zpcimon: Add NVMe SMART data monitor

Add a new monitor which uses libnvme to collect SMART data from NVMes.
This monitor only implements the .collect_adapter_data operation since
it carries no state across data collections. Nevertheless for future
symmetry and possible future expansion also add an empty struct
nvmemon_ctx.

Reviewed-by: Jan Höppner <hoeppner@linux.ibm.com>
Signed-off-by: Niklas Schnelle <schnelle@linux.ibm.com>
Signed-off-by: Jan Höppner <hoeppner@linux.ibm.com>

README.md

@@ -327,6 +327,7 @@ build options:
 | systemd      | `HAVE_SYSTEMD`     | hsavmcore                              |
 | libudev      | `HAVE_LIBUDEV`     | cpacfstatsd                            |
 | libnl3       | `HAVE_LIBNL3`      | zpcimon                                |
+| libnvme      | `HAVE_LIBNVME`     | zpcimon                                |
 
 This table lists additional build or install options:
 
@@ -375,10 +376,10 @@ the different tools are provided:
   The runtime requirements are: openssl-libs (>= 1.1.1) and libcurl.
 
 * zpcimon:
-  For building zpcimon OpenSSL and the Netlink Library Suite (libnl3) are
+  For building zpcimon OpenSSL, the Netlink Library Suite (libnl3),
-  required.
+  and libnvme are required.
   Tip: you may skip the zpcimon build by adding
-  `HAVE_OPENSSL=0` or `HAVE_LIBNL3=0`
+  `HAVE_OPENSSL=0` or `HAVE_LIBNL3=0` or `HAVE_LIBNVME=0`.
 
 * osasnmpd:
   You need at least the NET-SNMP 5.1.x package (net-snmp-devel.rpm)

include/lib/pci_sclp.h

@@ -21,9 +21,11 @@
 #define SCLP_ERRNOTIFY_AQ_DECONF		1
 #define SCLP_ERRNOTIFY_AQ_REPORT_ERR		2
 #define SCLP_ERRNOTIFY_AQ_OPTICS_DATA		3
+#define SCLP_ERRNOTIFY_AQ_NVME_SMART_DATA	4
 
 #define SCLP_ERRNOTIFY_ID_ZPCICTL		0x4713
 #define SCLP_ERRNOTIFY_ID_OPTICSMON		0x4714
+#define SCLP_ERRNOTIFY_ID_NVMEMON		0x4715
 
 #define SCLP_ERRNOTIFY_DATA_SIZE		4054
 

zpcimon/Makefile

@@ -29,6 +29,16 @@ check_dep_libnl3:
 BUILDTARGET += check_dep_libnl3
 endif # HAVE_LIBNL3
 
+ifneq (${HAVE_LIBNVME},0)
+check_dep_libnvme:
+	$(call check_dep, \
+		"zpcimon", \
+		"libnvme.h", \
+		"libnvme-devel", \
+		"HAVE_LIBNVME=0")
+BUILDTARGET += check_dep_libnvme
+endif # HAVE_LIBNVME
+
 ifeq (${HAVE_OPENSSL},0)
 
 all:
@@ -44,6 +54,13 @@ all:
 install:
 	$(SKIP) HAVE_LIBNL3=0
 
+else ifeq (${HAVE_LIBNVME},0)
+all:
+	$(SKIP) HAVE_LIBNVME=0
+
+install:
+	$(SKIP) HAVE_LIBNVME=0
+
 else
 
 ifneq ($(shell sh -c 'command -v pkg-config'),)
@@ -58,12 +75,15 @@ LIB_LFLAGS += $(shell $(PKG_CONFIG) --silence-errors --libs libnl-genl-3.0)
 LIB_LFLAGS += $(shell $(PKG_CONFIG) --silence-errors --libs libnl-route-3.0)
 
 LIB_LFLAGS += $(shell $(PKG_CONFIG) --silence-errors --libs libcrypto)
+LIB_LFLAGS += $(shell $(PKG_CONFIG) --silence-errors --libs libnvme)
 else
 LIB_CFLAGS += -I /usr/include/libnl3/
 LIB_LFLAGS += -lnl-route-3 -lnl-genl-3 -lnl-3
 
 LIB_CFLAGS += -I /usr/include/openssl/
 LIB_LFLAGS += -lcrypto
+
+LIB_LFLAGS += -lnvme
 endif
 
 ALL_CPPFLAGS += $(LIB_CFLAGS)
@@ -73,7 +93,7 @@ BUILDTARGET += zpcimon
 
 all: ${BUILDTARGET}
 
-zpcimon: zpcimon.o opticsmon.o optics_info.o optics_sclp.o ethtool.o link_mon.o $(libs)
+zpcimon: zpcimon.o nvmemon.o opticsmon.o optics_info.o optics_sclp.o ethtool.o link_mon.o $(libs)
 
 install: all
 	$(INSTALL) -d -m 755 $(DESTDIR)$(BINDIR) $(DESTDIR)$(MANDIR)/man8

zpcimon/nvmemon.c

@@ -0,0 +1,98 @@
+/*
+ * zpcimon - Report monitoring data to firmware
+ *
+ * Copyright IBM Corp. 2025
+ *
+ * s390-tools is free software; you can redistribute it and/or modify
+ * it under the terms of the MIT license. See LICENSE for details.
+ */
+#include <err.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <stdbool.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <libnvme.h>
+
+#include "lib/pci_list.h"
+#include "lib/pci_sclp.h"
+#include "lib/util_fmt.h"
+
+#include "nvmemon.h"
+#include "zpcimon.h"
+
+static void nvme_json_print(struct zpcimon_ctx *ctx, struct zpci_dev *zdev, const char *name,
+			    struct nvme_smart_log *log)
+{
+	zpci_adapter_json_print_start(zdev);
+	util_fmt_obj_start(FMT_DEFAULT, "nvmedev");
+	util_fmt_pair(FMT_QUOTE, "dev", name);
+	if (ctx->opts.smart_blob)
+		zpcimon_json_base64_pair("smart-log-raw", (uint8_t *)log, sizeof(*log));
+	util_fmt_obj_end();
+	zpci_adapter_json_print_end();
+	fflush(stdout);
+}
+
+static int sclp_issue_nvme_smart_report(struct zpci_dev *zdev, const uint8_t *smart, int smart_len)
+{
+	char *pci_addr;
+	int rc;
+
+	if (zdev->pft != ZPCI_PFT_NVME)
+		return -ENOTSUP;
+	pci_addr = zpci_pci_addr(zdev);
+	rc = zpci_sclp_issue_action(pci_addr, SCLP_ERRNOTIFY_AQ_NVME_SMART_DATA,
+				    (char *)smart, smart_len, SCLP_ERRNOTIFY_ID_NVMEMON);
+	free(pci_addr);
+	return rc;
+}
+
+static int nvmemon_collect_adapter_data(struct zpcimon_ctx *ctx, struct zpci_dev *zdev)
+{
+	struct nvme_smart_log log = {};
+	int nvme_fd, rc = -ENODEV;
+	char *dev, *pci_addr;
+
+	if (zdev->pft != ZPCI_PFT_NVME)
+		return -ENODEV;
+
+	pci_addr = zpci_pci_addr(zdev);
+	dev = zpci_get_nvme_device_node(pci_addr);
+	if (!dev)
+		goto exit_free_addr;
+
+	nvme_fd = openat(AT_FDCWD, dev, O_RDONLY);
+	if (nvme_fd < 0) {
+		warn("Failed to open %s", dev);
+		rc = -errno;
+		goto exit_free_dev;
+	}
+
+	rc = nvme_get_log_smart(nvme_fd, NVME_NSID_ALL, false, &log);
+	if (rc) {
+		warnx("Getting NVMe SMART log failed %d", rc);
+		goto exit_close;
+	}
+	if (!ctx->opts.quiet)
+		nvme_json_print(ctx, zdev, dev, &log);
+	if (ctx->opts.report) {
+		rc = sclp_issue_nvme_smart_report(zdev, (uint8_t *)&log, sizeof(log));
+		if (rc < 0 && rc != -ENOTSUP)
+			warnx("Error issuing SCLP for NVMe SMART log failed");
+	}
+
+exit_close:
+	close(nvme_fd);
+exit_free_dev:
+	free(dev);
+exit_free_addr:
+	free(pci_addr);
+	return rc;
+}
+
+const struct zpcimon_ops nvmemon_ops = {
+	.collect_adapter_data = nvmemon_collect_adapter_data,
+};

zpcimon/nvmemon.h

@@ -0,0 +1,13 @@
+/*
+ * Copyright IBM Corp. 2025
+ *
+ * s390-tools is free software; you can redistribute it and/or modify
+ * it under the terms of the MIT license. See LICENSE for details.
+ */
+#ifndef ZPCIMON_NVMEMON_H
+#define ZPCIMON_NVMEMON_H
+
+struct nvmemon_ctx {};
+
+extern const struct zpcimon_ops nvmemon_ops;
+#endif /* ZPCIMON_NVMEMON_H */

zpcimon/zpcimon.8

@@ -33,7 +33,7 @@
 .TH zpcimon 8 "Oct 2024" s390-tools zpcictl
 .
 .SH NAME
-zpcimon - Monitor optical modules for directly attached PCI based NICs
+zpcimon - Monitor health of directly attached PCI devices including NVMes and NICs.
 .
 .
 .SH SYNOPSIS
@@ -45,26 +45,29 @@ zpcimon - Monitor optical modules for directly attached PCI based NICs
 Use
 .B zpcimon
 to monitor the health of the optical modules of directly attached PCI based
-NICs. When executed without the \fB--monitor\fR option it will collect optical
+NICs as well as the health of NVMe devices using their SMART data. When
-module data from all available PCI network interface physical functions and
+executed without the \fB--monitor\fR option it will collect health data from
-print a summary in JSON format. Add the \fB--send-report\fR option to report
+all available and relevant PCI devices and print a summary in JSON format. Add
-this data to the support element.
+the \fB--send-report\fR option to report this data to the support element.
-
 
 .
 .
 .SH OPTIONS
 .SS Operation Options
 .OD monitor "m"
-Run continuously and report on link state changes and periodically
+Run continuously and monitor all supported devices. On link state changes
+optical module data is collected and NVMe SMART data is collected on hotplug.
+Moreover every
+.B --interval
+seconds data from all adapters is collected as a whole system snapshot.
 .PP
 .
 .OD send-report "r"
-Report the optics health data to the Support Element (SE)
+Report the optics and NVMe health data to the Support Element (SE)
 .PP
 .
 .OD quiet "q"
-Be quiet and don't print optics health summary
+Be quiet and don't print health summaries.
 .PP
 .
 .OD interval "i" "seconds"
@@ -84,6 +87,11 @@ on".
 .PP
 .PP
 .
+.OD smart-log-raw ""
+Include a base64 encoded binary dump of each NVMe's SMART data. This matches
+"nvme smart-log --output-format=binary /dev/nvmeX".
+.PP
+.
 .SS Output Format Options
 .OD format "" <FORMAT>
 Set the output format out of json, json-seq, jsonl, pairs, and csv. Note that

zpcimon/zpcimon.c

@@ -28,6 +28,7 @@
 #include "lib/util_time.h"
 #include "lib/zt_common.h"
 
+#include "nvmemon.h"
 #include "opticsmon.h"
 #include "zpcimon.h"
 #include "zpcimon_cli.h"
@@ -42,6 +43,7 @@ struct zpcimon_monitor {
 
 static struct zpcimon_monitor monitors[] = {
 	{.ops = &opticsmon_ops, .initialized = 0, .opened = 0},
+	{.ops = &nvmemon_ops, .initialized  = 0, .opened = 0},
 };
 
 static const struct util_prg prg = {
@@ -79,6 +81,9 @@ static void parse_cmdline(int argc, char *argv[], struct options *opts)
 		case OPT_DUMP:
 			opts->module_info = true;
 			break;
+		case OPT_SMART_DUMP:
+			opts->smart_blob = true;
+			break;
 		case OPT_FORMAT:
 			if (!util_fmt_name_to_type(optarg, &fmt))
 				errx(EXIT_FAILURE, "Unknown format %s", optarg);

zpcimon/zpcimon.h

@@ -11,6 +11,7 @@
 
 #include "lib/util_fmt.h"
 
+#include "nvmemon.h"
 #include "opticsmon.h"
 
 #define API_LEVEL 1
@@ -26,12 +27,15 @@ struct options {
 
 	/* Optics Monitoring Specific */
 	bool module_info;
+	/* NVMe Monitoring Specific */
+	bool smart_blob;
 };
 
 struct zpcimon_ctx {
 	struct options opts;
 	struct util_list *zpci_list;
 	struct opticsmon_ctx opticsmon_ctx;
+	struct nvmemon_ctx nvmemon_ctx;
 };
 
 struct zpcimon_ops {

zpcimon/zpcimon_cli.h

@@ -12,6 +12,7 @@
 
 #define OPT_DUMP 128
 #define OPT_FORMAT 129
+#define OPT_SMART_DUMP 130
 
 static struct util_opt opt_vec[] = {
 	UTIL_OPT_SECTION("OPERATION OPTIONS"),
@@ -35,6 +36,13 @@ static struct util_opt opt_vec[] = {
 			"This matches 'ethtool --module-info <netdev> raw on'",
 		.flags = UTIL_OPT_FLAG_NOSHORT,
 	},
+	{
+		.option = { "smart-log-raw", no_argument, NULL, OPT_SMART_DUMP },
+		.desc = "Include a base64 encoded binary dump of the "
+			"SMART log data for each NVMe. "
+			"This matches 'nvme smart-log --raw-binary <dev>'",
+		.flags = UTIL_OPT_FLAG_NOSHORT,
+	},
 	UTIL_OPT_SECTION("OPTIONS WITH ARGUMENTS"),
 	{
 		.option = { "interval", required_argument, NULL, 'i' },