libzpci: Introduce libzpci

This introduces libzpci which is a library intended to handle s390x
specific PCI attributes and peculiarities. As a first step it introduces
code to list PCI devices on s390x including an initial set of s390x
specific attributes like FID, PFT, UID etc. It also collects information
on network device associated with a particular PCI function. The
included example serves as a demonstration of using libzpci to list PCI
devices.

Reviewed-by: Halil Pasic <pasic@linux.ibm.com>
Signed-off-by: Niklas Schnelle <schnelle@linux.ibm.com>
Signed-off-by: Jan Höppner <hoeppner@linux.ibm.com>

Makefile

@@ -9,7 +9,7 @@ include common.mak
 ifeq ($(HOST_ARCH),s390x)
 BASELIB_DIRS = libutil libseckey
 LIB_DIRS = libvtoc libzds libdasd libccw libvmcp libekmfweb \
-	   libkmipclient libcpumf libap libpv
+	   libkmipclient libcpumf libap libpv libzpci
 TOOL_DIRS = zipl zdump fdasd dasdfmt dasdview tunedasd \
 	   tape390 osasnmpd qetharp ip_watcher qethconf scripts zconf \
 	   vmcp man mon_tools dasdinfo vmur cpuplugd ipl_tools \

common.mak

@@ -504,6 +504,10 @@ $(rootdir)/libpv/libpv.a: $(rootdir)/libpv
 	$(MAKE) -C $(rootdir)/libpv libpv.a
 .PHONY: $(rootdir)/libpv
 
+$(rootdir)/libzpci/libzpci.a: $(rootdir)/libzpci
+	$(MAKE) -C $(rootdir)/libzpci libzpci.a
+.PHONY: $(rootdir)/libzpci
+
 $(rootdir)/zipl/boot/.loaders:
 	$(MAKE) -C $(rootdir)/zipl/boot/ .loaders
 

include/lib/pci_list.h

@@ -0,0 +1,70 @@
+/**
+ * @defgroup pci_list_h libzpci: zPCI device handling
+ * @{
+ * @brief Work with zPCI devices
+ *
+ * Copyright IBM Corp. 2024
+ *
+ * s390-tools is free software; you can redistribute it and/or modify
+ * it under the terms of the MIT license. See LICENSE for details.
+ */
+
+#ifndef LIB_ZPCI_PCI_LIST_H
+#define LIB_ZPCI_PCI_LIST_H
+
+#include <stdint.h>
+#include <stdbool.h>
+
+#include "util_list.h"
+
+enum zpci_pft {
+	ZPCI_PFT_UNCLASSIFIED = 0x00,
+
+	ZPCI_PFT_ROCE_EXPRESS = 0x02,
+	ZPCI_PFT_ROCE_EXPRESS2 = 0x0a,
+	ZPCI_PFT_CNW = 0x0d,
+	ZPCI_PFT_NETH =	0x0c,
+	ZPCI_PFT_NETD = 0x0f,
+
+	ZPCI_PFT_NVME = 0x0b,
+	ZPCI_PFT_ISM = 0x05
+};
+
+struct zpci_dev {
+	struct util_list_node entry;
+	/* PCI Domain */
+	uint32_t domain_nr;
+	/* PCI Bus (8 bits), Device (5 bits), Function (3 bits) */
+	union {
+		uint16_t val;
+		struct {
+			uint16_t bus : 8;
+			uint16_t dev : 5;
+			uint16_t fn : 3;
+		};
+	} bdf;
+
+	/* Function attributes (see linux/Documentation/arch/s390/pci.rst) */
+	uint32_t fid;
+	uint32_t uid;
+	uint16_t pchid;
+	uint16_t vfn;
+	uint8_t port;
+	enum zpci_pft pft;
+	bool uid_is_unique;
+	/* Configuration state 0 - Standby, 1 Configured */
+	bool conf;
+
+	/* Associated netdevs if any */
+	int num_netdevs;
+	char **netdevs;
+};
+
+struct util_list *zpci_dev_list(void);
+void zpci_free_dev_list(struct util_list *zpci_list);
+void zpci_free_dev(struct zpci_dev *zdev);
+
+char *zpci_pci_addr(struct zpci_dev *zdev);
+const char *zpci_pft_str(struct zpci_dev *zdev);
+
+#endif /* LIB_ZPCI_PCI_LIST_H */

libzpci/Makefile

@@ -0,0 +1,20 @@
+include ../common.mak
+
+lib = libzpci.a
+
+all: $(lib)
+
+objects = pci_list.o
+
+examples := $(patsubst %.c,%,$(wildcard *_example.c))
+
+examples: $(examples)
+$(examples): %: %.o $(lib) $(rootdir)/libutil/libutil.a
+
+$(lib): ALL_CFLAGS += -fPIC -std=c11
+$(lib): $(objects)
+
+install: all
+
+clean:
+	rm -f *.o $(lib) $(examples)

libzpci/libzpci_example.c

@@ -0,0 +1,47 @@
+#include <stdio.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <err.h>
+
+#include "lib/util_list.h"
+#include "lib/pci_list.h"
+
+static void zpci_print(struct zpci_dev *zdev)
+{
+	char *pci_addr = zpci_pci_addr(zdev);
+	int i;
+
+	if (!zdev->conf) {
+		printf("fid: %8x address: %s\n", zdev->fid, pci_addr);
+	} else {
+		printf("fid: %8x address: %s uid: %4x%s pchid: %4x vfn: %4d port: %1d pft: %s ",
+		       zdev->fid, pci_addr, zdev->uid, (zdev->uid_is_unique) ? " (unique)" : "",
+		       zdev->pchid, zdev->vfn, zdev->port, zpci_pft_str(zdev));
+		if (zdev->num_netdevs) {
+			printf("netdevs: ");
+			for (i = 0; i < zdev->num_netdevs; i++) {
+				printf("%s", zdev->netdevs[i]);
+				if (i + 1 < zdev->num_netdevs)
+					printf(", ");
+			}
+		}
+		printf("\n");
+	}
+	free(pci_addr);
+}
+
+int main(void)
+{
+	struct util_list *zpci_list;
+	struct zpci_dev *zdev;
+
+	zpci_list = zpci_dev_list();
+	if (!zpci_list)
+		errx(EXIT_FAILURE, "Error getting list of zPCI devices");
+
+	util_list_iterate(zpci_list, zdev)
+		zpci_print(zdev);
+
+	zpci_free_dev_list(zpci_list);
+	return EXIT_SUCCESS;
+}

libzpci/pci_list.c

@@ -0,0 +1,287 @@
+/**
+ * libzpci - Functions to handle zPCI devices and their properties
+ *
+ * Copyright IBM Corp. 2023
+ *
+ * s390-tools is free software; you can redistribute it and/or modify
+ * it under the terms of the MIT license. See LICENSE for details.
+ */
+
+#include <err.h>
+#include <errno.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "lib/pci_list.h"
+#include "lib/util_file.h"
+#include "lib/util_libc.h"
+#include "lib/util_list.h"
+#include "lib/util_path.h"
+#include "lib/util_scandir.h"
+
+/**
+ * Get the function type name for the given device
+ *
+ * The device type name is suitable for presentation to a user.
+ *
+ * @param[in]	zdev	The device in question
+ *
+ * @return a string representing the PCI device type
+ */
+const char *zpci_pft_str(struct zpci_dev *zdev)
+{
+	switch (zdev->pft) {
+	case ZPCI_PFT_UNCLASSIFIED:
+		return "unclassified";
+	case ZPCI_PFT_ROCE_EXPRESS:
+		return "RoCE Express";
+	case ZPCI_PFT_ROCE_EXPRESS2:
+		return "RoCE Express-2";
+	case ZPCI_PFT_CNW:
+		return "Cloud Network Adapter";
+	case ZPCI_PFT_NETH:
+		return "Network Express Hybrid";
+	case ZPCI_PFT_NETD:
+		return "Network Express Dedicated";
+	case ZPCI_PFT_NVME:
+		return "NVMe";
+	case ZPCI_PFT_ISM:
+		return "ISM";
+	default:
+		return "unknown";
+	}
+}
+
+/**
+ * Get a textual representation of the device's PCI address
+ *
+ * The representation has extended "DDDD:bb:dd.f" format used
+ * by Linux tooling such as lspci.
+ *
+ * @param[in]	zdev	The device in question
+ *
+ * @return the string representing the PCI address
+ */
+char *zpci_pci_addr(struct zpci_dev *zdev)
+{
+	uint8_t bus = zdev->bdf.bus;
+	uint8_t dev = zdev->bdf.dev;
+	uint8_t fn = zdev->bdf.fn;
+	char *pci_addr;
+
+	util_asprintf(&pci_addr, "%04x:%02x:%02x.%x", zdev->domain_nr, bus, dev, fn);
+	return pci_addr;
+}
+
+static int zpci_populate_from_slot_dir(struct zpci_dev *zdev, const char *slot_dir,
+				       const char *slot_name)
+{
+	char buf_addr[11]; /* "dddd:bb:dd\0" */
+	uint8_t bus, df;
+	uint32_t domain;
+	int val, rc;
+
+	rc = sscanf(slot_name, "%x", &zdev->fid);
+	if (rc != 1)
+		return -EINVAL;
+
+	rc = util_file_read_line(buf_addr, sizeof(buf_addr), "%s/%s/address", slot_dir, slot_name);
+	if (rc) {
+		warn("Reading address from slot %s/%s", slot_dir, slot_name);
+		return rc;
+	}
+	rc = sscanf(buf_addr, "%04x:%02hhx:%02hhx", &domain, &bus, &df);
+	if (rc != 3)
+		return -EINVAL;
+	zdev->domain_nr = domain;
+	zdev->bdf.val = (((uint16_t)bus) << 8) | df;
+
+	rc = util_file_read_i(&val, 10, "%s/%s/power", slot_dir, slot_name);
+	if (rc) {
+		warn("Reading power from slot %s/%s", slot_dir, slot_name);
+		return rc;
+	}
+	zdev->conf = val > 0;
+
+	return 0;
+}
+
+static int zpci_populate_netdevices(struct zpci_dev *zdev, const char *dev_dir)
+{
+	const char *netdev_patt = "en.*";
+	struct dirent **de_vec;
+	int count, i, rc = 0;
+	char *net_dir;
+
+	util_asprintf(&net_dir, "%s/net", dev_dir);
+	count = util_scandir(&de_vec, alphasort, net_dir, netdev_patt);
+	if (count == -1) {
+		warn("Reading netdevice information for %s/net failed", dev_dir);
+		rc = -EINVAL;
+		goto out_net_dir;
+	}
+	/* A directory per netdev */
+	for (i = 0; i < count; i++) {
+		if (de_vec[i]->d_type != DT_DIR) {
+			rc = -EINVAL;
+			goto out_scan_dir;
+		}
+	}
+	zdev->num_netdevs = count;
+	if (!count)
+		goto out_scan_dir;
+	zdev->netdevs = util_zalloc(sizeof(char *) * zdev->num_netdevs);
+	for (i = 0; i < count; i++)
+		zdev->netdevs[i] = util_strdup(de_vec[i]->d_name);
+
+out_scan_dir:
+	util_scandir_free(de_vec, count);
+out_net_dir:
+	free(net_dir);
+	return rc;
+}
+
+static int zpci_populate_from_dev_dir(struct zpci_dev *zdev)
+{
+	char *pci_addr = zpci_pci_addr(zdev);
+	int rc, val;
+	char *path;
+
+	path = util_path_sysfs("bus/pci/devices/%s", pci_addr);
+	if (!path) {
+		rc = -EINVAL;
+		goto out_pci_addr;
+	}
+	if (!util_path_exists(path)) {
+		rc = -ENODEV;
+		goto out_path;
+	}
+
+	rc = util_file_read_i(&val, 16, "%s/uid", path);
+	if (rc)
+		goto out_path;
+	zdev->uid = val;
+
+	/* In old Linux versions uid_is_unique doesn't exist
+	 * so don't treat this as an error.
+	 */
+	rc = util_file_read_i(&val, 10, "%s/uid_is_unique", path);
+	if (!rc)
+		zdev->uid_is_unique = !!val;
+
+	rc = util_file_read_i(&val, 16, "%s/pchid", path);
+	if (rc)
+		goto out_path;
+	zdev->pchid = val;
+
+	rc = util_file_read_i(&val, 16, "%s/vfn", path);
+	if (rc)
+		goto out_path;
+	zdev->vfn = val;
+
+	rc = util_file_read_i(&val, 10, "%s/port", path);
+	if (rc)
+		goto out_path;
+	zdev->port = val;
+
+	rc = util_file_read_i(&val, 16, "%s/pft", path);
+	if (rc)
+		goto out_path;
+	zdev->pft = val;
+
+	if (util_path_is_readable("%s/net", path)) {
+		rc = zpci_populate_netdevices(zdev, path);
+		if (rc)
+			goto out_path;
+	}
+out_path:
+	free(path);
+out_pci_addr:
+	free(pci_addr);
+	return rc;
+}
+
+/**
+ * Get a list of all configured and standby PCI devices
+ *
+ * @return a list of struct zpci_dev in case of success,
+ *	   NULL in case of failure
+ */
+struct util_list *zpci_dev_list(void)
+{
+	char *path = util_path_sysfs("bus/pci/slots/");
+	const char *zpci_slot_patt = "[0-9a-f]{8}";
+	struct util_list *zpci_list = NULL;
+	struct dirent **de_vec;
+	struct zpci_dev *zdev;
+	int count, i, rc;
+
+	count = util_scandir(&de_vec, alphasort, path, zpci_slot_patt);
+	if (count == -1) {
+		warn("util_scandir failed");
+		goto error_path;
+	}
+	zpci_list = util_list_new(struct zpci_dev, entry);
+
+	for (i = 0; i < count; i++) {
+		if (de_vec[i]->d_type != DT_DIR)
+			continue;
+		zdev = util_zalloc(sizeof(*zdev));
+		rc = zpci_populate_from_slot_dir(zdev, path, de_vec[i]->d_name);
+		if (rc) {
+			free(zdev);
+			continue;
+		}
+		if (zdev->conf) {
+			rc = zpci_populate_from_dev_dir(zdev);
+			if (rc) {
+				free(zdev);
+				continue;
+			}
+		}
+		util_list_add_tail(zpci_list, zdev);
+	}
+
+	util_scandir_free(de_vec, count);
+error_path:
+	free(path);
+	return zpci_list;
+}
+
+/**
+ * Free a PCI device struct
+ *
+ * This frees both the struct zpci_dev and its associated netdevs array
+ *
+ * @param[in]	zdev		The device struct to free
+ */
+void zpci_free_dev(struct zpci_dev *zdev)
+{
+	int i;
+
+	if (zdev->num_netdevs) {
+		for (i = 0; i < zdev->num_netdevs; i++)
+			free(zdev->netdevs[i]);
+		free(zdev->netdevs);
+	}
+	free(zdev);
+}
+
+/**
+ * Free a PCI device list
+ *
+ * This frees all elements in the list
+ *
+ * @param[in]	zpci_list	The device list to free
+ */
+void zpci_free_dev_list(struct util_list *zpci_list)
+{
+	struct zpci_dev *zdev, *tmp;
+
+	util_list_iterate_safe(zpci_list, zdev, tmp) {
+		util_list_remove(zpci_list, zdev);
+		zpci_free_dev(zdev);
+	}
+	util_list_free(zpci_list);
+}