From 3a4c20b443e969497065a82f54593d1f07ae2070 Mon Sep 17 00:00:00 2001 From: Niklas Schnelle Date: Tue, 14 Nov 2023 17:29:00 +0100 Subject: [PATCH] libzpci: Introduce libzpci MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This introduces libzpci which is a library intended to handle s390x specific PCI attributes and peculiarities. As a first step it introduces code to list PCI devices on s390x including an initial set of s390x specific attributes like FID, PFT, UID etc. It also collects information on network device associated with a particular PCI function. The included example serves as a demonstration of using libzpci to list PCI devices. Reviewed-by: Halil Pasic Signed-off-by: Niklas Schnelle Signed-off-by: Jan Höppner --- Makefile | 2 +- common.mak | 4 + include/lib/pci_list.h | 70 ++++++++++ libzpci/Makefile | 20 +++ libzpci/libzpci_example.c | 47 +++++++ libzpci/pci_list.c | 287 ++++++++++++++++++++++++++++++++++++++ 6 files changed, 429 insertions(+), 1 deletion(-) create mode 100644 include/lib/pci_list.h create mode 100644 libzpci/Makefile create mode 100644 libzpci/libzpci_example.c create mode 100644 libzpci/pci_list.c diff --git a/Makefile b/Makefile index caf7d6d7..94efe173 100644 --- a/Makefile +++ b/Makefile @@ -9,7 +9,7 @@ include common.mak ifeq ($(HOST_ARCH),s390x) BASELIB_DIRS = libutil libseckey LIB_DIRS = libvtoc libzds libdasd libccw libvmcp libekmfweb \ - libkmipclient libcpumf libap libpv + libkmipclient libcpumf libap libpv libzpci TOOL_DIRS = zipl zdump fdasd dasdfmt dasdview tunedasd \ tape390 osasnmpd qetharp ip_watcher qethconf scripts zconf \ vmcp man mon_tools dasdinfo vmur cpuplugd ipl_tools \ diff --git a/common.mak b/common.mak index b44c0861..5e01e496 100644 --- a/common.mak +++ b/common.mak @@ -504,6 +504,10 @@ $(rootdir)/libpv/libpv.a: $(rootdir)/libpv $(MAKE) -C $(rootdir)/libpv libpv.a .PHONY: $(rootdir)/libpv +$(rootdir)/libzpci/libzpci.a: $(rootdir)/libzpci + $(MAKE) -C $(rootdir)/libzpci libzpci.a +.PHONY: $(rootdir)/libzpci + $(rootdir)/zipl/boot/.loaders: $(MAKE) -C $(rootdir)/zipl/boot/ .loaders diff --git a/include/lib/pci_list.h b/include/lib/pci_list.h new file mode 100644 index 00000000..47dedf67 --- /dev/null +++ b/include/lib/pci_list.h @@ -0,0 +1,70 @@ +/** + * @defgroup pci_list_h libzpci: zPCI device handling + * @{ + * @brief Work with zPCI devices + * + * Copyright IBM Corp. 2024 + * + * s390-tools is free software; you can redistribute it and/or modify + * it under the terms of the MIT license. See LICENSE for details. + */ + +#ifndef LIB_ZPCI_PCI_LIST_H +#define LIB_ZPCI_PCI_LIST_H + +#include +#include + +#include "util_list.h" + +enum zpci_pft { + ZPCI_PFT_UNCLASSIFIED = 0x00, + + ZPCI_PFT_ROCE_EXPRESS = 0x02, + ZPCI_PFT_ROCE_EXPRESS2 = 0x0a, + ZPCI_PFT_CNW = 0x0d, + ZPCI_PFT_NETH = 0x0c, + ZPCI_PFT_NETD = 0x0f, + + ZPCI_PFT_NVME = 0x0b, + ZPCI_PFT_ISM = 0x05 +}; + +struct zpci_dev { + struct util_list_node entry; + /* PCI Domain */ + uint32_t domain_nr; + /* PCI Bus (8 bits), Device (5 bits), Function (3 bits) */ + union { + uint16_t val; + struct { + uint16_t bus : 8; + uint16_t dev : 5; + uint16_t fn : 3; + }; + } bdf; + + /* Function attributes (see linux/Documentation/arch/s390/pci.rst) */ + uint32_t fid; + uint32_t uid; + uint16_t pchid; + uint16_t vfn; + uint8_t port; + enum zpci_pft pft; + bool uid_is_unique; + /* Configuration state 0 - Standby, 1 Configured */ + bool conf; + + /* Associated netdevs if any */ + int num_netdevs; + char **netdevs; +}; + +struct util_list *zpci_dev_list(void); +void zpci_free_dev_list(struct util_list *zpci_list); +void zpci_free_dev(struct zpci_dev *zdev); + +char *zpci_pci_addr(struct zpci_dev *zdev); +const char *zpci_pft_str(struct zpci_dev *zdev); + +#endif /* LIB_ZPCI_PCI_LIST_H */ diff --git a/libzpci/Makefile b/libzpci/Makefile new file mode 100644 index 00000000..344f32b4 --- /dev/null +++ b/libzpci/Makefile @@ -0,0 +1,20 @@ +include ../common.mak + +lib = libzpci.a + +all: $(lib) + +objects = pci_list.o + +examples := $(patsubst %.c,%,$(wildcard *_example.c)) + +examples: $(examples) +$(examples): %: %.o $(lib) $(rootdir)/libutil/libutil.a + +$(lib): ALL_CFLAGS += -fPIC -std=c11 +$(lib): $(objects) + +install: all + +clean: + rm -f *.o $(lib) $(examples) diff --git a/libzpci/libzpci_example.c b/libzpci/libzpci_example.c new file mode 100644 index 00000000..ade86758 --- /dev/null +++ b/libzpci/libzpci_example.c @@ -0,0 +1,47 @@ +#include +#include +#include +#include + +#include "lib/util_list.h" +#include "lib/pci_list.h" + +static void zpci_print(struct zpci_dev *zdev) +{ + char *pci_addr = zpci_pci_addr(zdev); + int i; + + if (!zdev->conf) { + printf("fid: %8x address: %s\n", zdev->fid, pci_addr); + } else { + printf("fid: %8x address: %s uid: %4x%s pchid: %4x vfn: %4d port: %1d pft: %s ", + zdev->fid, pci_addr, zdev->uid, (zdev->uid_is_unique) ? " (unique)" : "", + zdev->pchid, zdev->vfn, zdev->port, zpci_pft_str(zdev)); + if (zdev->num_netdevs) { + printf("netdevs: "); + for (i = 0; i < zdev->num_netdevs; i++) { + printf("%s", zdev->netdevs[i]); + if (i + 1 < zdev->num_netdevs) + printf(", "); + } + } + printf("\n"); + } + free(pci_addr); +} + +int main(void) +{ + struct util_list *zpci_list; + struct zpci_dev *zdev; + + zpci_list = zpci_dev_list(); + if (!zpci_list) + errx(EXIT_FAILURE, "Error getting list of zPCI devices"); + + util_list_iterate(zpci_list, zdev) + zpci_print(zdev); + + zpci_free_dev_list(zpci_list); + return EXIT_SUCCESS; +} diff --git a/libzpci/pci_list.c b/libzpci/pci_list.c new file mode 100644 index 00000000..8ced8778 --- /dev/null +++ b/libzpci/pci_list.c @@ -0,0 +1,287 @@ +/** + * libzpci - Functions to handle zPCI devices and their properties + * + * Copyright IBM Corp. 2023 + * + * s390-tools is free software; you can redistribute it and/or modify + * it under the terms of the MIT license. See LICENSE for details. + */ + +#include +#include +#include +#include +#include + +#include "lib/pci_list.h" +#include "lib/util_file.h" +#include "lib/util_libc.h" +#include "lib/util_list.h" +#include "lib/util_path.h" +#include "lib/util_scandir.h" + +/** + * Get the function type name for the given device + * + * The device type name is suitable for presentation to a user. + * + * @param[in] zdev The device in question + * + * @return a string representing the PCI device type + */ +const char *zpci_pft_str(struct zpci_dev *zdev) +{ + switch (zdev->pft) { + case ZPCI_PFT_UNCLASSIFIED: + return "unclassified"; + case ZPCI_PFT_ROCE_EXPRESS: + return "RoCE Express"; + case ZPCI_PFT_ROCE_EXPRESS2: + return "RoCE Express-2"; + case ZPCI_PFT_CNW: + return "Cloud Network Adapter"; + case ZPCI_PFT_NETH: + return "Network Express Hybrid"; + case ZPCI_PFT_NETD: + return "Network Express Dedicated"; + case ZPCI_PFT_NVME: + return "NVMe"; + case ZPCI_PFT_ISM: + return "ISM"; + default: + return "unknown"; + } +} + +/** + * Get a textual representation of the device's PCI address + * + * The representation has extended "DDDD:bb:dd.f" format used + * by Linux tooling such as lspci. + * + * @param[in] zdev The device in question + * + * @return the string representing the PCI address + */ +char *zpci_pci_addr(struct zpci_dev *zdev) +{ + uint8_t bus = zdev->bdf.bus; + uint8_t dev = zdev->bdf.dev; + uint8_t fn = zdev->bdf.fn; + char *pci_addr; + + util_asprintf(&pci_addr, "%04x:%02x:%02x.%x", zdev->domain_nr, bus, dev, fn); + return pci_addr; +} + +static int zpci_populate_from_slot_dir(struct zpci_dev *zdev, const char *slot_dir, + const char *slot_name) +{ + char buf_addr[11]; /* "dddd:bb:dd\0" */ + uint8_t bus, df; + uint32_t domain; + int val, rc; + + rc = sscanf(slot_name, "%x", &zdev->fid); + if (rc != 1) + return -EINVAL; + + rc = util_file_read_line(buf_addr, sizeof(buf_addr), "%s/%s/address", slot_dir, slot_name); + if (rc) { + warn("Reading address from slot %s/%s", slot_dir, slot_name); + return rc; + } + rc = sscanf(buf_addr, "%04x:%02hhx:%02hhx", &domain, &bus, &df); + if (rc != 3) + return -EINVAL; + zdev->domain_nr = domain; + zdev->bdf.val = (((uint16_t)bus) << 8) | df; + + rc = util_file_read_i(&val, 10, "%s/%s/power", slot_dir, slot_name); + if (rc) { + warn("Reading power from slot %s/%s", slot_dir, slot_name); + return rc; + } + zdev->conf = val > 0; + + return 0; +} + +static int zpci_populate_netdevices(struct zpci_dev *zdev, const char *dev_dir) +{ + const char *netdev_patt = "en.*"; + struct dirent **de_vec; + int count, i, rc = 0; + char *net_dir; + + util_asprintf(&net_dir, "%s/net", dev_dir); + count = util_scandir(&de_vec, alphasort, net_dir, netdev_patt); + if (count == -1) { + warn("Reading netdevice information for %s/net failed", dev_dir); + rc = -EINVAL; + goto out_net_dir; + } + /* A directory per netdev */ + for (i = 0; i < count; i++) { + if (de_vec[i]->d_type != DT_DIR) { + rc = -EINVAL; + goto out_scan_dir; + } + } + zdev->num_netdevs = count; + if (!count) + goto out_scan_dir; + zdev->netdevs = util_zalloc(sizeof(char *) * zdev->num_netdevs); + for (i = 0; i < count; i++) + zdev->netdevs[i] = util_strdup(de_vec[i]->d_name); + +out_scan_dir: + util_scandir_free(de_vec, count); +out_net_dir: + free(net_dir); + return rc; +} + +static int zpci_populate_from_dev_dir(struct zpci_dev *zdev) +{ + char *pci_addr = zpci_pci_addr(zdev); + int rc, val; + char *path; + + path = util_path_sysfs("bus/pci/devices/%s", pci_addr); + if (!path) { + rc = -EINVAL; + goto out_pci_addr; + } + if (!util_path_exists(path)) { + rc = -ENODEV; + goto out_path; + } + + rc = util_file_read_i(&val, 16, "%s/uid", path); + if (rc) + goto out_path; + zdev->uid = val; + + /* In old Linux versions uid_is_unique doesn't exist + * so don't treat this as an error. + */ + rc = util_file_read_i(&val, 10, "%s/uid_is_unique", path); + if (!rc) + zdev->uid_is_unique = !!val; + + rc = util_file_read_i(&val, 16, "%s/pchid", path); + if (rc) + goto out_path; + zdev->pchid = val; + + rc = util_file_read_i(&val, 16, "%s/vfn", path); + if (rc) + goto out_path; + zdev->vfn = val; + + rc = util_file_read_i(&val, 10, "%s/port", path); + if (rc) + goto out_path; + zdev->port = val; + + rc = util_file_read_i(&val, 16, "%s/pft", path); + if (rc) + goto out_path; + zdev->pft = val; + + if (util_path_is_readable("%s/net", path)) { + rc = zpci_populate_netdevices(zdev, path); + if (rc) + goto out_path; + } +out_path: + free(path); +out_pci_addr: + free(pci_addr); + return rc; +} + +/** + * Get a list of all configured and standby PCI devices + * + * @return a list of struct zpci_dev in case of success, + * NULL in case of failure + */ +struct util_list *zpci_dev_list(void) +{ + char *path = util_path_sysfs("bus/pci/slots/"); + const char *zpci_slot_patt = "[0-9a-f]{8}"; + struct util_list *zpci_list = NULL; + struct dirent **de_vec; + struct zpci_dev *zdev; + int count, i, rc; + + count = util_scandir(&de_vec, alphasort, path, zpci_slot_patt); + if (count == -1) { + warn("util_scandir failed"); + goto error_path; + } + zpci_list = util_list_new(struct zpci_dev, entry); + + for (i = 0; i < count; i++) { + if (de_vec[i]->d_type != DT_DIR) + continue; + zdev = util_zalloc(sizeof(*zdev)); + rc = zpci_populate_from_slot_dir(zdev, path, de_vec[i]->d_name); + if (rc) { + free(zdev); + continue; + } + if (zdev->conf) { + rc = zpci_populate_from_dev_dir(zdev); + if (rc) { + free(zdev); + continue; + } + } + util_list_add_tail(zpci_list, zdev); + } + + util_scandir_free(de_vec, count); +error_path: + free(path); + return zpci_list; +} + +/** + * Free a PCI device struct + * + * This frees both the struct zpci_dev and its associated netdevs array + * + * @param[in] zdev The device struct to free + */ +void zpci_free_dev(struct zpci_dev *zdev) +{ + int i; + + if (zdev->num_netdevs) { + for (i = 0; i < zdev->num_netdevs; i++) + free(zdev->netdevs[i]); + free(zdev->netdevs); + } + free(zdev); +} + +/** + * Free a PCI device list + * + * This frees all elements in the list + * + * @param[in] zpci_list The device list to free + */ +void zpci_free_dev_list(struct util_list *zpci_list) +{ + struct zpci_dev *zdev, *tmp; + + util_list_iterate_safe(zpci_list, zdev, tmp) { + util_list_remove(zpci_list, zdev); + zpci_free_dev(zdev); + } + util_list_free(zpci_list); +}