New release s390-tools-2.23.0

Signed-off-by: Jan Höppner <hoeppner@linux.ibm.com>

.editorconfig

@@ -0,0 +1,21 @@
+# Check https://editorconfig.org for details
+
+root = true
+
+[*]
+end_of_line = lf
+insert_final_newline = true
+charset = utf-8
+indent_style = tab
+tab_width = 8
+
+[*.sh]
+shell_variant = bash  # used by `shfmt`
+
+[*.y{a,}ml]
+indent_style = space
+indent_size = 2
+
+[*.py]
+indent_style = space
+indent_size = 4

.gitignore

@@ -16,12 +16,14 @@ TAGS
 #
 # Ignore generated executables and other generated files
 #
+ap_tools/ap-check
 cmsfs-fuse/cmsfs-fuse
 cpacfstats/cpacfstats
 cpacfstats/cpacfstatsd
 cpumf/chcpumf
 cpumf/lscpumf
 cpumf/lshwc
+cpumf/pai
 cpuplugd/cpuplugd
 dasdfmt/dasdfmt
 dasdinfo/dasdinfo
@@ -45,6 +47,8 @@ iucvterm/src/iucvconn
 iucvterm/src/iucvtty
 iucvterm/src/ttyrun
 iucvterm/test/test_afiucv
+libap/check-dep-lock
+libap/check-dep-json
 libekmfweb/check-dep-libekmfweb
 libekmfweb/detect-openssl-version.dep
 libekmfweb/libekmfweb.so
@@ -64,6 +68,8 @@ lsstp/lsstp
 mon_tools/mon_fsstatd
 mon_tools/mon_procd
 osasnmpd/osasnmpd
+pvattest/src/pvattest
+pvattest/tools/exchange_info
 qetharp/qetharp
 qethqoat/qethqoat
 systemd/cpacfstatsd.service
@@ -117,3 +123,4 @@ zkey/kmip/zkey-kmip.so
 zkey/zkey
 zkey/zkey-cryptsetup
 zpcictl/zpcictl
+**/.detect-openssl.dep.c

AUTHORS.md

@@ -2,12 +2,14 @@ List of all individuals having contributed content to s390-tools
 ----------------------------------------------------------------
 
 - Alexander Egorenkov
+- Alexandra Winter
 - Alexey Ishchuk
 - Andreas Herrmann
-- Andre Wild
+- André Wild
 - Antoinette Kaschner
 - Arnd Bergmann
 - Axel Wirbser
+- Balint Reczey
 - Benjamin Block
 - Brian C. Lane
 - Carsten Otte
@@ -18,6 +20,7 @@ List of all individuals having contributed content to s390-tools
 - Clemens von Mann
 - Colin Walters
 - Dan Horak
+- Dan Horák
 - Despina Papadopoulou
 - Dimitri John Ledkov
 - Eberhard Pasch
@@ -32,11 +35,13 @@ List of all individuals having contributed content to s390-tools
 - Fedor Loshakov
 - Felix Beck
 - Frank Blaschka
+- Frank Heimes
 - Frank Munzert
 - Frank Pavlic
 - Fritz Elfert
 - Gerald Schaefer
 - Gerhard Tonn
+- Graham Inggs
 - Guevenc Guelce
 - Hannes Reinecke
 - Hans-Joachim Picht
@@ -51,7 +56,7 @@ List of all individuals having contributed content to s390-tools
 - Ingo Franzki
 - Ingo Tuchscherer
 - Jan Glauber
-- Jan Hoeppner
+- Jan Höppner
 - Jan Willeke
 - Jason J. Herne
 - Javier Martinez Canillas
@@ -73,8 +78,10 @@ List of all individuals having contributed content to s390-tools
 - Martin Peschke
 - Martin Petermann
 - Martin Schwidefsky
+- Matthew Rosato
 - Maxim Shchetynin
 - Melissa Howland
+- Mete Durlu
 - Michael Ernst
 - Michael Holzheu
 - Michael Mueller
@@ -102,6 +109,7 @@ List of all individuals having contributed content to s390-tools
 - Stefan Raspl
 - Stefan Reimbold
 - Stefan Weinhuber
+- Steffen Eiden
 - Steffen Maier
 - Steffen Thoss
 - Susanne Wintenberger
@@ -110,6 +118,7 @@ List of all individuals having contributed content to s390-tools
 - Swen Schillig
 - Taraka R. Bodireddy
 - Thomas Heidrich
+- Thomas Huth
 - Thomas Richter
 - Thomas Spatzier
 - Thomas Weber

CHANGELOG.md

@@ -1,5 +1,137 @@
 Release history for s390-tools (MIT version)
 --------------------------------------------
+
+* __v2.23.0 (2022-08-18)__
+
+  For Linux kernel version: 5.19
+
+  Changes of existing tools:
+  - Makefile: use common Make definition for DRACUTDIR
+  - Makefile: use common Make definition for UDEVDIR and UDEVRULESDIR
+  - cpacfstats: Add PAI and hotplug support
+  - cpumf/pai: Omit file write progress information
+  - dbginfo.sh: Get more details on lspci command
+  - dumpconf: Prevent running the service in containers
+  - libcpumf: Detect PMU named pai_ext
+  - pvattest: Improve error reporting and logging
+  - zdev: Add some --type ap examples to manpages
+  - zkey: Use default benchmarked Argon2i with LUKS2
+
+  Bug Fixes:
+  - dbginfo.sh: Fix accidental ftrace buffer shrinkage/free
+  - genprotimg: Fix BIO_reset() returncode handling
+  - libpv: Fix dependency checking
+  - pvattest: Fix dependency checking
+  - zipl: Fix segmentation fault when no parmline is provided
+
+
+* __v2.22.0 (2022-06-20)__
+
+  For Linux kernel version: 5.18
+
+  Add new tools / libraries:
+  - ap_tools: Introduce ap_tools and the ap-check tool
+  - cpumf/pai: Add Processor Activity Instrumentation tool
+  - libpv: New library for PV tools
+  - pvattest: Add new tool to create, perform, and verify attestation measurements
+  - zipl/zdump: Add Next Gen Dump (NGDump) support
+
+  Changes of existing tools:
+  - Move man pages to System commands section (lscpumf, lshwc, pai, dbginfo.sh, zfcpdbf, zipl-switch-to-blscfg)
+  - README.md: Add 70-chreipl-fcp-mpath.rules to the list of udev rule descriptions
+  - Remove SysV related daemon scripts (cpacfstatsd, cpuplugd, mon_statd)
+  - genprotimg: Move man page to section 1 for user commands
+  - hyptop: increase initial update interval
+  - libseckey: Adapt keymgmt_match() implementation to OpenSSL
+  - libutil: Add util_exit_code
+  - libutil: Introduce util_udev
+  - zdev: Introduce the ap device type
+  - zipl-editenv: Add zIPL multienvironment support
+  - zipl: Implement sorting BLS entries by versions
+  - zkey: Add initramfs hook
+
+  Bug Fixes:
+  - cmsfs-fuse: Fix enabling of hard_remove option
+  - s390-tools: Fix typos that were detected by lintian as 'typo-in-manual-page'
+  - zkey-kmip: Fix possible use after free
+  - zkey: Fix EP11 host library version checking
+  - zkey_kmip: Setup ext-lib once the APQNs have been configured
+
+* __v2.21.0 (2022-04-20)__
+
+  For Linux kernel version: 5.17
+
+  Add new tools / libraries:
+  - libcpumf: Create library libcpumf for CPU Measurement functions
+
+  Changes of existing tools:
+  - chreipl-fcp-mpath: bundle a pre-cooked version of the manpage for build
+                       environments without access to `pandoc`
+  - dbginfo.sh: Add multipath info to map paths to FC addressing and prio group
+  - dbginfo.sh: Collect config files of systemd-modules-load.service
+  - dbginfo.sh: Sort list of environment variables for readability
+  - dbginfo.sh: Replace "which" by builtin command "type"
+  - dbginfo.sh: Rework script formatting (indents, order)
+  - dbginfo.sh: Update sysfs collection (excludes, messages)
+  - genprotimg: Add Protected Virtualization (PV) dump support
+  - genprotimg: Remove DigiCert root CA pinning
+  - lszcrypt: Add CEX8S support
+  - zcryptctl: Add control domain handling
+  - zcryptstats: Add CEX8 support
+  - zipl: Allow optional entries that are left out when files are missing
+  - zipl: make IPL sections defined with BLS to inherit a target field
+  - zpcictl: Add option to trigger firmware reset
+
+  Bug Fixes:
+  - cpictl: Handle excessive kernel version numbers
+  - dbginfo.sh: Collect all places where modprobe.d config files could exist
+  - fdasd: Fix endless menu loop on EOF
+  - zdump/dfi: Fix segfault due to double free
+  - zdump: Fix /dev/mem reading
+  - zpcictl: Fix race of SCLP reset and Linux recovery
+
+* __v2.20.0 (2022-02-04)__
+
+  For Linux kernel version: 5.16
+
+  Add new tools / libraries:
+  - Add EditorConfig configuration
+
+  Changes of existing tools:
+  - s390-tools switches to Fuse 3 as Fuse 2 is deprecated.
+        Affected tools: cmsfs, hmcdrvfs, hsavmcore, zdsfs, zdump
+  - chreipl-fcp-mpath: don't compress the manpage before installing it
+  - cpictl: Report extended version information
+  - genprotimg: Add extended kernel command line support
+  - zdev: modify the lsblk output parser in lszdev
+  - zipl: Add support for longer kernel command lines (now supports up to 64k length)
+
+  Bug Fixes:
+  - cpictl: Suppress messages for unwritable sysfs files
+  - dbginfo.sh: Fix missing syslog for step create_package
+  - lshwc: Fix CPU list parameter setup for device driver
+  - zdev: Check for errors when removing a devtype setting
+  - zdev: Fix path resolution for multi-mount point file systems
+
+* __v2.19.0 (2021-11-10)__
+
+  For Linux kernel version: 5.15
+
+  Add new tools / libraries:
+  - chreipl-fcp-mpath: New toolset that uses multipath information to change
+      the configured FCP re-IPL path on detecting issues with the current path
+
+  Changes of existing tools:
+  - dbginfo.sh: Add retry timeout and remove possible blocking "blockdev --report"
+  - dbginfo.sh: Collect config- and debug-data for chreipl-fcp-mpath
+  - hsci: Add support for multiple MAC addresses
+
+  Bug Fixes:
+  - lshwc: Fix compile error for gcc <8.1
+  - zdump: Various clean-ups and fixes
+  - ziomon: Correct throughput calculation in ziorep_printers
+  - zipl: Fix segmentation fault when setting stage3_parms
+
 * __v2.18.0 (2021-10-01)__
 
   For Linux kernel version: 5.14

Makefile

@@ -10,13 +10,13 @@ include common.mak
 #
 BASELIB_DIRS = libutil libseckey
 LIB_DIRS = libvtoc libzds libdasd libvmdump libccw libvmcp libekmfweb \
-	   libkmipclient
+	   libkmipclient libcpumf libap libpv
 TOOL_DIRS = zipl zdump fdasd dasdfmt dasdview tunedasd \
 	   tape390 osasnmpd qetharp ip_watcher qethconf scripts zconf \
 	   vmconvert vmcp man mon_tools dasdinfo vmur cpuplugd ipl_tools \
 	   ziomon iucvterm hyptop cmsfs-fuse qethqoat zfcpdump zdsfs cpumf \
 	   systemd hmcdrvfs cpacfstats zdev dump2tar zkey netboot etc zpcictl \
-	   genprotimg lsstp hsci hsavmcore
+	   genprotimg lsstp hsci hsavmcore chreipl-fcp-mpath ap_tools pvattest
 
 SUB_DIRS = $(BASELIB_DIRS) $(LIB_DIRS) $(TOOL_DIRS)
 
@@ -33,7 +33,7 @@ endif
 
 #
 # We have to build the base libraries before the other libraries are built,
-# and then build the other libraries before the tools are built. Otherwise the 
+# and then build the other libraries before the tools are built. Otherwise the
 # other libraries and tools would trigger parallel "make -C" builds for the
 # base libraries and the other libraries in case of "make -j".
 #

README.md

@@ -33,6 +33,9 @@ Package contents
  * genprotimg:
    Create a protected virtualization image.
 
+ * pvattest:
+   Create, perform, and verify protected virtualization attestation measurements.
+
  * udev rules:
    - 59-dasd.rules: rules for unique DASD device nodes created in /dev/disk/.
    - 57-osasnmpd.rules: udev rules for osasnmpd.
@@ -40,6 +43,9 @@ Package contents
    - 40-z90crypt.rules: udev rules for z90crypt driver
    - 90-cpi.rules: udev rule to update Control-Program-Information when KVM is
                    used.
+   - 70-chreipl-fcp-mpath.rules: udev rules to monitor multipath events for
+                                 re-IPL path failover and to adjust the re-IPL
+                                 device in case needed.
 
  * systemd units:
    - cpi.service: Unit to apply CPI settings
@@ -50,10 +56,10 @@ Package contents
    - ttyrun-getty@.service: Instance unit to manage ttyrun
 
  * zipl:
-   Make DASDs or tapes bootable for system IPL or system dump.
+   Make DASDs, SCSIs, NVMes or tapes bootable for system IPL or system dump.
 
  * zgetdump:
-   Retrieve system dumps from either tapes or DASDs.
+   Retrieve system dumps from either tapes, DASDs, SCSIs or NVMes.
 
  * qetharp:
    Read and flush the ARP cache on OSA Express network cards.
@@ -270,6 +276,14 @@ Package contents
    can be time consuming and prevent the HSA memory from being reused
    by other LPARs.
 
+ * chreipl-fcp-mpath:
+   Use multipath information to change the configured FCP re-IPL path on
+   detecting issues with the current path.
+
+ * ap-check:
+   A utility called by mdevctl to assist in managing vfio_ap-passthrough
+   devices.
+
 For more information refer to the following publications:
 
   * "Device Drivers, Features, and Commands" chapter "Useful Linux commands"
@@ -288,30 +302,33 @@ build options:
 
 | __LIBRARY__    | __BUILD OPTION__   | __TOOLS__                             |
 |----------------|:------------------:|:-------------------------------------:|
-| fuse           | `HAVE_FUSE`        | cmsfs-fuse, zdsfs, hmcdrvfs, zgetdump,|
+| fuse3          | `HAVE_FUSE`        | cmsfs-fuse, zdsfs, hmcdrvfs, zgetdump,|
 |                |                    | hsavmcore                             |
 | zlib           | `HAVE_ZLIB`        | zgetdump, dump2tar                    |
 | ncurses        | `HAVE_NCURSES`     | hyptop                                |
-| pfm            | `HAVE_PFM`         | cpacfstats                            |
 | net-snmp       | `HAVE_SNMP`        | osasnmpd                              |
 | glibc-static   | `HAVE_LIBC_STATIC` | zfcpdump                              |
 | openssl        | `HAVE_OPENSSL`     | genprotimg, zkey, libekmfweb,         |
-|                |                    | libkmipclient                         |
+|                |                    | libkmipclient, pvattest               |
 | cryptsetup     | `HAVE_CRYPTSETUP2` | zkey-cryptsetup                       |
 | json-c         | `HAVE_JSONC`       | zkey-cryptsetup, libekmfweb,          |
 |                |                    | libkmipclient                         |
-| glib2          | `HAVE_GLIB2`       | genprotimg                            |
-| libcurl        | `HAVE_LIBCURL`     | genprotimg, libekmfweb, libkmipclient |
+| glib2          | `HAVE_GLIB2`       | genprotimg, pvattest                  |
+| libcurl        | `HAVE_LIBCURL`     | genprotimg, libekmfweb, libkmipclient,|
+|                |                    | pvattest                              |
 | libxml2        | `HAVE_LIBXML2`     | libkmipclient                         |
 | systemd        | `HAVE_SYSTEMD`     | hsavmcore                             |
+| liblockfile    | `HAVE_LOCKFILE`    | ap-check                              |
+| libudev        | `HAVE_LIBUDEV`     | cpacfstatsd                           |
 
 This table lists additional build or install options:
 
-| __COMPONENT__    | __OPTION__                   | __TOOLS__      |
-|------------------|:----------------------------:|:--------------:|
-| dracut           | `HAVE_DRACUT`                | zdev           |
-| initramfs-tools  | `HAVE_INITRAMFS`             | zdev           |
-|                  | `ZDEV_ALWAYS_UPDATE_INITRD`  | zdev           |
+| __COMPONENT__    | __OPTION__                   | __TOOLS__                |
+|------------------|:----------------------------:|:------------------------:|
+| dracut           | `HAVE_DRACUT`                | zdev, chreipl-fcp-mpath, |
+|                  |                              | zipl                     |
+| initramfs-tools  | `HAVE_INITRAMFS`             | zdev, zipl               |
+|                  | `ZDEV_ALWAYS_UPDATE_INITRD`  | zdev                     |
 
 The s390-tools build process uses "pkg-config" if available and hard-coded
 compiler and linker options otherwise.
@@ -333,6 +350,16 @@ the different tools are provided:
 
   The runtime requirements are: openssl-libs (>= 1.1.0) and glib2.
 
+* pvattest:
+  For building pvattest you need OpenSSL version 1.1.1 or newer
+  installed (openssl-devel.rpm). Also required is glib2.56 or newer
+  (glib2-devel.rpm) and libcurl.
+  Tip: you may skip the pvattest build by adding
+  `HAVE_OPENSSL=0`, `HAVE_LIBCURL=0`, or `HAVE_GLIB2=0`.
+
+  The runtime requirements are: openssl-libs (>= 1.1.1) and
+  glib2.56 or newer.
+
 * osasnmpd:
   You need at least the NET-SNMP 5.1.x package (net-snmp-devel.rpm)
   installed, before building the osasnmpd subagent.
@@ -351,12 +378,12 @@ the different tools are provided:
 
 * cmsfs-fuse/zdsfs/hmcdrvfs/zgetdump:
   The tools cmsfs-fuse, zdsfs, hmcdrvfs, and zgetdump depend on FUSE.
-  FUSE is provided by installing the fuse and libfuse packages and by a
+  FUSE is provided by installing the fuse3 and libfuse3 packages and by a
   kernel compiled with `CONFIG_FUSE_FS`. For compiling the s390-tools package
-  the fuse-devel package is required.
-  The cmsfs-fuse tool requires FUSE version 2.8.1 or newer for full
+  the fuse3-devel package is required.
+  The cmsfs-fuse tool requires FUSE version 3.0 or newer for full
   functionality.
-  For further information about FUSE see: http://fuse.sourceforge.net
+  For further information about FUSE see: https://github.com/libfuse/libfuse
 
 * hyptop:
   The ncurses-devel package is required to build hyptop.
@@ -376,14 +403,13 @@ the different tools are provided:
   As of s390-tools-1.13.0, the minimum required kernel level is 2.6.38.
 
 * cpacfstats:
-  For building the cpacfstats tools you need libpfm version 4 or
-  newer installed (libpfm-devel.rpm). Tip: you may skip the cpacfstats
-  build by adding `HAVE_PFN=0` to the make invocation. To run the
-  cpacfstats daemon the kernel needs to have performance
-  events enabled (check for `CONFIG_PERF_EVENTS=y`) and you need libpfm
-  version 4 or newer installed. A new group 'cpacfstats' needs to be
-  created and all users intending to use the tool should be added to
-  this group.
+  For building the cpacfstats tools you need libudev installed
+  (systemd-devel.rpm). Tip: you may skip the cpacfstats build by
+  adding `HAVE_LIBUDEV=0` to the make invocation. To run the
+  cpacfstats daemon the kernel needs to have performance events
+  enabled (check for `CONFIG_PERF_EVENTS=y`). A new group 'cpacfstats'
+  needs to be created and all users intending to use the tool should
+  be added to this group.
 
 * zdev:
   Depending on the boot loader and initial RAM-disk mechanism used by a
@@ -452,7 +478,7 @@ the different tools are provided:
   `HAVE_JSONC=0`, or `HAVE_LIBCURL=0` to the make invocation.
 
 * hsavmcore:
-  For building the hsavmcore tool you need fuse version 2.6 and optionally
+  For building the hsavmcore tool you need fuse version 3.0 and optionally
   systemd which is enabled by default, to disable systemd support,
   add `HAVE_SYSTEMD=0` to the make invocation.
   Tip: you may skip the hsavmcore build by adding `HAVE_FUSE=0`
@@ -465,3 +491,22 @@ the different tools are provided:
   (libxml2-devel.rpm), and libcurl version 7.59 or newer (libcurl-devel.rpm).
   Tip: you may skip the libkmipclient build by adding `HAVE_OPENSSL=0`,
   `HAVE_JSONC=0`, `HAVE_LIBXML2=0`, or `HAVE_LIBCURL=0` to the make invocation.
+
+* chreipl-fcp-mpath:
+  For a complete list and documentation of the requirements, installation and
+  uninstallation, please see
+  [chreipl-fcp-mpath/README.md](chreipl-fcp-mpath/README.md).
+
+  Summarized: chreipl-fcp-mpath requires GNU Bash, GNU Core Utilities,
+  util-linux, udev, and multipath-tools. When using `HAVE_DRACUT=1` with the
+  make invocation, it also requires dracut. When using `ENABLE_DOC=1` with the
+  make invocation to build a fresh man page (instead of using the pre-cooked
+  version) and render the README.md as HTML, make further requires pandoc and
+  GNU awk for the build process.
+
+* ap-check:
+  For building the ap-check mdevctl callout utility you need liblockfile
+  version 1.14 or newer installed (liblockfile-devel.rpm).  Also required is
+  json-c version 0.13 or newer (json-c-devel.rpm).
+  Tip: you may skip ap-check build by adding `HAVE_LOCKFILE=0` or `HAVE_JSONC=0`
+  to the make invocation.

ap_tools/Makefile

@@ -0,0 +1,47 @@
+include ../common.mak
+
+MDEVCTL_DIR = /etc/mdevctl.d/
+MDEVCTL_SCRIPTS = /etc/mdevctl.d/scripts.d/
+MDEVCTL_CALLOUTS = /etc/mdevctl.d/scripts.d/callouts/
+
+libs =	$(rootdir)/libap/libap.a \
+	$(rootdir)/libutil/libutil.a
+
+ifeq (${HAVE_LOCKFILE},0)
+all:
+	$(SKIP) HAVE_LOCKFILE=0
+
+install:
+	$(SKIP) HAVE_LOCKFILE=0
+
+else ifeq (${HAVE_JSONC},0)
+all:
+	$(SKIP) HAVE_JSONC=0
+
+install:
+	$(SKIP) HAVE_JSONC=0
+else
+LDLIBS += -llockfile -ljson-c
+
+all: ap-check
+
+ap-check: ap-check.o $(libs)
+
+install: all
+	@if [ ! -d $(DESTDIR)$(MDEVCTL_CALLOUTS) ]; then \
+		mkdir -p $(DESTDIR)$(MDEVCTL_CALLOUTS); \
+		chown $(OWNER).$(GROUP) $(DESTDIR)$(MDEVCTL_DIR); \
+		chown $(OWNER).$(GROUP) $(DESTDIR)$(MDEVCTL_SCRIPTS); \
+		chown $(OWNER).$(GROUP) $(DESTDIR)$(MDEVCTL_CALLOUTS); \
+		chmod 755 $(DESTDIR)$(MDEVCTL_DIR); \
+		chmod 755 $(DESTDIR)$(MDEVCTL_SCRIPTS); \
+		chmod 755 $(DESTDIR)$(MDEVCTL_CALLOUTS); \
+	fi; \
+	$(INSTALL) -g $(GROUP) -o $(OWNER) -m 755 ap-check \
+		$(DESTDIR)$(MDEVCTL_CALLOUTS)
+endif
+
+clean:
+	rm -f *.o *~ ap-check core
+
+.PHONY: all install clean

ap_tools/ap-check.c

@@ -0,0 +1,911 @@
+/*
+ * ap-check - Validate vfio-ap mediated device configuration changes
+ *
+ * This tool in intended to be driven via the callout API of the mdevctl
+ * utility (https://github.com/mdevctl/mdevctl/)
+ *
+ * Copyright IBM Corp. 2022
+ *
+ * s390-tools is free software; you can redistribute it and/or modify
+ * it under the terms of the MIT license. See LICENSE for details.
+ */
+
+#include <dirent.h>
+#include <err.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <getopt.h>
+#include <stdio.h>
+#include <string.h>
+#include <unistd.h>
+
+#include "lib/ap.h"
+#include "lib/util_base.h"
+#include "lib/util_libc.h"
+#include "lib/util_opt.h"
+#include "lib/util_path.h"
+
+#include "ap-check.h"
+
+static const struct mdevctl_action mdevctl_action_table[NUM_MDEVCTL_ACTIONS] = {
+	{MDEVCTL_ACTION_DEFINE, "define"},
+	{MDEVCTL_ACTION_LIST, "list"},
+	{MDEVCTL_ACTION_MODIFY, "modify"},
+	{MDEVCTL_ACTION_START, "start"},
+	{MDEVCTL_ACTION_STOP, "stop"},
+	{MDEVCTL_ACTION_TYPES, "types"},
+	{MDEVCTL_ACTION_UNDEFINE, "undefine"},
+	{MDEVCTL_ACTION_ATTRIBUTES, "attributes"}
+};
+
+static const struct mdevctl_event mdevctl_event_table[NUM_MDEVCTL_EVENTS] = {
+	{MDEVCTL_EVENT_PRE, "pre"},
+	{MDEVCTL_EVENT_POST, "post"},
+	{MDEVCTL_EVENT_GET, "get"}
+};
+
+/*
+ * Convert mdevctl action string to an enumerated value
+ */
+static enum mdevctl_action_id validate_action(char *action)
+{
+	int i;
+
+	for (i = 0; i < NUM_MDEVCTL_ACTIONS; i++) {
+		if (strcmp(action, mdevctl_action_table[i].action) == 0)
+			return mdevctl_action_table[i].id;
+	}
+
+	return MDEVCTL_ACTION_UNKNOWN;
+}
+
+/*
+ * Convert mdevctl event string to an enumerated value
+ */
+static enum mdevctl_event_id validate_event(char *event)
+{
+	int i;
+
+	for (i = 0; i < NUM_MDEVCTL_EVENTS; i++) {
+		if (strcmp(event, mdevctl_event_table[i].event) == 0)
+			return mdevctl_event_table[i].id;
+	}
+
+	return MDEVCTL_EVENT_UNKNOWN;
+}
+
+static struct util_opt opt_vec[] = {
+	UTIL_OPT_SECTION("DEVICE"),
+	{
+		.option = { "e", required_argument, NULL, 'e' },
+		.argument = "EVENT",
+		.desc = "The type of callout being issued",
+	},
+	{
+		.option = { "a", required_argument, NULL, 'a' },
+		.argument = "ACTION",
+		.desc = "The action being performed on the specified device",
+	},
+	{
+		.option = { "s", required_argument, NULL, 's' },
+		.argument = "STATE",
+		.desc = "The state of the associated mdevctl command",
+	},
+	{
+		.option = { "u", required_argument, NULL, 'u' },
+		.argument = "UUID",
+		.desc = "Universally Unique ID for the mediated device",
+	},
+	{
+		.option = { "p", required_argument, NULL, 'p' },
+		.argument = "PDEV",
+		.desc = "Parent device name, e.g. matrix",
+	},
+	{
+		.option = { "t", required_argument, NULL, 't' },
+		.argument = "TYPE",
+		.desc = "Mediated device type, e.g. vfio_ap-passthrough",
+	},
+	UTIL_OPT_END
+};
+
+/*
+ * Initialize the ap_check anchor struct.
+ */
+static void ap_check_init(struct ap_check_anchor *anc)
+{
+	anc->uuid = anc->parent = anc->type = NULL;
+	anc->dev = vfio_ap_device_new();
+	anc->cleanup_lock = false;
+}
+
+/*
+ * Free memory of ap_check anchor struct.
+ */
+static void ap_check_cleanup(struct ap_check_anchor *anc)
+{
+	if (anc->uuid)
+		free(anc->uuid);
+	if (anc->parent)
+		free(anc->parent);
+	if (anc->type)
+		free(anc->type);
+	if (anc->dev)
+		vfio_ap_device_free(anc->dev);
+	if (anc->cleanup_lock)
+		ap_release_lock();
+}
+
+/*
+ * Exit ap_check
+ */
+static void __noreturn ap_check_exit(struct ap_check_anchor *anc, int rc)
+{
+	ap_check_cleanup(anc);
+	exit(rc);
+}
+
+/*
+ * parses the command line
+ */
+static void ap_check_parse(struct ap_check_anchor *anc,
+			   int argc, char *argv[])
+{
+	bool action = false, event = false, state = false, bad_opts = false;
+	int opt;
+
+	util_opt_init(opt_vec, NULL);
+
+	while (1) {
+		opt = util_opt_getopt_long(argc, argv);
+		if (opt == -1)
+			break;
+		switch (opt) {
+		case 'e':
+			if (event) {
+				bad_opts = true;
+			} else {
+				anc->event = validate_event(optarg);
+				event = true;
+			}
+			break;
+		case 'a':
+			if (action) {
+				bad_opts = true;
+			} else {
+				anc->action = validate_action(optarg);
+				action = true;
+			}
+			break;
+		case 's':
+			if (state) {
+				bad_opts = true;
+			} else {
+				/* Ignore the state */
+				state = true;
+			}
+			break;
+		case 'u':
+			if (anc->uuid)
+				bad_opts = true;
+			else
+				anc->uuid = util_strdup(optarg);
+			break;
+		case 'p':
+			if (anc->parent)
+				bad_opts = true;
+			else
+				anc->parent = util_strdup(optarg);
+			break;
+		case 't':
+			if (anc->type)
+				bad_opts = true;
+			else
+				anc->type = util_strdup(optarg);
+			break;
+		default:
+			fprintf(stderr, "Unknown operand\n");
+			ap_check_exit(anc, EXIT_FAILURE);
+		}
+	}
+
+	/* Make sure we got all expected input values */
+	if (!(action && event && state && anc->uuid && anc->parent &&
+	      anc->type) || bad_opts) {
+		fprintf(stderr, "Duplicate or missing operand\n");
+		ap_check_exit(anc, EXIT_FAILURE);
+	}
+
+	/* Check for invalid UUID */
+	if (!is_valid_uuid(anc->uuid)) {
+		fprintf(stderr, "Invalid UUID specified\n");
+		ap_check_exit(anc, EXIT_FAILURE);
+	}
+	anc->dev->uuid = util_strdup(anc->uuid);
+
+	/* Check for valid type */
+	if (strcmp(anc->type, VFIO_AP_TYPE) != 0)
+		ap_check_exit(anc, APC_EXIT_UNKNOWN_TYPE);
+
+	/* Check for invalid parent - currently only 'matrix' supported */
+	if (strcmp(anc->parent, "matrix") != 0) {
+		fprintf(stderr, "Invalid parent specified\n");
+		ap_check_exit(anc, EXIT_FAILURE);
+	}
+}
+
+/*
+ * Call a function for each entry in a directory:
+ * int callback(const char *abs_path, const char *rel_path, void *data)
+ * Continues for all entries in the directory regardless of callback return
+ * code.  Will return 0 or, if one or more callbacks failed, the first nonzero
+ * rc received.
+ */
+static int path_for_each(const char *path,
+			 int (*callback)(const char *, const char *, void *),
+			 void *data)
+{
+	struct dirent *de;
+	int rc = 0;
+	int r = 0;
+	DIR *dir;
+	char *p;
+
+	dir = opendir(path);
+	if (!dir)
+		return -1;
+
+	while ((de = readdir(dir))) {
+		if (strcmp(de->d_name, ".") == 0 ||
+		    strcmp(de->d_name, "..") == 0)
+			continue;
+		util_asprintf(&p, "%s/%s", path, de->d_name);
+		r = callback(p, de->d_name, data);
+		/* Save first nonzero return code for caller */
+		if (rc == 0 && r != 0)
+			rc = r;
+		free(p);
+	}
+
+	closedir(dir);
+
+	return rc;
+}
+
+/*
+ * Report an error message when the specified configuration will conflict
+ * with an existing device
+ */
+static void conflict_error(const char *uuid, unsigned int a, unsigned int d,
+			   bool persistent)
+{
+	if (uuid) {
+		if (persistent) {
+			fprintf(stderr,
+				"APQN %u.%u is defined for autostart by %s\n",
+				a, d, uuid);
+		} else {
+			fprintf(stderr, "APQN %u.%u already in use by %s\n",
+				a, d, uuid);
+		}
+	} else {
+		if (persistent) {
+			fprintf(stderr, "AQPN %u.%u is not defined for "
+				"vfio_ap-passthrough use by the persistent "
+				"ap bus mask settings\n", a, d);
+		} else {
+			fprintf(stderr, "AQPN %u.%u is not allowed for "
+				"vfio_ap-passthrough use by the active ap "
+				"bus mask settings\n", a, d);
+		}
+	}
+}
+
+/*
+ * Compare the list of adapters and domains for two devices, reporting error
+ * messages for any conflicts that occur.  A conflict occurs when both devices
+ * have the same adapter + domain pair.
+ * The function below takes advantage of the fact that the lists are known to
+ * be sorted in numeric order; therefore we can use this information to run
+ * the lists in parallel rather than always starting from the beginning.
+ */
+static int find_apqn_conflicts(const char *uuid,
+			       struct util_list *adapters,
+			       struct util_list *domains,
+			       struct util_list *adapters2,
+			       struct util_list *domains2,
+			       bool persistent)
+{
+	struct vfio_ap_node *a, *a2, *d, *d2;
+	int rc = 0;
+
+	/* Checks for conflicts with the device */
+	a = util_list_start(adapters);
+	a2 = util_list_start(adapters2);
+	while ((a != NULL) && (a2 != NULL)) {
+		if (a->id == a2->id) {
+			d = util_list_start(domains);
+			d2 = util_list_start(domains2);
+			while ((d != NULL) && (d2 != NULL)) {
+				if (d->id == d2->id) {
+					/* Report error, look for more */
+					conflict_error(uuid, a->id, d->id,
+						       persistent);
+					rc = -1;
+					d = util_list_next(domains, d);
+					d2 = util_list_next(domains2, d2);
+				} else if (d->id > d2->id) {
+					d2 = util_list_next(domains2, d2);
+				} else {
+					d = util_list_next(domains, d);
+				}
+			}
+			a = util_list_next(adapters, a);
+			a2 = util_list_next(adapters2, a2);
+		} else if (a->id > a2->id) {
+			a2 = util_list_next(adapters2, a2);
+		} else {
+			a = util_list_next(adapters, a);
+		}
+	}
+
+	return rc;
+}
+
+/*
+ * If the provided path maps to a valid vfio-ap device configuration,
+ * determine if its current configuration will conflict with the proposed
+ * changes.
+ */
+static int check_other_mdev_cfg_cb(const char *path,
+				   const char *filename,
+				   void *data)
+{
+	struct other_mdev_cb_data *cbdata = data;
+	struct vfio_ap_device *dev = cbdata->dev;
+	struct vfio_ap_device *dev2 = NULL;
+	int rc = 0;
+
+	/* Skip anything that isn't an mdev config */
+	if (!is_valid_uuid(filename))
+		goto out;
+
+	/* Skip if this is the input device */
+	if (strcasecmp(cbdata->uuid, filename) == 0)
+		goto out;
+
+	/* Read the device config */
+	dev2 = vfio_ap_device_new();
+	if (vfio_ap_read_device_config(path, dev2) != 0)
+		goto out;
+
+	/* If wrong device type, skip */
+	if (strcmp(dev2->type, VFIO_AP_TYPE) != 0)
+		goto out;
+
+	/* If not AUTO device, skip */
+	if (dev2->manual)
+		goto out;
+
+	/* Perform mdev-to-mdev apqn conflict analysis */
+	rc = find_apqn_conflicts(filename, dev->adapters, dev->domains,
+				 dev2->adapters, dev2->domains, true);
+
+out:
+	if (dev2 != NULL)
+		vfio_ap_device_free(dev2);
+	return rc;
+}
+
+/*
+ * Perform conflict analysis against all other vfio-ap persistent
+ * configurations.
+ */
+int check_other_mdevs_cfg(struct ap_check_anchor *anc)
+{
+	struct other_mdev_cb_data cb_data;
+
+	if (!util_path_is_dir(VFIO_AP_CONFIG_PATH))
+		return 0;
+
+	cb_data.uuid = anc->uuid;
+	cb_data.dev = anc->dev;
+
+	return path_for_each(VFIO_AP_CONFIG_PATH, check_other_mdev_cfg_cb,
+			     &cb_data);
+}
+
+/*
+ * If the provided path maps to a valid device, determine if its current
+ * configuration will conflict with the proposed changes.
+ */
+static int check_other_mdev_sysfs_cb(const char *path, const char *filename,
+				     void *data)
+{
+	struct other_mdev_cb_data *cbdata = data;
+	struct vfio_ap_device *dev = cbdata->dev;
+	struct vfio_ap_device *dev2;
+	char *matrix_path;
+	char buf[80];
+	int rc = 0;
+	FILE *f;
+
+	if (!is_valid_uuid(filename) || path == NULL ||
+	    strcasecmp(filename, cbdata->uuid) == 0)
+		return 0;
+
+	dev2 = vfio_ap_device_new();
+	matrix_path = path_get_vfio_ap_attr(filename, "matrix");
+	f = fopen(matrix_path, "r");
+	while (fgets(buf, sizeof(buf), f))
+		vfio_ap_parse_matrix(dev2, buf);
+	vfio_ap_sort_matrix_results(dev2);
+	fclose(f);
+	free(matrix_path);
+
+	/* Look for conflicts between target device and this device */
+	rc = find_apqn_conflicts(filename, dev->adapters, dev->domains,
+				 dev2->adapters, dev2->domains, false);
+
+	vfio_ap_device_free(dev2);
+
+	return rc;
+}
+
+/* Run conflict analysis against all other active vfio-ap devices */
+static int check_other_mdevs_sysfs(struct ap_check_anchor *anc)
+{
+	struct other_mdev_cb_data cb_data;
+	char *root;
+	int rc = 0;
+
+	cb_data.uuid = anc->uuid;
+	cb_data.dev = anc->dev;
+
+	root = path_get_vfio_ap_mdev("");
+	if (util_path_is_dir(root))
+		rc = path_for_each(root, check_other_mdev_sysfs_cb, &cb_data);
+
+	free(root);
+	return rc;
+}
+
+/*
+ * Determine if there are any conflicts between the specified device and
+ * the active apmask/aqmask settings.  This is done by treating the masks
+ * as a temporary vfio_ap_device with all of the associated APQNs owned by
+ * the system.
+ */
+static int check_sysfs_mask_conflicts(struct ap_check_anchor *anc)
+{
+	struct vfio_ap_device *sysdev = vfio_ap_device_new();
+	char *apmask = util_zalloc(AP_MASK_SIZE);
+	char *aqmask = util_zalloc(AP_MASK_SIZE);
+	int rc = 0;
+
+	if (ap_read_sysfs_masks(apmask, aqmask, AP_MASK_SIZE) != 0) {
+		fprintf(stderr, "Error reading system AP settings\n");
+		rc = -1;
+		goto out;
+	}
+
+	/* Convert the masks to a device with the associated APQNs */
+	ap_mask_to_list(apmask, sysdev->adapters);
+	ap_mask_to_list(aqmask, sysdev->domains);
+
+	/* Perform conflict analysis */
+	rc = find_apqn_conflicts(NULL, anc->dev->adapters,
+				 anc->dev->domains, sysdev->adapters,
+				 sysdev->domains, false);
+out:
+	free(apmask);
+	free(aqmask);
+	vfio_ap_device_free(sysdev);
+
+	return rc;
+}
+
+/*
+ * Determine if there are any conflicts between the specified device and
+ * the apmask/aqmask settings stored in udev.  This is done by treating
+ * the masks as a temporary vfio_ap_device with all of the associated
+ * AQPNs owned by the system.
+ */
+static int check_cfg_mask_conflicts(struct ap_check_anchor *anc)
+{
+	struct vfio_ap_device *sysdev = vfio_ap_device_new();
+	char *apmask = util_zalloc(AP_MASK_SIZE);
+	char *aqmask = util_zalloc(AP_MASK_SIZE);
+	bool read_ap = false, read_aq = false;
+	char *path;
+	int rc = 0;
+
+	path = path_get_ap_udev();
+	if (!ap_read_udev_masks(path, apmask, aqmask, &read_ap, &read_aq)) {
+		fprintf(stderr, "Error reading system AP settings\n");
+		rc = -1;
+		goto out;
+	}
+
+	/* Convert the masks to a device with the associated APQNs */
+	ap_mask_to_list(apmask, sysdev->adapters);
+	ap_mask_to_list(aqmask, sysdev->domains);
+
+	/* Perform conflict analysis */
+	rc = find_apqn_conflicts(NULL, anc->dev->adapters,
+				 anc->dev->domains, sysdev->adapters,
+				 sysdev->domains, true);
+out:
+	free(apmask);
+	free(aqmask);
+	free(path);
+	vfio_ap_device_free(sysdev);
+
+	return rc;
+}
+
+/* Subroutine to handle checking shared between DEFINE and MODIFY actions. */
+static int ap_check_changes(struct ap_check_anchor *anc)
+{
+	int rc = 0, rc2;
+
+	rc = ap_get_lock_callout();
+	if (rc) {
+		fprintf(stderr, "Failed to acquire configuration lock %d\n",
+			rc);
+		rc = -1;
+		goto out;
+	}
+	anc->cleanup_lock = true;
+
+	if (vfio_ap_read_device_config(NULL, anc->dev) != 0) {
+		fprintf(stderr, "Failed to read device config\n");
+		rc = -1;
+		goto out;
+	}
+
+	if (strcmp(anc->dev->type, anc->type) != 0) {
+		fprintf(stderr, "Invalid mdev_type: %s\n", anc->dev->type);
+		rc = -1;
+		goto out;
+	}
+
+	if (!anc->dev->manual) {
+		/* Check against all other AUTO config files */
+		rc = check_other_mdevs_cfg(anc);
+		/* Check against the system UDEV rule for apmask/aqmask */
+		rc2 = check_cfg_mask_conflicts(anc);
+		/* If either hit an error, reflect this */
+		rc = rc != 0 ? rc : rc2;
+	}
+
+	/* If successful, lock must remain held until post callout */
+	if (rc == 0)
+		anc->cleanup_lock = false;
+
+out:
+	return rc;
+}
+
+/*
+ * Determine if defining the specified device is a valid operation.
+ * mdevctl can reach us for a DEFINE under the following circumstances:
+ * 1) the device does not exist
+ * 2) the device is active but does not have a config file, so this action
+ *    would be to generate a config file based upon the active device.
+ * DEFINE has no effect on an active device (if one exists) it only creates
+ * the configuration file.  The config file might be empty or may have various
+ * attributes if being fed by --jsonfile or an active device.
+ */
+static int ap_check_handle_define(struct ap_check_anchor *anc)
+{
+	char *path = path_get_vfio_ap_mdev_config(anc->uuid);
+
+	if (util_path_is_readable(path)) {
+		fprintf(stderr, "Config already exists\n");
+		free(path);
+		return -1;
+	}
+
+	free(path);
+
+	return ap_check_changes(anc);
+}
+
+/*
+ * Determine if modifying the specified device is a valid operation.
+ * mdevctl can reach us for a MODIFY under the following circumstances:
+ * 1) Modifying a MANUAL device
+ * 2) Modifying an AUTO device
+ * In the case of MANUAL, we don't take any action because changes made via
+ * MODIFY don't take affect on the active mdev until a STOP/START cycle.
+ * In the case of AUTO, we must compare the contents of the proposed device
+ * with the contents of stashed AUTO mdev configurations + the system.
+ */
+static int ap_check_handle_modify(struct ap_check_anchor *anc)
+{
+	char *path = path_get_vfio_ap_mdev_config(anc->uuid);
+	FILE *fd = fopen(path, "r");
+
+	/* Determine if a base config file already exists for UUID */
+	free(path);
+	if (fd == NULL) {
+		fprintf(stderr, "Config doesn't exist\n");
+		return -1;
+	}
+	fclose(fd);
+
+	return ap_check_changes(anc);
+}
+
+/*
+ * Determine if starting the specified device is a valid operation.
+ * mdevctl can reach us for a START under the following circumstances:
+ * 1) STARTing a MANUAL device
+ *    1a) Where the MANUAL device is defined (has a config file)
+ *    1b) Where the MANUAL device is NOT defined (no config file).
+ *        For vfio-ap this case provides an mdev with no adapters/domains.
+ *    1c) Where the MANUAL device is NOT defined but a full configuration is
+ *        provided via --jsonfile
+ * 2) STARTing an AUTO device
+ *    2a) Where the AUTO device is defined (has a config file)
+ *    2b) Where the AUTO device is NOT defined but a full configuration is
+ *        provided via --jsonfile
+ * In each case, we must compare the proposed device with the contents of
+ * active mdevs + the system.
+ */
+static int ap_check_handle_start(struct ap_check_anchor *anc)
+{
+	int rc = 0, rc2;
+
+	/* Can only start a device if vfio_ap is built-in or loaded */
+	if (!util_path_is_dir(VFIO_AP_PATH)) {
+		fprintf(stderr, "vfio_ap module is not loaded\n");
+		ap_check_exit(anc, EXIT_FAILURE);
+	}
+
+	rc = ap_get_lock_callout();
+	if (rc) {
+		fprintf(stderr, "Failed to acquire configuration lock %d\n",
+			rc);
+		rc = -1;
+		goto out;
+	}
+	anc->cleanup_lock = true;
+
+	if (vfio_ap_read_device_config(NULL, anc->dev) != 0) {
+		fprintf(stderr, "Failed to read device config\n");
+		rc = -1;
+		goto out;
+	}
+
+	if (strcmp(anc->dev->type, anc->type) != 0) {
+		fprintf(stderr, "Invalid mdev_type: %s\n", anc->dev->type);
+		rc = -1;
+		goto out;
+	}
+
+	/* Check against all other active vfio-ap devices */
+	rc = check_other_mdevs_sysfs(anc);
+	/* Check against the system sysfs values for apmask/aqmask */
+	rc2 = check_sysfs_mask_conflicts(anc);
+	/* If either hit an error, reflect this */
+	rc = rc != 0 ? rc : rc2;
+
+	/* If successful, lock must remain held until post callout */
+	if (rc == 0)
+		anc->cleanup_lock = false;
+
+out:
+	return rc;
+}
+
+/*
+ * Acquire the appropriate serialization so that the specified device can be
+ * STOPped.
+ */
+static int ap_check_handle_stop(void)
+{
+	int rc;
+
+	rc = ap_get_lock_callout();
+	if (rc) {
+		fprintf(stderr, "Failed to acquire configuration lock %d\n",
+			rc);
+		return -1;
+	}
+
+	/* The lock must remain held until post callout */
+	return 0;
+}
+
+/*
+ * Determine if UNDEFINEing the specified device is a valid operation.
+ * mdevctl can reach us for an UNDEFINE under the following circumstances:
+ * 1) UNDEFINEing an active device
+ * 2) UNDEFINEing an inactive device
+ * UNDEFINE has no effect on the active device, it only removes the config
+ * file.
+ */
+static int ap_check_handle_undefine(struct ap_check_anchor *anc)
+{
+	char *path = path_get_vfio_ap_mdev_config(anc->uuid);
+	int rc = 0;
+
+	rc = ap_get_lock_callout();
+	if (rc) {
+		fprintf(stderr, "Failed to acquire configuration lock %d\n",
+			rc);
+		rc = -1;
+		goto out;
+	}
+	anc->cleanup_lock = true;
+
+	if (vfio_ap_read_device_config(path, anc->dev) != 0) {
+		fprintf(stderr, "Failed to read device config\n");
+		rc = -1;
+		goto out;
+	}
+
+	if (strcmp(anc->dev->type, anc->type) != 0) {
+		fprintf(stderr, "Invalid mdev_type: %s\n", anc->dev->type);
+		rc = -1;
+		goto out;
+	}
+
+	/* Success: lock must remain held until post callout */
+	anc->cleanup_lock = false;
+
+out:
+	free(path);
+	return rc;
+}
+
+/*
+ * For callouts where the "pre" callout would have acquired the lock, it is
+ * now safe to remove the lock as all changes have been committed.
+ */
+static int ap_check_handle_post(void)
+{
+	return ap_release_lock();
+}
+
+/* For the specified device, print the attributes to stdout in JSON format */
+static int ap_check_handle_get_attributes(struct ap_check_anchor *anc)
+{
+	struct vfio_ap_device *dev = anc->dev;
+	struct vfio_ap_node *node;
+	bool has_attr = false;
+	char buf[80];
+	char *path;
+	FILE *f;
+
+	path = path_get_vfio_ap_attr(anc->uuid, "matrix");
+	f = fopen(path, "r");
+	while (fgets(buf, sizeof(buf), f))
+		vfio_ap_parse_matrix(dev, buf);
+	vfio_ap_sort_matrix_results(dev);
+	fclose(f);
+	free(path);
+
+	path = path_get_vfio_ap_attr(anc->uuid, "control_domains");
+	f = fopen(path, "r");
+	while (fgets(buf, sizeof(buf), f))
+		vfio_ap_parse_control(dev, buf);
+	fclose(f);
+	free(path);
+
+	printf("[{");
+
+	if (!util_list_is_empty(dev->adapters)) {
+		util_list_iterate(dev->adapters, node) {
+			if (has_attr)
+				printf("},{");
+			printf("\"assign_adapter\": \"%u\"", node->id);
+			has_attr = true;
+		}
+	}
+
+	if (!util_list_is_empty(dev->domains)) {
+		util_list_iterate(dev->domains, node) {
+			if (has_attr)
+				printf("},{");
+			printf("\"assign_domain\": \"%u\"", node->id);
+			has_attr = true;
+		}
+	}
+
+	if (!util_list_is_empty(dev->controls)) {
+		util_list_iterate(dev->controls, node) {
+			if (has_attr)
+				printf("},{");
+			printf("\"assign_control_domain\": \"%u\"", node->id);
+			has_attr = true;
+		}
+	}
+
+	printf("}]\n");
+
+	return 0;
+}
+
+/*
+ * Determine which mdevctl action is being checked and handle accordingly.
+ */
+static int ap_check_handle_action(struct ap_check_anchor *anc)
+{
+	int rc = 0;
+
+	switch (anc->event) {
+	case MDEVCTL_EVENT_PRE:
+		switch (anc->action) {
+		case MDEVCTL_ACTION_DEFINE:
+			rc = ap_check_handle_define(anc);
+			break;
+		case MDEVCTL_ACTION_MODIFY:
+			rc = ap_check_handle_modify(anc);
+			break;
+		case MDEVCTL_ACTION_START:
+			rc = ap_check_handle_start(anc);
+			break;
+		case MDEVCTL_ACTION_STOP:
+			rc = ap_check_handle_stop();
+			break;
+		case MDEVCTL_ACTION_UNDEFINE:
+			rc = ap_check_handle_undefine(anc);
+			break;
+		case MDEVCTL_ACTION_LIST:
+		case MDEVCTL_ACTION_TYPES:
+		default:
+			/* Ignore some actions including unknown ones */
+			break;
+		}
+		break;
+	case MDEVCTL_EVENT_POST:
+		switch (anc->action) {
+		case MDEVCTL_ACTION_DEFINE:
+		case MDEVCTL_ACTION_MODIFY:
+		case MDEVCTL_ACTION_START:
+		case MDEVCTL_ACTION_STOP:
+		case MDEVCTL_ACTION_UNDEFINE:
+			ap_check_handle_post();
+			break;
+		default:
+			/* Ignore other post events */
+			break;
+		}
+		break;
+	case MDEVCTL_EVENT_GET:
+		switch (anc->action) {
+		case MDEVCTL_ACTION_ATTRIBUTES:
+			rc = ap_check_handle_get_attributes(anc);
+			break;
+		default:
+			/* Ignore some actions including unknown ones */
+			break;
+		}
+		break;
+	default:
+		/* Ignore any unknown events */
+		break;
+	}
+
+	return rc;
+}
+
+/*
+ *
+ */
+int main(int argc, char *argv[])
+{
+	struct ap_check_anchor anchor;
+	int rc;
+
+	ap_check_init(&anchor);
+
+	ap_check_parse(&anchor, argc, argv);
+
+	rc = ap_check_handle_action(&anchor);
+
+	ap_check_exit(&anchor, rc);
+}

ap_tools/ap-check.h

@@ -0,0 +1,74 @@
+/*
+ * ap-check - Validate vfio-ap mediated device configuration changes
+ *
+ * Copyright IBM Corp. 2022
+ *
+ * s390-tools is free software; you can redistribute it and/or modify
+ * it under the terms of the MIT license. See LICENSE for details.
+ */
+
+#ifndef AP_CHECK_H
+#define AP_CHECK_H
+
+#include <stdbool.h>
+
+/*
+ * List of all of the supported mdevctl actions
+ */
+enum mdevctl_action_id {
+	MDEVCTL_ACTION_DEFINE = 0,
+	MDEVCTL_ACTION_LIST,
+	MDEVCTL_ACTION_MODIFY,
+	MDEVCTL_ACTION_START,
+	MDEVCTL_ACTION_STOP,
+	MDEVCTL_ACTION_TYPES,
+	MDEVCTL_ACTION_UNDEFINE,
+	MDEVCTL_ACTION_ATTRIBUTES,
+	/* UNKNOWN must always be the last in the list */
+	MDEVCTL_ACTION_UNKNOWN,
+};
+#define NUM_MDEVCTL_ACTIONS MDEVCTL_ACTION_UNKNOWN
+
+struct mdevctl_action {
+	enum mdevctl_action_id id;
+	const char action[32];
+};
+
+enum mdevctl_event_id {
+	MDEVCTL_EVENT_PRE = 0,
+	MDEVCTL_EVENT_POST,
+	MDEVCTL_EVENT_GET,
+	MDEVCTL_EVENT_UNKNOWN,
+};
+#define NUM_MDEVCTL_EVENTS MDEVCTL_EVENT_UNKNOWN
+
+struct mdevctl_event {
+	enum mdevctl_event_id id;
+	const char event[32];
+};
+
+/* ap-check special exit codes */
+#define APC_EXIT_UNKNOWN_TYPE 2
+
+struct ap_check_anchor {
+	enum mdevctl_event_id event;
+	enum mdevctl_action_id action;
+	char *uuid;
+	char *parent;
+	char *type;
+	struct vfio_ap_device *dev;
+	/* Active Masks */
+	char apmask[80];
+	char aqmask[80];
+	/* Persistent Masks */
+	char p_apmask[80];
+	char p_aqmask[80];
+	bool cleanup_lock;
+};
+
+struct other_mdev_cb_data {
+	const char *uuid;
+	struct vfio_ap_device *dev;
+};
+
+#endif /* AP_CHECK_H */

bin/mk-authors

@@ -32,7 +32,7 @@ Usage: $cmd [-t LAST_GIT_TAG]
 Add all new authors since last release to the $authors_file file. Without
 the -t option the tool uses the last available git tag as last release.
 
-The tool should be called form the s390-tools master branch.
+The tool runs on the s390-tools master branch.
 
 OPTIONS
 -t, --tag LAST_GIT_TAG  Use git tag LAST_GIT_TAG as starting point
@@ -116,7 +116,7 @@ echo "$cmd: Adding new authors since tag: $release_tag_last"
 	# Print authors list without header
 	tail -n +4 $authors_file_path
 	# Add the new authors
-	git log --format="- %an" $release_tag_last..
+	git log --format="- %an" $release_tag_last..master
 	# Then sort everything and remove duplicates
 } | sort | uniq > $authors_file_tmp
 # Create new AUTHORS file with header ...

chreipl-fcp-mpath/.chreipl-fcp-mpath.7.cksum

@@ -0,0 +1 @@
+8e604dab39577678bc9f24bfe5e2ee35354b2f4b1759edd16093157b1ecce9d3  README.md

chreipl-fcp-mpath/.gitignore

@@ -0,0 +1,11 @@
+# build artifacts
+/chreipl-fcp-mpath-common.sh
+/chreipl-fcp-mpath-is-ipl-tgt
+/chreipl-fcp-mpath-is-ipl-vol
+/chreipl-fcp-mpath-is-reipl-zfcp
+/chreipl-fcp-mpath-record-volume-identifier
+/chreipl-fcp-mpath-try-change-ipl-path
+/dracut/dracut.conf.d/70-chreipl-fcp-mpath.conf
+/README.html
+/README.pdf
+/chreipl-fcp-mpath.md

chreipl-fcp-mpath/Makefile

@@ -0,0 +1,274 @@
+# SPDX-License-Identifier: MIT
+#
+# chreipl-fcp-mpath: use multipath information to change FCP IPL target
+# (C) Copyright IBM Corp. 2021
+#
+# Uses the following system-utilities (and shell-builtins):
+#   Utilities list in GNU Make Conventions:
+#     https://www.gnu.org/software/make/manual/make.html#Utilities-in-Makefiles
+#   Those necessary for sourced Makefiles:
+#     - ../common.mak
+#     - chreipl-fcp-mpath.mak
+#   bash:
+#     - bash
+#   GNU coreutils:
+#     - sha256sum
+# If $(ENABLE_DOC) is `1`:
+#   GNU awk:
+#     - gawk
+
+override SHELL		:= /bin/bash
+override .SHELLFLAGS	:= -O globstar -O nullglob -O extglob -c
+
+# Include common s390-tools definitions
+include ../common.mak
+
+# Include common chreipl-fcp-mpath definitions
+include chreipl-fcp-mpath.mak
+
+# Local setting: .make.config
+#	You may create a file named like this in the same directory as this
+#	Makefile, and customize the build this way (e.g. re-define variables
+#	set in `chreipl-fcp-mpath.mak`, or define a `CHREIPLZFCPMP_POST_INSTALL`
+#	that is automatically called after each installation)
+ifneq ($(wildcard .make.config),)
+include $(wildcard .make.config)
+endif
+
+#
+## Build
+#
+
+.PHONY: chreipl-fcp-mpath chreipl-fcp-mpath-clean
+chreipl-fcp-mpath:
+chreipl-fcp-mpath-clean:
+all: chreipl-fcp-mpath
+clean: chreipl-fcp-mpath-clean
+
+# common function used in the helper scripts
+CHREIPL_FCP_MPATH_COMMON	:=					\
+	chreipl-fcp-mpath-common.sh
+CHREIPL_FCP_MPATH_UDEV_HELPER	:=					\
+	chreipl-fcp-mpath-is-ipl-tgt					\
+	chreipl-fcp-mpath-is-ipl-vol					\
+	chreipl-fcp-mpath-is-reipl-zfcp					\
+	chreipl-fcp-mpath-record-volume-identifier			\
+	chreipl-fcp-mpath-try-change-ipl-path
+
+$(CHREIPL_FCP_MPATH_UDEV_HELPER) $(CHREIPL_FCP_MPATH_COMMON): $(MAKEFILE_LIST)
+$(CHREIPL_FCP_MPATH_UDEV_HELPER) $(CHREIPL_FCP_MPATH_COMMON): % : %.in
+	$(call chreiplzfcpmp-sed-buildvar-replace,$(<),$(@))
+	chmod a+x $(@)
+
+$(CHREIPL_FCP_MPATH_UDEV_HELPER): $(CHREIPL_FCP_MPATH_COMMON)
+
+.PHONY: chreipl-fcp-mpath-udev-helper-clean
+chreipl-fcp-mpath-udev-helper-clean:
+	rm -f $(CHREIPL_FCP_MPATH_UDEV_HELPER) $(CHREIPL_FCP_MPATH_COMMON)
+
+udev/rules.d/70-chreipl-fcp-mpath.rules: $(CHREIPL_FCP_MPATH_UDEV_HELPER)
+
+chreipl-fcp-mpath: udev/rules.d/70-chreipl-fcp-mpath.rules
+chreipl-fcp-mpath-clean: chreipl-fcp-mpath-udev-helper-clean
+
+dracut/dracut.conf.d/70-chreipl-fcp-mpath.conf: $(MAKEFILE_LIST)
+dracut/dracut.conf.d/70-chreipl-fcp-mpath.conf: % : %.in
+	$(call chreiplzfcpmp-sed-buildvar-replace,$(<),$(@))
+
+.PHONY: chreipl-fcp-mpath-dracut-clean
+chreipl-fcp-mpath-dracut-clean:
+	rm -f dracut/dracut.conf.d/70-chreipl-fcp-mpath.conf
+
+chreipl-fcp-mpath: dracut/dracut.conf.d/70-chreipl-fcp-mpath.conf
+chreipl-fcp-mpath-clean: chreipl-fcp-mpath-dracut-clean
+
+ifeq ($(ENABLE_DOC),1)
+
+.PHONY: chreipl-fcp-mpath-doc
+chreipl-fcp-mpath-doc: README.html chreipl-fcp-mpath.7
+
+ifeq ($(ENABLE_DOC_PDF),1)
+chreipl-fcp-mpath-doc: README.pdf
+
+README.pdf: PANDOCFLAGS += -M title="chreipl-fcp-mpath" -M subtitle="README"
+endif
+
+README.html: PANDOCFLAGS += -M title="chreipl-fcp-mpath" -M subtitle="README"
+
+chreiplzfcpmp-doc-man-meta =						\
+	-M title="CHREIPL-FCP-MPATH"					\
+	-V header="Administrator Manual"				\
+	-V section="7"							\
+	-V footer="s390-tools $(S390_TOOLS_RELEASE)"
+
+chreipl-fcp-mpath.7: PANDOCFLAGS += $(chreiplzfcpmp-doc-man-meta)
+chreipl-fcp-mpath.7: .chreipl-fcp-mpath.7.cksum
+
+.INTERMEDIATE: chreipl-fcp-mpath.md
+chreipl-fcp-mpath.md: README.md
+	gawk -- '/NOT-IN-MAN \{/,/NOT-IN-MAN \}/ { next } { print }' $(<) > $(@)
+
+.chreipl-fcp-mpath.7.cksum: README.md
+	sha256sum $(<) > $(@)
+
+.PHONY: chreipl-fcp-mpath-doc-clean
+chreipl-fcp-mpath-doc-clean:
+	rm -f README.html README.pdf chreipl-fcp-mpath.md
+
+chreipl-fcp-mpath: chreipl-fcp-mpath-doc
+chreipl-fcp-mpath-clean: chreipl-fcp-mpath-doc-clean
+
+else # $(ENABLE_DOC) != 1
+
+# We bundle a pre-cooked man page with the source-code so that distributions
+# don't need `pandoc` in order to be able to ship the man page. As of this
+# writing multiple distributions don't have a packaged version of it.
+#
+# In order to remember to regenerate this pre-cooked version whenever the
+# README.md is changed, we also generate a checksum of the README.md, bundle
+# that as well, and compare that whenever `make` is called. This way, the
+# freshness of the man page can be checked, even if `pandoc` is not available,
+# or ENABLE_DOC disabled.
+
+MANPAGE_FRESH	:= $(shell sha256sum --check .chreipl-fcp-mpath.7.cksum >/dev/null && echo 1 || echo 0)
+ifeq ($(MANPAGE_FRESH),0)
+$(warning chreipl-fcp-mpath.7 is outdated, please regenerate it by calling `make ENABLE_DOC=1`)
+endif
+
+endif # $(ENABLE_DOC) == 1
+
+#
+## Install
+#
+
+.PHONY: chreipl-fcp-mpath-install
+# The content of `CHREIPLZFCPMP_POST_INSTALL` (bash script) is automatically
+# called *after* installing chreipl-fcp-mpath during `make install`. If not
+# defined (the default), nothing happens. You may define this on the make
+# command line, or by creating a `.make.config` and defining the variable in
+# there.
+chreipl-fcp-mpath-install:
+	$(CHREIPLZFCPMP_POST_INSTALL)
+
+install: chreipl-fcp-mpath-install
+
+# install udev rules
+INSTDIRS += $(UDEVRULESDIR)
+$(DESTDIR)$(UDEVRULESDIR): install_dirs
+
+.PHONY: chreipl-fcp-mpath-install-udev-rules
+chreipl-fcp-mpath-install-udev-rules: | $(DESTDIR)$(UDEVRULESDIR)
+chreipl-fcp-mpath-install-udev-rules: udev/rules.d/70-chreipl-fcp-mpath.rules
+	$(INSTALL_DATA) -t $(DESTDIR)$(UDEVRULESDIR)			\
+		udev/rules.d/70-chreipl-fcp-mpath.rules
+
+chreipl-fcp-mpath-install: chreipl-fcp-mpath-install-udev-rules
+
+# install udev helper programs
+INSTDIRS += $(UDEVDIR)
+$(DESTDIR)$(UDEVDIR): install_dirs
+
+.PHONY: chreipl-fcp-mpath-install-udev-helper
+chreipl-fcp-mpath-install-udev-helper: | $(DESTDIR)$(UDEVDIR)
+chreipl-fcp-mpath-install-udev-helper: $(CHREIPL_FCP_MPATH_UDEV_HELPER)
+	$(INSTALL_EXEC) -t $(DESTDIR)$(UDEVDIR) $(CHREIPL_FCP_MPATH_UDEV_HELPER)
+
+chreipl-fcp-mpath-install: chreipl-fcp-mpath-install-udev-helper
+
+# install common library files
+INSTDIRS += $(CHREIPLZFCPMPDIR)
+$(DESTDIR)$(CHREIPLZFCPMPDIR): install_dirs
+
+.PHONY: chreipl-fcp-mpath-install-libfiles
+chreipl-fcp-mpath-install-libfiles: | $(DESTDIR)$(CHREIPLZFCPMPDIR)
+chreipl-fcp-mpath-install-libfiles: $(CHREIPL_FCP_MPATH_COMMON)
+	$(INSTALL_DATA) -t $(DESTDIR)$(CHREIPLZFCPMPDIR)		\
+		$(CHREIPL_FCP_MPATH_COMMON)
+
+chreipl-fcp-mpath-install: chreipl-fcp-mpath-install-libfiles
+
+ifeq ($(HAVE_DRACUT),1)
+
+# install dracut config files
+INSTDIRS += $(DRACUTCONFDIR)
+$(DESTDIR)$(DRACUTCONFDIR): install_dirs
+
+.PHONY: chreipl-fcp-mpath-install-dracut-config
+chreipl-fcp-mpath-install-dracut-config: | $(DESTDIR)$(DRACUTCONFDIR)
+chreipl-fcp-mpath-install-dracut-config: dracut/dracut.conf.d/70-chreipl-fcp-mpath.conf
+	$(INSTALL_DATA) -t $(DESTDIR)$(DRACUTCONFDIR)			\
+		dracut/dracut.conf.d/70-chreipl-fcp-mpath.conf
+
+chreipl-fcp-mpath-install: chreipl-fcp-mpath-install-dracut-config
+
+endif
+
+# chreipl-fcp-mpath: install man page
+INSTDIRS += $(MANDIR)
+$(DESTDIR)$(MANDIR)/man7: install_dirs
+
+.PHONY: chreipl-fcp-mpath-install-man-page
+chreipl-fcp-mpath-install-man-page: | $(DESTDIR)$(MANDIR)/man7
+chreipl-fcp-mpath-install-man-page: chreipl-fcp-mpath.7
+	$(INSTALL_DATA) -t $(DESTDIR)$(MANDIR)/man7			\
+		chreipl-fcp-mpath.7
+
+chreipl-fcp-mpath-install: chreipl-fcp-mpath-install-man-page
+
+#
+## Utility
+#
+
+# Utilities for the debug feature of chreipl-fcp-mpath-common.sh.
+#
+# When `chreipl-fcp-mpath` is built with D=1 (default is D=0), each run
+# of one of the helper scripts will create a debug log in $(DEBUGOUTDIR)
+# (default: /run/udev) which among other things contains the complete shell
+# trace of that script run, with some added information that would not be
+# inspectable otherwise with just the trace.
+#
+# There is currently no way of enabling/disabling this at runtime.
+#
+# chreipl-fcp-mpath-common.sh defines debug log file name as
+# "chreiplzfcpmp-${debug_trace_tag}-${SEQNUM:-0}.XXXXXXXXXX" where SEQNUM is a
+# udev rule environment variable and each X is replaced with some [[:alnum:]]
+# by mktemp. For a definition of `debug_trace_tag`, please see the comments
+# in the source.
+#
+# The following targets can be used for some simple access and filtering of the
+# logs during development.
+
+DEBUG_LOG_GLOB = $(DEBUGOUTDIR)/chreiplzfcpmp-[[:digit:]][[:digit:]][[:alpha:]][[:alpha:]][[:alpha:]][[:alpha:]]-+([[:digit:]]).[[:alnum:]][[:alnum:]][[:alnum:]][[:alnum:]][[:alnum:]][[:alnum:]][[:alnum:]][[:alnum:]][[:alnum:]][[:alnum:]]
+
+# display all debug log files on the system
+.PHONY: chreipl-fcp-mpath-debug-logs
+chreipl-fcp-mpath-debug-logs:
+	@ls -1d $(DEBUG_LOG_GLOB)
+
+# display only debug log files of script runs that exited with status 0 (= good)
+#
+# XXX: `sed -n -e '$p'` is used instead of `tail -n1` to prevent an other
+#      dependency just for that (both invocations do the same thing).
+.PHONY: chreipl-fcp-mpath-debug-logs-filter-good
+chreipl-fcp-mpath-debug-logs-filter-good:
+	@for lg in $(DEBUG_LOG_GLOB); do				\
+		sed -e '/^+ trap_exit$$/,/^+ trap - EXIT$$/d' "$${lg}"	\
+		    | sed -n -e '$$p'					\
+		    | grep -q -e '^+ exit 0$$'		|| continue;	\
+		ls -d "$${lg}";						\
+	done
+
+# display only debug log files of script runs that didn't exit with status 0
+# (= bad)
+.PHONY: chreipl-fcp-mpath-debug-logs-filter-bad
+chreipl-fcp-mpath-debug-logs-filter-bad:
+	@for lg in $(DEBUG_LOG_GLOB); do				\
+		sed -e '/^+ trap_exit$$/,/^+ trap - EXIT$$/d' "$${lg}"	\
+		    | sed -n -e '$$p'					\
+		    | grep -q -e '^+ exit 0$$'		&& continue;	\
+		ls -d "$${lg}";						\
+	done
+
+.PHONY: chreipl-fcp-mpath-debug-logs-clean
+chreipl-fcp-mpath-debug-logs-clean:
+	rm -f $(DEBUG_LOG_GLOB)

chreipl-fcp-mpath/README.md

@@ -0,0 +1,270 @@
+<!-- markdown documentation: https://github.github.com/gfm/ -->
+
+NAME
+====
+
+chreipl-fcp-mpath - use multipath information for re-IPL path failover on a
+running Linux instance
+
+DESCRIPTION
+===========
+
+The IPL process of Linux on Z or LinuxONE from an FCP-attached SCSI volume uses
+exactly one path to the volume. If this path is unavailable, the IPL fails.
+
+The **chreipl-fcp-mpath** toolset monitors **udev** events about paths to the
+re-IPL volume. If the currently configured re-IPL path becomes unavailable, the
+toolset checks for operational paths to the same volume. If available, it
+reconfigures the re-IPL settings to use an operational path.
+
+Thus, re-IPL from an FCP-attached SCSI volume can be successful despite path
+failures on a running Linux instance if at least one path to the re-IPL volume
+remains operational.
+
+**Chreipl-fcp-mpath** requires **udev**, **multipathd** and **dm-multipath**.
+Once installed, the toolset runs automatically and autonomously. No user
+intervention is possible or required.
+
+Other than installing the toolset, there is no user interface for
+**chreipl-fcp-mpath**.
+
+Requirements
+------------
+
+The **chreipl-fcp-mpath** tool has the following requirements on the
+Linux instance that is being monitored:
+
+  - The Linux instance must have started successfully, during IPL.
+
+  - The running Linux instance must use **dm-multipath** and **multipathd** for
+    the configured re-IPL volume - a volume that contains a zipl boot record
+    and has one of its paths used in the re-IPL configuration.
+
+  - **udev** must run.
+
+  - The toolset must observe at least one event about the configured re-IPL
+    path. Examples for such events are: the SCSI disk comes online, or a path
+    of the corresponding multipath device goes down or comes back online.
+
+    - The WWID of the re-IPL volume must not change while the Linux instance is
+      running.
+
+  - When the configured re-IPL path becomes unavailable while the Linux
+    instance is running, at least one operational path to the re-IPL volume
+    must be available, or must become available. If no such path is available
+    when the Linux instance is rebooted, the re-IPL path is not changed.
+
+  - The tool assumes that any manually reconfigured re-IPL device is valid and
+    operational.
+
+    The tool treats a newly configured re-IPL device like the initially
+    configured re-IPL device. In particular, if the newly configured re-IPL
+    device fulfills the requirements of the tool, re-IPL path failover takes
+    place if the configured re-IPL path becomes unavailable.
+
+Caution with Manual Changes to the Configured re-IPL Target
+-----------------------------------------------------------
+
+**chreipl-fcp-mpath** is designed to accept operator-inititated changes of the
+re-IPL device. However, concurrent changes by the operator and tool driven
+changes can result in the operator change being overwritten.
+
+To avoid this problem, change the re-IPL device only during steady-state
+operations, when no path events happen. Alternatively, make sure that no events
+are processed while you change the device. See [EXAMPLES](#examples) for one
+way to suspend event processing.
+
+MESSAGES
+========
+
+During monitoring and event processing, **chreipl-fcp-mpath** writes messages
+to the syslog.
+
+When the configured re-IPL path is changed to a different path to the same
+volume (priority *daemon.notice*):
+
+  > Changed re-IPL path to: \<device-bus-id\>:\<wwpn\>:\<lun\>.
+
+When a path event indicates that the last available path has become
+non-operational (priority *daemon.alert*):
+
+  > The re-IPL device cannot be changed because no operational path to the
+  > re-IPL volume remains. The next re-IPL might fail unless you re-attach or
+  > enable at least one valid path to the re-IPL volume.
+
+When changing the configured re-IPL device failed because of an error with the
+used Linux kernel interface (priority *daemon.crit*):
+
+  > Changing the re-IPL device failed. The current re-IPL settings might be
+  > inconsistent. Check and correct the settings (see the README.md of
+  > chreipl-fcp-mpath) to make sure that the current re-IPL device is valid.
+
+A failure to change the re-IPL device can indicate an inconsistent setting that
+cannot be corrected automatically by **chreipl-fcp-mpath**. As a result, the
+next re-IPL might fail or might not use the intended re-IPL device.
+
+You can use the following tools to check and correct the current settings:
+
+  - **lsreipl** to confirm that the intended re-IPL device is configured;
+  - **chreipl** to change the re-IPL device;
+  - **lszfcp** to inspect the state of available paths to the re-IPL device.
+
+<!-- NOT-IN-MAN { -->
+
+SOFTWARE REQUIREMENTS
+=====================
+
+**chreipl-fcp-mpath** integrates into s390-tools's build and install
+infrastructure. Use **make** to build it. No explicit dependency management is
+in place, but the toolset has some software dependencies besides the
+requirements in section [Requirements](#requirements):
+
+  - GNU Bash;
+  - GNU Core Utilities (mktemp, readlink, sync, truncate, sha256sum);
+  - util-linux (flock, hexdump, logger);
+  - udev / systemd-udev;
+  - multipath-tools.
+
+To make use of the optional dracut configuration you need: dracut.
+
+To build a fresh version of the documentation (man page) you need:
+
+  - pandoc;
+  - GNU Core Utilities (date);
+  - GNU awk;
+
+otherwise the pre-cooked version shipped with the source will be used.
+
+INSTALLATION
+============
+
+If your distribution includes a packaged version of **chreipl-fcp-mpath**,
+either as a separate package or as part of a **s390-tools** package, install
+that package. Otherwise, you can either install it from source as part of
+**s390-tools** or separately.
+
+To install **chreipl-fcp-mpath** as part of **s390-tools**, use **make** on the
+top-level directory of your **s390-tools** distribution. Installing the entire
+distribution might overwrite other already installed tools.
+
+To install the tool separately, change into the **chreipl-fcp-mpath**
+directory, and use **make** there.
+
+You need *root* privileges to install the tool into the root file system.
+
+Calling **make** runs the build steps. Calling **make install** runs the build
+steps and copies the resulting components to their final destination.
+**s390-tools** offers more options and targets to customize the build (see
+**make help**).
+
+**chreipl-fcp-mpath** has the following optional build options:
+
+| Option      | Values | Default | Effect
+| :-----      | :----: | :-----: | :-----
+| HAVE_DRACUT | 0, 1   | 0       | Install a dracut configuration file that includes **chreipl-fcp-mpath** in the initial ramdisks built with **dracut**.
+| ENABLE_DOC  | 0, 1   | 0       | Build a fresh version of the man page for **chreipl-fcp-mpath**.
+
+Specify any options as arguments for both the **make** and **make install**
+command as shown in the following example:
+
+    ~ # cd chreipl-fcp-mpath/
+    ~ # make HAVE_DRACUT=1 ENABLE_DOC=1
+    ~ # make HAVE_DRACUT=1 ENABLE_DOC=1 install
+
+After the installation, reload the udev rules database:
+
+    ~ # udevadm control --reload
+
+*The toolset is now active on your running Linux instance.*
+
+If you use the *HAVE_DRACUT=1* option, also rebuild your
+initial ramdisk, to immediately include the toolset instead of
+waiting for the next kernel update.
+
+How to rebuild the initial ramdisk and the naming scheme for the
+resulting file or files depends on your distribution.
+The following example applies to Fedora and to Red Hat Enterprise Linux:
+
+    ~ # dracut --force /boot/initramfs-"$(uname -r)".img "$(uname -r)"
+
+For SUSE Linux Enterprise Server run for example:
+
+    ~ # dracut --hostonly --force /boot/initrd-"$(uname -r)" "$(uname -r)"
+
+These commands replace the initial ramdisk for the currently running kernel.
+If your distribution uses **zipl** as its boot loader, run **zipl** to refresh
+the boot record to find the new initial ramdisk.
+
+    ~ # zipl
+
+With dracut enabled, **make install** deploys the following files to these
+default locations:
+
+    /usr/lib/chreipl-fcp-mpath/chreipl-fcp-mpath-common.sh
+    /usr/lib/dracut/dracut.conf.d/70-chreipl-fcp-mpath.conf
+    /usr/lib/udev/chreipl-fcp-mpath-is-ipl-tgt
+    /usr/lib/udev/chreipl-fcp-mpath-is-ipl-vol
+    /usr/lib/udev/chreipl-fcp-mpath-is-reipl-zfcp
+    /usr/lib/udev/chreipl-fcp-mpath-record-volume-identifier
+    /usr/lib/udev/chreipl-fcp-mpath-try-change-ipl-path
+    /usr/lib/udev/rules.d/70-chreipl-fcp-mpath.rules
+    /usr/share/man/man7/chreipl-fcp-mpath.7
+
+UNINSTALL
+=========
+
+If your distribution includes a separately from **s390-tools** packaged version
+of **chreipl-fcp-mpath**, uninstall that package.
+
+For installations without distribution packaging, you cannot uninstall
+**chreipl-fcp-mpath** with **make**.
+
+Instead, remove the toolset by deleting the installed files as listed in
+[INSTALLATION](#installation)), reload the udev rules database, and rebuild all
+modified initial ramdisks as described in [INSTALLATION](#installation)).
+
+<!-- NOT-IN-MAN } -->
+
+EXAMPLES
+========
+
+Manual Changes to the Configured re-IPL Device
+----------------------------------------------
+
+As outlined in [DESCRIPTION](#description), be cautious when manually changing
+the configured re-IPL device. Assure that your reconfiguration actions do not
+collide with concurrent automatic event processing by **chreipl-fcp-mpath**.
+You can avoid such collisions, by stopping event processing, making your
+changes, and then re-enabling event processing.  You need *root* privileges for
+running the commands in the following example:
+
+    ~ # udevadm settle
+    ~ # udevadm control --stop-exec-queue
+    ~ # chreipl ...
+    ~ # udevadm control --start-exec-queue
+
+Listing messages with journalctl
+--------------------------------
+
+If your Linux instance includes **journalctl**, use the following command to
+list all messages that are issued by **chreipl-fcp-mpath**:
+
+    ~ # journalctl -t chreipl-fcp-mpath
+
+To list only messages that were issued since the last IPL, use this command:
+
+    ~ # journalctl -t chreipl-fcp-mpath -b
+
+REPORTING BUGS
+==============
+
+Use the **Issues** functionality on GitHub to report any bugs in
+**chreipl-fcp-mpath**:
+[s390-tools Issues](<https://github.com/ibm-s390-linux/s390-tools/issues> "Link to the s390-tools Issues page").
+
+SEE ALSO
+========
+
+**chreipl**(8), **dracut**(8), **journalctl**(1), **lsreipl(8)**,
+**lszfcp**(8), **multipath**(8), **multipathd**(8), **udev**(7),
+**udevadm**(8), **zipl**(8)

chreipl-fcp-mpath/chreipl-fcp-mpath-common.sh.in

@@ -0,0 +1,446 @@
+# SPDX-License-Identifier: MIT
+#
+# chreipl-fcp-mpath: use multipath information to change FCP IPL target
+# (C) Copyright IBM Corp. 2021
+#
+# Uses the following system-utilities (and shell-builtins):
+#   GNU coreutils:
+#     - mktemp
+#     - readlink
+#     - sync
+#   util-linux:
+#     - flock
+#     - logger
+
+# Makes use of udev event environment variables:
+#	SEQNUM
+
+# (1) expand failed globs to an empty string
+# (2) extended pattern matching to strip leading/trailing whitespaces
+shopt -s nullglob extglob
+# (1) don't overwrite existing files using redirects (e.g.: `>`)
+set -o noclobber
+
+# make sure any state files created are only writeable by the owning user
+umask 027
+
+# create log if DEBUG is enabled (with Make: D=1)
+#
+# Each script importing this library and expecting a debug log to be created
+# must declare a *trace tag* in a variable `debug_trace_tag`. This is used as
+# identifier in the log file name. The format is:
+#
+#     [[:digit:]][[:digit:]][[:alpha:]][[:alpha:]][[:alpha:]][[:alpha:]]
+#     \                    /\                                          /
+#      --------\  /--------  -------------------\  /-------------------
+#               \/                               \/
+#      relative position of  some unique abbreviation for the script
+#      execution in the      name, excluding any common prefix
+#      udev rules
+if '@DEBUG@' && [ -v debug_trace_tag ] && tlg="$(
+	mktemp -p '@debugoutdir@'					\
+		"chreiplzfcpmp-${debug_trace_tag}-${SEQNUM:-0}.XXXXXXXXXX" \
+		2>/dev/null)"
+then
+	readonly tlg
+	exec >|"${tlg}" 2>&1
+	set -x
+	set
+else
+	unset tlg
+fi
+
+declare -gr ID_FILE='@chreiplzfcpmp-id-file@'
+declare -gr FW_LOCK_FILE='@chreiplzfcpmp-fwlock-file@'
+
+declare -gA TRAP_EXIT_FN=()
+declare -gf trap_exit 1>/dev/null
+function trap_exit() {
+	local fn
+
+	for fn in "${TRAP_EXIT_FN[@]}"; do
+		"${fn}"
+	done
+
+	trap - EXIT
+}
+trap trap_exit EXIT
+
+# Output variables:
+#  id_file_unlock_exclusive_create() - call to unlock when finished with
+#                                      critical section
+#
+# XXX: `id_file_lock_*` can't be taken recursively
+function id_file_lock_exclusive_create() {
+	declare -g ID_FILE_LOCK=""
+
+	# prevent concurrent file creation
+	#
+	# First, open the file defined in ${ID_FILE} for writing; this will
+	# succeed and create the file only if it doesn't exist already. If the
+	# file already exist, the first open attempt will fail and we fall
+	# back to opening it only for reading; this will always succeed if the
+	# file already exists (the reason why the first attempty failed). In
+	# both cases store the corresponding file descriptor in
+	# ${ID_FILE_LOCK}.
+	#
+	# XXX: This should be race free.
+	#      open() with O_EXCL... is atomic (we set `noclobber` as shell
+	#      option); at least as long as we talk about a local FS.
+	if ! { exec {ID_FILE_LOCK}>"${ID_FILE}"; } 2>/dev/null; then
+		{ exec {ID_FILE_LOCK}<"${ID_FILE}"; } 2>/dev/null	\
+								|| return 1
+	fi
+
+	declare -gf id_file_unlock_exclusive_create 1>/dev/null
+	function id_file_unlock_exclusive_create() {
+		if [ -v ID_FILE_LOCK ]; then
+			sync "${ID_FILE}" 2>/dev/null
+			# release file and implicitly the lock, if taken
+			exec {ID_FILE_LOCK}>&-
+			unset ID_FILE_LOCK
+		fi
+
+		unset "TRAP_EXIT_FN[id_file_unlock_exclusive_create]"
+	}
+	TRAP_EXIT_FN+=(
+		[id_file_unlock_exclusive_create]=id_file_unlock_exclusive_create
+	)
+
+	flock --exclusive --timeout 5 "${ID_FILE_LOCK}"		|| return 2
+
+	return 0
+}
+
+# Output variables:
+#  id_file_unlock_exclusive_no_create() - call to unlock when finished with
+#                                         critical section
+#
+# XXX: `id_file_lock_*` can't be taken recursively
+function id_file_lock_exclusive_no_create() {
+	declare -g ID_FILE_LOCK=""
+
+	# Open the file defined in ${ID_FILE} for reading, and store the
+	# corresponding file descriptor in ${ID_FILE_LOCK}.
+	#
+	# XXX: return code is used in `chreipl-fcp-mpath-try-change-ipl-path`
+	{ exec {ID_FILE_LOCK}<"${ID_FILE}"; } 2>/dev/null	|| return 1
+
+	declare -gf id_file_unlock_exclusive_no_create 1>/dev/null
+	function id_file_unlock_exclusive_no_create() {
+		if [ -v ID_FILE_LOCK ]; then
+			sync "${ID_FILE}" 2>/dev/null
+			# release file and implicitly the lock, if taken
+			exec {ID_FILE_LOCK}<&-
+			unset ID_FILE_LOCK
+		fi
+
+		unset "TRAP_EXIT_FN[id_file_unlock_exclusive_no_create]"
+	}
+	TRAP_EXIT_FN+=(
+		[id_file_unlock_exclusive_no_create]=id_file_unlock_exclusive_no_create
+	)
+
+	flock --exclusive --timeout 5 "${ID_FILE_LOCK}"		|| return 2
+
+	return 0
+}
+
+# Output variables:
+#  id_file_unlock_shared_no_create() - call to unlock when finished with
+#                                      critical section
+#
+# XXX: `id_file_lock_*` can't be taken recursively
+function id_file_lock_shared_no_create() {
+	declare -g ID_FILE_LOCK=""
+
+	# Open the file defined in ${ID_FILE} for reading, and store the
+	# corresponding file descriptor in ${ID_FILE_LOCK}.
+	{ exec {ID_FILE_LOCK}<"${ID_FILE}"; } 2>/dev/null	|| return 1
+
+	declare -gf id_file_unlock_shared_no_create 1>/dev/null
+	function id_file_unlock_shared_no_create() {
+		if [ -v ID_FILE_LOCK ]; then
+			# release file and implicitly the lock, if taken
+			exec {ID_FILE_LOCK}<&-
+			unset ID_FILE_LOCK
+		fi
+
+		unset "TRAP_EXIT_FN[id_file_unlock_shared_no_create]"
+	}
+	TRAP_EXIT_FN+=(
+		[id_file_unlock_shared_no_create]=id_file_unlock_shared_no_create
+	)
+
+	flock --shared --timeout 5 "${ID_FILE_LOCK}"		|| return 2
+
+	return 0
+}
+
+# Output variables:
+#  firmware_unlock_exclusive() - call to unlock when finished with critical section
+#
+# XXX: `firmware_lock_*` can't be taken recursively
+function firmware_lock_exclusive() {
+	declare -g FIRMWARE_LOCK=""
+
+	# Open the file defined in ${FW_LOCK_FILE} for reading, and store the
+	# corresponding file descriptor in ${FIRMWARE_LOCK} (it doesn't matter
+	# whether this is a normal file or directory). This file descriptor
+	# will only be used for locking - not for actual I/O.
+	{ exec {FIRMWARE_LOCK}<"${FW_LOCK_FILE}"; } 2>/dev/null	|| return 1
+
+	declare -gf firmware_unlock_exclusive 1>/dev/null
+	function firmware_unlock_exclusive() {
+		if [ -v FIRMWARE_LOCK ]; then
+			# release file and implicitly the lock, if taken
+			exec {FIRMWARE_LOCK}<&-
+			unset FIRMWARE_LOCK
+		fi
+
+		unset "TRAP_EXIT_FN[firmware_unlock_exclusive]"
+	}
+	TRAP_EXIT_FN+=([firmware_unlock_exclusive]=firmware_unlock_exclusive)
+
+	flock --exclusive --timeout 5 "${FIRMWARE_LOCK}"	|| return 2
+
+	return 0
+}
+
+# Output variables:
+#  firmware_unlock_shared() - call to unlock when finished with critical section
+#
+# XXX: `firmware_lock_*` can't be taken recursively
+function firmware_lock_shared() {
+	declare -g FIRMWARE_LOCK=""
+
+	# Open the file defined in ${FW_LOCK_FILE} for reading, and store the
+	# corresponding file descriptor in ${FIRMWARE_LOCK} (it doesn't matter
+	# whether this is a normal file or directory). This file descriptor
+	# will only be used for locking - not for actual I/O.
+	{ exec {FIRMWARE_LOCK}<"${FW_LOCK_FILE}"; } 2>/dev/null	|| return 1
+
+	declare -gf firmware_unlock_shared 1>/dev/null
+	function firmware_unlock_shared() {
+		if [ -v FIRMWARE_LOCK ]; then
+			# release file and implicitly the lock, if taken
+			exec {FIRMWARE_LOCK}<&-
+			unset FIRMWARE_LOCK
+		fi
+
+		unset "TRAP_EXIT_FN[firmware_unlock_shared]"
+	}
+	TRAP_EXIT_FN+=([firmware_unlock_shared]=firmware_unlock_shared)
+
+	flock --shared --timeout 5 "${FIRMWARE_LOCK}"		|| return 2
+
+	return 0
+}
+
+# Output variables:
+#  IPL_TYPE
+#  IPL_BUSID
+#  IPL_WWPN
+#  IPL_LUN
+function firmware_get_ipl_information() {
+	declare -g IPL_TYPE="" IPL_BUSID="" IPL_WWPN="" IPL_LUN=""
+
+	# Take lock so we don't see any intermediate state from other helpers
+	# running in parallel
+	firmware_lock_shared					|| return 5
+
+	{ read -r IPL_TYPE _ < /sys/firmware/reipl/reipl_type; } 2>/dev/null \
+								|| return 1
+	{ read -r IPL_BUSID _ < /sys/firmware/reipl/fcp/device; } 2>/dev/null \
+								|| return 2
+	{ read -r IPL_WWPN _ < /sys/firmware/reipl/fcp/wwpn; } 2>/dev/null \
+								|| return 3
+	{ read -r IPL_LUN _ < /sys/firmware/reipl/fcp/lun; } 2>/dev/null \
+								|| return 4
+
+	firmware_unlock_shared
+
+	# show read values in debug log if enabled
+	if '@DEBUG@'; then
+		declare -p IPL_TYPE IPL_BUSID IPL_WWPN IPL_LUN 1>&2
+	fi
+
+	return 0
+}
+
+# Input:
+#  1: absolute canonical path to the scsi device in sysfs, e.g.:
+#     /sys/devices/css0/0.0.0014/0.0.1700/host1/rport-1:0-0/target1:0:0/1:0:0:1075789848
+# Output variables:
+#  SDEV_LUN
+function sdev_get_lun() {
+	local sdev="${1}" sdev_lun_str
+	# bash uses `intmax_t` as width for integer variables, and glibc
+	# defines this either as `long int` on 64 bit systems, or
+	# `long long int` on other.
+	local -i sdev_lun=0 fcp_lun=0
+
+	sdev_lun_str="${sdev##*:}"
+	# e.g.: 1075789848
+	[[ "${sdev_lun_str}" == +([[:digit:]]) ]]		|| return 1
+	# "cast" to integer
+	sdev_lun="${sdev_lun_str}"
+
+	# convert the Linux integer LUN format to the hexadecimal 64 bit T10
+	# LUN representation format used by many s390x interfaces
+	(( fcp_lun = (((sdev_lun >> 0) & 0xffff) << 48)
+			| (((sdev_lun >> 16) & 0xffff) << 32)
+			| (((sdev_lun >> 32) & 0xffff) << 16)
+			| (((sdev_lun >> 48) & 0xffff) << 0) ))
+	# the '0x' prefix is part of the length
+	printf -v SDEV_LUN "%#018llx" "${fcp_lun}"
+
+	# show read values in debug log if enabled
+	if '@DEBUG@'; then
+		declare -p SDEV_LUN 1>&2
+	fi
+
+	return 0
+}
+
+# Input:
+#  1: path to the scsi device in sysfs, e.g.:
+#     /sys/devices/css0/0.0.0014/0.0.1700/host1/rport-1:0-0/target1:0:0/1:0:0:1075789848
+#     , or a symlink pointing to the scsi device, e.g.:
+#     /sys/class/block/sds/device
+# Output variables:
+#  SDEV_BUSID
+#  SDEV_WWPN
+#  SDEV_LUN
+function sdev_get_fcp_addressing() {
+	local sdev="${1}" fcp_lun rport rport_wwpn zfcp_dev
+
+	declare -g SDEV_BUSID="" SDEV_WWPN="" SDEV_LUN=""
+
+	sdev="$(readlink -se "${sdev}")"			|| return 1
+
+	# get the LUN for this SDEV
+	#
+	# sets ${SDEV_LUN}
+	sdev_get_lun "${sdev}"					|| return 2
+
+	# get the WWPN of the remote port this SDEV is attached to
+	printf -v rport "%s" "${sdev}"/../../fc_remote_ports/rport-*:*-*
+	# e.g.: /sys/devices/css0/0.0.0016/0.0.1740/host0/rport-0:0-1/fc_remote_ports/rport-0:0-1
+	[ "${rport}" != "" ]					|| return 3
+
+	# XXX: This works even if the rport is currently in a bad
+	#      state, so e.g. when it has just gone down because of a
+	#      cable pull.
+	{ read -r rport_wwpn _ < "${rport}"/port_name; } 2>/dev/null	\
+								|| return 4
+
+	# The Linux kernel doesn't guarantee the same format as in
+	# /sys/firmware/..., so make sure it is the one we expect.
+	[[ "${rport_wwpn}" =~ ^0x[[:xdigit:]]{1,16}$ ]]		|| return 5
+	# the '0x' prefix is part of the length
+	printf -v rport_wwpn "%#018llx" "${rport_wwpn}"
+
+	# get the Device Bus-ID of the device via which this SDEV is attached
+	zfcp_dev="$(readlink -se "${sdev}"/../../../..)"	|| return 6
+	# e.g.: /sys/devices/css0/0.0.0016/0.0.1740
+	zfcp_dev="${zfcp_dev##*/}"
+
+	# shellcheck disable=2034
+	SDEV_BUSID="${zfcp_dev}"
+	# shellcheck disable=2034
+	SDEV_WWPN="${rport_wwpn}"
+	return 0
+}
+
+# Input:
+#  1: path to the scsi device in sysfs
+# Output variables:
+#  SDEV_WWID
+function sdev_get_wwid() {
+	local sdev="${1}"
+	local -a wwid
+
+	declare -g SDEV_WWID=""
+
+	# read the volume identifier without stripping any content
+	#
+	# XXX: we can read the WWID file, even if the SDEV is currently not
+	#      operational (e.g.: due to the path has gone away), as long as
+	#      the VPD PG 83 is still cached in the kernel; and the page gets
+	#      only released on SDEV device release.
+	{ readarray -d "" -t wwid < "${sdev}"/wwid; } 2>/dev/null	\
+								|| return 1
+	if '@DEBUG@'; then declare -p wwid 1>&2; fi
+
+	# test whether we read something
+	#
+	# This strips all leading spaces from the beginning of the read WWID
+	# (until the first non-space or NUL character), and checks whether the
+	# result is empty.
+	# Hence, we return early if the WWID consists of only whitespace.
+	#
+	# XXX: there could be unexpected characters in the returned ID.
+	#      `scsi_id` from the udev helpers sanitizes the strings it reads
+	#      from the devices, so they can be used in environment variables
+	#      without much danger.
+	#      But we don't export anything here, so it should be fine.
+	[ "${wwid[0]/#*([[:space:]])}" != "" ]			|| return 2
+
+	# shellcheck disable=2034
+	SDEV_WWID="${wwid[0]}"
+	return 0
+}
+
+# Input:
+#  1: path to the scsi device in sysfs
+# Return Value:
+#  == 0: SDEV referenced by `1` in good state
+#  != 0: otherwise
+function sdev_test_path_state() {
+	local sdev="${1}" state zfcp_failed zfcp_in_recovery rport port_state
+
+	sdev="$(readlink -se "${sdev}")"			|| return 1
+
+	{ read -r state _ < "${sdev}"/state; } 2>/dev/null	|| return 2
+	{ read -r zfcp_failed _ < "${sdev}"/zfcp_failed; } 2>/dev/null	\
+								|| return 3
+	{ read -r zfcp_in_recovery _ < "${sdev}"/zfcp_in_recovery; } 2>/dev/null \
+								|| return 4
+
+	printf -v rport "%s" "${sdev}"/../../fc_remote_ports/rport-*:*-*
+	# e.g.: /sys/devices/css0/0.0.0016/0.0.1740/host0/rport-0:0-1/fc_remote_ports/rport-0:0-1
+	[ "${rport}" != "" ]					|| return 5
+	{ read -r port_state _ < "${rport}"/port_state; } 2>/dev/null	\
+								|| return 6
+
+	if '@DEBUG@'; then
+		declare -p state zfcp_failed zfcp_in_recovery port_state 1>&2
+	fi
+
+	[ "${state}" = "running" ]				|| return 7
+	[ "${zfcp_failed}" = "0" ]				|| return 8
+	[ "${zfcp_in_recovery}" = "0" ]				|| return 9
+	{ [ "${port_state}" = "Online" ]				\
+	  || [ "${port_state}" = "Marginal" ]; }		|| return 10
+
+	return 0
+}
+
+# Input:
+#  *: all input parameters are used as quoted message
+function log_note() {
+	logger -p 'daemon.notice' -t 'chreipl-fcp-mpath' "${*}" &>/dev/null
+}
+
+# Input:
+#  *: all input parameters are used as quoted message
+function log_crit() {
+	logger -p 'daemon.crit' -t 'chreipl-fcp-mpath' "${*}" &>/dev/null
+}
+
+# Input:
+#  *: all input parameters are used as quoted message
+function log_alert() {
+	logger -p 'daemon.alert' -t 'chreipl-fcp-mpath' "${*}" &>/dev/null
+}

chreipl-fcp-mpath/chreipl-fcp-mpath-is-ipl-tgt.in

@@ -0,0 +1,56 @@
+#!/bin/bash
+# SPDX-License-Identifier: MIT
+#
+# chreipl-fcp-mpath: use multipath information to change FCP IPL target
+# (C) Copyright IBM Corp. 2021
+#
+# Uses the following system-utilities (and shell-builtins):
+#   Those necessary for sourced library:
+#     - chreipl-fcp-mpath-common.sh
+
+# Find out whether the device in udev event environment variable ${DEVPATH}
+# represents the device we want to re-IPL from. We do this by comparing
+# Device-Bus-ID/Target-WWPN/LUN of the individual SDEVs to the parameters set
+# in `/sys/firmware/reipl/fcp/`.
+#
+# Makes use of udev event environment variables:
+#	DM_UUID
+#	SUBSYSTEM
+#	DEVPATH
+
+# shellcheck disable=SC2034
+declare -gr debug_trace_tag=05iilt
+# shellcheck disable=SC1091
+source '@chreiplzfcpmp-lib@' || exit 127
+
+firmware_get_ipl_information					|| exit 1
+
+if [[ "${DM_UUID}" == mpath-* ]]; then
+	# Assume Multipath Device Mapper Device;
+	# e.g.: DEVPATH = /devices/virtual/block/dm-0
+	declare sdev
+
+	# depends on `nullglob` from `chreipl-fcp-mpath-common.sh`
+	for sdev in /sys/"${DEVPATH}"/slaves/sd*/device; do
+		sdev_get_fcp_addressing "${sdev}"		|| continue
+		[ "${SDEV_LUN}" = "${IPL_LUN}" ]		|| continue
+		[ "${SDEV_WWPN}" = "${IPL_WWPN}" ]		|| continue
+		[ "${SDEV_BUSID}" = "${IPL_BUSID}" ]		|| continue
+
+		exit 0
+	done
+	unset sdev
+
+elif [ "${SUBSYSTEM}" = block ]; then
+	# Assume SCSI Disk;
+	# e.g.: DEVPATH = /devices/css0/0.0.0014/0.0.1700/host0/rport-0:0-0/target0:0:0/0:0:0:1074806808/block/sds
+
+	sdev_get_fcp_addressing /sys/"${DEVPATH}"/device	|| exit 2
+	[ "${SDEV_LUN}" = "${IPL_LUN}" ]			|| exit 3
+	[ "${SDEV_WWPN}" = "${IPL_WWPN}" ]			|| exit 4
+	[ "${SDEV_BUSID}" = "${IPL_BUSID}" ]			|| exit 5
+
+	exit 0
+fi
+
+exit 6

chreipl-fcp-mpath/chreipl-fcp-mpath-is-ipl-vol.in

@@ -0,0 +1,86 @@
+#!/bin/bash
+# SPDX-License-Identifier: MIT
+#
+# chreipl-fcp-mpath: use multipath information to change FCP IPL target
+# (C) Copyright IBM Corp. 2021
+#
+# Uses the following system-utilities (and shell-builtins):
+#   Those necessary for sourced library:
+#     - chreipl-fcp-mpath-common.sh
+
+# Find out whether the device in environment variable ${DEVPATH} represents the
+# _volume_ that we IPL'ed from. We do this by comparing its WWID to the one
+# recorded in `@chreiplzfcpmp-id-file@`.
+#
+# Makes use of udev event environment variables:
+#	DM_UUID
+#	SUBSYSTEM
+#	DEVPATH
+
+# shellcheck disable=SC2034
+declare -gr debug_trace_tag=11iilv
+# shellcheck disable=SC1091
+source '@chreiplzfcpmp-lib@' || exit 127
+
+function id_file_read_ipl_information() {
+	local -a records
+
+	declare -g REC_WWID="" REC_BUSID="" REC_WWPN="" REC_LUN=""
+
+	# lock file before reading ID, so we don't see any intermediate state
+	id_file_lock_shared_no_create				|| return 1
+
+	{ readarray -d "" -t -u "${ID_FILE_LOCK}" records; } 2>/dev/null \
+								|| return 2
+	if '@DEBUG@'; then declare -p records 1>&2; fi
+
+	id_file_unlock_shared_no_create
+
+	[ "${#records[@]}" = "4" ]				|| return 3
+	# check that none of the array fields contains whitespace only
+	[ "${records[0]/#*([[:space:]])}" != "" ]		|| return 4
+	[ "${records[1]/#*([[:space:]])}" != "" ]		|| return 5
+	[ "${records[2]/#*([[:space:]])}" != "" ]		|| return 6
+	[ "${records[3]/#*([[:space:]])}" != "" ]		|| return 7
+
+	REC_WWID="${records[0]}"
+	REC_BUSID="${records[1]}"
+	REC_WWPN="${records[2]}"
+	REC_LUN="${records[3]}"
+	return 0
+}
+
+id_file_read_ipl_information					|| exit 1
+
+if [[ "${DM_UUID}" == mpath-* ]]; then
+	# Assume Multipath Device Mapper Device;
+	# e.g.: DEVPATH = /devices/virtual/block/dm-0
+	declare sdev found=false
+
+	for sdev in /sys/"${DEVPATH}"/slaves/sd*/device; do
+		if sdev_get_wwid "${sdev}"; then
+			found=true
+			break
+		fi
+	done
+	unset sdev
+
+	"${found}"						|| exit 2
+
+elif [ "${SUBSYSTEM}" = block ]; then
+	# Assume SCSI Disk;
+	# e.g.: DEVPATH = /devices/css0/0.0.0014/0.0.1700/host0/rport-0:0-0/target0:0:0/0:0:0:1074806808/block/sds
+
+	sdev_get_wwid /sys/"${DEVPATH}"/device			|| exit 3
+fi
+
+# set by `sdev_get_wwid` and `id_file_read_ipl_information`
+[ "${SDEV_WWID}" = "${REC_WWID}" ]				|| exit 4
+
+firmware_get_ipl_information					|| exit 5
+# set by `firmware_get_ipl_information` and `id_file_read_ipl_information`
+[ "${IPL_BUSID}" = "${REC_BUSID}" ]				|| exit 6
+[ "${IPL_WWPN}" = "${REC_WWPN}" ]				|| exit 7
+[ "${IPL_LUN}" = "${REC_LUN}" ]					|| exit 8
+
+exit 0

chreipl-fcp-mpath/chreipl-fcp-mpath-is-reipl-zfcp.in

@@ -0,0 +1,25 @@
+#!/bin/bash
+# SPDX-License-Identifier: MIT
+#
+# chreipl-fcp-mpath: use multipath information to change FCP IPL target
+# (C) Copyright IBM Corp. 2021
+#
+# Uses the following system-utilities (and shell-builtins):
+#   Those necessary for sourced library:
+#     - chreipl-fcp-mpath-common.sh
+
+# Find out whether ReIPL is gonna happen from a SCSI volume attached via zFCP
+
+# shellcheck disable=SC2034
+declare -gr debug_trace_tag=00iriz
+# shellcheck disable=SC1091
+source '@chreiplzfcpmp-lib@' || exit 127
+
+declare reipl_type
+
+{ read -r reipl_type _ < /sys/firmware/reipl/reipl_type; } 2>/dev/null || exit 1
+if '@DEBUG@'; then declare -p reipl_type 1>&2; fi
+
+[ "${reipl_type}" = "fcp" ]					|| exit 2
+
+exit 0

chreipl-fcp-mpath/chreipl-fcp-mpath-record-volume-identifier.in

@@ -0,0 +1,73 @@
+#!/bin/bash
+# SPDX-License-Identifier: MIT
+#
+# chreipl-fcp-mpath: use multipath information to change FCP IPL target
+# (C) Copyright IBM Corp. 2021
+#
+# Uses the following system-utilities (and shell-builtins):
+#   Those necessary for sourced library:
+#     - chreipl-fcp-mpath-common.sh
+#   GNU coreutils:
+#     - truncate
+#   util-linux:
+#     - hexdump
+
+# Record the identification of the volume we want to re-IPL from
+#
+# Makes use of udev event environment variables:
+#	DM_UUID
+#	SUBSYSTEM
+#	DEVPATH
+
+# shellcheck disable=SC2034
+declare -gr debug_trace_tag=10rvid
+# shellcheck disable=SC1091
+source '@chreiplzfcpmp-lib@' || exit 127
+
+function id_file_record_ipl_information() {
+	local sdev_wwid="${1}" ipl_busid="${2}" ipl_wwpn="${3}" ipl_lun="${4}"
+
+	# lock file before writing ID, so noone sees any intermediate state
+	id_file_lock_exclusive_create				|| return 1
+
+	# reset ID without removing the file (necessary for the locking to work
+	# properly, since the FD we use for locking is on this file/inode)
+	truncate --no-create --size=0 "${ID_FILE}"		|| return 3
+	echo -ne "${sdev_wwid}\x00${ipl_busid}\x00${ipl_wwpn}\x00${ipl_lun}\x00" \
+		>>"${ID_FILE}"					|| return 4
+
+	if '@DEBUG@'; then hexdump -vC "${ID_FILE}" 1>&2; fi
+
+	id_file_unlock_exclusive_create
+
+	return 0
+}
+
+if [[ "${DM_UUID}" == mpath-* ]]; then
+	# Assume Multipath Device Mapper Device;
+	# e.g.: DEVPATH = /devices/virtual/block/dm-0
+	declare sdev
+
+	for sdev in /sys/"${DEVPATH}"/slaves/sd*/device; do
+		if sdev_get_wwid "${sdev}"; then
+			break
+		fi
+	done
+	unset sdev
+
+elif [ "${SUBSYSTEM}" = block ]; then
+	# Assume SCSI Disk;
+	# e.g.: DEVPATH = /devices/css0/0.0.0014/0.0.1700/host0/rport-0:0-0/target0:0:0/0:0:0:1074806808/block/sds
+
+	sdev_get_wwid /sys/"${DEVPATH}"/device
+fi
+# shellcheck disable=SC2153
+[ "${SDEV_WWID}" != "" ]					|| exit 1
+
+firmware_get_ipl_information					|| exit 2
+# shellcheck disable=SC2153
+id_file_record_ipl_information						\
+	"${SDEV_WWID}" "${IPL_BUSID}" "${IPL_WWPN}" "${IPL_LUN}"	\
+								|| exit 3
+
+exit 0

chreipl-fcp-mpath/chreipl-fcp-mpath-try-change-ipl-path.in

@@ -0,0 +1,158 @@
+#!/bin/bash
+# SPDX-License-Identifier: MIT
+#
+# chreipl-fcp-mpath: use multipath information to change FCP IPL target
+# (C) Copyright IBM Corp. 2021
+#
+# Uses the following system-utilities (and shell-builtins):
+#   Those necessary for sourced library:
+#     - chreipl-fcp-mpath-common.sh
+#   GNU coreutils:
+#     - truncate
+#   util-linux:
+#     - hexdump
+
+# Try to change the current re-IPL target to a dfferent, operational path to
+# the same volume.
+#
+# Makes use of udev event environment variables:
+#	DM_UUID
+#	SUBSYSTEM
+#	DEVPATH
+#	CHREIPL_FCP_MPATH_IS_TGT
+
+# shellcheck disable=SC2034
+declare -gr debug_trace_tag=15tcip
+# shellcheck disable=SC1091
+source '@chreiplzfcpmp-lib@' || exit 127
+
+function apply_ipl_information() {
+	local sdev_wwid="${1}"
+	local sdev_busid="${2}" sdev_wwpn="${3}" sdev_lun="${4}"
+	local ipl_type="${5}" ipl_busid="${6}" ipl_wwpn="${7}" ipl_lun="${8}"
+	local -a records
+	local try_update_id_file=true
+
+	[ "${ipl_type}" = "fcp" ]				|| return 1
+
+	[[ "${sdev_busid}" =~ ^[[:xdigit:]]{1,3}\.[[:xdigit:]]\.[[:xdigit:]]{1,4}$ ]] \
+								|| return 2
+	[[ "${sdev_wwpn}" =~ ^0x[[:xdigit:]]{16}$ ]]		|| return 3
+	[[ "${sdev_lun}" =~ ^0x[[:xdigit:]]{16}$ ]]		|| return 4
+
+	# After updating the firmware re-IPL information below we also try to
+	# update the information stored in the ID file (necessary, so it
+	# contains the correct Device-Bus-ID/WWPN/LUN after the update). For
+	# the update of the ID file we try to grab an exclusive lock, so there
+	# are no overlapping reads/writes.
+	#
+	# In case we can't get the lock because the ID file is missing, but we
+	# have a direct TGT match, we may still try to change the re-IPL
+	# information, but skip the ID file update.
+	#
+	# "direct match" means, the event subject is either the SDEV that is
+	# currently set as re-IPL target, or it is the dm-multipath device that
+	# currently contains the re-IPL target.
+	if ! id_file_lock_exclusive_no_create; then
+		# rc == 1       --> could not read ${ID_FILE}
+		[ "${PIPESTATUS[0]}" -eq 1 ]			|| return 5
+		# if true, we are dealing with a direct TGT match
+		[ "${CHREIPL_FCP_MPATH_IS_TGT}" = "true" ]	|| return 6
+		try_update_id_file=false
+	fi
+
+	# If we have a direct match (see in the comment above), we know
+	# that we have a path to the current re-IPL volume - no matter of the
+	# WWID. Otherwise, we got here by comparing the WWID of the event
+	# subject with the one recorded in the ID file; in this case we try to
+	# make sure the information is still up-to-date.
+	if [ "${CHREIPL_FCP_MPATH_IS_TGT}" != "true" ]; then
+		# last bail to make sure we don't overwrite user choices..
+		#
+		# XXX: this will *NOT* prevent the race completely, but at least
+		#      make it less likely
+		{ readarray -d "" -t -u "${ID_FILE_LOCK}" records; } 2>/dev/null \
+								|| return 7
+		if '@DEBUG@'; then declare -p records 1>&2; fi
+
+		[ "${#records[@]}" = "4" ]			|| return 8
+		[ "${records[0]}" = "${sdev_wwid}" ]		|| return 9
+		[ "${records[1]}" = "${ipl_busid}" ]		|| return 10
+		[ "${records[2]}" = "${ipl_wwpn}" ]		|| return 11
+		[ "${records[3]}" = "${ipl_lun}" ]		|| return 12
+	fi
+
+	# Take lock so we don't see any intermediate state from other helpers
+	# running in parallel
+	firmware_lock_exclusive					|| return 13
+
+	if ! { echo "${sdev_busid}" >| /sys/firmware/reipl/fcp/device	\
+	       && echo "${sdev_wwpn}" >| /sys/firmware/reipl/fcp/wwpn	\
+	       && echo "${sdev_lun}" >| /sys/firmware/reipl/fcp/lun; };
+	then
+		log_alert "Changing the re-IPL device failed. The current re-IPL settings might be inconsistent. Check and correct the settings (see the README.md of chreipl-fcp-mpath) to make sure that the current re-IPL device is valid."
+		return 14
+	fi
+
+	firmware_unlock_exclusive
+
+	if [ "${sdev_busid}" != "${ipl_busid}" ]			\
+	   || [ "${sdev_wwpn}" != "${ipl_wwpn}" ]			\
+	   || [ "${sdev_lun}" != "${ipl_lun}" ]; then
+		log_note "Changed re-IPL path to: ${sdev_busid}:${sdev_wwpn}:${sdev_lun}."
+	fi
+
+	# Try to update the information in the ID file if we have gotten the
+	# lock for it.
+	if ${try_update_id_file}; then
+		# reset ID without removing the file
+		truncate --no-create --size=0 "${ID_FILE}"	|| return 15
+		echo -ne "${sdev_wwid}\x00${sdev_busid}\x00${sdev_wwpn}\x00${sdev_lun}\x00" \
+			>>"${ID_FILE}"				|| return 16
+
+		id_file_unlock_exclusive_no_create
+
+		if '@DEBUG@'; then hexdump -vC "${ID_FILE}" 1>&2; fi
+	fi
+
+	return 0
+}
+
+declare -g SDEV=""
+if [[ "${DM_UUID}" == mpath-* ]]; then
+	# Assume Multipath Device Mapper Device;
+	# e.g.: DEVPATH = /devices/virtual/block/dm-0
+
+	for sdev in /sys/"${DEVPATH}"/slaves/sd*/device; do
+		if sdev_test_path_state "${sdev}"; then
+			SDEV="${sdev}"
+			break
+		fi
+	done
+
+	# No path of the multipath-device that represents the IPL volume is
+	# online.
+	if [ "${SDEV}" = "" ]; then
+		log_crit "The re-IPL device cannot be changed because no operational path to the re-IPL volume remains. The next re-IPL might fail unless you re-attach or enable at least one valid path to the re-IPL volume."
+	fi
+
+elif [ "${SUBSYSTEM}" = block ]; then
+	# Assume SCSI Disk;
+	# e.g.: DEVPATH = /devices/css0/0.0.0014/0.0.1700/host0/rport-0:0-0/target0:0:0/0:0:0:1074806808/block/sds
+	if sdev_test_path_state /sys/"${DEVPATH}"/device; then
+		SDEV=/sys/"${DEVPATH}"/device
+	fi
+
+fi
+[ "${SDEV}" != "" ]						|| exit 0
+
+sdev_get_wwid "${SDEV}"						|| exit 0
+sdev_get_fcp_addressing "${SDEV}"				|| exit 0
+firmware_get_ipl_information					|| exit 0
+
+# shellcheck disable=SC2153
+apply_ipl_information							\
+	"${SDEV_WWID}" "${SDEV_BUSID}" "${SDEV_WWPN}" "${SDEV_LUN}"	\
+	"${IPL_TYPE}" "${IPL_BUSID}" "${IPL_WWPN}" "${IPL_LUN}"	|| exit 0
+
+exit 0

chreipl-fcp-mpath/chreipl-fcp-mpath.7

@@ -0,0 +1,183 @@
+.\" Automatically generated by Pandoc 2.16.1
+.\"
+.TH "CHREIPL-FCP-MPATH" "7" "2022-02-24" "s390-tools 2.20.0-build-20220224" "Administrator Manual"
+.hy
+.SH NAME
+.PP
+chreipl-fcp-mpath - use multipath information for re-IPL path failover
+on a running Linux instance
+.SH DESCRIPTION
+.PP
+The IPL process of Linux on Z or LinuxONE from an FCP-attached SCSI
+volume uses exactly one path to the volume.
+If this path is unavailable, the IPL fails.
+.PP
+The \f[B]chreipl-fcp-mpath\f[R] toolset monitors \f[B]udev\f[R] events
+about paths to the re-IPL volume.
+If the currently configured re-IPL path becomes unavailable, the toolset
+checks for operational paths to the same volume.
+If available, it reconfigures the re-IPL settings to use an operational
+path.
+.PP
+Thus, re-IPL from an FCP-attached SCSI volume can be successful despite
+path failures on a running Linux instance if at least one path to the
+re-IPL volume remains operational.
+.PP
+\f[B]Chreipl-fcp-mpath\f[R] requires \f[B]udev\f[R],
+\f[B]multipathd\f[R] and \f[B]dm-multipath\f[R].
+Once installed, the toolset runs automatically and autonomously.
+No user intervention is possible or required.
+.PP
+Other than installing the toolset, there is no user interface for
+\f[B]chreipl-fcp-mpath\f[R].
+.SS Requirements
+.PP
+The \f[B]chreipl-fcp-mpath\f[R] tool has the following requirements on
+the Linux instance that is being monitored:
+.IP \[bu] 2
+The Linux instance must have started successfully, during IPL.
+.IP \[bu] 2
+The running Linux instance must use \f[B]dm-multipath\f[R] and
+\f[B]multipathd\f[R] for the configured re-IPL volume - a volume that
+contains a zipl boot record and has one of its paths used in the re-IPL
+configuration.
+.IP \[bu] 2
+\f[B]udev\f[R] must run.
+.IP \[bu] 2
+The toolset must observe at least one event about the configured re-IPL
+path.
+Examples for such events are: the SCSI disk comes online, or a path of
+the corresponding multipath device goes down or comes back online.
+.RS 2
+.IP \[bu] 2
+The WWID of the re-IPL volume must not change while the Linux instance
+is running.
+.RE
+.IP \[bu] 2
+When the configured re-IPL path becomes unavailable while the Linux
+instance is running, at least one operational path to the re-IPL volume
+must be available, or must become available.
+If no such path is available when the Linux instance is rebooted, the
+re-IPL path is not changed.
+.IP \[bu] 2
+The tool assumes that any manually reconfigured re-IPL device is valid
+and operational.
+.RS 2
+.PP
+The tool treats a newly configured re-IPL device like the initially
+configured re-IPL device.
+In particular, if the newly configured re-IPL device fulfills the
+requirements of the tool, re-IPL path failover takes place if the
+configured re-IPL path becomes unavailable.
+.RE
+.SS Caution with Manual Changes to the Configured re-IPL Target
+.PP
+\f[B]chreipl-fcp-mpath\f[R] is designed to accept operator-inititated
+changes of the re-IPL device.
+However, concurrent changes by the operator and tool driven changes can
+result in the operator change being overwritten.
+.PP
+To avoid this problem, change the re-IPL device only during steady-state
+operations, when no path events happen.
+Alternatively, make sure that no events are processed while you change
+the device.
+See EXAMPLES for one way to suspend event processing.
+.SH MESSAGES
+.PP
+During monitoring and event processing, \f[B]chreipl-fcp-mpath\f[R]
+writes messages to the syslog.
+.PP
+When the configured re-IPL path is changed to a different path to the
+same volume (priority \f[I]daemon.notice\f[R]):
+.RS
+.PP
+Changed re-IPL path to: <device-bus-id>:<wwpn>:<lun>.
+.RE
+.PP
+When a path event indicates that the last available path has become
+non-operational (priority \f[I]daemon.alert\f[R]):
+.RS
+.PP
+The re-IPL device cannot be changed because no operational path to the
+re-IPL volume remains.
+The next re-IPL might fail unless you re-attach or enable at least one
+valid path to the re-IPL volume.
+.RE
+.PP
+When changing the configured re-IPL device failed because of an error
+with the used Linux kernel interface (priority \f[I]daemon.crit\f[R]):
+.RS
+.PP
+Changing the re-IPL device failed.
+The current re-IPL settings might be inconsistent.
+Check and correct the settings (see the README.md of chreipl-fcp-mpath)
+to make sure that the current re-IPL device is valid.
+.RE
+.PP
+A failure to change the re-IPL device can indicate an inconsistent
+setting that cannot be corrected automatically by
+\f[B]chreipl-fcp-mpath\f[R].
+As a result, the next re-IPL might fail or might not use the intended
+re-IPL device.
+.PP
+You can use the following tools to check and correct the current
+settings:
+.IP \[bu] 2
+\f[B]lsreipl\f[R] to confirm that the intended re-IPL device is
+configured;
+.IP \[bu] 2
+\f[B]chreipl\f[R] to change the re-IPL device;
+.IP \[bu] 2
+\f[B]lszfcp\f[R] to inspect the state of available paths to the re-IPL
+device.
+.SH EXAMPLES
+.SS Manual Changes to the Configured re-IPL Device
+.PP
+As outlined in DESCRIPTION, be cautious when manually changing the
+configured re-IPL device.
+Assure that your reconfiguration actions do not collide with concurrent
+automatic event processing by \f[B]chreipl-fcp-mpath\f[R].
+You can avoid such collisions, by stopping event processing, making your
+changes, and then re-enabling event processing.
+You need \f[I]root\f[R] privileges for running the commands in the
+following example:
+.IP
+.nf
+\f[C]
+\[ti] # udevadm settle
+\[ti] # udevadm control --stop-exec-queue
+\[ti] # chreipl ...
+\[ti] # udevadm control --start-exec-queue
+\f[R]
+.fi
+.SS Listing messages with journalctl
+.PP
+If your Linux instance includes \f[B]journalctl\f[R], use the following
+command to list all messages that are issued by
+\f[B]chreipl-fcp-mpath\f[R]:
+.IP
+.nf
+\f[C]
+\[ti] # journalctl -t chreipl-fcp-mpath
+\f[R]
+.fi
+.PP
+To list only messages that were issued since the last IPL, use this
+command:
+.IP
+.nf
+\f[C]
+\[ti] # journalctl -t chreipl-fcp-mpath -b
+\f[R]
+.fi
+.SH REPORTING BUGS
+.PP
+Use the \f[B]Issues\f[R] functionality on GitHub to report any bugs in
+\f[B]chreipl-fcp-mpath\f[R]: s390-tools
+Issues (https://github.com/ibm-s390-linux/s390-tools/issues).
+.SH SEE ALSO
+.PP
+\f[B]chreipl\f[R](8), \f[B]dracut\f[R](8), \f[B]journalctl\f[R](1),
+\f[B]lsreipl(8)\f[R], \f[B]lszfcp\f[R](8), \f[B]multipath\f[R](8),
+\f[B]multipathd\f[R](8), \f[B]udev\f[R](7), \f[B]udevadm\f[R](8),
+\f[B]zipl\f[R](8)

chreipl-fcp-mpath/chreipl-fcp-mpath.mak

@@ -0,0 +1,89 @@
+# SPDX-License-Identifier: MIT
+#
+# chreipl-fcp-mpath: use multipath information to change FCP IPL target
+# (C) Copyright IBM Corp. 2021
+#
+# Uses the following system-utilities (and shell-builtins):
+#   Utilities list in GNU Make Conventions:
+#     https://www.gnu.org/software/make/manual/make.html#Utilities-in-Makefiles
+#   GNU coreutils:
+#     - mktemp
+# If $(ENABLE_DOC) is `1`:
+#   Pandoc:
+#     - pandoc
+#   GNU coreutils:
+#     - date
+
+#
+## Paths and Build Variables
+#
+
+# Install the configuration file for dracut, to automatically pull in the
+# toolset into the initial ramdisk, when built with it.
+HAVE_DRACUT	 = 0
+
+# Build documentation; requires: Pandoc
+ENABLE_DOC	 = 0
+
+# https://www.gnu.org/software/make/manual/make.html#Directory-Variables
+CHREIPLZFCPMPDIR = $(USRLIBDIR)/chreipl-fcp-mpath
+UDEVRUNDIR	 = /run/udev
+DEBUGOUTDIR	 = $(UDEVRUNDIR)
+
+INSTALL_EXEC	 = $(INSTALL) -g $(GROUP) -o $(OWNER) --preserve-timestamps
+INSTALL_DATA	 = $(INSTALL_EXEC) --mode=0644
+
+# used for data exchange and synchronization across the different helpers
+chreiplzfcpmp-id-file		 = $(UDEVRUNDIR)/chreiplzfcpmp-ipl-volume-id
+# file used to implement mutual exclusion when accessing firmware IPL info:
+#   - this should be something that is (practically) always available, so we
+#     dont have to worry about fallbacks or error-handling;
+#   - at the same time, it should not be used by anything else with flock(2) to
+#     hold a lock for long periods.
+chreiplzfcpmp-fwlock-file	 = /sys/firmware/reipl
+
+.DELETE_ON_ERROR:
+
+# export build-time definitions to the scripts/built-components
+define chreiplzfcpmp-sed-buildvar-replace =
+tmpout=$$(mktemp -p ./ .make.tmp.XXXXXXXXXXXXXXXX) && {			\
+	$(SED) -E 							\
+		-e 's|@DEBUG@|$(if $(filter 1,$(D)),true,false)|g'	\
+		-e 's|@chreiplzfcpmp-id-file@|$(chreiplzfcpmp-id-file)|g' \
+		-e 's|@chreiplzfcpmp-fwlock-file@|$(chreiplzfcpmp-fwlock-file)|g' \
+		-e 's|@chreiplzfcpmp-lib@|$(CHREIPLZFCPMPDIR)/chreipl-fcp-mpath-common.sh|g' \
+		-e 's|@debugoutdir@|$(DEBUGOUTDIR)|g'			\
+		-e 's|@udevdir@|$(UDEVDIR)|g'				\
+		-e 's|@udevrulesdir@|$(UDEVRULESDIR)|g'			\
+		$(1) > $${tmpout}					\
+	    && mv $${tmpout} $(2)					\
+	    || { rm $${tmpout}; false; }				\
+; }
+endef
+
+.PHONY: clean-mk-temp
+clean: clean-mk-temp
+clean-mk-temp:
+	rm -f .make.tmp.[[:alnum:]][[:alnum:]][[:alnum:]][[:alnum:]][[:alnum:]][[:alnum:]][[:alnum:]][[:alnum:]][[:alnum:]][[:alnum:]][[:alnum:]][[:alnum:]][[:alnum:]][[:alnum:]][[:alnum:]][[:alnum:]]
+
+# Definitions for generating documentation when $(ENABLE_DOC) is set to `1`
+
+PANDOCFLAGS	 = --fail-if-warnings
+ALL_PANDOCFLAGS	 = --preserve-tabs --tab-stop=8 --strip-comments	\
+		   --standalone --self-contained			\
+		   -M date="$(shell date +'%Y-%m-%d')"			\
+		   $(PANDOCFLAGS)
+
+$(eval $(call cmd_define, PANDOC,"  PANDOC  ",pandoc))
+
+%.html : ALL_PANDOCFLAGS += -t html
+%.html : %.md
+	$(PANDOC) $(ALL_PANDOCFLAGS) -f gfm -o $(@) $(<)
+
+%.pdf : ALL_PANDOCFLAGS += -t latex --toc
+%.pdf : %.md
+	$(PANDOC) $(ALL_PANDOCFLAGS) -f gfm -o $(@) $(<)
+
+%.7 : ALL_PANDOCFLAGS += -t man
+%.7 : %.md
+	$(PANDOC) $(ALL_PANDOCFLAGS) -f gfm -o $(@) $(<)

chreipl-fcp-mpath/dracut/dracut.conf.d/70-chreipl-fcp-mpath.conf.in

@@ -0,0 +1,16 @@
+# SPDX-License-Identifier: MIT
+#
+# chreipl-fcp-mpath: use multipath information to change FCP IPL target
+# (C) Copyright IBM Corp. 2021
+
+add_dracutmodules+=" bash multipath udev-rules "
+
+install_items+=" @udevrulesdir@/70-chreipl-fcp-mpath.rules "
+install_items+=" @chreiplzfcpmp-lib@ "
+install_items+=" @udevdir@/chreipl-fcp-mpath-is-ipl-tgt "
+install_items+=" @udevdir@/chreipl-fcp-mpath-is-ipl-vol "
+install_items+=" @udevdir@/chreipl-fcp-mpath-is-reipl-zfcp "
+install_items+=" @udevdir@/chreipl-fcp-mpath-record-volume-identifier "
+install_items+=" @udevdir@/chreipl-fcp-mpath-try-change-ipl-path "
+
+install_items+=" flock hexdump logger mktemp readlink sync truncate "

chreipl-fcp-mpath/udev/rules.d/70-chreipl-fcp-mpath.rules

@@ -0,0 +1,101 @@
+# SPDX-License-Identifier: MIT
+#
+# chreipl-fcp-mpath: use multipath information to change FCP IPL target
+# (C) Copyright IBM Corp. 2021
+
+# Did the event affect a multipath or scsi disk device?
+ACTION=="change", KERNEL=="dm-[0-9]*", SUBSYSTEM=="block",		\
+	ENV{DM_UUID}=="mpath-*", ENV{DM_ACTION}=="PATH_FAILED",		\
+	GOTO="chreipl_fcp_mpath_path_change"
+ACTION=="change", KERNEL=="dm-[0-9]*", SUBSYSTEM=="block",		\
+	ENV{DM_UUID}=="mpath-*", ENV{DM_ACTION}=="PATH_REINSTATED",	\
+	GOTO="chreipl_fcp_mpath_path_change"
+ACTION=="add", KERNEL=="sd[a-z]*", SUBSYSTEM=="block",			\
+	GOTO="chreipl_fcp_mpath_path_change"
+GOTO="chreipl_fcp_mpath_end"
+LABEL="chreipl_fcp_mpath_path_change"
+
+# Is this system IPL'ed (IOW, are we on s390x)? And do we ReIPL via zFCP?
+#
+#    udev(7): If no absolute path is given, the program is expected to live
+#             in /usr/lib/udev; otherwise, the absolute path must be
+#             specified.
+TEST!="/sys/firmware/ipl", GOTO="chreipl_fcp_mpath_end"
+PROGRAM!="chreipl-fcp-mpath-is-reipl-zfcp", GOTO="chreipl_fcp_mpath_end"
+
+# Consider the following scenarios.
+# Either:
+#
+# (A) We recognized a new SCSI Disk. This might represent:
+#     (a) the path we want to ReIPL from;
+#     (b) an alternative path to the volume we want to ReIPL from;
+#     (c) a path to some unrelated volume.
+#
+# Or:
+#
+# (B) We recognized a PATH_ event for a multipath device. This might represent:
+#     the path we want to ReIPL from:
+#         (a) went away;
+#         (b) came back online;
+#     an alternative path to the volume we want to ReIPL from:
+#         (c) went away;
+#         (d) came back online;
+#     (e) some unrelated multipath device saw an event.
+
+# Test whether the affected device is, or contains, the current IPL target.
+#
+# This covers scenarios:
+#     (A) (a),
+#     (B) (a)/(b)/(c)/(d)
+PROGRAM!="chreipl-fcp-mpath-is-ipl-tgt",				\
+	ENV{CHREIPL_FCP_MPATH_IS_TGT}="false",				\
+	GOTO="chreipl_fcp_mpath_not_direct_match"
+ENV{CHREIPL_FCP_MPATH_IS_TGT}="true"
+
+# Record the WWID, Device-Bus-ID, Remote WWPN, and LUN of the ReIPL target
+# (see `chreipl-fcp-mpath-is-ipl-vol` for usecases). This information
+# might change, depending on whether the machine operator changes the ReIPL
+# target to a different volume.
+#
+# XXX: Because the kernel doesn't generate any events upon changing of
+#      the ReIPL target, the chreipl-fcp-mpath toolset can't take any
+#      actions until the next path event for the new target is
+#      generated. Following that, we assume that when the machine
+#      operator changes the ReIPL target, the new target is reachable
+#      and in a good state at this point in time.
+PROGRAM!="chreipl-fcp-mpath-record-volume-identifier",			\
+	GOTO="chreipl_fcp_mpath_try_change_ipl_path"
+GOTO="chreipl_fcp_mpath_try_change_ipl_path"
+
+# If the even subject is not a direct match (not the sdev that is the current
+# ReIPL target, and not a mpath device that contains the current ReIPL target)
+LABEL="chreipl_fcp_mpath_not_direct_match"
+
+# While this sdev/mpath device doesn't directly correspond to the path
+# currently set as ReIPL target, it might still point to the same volume.
+#
+# For mpath devices this can happen if the original ReIPL target is completely
+# gone from the machine, and so there is no way we can successfully, directly
+# compare the ReIPL parameters to the sdevs of the mpath device.
+#
+# For cases like these we recorded the volume identifier, which we now can
+# compare, and so still decide whether we are addressing the correct volume.
+#
+# This covers scenarios:
+#     (A) (b)/(c),
+#     (B) (c)/(d)/(e)
+#
+# XXX: we recorded WWID, Device-Bus-ID, Remote WWPN, LUN of the ReIPL target at
+#      the time; if the latter three don't match the current ReIPL setting
+#      anymore, we have to assume that someone changed the ReIPL target
+#      manually, and we cannot use the WWID anymore since we can't possibly
+#      know whether that stayed the same when the change was done.
+PROGRAM!="chreipl-fcp-mpath-is-ipl-vol", GOTO="chreipl_fcp_mpath_end"
+
+# We are here because of scenarios:
+#     (A) (a)/(b),
+#     (B) (a)/(b)/(c)/(d)
+LABEL="chreipl_fcp_mpath_try_change_ipl_path"
+RUN{program}+="chreipl-fcp-mpath-try-change-ipl-path"
+
+LABEL="chreipl_fcp_mpath_end"

cmsfs-fuse/Makefile

@@ -16,17 +16,18 @@ check_dep:
 	$(call check_dep, \
 		"cmsfs-fuse", \
 		"fuse.h", \
-		"fuse-devel or libfuse-dev", \
-		"HAVE_FUSE=0")
+		"fuse3-devel or libfuse3-dev", \
+		"HAVE_FUSE=0", \
+		"-DFUSE_USE_VERSION=30")
 
 all: check_dep cmsfs-fuse
 
 ifneq ($(shell sh -c 'command -v pkg-config'),)
-FUSE_CFLAGS = $(shell pkg-config --silence-errors --cflags fuse)
-FUSE_LDLIBS = $(shell pkg-config --silence-errors --libs fuse)
+FUSE_CFLAGS = $(shell pkg-config --silence-errors --cflags fuse3)
+FUSE_LDLIBS = $(shell pkg-config --silence-errors --libs fuse3)
 else
-FUSE_CFLAGS = -D_FILE_OFFSET_BITS=64 -I/usr/include/fuse
-FUSE_LDLIBS = -lfuse
+FUSE_CFLAGS = -D_FILE_OFFSET_BITS=64 -I/usr/include/fuse3
+FUSE_LDLIBS = -lfuse3
 endif
 ALL_CFLAGS += -DHAVE_SETXATTR $(FUSE_CFLAGS)
 LDLIBS += $(FUSE_LDLIBS) -lm

cmsfs-fuse/cmsfs-fuse.1

@@ -80,9 +80,6 @@ Allow access by other users
 \fB\-o\fR allow_root
 Allow access by root
 .TP
-\fB\-o\fR nonempty
-Allow mounts over non\-empty file/dir
-.TP
 \fB\-o\fR default_permissions 
 Enable permission checking by kernel
 .TP

cmsfs-fuse/cmsfs-fuse.c

@@ -9,7 +9,7 @@
  * it under the terms of the MIT license. See LICENSE for details.
  */
 
-#define FUSE_USE_VERSION 26
+#define FUSE_USE_VERSION 30
 #include <assert.h>
 #include <ctype.h>
 #include <errno.h>
@@ -1516,7 +1516,7 @@ static void walk_dir_block(struct fst_entry *fst, struct walk_file *walk,
 				decode_edf_name(file, fst->name, fst->type);
 				if (!file_unlinked(file)) {
 					cache_fst_addr(walk->addr, file);
-					walk->filler(walk->buf, file, NULL, 0);
+					walk->filler(walk->buf, file, NULL, 0, 0);
 				}
 			}
 		}
@@ -1728,8 +1728,11 @@ static off_t get_file_size_logical(struct fst_entry *fst)
 	return total;
 }
 
-static int cmsfs_getattr(const char *path, struct stat *stbuf)
+static int cmsfs_getattr(const char *path, struct stat *stbuf,
+			 struct fuse_file_info *fi)
 {
+	(void) fi;
+
 	int mask = (cmsfs.allow_other) ? 0444 : 0440;
 	struct fst_entry fst;
 
@@ -1783,13 +1786,15 @@ static int cmsfs_getattr(const char *path, struct stat *stbuf)
 }
 
 static int cmsfs_readdir(const char *path, void *buf, fuse_fill_dir_t filler,
-			 off_t offset, struct fuse_file_info *fi)
+			 off_t offset, struct fuse_file_info *fi,
+			 enum fuse_readdir_flags flags)
 {
 	struct walk_file walk;
 	struct fst_entry fst;
 
 	(void) offset;
 	(void) fi;
+	(void) flags;
 
 	/*
 	 * Offset is ignored and 0 passed to the filler fn so the whole
@@ -1800,8 +1805,8 @@ static int cmsfs_readdir(const char *path, void *buf, fuse_fill_dir_t filler,
 	if (strcmp(path, "/") != 0)
 		return -ENOENT;
 
-	filler(buf, ".", NULL, 0);
-	filler(buf, "..", NULL, 0);
+	filler(buf, ".", NULL, 0, 0);
+	filler(buf, "..", NULL, 0, 0);
 
 	memset(&walk, 0, sizeof(walk));
 	/* readdir is possible without open so fi->fh is not set */
@@ -2683,13 +2688,16 @@ static int cmsfs_statfs(const char *path, struct statvfs *buf)
 	return 0;
 }
 
-static int cmsfs_utimens(const char *path, const struct timespec ts[2])
+static int cmsfs_utimens(const char *path, const struct timespec ts[2],
+			 struct fuse_file_info *fi)
 {
 	struct fst_entry fst;
 	off_t fst_addr;
 	struct tm tm;
 	int rc;
 
+	(void) fi;
+
 	if (cmsfs.readonly)
 		return -EACCES;
 
@@ -2825,7 +2833,8 @@ error:
 	return rc;
 }
 
-static int cmsfs_rename(const char *path, const char *new_path)
+static int cmsfs_rename(const char *path, const char *new_path,
+			unsigned int flags)
 {
 	struct fst_entry fst, fst_new;
 	off_t fst_addr, fst_addr_new;
@@ -2835,6 +2844,8 @@ static int cmsfs_rename(const char *path, const char *new_path)
 	struct file *f;
 	int rc;
 
+	(void) flags;
+
 	if (cmsfs.readonly)
 		return -EACCES;
 
@@ -3232,13 +3243,16 @@ static void update_fst(struct file *f, off_t addr)
 	unhide_null_blocks(f);
 }
 
-static int cmsfs_truncate(const char *path, off_t size)
+static int cmsfs_truncate(const char *path, off_t size,
+			  struct fuse_file_info *fi)
 {
 	struct fst_entry fst;
 	off_t fst_addr, len;
 	struct file *f;
 	int rc = 0;
 
+	(void) fi;
+
 	if (cmsfs.readonly)
 		return -EROFS;
 
@@ -4478,6 +4492,15 @@ static void destroy_file_object(struct file *f)
 	free(f);
 }
 
+static void *cmsfs_oper_init(struct fuse_conn_info *conn, struct fuse_config *cfg)
+{
+	(void)conn;
+
+	/* force immediate file removal */
+	cfg->hard_remove = 1;
+	return NULL;
+}
+
 static struct file_operations fops_fixed = {
 	.cache_data = cache_file_fixed,
 	.write_data = extend_block_fixed,
@@ -4493,6 +4516,7 @@ static struct file_operations fops_variable = {
 };
 
 static struct fuse_operations cmsfs_oper = {
+	.init		= cmsfs_oper_init,
 	.getattr	= cmsfs_getattr,
 	.statfs		= cmsfs_statfs,
 	.readdir	= cmsfs_readdir,
@@ -4669,8 +4693,6 @@ int main(int argc, char *argv[])
 		fuse_opt_add_arg(&args, "-oro");
 	/* force single threaded mode which requires no locking */
 	fuse_opt_add_arg(&args, "-s");
-	/* force immediate file removal */
-	fuse_opt_add_arg(&args, "-ohard_remove");
 
 	if (cmsfs.mode == BINARY_MODE &&
 	    (cmsfs.codepage_from != NULL || cmsfs.codepage_to != NULL))

common.mak

@@ -5,7 +5,7 @@ COMMON_INCLUDED = true
 # The variable "DISTRELEASE" should be overwritten in rpm spec files with:
 # "make DISTRELEASE=%{release}" and "make install DISTRELEASE=%{release}"
 VERSION            = 2
-RELEASE            = 18
+RELEASE            = 23
 PATCHLEVEL         = 0
 DISTRELEASE        = build-$(shell date +%Y%m%d)
 S390_TOOLS_RELEASE = $(VERSION).$(RELEASE).$(PATCHLEVEL)-$(DISTRELEASE)
@@ -59,11 +59,8 @@ define cmd_define
 endef
 
 $(eval $(call cmd_define,     AS,"  AS      ",$(CROSS_COMPILE)as))
-$(eval $(call cmd_define,   LINK,"  LINK    ",$(CROSS_COMPILE)gcc))
-$(eval $(call cmd_define,     LD,"  LD      ",$(CROSS_COMPILE)ld))
 $(eval $(call cmd_define,     CC,"  CC      ",$(CROSS_COMPILE)gcc))
 $(eval $(call cmd_define, HOSTCC,"  HOSTCC  ",gcc))
-$(eval $(call cmd_define, LINKXX,"  LINKXX  ",$(CROSS_COMPILE)g++))
 $(eval $(call cmd_define,    CXX,"  CXX     ",$(CROSS_COMPILE)g++))
 $(eval $(call cmd_define,    CPP,"  CPP     ",$(CROSS_COMPILE)gcc -E))
 $(eval $(call cmd_define,     AR,"  AR      ",$(CROSS_COMPILE)ar))
@@ -84,6 +81,8 @@ CHECKTOOL       = $(call echocmd,"  CHECK   ",/$@)$(CHECK_SILENT)
 SKIP            = echo           "  SKIP    $(call reldir) due to"
 
 INSTALL         = install
+LINK            = $(CC)
+LINKXX          = $(CXX)
 CP              = cp
 ifneq ("${V}","1")
 	MAKEFLAGS += --quiet
@@ -92,13 +91,12 @@ ifneq ("${V}","1")
 else
 	echocmd=
 endif
+DEFAULT_CFLAGS = -g -fstack-protector-all -W -Wall -Wformat-security
 ifeq ("${W}","1")
-	DEFAULT_CFLAGS = -g -rdynamic -fstack-protector-all -W -Wall -Wformat-security -Wextra
-else
-	DEFAULT_CFLAGS = -g -rdynamic -fstack-protector-all -W -Wall -Wformat-security
+	DEFAULT_CFLAGS += -Wextra -Wshadow -Wundef -Wuninitialized -Wdouble-promotion -Wconversion
 endif
 ifeq ("${D}","1")
-	DEFAULT_CFLAGS += -Og
+	DEFAULT_CFLAGS += -Og -g3 -ggdb3
 else
 	DEFAULT_CFLAGS += -O3
 endif
@@ -120,7 +118,7 @@ endif
 # Returns "yes" on success and nothing otherwise
 #
 define check_header_prereq
-$(shell printf "#include <%s>\n int main(void) {return 0;}" $1 | \
+$(shell printf "#include <%s>\n int main(void) {return 0;}\n" $1 | \
         ( $(CC) $(filter-out --coverage, $(ALL_CFLAGS)) $(ALL_CPPFLAGS) \
                 $2 -o /dev/null -xc - ) >/dev/null 2>&1 && echo -n yes)
 endef
@@ -135,7 +133,7 @@ endef
 # $5: Additional compiler & linker options (optional)
 #
 check_dep=\
-printf "\#include <%s>\n int main(void) {return 0;}" $2 | ( $(CC) $(filter-out --coverage, $(ALL_CFLAGS)) $(ALL_CPPFLAGS) $5 -o /dev/null -xc - ) > /dev/null 2>&1; \
+printf "\#include <%s>\n int main(void) {return 0;}\n" $2 | ( $(CC) $(filter-out --coverage, $(ALL_CFLAGS)) $(ALL_CPPFLAGS) $5 -o /dev/null -xc - ) > /dev/null 2>&1; \
 if [ $$? != 0 ]; \
 then \
 	printf "  REQCHK  %s (%s)\n" $1 $2; \
@@ -182,6 +180,7 @@ USRSBINDIR      = $(INSTALLDIR)/usr/sbin
 USRBINDIR       = $(INSTALLDIR)/usr/bin
 BINDIR          = $(INSTALLDIR)/sbin
 LIBDIR          = $(INSTALLDIR)/lib
+USRLIBDIR	= $(INSTALLDIR)/usr/lib
 USRLIB64DIR     = $(INSTALLDIR)/usr/lib64
 SYSCONFDIR      = $(INSTALLDIR)/etc
 MANDIR          = $(INSTALLDIR)/usr/share/man
@@ -194,6 +193,11 @@ ZFCPDUMP_DIR    = $(TOOLS_LIBDIR)/zfcpdump
 SYSTEMDSYSTEMUNITDIR =
 USRINCLUDEDIR   = $(INSTALLDIR)/usr/include
 ZKEYKMSPLUGINDIR = $(USRLIB64DIR)/zkey
+UDEVDIR		= $(USRLIBDIR)/udev
+UDEVRULESDIR	= $(UDEVDIR)/rules.d
+DRACUTDIR	= $(USRLIBDIR)/dracut
+DRACUTCONFDIR   = $(DRACUTDIR)/dracut.conf.d
+DRACUTMODDIR	= $(DRACUTDIR)/modules.d
 
 ifeq ($(LIBDIR),$(INSTALLDIR)/lib)
 SOINSTALLDIR = $(USRLIB64DIR)
@@ -206,10 +210,10 @@ INSTDIRS        = $(USRSBINDIR) $(USRBINDIR) $(BINDIR) $(LIBDIR) $(MANDIR) \
 			$(TOOLS_LIBDIR) $(TOOLS_DATADIR) \
 			$(ZFCPDUMP_DIR) $(SYSTEMDSYSTEMUNITDIR) \
 			$(USRLIB64DIR) $(USRINCLUDEDIR) $(ZKEYKMSPLUGINDIR) \
-			$(SOINSTALLDIR)
+			$(SOINSTALLDIR) $(USRLIBDIR)
 OWNER           = $(shell id -un)
 GROUP		= $(shell id -gn)
-export INSTALLDIR BINDIR LIBDIR USRLIB64DIR MANDIR OWNER GROUP
+export INSTALLDIR BINDIR LIBDIR USRLIBDIR USRLIB64DIR MANDIR OWNER GROUP
 
 # Special defines for zfcpdump
 ZFCPDUMP_IMAGE	= zfcpdump-image
@@ -251,11 +255,9 @@ export AS LD CC CPP AR NM STRIP OBJCOPY OBJDUMP INSTALL CFLAGS CXXFLAGS \
 
 ifneq ($(shell $(CC_SILENT) -dumpspecs 2>/dev/null | grep -e '[^f]no-pie'),)
 	NO_PIE_CFLAGS := -fno-pie
-	NO_PIE_LINKFLAGS := -no-pie
 	NO_PIE_LDFLAGS := -no-pie
 else
 	NO_PIE_CFLAGS :=
-	NO_PIE_LINKFLAGS :=
 	NO_PIE_LDFLAGS :=
 endif
 
@@ -370,6 +372,10 @@ $(rootdir)/libvmcp/libvmcp.a: $(rootdir)/libvmcp
 	$(MAKE) -C $(rootdir)/libvmcp/ libvmcp.a
 .PHONY: $(rootdir)/libvmcp
 
+$(rootdir)/libcpumf/libcpumf.a: $(rootdir)/libcpumf
+	$(MAKE) -C $(rootdir)/libcpumf/ libcpumf.a
+.PHONY: $(rootdir)/libcpumf
+
 $(rootdir)/libekmfweb/libekmfweb.so: $(rootdir)/libekmfweb
 	$(MAKE) -C $(rootdir)/libekmfweb/ libekmfweb.so
 .PHONY: $(rootdir)/libekmfweb
@@ -382,6 +388,14 @@ $(rootdir)/libkmipclient/libkmipclient.so: $(rootdir)/libkmipclient
 	$(MAKE) -C $(rootdir)/libkmipclient/ libkmipclient.so
 .PHONY: $(rootdir)/libkmipclient
 
+$(rootdir)/libap/libap.a: $(rootdir)/libap
+	$(MAKE) -C $(rootdir)/libap/ libap.a
+.PHONY: $(rootdir)/libap
+
+$(rootdir)/libpv/libpv.a: $(rootdir)/libpv
+	$(MAKE) -C $(rootdir)/libpv libpv.a
+.PHONY: $(rootdir)/libpv
+
 $(rootdir)/zipl/boot/data.o:
 	$(MAKE) -C $(rootdir)/zipl/boot/ data.o
 

cpacfstats/Makefile

@@ -1,11 +1,28 @@
 include ../common.mak
 
+ifeq (${HAVE_LIBUDEV},0)
+
+all:
+	$(SKIP) HAVE_LIBUDEV=0
+
+install:
+	$(SKIP) HAVE_LIBUDEV=0
+
+else
+
+check_dep:
+	$(call check_dep, \
+		"cpacfstatsd", \
+		"libudev.h", \
+		"systemd-devel", \
+		"HAVE_LIBUDEV=0")
+
 ALL_CPPFLAGS += -DVERSION=$(VERSION)
 
-all:		cpacfstats cpacfstatsd
+all:		check_dep cpacfstats cpacfstatsd
 
 cpacfstatsd:	cpacfstatsd.o stats_sock.o perf_crypto.o
-		$(LINK) $(ALL_LDFLAGS) $^ $(LDLIBS) -o $@
+		$(LINK) $(ALL_LDFLAGS) $^ $(LDLIBS) -ludev -lpthread -o $@
 
 cpacfstats:	cpacfstats.o stats_sock.o
 		$(LINK) $(ALL_LDFLAGS) $^ $(LDLIBS) -o $@
@@ -16,6 +33,8 @@ install: 	all
 		$(INSTALL) -m 644 cpacfstatsd.8 $(DESTDIR)$(MANDIR)/man8
 		$(INSTALL) -m 644 cpacfstats.1  $(DESTDIR)$(MANDIR)/man1
 
+endif
+
 clean:
 		rm -f *.o *~ cpacfstatsd cpacfstats
 

cpacfstats/cpacfstats.1

@@ -1,6 +1,6 @@
 .\" cpacfstats.1
 .\"
-.\" Copyright IBM Corp. 2015, 2020
+.\" Copyright IBM Corp. 2015, 2022
 .\" s390-tools is free software; you can redistribute it and/or modify
 .\" it under the terms of the MIT license. See LICENSE for details.
 .\"
@@ -32,7 +32,9 @@ cpacfstats \- enable, disable and display CPACF statistical data
 .RB ]
 .RB [ \-p | \-\-print
 .I counter
+.RB [ \-n | \-\-nonzero]
 .RB ]
+.RB [ \-j | \-\-json ]
 .
 .SH DESCRIPTION
 The cpacfstats client application interacts with the cpacfstatsd daemon and
@@ -44,12 +46,347 @@ All counters are initially disabled and must be switched on to measure
 CPACF activities of the system. There is a slight performance penalty with
 CPACF counters enabled.
 
-Note: All cryptographic counters are bound to physical CPUs. If you use
-dedicated CPUs for your LPAR, you obtain only your CPACF activities. If
-your CPUs are shared with other LPARs, their cryptographic activities might
-distort the results. CPACF performance counters are available on LPARs
-only. For security reasons only members of the group \fIcpacfstats\fR are
-allowed to run the cpacfstats client application.
+CPACF activity counters come in two flavors: CPU-MF and PAI.  CPU-MF
+counters are only available on LPARs and have to be authorized.  If
+they are available, the counters
+.B des,
+.B aes,
+.B sha,
+.B rng,
+and
+.B ecc
+are made available.  These counters can individually be activated,
+reset, printed, or deactivated.  PAI counters are a lot more detailed.
+The user interface only offers the counters
+.B pai_user
+and
+.B pai_kernel
+to count CPACF usage in user-space or kernel-space.  When printing
+these counters, detailed counters are shown.  The detailed names are:
+.RS
+.IP \(bu
+KM DES,
+.IP \(bu
+KM 2key TDES,
+.IP \(bu
+KM TDES,
+.IP \(bu
+KM DES protected key,
+.IP \(bu
+KM 2key TDES protected key,
+.IP \(bu
+KM TDES protected key,
+.IP \(bu
+KM AES 128bit,
+.IP \(bu
+KM AES 192bit,
+.IP \(bu
+KM AES 256bit,
+.IP \(bu
+KM AES 128bit protected key,
+.IP \(bu
+KM AES 192bit protected key,
+.IP \(bu
+KM AES 256bit protected key,
+.IP \(bu
+KM AES-XTS 128bit,
+.IP \(bu
+KM AES-XTS 256bit,
+.IP \(bu
+KM AES-XTS 128bit protected key,
+.IP \(bu
+KM AES-XTS 256bit protected key,
+.IP \(bu
+KMC DES,
+.IP \(bu
+KMC 2key TDES,
+.IP \(bu
+KMC TDES,
+.IP \(bu
+KMC DES protected key,
+.IP \(bu
+KMC 2key TDES protected key,
+.IP \(bu
+KMC TDES protected key,
+.IP \(bu
+KMC AES 128bit,
+.IP \(bu
+KMC AES 192bit,
+.IP \(bu
+KMC AES 256bit,
+.IP \(bu
+KMC AES 128bit protected key,
+.IP \(bu
+KMC AES 192bit protected key,
+.IP \(bu
+KMC AES 256bit protected key,
+.IP \(bu
+KMC PRNG,
+.IP \(bu
+KMA AES 128bit,
+.IP \(bu
+KMA AES 192bit,
+.IP \(bu
+KMA AES 256bit,
+.IP \(bu
+KMA AES 128bit protected key,
+.IP \(bu
+KMA AES 192bit protected key,
+.IP \(bu
+KMA AES 256bit protected key,
+.IP \(bu
+KMF DES,
+.IP \(bu
+KMF 2key TDES,
+.IP \(bu
+KMF TDES,
+.IP \(bu
+KMF DES protected key,
+.IP \(bu
+KMF 2key TDES protected key,
+.IP \(bu
+KMF TDES protected key,
+.IP \(bu
+KMF AES 128bit,
+.IP \(bu
+KMF AES 192bit,
+.IP \(bu
+KMF AES 256bit,
+.IP \(bu
+KMF AES 128bit protected key,
+.IP \(bu
+KMF AES 192bit protected key,
+.IP \(bu
+KMF AES 256bit protected key,
+.IP \(bu
+KMCTR DES,
+.IP \(bu
+KMCTR 2key TDES,
+.IP \(bu
+KMCTR TDES,
+.IP \(bu
+KMCTR DES protected key,
+.IP \(bu
+KMCTR 2key TDES protected key,
+.IP \(bu
+KMCTR TDES protected key,
+.IP \(bu
+KMCTR AES 128bit,
+.IP \(bu
+KMCTR AES 192bit,
+.IP \(bu
+KMCTR AES 256bit,
+.IP \(bu
+KMCTR AES 128bit protected key,
+.IP \(bu
+KMCTR AES 192bit protected key,
+.IP \(bu
+KMCTR AES 256bit protected key,
+.IP \(bu
+KMO DES,
+.IP \(bu
+KMO 2key TDES,
+.IP \(bu
+KMO TDES,
+.IP \(bu
+KMO DES protected key,
+.IP \(bu
+KMO 2key TDES protected key,
+.IP \(bu
+KMO TDES protected key,
+.IP \(bu
+KMO AES 128bit,
+.IP \(bu
+KMO AES 192bit,
+.IP \(bu
+KMO AES 256bit,
+.IP \(bu
+KMO AES 128bit protected key,
+.IP \(bu
+KMO AES 192bit protected key,
+.IP \(bu
+KMO AES 256bit protected key,
+.IP \(bu
+KIMD SHA1,
+.IP \(bu
+KIMD SHA256,
+.IP \(bu
+KIMD SHA512,
+.IP \(bu
+KIMD SHA3-224,
+.IP \(bu
+KIMD SHA3-256,
+.IP \(bu
+KIMD SHA3-384,
+.IP \(bu
+KIMD SHA3-512,
+.IP \(bu
+KIMD SHAKE 128,
+.IP \(bu
+KIMD SHAKE 256,
+.IP \(bu
+KIMD GHASH,
+.IP \(bu
+KLMD SHA1,
+.IP \(bu
+KLMD SHA256,
+.IP \(bu
+KLMD SHA512,
+.IP \(bu
+KLMD SHA3-224,
+.IP \(bu
+KLMD SHA3-256,
+.IP \(bu
+KLMD SHA3-384,
+.IP \(bu
+KLMD SHA3-512,
+.IP \(bu
+KLMD SHAKE 128,
+.IP \(bu
+KLMD SHAKE 256,
+.IP \(bu
+KMAC DES,
+.IP \(bu
+KMAC 2key TDES,
+.IP \(bu
+KMAC TDES,
+.IP \(bu
+KMAC DES protected key,
+.IP \(bu
+KMAC 2key TDES protected key,
+.IP \(bu
+KMAC TDES protected key,
+.IP \(bu
+KMAC AES 128bit,
+.IP \(bu
+KMAC AES 192bit,
+.IP \(bu
+KMAC AES 256bit,
+.IP \(bu
+KMAC AES 128bit protected key,
+.IP \(bu
+KMAC AES 192bit protected key,
+.IP \(bu
+KMAC AES 256bit protected key,
+.IP \(bu
+PCC Last Block CMAC DES,
+.IP \(bu
+PCC Last Block CMAC 2key TDES,
+.IP \(bu
+PCC Last Block CMAC TDES,
+.IP \(bu
+PCC Last Block CMAC DES protected key,
+.IP \(bu
+PCC Last Block CMAC 2key TDES protected key,
+.IP \(bu
+PCC Last Block CMAC TDES protected key,
+.IP \(bu
+PCC Last Block CMAC AES 128bit,
+.IP \(bu
+PCC Last Block CMAC AES 192bit,
+.IP \(bu
+PCC Last Block CMAC AES 256bit,
+.IP \(bu
+PCC Last Block CMAC AES 128bit protected key,
+.IP \(bu
+PCC Last Block CMAC AES 192bit protected key,
+.IP \(bu
+PCC Last Block CMAC AES 256bit protected key,
+.IP \(bu
+PCC XTS Parameter AES 128bit,
+.IP \(bu
+PCC XTS Parameter AES 256bit,
+.IP \(bu
+PCC XTS Parameter AES 128bit protected key,
+.IP \(bu
+PCC XTS Parameter AES 256bit protected key,
+.IP \(bu
+PCC Scalar Mult P256,
+.IP \(bu
+PCC Scalar Mult P384,
+.IP \(bu
+PCC Scalar Mult P521,
+.IP \(bu
+PCC Scalar Mult Ed25519,
+.IP \(bu
+PCC Scalar Mult Ed448,
+.IP \(bu
+PCC Scalar Mult X25519,
+.IP \(bu
+PCC Scalar Mult X448,
+.IP \(bu
+PRNO SHA512 DRNG,
+.IP \(bu
+PRNO TRNG Query Ratio,
+.IP \(bu
+PRNO TRNG,
+.IP \(bu
+KDSA ECDSA Verify P256,
+.IP \(bu
+KDSA ECDSA Verify P384,
+.IP \(bu
+KDSA ECDSA Verify P521,
+.IP \(bu
+KDSA ECDSA Sign P256,
+.IP \(bu
+KDSA ECDSA Sign P384,
+.IP \(bu
+KDSA ECDSA Sign P521,
+.IP \(bu
+KDSA ECDSA Sign P256 protected key,
+.IP \(bu
+KDSA ECDSA Sign P384 protected key,
+.IP \(bu
+KDSA ECDSA Sign P521 protected key,
+.IP \(bu
+KDSA EdDSA Verify Ed25519,
+.IP \(bu
+KDSA EdDSA Verify Ed448,
+.IP \(bu
+KDSA EdDSA Sign Ed25519,
+.IP \(bu
+KDSA EdDSA Sign Ed448,
+.IP \(bu
+KDSA EdDSA Sign Ed25519 protected key,
+.IP \(bu
+KDSA EdDSA Sign Ed448 protected key,
+.IP \(bu
+PCKMO DES,
+.IP \(bu
+PCKMO 2key TDES,
+.IP \(bu
+PCMKO TDES,
+.IP \(bu
+PCKMO AES 128bit,
+.IP \(bu
+PCKMO AES 192bit,
+.IP \(bu
+PCMKO AES 256bit,
+.IP \(bu
+PCKMO ECC P256,
+.IP \(bu
+PCKMO ECC P384,
+.IP \(bu
+PCKMO ECC P521,
+.IP \(bu
+PCKMO ECC Ed25519,
+.IP \(bu
+PCKMO ECC Ed448,
+.IP \(bu
+Reserved 1, and
+.IP \(bu
+Reserved 2.
+.RE
+Note that the counters starting with PCKMO and Reserved are only
+available in the pai_kernel set.  Also note that the counters are
+designed to count successful operations.  In the case of KMA this
+means only complete GCM operations including final hashing are
+counted.
+
+Note: CPU-MF based CPACF performance counters are available on LPARs
+only. PAI counters are available on all hypervisors. For security
+reasons only members of the group \fIcpacfstats\fR are allowed to run
+the cpacfstats client application.
 
 Example usage scenario:
 .P
@@ -101,12 +438,34 @@ argument can be one of: \fBdes\fR, \fBaes\fR, \fBsha\fR, \fBprng\fR,
 \fBecc\fR, or \fBall\fR. If the counter argument is omitted, all
 performance counters are reset to 0.
 .TP
-\fB\-p\fR or \fB\-\-print\fR [counter]
+\fB\-p\fR or \fB\-\-print\fR  [\fB\-n\fR or \fB\-\-nonzero\fR] [counter]
 Display the value of one or all CPACF performance counters. The
 optional counter argument can be one of: \fBdes\fR, \fBaes\fR,
-\fBsha\fR, \fBprng\fR, \fBecc\fR, or \fBall\fR. If the counter
-argument is omitted or if there is no argument, all performance
-counters are displayed.
+\fBsha\fR, \fBprng\fR, \fBecc\fR, \fBpai_user\fR, \fBpai_kernel\fR, or
+\fBall\fR. If the counter argument is omitted or if there is no
+argument, all performance counters are displayed.  If the optional
+\fB\-n\fR or \fB\-\-nonzero\fR argument is given, then only PAI
+counters that have a non-zero value are printed.
+.TP
+\fB\-j\fR or \fB\-\-json\fR
+Display all activated counters in JSON format.  The JSON contains an
+array of counter objects.  Each object contains the property
+.B counter
+specifying either a CPU-MF counter of one of the detailed PAI counter.
+Additional properties include
+.B error
+an error number if the counter could not be read,
+.B value
+the counter value if the counter could be read,
+.B space
+for PAI counters to specify
+.B user
+or
+.B kernel
+space counter set, and
+.B counterid
+for PAI counters to specify the PAI counter number as specified in the
+Principles of Operation.
 .TP
 The default command is --print all.
 .

cpacfstats/cpacfstats.c

@@ -3,7 +3,7 @@
  *
  * cpacfstats client implementation
  *
- * Copyright IBM Corp. 2015, 2020
+ * Copyright IBM Corp. 2015, 2022
  *
  * s390-tools is free software; you can redistribute it and/or modify
  * it under the terms of the MIT license. See LICENSE for details.
@@ -39,17 +39,187 @@ static const char *const usage =
 	"\t-d, --disable [counter]   Disable one or all counters\n"
 	"\t-r, --reset   [counter]   Reset one or all counter values\n"
 	"\t-p, --print   [counter]   Print one or all counter values\n"
-	"\tcounter can be: 'aes' 'des' 'rng' 'sha' 'ecc' or 'all'\n";
+	"\t-n, --nonzero             Print all PAI counters\n"
+	"\t-j, --json                Print all counter values in JSON format\n"
+	"\tcounter can be: 'aes' 'des' 'rng' 'sha' 'ecc'\n"
+	"\t                'pai_user' 'pai_kernel' or 'all'\n";
 
 static const char *const counter_str[] = {
 	[DES_FUNCTIONS]  = "des",
 	[AES_FUNCTIONS]  = "aes",
 	[SHA_FUNCTIONS]  = "sha",
 	[PRNG_FUNCTIONS] = "rng",
-	[ECC_FUNCTIONS] = "ecc",
-	[ALL_COUNTER]    = "all"
+	[ECC_FUNCTIONS]  = "ecc",
+	[ALL_COUNTER]    = "all",
+	[PAI_USER]       = "pai_user",
+	[PAI_KERNEL]     = "pai_kernel"
 };
 
+/* Strings for the pai counter details.  Note that this is 0-based
+ * while PoP is 1-based.
+ */
+static const char *const pai_str[] = {
+	[  0] = "KM DES",
+	[  1] = "KM 2key TDES",
+	[  2] = "KM TDES",
+	[  3] = "KM DES protected key",
+	[  4] = "KM 2key TDES protected key",
+	[  5] = "KM TDES protected key",
+	[  6] = "KM AES 128bit",
+	[  7] = "KM AES 192bit",
+	[  8] = "KM AES 256bit",
+	[  9] = "KM AES 128bit protected key",
+	[ 10] = "KM AES 192bit protected key",
+	[ 11] = "KM AES 256bit protected key",
+	[ 12] = "KM AES-XTS 128bit",
+	[ 13] = "KM AES-XTS 256bit",
+	[ 14] = "KM AES-XTS 128bit protected key",
+	[ 15] = "KM AES-XTS 256bit protected key",
+	[ 16] = "KMC DES",
+	[ 17] = "KMC 2key TDES",
+	[ 18] = "KMC TDES",
+	[ 19] = "KMC DES protected key",
+	[ 20] = "KMC 2key TDES protected key",
+	[ 21] = "KMC TDES protected key",
+	[ 22] = "KMC AES 128bit",
+	[ 23] = "KMC AES 192bit",
+	[ 24] = "KMC AES 256bit",
+	[ 25] = "KMC AES 128bit protected key",
+	[ 26] = "KMC AES 192bit protected key",
+	[ 27] = "KMC AES 256bit protected key",
+	[ 28] = "KMC PRNG",
+	[ 29] = "KMA AES 128bit",
+	[ 30] = "KMA AES 192bit",
+	[ 31] = "KMA AES 256bit",
+	[ 32] = "KMA AES 128bit protected key",
+	[ 33] = "KMA AES 192bit protected key",
+	[ 34] = "KMA AES 256bit protected key",
+	[ 35] = "KMF DES",
+	[ 36] = "KMF 2key TDES",
+	[ 37] = "KMF TDES",
+	[ 38] = "KMF DES protected key",
+	[ 39] = "KMF 2key TDES protected key",
+	[ 40] = "KMF TDES protected key",
+	[ 41] = "KMF AES 128bit",
+	[ 42] = "KMF AES 192bit",
+	[ 43] = "KMF AES 256bit",
+	[ 44] = "KMF AES 128bit protected key",
+	[ 45] = "KMF AES 192bit protected key",
+	[ 46] = "KMF AES 256bit protected key",
+	[ 47] = "KMCTR DES",
+	[ 48] = "KMCTR 2key TDES",
+	[ 49] = "KMCTR TDES",
+	[ 50] = "KMCTR DES protected key",
+	[ 51] = "KMCTR 2key TDES protected key",
+	[ 52] = "KMCTR TDES protected key",
+	[ 53] = "KMCTR AES 128bit",
+	[ 54] = "KMCTR AES 192bit",
+	[ 55] = "KMCTR AES 256bit",
+	[ 56] = "KMCTR AES 128bit protected key",
+	[ 57] = "KMCTR AES 192bit protected key",
+	[ 58] = "KMCTR AES 256bit protected key",
+	[ 59] = "KMO DES",
+	[ 60] = "KMO 2key TDES",
+	[ 61] = "KMO TDES",
+	[ 62] = "KMO DES protected key",
+	[ 63] = "KMO 2key TDES protected key",
+	[ 64] = "KMO TDES protected key",
+	[ 65] = "KMO AES 128bit",
+	[ 66] = "KMO AES 192bit",
+	[ 67] = "KMO AES 256bit",
+	[ 68] = "KMO AES 128bit protected key",
+	[ 69] = "KMO AES 192bit protected key",
+	[ 70] = "KMO AES 256bit protected key",
+	[ 71] = "KIMD SHA1",
+	[ 72] = "KIMD SHA256",
+	[ 73] = "KIMD SHA512",
+	[ 74] = "KIMD SHA3-224",
+	[ 75] = "KIMD SHA3-256",
+	[ 76] = "KIMD SHA3-384",
+	[ 77] = "KIMD SHA3-512",
+	[ 78] = "KIMD SHAKE 128",
+	[ 79] = "KIMD SHAKE 256",
+	[ 80] = "KIMD GHASH",
+	[ 81] = "KLMD SHA1",
+	[ 82] = "KLMD SHA256",
+	[ 83] = "KLMD SHA512",
+	[ 84] = "KLMD SHA3-224",
+	[ 85] = "KLMD SHA3-256",
+	[ 86] = "KLMD SHA3-384",
+	[ 87] = "KLMD SHA3-512",
+	[ 88] = "KLMD SHAKE 128",
+	[ 89] = "KLMD SHAKE 256",
+	[ 90] = "KMAC DES",
+	[ 91] = "KMAC 2key TDES",
+	[ 92] = "KMAC TDES",
+	[ 93] = "KMAC DES protected key",
+	[ 94] = "KMAC 2key TDES protected key",
+	[ 95] = "KMAC TDES protected key",
+	[ 96] = "KMAC AES 128bit",
+	[ 97] = "KMAC AES 192bit",
+	[ 98] = "KMAC AES 256bit",
+	[ 99] = "KMAC AES 128bit protected key",
+	[100] = "KMAC AES 192bit protected key",
+	[101] = "KMAC AES 256bit protected key",
+	[102] = "PCC Last Block CMAC DES",
+	[103] = "PCC Last Block CMAC 2key TDES",
+	[104] = "PCC Last Block CMAC TDES",
+	[105] = "PCC Last Block CMAC DES protected key",
+	[106] = "PCC Last Block CMAC 2key TDES protected key",
+	[107] = "PCC Last Block CMAC TDES protected key",
+	[108] = "PCC Last Block CMAC AES 128bit",
+	[109] = "PCC Last Block CMAC AES 192bit",
+	[110] = "PCC Last Block CMAC AES 256bit",
+	[111] = "PCC Last Block CMAC AES 128bit protected key",
+	[112] = "PCC Last Block CMAC AES 192bit protected key",
+	[113] = "PCC Last Block CMAC AES 256bit protected key",
+	[114] = "PCC XTS Parameter AES 128bit",
+	[115] = "PCC XTS Parameter AES 256bit",
+	[116] = "PCC XTS Parameter AES 128bit protected key",
+	[117] = "PCC XTS Parameter AES 256bit protected key",
+	[118] = "PCC Scalar Mult P256",
+	[119] = "PCC Scalar Mult P384",
+	[120] = "PCC Scalar Mult P521",
+	[121] = "PCC Scalar Mult Ed25519",
+	[122] = "PCC Scalar Mult Ed448",
+	[123] = "PCC Scalar Mult X25519",
+	[124] = "PCC Scalar Mult X448",
+	[125] = "PRNO SHA512 DRNG",
+	[126] = "PRNO TRNG Query Ratio",
+	[127] = "PRNO TRNG",
+	[128] = "KDSA ECDSA Verify P256",
+	[129] = "KDSA ECDSA Verify P384",
+	[130] = "KDSA ECDSA Verify P521",
+	[131] = "KDSA ECDSA Sign P256",
+	[132] = "KDSA ECDSA Sign P384",
+	[133] = "KDSA ECDSA Sign P521",
+	[134] = "KDSA ECDSA Sign P256 protected key",
+	[135] = "KDSA ECDSA Sign P384 protected key",
+	[136] = "KDSA ECDSA Sign P521 protected key",
+	[137] = "KDSA EdDSA Verify Ed25519",
+	[138] = "KDSA EdDSA Verify Ed448",
+	[139] = "KDSA EdDSA Sign Ed25519",
+	[140] = "KDSA EdDSA Sign Ed448",
+	[141] = "KDSA EdDSA Sign Ed25519 protected key",
+	[142] = "KDSA EdDSA Sign Ed448 protected key",
+	[143] = "PCKMO DES",
+	[144] = "PCKMO 2key TDES",
+	[145] = "PCMKO TDES",
+	[146] = "PCKMO AES 128bit",
+	[147] = "PCKMO AES 192bit",
+	[148] = "PCMKO AES 256bit",
+	[149] = "PCKMO ECC P256",
+	[150] = "PCKMO ECC P384",
+	[151] = "PCKMO ECC P521",
+	[152] = "PCKMO ECC Ed25519",
+	[153] = "PCKMO ECC Ed448",
+	[154] = "Reserved 1",
+	[155] = "Reserved 2"
+};
+
+
+static int paiprintnonzero;
+
 
 static int send_query(int s, enum cmd_e cmd, enum ctr_e ctr)
 {
@@ -62,7 +232,7 @@ static int send_query(int s, enum cmd_e cmd, enum ctr_e ctr)
 	m.query.m_ctr = ctr;
 	m.query.m_cmd = cmd;
 
-	return send_msg(s, &m);
+	return send_msg(s, &m, 0);
 }
 
 
@@ -71,7 +241,7 @@ static int recv_answer(int s, int *ctr, int *state, uint64_t *value)
 	struct msg m;
 	int rc;
 
-	rc = recv_msg(s, &m);
+	rc = recv_msg(s, &m, 0);
 	if (rc == 0) {
 		if (m.head.m_ver != VERSION) {
 			eprint("Received msg with wrong version %d != %d\n",
@@ -92,9 +262,139 @@ static int recv_answer(int s, int *ctr, int *state, uint64_t *value)
 }
 
 
-static void print_answer(int ctr, int state, uint64_t value)
+static void printjsonsep(void)
 {
-	if (state < 0)
+	static const char *jsonsep = "";
+
+	fputs(jsonsep, stdout);
+	jsonsep = ",";
+}
+
+
+static void json_print_virtual_counter_answer(int s, int ctr,
+					      int state, uint64_t value)
+{
+	int paictr = 0, paistate = 0, ec;
+	uint64_t i, paivalue = 0, maxnum;
+	const char *space;
+
+	switch (ctr) {
+	case HOTPLUG_DETECTED:
+		printjsonsep();
+		printf("{\"counter\":\"hotplug detected\",");
+		if (state < 0)
+			printf("\"error\":%d}", state);
+		else
+			printf("\"value\":%d}", !!value);
+		return;
+	case PAI_USER:
+		maxnum = NUM_PAI_USER;
+		space = "user";
+		break;
+	case PAI_KERNEL:
+		maxnum = NUM_PAI_KERNEL;
+		space = "kernel";
+		break;
+	default:
+		return;
+	}
+	/* Here, we have validated the PAI counter retrieved, but not
+	 * yet printed. */
+	if (state != ENABLED)
+		return;
+	if (value > maxnum) {
+		eprint("Incompatible versions detected!\n");
+		eprint("Expected %"PRIu64" counter space for %s, but got %"PRIu64"\n",
+			maxnum, space, value);
+		exit(EXIT_FAILURE);
+	}
+	for (i = 0; i < value; ++i) {
+		ec = recv_answer(s, &paictr, &paistate, &paivalue);
+		if (ec < 0 || paistate < 0) {
+			eprint("Error on receiving answer message from daemon\n");
+			/* No more data for this virtual event after error. */
+			return;
+		}
+		if (paictr > NUM_PAI_KERNEL) {
+			eprint("Pai counter number too big: %d\n", paictr);
+		} else {
+			printjsonsep();
+			printf("{\"counter\":\"%s\",\"space\":\"%s\",\"counterid\":%d,",
+				pai_str[paictr], space, paictr + 1);
+			if (paistate < 0) {
+				printf("\"error\":%d}", paistate);
+				/* Protocol does not send furter counters. */
+				return;
+			}
+			printf("\"value\":%"PRIu64"}", paivalue);
+		}
+	}
+}
+
+
+static void print_virtual_counter_answer(int s,
+					 int ctr, int state, uint64_t value)
+{
+	static const char *const states[] = {
+		[DISABLED]    = "disabled",
+		[ENABLED]     = "enabled",
+		[UNSUPPORTED] = "unsupported"
+	};
+	int paictr = 0, paistate = 0, ec;
+	uint64_t i, paivalue = 0, maxnum;
+	const char *ctrstr;
+
+	switch (ctr) {
+	case HOTPLUG_DETECTED:
+		if (state >= 0 && value > 0)
+			printf(" hotplug detected\n");
+		return;
+	case PAI_USER:
+		maxnum = NUM_PAI_USER;
+		ctrstr = "pai_user";
+		break;
+	case PAI_KERNEL:
+		maxnum = NUM_PAI_KERNEL;
+		ctrstr = "pai_kernel";
+		break;
+	default:
+		return;
+	}
+	/* Here, we have validated the PAI counter retrieved, but not
+	 * yet printed. */
+	if (state < 0 || state > UNSUPPORTED) {
+		printf(" %11s: error state %d\n", ctrstr, state);
+		/* No details follow if counter in error state. */
+		return;
+	}
+	printf(" %-11s: %s\n", ctrstr, states[state]);
+	if (state != ENABLED)
+		return;
+	if (value > maxnum) {
+		eprint("Incompatible versions detected!\n");
+		eprint("Expected %"PRIu64" counters for %s, but got %"PRIu64"\n",
+		       maxnum, ctrstr, value);
+		exit(EXIT_FAILURE);
+	}
+	for (i = 0; i < value; ++i) {
+		ec = recv_answer(s, &paictr, &paistate, &paivalue);
+		if (ec < 0 || paistate < 0) {
+			eprint("Error on receiving answer message from daemon\n");
+			/* No more data for this virtual event after error. */
+			return;
+		}
+		if (paictr > NUM_PAI_KERNEL)
+			eprint("Pai counter number too big: %d\n", paictr);
+		else if (!paiprintnonzero || paivalue > 0)
+			printf("  %-45s: %"PRIu64"\n", pai_str[paictr], paivalue);
+	}
+}
+
+static void print_answer(int s, int ctr, int state, uint64_t value)
+{
+	if (ctr > ALL_COUNTER)
+		print_virtual_counter_answer(s, ctr, state, value);
+	else if (state < 0)
 		printf(" %s counter: error state %d\n",
 		       counter_str[ctr], state);
 	else if (state == DISABLED)
@@ -106,6 +406,22 @@ static void print_answer(int ctr, int state, uint64_t value)
 }
 
 
+static void json_print_answer(int s, int ctr, int state, uint64_t value)
+{
+	if (ctr > ALL_COUNTER) {
+		json_print_virtual_counter_answer(s, ctr, state, value);
+	} else if (state < 0) {
+		printjsonsep();
+		printf("{\"counter\":\"%s\",", counter_str[ctr]);
+		printf("\"error\":%d}\n", state);
+	} else if (state == ENABLED) {
+		printjsonsep();
+		printf("{\"counter\":\"%s\",", counter_str[ctr]);
+		printf("\"value\":%"PRIu64"}", value);
+	}
+}
+		
+
 int eprint(const char *format, ...)
 {
 	char buf[1024];
@@ -128,7 +444,7 @@ int main(int argc, char *argv[])
 {
 	enum ctr_e ctr = ALL_COUNTER;
 	enum cmd_e cmd = PRINT;
-	int i, j, s, state;
+	int i, j, s, state, num, json = 0;
 	uint64_t value;
 
 	if (argc > 1) {
@@ -140,98 +456,108 @@ int main(int argc, char *argv[])
 			{ "disable", 0, NULL, 'd' },
 			{ "reset", 0, NULL, 'r' },
 			{ "print", 0, NULL, 'p' },
+			{ "nonzero", 0, NULL, 'n' },
+			{ "json", 0, NULL, 'j' },
 			{ NULL, 0, NULL, 0 } };
 		while (1) {
 			opt = getopt_long(argc, argv,
-					  "hvedrp", long_opts, &idx);
+					  "hvedrpnj", long_opts, &idx);
 			if (opt == -1)
 				break; /* no more arguments */
 			switch (opt) {
 			case 'h':
 				printf(usage, name);
-				exit(0);
-				break;
+				return 0;
 			case 'v':
 				printf("%s: Linux on System z CPACF Crypto Activity Counters Client\n"
 				       "Version %s\n%s\n",
 				       name, RELEASE_STRING, COPYRIGHT);
-				exit(0);
-				break;
+				return 0;
 			case 'e':
 				cmd = ENABLE;
+				json = 0;
 				break;
 			case 'd':
 				cmd = DISABLE;
+				json = 0;
 				break;
 			case 'r':
 				cmd = RESET;
 				break;
 			case 'p':
 				cmd = PRINT;
+				json = 0;
+				break;
+			case 'n':
+				paiprintnonzero = 1;
+				break;
+			case 'j':
+				cmd = PRINT;
+				json = 1;
 				break;
 			default:
 				eprint("Invalid argument, try -h or --help for more information\n");
-				exit(1);
-				break;
+				return EXIT_FAILURE;
 			}
 		}
 		/* there may be an optional counter argument */
 		if (optind > 0 && optind < argc) {
-			for (i = 0; i <= ALL_COUNTER; i++)
+			for (i = 0; i < NUM_COUNTER; i++)
 				if (strcmp(argv[optind], counter_str[i]) == 0)
 					break;
-			if (i > ALL_COUNTER) {
+			if (i >= NUM_COUNTER) {
 				eprint("Unknown counter '%s'\n", argv[optind]);
-				exit(1);
+				return EXIT_FAILURE;
 			}
 			ctr = (enum ctr_e) i;
 		}
 	}
+	if (json)
+		ctr = ALL_COUNTER;
 
 	/* try to open and connect socket to the cpacfstatsd daemon */
 	s = open_socket(CLIENT);
 	if (s < 0) {
 		eprint("Can't connect to daemon\n");
-		exit(1);
+		return EXIT_FAILURE;
 	}
 
 	/* send query */
 	if (send_query(s, cmd, ctr) != 0) {
 		eprint("Error on sending query message to daemon\n");
 		close(s);
-		exit(1);
+		return EXIT_FAILURE;
 	}
 
 	if (ctr == ALL_COUNTER) {
-		for (i = 0; i < ALL_COUNTER; i++) {
-			/* receive answer */
-			if (recv_answer(s, &j, &state, &value) != 0) {
-				eprint("Error on receiving answer message from daemon\n");
-				exit(1);
-			}
-			if (state < 0) {
-				eprint("Received bad status code %d from daemon\n",
-				       state);
-				close(s);
-				exit(1);
-			}
-			print_answer(j, state, value);
-		}
+		/* The -1 is for ALL_COUNTER which is not sent, +1 for
+		 * hotplug state. */
+		num = NUM_COUNTER - 1 + 1;
 	} else {
+		/* +1 for hotplug state */
+		num = 1 + 1;
+	}
+	if (json)
+		putchar('[');
+	for (i = 0; i < num; i++) {
 		/* receive answer */
 		if (recv_answer(s, &j, &state, &value) != 0) {
 			eprint("Error on receiving answer message from daemon\n");
-			close(s);
-			exit(1);
+			return EXIT_FAILURE;
 		}
 		if (state < 0) {
 			eprint("Received bad status code %d from daemon\n",
-			       state);
+				state);
 			close(s);
-			exit(1);
+			return EXIT_FAILURE;
 		}
-		print_answer(j, state, value);
+		if (json)
+			json_print_answer(s, j, state, value);
+		else
+			print_answer(s, j, state, value);
 	}
+	if (json)
+		putchar(']');
 
 	/* close connection */
 	close(s);

cpacfstats/cpacfstats.h

@@ -3,7 +3,7 @@
  *
  * common function prototypes and definitions
  *
- * Copyright IBM Corp. 2015, 2020
+ * Copyright IBM Corp. 2015, 2022
  *
  * s390-tools is free software; you can redistribute it and/or modify
  * it under the terms of the MIT license. See LICENSE for details.
@@ -14,13 +14,27 @@
 
 #include "lib/zt_common.h"
 
-#define COPYRIGHT "Copyright IBM Corp. 2015, 2020"
+#define COPYRIGHT "Copyright IBM Corp. 2015, 2022"
+
+#define DEFAULT_SEND_TIMEOUT  (30 * 1000)
+#define DEFAULT_RECV_TIMEOUT  (30 * 1000)
+
+/*
+ * Number of PAI counters for user space.  This excludes PCKMO since
+ * this instruction is privileged.
+ */
+#define NUM_PAI_USER     143
+/* Number of PAI counters for kernel space.  Contains all counters. */
+#define NUM_PAI_KERNEL   156
 
 int eprint(const char *format, ...);
 
 /*
  * Counter names
- * ALL_COUNTER must always be the last member of the enum
+ * ALL_COUNTER specifies the number of physical counters.  Virtual
+ * counters might be added afterwards.  NUM_COUNTER is the last
+ * managed counter (i.e., a counter that can be activated, reset,
+ * deactivated).
  */
 enum ctr_e {
 	DES_FUNCTIONS = 0,
@@ -28,7 +42,11 @@ enum ctr_e {
 	SHA_FUNCTIONS,
 	PRNG_FUNCTIONS,
 	ECC_FUNCTIONS,
-	ALL_COUNTER
+	ALL_COUNTER,
+	PAI_USER,
+	PAI_KERNEL,
+	NUM_COUNTER,
+	HOTPLUG_DETECTED = 0xffff
 };
 
 enum type_e {
@@ -50,7 +68,7 @@ enum state_e {
 };
 
 /*
- * query send from clent to daemon
+ * query send from client to daemon
  * Consist of:
  * enum counter
  * enum command
@@ -63,7 +81,7 @@ struct msg_query {
 /*
  * answer send from daemon to client
  * Consist of:
- * enum counter
+ * enum counter or PAI counter number if following PAI_USER or PAI_KERNEL
  * status code: < 0 error, 0 disabled, > 0 enabled
  * counter value
  */
@@ -99,17 +117,20 @@ struct msg {
 } __packed;
 
 int open_socket(int mode);
-int send_msg(int sfd, struct msg *m);
-int recv_msg(int sfd, struct msg *m);
+int send_msg(int sfd, struct msg *m, int timeout);
+int recv_msg(int sfd, struct msg *m, int timeout);
 
 /* perf_crypto.c */
 
 int  perf_init(void);
+void perf_stop(void);
 void perf_close(void);
 int  perf_enable_ctr(enum ctr_e ctr);
 int  perf_disable_ctr(enum ctr_e ctr);
-int  perf_reset_ctr(enum ctr_e ctr);
+int  perf_reset_ctr(enum ctr_e ctr, uint64_t *value);
 int  perf_read_ctr(enum ctr_e ctr, uint64_t *value);
 int  perf_ecc_supported(void);
+int  perf_ctr_state(enum ctr_e ctr);
+int  perf_read_pai_ctr(unsigned int ctrnum, int user, uint64_t *value);
 
 #endif

cpacfstats/cpacfstatsd.c

@@ -3,7 +3,7 @@
  *
  * cpacfstatsd daemon implementation
  *
- * Copyright IBM Corp. 2015, 2017
+ * Copyright IBM Corp. 2015, 2022
  *
  * s390-tools is free software; you can redistribute it and/or modify
  * it under the terms of the MIT license. See LICENSE for details.
@@ -28,6 +28,7 @@
 #include "lib/zt_common.h"
 #include "cpacfstats.h"
 
+static volatile int stopsig;
 
 static const char *const name = "cpacfstatsd";
 
@@ -43,15 +44,12 @@ static const char *const usage =
 
 static int daemonized;
 
-static int ctr_state[ALL_COUNTER];
-
-
 static int recv_query(int s, enum ctr_e *ctr, enum cmd_e *cmd)
 {
 	struct msg m;
 	int rc;
 
-	rc = recv_msg(s, &m);
+	rc = recv_msg(s, &m, DEFAULT_RECV_TIMEOUT);
 	if (rc == 0) {
 		if (m.head.m_ver != VERSION) {
 			eprint("Received msg with wrong version %d != %d\n",
@@ -83,38 +81,82 @@ static int send_answer(int s, int ctr, int state, uint64_t value)
 	m.answer.m_state = state;
 	m.answer.m_value = value;
 
-	return send_msg(s, &m);
+	return send_msg(s, &m, DEFAULT_SEND_TIMEOUT);
+}
+
+
+/*
+ * Print according to protocol for PAI:
+ * - first the state and the number of PAI counters that follow
+ * - if state is ENABLED:
+ *   - for each PAI counter the value with state ENABLED
+ * Note that the PAI counters are 0-based, not 1 based as in PoP!
+ * Sending ends with the first error.
+ */
+static int do_send_pai(int s, int user)
+{
+	int ctr, state, i, maxctr, rc = 0;
+	uint64_t value;
+	
+	if (user) {
+		ctr = PAI_USER;
+		maxctr = NUM_PAI_USER;
+	} else {
+		ctr = PAI_KERNEL;
+		maxctr = NUM_PAI_KERNEL;
+	}
+	state = perf_ctr_state(ctr);
+	if (state != ENABLED)
+		return rc;
+	for (i = 0; i < maxctr; ++i) {
+		rc = perf_read_pai_ctr(i, user, &value);
+		if (rc != 0) {
+			send_answer(s, i, rc, 0);
+			break;
+		}
+		send_answer(s, i, state, value);
+	}
+	return rc;
 }
 
 
 static int do_enable(int s, enum ctr_e ctr)
 {
-	uint64_t value;
+	uint64_t value = 0;
 	int i, rc = 0;
+	int state;
 
-	for (i = 0; i < ALL_COUNTER; i++) {
+	for (i = 0; i < NUM_COUNTER; i++) {
+		if (i == ALL_COUNTER)
+			continue;
 		if (i == (int) ctr || ctr == ALL_COUNTER) {
-			if (ctr_state[i] == DISABLED) {
+			state = perf_ctr_state(i);
+			if (state == DISABLED) {
 				rc = perf_enable_ctr(i);
 				if (rc != 0) {
 					send_answer(s, i, rc, 0);
 					break;
 				}
-				ctr_state[i] = ENABLED;
+				state = ENABLED;
 			}
-			if (ctr_state[i] == UNSUPPORTED) {
-				send_answer(s, i, UNSUPPORTED, 0);
-			} else {
+			if (state != UNSUPPORTED) {
 				rc = perf_read_ctr(i, &value);
 				if (rc != 0) {
 					send_answer(s, i, rc, 0);
 					break;
 				}
-				send_answer(s, i, ENABLED, value);
 			}
+			send_answer(s, i, state, value);
+			if (i == PAI_USER)
+				rc = do_send_pai(s, 1);
+			if (i == PAI_KERNEL)
+				rc = do_send_pai(s, 0);
 		}
 	}
-
+	if (rc == 0) {
+		rc = perf_read_ctr(HOTPLUG_DETECTED, &value);
+		send_answer(s, HOTPLUG_DETECTED, rc, value);
+	}
 	return rc;
 }
 
@@ -122,67 +164,90 @@ static int do_enable(int s, enum ctr_e ctr)
 static int do_disable(int s, enum ctr_e ctr)
 {
 	int i, rc = 0;
+	uint64_t value;
 
-	for (i = 0; i < ALL_COUNTER; i++) {
+	for (i = 0; i < NUM_COUNTER; i++) {
+		if (i == ALL_COUNTER)
+			continue;
 		if (i == (int) ctr || ctr == ALL_COUNTER) {
-			if (ctr_state[i] == ENABLED) {
+			if (perf_ctr_state(i) == ENABLED) {
 				rc = perf_disable_ctr(i);
 				if (rc != 0) {
 					send_answer(s, i, rc, 0);
 					break;
 				}
-				ctr_state[i] = 0;
 			}
-			send_answer(s, i, ctr_state[i], 0);
+			send_answer(s, i, perf_ctr_state(i), 0);
 		}
 	}
-
+	if (rc == 0) {
+		rc = perf_read_ctr(HOTPLUG_DETECTED, &value);
+		send_answer(s, HOTPLUG_DETECTED, rc, value);
+	}
 	return rc;
 }
 
 
 static int do_reset(int s, enum ctr_e ctr)
 {
-	int i, rc = 0;
+	int i, rc = 0, state;
+	uint64_t value;
 
-	for (i = 0; i < ALL_COUNTER; i++) {
+	for (i = 0; i < NUM_COUNTER; i++) {
+		if (i == ALL_COUNTER)
+			continue;
 		if (i == (int) ctr || ctr == ALL_COUNTER) {
-			if (ctr_state[i] == ENABLED) {
-				rc = perf_reset_ctr(i);
+			state = perf_ctr_state(i);
+			if (state == ENABLED) {
+				rc = perf_reset_ctr(i, &value);
 				if (rc != 0) {
 					send_answer(s, i, rc, 0);
 					break;
 				}
-				send_answer(s, i, ENABLED, 0);
-			} else {
-				send_answer(s, i, ctr_state[i], 0);
 			}
+			send_answer(s, i, state, value);
+			if (i == PAI_USER)
+				rc = do_send_pai(s, 1);
+			if (i == PAI_KERNEL)
+				rc = do_send_pai(s, 0);
 		}
 	}
-
+	if (rc == 0) {
+		rc = perf_read_ctr(HOTPLUG_DETECTED, &value);
+		send_answer(s, HOTPLUG_DETECTED, rc, value);
+	}
 	return rc;
 }
 
+
 static int do_print(int s, enum ctr_e ctr)
 {
-	int i, rc = 0;
-	uint64_t value;
+	int i, rc = 0, state;
+	uint64_t value = 0;
 
-	for (i = 0; i < ALL_COUNTER; i++) {
+	for (i = 0; i < NUM_COUNTER; i++) {
+		if (i == ALL_COUNTER)
+			continue;
 		if (i == (int) ctr || ctr == ALL_COUNTER) {
-			if (ctr_state[i] == ENABLED) {
+			state = perf_ctr_state(i);
+			if (state == ENABLED) {
 				rc = perf_read_ctr(i, &value);
 				if (rc != 0) {
 					send_answer(s, i, rc, 0);
 					break;
 				}
-				send_answer(s, i, ENABLED, value);
-			} else {
-				send_answer(s, i, ctr_state[i], 0);
 			}
+			send_answer(s, i, state, value);
+			if (i == PAI_USER)
+				rc = do_send_pai(s, 1);
+			if (i == PAI_KERNEL)
+				rc = do_send_pai(s, 0);
 		}
 	}
-
+	if (rc == 0) {
+		rc = perf_read_ctr(HOTPLUG_DETECTED, &value);
+		send_answer(s, HOTPLUG_DETECTED, rc, value);
+	}
 	return rc;
 }
 
@@ -324,18 +389,8 @@ static void remove_pidfile(void)
 
 void signalhandler(int sig)
 {
-	if (sig == SIGTERM)
-		eprint("Caught signal SIGTERM, terminating...\n");
-	else if (sig == SIGINT)
-		eprint("Caught signal SIGINT, terminating...\n");
-	else
-		eprint("Caught signal %d, terminating...\n", sig);
-
-	remove_sock();
-	perf_close();
-	remove_pidfile();
-
-	exit(0);
+	perf_stop();
+	stopsig = sig;
 }
 
 
@@ -381,7 +436,7 @@ int main(int argc, char *argv[])
 			switch (opt) {
 			case 'h':
 				printf(usage, name);
-				exit(0);
+				return 0;
 			case 'f':
 				foreground = 1;
 				break;
@@ -389,40 +444,37 @@ int main(int argc, char *argv[])
 				printf("%s: Linux on System z CPACF Crypto Activity Counters Daemon\n"
 				       "Version %s\n%s\n",
 				       name, RELEASE_STRING, COPYRIGHT);
-				exit(0);
+				return 0;
 			default:
 				printf("%s: Invalid argument, try -h or --help for more information\n",
 					name);
-				exit(1);
+				return EXIT_FAILURE;
 			}
 		}
 	}
 
 	if (check_pidfile() != 0) {
 		eprint("Stalled pid file or daemon allready running, terminating\n");
-		exit(1);
+		return EXIT_FAILURE;
 	}
 
 	if (!foreground) {
 		if (become_daemon() != 0) {
 			eprint("Couldn't daemonize\n");
-			exit(1);
+			return EXIT_FAILURE;
 		}
 	}
 
 	if (perf_init() != 0) {
 		eprint("Couldn't initialize perf lib\n");
-		exit(1);
+		return EXIT_FAILURE;
 	}
 	atexit(perf_close);
 
-	if (!perf_ecc_supported())
-		ctr_state[ECC_FUNCTIONS] = UNSUPPORTED;
-
 	sfd = open_socket(SERVER);
 	if (sfd < 0) {
 		eprint("Couldn't initialize server socket\n");
-		exit(1);
+		return EXIT_FAILURE;
 	}
 	atexit(remove_sock);
 
@@ -432,17 +484,19 @@ int main(int argc, char *argv[])
 	if (sigaction(SIGINT, &act, 0) != 0) {
 		eprint("Couldn't establish signal handler for SIGINT, errno=%d [%s]\n",
 		       errno, strerror(errno));
-		exit(1);
+		return EXIT_FAILURE;
 	}
 	if (sigaction(SIGTERM, &act, 0) != 0) {
 		eprint("Couldn't establish signal handler for SIGTERM, errno=%d [%s]\n",
 		       errno, strerror(errno));
-		exit(1);
+		return EXIT_FAILURE;
 	}
+	/* Ignore SIGPIPE such that we see EPIPE as return from write. */
+	signal(SIGPIPE, SIG_IGN);
 
 	eprint("Running\n");
 
-	while (1) {
+	while (!stopsig) {
 		enum ctr_e ctr;
 		enum cmd_e cmd;
 		int s;
@@ -453,7 +507,7 @@ int main(int argc, char *argv[])
 				continue;
 			eprint("Accept() failure, errno=%d [%s]\n",
 			       errno, strerror(errno));
-			exit(1);
+			return EXIT_FAILURE;
 		}
 
 		rc = recv_query(s, &ctr, &cmd);
@@ -480,5 +534,13 @@ cleanup:
 		close(s);
 	}
 
+	if (stopsig == SIGTERM)
+		eprint("Caught signal SIGTERM, terminating...\n");
+	else if (stopsig == SIGINT)
+		eprint("Caught signal SIGINT, terminating...\n");
+	else
+		eprint("Caught signal %d, terminating...\n", stopsig);
+	remove_pidfile();
+
 	return 0;
 }

cpacfstats/perf_crypto.c

@@ -3,7 +3,7 @@
  *
  * low level perf functions
  *
- * Copyright IBM Corp. 2015, 2020
+ * Copyright IBM Corp. 2015, 2022
  *
  * s390-tools is free software; you can redistribute it and/or modify
  * it under the terms of the MIT license. See LICENSE for details.
@@ -16,14 +16,21 @@
 #include <inttypes.h>
 #include <limits.h>
 #include <linux/perf_event.h>
+#include <pthread.h>
+#include <poll.h>
+#include <signal.h>
 #include <stdint.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
 #include <sys/ioctl.h>
+#include <sys/resource.h>
 #include <unistd.h>
 
+#include <libudev.h>
+
 #include "cpacfstats.h"
+#include "../include/lib/zt_common.h"
 
 /* correlation between counter and perf counter string */
 static const struct {
@@ -38,25 +45,83 @@ static const struct {
 	{"cpum_cf", "ECC_FUNCTION_COUNT", ECC_FUNCTIONS}
 };
 
-/*
- * We need one filedescriptor per CPU per counter.
- * So perf_init builds this:
- *
- * ctr_fds - is an array of pointers to file descriptor arrays.
- * Each file descriptor array has space for number of logical CPUs + 1
- * filedescriptors (int values). The last element of each file descriptor
- * array is always 0, assuming there will never appear a filedescriptor
- * with value 0:
- *
- * ctr_fds:
- *   ctr_fds[0]             -> [file descriptor 0] [fd1] ... [fd cpus-1][0]
- *   ctr_fds[1]             -> [file descriptor 0] [fd1] ... [fd cpus-1][0]
- *   ...
- *   ctr_fds[ALL_COUNTER-1] -> [file descriptor 0] [fd1] ... [fd cpus-1][0]
- */
-static int *ctr_fds[ALL_COUNTER];
+static struct pmf_data {
+	int pmutype;
+	int eventid;
+} pmf_counter_data[ALL_COUNTER];
 
-static int ecc_supported;
+struct percpucounter {
+	int                   ctr_fds[ALL_COUNTER];
+	int                   pai_user[NUM_PAI_USER];
+	int                   pai_kernel[NUM_PAI_KERNEL];
+	unsigned int          cpunum;
+	struct percpucounter *next;
+};
+
+static struct percpucounter *root;
+pthread_mutex_t rootmux = PTHREAD_MUTEX_INITIALIZER;
+static volatile int hotplugdetected;
+static unsigned int enabledcounter;
+pthread_t hotplugthread;
+
+#define foreachcpu(PCPU) if (pthread_mutex_lock(&rootmux)) return -1; \
+	for ((PCPU) = root; (PCPU) != NULL; (PCPU) = (PCPU)->next)
+#define endforeachcpu() pthread_mutex_unlock(&rootmux)
+
+static int ctr_state[NUM_COUNTER];
+
+static volatile int stoprequested;
+
+static int paipmutype, paipmueventstart;
+
+static struct percpucounter *allocpercpucounter(unsigned int cpunum)
+{
+	struct percpucounter *ppc;
+
+	ppc = malloc(sizeof(struct percpucounter));
+	if (ppc) {
+		int i;
+
+		for (i = 0; i < ALL_COUNTER; ++i)
+			ppc->ctr_fds[i] = -1;
+		for (i = 0; i < NUM_PAI_USER; ++i)
+			ppc->pai_user[i] = -1;
+		for (i = 0; i < NUM_PAI_KERNEL; ++i)
+			ppc->pai_kernel[i] = -1;
+		ppc->cpunum = cpunum;
+		ppc->next = NULL;
+	}
+	return ppc;
+}
+
+static void freepercpucounter(struct percpucounter *pcpu)
+{
+	int i;
+
+	for (i = 0; i < ALL_COUNTER; ++i)
+		(void)close(pcpu->ctr_fds[i]);
+	for (i = 0; i < NUM_PAI_USER; ++i)
+		(void)close(pcpu->pai_user[i]);
+	for (i = 0; i < NUM_PAI_KERNEL; ++i)
+		(void)close(pcpu->pai_kernel[i]);
+	free(pcpu);
+}
+
+static struct percpucounter *findcpu(unsigned int cpunum, int unlinkflag)
+{
+	struct percpucounter **prev = &root, *walk = root;
+
+	while (walk) {
+		if (walk->cpunum == cpunum) {
+			if (unlinkflag)
+				*prev = walk->next;
+			return walk;
+		}
+		prev = &(walk->next);
+		walk = walk->next;
+	}
+	return NULL;
+}
 
 static long perf_event_open(struct perf_event_attr *hw_event, pid_t pid,
 			    int cpu, int group_fd, unsigned long flags)
@@ -85,12 +150,43 @@ static int perf_counter_supported(const char *pmu, const char *counter)
 	return !access(buf, R_OK);
 }
 
-static int perf_event_encode(struct perf_event_attr *attr,
+static int cpumf_authorized(void)
+{
+	unsigned vermin, vermax, auth;
+	int res = 0, found = 0;
+	size_t linesize = 0;
+	char *line = NULL;
+	FILE *f;
+
+	f = fopen("/proc/service_levels", "r");
+	if (f == NULL) {
+		eprint("Failed to open /proc/service_levels (%d:%s)\n",
+			errno, strerror(errno));
+		return 0;
+	}
+	while (getline(&line, &linesize, f) >= 0) {
+		if (sscanf(line,
+			   "CPU-MF: Counter facility: version=%d.%d authorization=%x",
+			   &vermin, &vermax, &auth) == 3) {
+			if (auth & 0x8)
+				res = 1;
+			else
+				eprint("CPU-MF counters not authorized.\n");
+			found = 1;
+			break;
+		}
+	}
+	if (!found)
+		eprint("CPU-MF counters not available.\n");
+	free(line);
+	fclose(f);
+	return res;
+}
+
+static int perf_event_encode(int *pmutype, int *eventid,
 			     const char *pmu, const char *event)
 {
 	FILE *f;
-	int eventid;
-	int pmutype;
 	char buf[PATH_MAX];
 
 	if (snprintf(buf, PATH_MAX, "/sys/bus/event_source/devices/%s/events/%s",
@@ -104,7 +200,7 @@ static int perf_event_encode(struct perf_event_attr *attr,
 		       errno, strerror(errno));
 		return -1;
 	}
-	if (fscanf(f, "event=0x%x\n", &eventid) != 1) {
+	if (fscanf(f, "event=0x%x\n", eventid) != 1) {
 		fclose(f);
 		eprint("Event file %s has invalid format\n", buf);
 		return -1;
@@ -121,21 +217,257 @@ static int perf_event_encode(struct perf_event_attr *attr,
 		       errno, strerror(errno));
 		return -1;
 	}
-	if (fscanf(f, "%d\n", &pmutype) != 1) {
+	if (fscanf(f, "%d\n", pmutype) != 1) {
 		fclose(f);
 		eprint("Type file %s has invalid format\n", buf);
 		return -1;
 	}
-	attr->type = pmutype;
-	attr->config = eventid;
 	return 0;
 }
 
+static int activatecpu(unsigned int cpu)
+{
+	struct perf_event_attr pfm_event;
+	struct percpucounter *ppc;
+	int fd, i, rc = 0;
+
+	ppc = allocpercpucounter(cpu);
+	if (ppc == NULL) {
+		eprint("Failed to allocate per cpu counter data");
+		return -1;
+	}
+	if (pthread_mutex_lock(&rootmux)) {
+		freepercpucounter(ppc);
+		return -1;
+	}
+	/* activate CPU-MF */
+	for (i = 0; i < ALL_COUNTER; ++i) {
+		if (ctr_state[i] == UNSUPPORTED)
+			continue;
+		memset(&pfm_event, 0, sizeof(pfm_event));
+		pfm_event.size = sizeof(pfm_event);
+		pfm_event.type = pmf_counter_data[i].pmutype;
+		pfm_event.config = pmf_counter_data[i].eventid;
+
+		/* fetch file descriptor for this perf event
+		 * the counter event should start disabled
+		 */
+		pfm_event.disabled = ctr_state[i] == DISABLED;
+		fd = perf_event_open(
+			&pfm_event,
+			-1,  /* pid -1 means all processes */
+			cpu,
+			-1,  /* group filedescriptor */
+			0);  /* flags */
+		if (fd < 0) {
+			eprint("Perf_event_open() failed with errno=%d [%s]\n",
+				errno, strerror(errno));
+			rc = -1;
+			ctr_state[i] = UNSUPPORTED;
+		} else {
+			ppc->ctr_fds[i] = fd;
+		}
+	}
+	/* activate pai_user and pai_kernel */
+	/* invariant:
+	   (ctr_state[PAI_USER] == UNSUPPORTED) ==
+	   (ctr_state[PAI_KERNEL] == UNSUPPORTED) */
+	if (ctr_state[PAI_USER] != UNSUPPORTED) {
+		for (i = 1; i <= NUM_PAI_USER; ++i) {
+			memset(&pfm_event, 0, sizeof(pfm_event));
+			pfm_event.size = sizeof(pfm_event);
+			pfm_event.type = paipmutype;
+			pfm_event.config = paipmueventstart + i;
+			pfm_event.exclude_kernel = 1;
+			pfm_event.exclude_user = 0;
+			pfm_event.disabled = ctr_state[PAI_USER] == DISABLED;
+			fd = perf_event_open(&pfm_event, -1, cpu, -1, 0);
+			if (fd < 0) {
+				eprint("Perf_event_open() failed with errno=%d [%s]\n",
+					errno, strerror(errno));
+				rc = -1;
+				ctr_state[PAI_USER] = UNSUPPORTED;
+				ctr_state[PAI_KERNEL] = UNSUPPORTED;
+				goto outevents;
+			} else {
+				ppc->pai_user[i - 1] = fd;
+			}
+			pfm_event.exclude_kernel = 0;
+			pfm_event.exclude_user = 1;
+			pfm_event.disabled = ctr_state[PAI_KERNEL] == DISABLED;
+			fd = perf_event_open(&pfm_event, -1, cpu, -1, 0);
+			if (fd < 0) {
+				eprint("Perf_event_open() failed with errno=%d [%s]\n",
+					errno, strerror(errno));
+				rc = -1;
+				ctr_state[PAI_USER] = UNSUPPORTED;
+				ctr_state[PAI_KERNEL] = UNSUPPORTED;
+				goto outevents;
+			} else {
+				ppc->pai_kernel[i - 1] = fd;
+			}
+		}
+		for (; i <= NUM_PAI_KERNEL; ++i) {
+			memset(&pfm_event, 0, sizeof(pfm_event));
+			pfm_event.size = sizeof(pfm_event);
+			pfm_event.type = paipmutype;
+			pfm_event.config = paipmueventstart + i;
+			pfm_event.exclude_kernel = 0;
+			pfm_event.exclude_user = 1;
+			pfm_event.disabled = ctr_state[PAI_KERNEL] == DISABLED;
+			fd = perf_event_open(&pfm_event, -1, cpu, -1, 0);
+			if (fd < 0) {
+				eprint("Perf_event_open() failed with errno=%d [%s]\n",
+					errno, strerror(errno));
+				rc = -1;
+				ctr_state[PAI_USER] = UNSUPPORTED;
+				ctr_state[PAI_KERNEL] = UNSUPPORTED;
+				goto outevents;
+			} else {
+				ppc->pai_kernel[i - 1] = fd;
+			}
+		}
+	}
+outevents:
+	ppc->next = root;
+	root = ppc;
+	if (enabledcounter)
+		hotplugdetected = 1;
+	pthread_mutex_unlock(&rootmux);
+	return rc;
+}
+
+static void deactivatecpu(unsigned int cpunum)
+{
+	struct percpucounter *pcpu;
+	int i;
+
+	if (pthread_mutex_lock(&rootmux))
+		return;
+	pcpu = findcpu(cpunum, 1);
+	if (pcpu != NULL) {
+		for (i = 0; i < ALL_COUNTER; ++i)
+			(void)close(pcpu->ctr_fds[i]);
+		for (i = 0; i < NUM_PAI_USER; ++i)
+			(void)close(pcpu->pai_user[i]);
+		for (i = 0; i < NUM_PAI_KERNEL; ++i)
+			(void)close(pcpu->pai_kernel[i]);
+		free(pcpu);
+		if (enabledcounter)
+			hotplugdetected = 1;
+	}
+	pthread_mutex_unlock(&rootmux);
+}
+
+static int addallcpus(void)
+{
+	unsigned int start, end;
+	int scanned, rc = 0;
+	FILE *fp;
+
+	/* comma separated list of intervals */
+	if ((fp = fopen("/sys/devices/system/cpu/online", "r")) == NULL) {
+		eprint("Failed to get online cpus (%d:%s)\n",
+			errno, strerror(errno));
+		return -1;
+	}
+		
+	while (!feof(fp)) {
+		/* scan all intervals of online cpus */
+		scanned = fscanf(fp, "%u-%u", &start, &end);
+		/* take care of singleton intervals */
+		if (scanned == 1)
+			end = start;
+		for (; start <= end; ++start) {
+			if (activatecpu(start)) {
+				rc = -1;
+				goto out;
+			}
+		}
+		/* Skip comma separator */
+		(void)fgetc(fp);
+	}
+out:
+	fclose(fp);
+	return rc;
+}
+
+static int perf_load_counter_data(void)
+{
+	int i, res = 0;
+
+	for (i = 0; i < ALL_COUNTER; ++i) {
+		if (ctr_state[i] != UNSUPPORTED)
+			res |= perf_event_encode(&pmf_counter_data[i].pmutype,
+						 &pmf_counter_data[i].eventid,
+						 pmf_counter_name[i].pmu,
+						 pmf_counter_name[i].pfm_name);
+	}
+	if (ctr_state[PAI_USER] != UNSUPPORTED)
+		res |= perf_event_encode(&paipmutype, &paipmueventstart,
+					 "pai_crypto", "CRYPTO_ALL");
+	return res;
+}
+
+static void *hotplughandler(void *UNUSED(unused))
+{
+	struct udev *hotplug;
+	struct udev_monitor *monitor;
+	struct pollfd item;
+	
+	hotplug = udev_new();
+	if (!hotplug) {
+		eprint("Failed to create hotplug device\n");
+		return NULL;
+	}
+	monitor = udev_monitor_new_from_netlink(hotplug, "udev");
+	udev_monitor_filter_add_match_subsystem_devtype(monitor, "cpu", NULL);
+	udev_monitor_enable_receiving(monitor);
+	item.fd = udev_monitor_get_fd(monitor);
+	item.events = POLLIN;
+	item.revents = 0;
+	while (!stoprequested) {
+		struct udev_device *dev;
+		const char *path, *action;
+		unsigned int cpunum;
+		int rc, on, off;
+
+		errno = 0;
+		rc = poll(&item, 1, -1);
+		if (rc == -1) {
+			if (errno == EINTR)
+				continue;
+			break;
+		}
+		dev = udev_monitor_receive_device(monitor);
+		if (dev == NULL)
+			continue;
+		action = udev_device_get_action(dev);
+		if (action == NULL)
+			continue;
+		off = strcmp(action, "offline") == 0;
+		on = strcmp(action, "online") == 0;
+		if (!on && !off)
+			continue;
+		path = udev_device_get_devpath(dev);
+		if (sscanf(path, "/devices/system/cpu/cpu%u", &cpunum) != 1)
+			continue;
+		if (on && activatecpu(cpunum))
+			eprint("Failed to attach to hotplugged CPU %u\n", cpunum);
+		if (off)
+			deactivatecpu(cpunum);
+	}
+	udev_monitor_unref(monitor);
+	udev_unref(hotplug);
+	return NULL;
+}
+
 int perf_init(void)
 {
-	int i, cpus, ctr, cpu, *fds;
-
-	memset(ctr_fds, 0, sizeof(ctr_fds));
+	int ecc_supported, i, num;
+	unsigned long maxfd;
+	struct rlimit rlim;
+	FILE *f;
 
 	/*  initialize performance monitoring library */
 	if (!perf_supported()) {
@@ -143,99 +475,105 @@ int perf_init(void)
 		return -1;
 	}
 
+	/* We currently support all cpumf counters plus two virtual
+	 * counters for PAI. */
+	num = ALL_COUNTER + 2;
 	/* Check if ECC is supported on current hardware */
 	ecc_supported = perf_counter_supported("cpum_cf", "ECC_FUNCTION_COUNT");
 
-	/* get number of logical processors */
-	cpus = sysconf(_SC_NPROCESSORS_ONLN);
-
-	/* for each counter */
-	for (ctr = 0; ctr < ALL_COUNTER; ctr++) {
-
-		/* Skip ECC counters completely if unsupported */
-		if (ctr == ECC_FUNCTIONS && !ecc_supported)
-			continue;
-
-		/*
-		 * allocate an array of ints to store for each CPU
-		 * one filedescriptor + a terminating 0
-		 */
-		fds = (int *) calloc(sizeof(int), cpus+1);
-		if (!fds) {
-			eprint("Malloc() of %d byte failed, errno=%d [%s]\n",
-			       (int)(sizeof(int) * (cpus+1)),
-			       errno, strerror(errno));
-			return -1;
-		}
-
-		ctr_fds[ctr] = fds;
-
-		/* search for the counter's corresponding pfm name */
-		for (i = ALL_COUNTER-1; i >= 0; i--)
-			if ((int) pmf_counter_name[i].ctr == ctr)
-				break;
-		if (i < 0) {
-			eprint("Pfm ctr name not found for counter %d, please adjust pmf_counter_name[] in %s\n",
-			       ctr, __FILE__);
-			return -1;
-		}
-
-		for (cpu = 0; cpu < cpus; cpu++) {
-			struct perf_event_attr pfm_event;
-			int fd;
-
-			memset(&pfm_event, 0, sizeof(pfm_event));
-			pfm_event.size = sizeof(pfm_event);
-			if (perf_event_encode(&pfm_event,
-					      pmf_counter_name[i].pmu,
-					      pmf_counter_name[i].pfm_name)) {
-				eprint("Failed to initialize counter %s for pmu %s\n",
-				       pmf_counter_name[i].pfm_name,
-				       pmf_counter_name[i].pmu);
-				return -1;
-			}
-
-			/* fetch file descriptor for this perf event
-			 * the counter event should start disabled
-			 */
-			pfm_event.disabled = 1;
-			fd = perf_event_open(
-				&pfm_event,
-				-1,  /* pid -1 means all processes */
-				cpu,
-				-1,  /* group filedescriptor */
-				0);  /* flags */
-			if (fd < 0) {
-				eprint("Perf_event_open() failed with errno=%d [%s]\n",
-				       errno, strerror(errno));
-				return -1;
-			}
-			fds[cpu] = fd;
-		}
+	if (!cpumf_authorized()) {
+		for (i = 0; i < ALL_COUNTER; ++i)
+			ctr_state[i] = UNSUPPORTED;
+		num -= ALL_COUNTER;
+	} else if (!ecc_supported) {
+		ctr_state[ECC_FUNCTIONS] = UNSUPPORTED;
+		--num;
 	}
 
-	return 0;
+	if (!perf_counter_supported("pai_crypto", "CRYPTO_ALL")) {
+		ctr_state[PAI_USER] = UNSUPPORTED;
+		ctr_state[PAI_KERNEL] = UNSUPPORTED;
+		num -= 2;
+	}
+
+	if (num == 0) {
+		eprint("No crypto counters supported!\n");
+		return -1;
+	}
+
+	if (perf_load_counter_data())
+		return -1;
+	/* We have to adjust the number of FDs possible since we might
+	 * need more than 1024 (the typical soft limit) */
+	f = fopen("/proc/sys/fs/nr_open", "r");
+	if (f == NULL) {
+		eprint("fopen failed for /proc/sys/fs/nr_open with errno=%d [%s]\n",
+			errno, strerror(errno));
+		return -1;
+	}
+	if (fscanf(f, "%lu", &maxfd) != 1) {
+		fclose(f);
+		eprint("Failed to parse /proc/sys/fs/nr_open\n");
+		return -1;
+	}
+	fclose(f);
+	rlim.rlim_cur = maxfd;
+	rlim.rlim_max = maxfd;
+	if (setrlimit(RLIMIT_NOFILE, &rlim) == -1) {
+		eprint("setrlimit failed with errno=%d [%s]\n",
+			errno, strerror(errno));
+		return -1;
+	}
+
+	if (pthread_create(&hotplugthread, NULL, hotplughandler, NULL)) {
+		eprint("Failed to start hotplug handler thread\n");
+		return -1;
+	}
+	return addallcpus();
+}
+
+
+void perf_stop(void)
+{
+	stoprequested = 1;
 }
 
 
 void perf_close(void)
 {
-	int ctr, *fds;
+	struct percpucounter *walk, *next;
 
-	for (ctr = 0; ctr < ALL_COUNTER; ctr++) {
-		for (fds = ctr_fds[ctr]; fds && *fds; fds++) {
-			close(*fds);
-			*fds = 0;
-		}
-		free(ctr_fds[ctr]);
-		ctr_fds[ctr] = NULL;
+	pthread_kill(hotplugthread, SIGINT);
+	pthread_join(hotplugthread, NULL);
+	walk = root;
+	while (walk) {
+		next = walk->next;
+		freepercpucounter(walk);
+		walk = next;
 	}
 }
 
 
+static int enable_array(int *arr, int size)
+{
+	int i, ec, rc = 0;
+
+	for (i = 0; i < size; ++i) {
+		ec = ioctl(arr[i], PERF_EVENT_IOC_ENABLE, 0);
+		if (ec < 0) {
+			eprint("Ioctl(PERF_EVENT_IOC_ENABLE) failed with errno=%d [%s]\n",
+				errno, strerror(errno));
+			rc = -1;
+		}
+	}
+	return rc;
+}
+
+
 int perf_enable_ctr(enum ctr_e ctr)
 {
-	int *fds, ec, rc = 0;
+	struct percpucounter *pcpu;
+	int ec, rc = 0;
 
 	if (ctr == ALL_COUNTER) {
 		for (ctr = 0; ctr < ALL_COUNTER; ctr++) {
@@ -243,24 +581,64 @@ int perf_enable_ctr(enum ctr_e ctr)
 			if (rc != 0)
 				return rc;
 		}
-	} else {
-		for (fds = ctr_fds[ctr]; fds && *fds; fds++) {
-			ec = ioctl(*fds, PERF_EVENT_IOC_ENABLE, 0);
+	} else if (ctr < ALL_COUNTER) {
+		foreachcpu(pcpu) {
+			ec = ioctl(pcpu->ctr_fds[ctr], PERF_EVENT_IOC_ENABLE, 0);
 			if (ec < 0) {
 				eprint("Ioctl(PERF_EVENT_IOC_ENABLE) failed with errno=%d [%s]\n",
 				       errno, strerror(errno));
 				rc = -1;
 			}
 		}
+		ctr_state[ctr] = ENABLED;
+		++enabledcounter;
+		endforeachcpu();
+	} else if (ctr == PAI_USER) {
+		foreachcpu(pcpu) {
+			ec = enable_array(pcpu->pai_user, NUM_PAI_USER);
+			if (ec < 0)
+				rc = -1;
+		}
+		ctr_state[ctr] = ENABLED;
+		++enabledcounter;
+		endforeachcpu();
+	} else if (ctr == PAI_KERNEL) {
+		foreachcpu(pcpu) {
+			ec = enable_array(pcpu->pai_kernel, NUM_PAI_KERNEL);
+			if (ec < 0)
+				rc = -1;
+		}
+		ctr_state[ctr] = ENABLED;
+		++enabledcounter;
+		endforeachcpu();
+	} else {
+		rc = -1;
 	}
 
 	return rc;
 }
 
 
+static int disable_array(int *arr, int size)
+{
+	int i, ec, rc = 0;
+
+	for (i = 0; i < size; ++i) {
+		ec = ioctl(arr[i], PERF_EVENT_IOC_DISABLE, 0);
+		if (ec < 0) {
+			eprint("Ioctl(PERF_EVENT_IOC_DISABLE) failed with errno=%d [%s]\n",
+				errno, strerror(errno));
+			rc = -1;
+		}
+	}
+	return rc;
+}
+
+
 int perf_disable_ctr(enum ctr_e ctr)
 {
-	int *fds, ec, rc = 0;
+	struct percpucounter *pcpu;
+	int ec, rc = 0;
 
 	if (ctr == ALL_COUNTER) {
 		for (ctr = 0; ctr < ALL_COUNTER; ctr++) {
@@ -268,70 +646,183 @@ int perf_disable_ctr(enum ctr_e ctr)
 			if (rc != 0)
 				return rc;
 		}
-	} else {
-		for (fds = ctr_fds[ctr]; fds && *fds; fds++) {
-			ec = ioctl(*fds, PERF_EVENT_IOC_DISABLE, 0);
+	} else if (ctr < ALL_COUNTER) {
+		foreachcpu(pcpu) {
+			ec = ioctl(pcpu->ctr_fds[ctr], PERF_EVENT_IOC_DISABLE, 0);
 			if (ec < 0) {
 				eprint("Ioctl(PERF_EVENT_IOC_DISABLE) failed with errno=%d [%s]\n",
 				       errno, strerror(errno));
 				rc = -1;
 			}
 		}
+		ctr_state[ctr] = DISABLED;
+		--enabledcounter;
+		if (enabledcounter == 0)
+			hotplugdetected = 0;
+		endforeachcpu();
+	} else if (ctr == PAI_USER) {
+		foreachcpu(pcpu) {
+			ec = disable_array(pcpu->pai_user, NUM_PAI_USER);
+			if (ec < 0)
+				rc = -1;
+		}
+		ctr_state[ctr] = DISABLED;
+		--enabledcounter;
+		if (enabledcounter == 0)
+			hotplugdetected = 0;
+		endforeachcpu();
+	} else if (ctr == PAI_KERNEL) {
+		foreachcpu(pcpu) {
+			ec = disable_array(pcpu->pai_kernel, NUM_PAI_KERNEL);
+			if (ec < 0)
+				rc = -1;
+		}
+		ctr_state[ctr] = DISABLED;
+		--enabledcounter;
+		if (enabledcounter == 0)
+			hotplugdetected = 0;
+		endforeachcpu();
+	} else {
+		rc = -1;
 	}
 
 	return rc;
 }
 
 
-int perf_reset_ctr(enum ctr_e ctr)
+static int reset_array(int *arr, int size)
 {
-	int *fds, ec, rc = 0;
+	int ec, rc = 0, i;
+
+	for (i = 0; i < size; ++i) {
+		ec = ioctl(arr[i], PERF_EVENT_IOC_RESET, 0);
+		if (ec < 0) {
+			eprint("Ioctl(PERF_EVENT_IOC_RESET) failed with errno=%d [%s]\n",
+				errno, strerror(errno));
+			rc = -1;
+		}
+	}
+	return rc;
+}
+
+
+int perf_reset_ctr(enum ctr_e ctr, uint64_t *value)
+{
+	struct percpucounter *pcpu;
+	int ec, rc = 0;
 
 	if (ctr == ALL_COUNTER) {
 		for (ctr = 0; ctr < ALL_COUNTER; ctr++) {
-			rc = perf_reset_ctr(ctr);
+			rc = perf_reset_ctr(ctr, value);
 			if (rc != 0)
 				return rc;
 		}
-	} else {
-		for (fds = ctr_fds[ctr]; fds && *fds; fds++) {
-			ec = ioctl(*fds, PERF_EVENT_IOC_RESET, 0);
+	} else if (ctr < ALL_COUNTER) {
+		foreachcpu(pcpu) {
+			ec = ioctl(pcpu->ctr_fds[ctr], PERF_EVENT_IOC_RESET, 0);
 			if (ec < 0) {
 				eprint("Ioctl(PERF_EVENT_IOC_RESET) failed with errno=%d [%s]\n",
 				       errno, strerror(errno));
 				rc = -1;
 			}
 		}
+		endforeachcpu();
+	} else if (ctr == PAI_USER) {
+		foreachcpu(pcpu) {
+			ec = reset_array(pcpu->pai_user, NUM_PAI_USER);
+			if (ec < 0)
+				rc = -1;
+		}
+		endforeachcpu();
+	} else if (ctr == PAI_KERNEL) {
+		foreachcpu(pcpu) {
+			ec = reset_array(pcpu->pai_kernel, NUM_PAI_KERNEL);
+			if (ec < 0)
+				rc = -1;
+		}
+		endforeachcpu();
+	} else {
+		rc = -1;
 	}
-
+	if (rc == 0)
+		rc = perf_read_ctr(ctr, value);
 	return rc;
 }
 
 
 int perf_read_ctr(enum ctr_e ctr, uint64_t *value)
 {
-	int *fds, ec, rc = -1;
+	struct percpucounter *pcpu;
+	int ec, rc = 0;
 	uint64_t val;
 
 	if (!value)
 		return -1;
+	if (ctr == HOTPLUG_DETECTED) {
+		*value = hotplugdetected;
+		return 0;
+	}
+	if (ctr == PAI_USER) {
+		*value = NUM_PAI_USER;
+		return 0;
+	}
+	if (ctr == PAI_KERNEL) {
+		*value = NUM_PAI_KERNEL;
+		return 0;
+	}
+	if (ctr >= ALL_COUNTER)
+		return -1;
 	*value = 0;
 
-	for (fds = ctr_fds[ctr]; fds && *fds; fds++) {
-		ec = read(*fds, &val, sizeof(val));
+	foreachcpu(pcpu) {
+		ec = read(pcpu->ctr_fds[ctr], &val, sizeof(val));
 		if (ec != sizeof(val)) {
 			eprint("Read() on perf file descriptor failed with errno=%d [%s]\n",
 			       errno, strerror(errno));
+			rc = -1;
 		} else {
 			*value += val;
-			rc = 0;
 		}
 	}
+	endforeachcpu();
 
 	return rc;
 }
 
 int  perf_ecc_supported(void)
 {
-	return ecc_supported;
+	return ctr_state[ECC_FUNCTIONS] != UNSUPPORTED;
+}
+
+int perf_ctr_state(enum ctr_e ctr) {
+	if (ctr < NUM_COUNTER)
+		return ctr_state[ctr];
+	return UNSUPPORTED;
+}
+
+
+int perf_read_pai_ctr(unsigned int ctrnum, int user, uint64_t *value)
+{
+	struct percpucounter *pcpu;
+	unsigned int maxctr;
+	int *arr, ec, rc = 0;
+	uint64_t val;
+
+	*value = 0;
+	maxctr = user ? NUM_PAI_USER : NUM_PAI_KERNEL;
+	if (ctrnum >= maxctr)
+		return -1;
+	foreachcpu(pcpu) {
+		arr = user ? pcpu->pai_user : pcpu->pai_kernel;
+		ec = read(arr[ctrnum], &val, sizeof(val));
+		if (ec != sizeof(val)) {
+			eprint("Read() on perf file descriptor failed with errno=%d [%s]\n",
+			       errno, strerror(errno));
+			rc = -1;
+		} else {
+			*value += val;
+		}
+	}
+	endforeachcpu();
+	return rc;
 }

cpacfstats/stats_sock.c

@@ -3,7 +3,7 @@
  *
  * basic socket and receive/send functions
  *
- * Copyright IBM Corp. 2015, 2017
+ * Copyright IBM Corp. 2015, 2022
  *
  * s390-tools is free software; you can redistribute it and/or modify
  * it under the terms of the MIT license. See LICENSE for details.
@@ -13,6 +13,7 @@
 #include <endian.h>
 #include <errno.h>
 #include <grp.h>
+#include <poll.h>
 #include <stdint.h>
 #include <stdio.h>
 #include <sys/socket.h>
@@ -148,7 +149,51 @@ static int __read(int fd, void *buf, int buflen)
 }
 
 
-int send_msg(int sfd, struct msg *m)
+static int __timedwrite(int fd, const void *buf, int buflen, int timeout)
+{
+	struct pollfd pfd = { .fd = fd, .events = POLLOUT };
+	int i = 0, n;
+
+	while (poll(&pfd, 1, timeout) == 1) {
+		n = write(fd, buf + i, buflen - i);
+		if (n < 0) {
+			if (errno == EINTR)
+				continue;
+			else
+				return n;
+		} else if (n == 0) {
+			return i;
+		}
+		i += n;
+		if (buflen == i)
+			return i;
+	}
+	return -1;
+}
+
+static int __timedread(int fd, void *buf, int buflen, int timeout)
+{
+	struct pollfd pfd = { .fd = fd, .events = POLLIN };
+	int i = 0, n;
+
+	while (poll(&pfd, 1, timeout) == 1) {
+		n = read(fd, buf + i, buflen - i);
+		if (n < 0) {
+			if (errno == EINTR)
+				continue;
+			else
+				return n;
+		} else if (n == 0) {
+			return i;
+		}
+		i += n;
+		if (buflen == i)
+			return i;
+	}
+	return -1;
+}
+
+int send_msg(int sfd, struct msg *m, int timeout)
 {
 	int n, len;
 
@@ -166,7 +211,7 @@ int send_msg(int sfd, struct msg *m)
 		return -1;
 	}
 
-	n = __write(sfd, m, len);
+	n = timeout ? __timedwrite(sfd, m, len, timeout) : __write(sfd, m, len);
 	if (n != len) {
 		eprint("Write() error: write()=%d expected %d, errno=%d [%s]\n",
 		       n, len, errno, strerror(errno));
@@ -176,12 +221,12 @@ int send_msg(int sfd, struct msg *m)
 	return 0;
 }
 
-int recv_msg(int sfd, struct msg *m)
+int recv_msg(int sfd, struct msg *m, int timeout)
 {
 	int n, len;
 
 	len = sizeof(m->head);
-	n = __read(sfd, m, len);
+	n = timeout ? __timedread(sfd, m, len, timeout) : __read(sfd, m, len);
 	if (n != len) {
 		eprint("Recv() error: read()=%d expected %d, errno=%d [%s]\n",
 		       n, len, errno, strerror(errno));
@@ -200,7 +245,8 @@ int recv_msg(int sfd, struct msg *m)
 		return -1;
 	}
 
-	n = __read(sfd, ((char *)m) + sizeof(m->head), len);
+	n = timeout ? __timedread(sfd, ((char *)m) + sizeof(m->head), len, timeout) :
+		__read(sfd, ((char *)m) + sizeof(m->head), len);
 	if (n != len) {
 		eprint("Recv() error: recv()=%d expected %d, errno=%d [%s]\n",
 		       n, len, errno, strerror(errno));

cpumf/Makefile

@@ -1,15 +1,16 @@
 include ../common.mak
 
-BIN_FILES = lscpumf chcpumf lshwc
-MAN_FILES = lscpumf.1 chcpumf.8 lshwc.1
+BIN_FILES = lscpumf chcpumf lshwc pai
+MAN_FILES = lscpumf.8 chcpumf.8 lshwc.8 pai.8
 
 all: $(BIN_FILES)
 
-libs =	$(rootdir)/libutil/libutil.a
+libs =	$(rootdir)/libcpumf/libcpumf.a $(rootdir)/libutil/libutil.a
 
 lscpumf: lscpumf.o $(libs)
 chcpumf: chcpumf.o $(libs)
 lshwc: lshwc.o $(libs)
+pai: pai.o $(libs)
 
 install: all install-man
 	$(INSTALL) -d -m 755 $(DESTDIR)$(BINDIR) $(DESTDIR)$(MANDIR)/man8
@@ -22,8 +23,8 @@ clean:
 
 install-man:
 	for man in $(MAN_FILES); do \
-	  msection=`echo $$man |sed 's/.*\.\([1-9]\)$$/man\1/'` ; \
-	  $(INSTALL) -g $(GROUP) -o $(OWNER) -m 644 -D man/$$man $(DESTDIR)$(MANDIR)/$$msection/$$man ; \
+		$(INSTALL) -g $(GROUP) -o $(OWNER) -m 644 man/$$man \
+		$(DESTDIR)$(MANDIR)/man8 ; \
 	done
 
 .PHONY: all install clean

cpumf/chcpumf.c

@@ -1,7 +1,7 @@
 /*
  * chcpumf -  Change CPU Measurement Facility Characteristics
  *
- * Copyright IBM Corp. 2020
+ * Copyright IBM Corp. 2020, 2022
  *
  * s390-tools is free software; you can redistribute it and/or modify
  * it under the terms of the MIT license. See LICENSE for details.
@@ -20,9 +20,9 @@
 #include "lib/util_prg.h"
 #include "lib/util_base.h"
 
-#include "defines.h"
+#include "lib/libcpumf.h"
 
-static int verbose;
+static unsigned int verbose;
 static unsigned long min_sdb, max_sdb;
 
 static struct util_opt opt_vec[] = {
@@ -85,55 +85,30 @@ static long parse_buffersize(char *string)
 	return bytes;
 }
 
-static void read_sfb(unsigned long *min, unsigned long *max)
-{
-	unsigned long cur_min_sdb, cur_max_sdb;
-	FILE *fp;
-
-	if (geteuid())
-		errx(EXIT_FAILURE, "Must run as root");
-	fp = fopen(PERF_SFB_SIZE, "r");
-	if (!fp)
-		err(EXIT_FAILURE, PERF_SFB_SIZE);
-	if (fscanf(fp, "%ld,%ld", &cur_min_sdb, &cur_max_sdb) != 2) {
-		fclose(fp);
-		errx(EXIT_FAILURE, "Can not parse file " PERF_SFB_SIZE);
-	} else {
-		if (*min == 0)
-			*min = cur_min_sdb;
-		if (*max == 0)
-			*max = cur_max_sdb;
-	}
-	fclose(fp);
-	if (*min >= *max)
-		errx(EXIT_FAILURE,
-		     "The specified maximum must be greater than the minimum");
-}
-
-static int write_sfb(unsigned long min, unsigned long max)
+static int write_sfb(unsigned int min, unsigned int max)
 {
 	int rc = EXIT_SUCCESS;
 	char text[64];
 	size_t len;
 	FILE *fp;
 
-	fp = fopen(PERF_SFB_SIZE, "w");
+	fp = fopen(S390_CPUMSF_BUFFERSZ, "w");
 	if (!fp)
-		err(EXIT_FAILURE, PERF_SFB_SIZE);
-	snprintf(text, sizeof text, "%ld,%ld", min, max);
+		err(EXIT_FAILURE, S390_CPUMSF_BUFFERSZ);
+	snprintf(text, sizeof(text), "%u,%u", min, max);
 	len = strlen(text) + 1;
 	if (fwrite(text, 1, len, fp) != len) {
-		warn(PERF_SFB_SIZE);
+		warn(S390_CPUMSF_BUFFERSZ);
 		rc = EXIT_FAILURE;
 	}
 	if (fclose(fp)) {
-		warn(PERF_SFB_SIZE);
+		warn(S390_CPUMSF_BUFFERSZ);
 		rc = EXIT_FAILURE;
 	}
 	if (verbose && rc != EXIT_FAILURE)
 		warnx("Sampling buffer sizes:\n"
-		      "    Minimum:%7ld sample-data-blocks\n"
-		      "    Maximum:%7ld sample-data-blocks\n",
+		      "    Minimum:%7d sample-data-blocks\n"
+		      "    Maximum:%7d sample-data-blocks\n",
 		      min, max);
 	return rc;
 }
@@ -183,15 +158,21 @@ static int parse_args(int argc, char **argv)
 
 int main(int argc, char **argv)
 {
-	struct stat sbuf;
+	unsigned long my_min, my_max;
 
 	util_prg_init(&prg);
 	util_opt_init(opt_vec, NULL);
 
 	parse_args(argc, argv);
-	if (stat(PERF_PATH PERF_SF, &sbuf))
+	if (geteuid())
+		errx(EXIT_FAILURE, "Must run as root");
+	if (!libcpumf_have_sfb())
 		errx(EXIT_FAILURE,
 		     "No CPU-measurement sampling facility detected");
-	read_sfb(&min_sdb, &max_sdb);
+	libcpumf_sfb_info(&my_min, &my_max);
+	if (!min_sdb)
+		min_sdb = my_min;
+	if (!max_sdb)
+		max_sdb = my_max;
 	return write_sfb(min_sdb, max_sdb);
 }

cpumf/defines.h

@@ -1,18 +0,0 @@
-/*
- * Defines for CPU Measurement Facility Characteristics
- *
- * Copyright IBM Corp. 2020
- *
- * s390-tools is free software; you can redistribute it and/or modify
- * it under the terms of the MIT license. See LICENSE for details.
- */
-
-#ifndef DEFINES_H
-#define	DEFINES_H
-
-#define	PERF_SFB_SIZE	"/sys/module/kernel/parameters/cpum_sfb_size"
-#define	PERF_PATH	"/sys/bus/event_source/devices/"
-#define	PERF_SF		"cpum_sf"
-#define	PERF_CF		"cpum_cf"
-
-#endif

cpumf/lscpumf.c

@@ -1,7 +1,7 @@
 /*
  * lscpumf -  Show CPU Measurement Facility Characteristics
  *
- * Copyright IBM Corp. 2020
+ * Copyright IBM Corp. 2020, 2022
  *
  * s390-tools is free software; you can redistribute it and/or modify
  * it under the terms of the MIT license. See LICENSE for details.
@@ -26,9 +26,10 @@
 #include "lib/util_arch.h"
 #include "lib/util_base.h"
 #include "lib/util_opt.h"
+#include "lib/util_path.h"
 #include "lib/util_prg.h"
 
-#include "defines.h"
+#include "lib/libcpumf.h"
 
 #define	ACTION_NONE	0
 #define	ACTION_INFO	1
@@ -45,9 +46,6 @@ static bool actions[ACTION_SAMPLE + 1];	/* Specified command line options */
 #define	PER_SDBT_SIZE	511
 
 /* File names to read data from */
-#define	SERVICELEVEL	"/proc/service_levels"
-#define	CPUMF_CF_TYPE	"/sys/devices/cpum_cf/type"
-#define	CPUMF_SF_TYPE	"/sys/devices/cpum_sf/type"
 
 static struct util_opt opt_vec[] = {
 	UTIL_OPT_SECTION("OPTIONS"),
@@ -92,19 +90,18 @@ static char prefix[32];			/* Counter prefix */
 static bool show_names;
 
 static struct cpumf_info {
-	unsigned int first_vn;		/* Counter facility first version nr */
-	unsigned int second_vn;		/* Counter facility second version nr */
-	unsigned int authorization;	/* Counter facility  authorization */
-	float version;
+	int first_vn;			/* Counter facility first version nr */
+	int second_vn;			/* Counter facility second version nr */
+	int authorization;		/* Counter facility  authorization */
 	unsigned long min_rate;		/* Minimum sampling rate */
 	unsigned long max_rate;		/* Maximum sampling rate */
 	unsigned long cpu_speed;	/* CPU Cycles per micro second */
-	unsigned int basic_sample_sz;	/* # of Bytes per basic sample */
-	unsigned int diag_sample_sz;	/* # of bytes per diagnostic sample */
-	unsigned char have_counter;	/* CPUM counter facility detected */
-	unsigned char have_samples;	/* CPUM sampling facility detected */
-	unsigned int min_sfb;		/* Minimum sampling buffer size */
-	unsigned int max_sfb;		/* Maximum sampling buffer size */
+	int basic_sample_sz;		/* # of Bytes per basic sample */
+	int diag_sample_sz;		/* # of bytes per diagnostic sample */
+	bool have_counter;		/* CPUM counter facility detected */
+	bool have_samples;		/* CPUM sampling facility detected */
+	unsigned long min_sfb;		/* Minimum sampling buffer size */
+	unsigned long max_sfb;		/* Maximum sampling buffer size */
 	unsigned short machine_type;	/* Machine Type */
 } cpumf;
 
@@ -2559,32 +2556,598 @@ static struct counters cpumcf_z15_counters[] = {
 	},
 };
 
+static struct counters cpumcf_z16_counters[] = {
+	{
+		.ctrnum = 128,
+		.ctrset = CPUMF_CTRSET_EXTENDED,
+		.name = "L1D_RO_EXCL_WRITES",
+		.desc = "A directory write to the Level-1 Data cache where"
+			"\n\t\tthe line was originally in a Read-Only state in the"
+			"\n\t\tcache but has been updated to be in the Exclusive"
+			"\n\t\tstate that allows stores to the cache line.",
+	},
+	{
+		.ctrnum = 129,
+		.ctrset = CPUMF_CTRSET_EXTENDED,
+		.name = "DTLB2_WRITES",
+		.desc = "A translation has been written into The Translation"
+			"\n\t\tLookaside Buffer 2 (TLB2) and the request was made"
+			"\n\t\tby the Level-1 Data cache. This is a replacement"
+			"\n\t\tfor what was provided for the DTLB on z13 and prior"
+			"\n\t\tmachines.",
+	},
+	{
+		.ctrnum = 130,
+		.ctrset = CPUMF_CTRSET_EXTENDED,
+		.name = "DTLB2_MISSES",
+		.desc = "A TLB2 miss is in progress for a request made by"
+			"\n\t\tthe Level-1 Data cache. Incremented by one for"
+			"\n\t\tevery TLB2 miss in progress for the Level-1 Data"
+			"\n\t\tcache on this cycle. This is a replacement for what"
+			"\n\t\twas provided for the DTLB on z13 and prior"
+			"\n\t\tmachines.",
+	},
+	{
+		.ctrnum = 131,
+		.ctrset = CPUMF_CTRSET_EXTENDED,
+		.name = "CRSTE_1MB_WRITES",
+		.desc = "A translation entry was written into the Combined"
+			"\n\t\tRegion and Segment Table Entry array in the Level-2"
+			"\n\t\tTLB for a one-megabyte page.",
+	},
+	{
+		.ctrnum = 132,
+		.ctrset = CPUMF_CTRSET_EXTENDED,
+		.name = "DTLB2_GPAGE_WRITES",
+		.desc = "A translation entry for a two-gigabyte page was"
+			"\n\t\twritten into the Level-2 TLB.",
+	},
+	{
+		.ctrnum = 134,
+		.ctrset = CPUMF_CTRSET_EXTENDED,
+		.name = "ITLB2_WRITES",
+		.desc = "A translation entry has been written into the"
+			"\n\t\tTranslation Lookaside Buffer 2 (TLB2) and the"
+			"\n\t\trequest was made by the instruction cache. This is"
+			"\n\t\ta replacement for what was provided for the ITLB on"
+			"\n\t\tz13 and prior machines.",
+	},
+	{
+		.ctrnum = 135,
+		.ctrset = CPUMF_CTRSET_EXTENDED,
+		.name = "ITLB2_MISSES",
+		.desc = "A TLB2 miss is in progress for a request made by"
+			"\n\t\tthe Level-1 Instruction cache. Incremented by one"
+			"\n\t\tfor every TLB2 miss in progress for the Level-1"
+			"\n\t\tInstruction cache in a cycle. This is a replacement"
+			"\n\t\tfor what was provided for the ITLB on z13 and prior"
+			"\n\t\tmachines.",
+	},
+	{
+		.ctrnum = 137,
+		.ctrset = CPUMF_CTRSET_EXTENDED,
+		.name = "TLB2_PTE_WRITES",
+		.desc = "A translation entry was written into the Page Table"
+			"\n\t\tEntry array in the Level-2 TLB.",
+	},
+	{
+		.ctrnum = 138,
+		.ctrset = CPUMF_CTRSET_EXTENDED,
+		.name = "TLB2_CRSTE_WRITES",
+		.desc = "Translation entries were written into the Combined"
+			"\n\t\tRegion and Segment Table Entry array and the Page"
+			"\n\t\tTable Entry array in the Level-2 TLB.",
+	},
+	{
+		.ctrnum = 139,
+		.ctrset = CPUMF_CTRSET_EXTENDED,
+		.name = "TLB2_ENGINES_BUSY",
+		.desc = "The number of Level-2 TLB translation engines busy"
+			"\n\t\tin a cycle.",
+	},
+	{
+		.ctrnum = 140,
+		.ctrset = CPUMF_CTRSET_EXTENDED,
+		.name = "TX_C_TEND",
+		.desc = "A TEND instruction has completed in a constrained"
+			"\n\t\ttransactional-execution mode.",
+	},
+	{
+		.ctrnum = 141,
+		.ctrset = CPUMF_CTRSET_EXTENDED,
+		.name = "TX_NC_TEND",
+		.desc = "A TEND instruction has completed in a non-"
+			"\n\t\tconstrained transactional-execution mode.",
+	},
+	{
+		.ctrnum = 143,
+		.ctrset = CPUMF_CTRSET_EXTENDED,
+		.name = "L1C_TLB2_MISSES",
+		.desc = "Increments by one for any cycle where a level-1"
+			"\n\t\tcache or level-2 TLB miss is in progress.",
+	},
+	{
+		.ctrnum = 145,
+		.ctrset = CPUMF_CTRSET_EXTENDED,
+		.name = "DCW_REQ",
+		.desc = "A directory write to the Level-1 Data cache"
+			"\n\t\tdirectory where the returned cache line was sourced"
+			"\n\t\tfrom the requestor’s Level-2 cache.",
+	},
+	{
+		.ctrnum = 146,
+		.ctrset = CPUMF_CTRSET_EXTENDED,
+		.name = "DCW_REQ_IV",
+		.desc = "A directory write to the Level-1 Data cache"
+			"\n\t\tdirectory where the returned cache line was sourced"
+			"\n\t\tfrom the requestor’s Level-2 cache with"
+			"\n\t\tintervention.",
+	},
+	{
+		.ctrnum = 147,
+		.ctrset = CPUMF_CTRSET_EXTENDED,
+		.name = "DCW_REQ_CHIP_HIT",
+		.desc = "A directory write to the Level-1 Data cache"
+			"\n\t\tdirectory where the returned cache line was sourced"
+			"\n\t\tfrom the requestor’s Level-2 cache after using"
+			"\n\t\tchip level horizontal persistence, Chip-HP hit.",
+	},
+	{
+		.ctrnum = 148,
+		.ctrset = CPUMF_CTRSET_EXTENDED,
+		.name = "DCW_REQ_DRAWER_HIT",
+		.desc = "A directory write to the Level-1 Data cache"
+			"\n\t\tdirectory where the returned cache line was sourced"
+			"\n\t\tfrom the requestor’s Level-2 cache after using"
+			"\n\t\tdrawer level horizontal persistence, Drawer-HP hit.",
+	},
+	{
+		.ctrnum = 149,
+		.ctrset = CPUMF_CTRSET_EXTENDED,
+		.name = "DCW_ON_CHIP",
+		.desc = "A directory write to the Level-1 Data cache"
+			"\n\t\tdirectory where the returned cache line was sourced"
+			"\n\t\tfrom an On-Chip Level-2 cache.",
+	},
+	{
+		.ctrnum = 150,
+		.ctrset = CPUMF_CTRSET_EXTENDED,
+		.name = "DCW_ON_CHIP_IV",
+		.desc = "A directory write to the Level-1 Data cache"
+			"\n\t\tdirectory where the returned cache line was sourced"
+			"\n\t\tfrom an On-Chip Level-2 cache with intervention.",
+	},
+	{
+		.ctrnum = 151,
+		.ctrset = CPUMF_CTRSET_EXTENDED,
+		.name = "DCW_ON_CHIP_CHIP_HIT",
+		.desc = "A directory write to the Level-1 Data cache"
+			"\n\t\tdirectory where the returned cache line was sourced"
+			"\n\t\tfrom an On-Chip Level-2 cache after using chip"
+			"\n\t\tlevel horizontal persistence, Chip-HP hit.",
+	},
+	{
+		.ctrnum = 152,
+		.ctrset = CPUMF_CTRSET_EXTENDED,
+		.name = "DCW_ON_CHIP_DRAWER_HIT",
+		.desc = "A directory write to the Level-1 Data cache"
+			"\n\t\tdirectory where the returned cache line was sourced"
+			"\n\t\tfrom an On-Chip Level-2 cache using drawer level"
+			"\n\t\thorizontal persistence, Drawer-HP hit.",
+	},
+	{
+		.ctrnum = 153,
+		.ctrset = CPUMF_CTRSET_EXTENDED,
+		.name = "DCW_ON_MODULE",
+		.desc = "A directory write to the Level-1 Data cache"
+			"\n\t\tdirectory where the returned cache line was sourced"
+			"\n\t\tfrom an On-Module Level-2 cache.",
+	},
+	{
+		.ctrnum = 154,
+		.ctrset = CPUMF_CTRSET_EXTENDED,
+		.name = "DCW_ON_DRAWER",
+		.desc = "A directory write to the Level-1 Data cache"
+			"\n\t\tdirectory where the returned cache line was sourced"
+			"\n\t\tfrom an On-Drawer Level-2 cache.",
+	},
+	{
+		.ctrnum = 155,
+		.ctrset = CPUMF_CTRSET_EXTENDED,
+		.name = "DCW_OFF_DRAWER",
+		.desc = "A directory write to the Level-1 Data cache"
+			"\n\t\tdirectory where the returned cache line was sourced"
+			"\n\t\tfrom an Off-Drawer Level-2 cache.",
+	},
+	{
+		.ctrnum = 156,
+		.ctrset = CPUMF_CTRSET_EXTENDED,
+		.name = "DCW_ON_CHIP_MEMORY",
+		.desc = "A directory write to the Level-1 Data cache"
+			"\n\t\tdirectory where the returned cache line was sourced"
+			"\n\t\tfrom On-Chip memory.",
+	},
+	{
+		.ctrnum = 157,
+		.ctrset = CPUMF_CTRSET_EXTENDED,
+		.name = "DCW_ON_MODULE_MEMORY",
+		.desc = "A directory write to the Level-1 Data cache"
+			"\n\t\tdirectory where the returned cache line was sourced"
+			"\n\t\tfrom On-Module memory.",
+	},
+	{
+		.ctrnum = 158,
+		.ctrset = CPUMF_CTRSET_EXTENDED,
+		.name = "DCW_ON_DRAWER_MEMORY",
+		.desc = "A directory write to the Level-1 Data cache"
+			"\n\t\tdirectory where the returned cache line was sourced"
+			"\n\t\tfrom On-Drawer memory.",
+	},
+	{
+		.ctrnum = 159,
+		.ctrset = CPUMF_CTRSET_EXTENDED,
+		.name = "DCW_OFF_DRAWER_MEMORY",
+		.desc = "A directory write to the Level-1 Data cache"
+			"\n\t\tdirectory where the returned cache line was sourced"
+			"\n\t\tfrom Off-Drawer memory.",
+	},
+	{
+		.ctrnum = 160,
+		.ctrset = CPUMF_CTRSET_EXTENDED,
+		.name = "IDCW_ON_MODULE_IV",
+		.desc = "A directory write to the Level-1 Data or Level-1"
+			"\n\t\tInstruction cache directory where the returned"
+			"\n\t\tcache line was sourced from an On-Module Level-2"
+			"\n\t\tcache with intervention.",
+	},
+	{
+		.ctrnum = 161,
+		.ctrset = CPUMF_CTRSET_EXTENDED,
+		.name = "IDCW_ON_MODULE_CHIP_HIT",
+		.desc = "A directory write to the Level-1 Data or Level-1"
+			"\n\t\tInstruction cache directory where the returned"
+			"\n\t\tcache line was sourced from an On-Module Level-2"
+			"\n\t\tcache using chip horizontal persistence, Chip-HP"
+			"\n\t\thit.",
+	},
+	{
+		.ctrnum = 162,
+		.ctrset = CPUMF_CTRSET_EXTENDED,
+		.name = "IDCW_ON_MODULE_DRAWER_HIT",
+		.desc = "A directory write to the Level-1 Data or Level-1"
+			"\n\t\tInstruction cache directory where the returned"
+			"\n\t\tcache line was sourced from an On-Module Level-2"
+			"\n\t\tcache using drawer level horizontal persistence,"
+			"\n\t\tDrawer-HP hit.",
+	},
+	{
+		.ctrnum = 163,
+		.ctrset = CPUMF_CTRSET_EXTENDED,
+		.name = "IDCW_ON_DRAWER_IV",
+		.desc = "A directory write to the Level-1 Data or Level-1"
+			"\n\t\tInstruction cache directory where the returned"
+			"\n\t\tcache line was sourced from an On-Drawer Level-2"
+			"\n\t\tcache with intervention.",
+	},
+	{
+		.ctrnum = 164,
+		.ctrset = CPUMF_CTRSET_EXTENDED,
+		.name = "IDCW_ON_DRAWER_CHIP_HIT",
+		.desc = "A directory write to the Level-1 Data or Level-1"
+			"\n\t\tinstruction cache directory where the returned"
+			"\n\t\tcache line was sourced from an On-Drawer Level-2"
+			"\n\t\tcache using chip level horizontal persistence, Chip-"
+			"\n\t\tHP hit.",
+	},
+	{
+		.ctrnum = 165,
+		.ctrset = CPUMF_CTRSET_EXTENDED,
+		.name = "IDCW_ON_DRAWER_DRAWER_HIT",
+		.desc = "A directory write to the Level-1 Data or Level-1"
+			"\n\t\tinstruction cache directory where the returned"
+			"\n\t\tcache line was sourced from an On-Drawer Level-2"
+			"\n\t\tcache using drawer level horizontal persistence,"
+			"\n\t\tDrawer-HP hit.",
+	},
+	{
+		.ctrnum = 166,
+		.ctrset = CPUMF_CTRSET_EXTENDED,
+		.name = "IDCW_OFF_DRAWER_IV",
+		.desc = "A directory write to the Level-1 Data or Level-1"
+			"\n\t\tinstruction cache directory where the returned"
+			"\n\t\tcache line was sourced from an Off-Drawer Level-2"
+			"\n\t\tcache with intervention.",
+	},
+	{
+		.ctrnum = 167,
+		.ctrset = CPUMF_CTRSET_EXTENDED,
+		.name = "IDCW_OFF_DRAWER_CHIP_HIT",
+		.desc = "A directory write to the Level-1 Data or Level-1"
+			"\n\t\tinstruction cache directory where the returned"
+			"\n\t\tcache line was sourced from an Off-Drawer Level-2"
+			"\n\t\tcache using chip level horizontal persistence, Chip-"
+			"\n\t\tHP hit.",
+	},
+	{
+		.ctrnum = 168,
+		.ctrset = CPUMF_CTRSET_EXTENDED,
+		.name = "IDCW_OFF_DRAWER_DRAWER_HIT",
+		.desc = "A directory write to the Level-1 Data or Level-1"
+			"\n\t\tInstruction cache directory where the returned"
+			"\n\t\tcache line was sourced from an Off-Drawer Level-2"
+			"\n\t\tcache using drawer level horizontal persistence,"
+			"\n\t\tDrawer-HP hit.",
+	},
+	{
+		.ctrnum = 169,
+		.ctrset = CPUMF_CTRSET_EXTENDED,
+		.name = "ICW_REQ",
+		.desc = "A directory write to the Level-1 Instruction cache"
+			"\n\t\tdirectory where the returned cache line was sourced"
+			"\n\t\tthe requestors Level-2 cache.",
+	},
+	{
+		.ctrnum = 170,
+		.ctrset = CPUMF_CTRSET_EXTENDED,
+		.name = "ICW_REQ_IV",
+		.desc = "A directory write to the Level-1 Instruction cache"
+			"\n\t\tdirectory where the returned cache line was sourced"
+			"\n\t\tfrom the requestors Level-2 cache with"
+			"\n\t\tintervention.",
+	},
+	{
+		.ctrnum = 171,
+		.ctrset = CPUMF_CTRSET_EXTENDED,
+		.name = "ICW_REQ_CHIP_HIT",
+		.desc = "A directory write to the Level-1 Instruction cache"
+			"\n\t\tdirectory where the returned cache line was sourced"
+			"\n\t\tfrom the requestors Level-2 cache using chip level"
+			"\n\t\thorizontal persistence, Chip-HP hit.",
+	},
+	{
+		.ctrnum = 172,
+		.ctrset = CPUMF_CTRSET_EXTENDED,
+		.name = "ICW_REQ_DRAWER_HIT",
+		.desc = "A directory write to the Level-1 Instruction cache"
+			"\n\t\tdirectory where the returned cache line was sourced"
+			"\n\t\tfrom the requestor’s Level-2 cache using drawer"
+			"\n\t\tlevel horizontal persistence, Drawer-HP hit.",
+	},
+	{
+		.ctrnum = 173,
+		.ctrset = CPUMF_CTRSET_EXTENDED,
+		.name = "ICW_ON_CHIP",
+		.desc = "A directory write to the Level-1 Instruction cache"
+			"\n\t\tdirectory where the returned cache line was sourced"
+			"\n\t\tfrom an On-Chip Level-2 cache.",
+	},
+	{
+		.ctrnum = 174,
+		.ctrset = CPUMF_CTRSET_EXTENDED,
+		.name = "ICW_ON_CHIP_IV",
+		.desc = "A directory write to the Level-1 Instruction cache"
+			"\n\t\tdirectory where the returned cache line was sourced"
+			"\n\t\tan On-Chip Level-2 cache with intervention.",
+	},
+	{
+		.ctrnum = 175,
+		.ctrset = CPUMF_CTRSET_EXTENDED,
+		.name = "ICW_ON_CHIP_CHIP_HIT",
+		.desc = "A directory write to the Level-1 Instruction cache"
+			"\n\t\tdirectory where the returned cache line was sourced"
+			"\n\t\tfrom an On-Chip Level-2 cache using chip level"
+			"\n\t\thorizontal persistence, Chip-HP hit.",
+	},
+	{
+		.ctrnum = 176,
+		.ctrset = CPUMF_CTRSET_EXTENDED,
+		.name = "ICW_ON_CHIP_DRAWER_HIT",
+		.desc = "A directory write to the Level-1 Instruction cache"
+			"\n\t\tdirectory where the returned cache line was sourced"
+			"\n\t\tfrom an On-Chip level 2 cache using drawer level"
+			"\n\t\thorizontal persistence, Drawer-HP hit.",
+	},
+	{
+		.ctrnum = 177,
+		.ctrset = CPUMF_CTRSET_EXTENDED,
+		.name = "ICW_ON_MODULE",
+		.desc = "A directory write to the Level-1 Instruction cache"
+			"\n\t\tdirectory where the returned cache line was sourced"
+			"\n\t\tfrom an On-Module Level-2 cache.",
+	},
+	{
+		.ctrnum = 178,
+		.ctrset = CPUMF_CTRSET_EXTENDED,
+		.name = "ICW_ON_DRAWER",
+		.desc = "A directory write to the Level-1 Instruction cache"
+			"\n\t\tdirectory where the returned cache line was sourced"
+			"\n\t\tan On-Drawer Level-2 cache.",
+	},
+	{
+		.ctrnum = 179,
+		.ctrset = CPUMF_CTRSET_EXTENDED,
+		.name = "ICW_OFF_DRAWER",
+		.desc = "A directory write to the Level-1 Instruction cache"
+			"\n\t\tdirectory where the returned cache line was sourced"
+			"\n\t\tan Off-Drawer Level-2 cache.",
+	},
+	{
+		.ctrnum = 180,
+		.ctrset = CPUMF_CTRSET_EXTENDED,
+		.name = "ICW_ON_CHIP_MEMORY",
+		.desc = "A directory write to the Level-1 Instruction cache"
+			"\n\t\tdirectory where the returned cache line was sourced"
+			"\n\t\tfrom On-Chip memory.",
+	},
+	{
+		.ctrnum = 181,
+		.ctrset = CPUMF_CTRSET_EXTENDED,
+		.name = "ICW_ON_MODULE_MEMORY",
+		.desc = "A directory write to the Level-1 Instruction cache"
+			"\n\t\tdirectory where the returned cache line was sourced"
+			"\n\t\tfrom On-Module memory.",
+	},
+	{
+		.ctrnum = 182,
+		.ctrset = CPUMF_CTRSET_EXTENDED,
+		.name = "ICW_ON_DRAWER_MEMORY",
+		.desc = "A directory write to the Level-1 Instruction cache"
+			"\n\t\tdirectory where the returned cache line was sourced"
+			"\n\t\tfrom On-Drawer memory.",
+	},
+	{
+		.ctrnum = 183,
+		.ctrset = CPUMF_CTRSET_EXTENDED,
+		.name = "ICW_OFF_DRAWER_MEMORY",
+		.desc = "A directory write to the Level-1 Instruction cache"
+			"\n\t\tdirectory where the returned cache line was sourced"
+			"\n\t\tfrom Off-Drawer memory.",
+	},
+	{
+		.ctrnum = 224,
+		.ctrset = CPUMF_CTRSET_EXTENDED,
+		.name = "BCD_DFP_EXECUTION_SLOTS",
+		.desc = "Count of floating point execution slots used for"
+			"\n\t\tfinished Binary Coded Decimal to Decimal Floating"
+			"\n\t\tPoint conversions. Instructions: CDZT, CXZT, CZDT,"
+			"\n\t\tCZXT.",
+	},
+	{
+		.ctrnum = 225,
+		.ctrset = CPUMF_CTRSET_EXTENDED,
+		.name = "VX_BCD_EXECUTION_SLOTS",
+		.desc = "Count of floating point execution slots used for"
+			"\n\t\tfinished vector arithmetic Binary Coded Decimal"
+			"\n\t\tinstructions. Instructions: VAP, VSP, VMP, VMSP,"
+			"\n\t\tVDP, VSDP, VRP, VLIP, VSRP, VPSOP, VCP, VTP, VPKZ,"
+			"\n\t\tVUPKZ, VCVB, VCVBG, VCVD, VCVDG.",
+	},
+	{
+		.ctrnum = 226,
+		.ctrset = CPUMF_CTRSET_EXTENDED,
+		.name = "DECIMAL_INSTRUCTIONS",
+		.desc = "Decimal instruction dispatched. Instructions: CVB,"
+			"\n\t\tCVD, AP, CP, DP, ED, EDMK, MP, SRP, SP, ZAP.",
+	},
+	{
+		.ctrnum = 232,
+		.ctrset = CPUMF_CTRSET_EXTENDED,
+		.name = "LAST_HOST_TRANSLATIONS",
+		.desc = "Last Host Translation done",
+	},
+	{
+		.ctrnum = 244,
+		.ctrset = CPUMF_CTRSET_EXTENDED,
+		.name = "TX_NC_TABORT",
+		.desc = "A transaction abort has occurred in a non-"
+			"\n\t\tconstrained transactional-execution mode.",
+	},
+	{
+		.ctrnum = 245,
+		.ctrset = CPUMF_CTRSET_EXTENDED,
+		.name = "TX_C_TABORT_NO_SPECIAL",
+		.desc = "A transaction abort has occurred in a constrained"
+			"\n\t\ttransactional-execution mode and the CPU is not"
+			"\n\t\tusing any special logic to allow the transaction to"
+			"\n\t\tcomplete.",
+	},
+	{
+		.ctrnum = 246,
+		.ctrset = CPUMF_CTRSET_EXTENDED,
+		.name = "TX_C_TABORT_SPECIAL",
+		.desc = "A transaction abort has occurred in a constrained"
+			"\n\t\ttransactional-execution mode and the CPU is using"
+			"\n\t\tspecial logic to allow the transaction to complete.",
+	},
+	{
+		.ctrnum = 248,
+		.ctrset = CPUMF_CTRSET_EXTENDED,
+		.name = "DFLT_ACCESS",
+		.desc = "Cycles CPU spent obtaining access to Deflate unit",
+	},
+	{
+		.ctrnum = 253,
+		.ctrset = CPUMF_CTRSET_EXTENDED,
+		.name = "DFLT_CYCLES",
+		.desc = "Cycles CPU is using Deflate unit",
+	},
+	{
+		.ctrnum = 256,
+		.ctrset = CPUMF_CTRSET_EXTENDED,
+		.name = "SORTL",
+		.desc = "Increments by one for every SORT LISTS instruction"
+			"\n\t\texecuted.",
+	},
+	{
+		.ctrnum = 265,
+		.ctrset = CPUMF_CTRSET_EXTENDED,
+		.name = "DFLT_CC",
+		.desc = "Increments by one for every DEFLATE CONVERSION CALL"
+			"\n\t\tinstruction executed.",
+	},
+	{
+		.ctrnum = 266,
+		.ctrset = CPUMF_CTRSET_EXTENDED,
+		.name = "DFLT_CCFINISH",
+		.desc = "Increments by one for every DEFLATE CONVERSION CALL"
+			"\n\t\tinstruction executed that ended in Condition Codes"
+			"\n\t\t0, 1 or 2.",
+	},
+	{
+		.ctrnum = 267,
+		.ctrset = CPUMF_CTRSET_EXTENDED,
+		.name = "NNPA_INVOCATIONS",
+		.desc = "Increments by one for every Neural Network"
+			"\n\t\tProcessing Assist instruction executed.",
+	},
+	{
+		.ctrnum = 268,
+		.ctrset = CPUMF_CTRSET_EXTENDED,
+		.name = "NNPA_COMPLETIONS",
+		.desc = "Increments by one for every Neural Network"
+			"\n\t\tProcessing Assist instruction executed that ended"
+			"\n\t\tin Condition Codes 0, 1 or 2.",
+	},
+	{
+		.ctrnum = 269,
+		.ctrset = CPUMF_CTRSET_EXTENDED,
+		.name = "NNPA_WAIT_LOCK",
+		.desc = "Cycles CPU spent obtaining access to IBM Z"
+			"\n\t\tIntegrated Accelerator for AI.",
+	},
+	{
+		.ctrnum = 270,
+		.ctrset = CPUMF_CTRSET_EXTENDED,
+		.name = "NNPA_HOLD_LOCK",
+		.desc = "Cycles CPU is using IBM Z Integrated Accelerator"
+			"\n\t\tfor AI.",
+	},
+	{
+		.ctrnum = 448,
+		.ctrset = CPUMF_CTRSET_MT_DIAG,
+		.name = "MT_DIAG_CYCLES_ONE_THR_ACTIVE",
+		.desc = "Cycle count with one thread active",
+	},
+	{
+		.ctrnum = 449,
+		.ctrset = CPUMF_CTRSET_MT_DIAG,
+		.name = "MT_DIAG_CYCLES_TWO_THR_ACTIVE",
+		.desc = "Cycle count with two threads active",
+	},
+};
+
 /* Return the type number of the CPU Measurement facility from the sysfs file.
  * If the type number is equal to PERF_TYPE_RAW, then the prefix is 'r' to
  * specify the raw counter number by the perf tool.
  * If perf_pmu_register() kernel function assigned any other (higher) type
  * number, set the prefix to <type-nr>:
  */
-static int read_cpumf_type(const char *filename, const char *type)
+static void set_prefix(int nr)
 {
-	int nr, rc = EXIT_FAILURE;
-	FILE *fp = fopen(filename, "r");
-
-	if (fp == NULL) {
-		warnx("No CPU-measurement %s facility detected", type);
-		return rc;
-	}
-	if (fscanf(fp, "%d", &nr) != 1) {
-		warnx("Can not parse file %s", filename);
-	} else {
-		rc = EXIT_SUCCESS;
-		if (nr == PERF_TYPE_RAW)
-			strcat(prefix, "r");
-		else
-			snprintf(prefix, sizeof prefix, "%d:", nr);
-	}
-	fclose(fp);
-	return rc;
+	if (nr == PERF_TYPE_RAW)
+		strcat(prefix, "r");
+	else
+		snprintf(prefix, sizeof(prefix), "%d:", nr);
 }
 
 /* Parse tool parameters. In case of --help or --version, print
@@ -2661,8 +3224,6 @@ static unsigned long div_ceil(unsigned long a, unsigned long b)
 
 static void show_info(struct cpumf_info *p, int details)
 {
-	struct stat sbuf;
-
 	if (!p->have_counter && !p->have_samples) {
 		warnx("No CPU-measurement facilities detected");
 		return;
@@ -2672,7 +3233,7 @@ static void show_info(struct cpumf_info *p, int details)
 		if (details) {
 			printf("----------------------------------------------"
 			       "----------------------------\n");
-			printf("Version: %3.1f\n\n", p->version);
+			printf("Version: %d.%d\n\n", p->first_vn, p->second_vn);
 			printf("Authorized counter sets:\n");
 			if (!p->authorization)
 				printf("    None\n");
@@ -2689,8 +3250,7 @@ static void show_info(struct cpumf_info *p, int details)
 			if (0x8000 & p->authorization)
 				printf("    Coprocessor Group counter Set\n");
 			printf("\nLinux perf event support: %s\n\n",
-				(stat(PERF_PATH PERF_CF, &sbuf)) ? "No" :
-				"Yes (PMU: " PERF_CF ")");
+				!p->have_counter ? "No" : "Yes (PMU: cpum_cf)");
 
 		}
 	} else
@@ -2718,19 +3278,18 @@ static void show_info(struct cpumf_info *p, int details)
 			       p->diag_sample_sz);
 
 			printf("\nLinux perf event support: %s\n\n",
-				(stat(PERF_PATH PERF_SF, &sbuf)) ? "No" :
-				"Yes (PMU: " PERF_SF ")");
+				!p->have_samples ? "No" : "Yes (PMU: cpum_sf)");
 
-			printf("Current sampling buffer settings for %s:\n",
-			       PERF_SF);
+			printf("Current sampling buffer settings for"
+			       " cpum_sf:\n");
 			printf("    Basic-sampling mode\n");
 			total = p->min_sfb + div_ceil(p->min_sfb, PER_SDBT_SIZE);
-			human(text, sizeof text, PAGE_SIZE * total);
-			printf("	Minimum: %6d"
+			human(text, sizeof(text), PAGE_SIZE * total);
+			printf("	Minimum: %6ld"
 			       " sample-data-blocks (%6s)\n", p->min_sfb, text);
 			total = p->max_sfb + div_ceil(p->max_sfb, PER_SDBT_SIZE);
-			human(text, sizeof text, PAGE_SIZE * total);
-			printf("	Maximum: %6d"
+			human(text, sizeof(text), PAGE_SIZE * total);
+			printf("	Maximum: %6ld"
 			       " sample-data-blocks (%6s)\n\n", p->max_sfb,
 			       text);
 
@@ -2739,13 +3298,13 @@ static void show_info(struct cpumf_info *p, int details)
 			fdiag = div_ceil(p->diag_sample_sz, p->basic_sample_sz);
 			total = fdiag * p->min_sfb
 				+ div_ceil(p->min_sfb, PER_SDBT_SIZE);
-			human(text, sizeof text, PAGE_SIZE * total);
+			human(text, sizeof(text), PAGE_SIZE * total);
 			printf("	Minimum: %6ld"
 			       " sample-data-blocks (%6s)\n",
 			       fdiag * p->min_sfb, text);
 			total = fdiag * p->max_sfb
 				+ div_ceil(p->max_sfb * fdiag, PER_SDBT_SIZE);
-			human(text, sizeof text, PAGE_SIZE * total);
+			human(text, sizeof(text), PAGE_SIZE * total);
 			printf("	Maximum: %6ld"
 			       " sample-data-blocks (%6s)\n", fdiag * p->max_sfb,
 			       text);
@@ -2755,43 +3314,23 @@ static void show_info(struct cpumf_info *p, int details)
 		warnx("No CPU-measurement sampling facility detected");
 }
 
-/* Read CPU Measurement sampling facility device driver minimum and maximum
- * buffer size
- */
-static int read_sfb(struct cpumf_info *p)
-{
-	int rc = EXIT_SUCCESS;
-	FILE *fp;
-
-	fp = fopen(PERF_SFB_SIZE, "r");
-	if (!fp) {
-		warn(PERF_SFB_SIZE);
-		return EXIT_FAILURE;
-	}
-	if (fscanf(fp, "%d,%d", &p->min_sfb, &p->max_sfb) != 2) {
-		warnx("Can not parse %s", PERF_SFB_SIZE);
-		rc = EXIT_FAILURE;
-	}
-	fclose(fp);
-	return rc;
-}
-
 /* Set the counter name for z15 counter numbered 265. It is either named
  * DFLT_CCERROR or DFLT_CCFINISH, depending on the linux version. The
  * counter was renamed from CCERROR to CCFINISH in linux version 5.8.
  * Check for existence of file /sys/devices/cpum_cf/events/DLFT_CCERROR.
  */
-#define	CCERROR		"/sys/devices/cpum_cf/events/DFLT_CCERROR"
 static void read_ccerror(struct counters *cp, size_t cp_cnt)
 {
+	char *ctrname, *path;
 	struct stat sbuf;
-	char *ctrname;
 	size_t i = 0;
 
-	if (!stat(CCERROR, &sbuf))
+	path = util_path_sysfs("devices/cpum_cf/events/DFLT_CCERROR");
+	if (!stat(path, &sbuf))
 		ctrname = "DFLT_CCERROR";
 	else
 		ctrname = "DFLT_CCFINISH";
+	free(path);
 	/* Find DFLT_CC{FINISH,ERROR} in table and set name */
 	for (struct counters *p = cp; i < cp_cnt; ++i, ++p)
 		if (p->ctrnum == 265) {
@@ -2803,81 +3342,23 @@ static void read_ccerror(struct counters *cp, size_t cp_cnt)
 /* Read allnecessary information from /sysfs file /proc/service_levels */
 static int read_info(void)
 {
-	char *linep = NULL;
-	size_t line_sz;
-	ssize_t nbytes;
-	FILE *slp;
-	int rc;
+	int rc = EXIT_FAILURE;
 
-	memset(&cpumf, 0, sizeof cpumf);
-	slp = fopen(SERVICELEVEL, "r");
-	if (!slp) {
-		warn(SERVICELEVEL);
-		return EXIT_FAILURE;
-	}
-
-	while ((nbytes = getline(&linep, &line_sz, slp)) != EOF) {
-		if (!strncmp(linep, "CPU-MF: Counter facility:", 25)) {
-			rc = sscanf(linep, "CPU-MF: Counter facility:"
-				    " version=%f authorization=%x",
-				    &cpumf.version, &cpumf.authorization);
-			if (rc != 2) {
-				warnx("Can not parse line %s", linep);
-				rc = EXIT_FAILURE;
-				goto out;
-			}
-			cpumf.have_counter = 1;
-			cpumf.first_vn = (int)cpumf.version;
-			cpumf.second_vn = ((int)(10 * cpumf.version) % 10);
-		}
-		if (!strncmp(linep, "CPU-MF: Sampling facility: min", 30)) {
-			rc = sscanf(linep, "CPU-MF: Sampling facility:"
-				    " min_rate=%ld max_rate=%ld cpu_speed=%ld",
-				    &cpumf.min_rate, &cpumf.max_rate,
-				    &cpumf.cpu_speed);
-			if (rc != 3) {
-				warnx("Can not parse line %s", linep);
-				rc = EXIT_FAILURE;
-				goto out;
-			}
-			cpumf.have_samples = 1;
-		}
-		if (!strncmp(linep, "CPU-MF: Sampling facility: mode=basic", 37)) {
-			rc = sscanf(linep, "CPU-MF: Sampling facility:"
-				    " mode=basic sample_size=%u",
-				    &cpumf.basic_sample_sz);
-			if (rc != 1) {
-				warnx("Can not parse line %s", linep);
-				rc = EXIT_FAILURE;
-				goto out;
-			}
-		}
-		if (!strncmp(linep, "CPU-MF: Sampling facility: mode=diag", 36)) {
-			rc = sscanf(linep, "CPU-MF: Sampling facility:"
-				    " mode=diagnostic sample_size=%u",
-				    &cpumf.diag_sample_sz);
-			if (rc != 1) {
-				warnx("Can not parse line %s", linep);
-				rc = EXIT_FAILURE;
-				goto out;
-			}
-		}
-	}
-	if (cpumf.have_samples) {
-		rc = read_sfb(&cpumf);
-		if (rc == EXIT_FAILURE)
-			goto out;
-	}
+	cpumf.have_counter = libcpumf_cpumcf_info(&cpumf.first_vn,
+						  &cpumf.second_vn,
+						  &cpumf.authorization);
+	cpumf.have_samples = libcpumf_cpumsf_info(&cpumf.min_rate,
+						  &cpumf.max_rate,
+						  &cpumf.cpu_speed,
+						  &cpumf.basic_sample_sz,
+						  &cpumf.diag_sample_sz);
+	if (cpumf.have_samples)
+		libcpumf_sfb_info(&cpumf.min_sfb, &cpumf.max_sfb);
 	cpumf.machine_type = util_arch_machine_type();
-	if (cpumf.machine_type == UTIL_ARCH_MACHINE_TYPE_UNKNOWN) {
+	if (cpumf.machine_type == UTIL_ARCH_MACHINE_TYPE_UNKNOWN)
 		rc = EXIT_FAILURE;
-		goto out;
-	}
-	rc = EXIT_SUCCESS;
-out:
-	fclose(slp);
-	free(linep);
-
+	else
+		rc = EXIT_SUCCESS;
 	return rc;
 }
 
@@ -2991,6 +3472,9 @@ static struct counters *get_counter(int ctrset, size_t *len)
 			*len = ARRAY_SIZE(cpumcf_z15_counters);
 			read_ccerror(cp, *len);
 			break;
+		case UTIL_ARCH_MACHINE_TYPE_Z16:
+			cp = cpumcf_z16_counters;
+			*len = ARRAY_SIZE(cpumcf_z16_counters);
 		}
 		break;
 	}
@@ -3051,14 +3535,18 @@ int main(int argc, char **argv)
 	case ACTION_CNT:
 	case ACTION_CNTALL:
 		all = ret == ACTION_CNTALL;
-		ret = read_cpumf_type(CPUMF_CF_TYPE, "counter");
-		if (ret == EXIT_SUCCESS)
+		ret = libcpumf_pmutype(S390_CPUMF_CF);
+		if (ret >= EXIT_SUCCESS) {
+			set_prefix(ret);
 			show_counter(all);
+		}
 		break;
 	case ACTION_SAMPLE:
-		ret = read_cpumf_type(CPUMF_SF_TYPE, "sampling");
-		if (ret == EXIT_SUCCESS)
+		ret = libcpumf_pmutype(S390_CPUMF_SF);
+		if (ret >= EXIT_SUCCESS) {
+			set_prefix(ret);
 			show_sample();
+		}
 		break;
 	case ACTION_NONE:
 	case ACTION_INFO:

cpumf/lshwc.c

@@ -38,17 +38,19 @@
 #include "lib/util_path.h"
 #include "lib/util_scandir.h"
 #include "lib/util_libc.h"
+#include "lib/util_file.h"
+#include "lib/libcpumf.h"
 
 #include "lshwc.h"
 
-#define SERVICELEVEL	"/proc/service_levels"
 #define CPUS_ONLINE	"/sys/devices/system/cpu/online"
 #define CPUS_POSSIBLE	"/sys/devices/system/cpu/possible"
 #define CPUS_KERNELMAX	"/sys/devices/system/cpu/kernel_max"
 #define MAXCTRS		512
+#define IOCTLSLEEP	60U
 
-static const unsigned int ioctlsleep = 60;
-static unsigned int read_interval = ioctlsleep, cfvn, csvn, authorization;
+static unsigned int read_interval = IOCTLSLEEP;
+static int cfvn, csvn, authorization;
 static unsigned long loop_count = 1;
 static unsigned char *ioctlbuffer;
 static bool allcpu;
@@ -61,25 +63,6 @@ static struct ctrname {		/* List of defined counters */
 	unsigned long *ccv;	/* Per CPU counter value */
 } ctrname[MAXCTRS];
 
-/* Open file and extract counter number */
-static int read_counter(const char *p)
-{
-	FILE *fp = fopen(p, "r");
-	int rc = 0, ctr;
-
-	if (fp) {
-		rc = fscanf(fp, "event=%x", &ctr);
-		fclose(fp);
-	}
-	return rc == 1 ? ctr : -EINVAL;
-}
-
-static int add_countername(char *name, int nr)
-{
-	ctrname[nr].name = strdup(name);
-	return ctrname[nr].name ? 0 : -ENOMEM;
-}
-
 static bool read_counternames(void)
 {
 	struct dirent **namelist = NULL;
@@ -95,13 +78,12 @@ static bool read_counternames(void)
 	}
 	for (i = 0; i < count && ctr >= 0; i++) {
 		util_asprintf(&ctrpath, "%s/%s", path, namelist[i]->d_name);
-		ctr = read_counter(ctrpath);
+		if (util_file_read_va(ctrpath, "event=%x", &ctr) == 1)
+			ctrname[ctr].name = util_strdup(namelist[i]->d_name);
+		else
+			warnx("Cannot parse %s", ctrpath);
 		free(ctrpath);
-		if (ctr >= 0)
-			ctr = add_countername(namelist[i]->d_name, ctr);
 	}
-	if (ctr < 0)
-		warnx("Cannot parse %s", path);
 	util_scandir_free(namelist, count);
 	free(path);
 	return ctr < 0 ? false : true;
@@ -151,20 +133,6 @@ static unsigned long getnumber(char *word, char stopchar)
 	return no;
 }
 
-/* Remove all whitespace from string.  */
-static void kill_whitespace(char *s)
-{
-	char *cp = s;
-
-	for (; *s != '\0'; ++s) {
-		if (isspace(*s))
-			continue;
-		if (isprint(*s))
-			*cp++ = *s;
-	}
-	*cp = '\0';
-}
-
 /* Read file to get all online CPUs */
 static bool get_cpus(char *file, char *buf, size_t bufsz)
 {
@@ -242,29 +210,34 @@ static char *show_ctrset(unsigned long set)
 static void parse_cpulist(char *parm, struct s390_hwctr_start *start)
 {
 	uint64_t *words = start->cpumask;
-	unsigned long i, no_a, no_b;
-	char *cp, *tokens[16];		/* Used to parse command line params */
-	char cpubuf[256];
+	unsigned int i, no_a, no_b;
+	cpu_set_t cpulist;
+	int rc;
 
+	CPU_ZERO(&cpulist);
 	start->data_bytes = 0;
-	if (parm)
-		kill_whitespace(parm);
-	if (!parm || *parm == ':') {
-		/* No CPU list or just counter sets */
-		if (!get_cpus(CPUS_ONLINE, cpubuf, sizeof(cpubuf)))
-			exit(EXIT_FAILURE);
-		if (parm)
-			strcat(cpubuf, parm);
-		parm = cpubuf;
+	start->counter_sets = S390_HWCTR_ALL;	/* Default all counter sets */
+
+	if (parm) {		/* CPU list with optional counter set */
+		char *cp = strchr(parm, ':');
+
+		if (cp) {			/* Handle counter set */
+			*cp = '\0';
+			start->counter_sets = parse_ctrset(++cp);
+		}
+
+		if (strlen(parm) > 0)		/* Handle CPU list */
+			rc = libcpumf_cpuset(parm, &cpulist);
+		else
+			rc = libcpumf_cpuset_fn(S390_CPUS_ONLINE, &cpulist);
+		if (rc)
+			errx(EXIT_FAILURE, "Cannot use CPU list %s", parm);
+	} else {		/* No CPU list and no counter sets */
+		rc = libcpumf_cpuset_fn(S390_CPUS_ONLINE, &cpulist);
+		if (rc)
+			err(EXIT_FAILURE, "Cannot read file " S390_CPUS_ONLINE);
 	}
 
-	cp = strchr(parm, ':');
-	if (cp) {		/* Handle counter set */
-		*cp = '\0';
-		start->counter_sets = parse_ctrset(++cp);
-	} else {
-		start->counter_sets = S390_HWCTR_ALL;
-	}
 	/* Check with authorized counter sets */
 	if ((start->counter_sets & authorization) != start->counter_sets) {
 		unsigned int noton = ~(start->counter_sets & authorization);
@@ -276,19 +249,10 @@ static void parse_cpulist(char *parm, struct s390_hwctr_start *start)
 		      show_ctrset(noton));
 	}
 
-	for (i = 0; i < ARRAY_SIZE(tokens) && (tokens[i] = strtok(parm, ","));
-	    ++i, parm = NULL) {
-		cp = strchr(tokens[i], '-');	/* Range character? */
-		if (cp) {
-			no_a = getnumber(tokens[i], *cp);
-			no_b = getnumber(++cp, '\0');
-		} else {
-			no_b = getnumber(tokens[i], '\0');
-			no_a = no_b;
-		}
-		if (!check_set(no_a, no_b, start->counter_sets))
-			errx(EXIT_FAILURE, "Invalid CPU list %s", tokens[i]);
-	}
+	for (rc = 0; rc < CPU_SETSIZE; ++rc)
+		if (CPU_ISSET(rc, &cpulist))
+			if (!check_set(rc, rc, start->counter_sets))
+				errx(EXIT_FAILURE, "Invalid CPU %d", rc);
 
 	/* Convert the CPU list to a bitmask for kernel cpumask_t */
 	for (i = 0, no_b = 0; i < max_possible_cpus; ++i) {
@@ -298,14 +262,8 @@ static void parse_cpulist(char *parm, struct s390_hwctr_start *start)
 			words[no_b] |= 1ULL << no_a;
 		}
 	}
-	/* no_b is highest used index, swap array */
-	start->cpumask_len = (no_b + 1) * 8;
-	for (no_a = 0; no_a < no_b; ++no_a, --no_b) {
-		uint64_t tmp =  words[no_a];
-
-		words[no_a] = words[no_b];
-		words[no_b] = tmp;
-	}
+	/* no_b is highest used index */
+	start->cpumask_len = (no_b + 1) * CHAR_BIT;
 	start->version = S390_HWCTR_START_VERSION;
 }
 
@@ -638,37 +596,6 @@ static int do_it(char *s)
 	return rc ? EXIT_FAILURE : EXIT_SUCCESS;
 }
 
-/* Read counter first and second version number */
-static bool get_cvn(void)
-{
-	char *linep = NULL;
-	bool good = false;
-	size_t line_sz;
-	ssize_t nbytes;
-	FILE *slp;
-
-	slp = fopen(SERVICELEVEL, "r");
-	if (!slp) {
-		warn(SERVICELEVEL);
-		return false;
-	}
-	while ((nbytes = getline(&linep, &line_sz, slp)) != EOF) {
-		if (!strncmp(linep, "CPU-MF: Counter facility:", 25)) {
-			int rc;
-
-			rc = sscanf(linep, "CPU-MF: Counter facility: version=%d.%d authorization=%x",
-				    &cfvn, &csvn, &authorization);
-			good = rc == 3;
-			if (!good)
-				warnx("Cannot parse line %s", linep);
-			break;
-		}
-	}
-	fclose(slp);
-	free(linep);
-	return good;
-}
-
 static struct util_opt opt_vec[] = {
 	UTIL_OPT_SECTION("OPTIONS"),
 	{
@@ -752,7 +679,7 @@ int main(int argc, char **argv)
 	}
 
 	have_support();
-	if (!get_cvn())
+	if (!libcpumf_cpumcf_info(&cfvn, &csvn, &authorization))
 		return EXIT_FAILURE;
 	if (!check_setpossible())
 		return EXIT_FAILURE;

cpumf/man/lscpumf.8

@@ -1,11 +1,11 @@
-\" lscpumf.1
+\" lscpumf.8
 .\"
 .\"
 .\" Copyright IBM Corp. 2014, 2020
 .\" s390-tools is free software; you can redistribute it and/or modify
 .\" it under the terms of the MIT license. See LICENSE for details.
 .\" ----------------------------------------------------------------------
-.TH lscpumf "1" "May 2020" "s390-tools" "CPU-MF management programs"
+.TH lscpumf "8" "May 2022" "s390-tools" "CPU-MF management programs"
 .
 .ds c \fBlscpumf\fP
 .

cpumf/man/lshwc.8

@@ -1,4 +1,4 @@
-.\" lshwc.1
+.\" lshwc.8
 .\"
 .\"
 .\" Copyright IBM Corp. 2021
@@ -7,7 +7,7 @@
 .\" ----------------------------------------------------------------------
 .ds c \fBlshwc\fP
 .
-.TH \*c "1" "February 2021" "s390-tools" "CPU-MF management programs"
+.TH \*c "8" "May 2022" "s390-tools" "CPU-MF management programs"
 .
 .SH NAME
 \*c \- extract CPU Measurement Facilities counter sets
@@ -130,5 +130,5 @@ Date,Time,CPU,PROBLEM_STATE_CPU_CYCLES(32),PROBLEM_STATE_INSTRUCTIONS(33)
 .ft
 .fi
 .SH "SEE ALSO"
-.BR lscpumf (1)
+.BR lscpumf (8)
 .BR chcpumf (8)

cpumf/man/pai.8

@@ -0,0 +1,268 @@
+.\" pai.8
+.\"
+.\"
+.\" Copyright IBM Corp. 2022
+.\" s390-tools is free software; you can redistribute it and/or modify
+.\" it under the terms of the MIT license. See LICENSE for details.
+.\" ----------------------------------------------------------------------
+.ds c \fBpai\fP
+.
+.TH \*c "8" "May 2022" "s390-tools" "PAI Management Programs"
+.
+.SH NAME
+\*c \- record and report Processor Activity Instrumentation (PAI) counters
+.
+.SH SYNOPSIS
+\*c
+.RB [ \-V ][ \-m | \-\-mapsize
+.IR size ]
+.RB [ \-i | \-\-interval
+.IR ms ]
+.BR  \-c | \-\-crypto [ \fIcpulist ][: \fIdata\fR "] [" \fIloops\fP ]
+.br
+\*c
+.RB [ \-V ][ \-m | \-\-mapsize
+.IR size ]
+.RB [ \-i | \-\-interval
+.IR ms ]
+.BR  \-n | \-\-nnpa [ \fIcpulist ][: \fIdata\fR "] [" \fIloops\fP ]
+.br
+\*c
+.RB [ \-V ][ \-H | \-\-humantime ][ \-S | \-\-summary "] " \-r | \-\-report " [" \fIfiles\fP ]
+.br
+\*c
+.BR \-h | \-\-help
+.br
+\*c
+.BR \-v | \-\-version
+.
+.
+.SH DESCRIPTION
+\*c
+counters tally calls for specific CPU instructions.
+The \*c command records PAI counters in a ring buffer.
+\*c can record counter data for all CPUs or for selected CPUs.
+The main command options are
+.B \-c
+for recording cryptographic CPU instructions,
+.B \-n
+for recording NNPA CPU instructions
+and
+.B \-r
+for reporting.
+If all three options are omitted, option
+.B \-r
+is assumed and a message is printed.
+Recording stores data, by CPU, in files
+.I paicrypto.<XXX>,
+for option
+.B \-c
+or
+.I painnpa.<XXX>,
+for option
+.B \-n
+where <XXX> specifies the CPU number with leading
+zeros.
+The files are created in the working directory,
+existing files are overwritten.
+Reporting evaluates files that are created by recording.
+
+.SH OPTIONS
+.TP
+.BR \-c ", " \-\-crypto "\fR[\fIcpulist\fR][:\fIdata\fR]"
+Records data for all (default) or a specified list of CPUs.
+The CPU list is a comma-separated list of CPU numbers and ranges.
+In a range, a hyphen separates the first CPU number
+from the last CPU number.
+By default \*c lists all CPUs.
+.RS
+The optional data specification
+follows the colon
+and determines additional collection of data.
+The specification consists of alphabetic
+characters that can be upper or lower case:
+.IP c|C
+Include task rename system calls
+.B exec
+and
+.BR prctl .
+.IP f|F
+Include task creation and deletion system calls
+.B fork
+and
+.BR exit .
+.IP s|S
+Include context switch records created by the kernel scheduler.
+.IP k|K
+Include only PAI cryptographic counters changed during system call
+execution.
+.IP u|U
+Include only PAI cryptographic counters changed during user space
+execution.
+.RE
+.
+.TP
+.BR \-n ", " \-\-nnpa "\fR[\fIcpulist\fR][:\fIdata\fR]"
+Records data for all (default) or a specified list of CPUs.
+The CPU list is a comma-separated list of CPU numbers and ranges.
+In a range, a hyphen separates the first CPU number
+from the last CPU number.
+By default \*c lists all CPUs.
+.RS
+The optional data specification
+follows the colon
+and determines additional collection of data.
+The specification consists of alphabetic
+characters that can be upper or lower case:
+.IP c|C
+Include task rename system calls
+.B exec
+and
+.BR prctl .
+.IP f|F
+Include task creation and deletion system calls
+.B fork
+and
+.BR exit .
+.IP s|S
+Include context switch records created by the kernel scheduler.
+.RE
+.
+.TP
+.BR \-r ", " \-\-report
+Generates a report from the specified files.
+Files is a list of blank-separated file names.
+If no files are specified,
+uses all files named
+.I paicrypto.<X>,
+in the working directory, where
+.I <X>
+is a CPU number that
+identifies the CPU for which the data was recorded.
+.
+.TP
+.BR \-h ", " \-\-help
+Displays help information, then exits.
+.
+.TP
+.BR \-v ", " \-\-version
+Displays version information, then exits.
+.
+.TP
+.BR \-V ", " \-\-verbose
+Displays the following information during reporting:
+the file name and the hexadecimal offset for each sample header.
+Displays the following information during recording:
+the read position in the ring buffer,
+the file name, and the hexadecimal offset for each sample header.
+.
+.TP
+.BR \-H ", " \-\-humantime
+Changes the time stamp format for sample entries from a hexadecimal
+number to a human readable <seconds>.<nanoseconds> format.
+The time stamp specifies the elapsed time since Linux was booted.
+.
+.TP
+.BR \-S ", " \-\-summary
+Prints a summary of counter names and
+counter numbers with non-zero values.
+The summary is the last line in the command output.
+It shows the sum of the counter values of all processed files.
+.
+.TP
+.BR \-i ", " \-\-interval "\ ms"
+Specifies the waiting time,
+in milliseconds,
+between ring buffer read operations during recording.
+The default is 1000 milliseconds.
+Argument
+.B loops
+is an integer that specifies the number of
+read operations during recording. The default is 1.
+.
+.TP
+.BR \-m ", " \-\-mapsize "\ size"
+Specifies the size of the ring buffer
+that holds the collected data.
+The value specifies the number of 4 KB pages
+to be allocated and must be a power of 2.
+The default size is 512 pages.
+The ring buffer is created with the
+.IR mmap (2)
+system call.
+.
+.SH ARGUMENT
+The command line options determine how command line
+arguments are interpreted.
+For option
+.B \-c
+to start recording, the argument specifies
+the number of read operations.
+If omitted, the default is one.
+For option
+.B \-r
+to start reporting, the argument specifies
+the file names to be read.
+If omitted all files in the current directory
+with file name
+.I paicryto.<X>
+are read.
+.SH "Concurrency with perf tool"
+The following concurrency restrictions apply
+for \*c and the
+.B perf
+tool.
+Both tools use the
+.IR perf_event_open ()
+system call and use the same device driver.
+The device driver supports limited concurrency:
+.IP Counting:
+Counting pai events can run in parallel.
+.IP Sampling:
+Only one sampling event pai_crypto/CRYPTO_ALL can be active at any one time.
+If a sampling event is active, no pai counting event can be active.
+Both tools stop with an error message if they detect a collision
+reported by the device driver.
+.SH "EXAMPLES"
+This example collects cryptographic counter on CPU 0.
+The program runs for 10 seconds
+(10 intervals of 1000 milliseconds).
+.sp 1
+.nf
+.ft CW
+# \*c -c0 10
+.ft R
+.fi
+.sp 1
+This example displays the data
+that is collected in the first example.
+Each data line shows the time stamp in jiffies,
+followed by the CPU number,
+the event number,
+process and thread identifier (pid/tid) separated by slash (/),
+and a comma-separated list of counter number, colon (:),
+and the counter value n hexadecimal notation.
+.sp 1
+.nf
+.ft CW
+# \*c -r
+0x62a668f2fa 0 event 4096 sample pid 4956/4956 9:0xa7,73:0x8,74:0x18
+0x6319c75653 0 event 4096 sample pid 4972/4972 32:0x1
+0x6319e2ddee 0 event 4096 sample pid 4972/4972 32:0x1
+0x631d3e44f5 0 event 4096 sample pid 4972/4972 32:0x2
+0x631d4cfc2e 0 event 4096 sample pid 4972/4972 32:0x3
+0x631d529fdd 0 event 4096 sample pid 4972/4972 32:0x2
+0x631d84cfa0 0 event 4096 sample pid 4972/4972 73:0x1
+0x636e9826bc 0 event 4096 sample pid 4984/4984 9:0xa7,73:0x8,74:0x18
+0x636f81a137 0 event 4096 sample pid 4984/4984 9:0x2,74:0x7
+0x6378026e54 0 event 4096 sample pid 4984/4984 73:0x2
+0x637bcdc8da 0 event 4096 sample pid 4984/4984 73:0x2
+0x637bd426cd 0 event 4096 sample pid 4984/4984 73:0x6
+0x637c503384 0 event 4096 sample pid 4984/4984 73:0x2
+0x64991d83ba 0 event 4096 sample pid 5026/5026 9:0xb0,73:0x8,74:0x18
+#
+.ft R
+.fi
+.sp 1
+.SH "SEE ALSO"
+perf(1)

cpumf/pai.h

@@ -0,0 +1,105 @@
+/*
+ * pai - Extract CPU Processor Activity Instrumentation (PAI) facility data.
+ *
+ * Copyright IBM Corp. 2022
+ *
+ * s390-tools is free software; you can redistribute it and/or modify
+ * it under the terms of the MIT license. See LICENSE for details.
+ */
+
+#ifndef PAI_H
+#define PAI_H
+
+#define S390_EVT_PAI_CRYPTO	0x1000
+
+enum {			/* Event attribute specifications */
+	S390_EVTATTR_CTX_SWITCH = 1,
+	S390_EVTATTR_COMM = 2,
+	S390_EVTATTR_FORK = 4,
+	S390_EVTATTR_USERSPACE = 8,
+	S390_EVTATTR_KERNELSPACE = 16
+};
+
+struct pai_event {
+	struct util_list_node node;	/* List node */
+	struct perf_event_attr attr;	/* Perf_event_open(2) attr */
+	int fd;				/* Perf event file descriptor */
+	void *map_addr;			/* Sampling data mapping address */
+	size_t map_size;		/* Sampling size mapping size */
+	char file_name[16];		/* File name of sampled data */
+	int file_fd;			/* Map data output file descriptor */
+	int cpu;			/* Perf_event_open(2) CPU */
+	unsigned long flags;		/* Perf_event_open(2) flags */
+};
+
+struct pai_event_out {		/* Output for CRYPTO_ALL event */
+	__u32 type;		/* Header type, see PERF_RECORD_xxx */
+	__u16 misc;		/* Header misc, value depends on type */
+	__u64 time;		/* Time stamp valid for all entries */
+	__u32 cpu;		/* CPU number valid for all entries */
+	union {
+		/* Fields from PERF_RECORD_FORK|EXIT */
+		struct {
+			__u32 pid, ppid;
+			__u32 tid, ptid;
+			__u64 time;
+		} s_fork;
+		/* Fields from PERF_RECORD_COMM */
+		struct {
+			__u32 pid, tid;
+			__u8 cmd[16];
+		} s_comm;
+		/* Fields from PERF_RECORD_SWITCH_CPU_WIDE */
+		struct {
+			__u32 next_prev_pid, next_prev_tid;
+		} s_cs;
+		/* Fields from PERF_RECORD_LOST_SAMPLES */
+		struct {
+			__u64 lost;
+		} s_lost;
+		/* Relevant fields from PERF_RECORD_SAMPLE, time and cpu
+		 * are stored above
+		 */
+		struct {
+			__u64 ip;
+			__u64 period;
+			__u32 pid, tid;
+		} s_sample;
+		/* Fields from PERF_RECORD_[UN]THROTTLE */
+		struct {
+			__u64 time;
+			__u64 id;
+			__u64 stream_id;
+		} s_throttle;
+	} u;
+	/* Information on last context switch out */
+	struct cs_switch {
+		unsigned char valid;
+		__u32 topid, totid;
+		__u32 frompid, fromtid;
+	} cs_switch;
+	void *raw;	/* Pointer to key/value array for crypto counters */
+};
+
+struct data_pos {			/* Perf event mapped ring buffer */
+	__u64   data_head;		/* Head in the data section */
+	__u64	data_tail;		/* User-space written tail */
+	__u64	data_offset;		/* Where the buffer starts */
+	__u64	data_size;		/* Data buffer size */
+};
+
+struct event_name {		/* Event list for number to name xlate */
+	char *name;		/* Event name */
+	__u64 config;		/* Event config value */
+	__u64 total;		/* Total counter value */
+};
+
+struct pmu_events {		/* Event list for PMU number to name xlate */
+	struct util_list_node node;	/* List node */
+	char *name;		/* PMU name */
+	int type;		/* PMU type */
+	int lstlen;		/* # of entries in lst */
+	unsigned long base;	/* Base event number */
+	struct event_name *lst;	/* List of event names */
+};
+#endif /* PAI_H */

dasdfmt/dasdfmt.c

@@ -1107,8 +1107,8 @@ static void dasdfmt_write_labels(volume_label_t *vlabel,
 	} else {
 		vlabel->ldl_version = 0xf2; /* EBCDIC '2' */
 		vlabel->formatted_blocks = cylinders * heads * geo.sectors;
-		rc = write(fd, &vlabel->vollbl, (sizeof(*vlabel)
-						 - sizeof(vlabel->volkey)));
+		rc = write(fd, (void *)vlabel + sizeof(vlabel->volkey),
+			   (sizeof(*vlabel) - sizeof(vlabel->volkey)));
 	}
 
 	if (((rc != sizeof(*vlabel) - sizeof(vlabel->formatted_blocks)) &&

dasdinfo/dasdinfo.8

@@ -47,7 +47,7 @@ Print DASD uid without z/VM minidisk token
 
 z/VM may provide an additional token that can be used to distinguish
 between different minidisks (see --extended-uid option).  To remain
-compatibile with systems that were installed on older Linux or z/VM
+compatible with systems that were installed on older Linux or z/VM
 levels, the -u option will print the uid excluding any z/VM-provided
 minidisk token.
 

dasdview/dasdview.c

@@ -29,6 +29,7 @@
 #include "lib/dasd_sys.h"
 #include "lib/libzds.h"
 #include "lib/util_base.h"
+#include "lib/util_libc.h"
 #include "lib/util_opt.h"
 #include "lib/util_prg.h"
 #include "lib/util_sys.h"
@@ -1788,10 +1789,8 @@ dasdview_print_format1(unsigned int size, unsigned char *dumpstr)
 	for (i = 0; i < size; i++) {
 		if ((i / 16) * 16 == i) {
 			printf("\n|  ");
-			strncpy(asc, (char *)dumpstr + i, 16);
-			strncpy(ebc, (char *)dumpstr + i, 16);
-			asc[16] = '\0';
-			ebc[16] = '\0';
+			util_strlcpy(asc, (char *)dumpstr + i, 16);
+			util_strlcpy(ebc, (char *)dumpstr + i, 16);
 		}
 		printf("%02X", dumpstr[i]);
 		if (((i + 1) / 4)  * 4  == i + 1)
@@ -1822,8 +1821,8 @@ dasdview_print_format2(unsigned int size, unsigned char *dumpstr,
 			       begin + (unsigned long long)i,
 			       begin + (unsigned long long)i);
 
-			strncpy(asc, (char *)dumpstr + i, 8);
-			strncpy(ebc, (char *)dumpstr + i, 8);
+			util_strlcpy(asc, (char *)dumpstr + i, 8);
+			util_strlcpy(ebc, (char *)dumpstr + i, 8);
 		}
 		printf("%02X", dumpstr[i]);
 		if (((i + 1) / 4) * 4 == i + 1)

etc/init.d/cpacfstatsd

@@ -1,89 +0,0 @@
-#!/bin/bash
-### BEGIN INIT INFO
-# Provides: cpacfstatsd
-# Required-Start: $local_fs
-# Required-Stop: $local_fs
-# Should-Start:
-# Should-Stop:
-# Default-Start: 2 3 5
-# Default-Stop: 0 1 6
-# Short-Description: Start the cpacfstatsd daemon for Linux on System z
-# Description: CPACF statistics collection daemon process for Linux on System z
-### END INIT INFO
-
-DAEMON=cpacfstatsd
-DAEMON_PATH=/usr/sbin/cpacfstatsd
-RUN_PID_FILE=/run/cpacfstatsd.pid
-RETVAL=0
-OPTIONS=""
-
-# source function library
-. /lib/lsb/init-functions
-
-start()
-{
-	if [ ! -f $RUN_PID_FILE ]; then
-		echo -n $"Starting $DAEMON:"
-		$DAEMON_PATH $OPTIONS
-		if [ $? == "0" ]; then
-			log_success_msg
-		else
-			log_failure_msg
-		fi
-		echo
-	else
-		echo "$DAEMON (pid $(cat $RUN_PID_FILE)) is already running..."
-		echo
-	fi
-}
-
-stop()
-{
-	echo -n $"Stopping $DAEMON:"
-	if [ -f $RUN_PID_FILE ]; then
-		killproc $DAEMON_PATH -TERM
-		log_success_msg
-		rm -f $RUN_PID_FILE
-	else
-		log_failure_msg
-	fi
-	echo
-}
-
-restart() {
-	stop
-	sleep 1
-	start
-}
-
-status()
-{
-	if [ ! -f $RUN_PID_FILE ]; then
-		echo "$DAEMON is not running."
-		echo
-	else
-		echo "$DAEMON (pid $(cat $RUN_PID_FILE)) is running."
-		echo
-	fi
-}
-
-# How are we called?
-case "$1" in
-	start)
-		start
-		;;
-	stop)
-		stop
-		;;
-	status)
-		status
-		;;
-	restart)
-		restart
-		;;
-	*)
-		echo "Usage: $DAEMON {start|stop|status|restart}"
-		RETVAL=1
-esac
-
-exit $RETVAL

etc/init.d/cpuplugd

@@ -1,116 +0,0 @@
-#!/bin/bash
-### BEGIN INIT INFO
-# Provides: cpuplugd
-# Required-Start: $local_fs $remote_fs
-# Required-Stop: $local_fs $remote_fs
-# Should-Start: 
-# Should-Stop: 
-# Default-Start: 2 3 5
-# Default-Stop: 0 1 6
-# Short-Description: Start the cpu hotplug daemon for Linux on System z
-# Description: Starts the cpuplugd. It uses the configuration
-#              file /etc/cpuplugd.conf
-### END INIT INFO
-
-# chkconfig: 235 01 99
-
-DAEMON=cpuplugd
-DAEMON_PATH=/usr/sbin/cpuplugd
-CONFIG_FILE=/etc/cpuplugd.conf
-RUN_PID_FILE=/run/cpuplugd.pid
-RETVAL=0
-OPTIONS="-c $CONFIG_FILE"
-
-# source function library
-. /lib/lsb/init-functions
-
-start()
-{
-	if [ ! -f $RUN_PID_FILE ]; then
-		echo -n $"Starting $DAEMON:"
-		$DAEMON_PATH $OPTIONS
-		if [ $? == "0" ]; then
-			log_success_msg
-		else
-			log_failure_msg
-		fi
-		echo
-	else
-		echo "$DAEMON (pid $(cat $RUN_PID_FILE)) is already running..."
-		echo
-	fi
-}
-
-stop()
-{
-	echo -n $"Stopping $DAEMON:"
-	if [ -f $RUN_PID_FILE ]; then
-		killproc $DAEMON_PATH -TERM
-		log_success_msg
-		rm -f $RUN_PID_FILE
-	else
-		log_failure_msg
-	fi
-	echo
-}
-
-restart() {
-	stop
-  	#
-        # We have to wait 2-3 seconds here. When the daemon is stopped it takes
-        # the time we sleep to reactivate cpus. If we restart to fast and
-        # cpuplugd wasn't able to restore some settings we may get a undesired
-        # online cpu count after cpuplugd shutdown
-        #
-	sleep 4
-	start
-}
-
-status()
-{
-	if [ ! -f $RUN_PID_FILE ]; then
-		echo "$DAEMON is not running."
-		echo
-	else
-		echo "$DAEMON (pid $(cat $RUN_PID_FILE), options: $OPTIONS) is running."
-		echo
-	fi
-}
-
-reload()                                                                        
-{
-        echo -n $"Reloading $DAEMON: "
-	if [ -f $RUN_PID_FILE ]; then
-		killproc $DAEMON_PATH -HUP
-	    log_success_msg
-        else
-            log_failure_msg
-        fi
-        RETVAL=$?
-        echo
-}
-
-
-# How are we called?
-case "$1" in
-	start)
-		start
-		;;
-	stop)
-		stop
-		;;
-	status)
-		status
-		;;
-	restart)
-		restart
-		;;
-	reload|force-reload)
-		reload
-		;;
-	*)
-		echo "Usage: $DAEMON {start|stop|status|restart|reload}"
-		RETVAL=1
-esac
-
-exit $RETVAL

etc/init.d/mon_statd

@@ -1,161 +0,0 @@
-#!/bin/bash
-### BEGIN INIT INFO
-# Provides: mon_statd
-# Required-Start: $local_fs $remote_fs
-# Required-Stop: $local_fs $remote_fs
-# Should-Start: 
-# Should-Stop: 
-# Default-Start: 2 3 5
-# Default-Stop: 0 1 6
-# Short-Description: Configure the mon_fsstatd and mon_procd daemons.
-# Description: Configures the mon_fsstatd and mon_procd daemons. It uses the
-#              configuration file /etc/sysconfig/mon_statd.
-### END INIT INFO
-
-# chkconfig: 235 01 99
-
-DAEMON=mon_statd
-FSSTATD=mon_fsstatd
-PROCD=mon_procd
-FSSTATD_PATH=/usr/sbin/$FSSTATD
-PROCD_PATH=/usr/sbin/$PROCD
-CONFIG_FILE=/etc/sysconfig/$DAEMON
-FSSTATD_PID_FILE=/run/$FSSTATD.pid
-PROCD_PID_FILE=/run/$PROCD.pid
-
-# source function library
-. /lib/lsb/init-functions
-
-# Source config file
-if [ -f $CONFIG_FILE ]; then
-	. $CONFIG_FILE
-fi
-
-UDEVSETTLE=/usr/bin/udevadm
-if [ ! -e $UDEVSETTLE ]
-then
-	UDEVSETTLE=/sbin/udevsettle
-	UDEVSETTLE_CALL="$UDEVSETTLE --timeout=10"
-else
-	UDEVSETTLE_CALL="$UDEVSETTLE settle --timeout=10"
-fi
-
-load_kernel_module()
-{
-	if [ ! -e /dev/monwriter ]; then
-		echo "Loading monwriter module..."
-		modprobe monwriter 2>&1
-		if [ $? -ne 0 ]; then
-			exit 1
-		fi
-		if [ -e $UDEVSETTLE ]
-		then
-			$UDEVSETTLE_CALL
-		fi
-	fi
-}
-
-start_daemon()
-{
-	local daemon_name=$1
-	local daemon_interval=$2
-	local daemon_pid_file=$3
-	local daemon_path=$4
-
-	if [ ! -f $daemon_pid_file ]; then
-		load_kernel_module
-		echo -n "Starting $daemon_name:"
-		$daemon_path -i $daemon_interval
-		if [ $? -eq 0 ]; then
-			log_success_msg
-		else
-			log_failure_msg
-		fi
-	else
-		echo "$daemon_name (pid $(cat $daemon_pid_file)) is already running..."
-	fi
-}
-	
-start()
-{
-	if [ "$FSSTAT" = "yes" ]; then
-		start_daemon $FSSTATD $FSSTAT_INTERVAL $FSSTATD_PID_FILE \
-			$FSSTATD_PATH
-	fi
-
-	if [ "$PROC" = "yes" ]; then
-		start_daemon $PROCD $PROC_INTERVAL $PROCD_PID_FILE \
-			$PROCD_PATH
-	fi
-}
-
-stop_daemon()
-{
-	local daemon_name=$1
-	local daemon_pid_file=$2
-	local daemon_path=$3
-
-	echo -n "Stopping $daemon_name:"
-	if [ -f $daemon_pid_file ]; then
-		killproc $daemon_path -TERM
-		log_success_msg
-		rm -f $daemon_pid_file
-	else
-		log_failure_msg
-	fi
-}
-
-stop()
-{
-	if [ "$FSSTAT" = "yes" ]; then
-		stop_daemon $FSSTATD $FSSTATD_PID_FILE $FSSTATD_PATH
-	fi
-	if [ "$PROC" = "yes" ]; then
-		stop_daemon $PROCD $PROCD_PID_FILE $PROCD_PATH
-	fi
-}
-
-restart() {
-	stop
-	start
-}
-
-status_daemon()
-{
-	local daemon_name=$1
-	local daemon_pid_file=$2
-	local daemon_interval=$3
-
-	if [ ! -f $daemon_pid_file ]; then
-		echo "$daemon_name is not running."
-	else
-		echo "$daemon_name (pid $(cat $daemon_pid_file), interval: $daemon_interval) is running."
-	fi
-}
-
-status()
-{
-	status_daemon $FSSTATD $FSSTATD_PID_FILE $FSSTAT_INTERVAL
-	status_daemon $PROCD $PROCD_PID_FILE $PROC_INTERVAL
-}
-
-# How are we called?
-case "$1" in
-	start)
-		start
-		;;
-	stop)
-		stop
-		;;
-	status)
-		status
-		;;
-	restart|reload|force-reload)
-		restart
-		;;
-	*)
-		echo "Usage: $DAEMON {start|stop|status|restart|reload}"
-		exit 1
-esac
-
-exit 0

fdasd/fdasd.c

@@ -408,8 +408,10 @@ static int read_line(void)
 {
 	bzero(line_buffer, LINE_LENGTH);
 	line_ptr = line_buffer;
-	if (!fgets(line_buffer, LINE_LENGTH, stdin))
+	if (!fgets(line_buffer, LINE_LENGTH, stdin)) {
+		clearerr(stdin);
 		return 0;
+	}
 	while (*line_ptr && !isgraph(*line_ptr))
 		line_ptr++;
 

genprotimg/boot/Makefile

@@ -15,7 +15,8 @@ ALL_CFLAGS := $(NO_PIE_CFLAGS) -Os -g \
 	-fno-delete-null-pointer-checks -fno-stack-protector \
 	-fexec-charset=IBM1047 -m64 -mpacked-stack \
 	-mstack-size=4096 -mstack-guard=128 -msoft-float \
-	-Wall -Wformat-security -Wextra -Werror
+	-Wall -Wformat-security -Wextra -Werror \
+	-Wno-array-bounds
 
 FILES := stage3a.bin stage3b.bin stage3b_reloc.bin
 
@@ -74,11 +75,11 @@ stage3b_reloc.elf:
 
 %.elf:	%.o
 	case $* in \
-		stage3a) SFLAGS="$(NO_PIE_LINKFLAGS) -nostdlib -Wl,-T,stage3a.lds";; \
-		stage3b) SFLAGS="$(NO_PIE_LINKFLAGS) -nostdlib -Wl,-T,stage3b.lds";; \
-		stage3b_reloc) SFLAGS="$(NO_PIE_LINKFLAGS) -nostdlib -Wl,-estage3b_reloc_start,-Ttext,0";; \
+		stage3a) SFLAGS="-Wl,-T,stage3a.lds";; \
+		stage3b) SFLAGS="-Wl,-T,stage3b.lds";; \
+		stage3b_reloc) SFLAGS="-Wl,-estage3b_reloc_start,-Ttext,0";; \
 	esac; \
-	$(LINK) $$SFLAGS -m64 $(filter %.o, $^) -o $@
+	$(LINK) $$SFLAGS $(NO_PIE_LDFLAGS) -m64 -static -nostdlib $(filter %.o, $^) -o $@
 	@chmod a-x $@
 
 %.bin:	%.elf

genprotimg/boot/stage3b.c

@@ -28,13 +28,23 @@ static inline void __noreturn load_psw(struct psw_t psw)
 		;
 }
 
+static unsigned long get_kernel_cmdline_size(void)
+{
+	unsigned long size = *(volatile unsigned long *)MAX_COMMAND_LINE_SIZE;
+
+	if (size != 0)
+		return size;
+
+	return LEGACY_COMMAND_LINE_SIZE;
+}
+
 void __noreturn start(void)
 {
 	volatile struct stage3b_args *args = &loader_parms;
 	volatile struct memblob *kernel = &args->kernel;
 	volatile struct memblob *cmdline = &args->cmdline;
 	volatile struct memblob *initrd = &args->initrd;
-	volatile struct psw_t psw = args->psw;
+	struct psw_t psw = args->psw;
 
 	/* set up ASCII and line-mode */
 	sclp_setup(SCLP_LINE_ASCII_INIT);
@@ -42,14 +52,14 @@ void __noreturn start(void)
 	if (kernel->size < IMAGE_LOAD_ADDRESS)
 		panic(EINTERNAL, "Invalid kernel\n");
 
-	if (cmdline->size > COMMAND_LINE_SIZE)
-		panic(EINTERNAL, "Command line is too large\n");
-
 	/* move the kernel and cut the kernel header */
 	memmove((void *)IMAGE_LOAD_ADDRESS,
 		(void *)(kernel->src + IMAGE_LOAD_ADDRESS),
 		kernel->size - IMAGE_LOAD_ADDRESS);
 
+	if (cmdline->size > get_kernel_cmdline_size())
+		panic(EINTERNAL, "Command line is too large\n");
+
 	/* move the kernel cmdline */
 	memmove((void *)COMMAND_LINE,
 		(void *)cmdline->src,

genprotimg/man/Makefile

@@ -4,8 +4,8 @@ include ../../common.mak
 all:
 
 install:
-	$(INSTALL) -d -m 755 $(DESTDIR)$(MANDIR)/man8
-	$(INSTALL) -m 644 -c genprotimg.8 $(DESTDIR)$(MANDIR)/man8
+	$(INSTALL) -d -m 755 $(DESTDIR)$(MANDIR)/man1
+	$(INSTALL) -m 644 -c genprotimg.1 $(DESTDIR)$(MANDIR)/man1
 
 clean:
 

genprotimg/man/genprotimg.1

@@ -2,7 +2,7 @@
 .\" s390-tools is free software; you can redistribute it and/or modify
 .\" it under the terms of the MIT license. See LICENSE for details.
 .\"
-.TH GENPROTIMG 8 "November 2020" "s390-tools"
+.TH GENPROTIMG 1 "May 2022" "s390-tools"
 .SH NAME
 genprotimg \- Create a protected virtualization image
 
@@ -62,7 +62,8 @@ host.
 Specifies the certificate that is used to establish a chain of trust
 for the verification of the host-key documents. Specify this option
 twice to specify the IBM Z signing key and the intermediate CA
-certificate (signed by the root CA). Required.
+certificate (signed by the root CA).
+Required. Ignored if \fB--no-verify\fP is specified.
 .TP
 \fB\-o\fR, \fB\-\-output\fR=\fI\,OUTPUT_FILE\/\fR
 Specifies the output file. Required.
@@ -87,12 +88,23 @@ CRLs. Optional.
 .TP
 \fB\-\-root\-ca\fR=\fI\,FILE\/\fR
 Specifies the root CA certificate for the verification. If omitted,
-the DigiCert root CA certificate installed on the system is used. Use
+the system wide root CAs installed on the system is used. Use
 this only if you trust the specified certificate. Optional.
 .TP
 \fB\-\-no-verify\fR
-Do not require the host-key documents to be valid. For testing
-purposes, do not use for a production image. Optional.
+Does not require the host-key documents to be valid.
+Do not use for a production image unless you verified
+the host-key document before. Optional.
+.TP
+\fB\-\-comm\-key\fR=\fI\,FILE\/\fR
+Specifies the encryption key you want to use for the PV guest dump. Use a
+secure, random, plaintext AES-256 GCM key. Optional.
+.TP
+\fB\-\-enable\-dump\fR
+Enable PV guest dumps. Requires the \fB\-\-comm-key\fR option. Optional.
+.TP
+\fB\-\-disable\-dump\fR
+Disable PV guest dumps. This is the default. Optional.
 .TP
 \fB\-\-enable\-pckmo\fR
 Enable the support for the DEA, TDEA, AES, and ECC PCKMO key encryption
@@ -105,21 +117,25 @@ functions. Optional.
 \fB\-v\fR, \fB\-\-version\fR
 Prints version information, then exits.
 
-.SH EXAMPLE
+.SH EXAMPLES
+
+These are examples to generate a protected virtualization image in
+\fI\,/boot/vmlinuz.pv\/\fR, using the kernel file \fI\,vmlinuz\/\fR, the
+initrd in \fI\,initramfs\/\fR, the kernel parameters contained in
+\fI\,parmfile\/\fR, the intermediate CA in \fI\,DigiCertCA.crt\/\fR, the
+IBM Z signing key in \fI\,ibm-z-host-key-signing.crt\/\fR, and the
+host-key document in \fI\,host_key.crt\/\fR. An AES-256 GCM key is stored in
+\fI\,comm-key\/\fR, which is used for the PV guest dump support in the second
+example.
+
+Generate a protected virtualization image:
+
 .PP
-Generate a protected virtualization image in
-\fI\,/boot/vmlinuz.pv\/\fR, using the kernel file \fI\,vmlinuz\/\fR,
-the initrd in \fI\,initramfs\/\fR, the kernel parameters contained in
-\fI\,parmfile\/\fR, the intermediate CA in \fI\,DigiCertCA.crt\/\fR,
-the IBM Z signing key in \fI\,ibm-z-host-key-signing.crt\/\fR, and the
-host-key document in \fI\,host_key.crt\/\fR:
-.PP
-.Vb 1
-.EX
-\&        genprotimg \-i \fI\,vmlinuz\/\fR \-r \fI\,initramfs\/\fR \-p \fI\,parmfile\/\fR \-k \fI\,host_key.crt\/\fR \-C \fI\,ibm-z-host-key-signing.crt\/\fR \-C \fI\,DigiCertCA.crt \-o \fI\,/boot/vmlinuz.pv\/\fR
-.EE
-.Ve
+.B genprotimg \-i \fI\,vmlinuz\/\fR \-r \fI\,initramfs\/\fR \-p \fI\,parmfile\/\fR \-k \fI\,host_key.crt\/\fR \-C \fI\,ibm-z-host-key-signing.crt\/\fR \-C \fI\,DigiCertCA.crt\fR \-o \fI\,/boot/vmlinuz.pv\/\fR
+
+Generate a protected virtualization image with PV guest dump support:
 .PP
+.B genprotimg \-i \fI\,vmlinuz\/\fR \-r \fI\,initramfs\/\fR \-p \fI\,parmfile\/\fR \-k \fI\,host_key.crt\/\fR \-C \fI\,ibm-z-host-key-signing.crt\/\fR \-C \fI\,DigiCertCA.crt\fR \-o \fI\,/boot/vmlinuz.pv\/\fR \-\-enable\-dump \-\-comm\-key \fI\,comm-key\fR
 
 .SH NOTES
 .IP "1." 4

genprotimg/samples/check_hostkeydoc

@@ -23,6 +23,7 @@ BODY_FILE=$(mktemp)
 ISSUER_DN_FILE=$(mktemp)
 SUBJECT_DN_FILE=$(mktemp)
 DEF_ISSUER_DN_FILE=$(mktemp)
+CANONICAL_ISSUER_DN_FILE=$(mktemp)
 CRL_SERIAL_FILE=$(mktemp)
 
 # Cleanup on exit
@@ -30,7 +31,7 @@ cleanup()
 {
     rm -f $ISSUER_PUBKEY_FILE $SIGNATURE_FILE $BODY_FILE \
         $ISSUER_DN_FILE $SUBJECT_DN_FILE $DEF_ISSUER_DN_FILE \
-        $CRL_SERIAL_FILE
+        $CANONICAL_ISSUER_DN_FILE $CRL_SERIAL_FILE
 }
 trap cleanup EXIT
 
@@ -47,13 +48,21 @@ set -e
 usage()
 {
 cat <<-EOF
-Usage: `basename $1` host-key-doc signing-key-cert [-c CA-cert] [-r CRL]
+Usage: `basename $1` [-d] [-c CA-cert] [-r CRL] host-key-doc signing-key-cert
 
 Verify an IBM Secure Execution host key document against
 a signing key.
 
+Options:
+-d         disable default issuer check of host-key-doc
+-c CA-cert trusted CA certificate
+-r CRL     list of revoked host-key-docs
+
 Note that in order to have the full trust chain verified
-it is necessary to provide the issueing CA's certificate.
+it is necessary to provide the issuing CA's certificate.
+The default issuer check may be disabled if a non-default
+signing key certificate needs to be verified against the
+CA certificate.
 
 EOF
 }
@@ -121,20 +130,34 @@ default_issuer()
     commonName                = International Business Machines Corporation
     countryName               = US
     localityName              = Poughkeepsie
-    organizationalUnitName    = IBM Z Host Key Signing Service
+    organizationalUnitName    = Key Signing Service
     organizationName          = International Business Machines Corporation
     stateOrProvinceName       = New York
 EOF
 }
 
-verify_issuer_files()
+# As organizationalUnitName can have an arbitrary prefix but must
+# end with "Key Signing Service" let's normalize the OU name by
+# stripping off the prefix
+verify_default_issuer()
 {
     default_issuer > $DEF_ISSUER_DN_FILE
 
-    if ! diff $ISSUER_DN_FILE $DEF_ISSUER_DN_FILE
+    sed "s/\(^[ ]*organizationalUnitName[ ]*=[ ]*\).*\(Key Signing Service$\)/\1\2/" \
+	$ISSUER_DN_FILE > $CANONICAL_ISSUER_DN_FILE
+
+    if ! diff $CANONICAL_ISSUER_DN_FILE $DEF_ISSUER_DN_FILE
     then
         echo Incorrect default issuer >&2 && exit 1
     fi
+}
+
+verify_issuer_files()
+{
+    if [ $1 -eq 1  ]
+    then
+       verify_default_issuer
+    fi
 
     if diff $ISSUER_DN_FILE $SUBJECT_DN_FILE
     then
@@ -196,14 +219,16 @@ check_file()
 # check args
 CRL_FILE=
 CA_FILE=
+CHECK_DEFAULT_ISSUER=1
 
-args=$(getopt -qu "r:c:h" $*)
+args=$(getopt -qu "dr:c:h" $*)
 if [ $? = 0 ]
 then
     set -- $args
     while [ $1 != "" ]
     do
         case $1 in
+            -d) CHECK_DEFAULT_ISSUER=0; shift;;
             -r) CRL_FILE=$2; shift 2;;
             -c) CA_FILE=$2; shift 2;;
             -h) usage $0; exit 0;;
@@ -244,7 +269,7 @@ exit 1
 # Verify the issuer
 canonical_dn x509 $HKD_FILE issuer $ISSUER_DN_FILE
 canonical_dn x509 $HKSK_FILE subject $SUBJECT_DN_FILE
-verify_issuer_files
+verify_issuer_files $CHECK_DEFAULT_ISSUER
 
 # Verify dates
 verify_dates $(cert_time $HKD_FILE startdate) $(cert_time $HKD_FILE enddate)
@@ -261,7 +286,7 @@ then
     echo -n "CRL "
     canonical_dn crl $CRL_FILE issuer $ISSUER_DN_FILE
     canonical_dn x509 $HKSK_FILE subject $SUBJECT_DN_FILE
-    verify_issuer_files
+    verify_issuer_files $CHECK_DEFAULT_ISSUER
 
     verify_dates $(crl_time $CRL_FILE lastupdate) $(crl_time $CRL_FILE nextupdate) 'CRL'
 

genprotimg/src/common.h

@@ -1,3 +1,10 @@
+/*
+ * Copyright IBM Corp. 2020, 2022
+ *
+ * s390-tools is free software; you can redistribute it and/or modify
+ * it under the terms of the MIT license. See LICENSE for details.
+ */
+
 #ifndef COMMON_H
 #define COMMON_H
 

genprotimg/src/include/pv_crypto_def.h

@@ -29,9 +29,6 @@
  */
 #define PV_CERTS_SECURITY_LEVEL 2
 
-/* SKID for DigiCert Assured ID Root CA */
-#define DIGICERT_ASSURED_ID_ROOT_CA_SKID "45EBA2AFF492CB82312D518BA7A7219DF36DC80F"
-
 union ecdh_pub_key {
 	struct {
 		uint8_t x[80];

genprotimg/src/include/pv_hdr_def.h

@@ -24,11 +24,15 @@
 #define PV_MAGIC_NUMBER 0x49424d5365634578ULL
 #define PV_VERSION_1	0x00000100U
 
+/* Internal helper macro */
+#define __PV_BIT(nr)		(1ULL << (63 - (nr)))
+
 /* Plaintext control flags */
-#define PV_PCF_PCKM_ECC		(1ULL << 5)  /* PCKMO encrypt-ECC-key functions allowed */
-#define PV_PCF_PCKMO_AES	(1ULL << 6)  /* PCKMO encrypt-AES-key functions allowed */
-#define PV_PCF_PCKMO_DEA_TDEA	(1ULL << 7)  /* PCKMO encrypt-DEA/TDEA-key functions allowed */
-#define PV_PCF_NO_DECRYPTION	(1ULL << 28) /* prevent Ultravisor decryption during unpack operation */
+#define PV_PCF_ALLOW_DUMPING	__PV_BIT(34) /* dumping of the configuration is allowed */
+#define PV_PCF_NO_DECRYPTION	__PV_BIT(35) /* prevent Ultravisor decryption during unpack operation */
+#define PV_PCF_PCKMO_DEA_TDEA	__PV_BIT(56) /* PCKMO encrypt-DEA/TDEA-key functions allowed */
+#define PV_PCF_PCKMO_AES	__PV_BIT(57) /* PCKMO encrypt-AES-key functions allowed */
+#define PV_PCF_PCKM_ECC	__PV_BIT(58) /* PCKMO encrypt-ECC-key functions allowed */
 
 /* maxima for the PV version 1 */
 #define PV_V1_IPIB_MAX_SIZE	PAGE_SIZE

genprotimg/src/pv/pv_args.c

@@ -64,13 +64,17 @@ static gint pv_args_validate_options(PvArgs *args, GError **err)
 {
 	PvComponentType KERNEL = PV_COMP_TYPE_KERNEL;
 
-	if (args->pcf && args->allow_pckmo != PV_NOT_SET) {
-		g_set_error(err, PV_PARSE_ERROR, PV_PARSE_ERROR_SYNTAX,
-			    _("The '--x-pcf' and '--(enable|disable)-pckmo' options are mutually"
-			      " exclusive.\nUse 'genprotimg --help' for more information"));
+	/* Check for mutually exclusive arguments */
+	if (args->pcf && !(args->allow_pckmo == PV_NOT_SET &&
+			   args->allow_dump == PV_NOT_SET)) {
+		g_set_error(
+			err, PV_PARSE_ERROR, PV_PARSE_ERROR_SYNTAX,
+			_("The '--x-pcf' option cannot be used with the '--(enable|disable)-pckmo' or"
+			  " '--(enable|disable)-dump' flags.\nUse 'genprotimg --help' for more information"));
 		return -1;
 	}
 
+	/* Check for unused arguments */
 	if (args->unused_values->len > 0) {
 		g_autofree gchar *unused = NULL;
 
@@ -88,6 +92,14 @@ static gint pv_args_validate_options(PvArgs *args, GError **err)
 		return -1;
 	}
 
+	/* Check for mandatory arguments */
+	if (args->allow_dump == PV_TRUE && !args->cust_comm_key_path) {
+		g_set_error(err, PV_PARSE_ERROR, PR_PARSE_ERROR_MISSING_ARGUMENT,
+			    _("Option '--allow-dump' requires the '--comm-key' option.\nUse 'genprotimg "
+			      "--help' for more information"));
+		return -1;
+	}
+
 	if (!args->output_path) {
 		g_set_error(err, PV_PARSE_ERROR, PR_PARSE_ERROR_MISSING_ARGUMENT,
 			    _("Option '--output' is required.\nUse 'genprotimg --help' for more information"));
@@ -111,7 +123,7 @@ static gint pv_args_validate_options(PvArgs *args, GError **err)
 	     g_strv_length(args->untrusted_cert_paths) == 0)) {
 		g_set_error(
 			err, PV_PARSE_ERROR, PR_PARSE_ERROR_MISSING_ARGUMENT,
-			_("Either specify the IBM Z signing key and (DigiCert) intermediate CA certificate\n"
+			_("Either specify the IBM Z signing key and intermediate CA certificate\n"
 			  "by using the '--cert' option, or use the '--no-verify' flag to disable the\n"
 			  "host-key document verification completely (at your own risk)."));
 		return -1;
@@ -155,14 +167,14 @@ static gboolean cb_set_string_option(const gchar *option, const gchar *value,
 {
 	gchar **args_option = NULL;
 
+	if (g_str_equal(option, "--comm-key"))
+		args_option = &args->cust_comm_key_path;
 	if (g_str_equal(option, "--root-ca"))
 		args_option = &args->root_ca_path;
 	if (g_str_equal(option, "-o") || g_str_equal(option, "--output"))
 		args_option = &args->output_path;
 	if (g_str_equal(option, "--x-comp-key"))
 		args_option = &args->xts_key_path;
-	if (g_str_equal(option, "--x-comm-key"))
-		args_option = &args->cust_comm_key_path;
 	if (g_str_equal(option, "--x-header-key"))
 		args_option = &args->cust_root_key_path;
 	if (g_str_equal(option, "--x-pcf"))
@@ -188,19 +200,6 @@ static gboolean cb_set_string_option(const gchar *option, const gchar *value,
 	return TRUE;
 }
 
-static gboolean cb_enable_disable_flag(const gchar *option, const gchar *value G_GNUC_UNUSED,
-				       PvArgs *args, GError **err G_GNUC_UNUSED)
-{
-	if (g_str_equal(option, "--enable-pckmo"))
-		args->allow_pckmo = PV_TRUE;
-	else if (g_str_equal(option, "--disable-pckmo"))
-		args->allow_pckmo = PV_FALSE;
-	else
-		g_assert_not_reached();
-
-	return TRUE;
-}
-
 static gboolean cb_set_log_level(const gchar *option G_GNUC_UNUSED,
 				 const gchar *value G_GNUC_UNUSED, PvArgs *args,
 				 GError **err G_GNUC_UNUSED)
@@ -217,8 +216,51 @@ static gboolean cb_remaining_values(const gchar *option G_GNUC_UNUSED,
 	return TRUE;
 }
 
+#define MUT_EXCL_BOOL_FLAG_CB_NAME(FLAG, VALUE) (cb_##FLAG##_##VALUE)
+#define DEFINE_MUT_EXCL_BOOL_FLAG_CB(FLAG, VALUE)                                        \
+	static gboolean MUT_EXCL_BOOL_FLAG_CB_NAME(FLAG, VALUE)(                         \
+		const gchar *option G_GNUC_UNUSED, const gchar *value G_GNUC_UNUSED,     \
+		PvArgs *args, GError **err)                                              \
+	{                                                                                \
+		if (!(args->allow_##FLAG == PV_NOT_SET ||                                \
+		      args->allow_##FLAG == VALUE)) {                                    \
+			g_set_error(err, G_OPTION_ERROR, G_OPTION_ERROR_FAILED,          \
+				    "'--enable-" #FLAG "' and '--disable-" #FLAG         \
+				    "' are mutually exclusive");                         \
+			return FALSE;                                                    \
+		}                                                                        \
+		args->allow_##FLAG = VALUE;                                              \
+		return TRUE;                                                             \
+	}
+
+#define DEFINE_MUT_EXCL_BOOL_FLAG_CBS(FLAG)                                    \
+	DEFINE_MUT_EXCL_BOOL_FLAG_CB(FLAG, PV_TRUE)                            \
+	DEFINE_MUT_EXCL_BOOL_FLAG_CB(FLAG, PV_FALSE)
+
+#define MUT_EXCL_BOOL_FLAG(FLAG, ENABLE_DESC, DISABLE_DESC)                    \
+	{                                                                      \
+		.long_name = "enable-" #FLAG,                                  \
+		.short_name = 0,                                               \
+		.flags = G_OPTION_FLAG_NO_ARG,                                 \
+		.arg = G_OPTION_ARG_CALLBACK,                                  \
+		.arg_data = MUT_EXCL_BOOL_FLAG_CB_NAME(FLAG, PV_TRUE),         \
+		.description = ENABLE_DESC,                                    \
+	},                                                                     \
+	{                                                                      \
+		.long_name = "disable-" #FLAG,                                 \
+		.short_name = 0,                                               \
+		.flags = G_OPTION_FLAG_NO_ARG,                                 \
+		.arg = G_OPTION_ARG_CALLBACK,                                  \
+		.arg_data = MUT_EXCL_BOOL_FLAG_CB_NAME(FLAG, PV_FALSE),        \
+		.description = DISABLE_DESC,                                   \
+	}
+
 #define INDENT "                                   "
 
+/* Define the callbacks for mutually exclusive command line flags */
+DEFINE_MUT_EXCL_BOOL_FLAG_CBS(dump)
+DEFINE_MUT_EXCL_BOOL_FLAG_CBS(pckmo)
+
 gint pv_args_parse_options(PvArgs *args, gint *argc, gchar **argv[],
 			   GError **err)
 {
@@ -238,7 +280,8 @@ gint pv_args_parse_options(PvArgs *args, gint *argc, gchar **argv[],
 		  .arg_data = &args->host_keys,
 		  .description =
 			_("FILE specifies a host-key document. At least\n" INDENT
-			  "one is required."),
+			  "one is required Specify this option multiple times\n" INDENT
+			  "to enable the image to run on more than one host."),
 		  .arg_description = _("FILE") },
 		{ .long_name = "cert",
 		  .short_name = 'C',
@@ -282,21 +325,28 @@ gint pv_args_parse_options(PvArgs *args, gint *argc, gchar **argv[],
 		  .description = _("Use the kernel parameters stored in PARMFILE\n" INDENT
 				   "(optional)."),
 		  .arg_description = _("PARMFILE") },
-		{.long_name = "enable-pckmo",
-		 .short_name = 0,
-		 .flags = G_OPTION_FLAG_NO_ARG,
-		 .arg = G_OPTION_ARG_CALLBACK,
-		 .arg_data = cb_enable_disable_flag,
-		 .description = _("Enable the support for the DEA, TDEA, AES, and\n" INDENT
-				  "ECC PCKMO key encryption functions (default)\n" INDENT
-				  "(optional).")},
-		{.long_name = "disable-pckmo",
-		 .short_name = 0,
-		 .flags = G_OPTION_FLAG_NO_ARG,
-		 .arg = G_OPTION_ARG_CALLBACK,
-		 .arg_data = cb_enable_disable_flag,
-		 .description = _("Disable the support for the DEA, TDEA, AES, and\n" INDENT
-				  "ECC PCKMO key encryption functions (optional).")},
+		MUT_EXCL_BOOL_FLAG(
+			dump,
+			_("Enable PV guest dumps (optional). This option\n" INDENT
+			  "requires the '--comm-key' option."),
+			_("Disable PV guest dumps (default) (optional).")),
+		MUT_EXCL_BOOL_FLAG(
+			pckmo,
+			_("Enable the support for the DEA, TDEA, AES, and\n" INDENT
+			  "ECC PCKMO key encryption functions (default)\n" INDENT
+			  "(optional)."),
+			_("Disable the support for the DEA, TDEA, AES, and\n" INDENT
+			  "ECC PCKMO key encryption functions (optional).")),
+		{ .long_name = "comm-key",
+		  .short_name = 0,
+		  .flags = G_OPTION_FLAG_FILENAME,
+		  .arg = G_OPTION_ARG_CALLBACK,
+		  .arg_data = cb_set_string_option,
+		  .description = _(
+			  "FILE contains the key with which you encrypt\n" INDENT
+			  "the PV guest dump (optional). Required by\n" INDENT
+			  "the '--enable-dump' option."),
+		  .arg_description = _("FILE") },
 		{ .long_name = "crl",
 		  .short_name = 0,
 		  .flags = G_OPTION_FLAG_NONE,
@@ -355,15 +405,6 @@ gint pv_args_parse_options(PvArgs *args, gint *argc, gchar **argv[],
 	};
 
 	GOptionEntry x_entries[] = {
-		{ .long_name = "x-comm-key",
-		  .short_name = 0,
-		  .flags = G_OPTION_FLAG_FILENAME,
-		  .arg = G_OPTION_ARG_CALLBACK,
-		  .arg_data = cb_set_string_option,
-		  .description = _(
-			  "Use FILE as the customer communication key.\n" INDENT
-			  "Optional; default: auto-generated."),
-		  .arg_description = _("FILE") },
 		{ .long_name = "x-comp-key",
 		  .short_name = 0,
 		  .flags = G_OPTION_FLAG_FILENAME,
@@ -446,6 +487,7 @@ PvArgs *pv_args_new(void)
 	g_autoptr(PvArgs) args = g_new0(PvArgs, 1);
 
 	args->unused_values = g_ptr_array_new_with_free_func(g_free);
+	args->allow_dump = PV_NOT_SET;
 	args->allow_pckmo = PV_NOT_SET;
 	return g_steal_pointer(&args);
 }

genprotimg/src/pv/pv_args.h

@@ -34,6 +34,7 @@ typedef struct {
 	gboolean offline;
 	gchar *pcf;
 	gchar *scf;
+	PvTristate allow_dump;
 	PvTristate allow_pckmo;
 	gchar *psw_addr; /* PSW address which will be used for the start of
 			  * the actual component (e.g. Linux kernel)

genprotimg/src/pv/pv_comp.c

@@ -248,19 +248,6 @@ gint pv_component_align(PvComponent *component, const gchar *tmp_path,
 	g_assert_not_reached();
 }
 
-/* Convert uint64_t address to byte array */
-static void uint64_to_uint8_buf(uint8_t dst[8], uint64_t addr)
-{
-	uint8_t *p = (uint8_t *)&addr;
-
-	g_assert(dst);
-
-	for (gint i = 0; i < 8; i++) {
-		/* cppcheck-suppress objectIndex */
-		dst[i] = p[i];
-	}
-}
-
 int64_t pv_component_update_ald(const PvComponent *comp, EVP_MD_CTX *ctx,
 				GError **err)
 {
@@ -273,11 +260,8 @@ int64_t pv_component_update_ald(const PvComponent *comp, EVP_MD_CTX *ctx,
 
 	do {
 		uint64_t cur_be = GUINT64_TO_BE(cur);
-		uint8_t addr_buf[8];
 
-		uint64_to_uint8_buf(addr_buf, cur_be);
-
-		if (EVP_DigestUpdate(ctx, addr_buf, sizeof(addr_buf)) != 1) {
+		if (EVP_DigestUpdate(ctx, &cur_be, sizeof(cur_be)) != 1) {
 			g_set_error(err, PV_CRYPTO_ERROR,
 				    PV_CRYPTO_ERROR_INTERNAL,
 				    _("EVP_DigestUpdate failed"));

genprotimg/src/pv/pv_image.c

@@ -229,7 +229,9 @@ static gint pv_img_set_psw_addr(PvImage *img, const gchar *psw_addr_s,
 }
 
 static gint pv_img_set_control_flags(PvImage *img, const gchar *pcf_s,
-				     const gchar *scf_s, PvTristate allow_pckmo, GError **err)
+				     const gchar *scf_s,
+				     PvTristate allow_dump,
+				     PvTristate allow_pckmo, GError **err)
 {
 	uint64_t flags;
 
@@ -247,6 +249,11 @@ static gint pv_img_set_control_flags(PvImage *img, const gchar *pcf_s,
 		img->scf = flags;
 	}
 
+	if (allow_dump == PV_TRUE)
+		img->pcf |= PV_PCF_ALLOW_DUMPING;
+	else if (allow_dump == PV_FALSE)
+		img->pcf &= ~PV_PCF_ALLOW_DUMPING;
+
 	if (allow_pckmo == PV_TRUE)
 		img->pcf |= PV_PCF_PCKM_ECC | PV_PCF_PCKMO_AES | PV_PCF_PCKMO_DEA_TDEA;
 	else if (allow_pckmo == PV_FALSE)
@@ -304,9 +311,10 @@ static gint pv_img_hostkey_verify(GSList *host_key_certs,
 	}
 
 	/* Load all untrusted certificates (e.g. IBM Z signing key and
-	 * DigiCert intermediate CA) that are required to establish a chain of
-	 * trust starting from the host-key document up to the root CA (if not
-	 * otherwise specified that's the DigiCert Assured ID Root CA).
+	 * intermediate CA) that are required to establish a chain of trust
+	 * starting from the host-key document up to the root CA (if not
+	 * otherwise specified that can be one of the system wide installed
+	 * root CAs, e.g. DigiCert).
 	 */
 	untrusted_certs_with_path = load_certificates(untrusted_cert_paths, err);
 	if (!untrusted_certs_with_path)
@@ -341,9 +349,8 @@ static gint pv_img_hostkey_verify(GSList *host_key_certs,
 	 * For this we must check:
 	 *
 	 * 1. Can a chain of trust be established ending in a root CA
-	 * 2. Is the correct root CA ued? It has either to be the
-	 *    'DigiCert Assured ID Root CA' or the root CA specified via
-	 *    command line.
+	 * 2. Is the correct root CA used? It has either to be a system CA
+	 *    or the root CA specified via command line.
 	 */
 	for (gint i = 0; i < sk_X509_num(ibm_signing_certs); ++i) {
 		X509 *ibm_signing_cert = sk_X509_value(ibm_signing_certs, i);
@@ -364,17 +371,12 @@ static gint pv_img_hostkey_verify(GSList *host_key_certs,
 		if (verify_cert(ibm_signing_cert, ctx, err) < 0)
 			goto error;
 
-		/* Verify the build chain of trust chain. If the user passes a
-		 * trusted root CA on the command line then the check for the
-		 * Subject Key Identifier (SKID) is skipped, otherwise let's
-		 * check if the SKID meets our expectation.
+		/* If there is a chain of trust using either the provided root
+		 * CA on the command line or a system wide trusted root CA.
 		 */
-		if (!root_ca_path &&
-		    check_chain_parameters(X509_STORE_CTX_get0_chain(ctx),
-					   get_digicert_assured_id_root_ca_skid(),
-					   err) < 0) {
+		if (check_chain_parameters(X509_STORE_CTX_get0_chain(ctx),
+					   err) < 0)
 			goto error;
-		}
 
 		ibm_signing_crls = store_ctx_find_valid_crls(ctx, ibm_signing_cert, err);
 		if (!ibm_signing_crls) {
@@ -588,7 +590,7 @@ PvImage *pv_img_new(PvArgs *args, const gchar *stage3a_path, GError **err)
 		g_warning(_("host-key document verification is disabled. Your workload is not secured."));
 
 	if (args->root_ca_path)
-		g_warning(_("A different root CA than the default DigiCert root CA is selected. Ensure that this root CA is trusted."));
+		g_warning(_("The root CA is selected through the command line. Ensure that this root CA is trusted."));
 
 	ret->comps = pv_img_comps_new(EVP_sha512(), EVP_sha512(), EVP_sha512(), err);
 	if (!ret->comps)
@@ -608,7 +610,9 @@ PvImage *pv_img_new(PvArgs *args, const gchar *stage3a_path, GError **err)
 		return NULL;
 
 	/* set the control flags: PCF and SCF */
-	if (pv_img_set_control_flags(ret, args->pcf, args->scf, args->allow_pckmo, err) < 0)
+	if (pv_img_set_control_flags(ret, args->pcf, args->scf,
+				     args->allow_dump, args->allow_pckmo,
+				     err) < 0)
 		return NULL;
 
 	/* read in the keys */

genprotimg/src/utils/crypto.c

@@ -440,14 +440,14 @@ static int check_signature_algo_match(const EVP_PKEY *pkey, const X509 *subject,
 
 static X509_CRL *load_crl_from_bio(BIO *bio)
 {
-	g_autoptr(X509_CRL) crl = PEM_read_bio_X509_CRL(bio, NULL, 0, NULL);
+	g_autoptr(X509_CRL) crl = PEM_read_bio_X509_CRL(bio, NULL, NULL, NULL);
 	gint rc;
 
 	if (crl)
 		return g_steal_pointer(&crl);
 	ERR_clear_error();
 	rc = BIO_reset(bio);
-	if (rc != 1 || (rc != 0 && BIO_method_type(bio) == BIO_TYPE_FILE))
+	if (rc != 1 && !(rc == 0 && BIO_method_type(bio) == BIO_TYPE_FILE))
 		return NULL;
 
 	/* maybe the CRL is stored in DER format */
@@ -533,7 +533,7 @@ X509 *load_cert_from_file(const char *path, GError **err)
 		return g_steal_pointer(&cert);
 	ERR_clear_error();
 	rc = BIO_reset(bio);
-	if (rc != 1 || (rc != 0 && BIO_method_type(bio) == BIO_TYPE_FILE)) {
+	if (rc != 1 && !(rc == 0 && BIO_method_type(bio) == BIO_TYPE_FILE)) {
 		g_set_error(err, PV_CRYPTO_ERROR, PV_CRYPTO_ERROR_READ_CERTIFICATE,
 			    _("unable to load certificate: '%s'"), path);
 		return NULL;
@@ -1079,8 +1079,8 @@ int store_set_verify_param(X509_STORE *store, GError **err)
 		g_abort();
 
 	/* The maximum depth level of the chain of trust for the verification of
-	 * the IBM Z signing key is 2, i.e. IBM Z signing key -> (DigiCert)
-	 * intermediate CA -> (DigiCert) root CA
+	 * the IBM Z signing key is 2, i.e. IBM Z signing key -> intermediate CA
+	 * -> root CA
 	 */
 	X509_VERIFY_PARAM_set_depth(param, 2);
 
@@ -1267,46 +1267,38 @@ static int security_level_to_bits(int level)
 	return security_bits[level];
 }
 
-static ASN1_OCTET_STRING *digicert_assured_id_root_ca;
-
-const ASN1_OCTET_STRING *get_digicert_assured_id_root_ca_skid(void)
-{
-	pv_crypto_init();
-	return digicert_assured_id_root_ca;
-}
-
 /* Used for the caching of the downloaded CRLs */
 static GHashTable *cached_crls;
 
 void pv_crypto_init(void)
 {
-	if (digicert_assured_id_root_ca)
+	if (cached_crls)
 		return;
-
 	cached_crls = g_hash_table_new_full(g_str_hash, g_str_equal, g_free,
 					    (GDestroyNotify)X509_CRL_free);
-	digicert_assured_id_root_ca = s2i_ASN1_OCTET_STRING(
-		NULL, NULL, DIGICERT_ASSURED_ID_ROOT_CA_SKID);
 }
 
 void pv_crypto_cleanup(void)
 {
-	if (!digicert_assured_id_root_ca)
+	if (!cached_crls)
 		return;
 	g_clear_pointer(&cached_crls, g_hash_table_destroy);
-	g_clear_pointer(&digicert_assured_id_root_ca, ASN1_OCTET_STRING_free);
 }
 
 gint check_chain_parameters(const STACK_OF_X509 *chain,
-			    const ASN1_OCTET_STRING *skid, GError **err)
+			    GError **err)
 {
-	const ASN1_OCTET_STRING *ca_skid = NULL;
+	const X509_NAME *ca_x509_subject = NULL;
+	g_autofree gchar *ca_subject = NULL;
 	gint len = sk_X509_num(chain);
 	X509 *ca = NULL;
 
-	g_assert(skid);
 	/* at least one root and one leaf certificate must be defined */
-	g_assert(len >= 2);
+	if (len < 2) {
+		g_set_error(err, PV_CRYPTO_ERROR, PV_CRYPTO_ERROR_INTERNAL,
+			    _("there must be at least on root and one leaf certificate in the chain of trust"));
+		return -1;
+	}
 
 	/* get the root certificate of the chain of trust */
 	ca = sk_X509_value(chain, len - 1);
@@ -1316,19 +1308,21 @@ gint check_chain_parameters(const STACK_OF_X509 *chain,
 		return -1;
 	}
 
-	ca_skid = X509_get0_subject_key_id(ca);
-	if (!ca_skid) {
-		g_set_error(err, PV_CRYPTO_ERROR, PV_CRYPTO_ERROR_MALFORMED_ROOT_CA,
-			    _("malformed root certificate"));
+	ca_x509_subject = X509_get_subject_name(ca);
+	if (!ca_x509_subject) {
+		g_set_error(err, PV_CRYPTO_ERROR, PV_CRYPTO_ERROR_INTERNAL,
+			    _("subject of the root CA cannot be retrieved"));
 		return -1;
 	}
 
-	if (ASN1_STRING_cmp(ca_skid, skid) != 0) {
-		g_set_error(err, PV_CRYPTO_ERROR, PV_CRYPTO_ERROR_WRONG_CA_USED,
-			    _("expecting DigiCert root CA to be used"));
+	ca_subject = X509_NAME_oneline(ca_x509_subject, NULL, 0);
+	if (!ca_subject) {
+		g_set_error(err, PV_CRYPTO_ERROR, PV_CRYPTO_ERROR_INTERNAL,
+			    _("subject name of the root CA cannot be retrieved"));
 		return -1;
 	}
 
+	g_info("Root CA used: '%s'", ca_subject);
 	return 0;
 }
 

genprotimg/src/utils/crypto.h

@@ -125,7 +125,6 @@ int check_crl_valid_for_cert(X509_CRL *crl, X509 *cert,
 			     gint verify_flags, GError **err);
 void pv_crypto_init(void);
 void pv_crypto_cleanup(void);
-const ASN1_OCTET_STRING *get_digicert_assured_id_root_ca_skid(void);
 gint verify_host_key(X509 *host_key, GSList *issuer_pairs,
 		     gint verify_flags, int level, GError **err);
 X509 *load_cert_from_file(const char *path, GError **err);
@@ -138,8 +137,7 @@ X509_STORE *store_setup(const gchar *root_ca_path,
 int store_set_verify_param(X509_STORE *store, GError **err);
 X509_CRL *load_crl_by_cert(X509 *cert, GError **err);
 STACK_OF_X509_CRL *try_load_crls_by_certs(GSList *certs_with_path);
-gint check_chain_parameters(const STACK_OF_X509 *chain,
-			    const ASN1_OCTET_STRING *skid, GError **err);
+gint check_chain_parameters(const STACK_OF_X509 *chain, GError **err);
 X509_NAME *c2b_name(const X509_NAME *name);
 
 STACK_OF_X509 *delete_ibm_signing_certs(STACK_OF_X509 *certs);

hmcdrvfs/Makefile

@@ -16,17 +16,17 @@ check_dep:
 	$(call check_dep, \
 		"hmcdrvfs", \
 		"fuse.h", \
-		"fuse-devel or libfuse-dev", \
+		"fuse3-devel or libfuse3-dev", \
 		"HAVE_FUSE=0")
 
 ifneq ($(shell sh -c 'command -v pkg-config'),)
-FUSE_CFLAGS = $(shell pkg-config --silence-errors --cflags fuse)
-FUSE_LDLIBS = $(shell pkg-config --silence-errors --libs fuse)
+FUSE_CFLAGS = $(shell pkg-config --silence-errors --cflags fuse3)
+FUSE_LDLIBS = $(shell pkg-config --silence-errors --libs fuse3)
 else
-FUSE_CFLAGS = -D_FILE_OFFSET_BITS=64 -I/usr/include/fuse
-FUSE_LDLIBS = -lfuse
+FUSE_CFLAGS = -D_FILE_OFFSET_BITS=64 -I/usr/include/fuse3
+FUSE_LDLIBS = -lfuse3
 endif
-ALL_CFLAGS += -DFUSE_USE_VERSION=26 -D_LARGEFILE_SOURCE $(FUSE_CFLAGS)
+ALL_CFLAGS += -DFUSE_USE_VERSION=30 -D_LARGEFILE_SOURCE $(FUSE_CFLAGS)
 LDLIBS += $(FUSE_LDLIBS) -lpthread -lrt -ldl -lm
 
 OBJECTS = hmcdrvfs.o

hmcdrvfs/hmcdrvfs.1

@@ -111,9 +111,6 @@ allow access by other users
 .B -o allow_root
 allow access by root
 .TP
-.B -o nonempty
-allow mounts over non-empty file/dir
-.TP
 .B -o default_permissions
 enable permission checking by kernel
 .TP

hmcdrvfs/hmcdrvfs.c

@@ -990,7 +990,7 @@ static int hmcdrv_cache_dir(const char *dir, fuse_fill_dir_t filler, void *buf)
 				hmcdrv_cache_refresh(path, &st, symlink);
 
 				if ((filler != NULL) &&
-				    (filler(buf, fname, &st, 0) != 0))
+				    (filler(buf, fname, &st, 0, 0) != 0))
 					filler = NULL; /* stop filling */
 #ifdef DEBUG
 				strftime(symlink, sizeof(symlink),
@@ -1109,7 +1109,8 @@ static struct hmcdrv_fuse_file *hmcdrv_file_get(const char *path)
  *
  * Note: The most important function which FUSE calls (very often).
  */
-static int hmcdrv_fuse_getattr(const char *path, struct stat *stbuf)
+static int hmcdrv_fuse_getattr(const char *path, struct stat *stbuf,
+			       struct fuse_file_info *UNUSED(fi))
 {
 	struct hmcdrv_fuse_file *fp;
 	int rc = 0;
@@ -1175,12 +1176,13 @@ static int hmcdrv_fuse_opendir(const char *UNUSED(path),
  */
 static int hmcdrv_fuse_readdir(const char *path, void *buf,
 			       fuse_fill_dir_t filler, off_t UNUSED(offset),
-			       struct fuse_file_info *UNUSED(fi))
+			       struct fuse_file_info *UNUSED(fi),
+			       enum fuse_readdir_flags UNUSED(flags))
 {
 	int ret;
 
-	filler(buf, ".", NULL, 0);
-	filler(buf, "..", NULL, 0);
+	filler(buf, ".", NULL, 0, 0);
+	filler(buf, "..", NULL, 0, 0);
 
 	pthread_mutex_lock(&hmcdrv_ctx.mutex);
 	ret = hmcdrv_cache_dir(path, filler, buf);
@@ -1232,7 +1234,8 @@ static int hmcdrv_fuse_read(const char *path, char *buf, size_t size,
  * Return: value to be passed in the private_data field of fuse_context to
  *         all file operations and as a parameter to the destroy() method
  */
-static void *hmcdrv_fuse_init(struct fuse_conn_info *UNUSED(conn))
+static void *hmcdrv_fuse_init(struct fuse_conn_info *UNUSED(conn),
+			      struct fuse_config *UNUSED(cfg))
 {
 	pthread_mutexattr_t attr;
 

hsavmcore/Makefile

@@ -23,11 +23,11 @@ else # HAVE_FUSE
 # FUSE
 #
 ifneq ($(shell sh -c 'command -v pkg-config'),)
-  FUSE_CFLAGS = $(shell pkg-config --silence-errors --cflags fuse)
-  FUSE_LDLIBS = $(shell pkg-config --silence-errors --libs fuse)
+  FUSE_CFLAGS = $(shell pkg-config --silence-errors --cflags fuse3)
+  FUSE_LDLIBS = $(shell pkg-config --silence-errors --libs fuse3)
 else
-  FUSE_CFLAGS = -I/usr/include/fuse
-  FUSE_LDLIBS = -lfuse
+  FUSE_CFLAGS = -I/usr/include/fuse3
+  FUSE_LDLIBS = -lfuse3
 endif
 
 #
@@ -67,8 +67,9 @@ check-dep-fuse:
 	$(call check_dep, \
 		"hsavmcore", \
 		"fuse.h", \
-		"fuse-devel or libfuse-dev", \
-		"HAVE_FUSE=0")
+		"fuse3-devel or libfuse3-dev", \
+		"HAVE_FUSE=0", \
+		"-DFUSE_USE_VERSION=30")
 	touch check-dep-fuse
 
 install: all

hsavmcore/initramfs/fedora-rhel/README.md

@@ -15,7 +15,7 @@ sudo reboot
 ## Dependencies
 
 ```shell
-sudo dnf install -y fuse fuse-devel systemd-devel
+sudo dnf install -y fuse3 fuse3-devel systemd-devel
 ```
 
 ## Build hsavmcore

hsavmcore/initramfs/sles/README.md

@@ -18,7 +18,7 @@ sudo reboot
 ## Dependencies
 
 ```shell
-sudo zypper install -y fuse fuse-devel systemd-devel
+sudo zypper install -y fuse3 fuse3-devel systemd-devel
 ```
 
 ## Build hsavmcore

hsavmcore/initramfs/ubuntu/README.md

@@ -11,7 +11,7 @@
 ## Dependencies
 
 ```shell
-sudo apt-get install -y make gcc kdump-tools fuse libfuse-dev libsystemd-dev
+sudo apt-get install -y make gcc kdump-tools fuse3 libfuse3-dev libsystemd-dev
 ```
 
 ## Build hsavmcore

hsavmcore/overlay.c

@@ -15,7 +15,7 @@
 #include <fcntl.h>
 #include <unistd.h>
 
-#define FUSE_USE_VERSION 26
+#define FUSE_USE_VERSION 30
 #include <fuse.h>
 
 #include "lib/util_log.h"
@@ -31,8 +31,11 @@ struct vmcore_overlay {
 	bool fuse_debug;
 };
 
-static int vmcore_fuse_getattr(const char *path, struct stat *stbuf)
+static int vmcore_fuse_getattr(const char *path, struct stat *stbuf,
+			       struct fuse_file_info *fi)
 {
+	(void)fi;
+
 	struct vmcore_overlay *overlay = fuse_get_context()->private_data;
 	int ret = 0;
 
@@ -54,18 +57,20 @@ static int vmcore_fuse_getattr(const char *path, struct stat *stbuf)
 
 static int vmcore_fuse_readdir(const char *path, void *buf,
 			       fuse_fill_dir_t filler, off_t offset,
-			       struct fuse_file_info *fi)
+			       struct fuse_file_info *fi,
+			       enum fuse_readdir_flags flags)
 {
 	(void)offset;
 	(void)fi;
+	(void)flags;
 
 	if (strcmp(path, ROOT_DIR) != 0)
 		return -ENOENT;
 
 	/* We have only one file */
-	filler(buf, ".", NULL, 0);
-	filler(buf, "..", NULL, 0);
-	filler(buf, VMCORE_FILE, NULL, 0);
+	filler(buf, ".", NULL, 0, 0);
+	filler(buf, "..", NULL, 0, 0);
+	filler(buf, VMCORE_FILE, NULL, 0, 0);
 
 	return 0;
 }

hsci/hsci

@@ -11,11 +11,36 @@
 hsdev=""
 ndev=""
 hsci=""
-hsdev_mac=""
+hscibr=""
+hscibp=""
+hsci_mac=""
 hsif_pnetid=""
 netif_pnetid=""
 hsci_pnetid=""
 
+##############################################################################
+#  Concept:
+#	     --------
+#	     | hsci |
+#	     --------
+#	     /       \
+#	--------    --------
+#	| ndev |    | hsdev |
+#	--------    --------
+#
+# Detail (bridge w/ bridgeports):
+#	           --------
+#	           | hsci |
+#	           --------
+#	               |
+#	----------------------
+#	|          |hsci-bp| |
+#	| hsci-br  --------- |
+#	|                    |
+#	--------   --------  |
+#	| ndev |   | hsdev | |
+#	----------------------
+
 function usage {
 cat <<-EOD
 Usage: hsci COMMAND [OPTION]
@@ -42,50 +67,53 @@ function prereqs_check {
 	fi
 }
 
+function get_pnetid {
+	local netdev=$1
+	local pnetid=""
+
+	#### ROCE:
+	if [ -e /sys/class/net/$netdev/device/util_string ]; then
+		pnetids="$(cat /sys/class/net/$netdev/device/util_string | tr -d '\000' | iconv -f IBM-1047 -t ASCII)"
+	else
+	#### OSA /HiperSockets:
+		if [ -e /sys/class/net/$netdev/device/chpid ]; then
+			chpid="$(cat /sys/class/net/$netdev/device/chpid | tr [:upper:] [:lower:])"
+			pnetids="$(cat /sys/devices/css0/chp0.$chpid/util_string | tr -d '\000' | iconv -f IBM-1047 -t ASCII)"
+		fi
+	fi
+	if [ "$pnetids" != "" ]; then
+		port_if="$(cat /sys/class/net/$netdev/dev_port)"
+		(( idx=16*$port_if+1 ))
+		(( end=$idx+15 ))
+		pnetid="$(echo "$pnetids" | cut -c $idx-$end | tr -d ' ')"
+	fi
+
+	echo $pnetid
+}
+
 function check_pnetids {
-	# get PNETID of the HS
-	local hsif_pnetids=""
-	local netif_pnetids=""
-
-	if [ -e /sys/class/net/$hsdev/device/util_string ]; then
-		hsif_pnetids="$(cat /sys/class/net/$hsdev/device/util_string | tr -d '\000' | iconv -f IBM-1047 -t ASCII)"
+	if [ "$hsdev" != "" ]; then
+		hsif_pnetid="$(get_pnetid $hsdev)"
 	else
-		if [ -e /sys/class/net/$hsdev/device/chpid ]; then
-			chpid="$(cat /sys/class/net/$hsdev/device/chpid | tr [:upper:] [:lower:])"
-			hsif_pnetids="$(cat /sys/devices/css0/chp0.$chpid/util_string | tr -d '\000' | iconv -f IBM-1047 -t ASCII)"
-		fi
+		hsif_pnetid=""
 	fi
-	if [ "$hsif_pnetids" != "" ]; then
-		port_hsif="$(cat /sys/class/net/$hsdev/dev_port)"
-		(( idx=16*$port_hsif+1 ))
-		(( end=$idx+15 ))
-		hsif_pnetid="$(echo "$hsif_pnetids" | cut -c $idx-$end | tr -d ' ')"
-	fi
-
-	# get PNETID of the NET_DEV
-	if [ -e /sys/class/net/$ndev/device/util_string ]; then
-		netif_pnetids="$(cat /sys/class/net/$ndev/device/util_string | tr -d '\000' | iconv -f IBM-1047 -t ASCII)"
+	if [ "$ndev" != "" ]; then
+		netif_pnetid="$(get_pnetid $ndev)"
 	else
-		if [ -e /sys/class/net/$ndev/device/chpid ]; then
-			chpid="$(cat /sys/class/net/$ndev/device/chpid | tr [:upper:] [:lower:])"
-			netif_pnetids="$(cat /sys/devices/css0/chp0.$chpid/util_string | tr -d '\000' | iconv -f IBM-1047 -t ASCII)"
-		fi
-	fi
-	if [ "$netif_pnetids" != "" ]; then
-		port_netif="$(cat /sys/class/net/$ndev/dev_port)"
-		(( idx=16*$port_netif+1 ))
-		(( end=$idx+15 ))
-		netif_pnetid="$(echo "$netif_pnetids" | cut -c $idx-$end | tr -d ' ')"
+		netif_pnetid=""
 	fi
 
 	#Check PNETIDs
 	if [ "$hsif_pnetid" != "" ] && [ "$netif_pnetid" != "" ] && [ "$netif_pnetid" != "$hsif_pnetid" ]; then
 		echo "Error: $hsdev and $ndev have different PNETIDs! They are $hsif_pnetid and $netif_pnetid respectively" >&2
 		return 1
-	fi
-
-	if [ "$hsif_pnetid" != "" ] && [ "$netif_pnetid" != "" ] && [ "$netif_pnetid" == "$hsif_pnetid" ]; then
-		hsci_pnetid=$hsif_pnetid
+	else
+		if [ "$hsif_pnetid" != "" ]; then
+			hsci_pnetid=$hsif_pnetid
+		else
+			hsci_pnetid=$netif_pnetid
+		fi
+		return 0
 	fi
 }
 
@@ -96,11 +124,11 @@ function verify_precon {
 		echo "Error: $hsdev does not exist" >&2
 		return 1
 	fi
-	if [ "$(cat /sys/class/net/$hsdev/device/card_type)" != "HiperSockets" ]; then
+	if [ "$(cat /sys/class/net/$hsdev/device/card_type 2>/dev/null)" != "HiperSockets" ]; then
 		echo "Error: $hsdev is not a HiperSockets device" >&2
 		return 1
 	fi
-	if [ "$(cat /sys/class/net/$hsdev/device/layer2)" != "1" ]; then
+	if [ "$(cat /sys/class/net/$hsdev/device/layer2 2>/dev/null)" != "1" ]; then
 		echo "Error: $hsdev is not in layer 2 mode" >&2
 		return 1
 	fi
@@ -108,7 +136,7 @@ function verify_precon {
 		echo "Error: Missing vnic-characteristics support" >&2
 		return 1
 	fi
-	if [ "$(cat /sys/class/net/$hsdev/device/vnicc/bridge_invisible)" == "n/a" ]; then
+	if [ "$(cat /sys/class/net/$hsdev/device/vnicc/bridge_invisible 2>/dev/null)" == "n/a" ]; then
 		echo "Error: $hsdev does not support vnicc" >&2
 		return 1
 	fi
@@ -121,8 +149,10 @@ function verify_precon {
 		return 1
 	fi
 	if [ $(ip link show $hsdev | grep master | wc -l) -ne 0 ]; then
-		echo "Error: $hsdev is already a bridge port" >&2
-		return 1
+		if [ $(ip link show $hsdev | grep "master $hsci" | wc -l) -eq 0 ]; then
+			echo "Error: $hsdev is already a subordinate to another master" >&2
+			return 1
+		fi
 	fi
 
 	#Pre-verify net_dev
@@ -130,17 +160,15 @@ function verify_precon {
 		echo "Error: $ndev does not exist" >&2
 		return 1
 	fi
-	if [ "$(cat /sys/class/net/$ndev/device/card_type)" == "HiperSockets" ]; then
-		echo "Error: $ndev is also a HiperSockets device" >&2
-		return 1
-	fi
 	if [ $(ip link show $ndev | grep UP | wc -l) -eq 0 ]; then
 		echo "Error: $ndev is not in state UP" >&2
 		return 1
 	fi
 	if [ $(ip link show $ndev | grep master | wc -l) -ne 0 ]; then
-		echo "Error: $ndev is already a bridge port" >&2
-		return 1
+		if [ $(ip link show $ndev | grep "master $hsci" | wc -l) -eq 0 ]; then
+			echo "Error: $ndev is already a subordinate to another master" >&2
+			return 1
+		fi
 	fi
 
 	#Check PNETIDs
@@ -153,10 +181,12 @@ function verify_precon {
 }
 
 function clean_up {
+	ip link del $hsci >/dev/null 2>&1
 	bridge link set dev $hsdev learning_sync off self >/dev/null 2>&1
 	echo 0 > /sys/class/net/$hsdev/device/vnicc/bridge_invisible >/dev/null 2>&1
-	bridge fdb del $hsdev_mac dev $ndev >/dev/null 2>&1
-	ip link del $hsci >/dev/null 2>&1
+	bridge fdb del $hsci_mac dev $hsdev self local >/dev/null 2>&1
+	bridge fdb del $hsci_mac dev $ndev self local >/dev/null 2>&1
+	ip link del $hscibr >/dev/null 2>&1
 }
 
 ##############################################################################
@@ -172,22 +202,33 @@ function add_hsci {
 	hsdev=$1
 	ndev=$2
 
+	hsci_postfix="$(readlink /sys/class/net/$hsdev/device/cdev0 | tail -c5)"
+	hscibr=hsci$hsci_postfix-br
+	hscibp=hsci$hsci_postfix-bp
+	hsci=hsci$hsci_postfix
+
 	#### Verify preconditions
 	verify_precon
 	if [ $? -ne 0 ]; then
 		return 1
 	fi
 
-	hsci_postfix="$(readlink /sys/class/net/$hsdev/device/cdev0 | tail -c5)"
-	hsci=hsci$hsci_postfix
-
 	echo "Adding $hsci with a HiperSockets dev $hsdev and an external dev $ndev"
 
-	#### Create bridge
-	ip link add name $hsci type bridge stp_state 0 >/dev/null 2>&1
-	if [ $? -ne 0 ]; then
-		echo "Error: Could not create a bridge" >&2
-		return 1
+	#### Create bridge (idempotent)
+	if [ ! -e /sys/class/net/$hscibr ]; then
+		# ageing_time of $hscibr defaults to 30000 (300 secs)
+		ip link add name $hscibr type bridge stp_state 0 >/dev/null 2>&1
+		if [ $? -ne 0 ]; then
+			echo "Error: Could not create a bridge" >&2
+			return 1
+		fi
+	else
+		ip link set dev $hscibr type bridge stp_state 0 >/dev/null 2>&1
+		if [ $? -ne 0 ]; then
+			echo "Error: $hscibr is not a bridge" >&2
+			return 1
+		fi
 	fi
 
 	#### Prepare hsdev
@@ -196,19 +237,24 @@ function add_hsci {
 	echo 1 > /sys/class/net/$hsdev/device/vnicc/bridge_invisible
 
 	#### Create bridge ports
-	ip link set dev $ndev master $hsci >/dev/null 2>&1
+	ip link set dev $ndev master $hscibr >/dev/null 2>&1
 	if [ $? -ne 0 ]; then
 		echo "Error: Could not set master for $ndev" >&2
 		clean_up
 		return 1
 	fi
-	ip link set dev $hsdev master $hsci >/dev/null 2>&1
+	ip link set dev $hsdev master $hscibr >/dev/null 2>&1
 	if [ $? -ne 0 ]; then
 		echo "Error: Could not set master for $hsdev" >&2
 		clean_up
 		return 1
 	fi
 
+# Do not learn from ndev, but do learn from hsci-bp:
+#  - First define hsdev and ndev as _isolated_ bridgeports
+#  - Then turn on learning_sync on self on hsdev
+#  - Then define hsci-bp as non-isolated veth bridgeport
+
 	# no forwarding between ndev and hsdev -> isolated on
 	# ndev is default for outgoing unknown targets -> flood on
 	# no need to learn external LAN targets into fdb -> learning off
@@ -230,29 +276,102 @@ function add_hsci {
 		clean_up
 		return 1
 	fi
-
 	# NOTE: Although not required, BCs will be sent out on hsdev.
 	# NOTE: We need to receive BCs on hsdev, as z/OS HSCI does ARP requests on HS.
 
-	hsdev_mac="$(cat /sys/class/net/$hsdev/address)"
-	echo "Set $hsdev MAC $hsdev_mac on $ndev and $hsci"
 
-	# set HS MAC on OSA as secondary MAC
-	bridge fdb add $hsdev_mac dev $ndev >/dev/null 2>&1
+	ip link set dev $hscibr up >/dev/null 2>&1
 	if [ $? -ne 0 ]; then
-		echo "Error: Failed to set HS MAC on OSA as secondary MAC" >&2
+		echo "Error: Failed to set $hscibr up" >&2
 		clean_up
 		return 1
 	fi
 
-	# set HS MAC (common MAC) on HSCI as primary MAC
-	ip link set address $hsdev_mac dev $hsci >/dev/null 2>&1
+	#### Turn on device for bridge notification
+	####  Toggle is required to learn full list of HS targets,
+	####   not only future changes.
+	bridge link set dev $hsdev learning_sync off self >/dev/null 2>&1
+	bridge link set dev $hsdev learning_sync on self >/dev/null 2>&1
 	if [ $? -ne 0 ]; then
-		echo "Error: Failed to set HiperSockets MAC (common MAC) on HSCI as primary MAC" >&2
+		echo "Error: Failed to turn on device for bridge notification" >&2
 		clean_up
 		return 1
 	fi
 
+	# define veth pair for hsci (idempotent)
+	if [ ! -e /sys/class/net/$hsci ]; then
+		ip link add dev $hsci type veth peer name $hscibp >/dev/null 2>&1
+		if [ $? -ne 0 ]; then
+			echo "Error: Could not create veth pair $hsci - $hscibp " >&2
+			clean_up
+			return 1
+		fi
+	else
+		if [ $hsci@$hscibp: != "$(ip -o -d link show dev $hsci | awk '/veth/ {print $2}' )" ]; then
+			echo "Error: $hsci@$hscibp is not a veth" >&2
+			clean_up
+			return 1
+		fi
+	fi
+
+	ip link set dev $hscibp master $hscibr >/dev/null 2>&1
+	if [ $? -ne 0 ]; then
+		echo "Error: Failed to add $hscibp to $hscibr" >&2
+		clean_up
+		return 1
+	fi
+	bridge link set dev $hscibp isolated off learning on flood on mcast_flood on >/dev/null 2>&1
+	if [ $? -ne 0 ]; then
+		echo "Error: Failed to set bridge parameters for $hscibp" >&2
+		clean_up
+		return 1
+	fi
+
+	#### Set a static forwarding rule for hsci MAC, so hsci can be used as a
+	####  single-MAC network interface without being subject to
+	####  ageing and re-learning
+        #### Wait for systemd to change the MAC of hsci, if it wants to:
+        sleep 1
+	hsci_mac="$(cat /sys/class/net/$hsci/address)"
+	####  (idempotent)
+	if [ $(bridge fdb show dev $hscibp | grep "$hsci_mac master $hscibr static" | wc -l) -eq 0 ]; then
+		bridge fdb add $hsci_mac dev $hscibp master static
+		if [ $? -ne 0 ]; then
+			echo "Error: Failed to set $hsci_mac to $hscibr fdb" >&2
+			clean_up
+			return 1
+		fi
+	fi
+
+	# Bridge-to-device learning will set this MAC on hsdev and ndev.
+	# Old kernel code doesn't do hsci bridge-to-device learning.
+	#  In this case: Set hsci_mac as local MAC of hsdev and ndev,
+	#  so at least the single-MAC scenario works.
+	if [ $(bridge fdb show dev $hsdev | grep "$hsci_mac self permanent" | wc -l) -eq 0 ]; then
+		echo "Warning: $hsci will support only its current static MAC address. Please upgrade your kernel to the latest level." >&2
+		bridge fdb add $hsci_mac dev $hsdev self local
+		if [ $? -ne 0 ]; then
+			echo "Error: Failed to add $hsci_mac to $hsdev" >&2
+			clean_up
+			return 1
+		fi
+	fi
+	if [ $(bridge fdb show dev $ndev | grep "$hsci_mac self permanent" | wc -l) -eq 0 ]; then
+		bridge fdb add $hsci_mac dev $ndev self local
+		if [ $? -ne 0 ]; then
+			echo "Error: Failed to add $hsci_mac to $ndev" >&2
+			clean_up
+			return 1
+		fi
+	fi
+
+	#### Set veth pair to UP
+	ip link set dev $hscibp up >/dev/null 2>&1
+	if [ $? -ne 0 ]; then
+		echo "Error: Failed to set $hscibp up" >&2
+		clean_up
+		return 1
+	fi
 	ip link set dev $hsci up >/dev/null 2>&1
 	if [ $? -ne 0 ]; then
 		echo "Error: Failed to set $hsci up" >&2
@@ -260,14 +379,7 @@ function add_hsci {
 		return 1
 	fi
 
-	# Turn on device for bridge notification
-	bridge link set dev $hsdev learning_sync on self >/dev/null 2>&1
-	if [ $? -ne 0 ]; then
-		echo "Error: Failed to turn on device for bridge notification" >&2
-		clean_up
-		return 1
-	fi
-	echo "Successfully added HSCI interface $hsci"
+	echo "Added HSCI interface $hsci"
 	return 0
 }
 
@@ -286,51 +398,97 @@ function del_hsci {
 		echo "Error: $hsci does not exit" >&2
 		return 1
 	fi
-	if [ $(ip link show | grep "master $hsci" | wc -l) -eq 0 ]; then
-		echo "Error: $hsci is not an active HSCI interface" >&2
-		return 1
+	hsci_mac="$(cat /sys/class/net/$hsci/address)"
+
+	#### Find hscibp and hscibr
+	hscibp="$(ip -o link show dev $hsci | awk '{print $2}')"
+	if [[ $hscibp != *@* ]]; then
+		#  $hsci has no HSCI veth peer
+		echo "Warning: $hsci may have been created by an older version of hsci" >&2
+		mvp=1
+		hscibp=""
+		hscibr=$hsci
+	else
+		mvp=0
+		hscibp=${hscibp##*@}
+		hscibp=${hscibp%:}
+		echo "$hsci is paired with $hscibp" >&2
+
+		hscibr="$(ip link show dev $hscibp | awk '{for(x=1;x<NF;x++) if($x~/master/) print $(x+1)}')"
 	fi
 
-	bports="$(ip link show | grep "master $hsci" | awk '{print $2}')"
+	#### Find hsdev and ndev
+	hsdev=""
+	ndev=""
+	bports="$(ip link show | grep "master $hscibr" | awk '{print $2}')"
 	for bport in $bports; do
 		bport=${bport%:}
-			if [[ $bport == *@* ]]; then
-				bport=${bport%@*}
-			fi
-		if [ $(bridge -d link show dev $bport | grep "learning_sync on" | wc -l) -ne 0 ]; then
-			hsdev=$bport
-		else
-			ndev=$bport
+		bport=${bport%@*}
+		if [ $(bridge -d link show dev $bport | grep "isolated on" | wc -l) -ne 0 ]; then
+				if [ $(bridge -d link show dev $bport | grep "learning_sync on" | wc -l) -ne 0 ]; then
+					hsdev=$bport
+				else
+					ndev=$bport
+				fi
 		fi
 	done
 	if [ "$hsdev" == "" ]; then
 		echo "Error: $hsci has no active HiperSockets port" >&2
-		return 1
 	fi
-	echo "Deleting HSCI interface $hsci with the HiperSockets $hsdev and the external $ndev"
-
-	bridge link set dev $hsdev learning_sync off self >/dev/null 2>&1
-	if [ $? -ne 0 ]; then
-		echo "Error: Failed to turn off learning_sync on $hsdev" >&2
-		return 1
-	fi
-	echo 0 > /sys/class/net/$hsdev/device/vnicc/bridge_invisible
-
-	hsdev_mac="$(cat /sys/class/net/$hsdev/address)"
-	echo "Deleting $hsev MAC $hsdev_mac on $ndev"
-	bridge fdb del $hsdev_mac dev $ndev >/dev/null 2>&1
-	if [ $? -ne 0 ]; then
-		echo "Error: Failed to delete $hsev MAC $hsdev_mac on $ndev" >&2
-		return 1
+	if [ "$ndev" == "" ]; then
+		echo "Error: $hsci has no active external port" >&2
 	fi
+	echo "Deleting HSCI interface $hsci with HiperSockets interface $hsdev and external interface $ndev"
 
+	#### Delete veth before resetting learning_sync and deleting bridge,
+	####  so fdb entries are cleaned up (synced)!
 	ip link del $hsci >/dev/null 2>&1
 	if [ $? -ne 0 ]; then
 		echo "Error: Failed to delete $hsci" >&2
-		return 1
 	fi
-	echo "Successfully deleted device $hsci"
+	# Bridge-to-device learning will remove learned MACs from hsdev and ndev.
+	# Old kernel code doesn't do hsci bridge-to-device learning.
+	#  In this case: Remove the hsci_mac that was added during 'add' from
+	#  hsdev and ndev.
+	if  [ "$hsdev" != "" ] && [ $mvp -eq 0 ]; then
+		if [ $(bridge fdb show dev $hsdev | grep "$hsci_mac self permanent" | wc -l) -ne 0 ]; then
+			echo "Warning: It seems your kernel does not support all hsci features, please upgrade." >&2
+			bridge fdb del $hsci_mac dev $hsdev self local
+			if [ $? -ne 0 ]; then
+				echo "Error: Failed to delete $hsci_mac from $hsdev" >&2
+			fi
+		fi
+	fi
+	# In the mvp case hsci_mac == hs_mac and was only set on ndev.
+	if [ "$ndev" != "" ]; then
+		if [ $(bridge fdb show dev $ndev | grep "$hsci_mac self permanent" | wc -l) -ne 0 ]; then
+			bridge fdb del $hsci_mac dev $ndev self local
+			if [ $? -ne 0 ]; then
+				echo "Error: Failed to delete $hsci_mac from $ndev" >&2
+			fi
+		fi
+	fi
 
+	#### Reset learning_sync
+	if [ "$hsdev" != "" ]; then
+		bridge link set dev $hsdev learning_sync off self >/dev/null 2>&1
+			if [ $? -ne 0 ]; then
+				echo "Error: Failed to turn off learning_sync on $hsdev" >&2
+			fi
+	fi
+	#### Delete bridge
+	if [ "$hscibr" != "" ]; then
+		ip link del $hscibr >/dev/null 2>&1
+		if [ $? -ne 0 ]; then
+			echo "Error: Failed to delete $hscibr" >&2
+		fi
+	fi
+
+	if [ "$hsdev" != "" ]; then
+		echo 0 > /sys/class/net/$hsdev/device/vnicc/bridge_invisible
+	fi
+
+	echo "Deleted $hsci"
 	return 0
 }
 
@@ -338,19 +496,63 @@ function del_hsci {
 ##	Show HSCI
 ##############################################################################
 
-function list_active {
-	hsdev=$1
-	local ext=""
+function print_row {
+	if [ $mvp -eq 0 ]; then
+		printf '%-8s  %-16s  %-15s  %-15s\n' "$hsci" "$hsci_pnetid" "$hsdev" "$ndev"
+	else
+		printf '%-8s  %-16s  %-15s  %-15s    (v1)\n' "$hsci" "$hsci_pnetid" "$hsdev" "$ndev"
+	fi
+}
 
-	hsci="$(ip link show dev $hsdev | awk '{for(x=1;x<NF;x++) if($x~/master/) print $(x+1)}')"
-	ext="$(ip link show | grep "master $hsci" | grep --invert-match $hsdev | awk '{print $2}')"
-	# remove trailing ':'
-	ndev="${ext%:}"
+function list_active {
+	hsci=""
+	hsdev=$1
+	ndev=""
+	hscibp=""
+	mvp=1
+
+	hsci_pnetid="$(get_pnetid $hsdev)"
+
+	hscibr="$(ip link show dev $hsdev | awk '{for(x=1;x<NF;x++) if($x~/master/) print $(x+1)}')"
+	if [ "$hscibr" == "" ]; then
+		print_row
+		return 0
+	fi
+
+	#### find ndev (the other isolated subordinate)
+	bports="$(ip link show | grep "master $hscibr" | awk '{print $2}')"
+	for bport in $bports; do
+		bport=${bport%:}
+		bport=${bport%@*}
+		if [ $(ip -d link show dev $bport | grep "isolated on" | wc -l) -ne 0 ]; then
+			if [ $bport != $hsdev ]; then
+				ndev=$bport
+			fi
+		else
+			hscibp=$bport
+			mvp=0
+		fi
+	done
 
 	check_pnetids
 
-	printf '%-8s  %-16s  %-15s  %-15s\n' "$hsci" "$hsci_pnetid" "$hsdev" "$ndev"
+	if [ $mvp -ne 0 ]; then
+		hsci=$hscibr
+		print_row
+		return 0
+	fi
 
+	#### find hsci (veth-peer of non-isolated subordinate of hscibr)
+	hsci="$(ip -o link show dev $hscibp | awk '{print $2}')"
+	if [[ $hsci != *@* ]]; then
+		echo "Error: $hscibp has no HSCI veth peer" >&2
+		print_row
+		return 1
+	fi
+	hsci=${hsci##*@}
+	hsci=${hsci%:}
+
+	print_row
 	return 0
 }
 
@@ -358,13 +560,14 @@ function print_header {
 	if [ $header -eq 0 ]; then
 		echo "HSCI      PNET_ID           HiperSockets     External       "
 		echo "------------------------------------------------------------"
+		header=1
 	fi
 }
 
 function list_one {
 	local hsnetdev=$1
 
-	if [ $(bridge -d link show dev $hsnetdev | grep	 "learning_sync on" | wc -l) -ne 0 ]; then
+	if [ $(bridge -d link show dev $hsnetdev 2>/dev/null | grep "learning_sync on" | wc -l) -ne 0 ]; then
 		print_header
 		list_active $hsnetdev
 	fi
@@ -433,4 +636,3 @@ while true; do
 	esac
 	shift
 done
-

hyptop/sd.h

@@ -17,7 +17,7 @@
 #include "helper.h"
 #include "table.h"
 
-#define SD_DG_INIT_INTERVAL_MS	200
+#define SD_DG_INIT_INTERVAL_SEC	1
 #define SD_SYS_ID_SIZE		9
 
 /*

hyptop/sd_core.c

@@ -150,7 +150,7 @@ void sd_update(void)
  */
 void sd_dg_register(struct sd_dg *dg, int has_core_data)
 {
-	struct timespec ts = {0, SD_DG_INIT_INTERVAL_MS * 1000000};
+	struct timespec ts = {SD_DG_INIT_INTERVAL_SEC, 0};
 	struct sd_sys_item *sys_item;
 	struct sd_cpu_item *cpu_item;
 	unsigned int i;

include/boot/boot_defs.h

@@ -0,0 +1,185 @@
+/*
+ * Boot and dump related definitions
+ *
+ * Copyright IBM Corp. 2022
+ *
+ * s390-tools is free software; you can redistribute it and/or modify
+ * it under the terms of the MIT license. See LICENSE for details.
+ */
+#ifndef BOOT_DEFS_H
+#define BOOT_DEFS_H
+
+#include <stdint.h>
+
+#include "lib/zt_common.h"
+
+#define ZIPL_MAGIC			"zIPL"
+#define ZIPL_MAGIC_SIZE			4
+
+/*
+ * ECKD dump parameter
+ */
+struct eckd_dump_param {
+	uint32_t blk_start;
+	uint32_t blk_end;
+	uint16_t blk_size;
+	uint8_t num_heads;
+	uint8_t bpt;
+	char reserved[4];
+} __packed;
+
+/*
+ * FBA dump parameter
+ */
+struct fba_dump_param {
+	uint32_t	res1;
+	uint32_t	blk_start;
+	uint32_t	res2;
+	uint32_t	blk_end;
+} __packed;
+
+/*
+ * SCSI dump parameter
+ */
+struct scsi_dump_param {
+	uint64_t block;
+	uint64_t reserved;
+} __packed;
+
+/*
+ * Layout of block pointer for linear devices
+ * e.g. SCSI
+ */
+struct linear_blockptr {
+	uint64_t blockno;
+	uint16_t size;
+	uint16_t blockct;
+	uint8_t reserved[4];
+} __packed;
+
+/*
+ * Layout of block pointer for cylinder/head/sector devices
+ * e.g. ECKD
+ */
+struct eckd_blockptr {
+	uint16_t cyl;
+	uint16_t head;
+	uint8_t sec;
+	uint16_t size;
+	uint8_t blockct;
+	uint8_t reserved[8];
+} __packed;
+
+typedef enum {
+	COMPONENT_EXECUTE = 0x01,
+	COMPONENT_LOAD = 0x02,
+	COMPONENT_SIGNATURE = 0x03
+} component_type;
+
+typedef enum {
+	COMPONENT_HEADER_IPL = 0x00,
+	COMPONENT_HEADER_DUMP = 0x01
+} component_header_type;
+
+struct component_header {
+	uint8_t magic[4];
+	uint8_t type;
+	uint8_t reserved[27];
+} __packed;
+
+struct signature_header {
+	uint8_t format;
+	uint8_t reserved[3];
+	uint32_t length;
+} __packed;
+
+typedef union {
+	uint64_t load_address;
+	uint64_t load_psw;
+	struct signature_header sig_head;
+} component_data;
+
+struct component_entry {
+	uint8_t data[23];
+	uint8_t type;
+	component_data compdat;
+} __packed;
+
+/* SCSI dump super block */
+
+struct scsi_dump_sb {
+	uint64_t        magic;
+	uint64_t        version;
+	uint64_t        part_start;
+	uint64_t        part_size;
+	uint64_t        dump_offset;
+	uint64_t        dump_size;
+	uint64_t        csum_offset;
+	uint64_t        csum_size;
+	uint64_t        csum;
+} __packed;
+
+#define SCSI_DUMP_SB_MAGIC	0x5a46435044554d50ULL /* ZFCPDUMP */
+/* To avoid a csum entry of 0 a seed is used */
+#define SCSI_DUMP_SB_SEED	0x12345678
+#define SCSI_DUMP_SB_CSUM_SIZE  4096
+
+/* Boot info */
+
+#define BOOT_INFO_VERSION		1
+#define BOOT_INFO_MAGIC			"zIPL"
+
+#define BOOT_INFO_DEV_TYPE_ECKD		0x00
+#define BOOT_INFO_DEV_TYPE_FBA		0x01
+#define BOOT_INFO_DEV_TYPE_SCSI         0x02
+
+#define BOOT_INFO_BP_TYPE_IPL		0x00
+#define BOOT_INFO_BP_TYPE_DUMP		0x01
+
+#ifdef __s390x__
+#define BOOT_INFO_FLAGS_ARCH		0x01
+#else
+#define BOOT_INFO_FLAGS_ARCH		0x00
+#endif
+
+struct boot_info_bp_dump {
+	union {
+		struct eckd_dump_param eckd;
+		struct fba_dump_param fba;
+		struct scsi_dump_param scsi;
+	} param;
+	uint8_t		unused[16];
+} __packed;
+
+struct boot_info_bp_ipl {
+	union {
+		struct eckd_blockptr eckd;
+		struct linear_blockptr lin;
+	} bm_ptr;
+	uint8_t		unused[16];
+} __packed;
+
+struct boot_info {
+	char		magic[4];
+	uint8_t		version;
+	uint8_t		bp_type;
+	uint8_t		dev_type;
+	uint8_t		flags;
+	union {
+		struct boot_info_bp_dump dump;
+		struct boot_info_bp_ipl ipl;
+	} bp;
+} __packed;
+
+#define DISK_LAYOUT_ID 0x00000001
+
+struct scsi_mbr {
+	uint8_t			magic[4];
+	uint32_t		version_id;
+	uint8_t			reserved[8];
+	struct linear_blockptr	program_table_pointer;
+	uint8_t			reserved2[0x50];
+	struct boot_info 	boot_info;
+}  __packed;
+
+#endif /* BOOT_DEFS_H */

include/boot/linux_layout.h

@@ -23,10 +23,11 @@
 #define INITRD_SIZE		_AC(0x10410, UL)
 #define OLDMEM_BASE		_AC(0x10418, UL)
 #define OLDMEM_SIZE		_AC(0x10420, UL)
+#define MAX_COMMAND_LINE_SIZE	_AC(0x10430, UL)
 #define COMMAND_LINE		_AC(0x10480, UL)
 
 /* Parameter sizes */
-#define COMMAND_LINE_SIZE	896
+#define LEGACY_COMMAND_LINE_SIZE	896
 
 
 #ifndef __ASSEMBLER__

include/boot/s390.h

@@ -16,6 +16,8 @@
 #define __LC_IPLDEV		0x0c6c
 #define __LC_OS_INFO		0x0e18
 
+#define LOWCORE_SIZE		_AC(0x2000, UL)
+
 #define PAGE_SIZE		_AC(4096, UL)
 
 /* Minimum size of a stack frame in bytes */
@@ -56,6 +58,26 @@ struct psw32_t {
 
 void load_wait_psw(uint64_t, struct psw_t *);
 
+struct subchannel_id {
+	uint32_t cssid:8;
+	uint32_t:4;
+	uint32_t m:1;
+	uint32_t ssid:2;
+	uint32_t one:1;
+	uint32_t sch_no:16;
+} __packed __aligned(4);
+
+struct tpi_info {
+	struct subchannel_id schid;
+	uint32_t intparm;
+	uint32_t adapter_IO:1;
+	uint32_t directed_irq:1;
+	uint32_t isc:3;
+	uint32_t:12;
+	uint32_t type:3;
+	uint32_t:12;
+} __packed __aligned(4);
+
 struct _lowcore {
 	uint8_t	pad_0x0000[0x0014-0x0000];	/* 0x0000 */
 	uint32_t	ipl_parmblock_ptr;		/* 0x0014 */
@@ -78,10 +100,15 @@ struct _lowcore {
 	uint8_t	pad_0x00a4[0x00a8-0x00a4];	/* 0x00a4 */
 	uint64_t	trans_exc_code;			/* 0x00a8 */
 	uint64_t	monitor_code;			/* 0x00b0 */
-	uint16_t	subchannel_id;			/* 0x00b8 */
-	uint16_t	subchannel_nr;			/* 0x00ba */
-	uint32_t	io_int_parm;			/* 0x00bc */
-	uint32_t	io_int_word;			/* 0x00c0 */
+	union {
+		struct {
+			uint16_t subchannel_id;		/* 0x00b8 */
+			uint16_t subchannel_nr;		/* 0x00ba */
+			uint32_t io_int_parm;		/* 0x00bc */
+			uint32_t io_int_word;		/* 0x00c0 */
+		};
+		struct tpi_info tpi_info;		/* 0x00b8 */
+	};
 	uint8_t	pad_0x00c4[0x00c8-0x00c4];	/* 0x00c4 */
 	uint32_t	stfl_fac_list;			/* 0x00c8 */
 	uint8_t	pad_0x00cc[0x00e8-0x00cc];	/* 0x00cc */
@@ -210,6 +237,7 @@ struct _lowcore {
 	/* align to the top of the prefix area */
 	uint8_t	pad_0x1900[0x2000-0x1900];	/* 0x1900 */
 } __packed __aligned(8192);
+STATIC_ASSERT(sizeof(struct _lowcore) == LOWCORE_SIZE)
 
 #define S390_lowcore (*((struct _lowcore *) 0))
 

include/lib/ap.h

@@ -0,0 +1,94 @@
+/*
+ * libap - A collection of tools for ap/vfio-ap management
+ *
+ * Copyright IBM Corp. 2022
+ *
+ * s390-tools is free software; you can redistribute it and/or modify
+ * it under the terms of the MIT license. See LICENSE for details.
+ */
+
+#ifndef LIB_AP_H
+#define LIB_AP_H
+
+#include <stdbool.h>
+
+#include "lib/util_list.h"
+
+#define VFIO_AP_PATH "/sys/devices/vfio_ap"
+#define VFIO_AP_PARENT_PATH "devices/vfio_ap/matrix"
+#define VFIO_AP_CONFIG_PATH "/etc/mdevctl.d/matrix"
+#define VFIO_AP_TYPE "vfio_ap-passthrough"
+#define AP_UDEV_FILE "/etc/udev/rules.d/41-ap.rules"
+#define AP_LOCKFILE "/run/lock/s390apconfig.lock"
+
+#define AP_LOCK_RETRIES 15
+
+/* apmask and aqmask are each represented as 67 character strings with:
+ *  '0x' leading characters
+ *  64 hex digits (to represent 256 bits)
+ *  terminating character
+ */
+#define AP_MASK_SIZE 67
+#define AP_MAX_MASK_VALUE 255
+
+/* List structure used for keeping track of lists of adapter/domain IDs */
+struct vfio_ap_node {
+	struct util_list_node node;	/* prev/next list info */
+	unsigned int id;		/* list entry (adapter, domain, etc) */
+};
+
+/*
+ * Structure used to represent a vfio_ap-passthrough device configuration.
+ * The list of adapters and domains can be used to derive the APQNs for the
+ * device.  The type value is used to verify that the device (when read from
+ * a mdevctl config file) is of type vfio_ap-passthrough.  The manual value
+ * represents whether the device is started on-demand after boot (true) or
+ * automatically during boot (false).
+ */
+struct vfio_ap_device {
+	char *uuid;			/* Unique ID for this mdev */
+	struct util_list *adapters;	/* List of adapters for device */
+	struct util_list *domains;	/* List of usage domains for device */
+	struct util_list *controls;	/* List of control domains for device */
+	char *type;			/* mdev type string */
+	bool manual;			/* manual/auto start setting for mdev */
+};
+
+/* General Utility Functions */
+void print_ap_device(struct vfio_ap_device *dev);
+bool is_valid_uuid(const char *uuid);
+int ap_test_bit(int n, const char *hexbytestr);
+void ap_set_bit(int n, char *hexbytestr, bool val);
+
+/* Path-related functions */
+char *path_get_vfio_ap_mdev(const char *uuid);
+char *path_get_vfio_ap_mdev_config(const char *uuid);
+char *path_get_vfio_ap_attr(const char *uuid, const char *attr);
+char *path_get_ap_udev(void);
+
+/* Functions for manipulating sysfs to read active device info */
+void vfio_ap_parse_matrix(struct vfio_ap_device *dev, char *matrix);
+void vfio_ap_sort_matrix_results(struct vfio_ap_device *dev);
+void vfio_ap_parse_control(struct vfio_ap_device *dev, char *control);
+
+/* Functions for reading JSON device config */
+int vfio_ap_read_device_config(const char *path, struct vfio_ap_device *dev);
+
+/* Functions for managing vfio_ap device structures */
+struct vfio_ap_device *vfio_ap_device_new(void);
+void vfio_ap_device_clear(struct vfio_ap_device *dev);
+void vfio_ap_device_free(struct vfio_ap_device *dev);
+
+/* Functions for acquiring current vfio_ap device info */
+int ap_read_sysfs_masks(char *ap, char *aq, int size);
+bool ap_read_udev_masks(char *path, char *ap, char *aq, bool *read_ap,
+			bool *read_aq);
+void ap_mask_to_list(char *mask, struct util_list *list);
+void ap_list_remove_all(struct util_list *list);
+
+/* Lock Functions */
+int ap_get_lock(void);
+int ap_get_lock_callout(void);
+int ap_release_lock(void);
+
+#endif /* LIB_AP_H */

include/lib/libcpumf.h

@@ -0,0 +1,166 @@
+/* Copyright IBM Corp. 2022
+ *
+ * s390-tools is free software; you can redistribute it and/or modify
+ * it under the terms of the MIT license. See LICENSE for details.
+ */
+
+#ifndef LIBCPUMF_H
+#define LIBCPUMF_H
+
+#include <sched.h>
+#include <stdbool.h>
+
+#define	S390_CPUMF_CF		"/sys/devices/cpum_cf/"
+#define	S390_CPUMF_CFDIAG	"/sys/devices/cpum_cf_diag/"
+#define	S390_CPUMF_SF		"/sys/devices/cpum_sf/"
+#define S390_CPUS_POSSIBLE	"/sys/devices/system/cpu/possible"
+#define S390_CPUS_ONLINE	"/sys/devices/system/cpu/online"
+#define S390_CPUMSF_BUFFERSZ	"/sys/module/kernel/parameters/cpum_sfb_size"
+#define S390_SYSFS_PAI_CRYPTO	"/sys/devices/pai_crypto/"
+#define S390_SYSFS_PAI_EXT	"/sys/devices/pai_ext/"
+#define S390_SYSFS_PAI_NNPA	S390_SYSFS_PAI_EXT "events/NNPA_ALL"
+
+/**
+ * Read out the PMU type from a given file.
+ *
+ * Return the PMU type number assigned to this PMU by the kernel. This is
+ * a non zero number.
+ *
+ * @param[in]    dirname   Name of the event directory in sysfs
+ * @retval       >=0       Number of PMU assigned by the kernel
+ * @retval       -1        PMU unknown to kernel
+ */
+int libcpumf_pmutype(const char *dirname);
+
+/**
+ * Read out the PMU name from a given type.
+ *
+ * Return the PMU name this PMU was registered in kernel. If the PMU was
+ * registered without a name, it is not listed in the directory.
+ * The caller must free the memory returned by name.
+ *
+ * @param[in]    wanted_type Type number of the PMU
+ * @param[out]   name        Name of the PMU (when retval is 0)
+ * @retval       0           PMU wanted_type detected and PMU name valid
+ * @retval       -1          No PMU with wanted_type
+ */
+int libcpumf_pmuname(unsigned int wanted_type, char **name);
+
+/**
+ * Read out the CPU list from a given file name, for example from files
+ * /sys/devices/system/cpu/online or /sys/devices/system/cpu/possible.
+ *
+ * Return the cpu_set_t created from parsing the CPU list in the second
+ * parameter.
+ *
+ * @param[in]    buffer   Comma separated string of a CPU list
+ * @param[in]    filename Name of a sysfs CPU list file name
+ * @param[out]   mask     Converted buffer into cpu_set_t mask structure
+ * @retval       0        Successful conversion of cpulist
+ * @retval       -1       Unsuccessful conversion of cpulist
+ */
+int libcpumf_cpuset(const char *buffer, cpu_set_t *mask);
+int libcpumf_cpuset_fn(const char *filename, cpu_set_t *mask);
+
+/**
+ * Read CPU Measurement Counting Facility hardware information
+ *
+ * Return true if CPU Measurement Counter facility information has been
+ * retrieved and is valid.
+ *
+ * Return false if the information could not be extracted from the file.
+ *
+ * @param[out]   cfvn     Contains CPUMF counter first version number
+ * @param[out]   csvn     Contains CPUMF counter second version number
+ * @param[out]   auth     Contains CPUMF counter set authorization level
+ * @retval       true     Information returned in parameters is valid
+ * @retval       false    Information could not be retrieved
+ */
+bool libcpumf_cpumcf_info(int *cfvn, int *csvn, int *auth);
+
+/**
+ * Return true if CPU Measurement Counter Facility is available.
+ *
+ * @retval       true     CPU Measurement Counter Facility is available
+ * @retval       false    CPU Measurement Counter Facility is not available
+ */
+bool libcpumf_have_cpumcf(void);
+
+/**
+ * Read CPU Measurement Sampling Facility hardware information
+ *
+ * Read all necessary information from /sysfs file /proc/service_levels
+ * to return CPU Measurement Counter Sampling facility information
+ * characteristics.
+ * Return true on success and false when the data can not be retrieved.
+ *
+ * @param[out]   min       Minimum supported sampling interval
+ * @param[out]   max       Maximum supported sampling interval
+ * @param[out]   speed     Current CPU speed, number of CPU cylces per
+ *                         microsecond
+ * @param[out]   basic_sz  Basic sample size in bytes
+ * @param[out]   diag_sz   Diagnostic sample size in bytes
+ * @retval       true      Information returned in parameters is valid
+ * @retval       false     Information could not be retrieved
+ */
+bool libcpumf_cpumsf_info(unsigned long *min, unsigned long *max,
+			  unsigned long *speed, int *basic_sz, int *diag_sz);
+
+/**
+ * Return true if CPU Measurement Sampling Facility is available.
+ *
+ * @retval       true     CPU Measurement Sampling Facility is available
+ * @retval       false    CPU Measurement Sampling Facility is not available
+ */
+bool libcpumf_have_cpumsf(void);
+
+/**
+ * Return true if CPU Measurement Sampling Facility buffer sizes are
+ * available.
+ *
+ * @retval       true     CPU Measurement Sampling Facility buffer sizes are
+ *                        available
+ * @retval       false    CPU Measurement Sampling Facility buffer sizes are
+ *                        not available
+ */
+bool libcpumf_have_sfb(void);
+
+/**
+ * Read CPU Measurement Sampling Facility supported sampling buffer sizes.
+ *
+ * Return the minimum and maximum CPU Measurement sampling facitity buffer
+ * sizes supported.
+ * Return true on success and false otherwise.
+ *
+ * @param[out]   min       Minimum supported sampling buffer size
+ * @param[out]   max       Maximum supported sampling buffer size
+ * @retval       true      Information returned in parameters is valid
+ * @retval       false     Information could not be retrieved
+ */
+bool libcpumf_sfb_info(unsigned long *min, unsigned long *max);
+
+/**
+ * Return true if PAI_CRYPTO counter Facility is supported.
+ *
+ * @retval       true     PAI_CRYPTO counter Facility is available
+ * @retval       false    PAI_CRYPTO counter Facility is not available
+ */
+bool libcpumf_have_pai_crypto(void);
+
+/**
+ * Return true if PAI_EXTENSION Facility is supported.
+ *
+ * @retval       true     PAI_EXTENSION counter Facility is available
+ * @retval       false    PAI_EXTENSION counter Facility is not available
+ */
+bool libcpumf_have_pai_ext(void);
+
+/**
+ * Return true if PAI_NNPA counter Facility is supported. This PMU facility
+ * supports the Neural Network Processing Assist (NNPA) counter set.
+ *
+ * @retval       true     PAI_NNPA counter Facility is available
+ * @retval       false    PAI_NNPA counter Facility is not available
+ */
+bool libcpumf_have_pai_nnpa(void);
+#endif

include/lib/util_arch.h

@@ -26,6 +26,7 @@ enum util_arch_machine_type {
 	UTIL_ARCH_MACHINE_TYPE_Z14_ZR1	= 3907,
 	UTIL_ARCH_MACHINE_TYPE_Z15	= 8561,
 	UTIL_ARCH_MACHINE_TYPE_Z15_T02	= 8562,
+	UTIL_ARCH_MACHINE_TYPE_Z16	= 3931,
 };
 
 int util_arch_machine_type(void);

include/lib/util_exit_code.h

@@ -0,0 +1,21 @@
+/**
+ * @defgroup util_exit_code_h util_exit_code: General purpose exit codes
+ * @{
+ * @brief General purpose exit codes
+ *
+ * Copyright IBM Corp. 2022
+ *
+ * s390-tools is free software; you can redistribute it and/or modify
+ * it under the terms of the MIT license. See LICENSE for details.
+ */
+
+#ifndef LIB_UTIL_EXIT_CODE_H
+#define LIB_UTIL_EXIT_CODE_H
+
+typedef enum {
+	UTIL_EXIT_OK		= 0,  /* Program finished successfully */
+	UTIL_EXIT_RUNTIME_ERROR	= 15, /* A run-time error occurred */
+	UTIL_EXIT_OUT_OF_MEMORY	= 22, /* Not enough available memory */
+} util_exit_code_t;
+
+#endif /** LIB_UTIL_EXIT_CODE_H @} */

include/lib/util_file.h

@@ -11,6 +11,8 @@
 #ifndef LIB_UTIL_FILE_H
 #define LIB_UTIL_FILE_H
 
+#include "lib/util_exit_code.h"
+
 int util_file_read_line(char *str, size_t size, const char *fmt, ...);
 int util_file_read_i(int *val, int base, const char *fmt, ...);
 int util_file_read_l(long *val, int base, const char *fmt, ...);
@@ -26,4 +28,10 @@ int util_file_write_ul(unsigned long val, int base, const char *fmt, ...);
 int util_file_write_ull(unsigned long long val, int base, const char *fmt, ...);
 
 int util_file_read_va(const char *path, const char *fmt, ...);
+
+util_exit_code_t util_file_read_fd_buf(FILE *fd, void **buffer_ptr,
+				       size_t *size_ptr);
+char *util_file_read_fd(FILE *fd, int chomp);
+char *util_file_read_text_file(const char *path, int chomp);
+
 #endif /** LIB_UTIL_FILE_H @} */

include/lib/util_udev.h

@@ -0,0 +1,44 @@
+/*
+ * @defgroup util_udev_h  util_udev: UDEV interface
+ * @{
+ * @brief Work with UDEV files
+ *
+ * Copyright IBM Corp. 2022
+ *
+ * s390-tools is free software; you can redistribute it and/or modify
+ * it under the terms of the MIT license. See LICENSE for details.
+ */
+
+#ifndef LIB_UTIL_UDEV_H
+#define LIB_UTIL_UDEV_H
+
+#include <stdbool.h>
+#include "lib/util_exit_code.h"
+#include "lib/util_list.h"
+
+/* Single key-operator-value entry in a udev rule line.*/
+struct util_udev_entry_node {
+	struct util_list_node node;
+	char *key;
+	char *op;
+	char *value;
+};
+
+/* Single udev line in a udev rule file. */
+struct util_udev_line_node {
+	struct util_list_node node;
+	struct util_list entries;
+	char *line;
+};
+
+/* Udev rule file. */
+struct util_udev_file {
+	struct util_list lines;
+};
+
+util_exit_code_t util_udev_read_file(const char *path,
+				     struct util_udev_file **file_ptr);
+void util_udev_free_file(struct util_udev_file *file);
+void util_udev_file_print(struct util_udev_file *file);
+
+#endif /** LIB_UTIL_UDEV_H @} */

include/lib/zt_common.h

@@ -21,11 +21,13 @@
  * Usage example:
  *  #define IMAGE_ENTRY _AC(0x10000, UL)
  */
+#ifndef _AC
 #ifdef __ASSEMBLER__
 #define _AC(X, TYPE)	X
 #else
 #define _AC(X, TYPE)	X##TYPE
 #endif
+#endif
 
 
 #ifndef __ASSEMBLER__

include/libpv/cert.h

@@ -0,0 +1,438 @@
+/*
+ * Certificate functions and definitions.
+ *
+ * Copyright IBM Corp. 2022
+ *
+ * s390-tools is free software; you can redistribute it and/or modify
+ * it under the terms of the MIT license. See LICENSE for details.
+ */
+#ifndef LIBPV_CERT_H
+#define LIBPV_CERT_H
+
+#include <openssl/x509v3.h>
+#include <openssl/err.h>
+
+#include "libpv/common.h"
+
+#define PV_IBM_Z_SUBJECT_COMMON_NAME "International Business Machines Corporation"
+#define PV_IBM_Z_SUBJECT_COUNTRY_NAME "US"
+#define PV_IBM_Z_SUBJECT_LOCALITY_NAME "Poughkeepsie"
+#define PV_IBM_Z_SUBJECT_ORGANIZATIONAL_UNIT_NAME_SUFFIX "Key Signing Service"
+#define PV_IBM_Z_SUBJECT_ORGANIZATION_NAME "International Business Machines Corporation"
+#define PV_IBM_Z_SUBJECT_STATE "New York"
+#define PV_IMB_Z_SUBJECT_ENTRY_COUNT 6
+
+/* Minimum security level for the keys/certificates used to establish a chain of
+ * trust (see https://www.openssl.org/docs/man1.1.1/man3/X509_VERIFY_PARAM_set_auth_level.html
+ * for details).
+ */
+#define PV_CERTS_SECURITY_LEVEL 2
+
+/** pv_cert_init:
+ *
+ * Should not be called by user.
+ * Use pv_init() instead which
+ * calls this function during creation.
+ *
+ * Sets up data structures for caching CRLs.
+ */
+void pv_cert_init(void);
+
+/** pv_cert_cleanup:
+ *
+ * Should not be called by user.
+ * Use pv_cleanup() instead which
+ * calls this function during creation.
+ *
+ * Cleans up data structures for caching CRLs.
+ */
+void pv_cert_cleanup(void);
+
+#define PV_CERT_ERROR g_quark_from_static_string("pv-cert-error-quark")
+typedef enum {
+	PV_CERT_ERROR_CERT_REVOKED,
+	PV_CERT_ERROR_CERT_SIGNATURE_INVALID,
+	PV_CERT_ERROR_CERT_SUBJECT_ISSUER_MISMATCH,
+	PV_CERT_ERROR_CRL_DOWNLOAD_FAILED,
+	PV_CERT_ERROR_CRL_SIGNATURE_INVALID,
+	PV_CERT_ERROR_CRL_SUBJECT_ISSUER_MISMATCH,
+	PV_CERT_ERROR_FAILED_DOWNLOAD_CRL,
+	PV_CERT_ERROR_INTERNAL,
+	PV_CERT_ERROR_INVALID_PARM,
+	PV_CERT_ERROR_INVALID_SIGNATURE_ALGORITHM,
+	PV_CERT_ERROR_INVALID_VALIDITY_PERIOD,
+	PV_CERT_ERROR_LOAD_CRL,
+	PV_CERT_ERROR_LOAD_DEFAULT_CA,
+	PV_CERT_ERROR_LOAD_ROOT_CA,
+	PV_CERT_ERROR_MALFORMED_CERTIFICATE,
+	PV_CERT_ERROR_MALFORMED_ROOT_CA,
+	PV_CERT_ERROR_NO_CRL,
+	PV_CERT_ERROR_NO_CRLDP,
+	PV_CERT_ERROR_NO_ISSUER_IBM_Z_FOUND,
+	PV_CERT_ERROR_NO_PUBLIC_KEY,
+	PV_CERT_ERROR_READ_CERTIFICATE,
+	PV_CERT_ERROR_READ_CRL,
+	PV_CERT_ERROR_SIGNATURE_ALGORITHM_MISMATCH,
+	PV_CERT_ERROR_SKID_AKID_MISMATCH,
+	PV_CERT_ERROR_VERIFICATION_FAILED,
+	PV_CERT_ERROR_WRONG_CA_USED,
+} PvCertErrors;
+
+/** PvX509WithPath - X509 certificate associated with a path
+ */
+typedef struct {
+	X509 *cert;
+	char *path;
+} PvX509WithPath;
+
+/** pv_x509_with_path_new:
+ *
+ *  @cert: X509 certificate
+ *  @path: Path of that X509 certificate
+ *
+ *  Returns: (nullable) (transfer full): new X509 with path
+ */
+PvX509WithPath *pv_x509_with_path_new(X509 *cert, const char *path);
+
+/** pv_x509_with_path_free:
+ *
+ * Frees the path and the PvX509WithPath; Decreases the refcount of the X509
+ */
+void pv_x509_with_path_free(PvX509WithPath *cert);
+
+typedef STACK_OF(DIST_POINT) STACK_OF_DIST_POINT;
+typedef STACK_OF(X509) STACK_OF_X509;
+typedef STACK_OF(X509_CRL) STACK_OF_X509_CRL;
+typedef GSList PvCertWithPathList;
+
+typedef struct {
+	X509 *cert;
+	STACK_OF_X509_CRL *crls;
+} PvX509Pair;
+
+/** pv_x509_pair_new_take:
+ * @cert: ptr to X509
+ * @crls: ptr to CRLs
+ *
+ * Takes a X509 and the associated CRLs and builds a pair.
+ * Both, *cert and *crls will be NULL afterwards, and owned by the pair.
+ *
+ * Returns: (nullable) (transfer full):	New PvX509Pair
+ */
+PvX509Pair *pv_x509_pair_new_take(X509 **cert, STACK_OF_X509_CRL **crls);
+
+/** pv_x509_pair_free:
+ *
+ * Decreases the refcount of the X509 and crls.
+ * Frees the PvX509Pair.
+ */
+void pv_x509_pair_free(PvX509Pair *pair);
+
+void STACK_OF_DIST_POINT_free(STACK_OF_DIST_POINT *stack);
+void STACK_OF_X509_free(STACK_OF_X509 *stack);
+void STACK_OF_X509_CRL_free(STACK_OF_X509_CRL *stack);
+
+/** pv_x509_from_pem_der_data:
+ *
+ * @data: GBytes containing the cert in PEM format
+ * @error: return location for a #GError
+ *
+ * Returns: (nullable) (transfer full): X509 cert
+ */
+X509 *pv_x509_from_pem_der_data(GBytes *data, GError **error);
+
+/** pv_x509_get_ec_pubkey:
+ *
+ * @cert: X509 to extract elliptic curve pubkey from
+ * @nid: numerical identifier of the expected curve
+ * @error: return location for a #GError
+ *
+ * Returns: (nullable) (transfer full):	corresponding pupkey for the given certificate
+ */
+EVP_PKEY *pv_x509_get_ec_pubkey(X509 *cert, int nid, GError **error);
+
+/** pv_get_ec_pubkeys:
+ *
+ * @certs_with_path: List of PvX509WithPath
+ * @nid: numerical identifier of the expected curve
+ * @error: return location for a #GError
+ *
+ * Returns: (nullable) (transfer full):	List of corresponding public keys for the given certificate
+ */
+GSList *pv_get_ec_pubkeys(PvCertWithPathList *certs_with_path, int nid, GError **error);
+
+/* pv_load_certificates:
+ *
+ * @cert_paths: list of cert paths.
+ * @error: return location for a #GError
+ *
+ * @cert_paths must contain at least one element, otherwise an error is
+ * reported.
+ *
+ * Returns: (nullable) (transfer full):	List of PvX509WithPath corresponding to the given paths
+ */
+PvCertWithPathList *pv_load_certificates(char **cert_paths, GError **error);
+
+/* pv_load_first_cert_from_file:
+ *
+ * @path: location of the x509
+ * @error: return location for a #GError
+ *
+ * This function reads in only the first certificate and ignores all other. This
+ * is only relevant for the PEM file format. For the host-key document and the
+ * root CA this behavior is expected.
+ *
+ * Returns: (nullable) (transfer full):	PvX509WithPath corresponding to the given path
+ */
+X509 *pv_load_first_cert_from_file(const char *path, GError **error);
+
+/* pv_load_first_crl_from_file:
+ *
+ * @path: location of the x509 CRL
+ * @error: return location for a #GError
+ *
+ * This function reads in only the first CRL and ignores all other. This
+ * is only relevant for the PEM file format.
+ *
+ * Returns: (nullable) (transfer full):	X509_CRL corresponding to the given path
+ */
+X509_CRL *pv_load_first_crl_from_file(const char *path, GError **error);
+
+/** pv_store_setup_crl_download:
+ *
+ * @st: X509_STORE
+ */
+void pv_store_setup_crl_download(X509_STORE *st);
+
+/** pv_load_first_crl_by_cert:
+ * @cert: X509 to specify the download location.
+ * @error: return location for a #GError
+ *
+ * This function returns the first X509_CRL found from the CRL distribution
+ * points specified in @cert.
+ *
+ * Returns: (nullable) (transfer full):	x509 CRL corresponding to the given X509
+ */
+X509_CRL *pv_load_first_crl_by_cert(X509 *cert, GError **error);
+
+/** pv_try_load_crls_by_certs:
+ *
+ * @certs_with_path: List of PvX509WithPath
+ *
+ * Returns: (nullable) (transfer full):	Stack of CRLs corresponding to the given X509
+ */
+STACK_OF_X509_CRL *pv_try_load_crls_by_certs(PvCertWithPathList *certs_with_path);
+
+/** pv_store_setup:
+ *
+ * @root_ca_path: Location of the rootCA or NULL if SystemRoot CA shall be used
+ * @crl_paths: List of CRL paths or NULL
+ * @cert_with_crl_paths: List of (untrusted) X509 paths
+ * @error: return location for a #GError
+ *
+ * The untrusted certs need to be verified before actually verifying a Host Key Document.
+ *
+ * Returns: (nullable) (transfer full):	X509_store with given input data.
+ *
+ */
+X509_STORE *pv_store_setup(char *root_ca_path, char **crl_paths, char **cert_with_crl_paths,
+			   GError **error);
+
+/** pv_get_x509_stack:
+ *
+ * x509_with_path_list: list of PvX509WithPath
+ *
+ * Returns: (nullable) (transfer full):	Stack of X509 corresponding to the given x509 with path
+ */
+STACK_OF_X509 *pv_get_x509_stack(const GSList *x509_with_path_list);
+
+/** pv_init_store_ctx:
+ *
+ * @ctx: a uninitialized Store CTX
+ * @trusted: X509_STORE with a trusted rootCA
+ * @chain: untrusted X509s
+ * @error: return location for a #GError
+ *
+ * Can be called multiple times on the same context if X509_STORE_CTX_cleanup(ctx)
+ * was called before.
+ *
+ * Returns:
+ * 	0 on success
+ * 	-1 in failure
+ */
+int pv_init_store_ctx(X509_STORE_CTX *ctx, X509_STORE *trusted, STACK_OF_X509 *chain,
+		      GError **error) PV_NONNULL(1, 2, 3);
+
+/** pv_init_store_ctx:
+ *
+ * @trusted: X509_STORE with a trusted rootCA
+ * @chain: untrusted X509s
+ * @error: return location for a #GError
+ *
+ * Returns: (nullable) (transfer full):	X509_STORE_CTX setup with the input data
+ */
+X509_STORE_CTX *pv_create_store_ctx(X509_STORE *trusted, STACK_OF_X509 *chain, GError **error)
+	PV_NONNULL(1, 2);
+/** pv_remove_ibm_signing_certs:
+ *
+ * @certs: Stack of X509s
+ *
+ * Returns: (transfer full):
+ * 	List of all IBM Z signing key certificates in @certs and remove them
+ * 		from the chain.
+ * 	Empty stack if no IBM Z signing key is found.
+ */
+STACK_OF_X509 *pv_remove_ibm_signing_certs(STACK_OF_X509 *certs);
+
+/** pv_c2b_name:
+ *
+ * Workaround to fix the mismatch between issuer name of the
+ * IBM Z signing CRLs and the IBM Z signing key subject name.
+ *
+ * In RFC 5280 the attributes of a (subject/issuer) name is not mandatory
+ * ordered. The problem is that our certificates are not consistent in the order
+ * (see https://tools.ietf.org/html/rfc5280#section-4.1.2.4 for details).
+ *
+ * This function tries to reorder the name attributes such that
+ * further OpenSSL calls can work with it.  The caller is
+ * responsible to free the returned value.
+ */
+X509_NAME *pv_c2b_name(const X509_NAME *name);
+
+/** pv_verify_host_key:
+ *
+ * @host_key: X509 to be verified
+ * @issuer_pairs: IBM signing key X509+CRLs Pairs used for verification
+ * @level: Security level. see PV_CERTS_SECURITY_LEVEL
+ * @error: return location for a #GError
+ *
+ * Returns:
+ * 	0 if Host key could be verified with one of the IBM signing keys
+ * 	-1 if no IBM signing key could verify the authenticity of the given host key
+ *
+ */
+int pv_verify_host_key(X509 *host_key, GSList *issuer_pairs, int verify_flags, int level,
+		       GError **error);
+
+/** pv_verify_cert:
+ *
+ * @ctx: trusted store ctx used for verification
+ * @cert: X509 to be verified
+ * @error: return location for a #GError
+ *
+ * Cannot be used to verify host keys with IBM signing keys, as IBM signing
+ * keys are no intermediate CAs. Use pv_verify_host_key() instead.
+ *
+ * Returns:
+ * 	0 if @cert could be verified
+ * 	-1 if @cert could not be verified
+ */
+int pv_verify_cert(X509_STORE_CTX *ctx, X509 *cert, GError **error) PV_NONNULL(1, 2);
+
+/** pv_check_crl_valid_for_cert:
+ *
+ * @crl: CRL to be verified
+ * @cert: Cert that probably issued the given CRL
+ * @verify_flags: X509 Verification flags (X509_V_FLAG_<TYPE>)
+ * @error: return location for a #GError
+ *
+ * Verify whether a revocation list @crl is valid and is issued by @cert. For
+ * this multiple steps must be done:
+ *
+ * 1. verify issuer of the CRL matches with the suject name of @cert
+ * 2. verify the validity period of the CRL
+ * 3. verify the signature of the CRL
+ *
+ * Important: This function does not verify whether @cert is allowed to issue a
+ * CRL.
+ *
+ * Returns:
+ * 	 0 if @crl is valid and issued by @cert
+ * 	 -1 otherwise
+ */
+int pv_verify_crl(X509_CRL *crl, X509 *cert, int verify_flags, GError **error);
+
+/** pv_check_chain_parameters:
+ *
+ * @chain: chain of trust to be validated
+ * @error: return location for a #GError
+ *
+ * Verifies that chain has at least a RootCA ans intermediate CA
+ * and logs the used ROD CA subject
+ *
+ * Returns:
+ * 	 0 @chain is valid
+ * 	 -1 otherwise
+ */
+int pv_check_chain_parameters(const STACK_OF_X509 *chain, GError **error);
+
+/** pv_store_set_verify_param:
+ *
+ * @store: X509_STORE to set parameters
+ * @error: return location for a #GError
+ *
+ * Returns:
+ * 	0 on success
+ * 	-1 on failure
+ */
+int pv_store_set_verify_param(X509_STORE *store, GError **error);
+
+/** pv_store_ctx_find_valid_crls:
+ *
+ * @ctx: STORE_CTX for searching CRLs
+ * @cert: X509 to match CRLs aggainst
+ * @error: return location for a #GError
+ *
+ * Returns: (nullable) (transfer full):	STACK of CRLs related to given @crl fin @ctx
+ */
+STACK_OF_X509_CRL *pv_store_ctx_find_valid_crls(X509_STORE_CTX *ctx, X509 *cert, GError **error)
+	PV_NONNULL(1, 2);
+
+/** pv_verify_host_key_doc:
+ *
+ * @host_key_certs_with_path: X509s to be verified
+ * @trusted. X509_STORE with a rusted RootCA
+ * @untrusted_certs: STACK OF untrusted X509s
+ * @online: true if CRLs shall be downloaded
+ * @error: return location for a #GError
+ *
+ * Returns:
+ * 	0 if all given HKDs could be verified using the chain of trust.
+ * 	-1 otherwise
+ */
+int pv_verify_host_key_doc(PvCertWithPathList *host_key_certs_with_path, X509_STORE *trusted,
+			   STACK_OF_X509 *untrusted_certs, gboolean online, GError **error)
+	PV_NONNULL(1, 2, 3);
+
+/** pv_verify_host_key_docs_by_path:
+ *
+ * @host_key_paths: locations of X509 to be verified
+ * @optional_root_ca_path: rootCA location or NULL if Default shall be used
+ * @optional_crl_paths: locations of CRLs or NULL
+ * @untrusted_cert_paths: locations of IntermediateCAs including the IBM signing key
+ * @online: true if CRLs shall be downloaded
+ * @error: return location for a #GError
+ *
+ * Returns:
+ * 	0 if all given HKDs could be verfied using the chain of trust.
+ * 	-1 otherwise
+ */
+int pv_verify_host_key_docs_by_path(char **host_key_paths, char *optional_root_ca_path,
+				    char **optional_crl_paths, char **untrusted_cert_paths,
+				    gboolean online, GError **error) PV_NONNULL(1, 4);
+
+WRAPPED_G_DEFINE_AUTOPTR_CLEANUP_FUNC(AUTHORITY_KEYID, AUTHORITY_KEYID_free)
+WRAPPED_G_DEFINE_AUTOPTR_CLEANUP_FUNC(PvX509WithPath, pv_x509_with_path_free)
+WRAPPED_G_DEFINE_AUTOPTR_CLEANUP_FUNC(STACK_OF_DIST_POINT, STACK_OF_DIST_POINT_free)
+WRAPPED_G_DEFINE_AUTOPTR_CLEANUP_FUNC(STACK_OF_X509, STACK_OF_X509_free)
+WRAPPED_G_DEFINE_AUTOPTR_CLEANUP_FUNC(STACK_OF_X509_CRL, STACK_OF_X509_CRL_free)
+WRAPPED_G_DEFINE_AUTOPTR_CLEANUP_FUNC(X509, X509_free)
+WRAPPED_G_DEFINE_AUTOPTR_CLEANUP_FUNC(X509_CRL, X509_CRL_free)
+WRAPPED_G_DEFINE_AUTOPTR_CLEANUP_FUNC(X509_LOOKUP, X509_LOOKUP_free)
+WRAPPED_G_DEFINE_AUTOPTR_CLEANUP_FUNC(X509_NAME, X509_NAME_free)
+WRAPPED_G_DEFINE_AUTOPTR_CLEANUP_FUNC(X509_VERIFY_PARAM, X509_VERIFY_PARAM_free)
+WRAPPED_G_DEFINE_AUTOPTR_CLEANUP_FUNC(PvX509Pair, pv_x509_pair_free)
+WRAPPED_G_DEFINE_AUTOPTR_CLEANUP_FUNC(X509_STORE, X509_STORE_free)
+WRAPPED_G_DEFINE_AUTOPTR_CLEANUP_FUNC(X509_STORE_CTX, X509_STORE_CTX_free)
+
+#endif /*  LIBPV_CERT_H */

include/libpv/common.h

@@ -0,0 +1,35 @@
+/*
+ * Libpv common definitions.
+ *
+ * Copyright IBM Corp. 2022
+ *
+ * s390-tools is free software; you can redistribute it and/or modify
+ * it under the terms of the MIT license. See LICENSE for details.
+ *
+ */
+#ifndef LIBPV_COMMON_H
+#define LIBPV_COMMON_H
+
+/* must be included before any (other) glib header to verify that
+ * the glib version is supported
+ */
+#include "libpv/glib-helper.h"
+
+#include <glib/gi18n-lib.h>
+
+#include "libpv/openssl-compat.h"
+#include "libpv/macros.h"
+
+/** pv_init:
+ *
+ * Must be called before any libpv call.
+ */
+int pv_init(void);
+
+/** pv_cleanup:
+ *
+ * Must be called when done with using libpv.
+ */
+void pv_cleanup(void);
+
+#endif /* LIBPV_COMMON_H */

include/libpv/crypto.h

@@ -0,0 +1,200 @@
+/*
+ * General cryptography helper functions and definitions
+ *
+ * Copyright IBM Corp. 2022
+ *
+ * s390-tools is free software; you can redistribute it and/or modify
+ * it under the terms of the MIT license. See LICENSE for details.
+ */
+#ifndef LIBPV_CRYPTO_H
+#define LIBPV_CRYPTO_H
+
+#include <openssl/bio.h>
+#include <openssl/bn.h>
+#include <openssl/ec.h>
+#include <openssl/evp.h>
+
+#include "libpv/common.h"
+
+typedef struct pv_cipher_parms {
+	const EVP_CIPHER *cipher;
+	size_t tag_size;
+	GBytes *key;
+	union {
+		GBytes *iv;
+		GBytes *tweak;
+	};
+} PvCipherParms;
+
+typedef union {
+	struct {
+		uint8_t x[80];
+		uint8_t y[80];
+	};
+	uint8_t data[160];
+} PvEcdhPubKey;
+G_STATIC_ASSERT(sizeof(PvEcdhPubKey) == 160);
+
+typedef GSList PvEvpKeyList;
+
+enum PvCryptoMode {
+	PV_ENCRYPT,
+	PV_DECRYPT,
+};
+
+/** pv_get_openssl_error:
+ *
+ * Returns: (transfer full): String representing the error.
+ */
+const char *pv_get_openssl_error(void);
+
+/**
+ * pv_BIO_reset:
+ * @b: BIO to reset
+ *
+ * Resets a BIO to its initial state.
+ *
+ * Returns: 0 in case of success, -1 otherwise.
+ */
+int pv_BIO_reset(BIO *b);
+
+/**
+ * pv_generate_rand_data:
+ * @size: number of generated random bytes using a crypographically secure pseudo random generator
+ * @error: return location for a #GError
+ *
+ * Creates a new #GBytes with @size random bytes using a cryptographically
+ * secure pseudo random generator.
+ *
+ * Returns: (nullable) (transfer full): a new #GBytes, or %NULL in case of an error
+ */
+GBytes *pv_generate_rand_data(size_t size, GError **error);
+
+/**
+ * pv_generate_key:
+ * @cipher: specifies the OpenSSL cipher for which a cryptographically secure key should be generated
+ * @error: return location for a #GError
+ *
+ * Creates a random key for @cipher using a cryptographically secure pseudo
+ * random generator.
+ *
+ * Returns: (nullable) (transfer full): a new #GBytes, or %NULL in case of an error
+ */
+GBytes *pv_generate_key(const EVP_CIPHER *cipher, GError **error) PV_NONNULL(1);
+
+/**
+ * pv_generate_iv:
+ * @cipher: specifies the OpenSSL cipher for which a cryptographically secure IV should be generated
+ * @error: return location for a #GError
+ *
+ * Creates a random IV for @cipher using a cryptographically secure pseudo
+ * random generator.
+ *
+ * Returns: (nullable) (transfer full): a new #GBytes, or %NULL in case of an error
+ */
+GBytes *pv_generate_iv(const EVP_CIPHER *cipher, GError **error) PV_NONNULL(1);
+
+/* Symmetric en/decryption functions */
+
+/**
+ * pv_gcm_encrypt:
+ * @plain: data to encrypt
+ * @aad: (optional): additional data that should be authenticated with the key
+ * @parms:
+ * @cipher: (out): location to store the ciphertext
+ * @tag: (out): location to store the generated GCM tag
+ * @error: return location for a #GError
+ *
+ * Encrypts the @plain data and authenticates @aad data.
+ *
+ * Returns: number of bytes, or -1 in case of an error
+ */
+int64_t pv_gcm_encrypt(GBytes *plain, GBytes *aad, const PvCipherParms *parms, GBytes **cipher,
+		       GBytes **tag, GError **error) PV_NONNULL(1, 3, 4, 5);
+
+/**
+ * pv_gcm_decrypt:
+ * @cipher: ciphertext to decrypt
+ * @aad: (optional): additional date to authenticate
+ * @tag: the GCM tag
+ * @parms:
+ * @plain: (out): location to store the decrypted data
+ * @error: return location for a #GError
+ *
+ * Decrypts the @cipher data and authenticates the @aad data.
+ *
+ * Returns: number of bytes, or -1 in case of an error
+ */
+int64_t pv_gcm_decrypt(GBytes *cipher, GBytes *aad, GBytes *tag, const PvCipherParms *parms,
+		       GBytes **plain, GError **error) PV_NONNULL(1, 3, 4, 5);
+
+/** pv_hkdf_extract_and_expand:
+ * @derived_key_len: size of the output key
+ * @key: input key
+ * @salt: salt for the extraction
+ * @info: infor for the expansion
+ * @md: EVP mode of operation
+ * @error: return location for a #GError
+ *
+ * Performs a RFC 5869 HKDF.
+ *
+ * Returns: (nullable) (transfer full): Result of RFC 5869 HKDF
+ *
+ */
+GBytes *pv_hkdf_extract_and_expand(size_t derived_key_len, GBytes *key, GBytes *salt, GBytes *info,
+				   const EVP_MD *md, GError **error) PV_NONNULL(2, 3, 4, 5);
+
+/** pv_generate_ec_key:
+ *
+ * @nid: Numerical identifier of the curve
+ * @error: return location for a #GError
+ *
+ * Returns: (nullable) (transfer full): new random key based on the given curve
+ */
+EVP_PKEY *pv_generate_ec_key(int nid, GError **error);
+
+/** pv_evp_pkey_to_ecdh_pub_key:
+ *
+ * @key: input key in EVP_PKEY format
+ * @error: return location for a #GError
+ *
+ * Returns: the public part of the input @key in ECDH format.
+ */
+PvEcdhPubKey *pv_evp_pkey_to_ecdh_pub_key(EVP_PKEY *key, GError **error) PV_NONNULL(1);
+
+/** pv_derive_exchange_key:
+ * @cust: Customer Key
+ * @host: Host key
+ * @error: return location for a #GError
+ *
+ * Returns: (nullable) (transfer full): Shared Secret of @cust and @host
+ */
+GBytes *pv_derive_exchange_key(EVP_PKEY *cust, EVP_PKEY *host, GError **error) PV_NONNULL(1, 2);
+
+GQuark pv_crypto_error_quark(void);
+#define PV_CRYPTO_ERROR pv_crypto_error_quark()
+typedef enum {
+	PV_CRYPTO_ERROR_DERIVE,
+	PV_CRYPTO_ERROR_HKDF_FAIL,
+	PV_CRYPTO_ERROR_INTERNAL,
+	PV_CRYPTO_ERROR_INVALID_KEY_SIZE,
+	PV_CRYPTO_ERROR_KEYGENERATION,
+	PV_CRYPTO_ERROR_RANDOMIZATION,
+	PV_CRYPTO_ERROR_READ_FILE,
+	PV_CRYPTO_ERROR_NO_IBM_Z_SIGNING_KEY,
+	PV_CRYPTO_ERROR_NO_MATCH_TAG,
+} PvCryptoErrors;
+
+WRAPPED_G_DEFINE_AUTOPTR_CLEANUP_FUNC(ASN1_INTEGER, ASN1_INTEGER_free)
+WRAPPED_G_DEFINE_AUTOPTR_CLEANUP_FUNC(ASN1_OCTET_STRING, ASN1_OCTET_STRING_free)
+WRAPPED_G_DEFINE_AUTOPTR_CLEANUP_FUNC(BIO, BIO_free_all)
+WRAPPED_G_DEFINE_AUTOPTR_CLEANUP_FUNC(BIGNUM, BN_free)
+WRAPPED_G_DEFINE_AUTOPTR_CLEANUP_FUNC(BN_CTX, BN_CTX_free)
+WRAPPED_G_DEFINE_AUTOPTR_CLEANUP_FUNC(EC_GROUP, EC_GROUP_free)
+WRAPPED_G_DEFINE_AUTOPTR_CLEANUP_FUNC(EC_KEY, EC_KEY_free)
+WRAPPED_G_DEFINE_AUTOPTR_CLEANUP_FUNC(EC_POINT, EC_POINT_free)
+WRAPPED_G_DEFINE_AUTOPTR_CLEANUP_FUNC(EVP_CIPHER_CTX, EVP_CIPHER_CTX_free)
+WRAPPED_G_DEFINE_AUTOPTR_CLEANUP_FUNC(EVP_PKEY, EVP_PKEY_free)
+WRAPPED_G_DEFINE_AUTOPTR_CLEANUP_FUNC(EVP_PKEY_CTX, EVP_PKEY_CTX_free)
+
+#endif /* LIBPV_CRYPTO_H */

include/libpv/curl.h

@@ -0,0 +1,53 @@
+/*
+ * Libcurl utils
+ *
+ * Copyright IBM Corp. 2022
+ *
+ * s390-tools is free software; you can redistribute it and/or modify
+ * it under the terms of the MIT license. See LICENSE for details.
+ */
+#ifndef LIBPV_CURL_H
+#define LIBPV_CURL_H
+
+#include <curl/curl.h>
+
+#include "libpv/common.h"
+
+#define CRL_DOWNLOAD_TIMEOUT_MS 3000
+#define CRL_DOWNLOAD_MAX_SIZE 0x100000
+
+WRAPPED_G_DEFINE_AUTOPTR_CLEANUP_FUNC(CURL, curl_easy_cleanup)
+
+/** curl_download:
+ * @url: URL to specify location of data
+ * @timeout_ms: time to wait until fail
+ * @max_size: Maximum size of the downloaded data
+ * @error: return location for a GError
+ *
+ * Returns: (nullable) (transfer full): Downloaded data as #GByteArray
+ */
+GByteArray *curl_download(const char *url, long timeout_ms, uint max_size, GError **err);
+
+/** pv_curl_init:
+ *
+ * Should not be called by user.
+ * Use pv_init() instead which
+ * calls this function during creation.
+ */
+int pv_curl_init(void);
+
+/** pv_curl_cleanup:
+ *
+ * Should not be called by user.
+ * Use pv_cleanup() instead which
+ * calls this function during creation.
+ */
+void pv_curl_cleanup(void);
+
+#define PV_CURL_ERROR g_quark_from_static_string("pv-curl-error-quark")
+typedef enum {
+	PV_CURL_ERROR_CURL_INIT_FAILED,
+	PV_CURL_ERROR_DOWNLOAD_FAILED,
+} PvCurlErrors;
+
+#endif /* LIBPV_CURL_H */

include/libpv/glib-helper.h

@@ -0,0 +1,117 @@
+/*
+ * Glib convenience functions
+ * Shall be used instead of manually including glib.
+ *
+ * Copyright IBM Corp. 2022
+ *
+ * s390-tools is free software; you can redistribute it and/or modify
+ * it under the terms of the MIT license. See LICENSE for details.
+ */
+#ifndef LIBPV_GLIB_HELPER_H
+#define LIBPV_GLIB_HELPER_H
+
+#if defined(GLIB_VERSION_MIN_REQUIRED)
+#if GLIB_VERSION_MIN_REQUIRED < GLIB_VERSION_2_56
+#error "GLIB_VERSION must be at least 2.56"
+#endif
+#else
+#define GLIB_VERSION_MIN_REQUIRED GLIB_VERSION_2_56
+#endif
+
+#include <glib.h>
+#include <gmodule.h>
+#include <stdio.h>
+
+#include "libpv/macros.h"
+
+#ifdef __clang__
+#define WRAPPED_G_DEFINE_AUTOPTR_CLEANUP_FUNC(...)                                                 \
+	DO_PRAGMA(clang diagnostic push)                                                           \
+	DO_PRAGMA(clang diagnostic ignored "-Wunused-function")                                    \
+	G_DEFINE_AUTOPTR_CLEANUP_FUNC(__VA_ARGS__)                                                 \
+	DO_PRAGMA(clang diagnostic pop)
+#else
+#define WRAPPED_G_DEFINE_AUTOPTR_CLEANUP_FUNC(...) G_DEFINE_AUTOPTR_CLEANUP_FUNC(__VA_ARGS__)
+#endif
+
+#define pv_wrapped_g_assert(__expr) g_assert(__expr)
+
+/** pv_sec_gbytes_new_take:
+ *
+ * #g_bytes_new_take() with secure cleanup
+ */
+GBytes *pv_sec_gbytes_new_take(void *data, size_t size);
+
+/** pv_sec_gbytes_new:
+ *
+ * #g_bytes_new() with secure cleanup
+ */
+GBytes *pv_sec_gbytes_new(const void *data, size_t size);
+
+/** pv_file_get_content_as_secure_bytes:
+ *
+ * @filename: path to file for reading in
+ *
+ * read file and save as secure gbytes
+ *
+ * Return:
+ * 	Content of file as #GBytes with secure cleanup
+ */
+GBytes *pv_file_get_content_as_secure_bytes(const char *filename);
+
+/** pv_file_get_content_as_g_bytes:
+ *
+ * @filename: path to file for reading in
+ *
+ * read file and save as gbytes
+ *
+ * Return:
+ * 	Content of file as #GBytes
+ */
+GBytes *pv_file_get_content_as_g_bytes(const char *filename, GError **error);
+
+/** pv_file_seek:
+ *
+ * fseek with error reporting
+ */
+int pv_file_seek(FILE *file, long offset, int whence, GError **error);
+
+/** pv_file_write:
+ *
+ * fwrite with error reporting
+ */
+size_t pv_file_write(FILE *file, const void *ptr, size_t size, GError **error);
+
+/** pv_file_close:
+ *
+ * fclose with error reporting
+ */
+long pv_file_close(FILE *file, GError **error);
+
+/** pv_file_tell:
+ *
+ * ftell with error reporting
+ */
+long pv_file_tell(FILE *file, GError **error);
+
+/** pv_file_open:
+ *
+ * fopen with error reporting
+ */
+FILE *pv_file_open(const char *filename, const char *mode, GError **error);
+
+/** pv_gbytes_memcpy:
+ *
+ * memcpy with size check.
+ */
+void *pv_gbytes_memcpy(void *dst, size_t dst_size, GBytes *src);
+
+#define PV_GLIB_HELPER_ERROR g_quark_from_static_string("pv-glib-helper_error-quark")
+typedef enum {
+	PV_GLIB_HELPER_FILE_ERROR,
+} pv_glib_helper_error_e;
+
+void pv_auto_close_file(FILE *file);
+WRAPPED_G_DEFINE_AUTOPTR_CLEANUP_FUNC(FILE, pv_auto_close_file)
+
+#endif /* LIBPV_GLIB_HELPER_H */

include/libpv/hash.h

@@ -0,0 +1,119 @@
+/*
+ * Hashing definitions.
+ *
+ * Copyright IBM Corp. 2022
+ *
+ * s390-tools is free software; you can redistribute it and/or modify
+ * it under the terms of the MIT license. See LICENSE for details.
+ */
+#ifndef LIBPV_HASH_H
+#define LIBPV_HASH_H
+
+#include <openssl/hmac.h>
+
+#include "libpv/common.h"
+
+/** pv_digest_ctx_new:
+ * @md: mode of digest, e.g. #EVP_sha256()
+ * @error: return location for a #GError
+ *
+ * Returns: (nullable) (transfer full): a new #EVP_MD_CTX, or %NULL in case of an error
+ */
+EVP_MD_CTX *pv_digest_ctx_new(const EVP_MD *md, GError **error);
+
+/** pv_digest_ctx_update:
+ * @ctx: EVP_MD_CTX to add data
+ * @data: #GBytes to add to the context
+ * @error: return location for a #GError
+ *
+ * Adds @data to the digest context. Can be called multiple times.
+ *
+ * Returns: 0 in case of success, -1 otherwise.
+ */
+int pv_digest_ctx_update(EVP_MD_CTX *ctx, GBytes *data, GError **error);
+
+/** pv_digest_ctx_update_raw:
+ * @ctx: #EVP_MD_CTX to add data
+ * @buf: data to add to the context
+ * @size: size of @buf
+ * @error: return location for a #GError
+ *
+ * Adds @buf to the digest context. Can be called multiple times.
+ *
+ * Returns: 0 in case of success, -1 otherwise.
+ */
+int pv_digest_ctx_update_raw(EVP_MD_CTX *ctx, const uint8_t *buf, size_t size, GError **error);
+
+/** pv_digest_ctx_finalize:
+ * @ctx: #EVP_MD_CTX with data to digest
+ * @error: return location for a #GError
+ *
+ * Calculates the digest of all previously added data. Do not use @ctx afterwards.
+ *
+ * Returns: (nullable) (transfer full): Digest of all data added before as #GBytes, or NULL in case of error.
+ */
+GBytes *pv_digest_ctx_finalize(EVP_MD_CTX *ctx, GError **error);
+
+/** pv_sha256_hash:
+ * @buf: data for which a sha256 hash sould be calculated
+ * @size: size of @buf
+ * @error: return location for a #GError
+ *
+ * Shorthand for initializing a sha256-digest ctx, updating, and finalizing.
+ *
+ * Returns: (nullable) (transfer full): SHA256 of @buf as #GBytes, or NULL in case of error.
+ */
+GBytes *pv_sha256_hash(uint8_t *buf, size_t size, GError **error);
+
+/** pv_hmac_ctx_new:
+ * @key: key used for the HMAC
+ * @md: mode of digest, e.g. #EVP_sha512()
+ * @error: return location for a #GError
+ *
+ * Returns: (nullable) (transfer full): New #HMAC_CTX or NULL in case of error
+ */
+HMAC_CTX *pv_hmac_ctx_new(GBytes *key, const EVP_MD *md, GError **error);
+
+/** pv_hmac_ctx_update_raw:
+ * @ctx: #HMAC_CTX to add data
+ * @buf: data to add to the context
+ * @size: size of @buf
+ * @error: return location for a #GError
+ *
+ * Adds @buf to the HMAC context. Can be called multiple times.
+ *
+ * Returns: 0 in case of success, -1 otherwise.
+ */
+int pv_hmac_ctx_update_raw(HMAC_CTX *ctx, const void *data, size_t size, GError **error);
+
+/** pv_hmac_ctx_update:
+ * @ctx: #HMAC_CTX to add data
+ * @data: #GBytes to add to the context
+ * @error: return location for a #GError
+ *
+ * Adds @data to the HMAC context. Can be called multiple times.
+ *
+ * Returns: 0 in case of success, -1 otherwise.
+ */
+
+int pv_hmac_ctx_update(HMAC_CTX *ctx, GBytes *data, GError **error);
+
+/** pv_hmac_ctx_finalize:
+ * @ctx: #HMAC_CTX with data to digest
+ * @error: return location for a #GError
+ *
+ * Calculates the HMAC of all previously added data. Do not use @ctx afterwards.
+ *
+ * Returns: (nullable) (transfer full): HMAC of all data added before as #GBytes, or NULL in case of error.
+ */
+GBytes *pv_hamc_ctx_finalize(HMAC_CTX *ctx, GError **error);
+
+#define PV_HASH_ERROR g_quark_from_static_string("pv-crypro-error-quark")
+typedef enum {
+	PV_HASH_ERROR_INTERNAL,
+} PvHashErrors;
+
+WRAPPED_G_DEFINE_AUTOPTR_CLEANUP_FUNC(EVP_MD_CTX, EVP_MD_CTX_free)
+WRAPPED_G_DEFINE_AUTOPTR_CLEANUP_FUNC(HMAC_CTX, HMAC_CTX_free)
+
+#endif /* LIBPV_HASH_H */

include/libpv/macros.h

@@ -0,0 +1,16 @@
+/*
+ * Libpv common macro definitions.
+ *
+ * Copyright IBM Corp. 2022
+ *
+ * s390-tools is free software; you can redistribute it and/or modify
+ * it under the terms of the MIT license. See LICENSE for details.
+ *
+ */
+#ifndef LIBPV_MACROS_H
+#define LIBPV_MACROS_H
+
+#define PV_NONNULL(...)
+#define DO_PRAGMA(x) _Pragma(#x)
+
+#endif /* LIBPV_MACROS_H */

include/libpv/openssl-compat.h

@@ -0,0 +1,29 @@
+/*
+ * OpenSSL compatibility utils
+ *
+ * Copyright IBM Corp. 2021
+ *
+ * s390-tools is free software; you can redistribute it and/or modify
+ * it under the terms of the MIT license. See LICENSE for details.
+ */
+#ifndef LIBPV_OPENSSL_COMPAT_H
+#define LIBPV_OPENSSL_COMPAT_H
+
+#include <openssl/opensslv.h>
+#include <openssl/x509.h>
+#include <openssl/x509_vfy.h>
+
+#if OPENSSL_VERSION_NUMBER >= 0x30000000L
+#define pv_X509_STORE_CTX_get_current_cert(ctx) X509_STORE_CTX_get_current_cert(ctx)
+#define pv_X509_STORE_CTX_get1_crls(ctx, nm) X509_STORE_CTX_get1_crls((ctx), (nm))
+#define pv_X509_STORE_set_lookup_crls(st, cb) X509_STORE_set_lookup_crls(st, cb)
+#elif OPENSSL_VERSION_NUMBER >= 0x10100000L
+#define pv_X509_STORE_CTX_get_current_cert(ctx)                                                    \
+	X509_STORE_CTX_get_current_cert((X509_STORE_CTX *)(ctx))
+#define pv_X509_STORE_CTX_get1_crls(ctx, nm)                                                       \
+	X509_STORE_CTX_get1_crls((X509_STORE_CTX *)(ctx), (X509_NAME *)(nm))
+#define pv_X509_STORE_set_lookup_crls(st, cb)                                                      \
+	X509_STORE_set_lookup_crls(st, (X509_STORE_CTX_lookup_crls_fn)(cb))
+#endif
+
+#endif /* LIBPV_OPENSSL_COMPAT_H */

include/libpv/se-hdr.h

@@ -0,0 +1,98 @@
+/*
+ * PV/SE header definitions
+ *
+ * Copyright IBM Corp. 2020
+ *
+ * s390-tools is free software; you can redistribute it and/or modify
+ * it under the terms of the MIT license. See LICENSE for details.
+ */
+#ifndef LIBPV_SE_HDR_H
+#define LIBPV_SE_HDR_H
+
+#include "libpv/common.h"
+
+#include <openssl/sha.h>
+
+#include "boot/s390.h"
+#include "libpv/crypto.h"
+
+/* Magic number which is used to identify the file containing the PV
+ * header
+ */
+#define PV_MAGIC_NUMBER 0x49424d5365634578ULL
+#define PV_VERSION_1 0x00000100U
+
+/* Internal helper macro */
+#define __PV_BIT(nr) (1ULL << (63 - (nr)))
+
+/* Plaintext control flags */
+/* dumping of the configuration is allowed */
+#define PV_PCF_ALLOW_DUMPING __PV_BIT(34)
+/* prevent Ultravisor decryption during unpack operation */
+#define PV_PCF_NO_DECRYPTION __PV_BIT(35)
+/* PCKMO encrypt-DEA/TDEA-key functions allowed */
+#define PV_PCF_PCKMO_DEA_TDEA __PV_BIT(56)
+/* PCKMO encrypt-AES-key functions allowed */
+#define PV_PCF_PCKMO_AES __PV_BIT(57)
+/* PCKMO encrypt-ECC-key functions allowed */
+#define PV_PCF_PCKM_ECC __PV_BIT(58)
+
+/* maxima for the PV version 1 */
+#define PV_V1_IPIB_MAX_SIZE PAGE_SIZE
+#define PV_V1_PV_HDR_MIN_SIZE                                                                      \
+	(sizeof(struct pv_hdr_head) + sizeof(struct pv_hdr_encrypted) +                            \
+	 sizeof(((struct pv_hdr *)0)->tag) + 1 * sizeof(struct pv_hdr_key_slot))
+#define PV_V1_PV_HDR_MAX_SIZE (2 * PAGE_SIZE)
+
+#define PV_IMAGE_ENCR_KEY_SIZE 64
+
+typedef struct pv_hdr_key_slot {
+	uint8_t digest_key[SHA256_DIGEST_LENGTH];
+	uint8_t wrapped_key[32];
+	uint8_t tag[16];
+} __packed PvHdrKeySlot;
+
+typedef struct pv_hdr_opt_item {
+	uint32_t otype;
+	uint8_t ibk[32];
+	uint8_t data[];
+} __packed PvHdrOptItem;
+
+/* integrity protected data (by GCM tag), but non-encrypted */
+struct pv_hdr_head {
+	uint64_t magic;
+	uint32_t version;
+	uint32_t phs;
+	uint8_t iv[12];
+	uint32_t res1;
+	uint64_t nks;
+	uint64_t sea;
+	uint64_t nep;
+	uint64_t pcf;
+	PvEcdhPubKey cust_pub_key;
+	uint8_t pld[SHA512_DIGEST_LENGTH];
+	uint8_t ald[SHA512_DIGEST_LENGTH];
+	uint8_t tld[SHA512_DIGEST_LENGTH];
+} __packed;
+
+/* Must not have any padding */
+struct pv_hdr_encrypted {
+	uint8_t cust_comm_key[32];
+	uint8_t img_enc_key_1[PV_IMAGE_ENCR_KEY_SIZE / 2];
+	uint8_t img_enc_key_2[PV_IMAGE_ENCR_KEY_SIZE / 2];
+	struct psw_t psw;
+	uint64_t scf;
+	uint32_t noi;
+	uint32_t res2;
+};
+G_STATIC_ASSERT(sizeof(struct pv_hdr_encrypted) == 32 + 32 + 32 + sizeof(struct psw_t) + 8 + 4 + 4);
+
+typedef struct pv_hdr {
+	struct pv_hdr_head head;
+	struct pv_hdr_key_slot *slots;
+	struct pv_hdr_encrypted *encrypted;
+	struct pv_hdr_opt_item **optional_items;
+	uint8_t tag[16];
+} PvHdr;
+
+#endif /* LIBPV_SE_HDR_H */

ipl_tools/man/chreipl.8

@@ -139,8 +139,20 @@ the same as 0.0.5000).
 .BR "\-L" " or " "\-\-loadparm"
 Specifies an entry in the
 .BR zipl (8)
-boot menu. If this option is omitted, the default menu entry is used.
+boot menu, and an active namespace in the installed zIPL environment block
+for evaluation of zIPL environment variables in the kernel command line.
 
+This option requires an argument, which has to be a string of the form
+"nSX", where optional number 'n' indicates the boot menu entry, and optional
+token 'SX' specifies the active namespace. This token consists of a mandatory
+leading capital 'S' and of an optional trailing character 'X', which is
+either any decimal digit, indicating ID of the active site, or a capital 'S'.
+In the last case, ID of the active namespace is determined as the SSID
+(Subchannel Set ID).
+
+If the menu entry is omitted in the argument, then the default menu entry
+is used. If the 'X' is omitted, or the whole token 'SX' is omitted, then the
+common namespace of the installed zIPL environment block will be activated.
 .TP
 .BR "\-c" " or " "\-\-clear"
 Specify whether memory should be cleared on re-IPL. Possible values are 0 to
@@ -158,9 +170,11 @@ loadparm:
 \fB# chreipl ccw 0.0.7e78\fP
 
 2. Next time reboot from the CCW device with the bus-ID 0.0.7e78
-using the first entry of the zipl boot menu:
+using the first entry of the zipl boot menu and the second site namespace
+in the installed zIPL environment block for evaluation of zIPL environment
+variables in the kernel command line:
 
-\fB# chreipl ccw -d 0.0.7e78 -L 1\fP
+\fB# chreipl ccw -d 0.0.7e78 -L 1S2\fP
 .SH fcp
 Use the fcp re-IPL target for SCSI disks that are accessed by the hardware
 using Fibre Channel Protocol (FCP) channels.
@@ -306,4 +320,5 @@ that you would otherwise have to specify with the ccw or fcp target.
 .BR lsreipl (8),
 .BR zipl (8),
 .BR zipl.conf (5),
+.BR zipl-editenv (8)
 .BR reboot (8)

iucvterm/include/iucvterm/functions.h

@@ -62,7 +62,7 @@ extern ssize_t __write(int, const void*, size_t);
 extern int strmatch(const char *, const char *);
 extern int is_regex_valid(const char *);
 extern int is_client_allowed(const char *, const struct iucvterm_cfg *);
-extern void userid_cpy(char [8], const char [8]);
+extern void userid_cpy(char [9], const char [8]);
 
 extern void iucv_msg_error(const char *, uint32_t);
 extern void program_error(const char *, const char *);

libap/Makefile

@@ -0,0 +1,37 @@
+include ../common.mak
+
+lib = libap.a
+
+check-dep-lock:
+	touch check-dep-lock
+ifneq (${HAVE_LOCKFILE},0)
+	$(call check_dep, \
+		"libap", \
+		"lockfile.h", \
+		"liblockfile-devel", \
+		"HAVE_LOCKFILE=0")
+ALL_CPPFLAGS += -DHAVE_LOCKFILE
+endif
+
+check-dep-json:
+	touch check-dep-json
+ifneq (${HAVE_JSONC},0)
+	$(call check_dep, \
+		"libap", \
+		"json-c/json.h", \
+		"json-c-devel", \
+		"HAVE_JSONC=0")
+ALL_CPPFLAGS += -DHAVE_JSONC
+endif
+
+all: $(lib)
+objects = ap.o
+
+$(lib): $(objects)
+
+$(objects): check-dep-lock check-dep-json
+
+install: all
+
+clean:
+	rm -f *.o check-dep-lock check-dep-json $(lib)

libap/ap.c

@@ -0,0 +1,760 @@
+/*
+ * libap - A collection of tools for ap/vfio-ap management
+ *
+ * Copyright IBM Corp. 2022
+ *
+ * s390-tools is free software; you can redistribute it and/or modify
+ * it under the terms of the MIT license. See LICENSE for details.
+ */
+
+#include <ctype.h>
+#include <dirent.h>
+#include <err.h>
+#include <errno.h>
+#include <stdarg.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+
+#ifdef HAVE_JSONC
+#include <json-c/json.h>
+#endif /* HAVE_JSONC */
+
+#ifdef HAVE_LOCKFILE
+#include <lockfile.h>
+#endif /* HAVE_LOCKFILE */
+
+#include "lib/ap.h"
+#include "lib/util_file.h"
+#include "lib/util_libc.h"
+#include "lib/util_panic.h"
+#include "lib/util_path.h"
+#include "lib/util_udev.h"
+
+/*
+ * Return sysfs path to a bus attribute
+ * Note: caller is responsible for freeing the returned string
+ */
+static char *path_get_bus_attr(const char *bus, const char *attr)
+{
+	return util_path_sysfs("bus/%s/%s", bus, attr);
+}
+
+/*
+ * Compare two vfio_ap nodes based upon their id value. Return:
+ * -1: a < b
+ *  1: a > b
+ *  0: a == b
+ */
+static int vfio_ap_node_cmp(void *a, void *b, void *UNUSED(data))
+{
+	struct vfio_ap_node *a_node = a, *b_node = b;
+
+	if (a_node->id < b_node->id)
+		return -1;
+	return (a_node->id > b_node->id);
+}
+
+static void vfio_ap_node_add_tail(struct util_list *list, unsigned int val)
+{
+	struct vfio_ap_node *node = util_zalloc(sizeof(struct vfio_ap_node));
+
+	node->id = val;
+	util_list_add_tail(list, node);
+}
+
+/* Remove duplicate entries from a sorted vfio_ap_node list */
+static void vfio_ap_node_remove_dupes(struct util_list *list)
+{
+	struct vfio_ap_node *node, *check, *next;
+
+	util_list_iterate(list, node) {
+		/* Remove any subsequent duplicates */
+		check = util_list_next(list, node);
+
+		while (check) {
+			next = util_list_next(list, check);
+			if (node->id == check->id) {
+				util_list_remove(list, check);
+				free(check);
+			}
+			check = next;
+		}
+	}
+}
+
+static bool starts_with(const char *str, const char *s)
+{
+	size_t len = strlen(s);
+
+	return (strncmp(str, s, len) == 0);
+}
+
+/*
+ * Pass a comma-delimited string of IDs (adapter, domains or control domains)
+ * and add these IDs to the input list.  The resulting list will be sorted
+ * and duplicates removed before returning.
+ */
+static void modify_device_attr(struct util_list *list, char *value)
+{
+	unsigned int val;
+	char *curr;
+
+	curr = strtok(value, ",");
+	if (curr == NULL)
+		return;
+
+	/* Create list from input setting */
+	while (curr != NULL) {
+		val = strtol(curr, NULL, 0);
+		vfio_ap_node_add_tail(list, val);
+		curr = strtok(NULL, ",");
+	}
+
+	/* Cleanup the list */
+	util_list_sort(list, vfio_ap_node_cmp, NULL);
+	vfio_ap_node_remove_dupes(list);
+}
+
+static void load_attr_to_device(struct vfio_ap_device *dev, char *attr,
+				const char *value)
+{
+	char *v = util_strdup(value);
+
+	if (strcmp(attr, "assign_adapter") == 0)
+		modify_device_attr(dev->adapters, v);
+	else if (strcmp(attr, "assign_domain") == 0)
+		modify_device_attr(dev->domains, v);
+	else if (strcmp(attr, "assign_control_domain") == 0)
+		modify_device_attr(dev->controls, v);
+
+	free(v);
+}
+
+/**
+ * Print the contents of the vfio-ap device struct to stderr. Used for
+ * debugging.
+ *
+ * @param[in]      dev        vfio-ap device strcucture to print
+ */
+void print_ap_device(struct vfio_ap_device *dev)
+{
+	struct vfio_ap_node *node;
+
+	warnx("Device %s:", dev->uuid);
+	warnx("Type: %s", dev->type);
+	if (dev->manual)
+		warnx("Start: MANUAL");
+	else
+		warnx("Start: AUTO");
+	if (util_list_is_empty(dev->adapters)) {
+		warnx("Adapters: (none)");
+	} else {
+		warnx("Adapters:");
+		util_list_iterate(dev->adapters, node) {
+			warnx("  %u", node->id);
+		}
+	}
+	if (util_list_is_empty(dev->domains)) {
+		warnx("Domains: (none)");
+	} else {
+		warnx("Domains:");
+		util_list_iterate(dev->domains, node) {
+			warnx("  %u", node->id);
+		}
+	}
+	if (util_list_is_empty(dev->controls)) {
+		warnx("Controls: (none)");
+	} else {
+		warnx("Controls:");
+		util_list_iterate(dev->controls, node) {
+			warnx("  %u", node->id);
+		}
+	}
+}
+
+/**
+ * Determine if the input string is a valid Universally Unique Identifier
+ *
+ * @param[in]      uuid       Character string to inspect
+ *
+ * @retval         true       Specified string is a valid UUID
+ * @retval         false      Specified string is not a valid UUID
+ */
+bool is_valid_uuid(const char *uuid)
+{
+	uint32_t s1, s2, s3, s4;
+	uint64_t s5;
+	char d;
+
+	return (strlen(uuid) == 36 && sscanf(uuid, "%8x-%4x-%4x-%4x-%12x %c",
+					     &s1, &s2, &s3, &s4,
+					     (unsigned int *) &s5, &d) == 5);
+}
+
+/*
+ * For an input string of hex characters, find the nth character and return its
+ * numeric value to the caller.
+ */
+static int get_hexbyte_value(int n, const char *hexbytestr)
+{
+	int i = 0, v = 0;
+	char c;
+
+	if (strncmp(hexbytestr, "0x", 2) == 0)
+		i = 2;
+
+	/*
+	 * The specified mask is only valid if it includes at least 64 hex
+	 * digits.  The specified mask may optionally include a leading '0x'
+	 */
+	util_assert((strlen(hexbytestr) >= (size_t)(AP_MASK_SIZE - i - 1)),
+		    "Invalid hex string provided for mask: %s", hexbytestr);
+
+	c = hexbytestr[i + n / 4];
+	if (c >= '0' && c <= '9')
+		v = c - '0';
+	else if (c >= 'a' && c <= 'f')
+		v = 10 + c - 'a';
+	else if (c >= 'A' && c <= 'F')
+		v = 10 + c - 'A';
+	else
+		util_assert(false, "Could not parse hex digit '%c'", c);
+
+	return v;
+}
+
+/**
+ * For an input hex string, determine if the specified nth bit is ON or OFF.
+ *
+ * @param[in]      n          Bit number to test
+ * @param[in, out] hexbytestr Character string of hex characters
+ *
+ * @retval         0          Specified bit is OFF
+ * @retval         != 0       Specified bit is ON
+ */
+int ap_test_bit(int n, const char *hexbytestr)
+{
+	int v;
+
+	v = get_hexbyte_value(n, hexbytestr);
+
+	return v & (1 << (3 - (n % 4)));
+}
+
+/**
+ * For an input hex string, set the nth bit true/false
+ *
+ * @param[in]      n          Bit number to set
+ * @param[in, out] hexbytestr Character string of hex characters
+ * @param[in]      val        Bit is to be set ON (true) or OFF (false)
+ */
+void ap_set_bit(int n, char *hexbytestr, bool val)
+{
+	char c = 0;
+	int v, m, i = 0;
+
+	v = get_hexbyte_value(n, hexbytestr);
+
+	/* Calculate the bit mask */
+	m = (1 << (3 - (n % 4)));
+
+	/* Return if bit already at correct value */
+	if (((val) && ((v & m) != 0)) || (!val && ((v & m) == 0)))
+		return;
+
+	if (val)
+		v = v + m;
+	else
+		v = v - m;
+
+	if (v < 10)
+		c = '0' + v;
+	else if (v >= 10 && v <= 15)
+		c = 'a' + (v - 10);
+	else
+		util_assert(false, "Could not set bit value '%d'", v);
+
+	if (strncmp(hexbytestr, "0x", 2) == 0)
+		i = 2;
+
+	/* Set the new value */
+	hexbytestr[i + n / 4] = c;
+}
+
+/**
+ * Return sysfs path to vfio_ap mdev
+ * Note: caller is responsible for freeing the returned string
+ *
+ * @param[in]      uuid       Character string containing an mdev UUID
+ *
+ * @retval         != 0       sysfs path for a vfio-ap device with this UUID
+ */
+char *path_get_vfio_ap_mdev(const char *uuid)
+{
+	return util_path_sysfs("%s/%s", VFIO_AP_PARENT_PATH, uuid);
+}
+
+/**
+ * Return path to mdevctl config file for specified UUID
+ * Note: caller is responsible for freeing the returned string
+ *
+ * @param[in]      uuid       Character string containing an mdev UUID
+ *
+ * @retval         != 0       config file path for vfio-ap device with this UUID
+ */
+char *path_get_vfio_ap_mdev_config(const char *uuid)
+{
+	char *path;
+
+	util_asprintf(&path, "%s/%s", VFIO_AP_CONFIG_PATH, uuid);
+	return path;
+}
+
+/**
+ * Return sysfs path to vfio_ap mdev attribute (matrix, remove, ...)
+ * Note: caller is responsible for freeing the returned string
+ *
+ * @param[in]      uuid       Character string containing an mdev UUID
+ * @param[in]      attr       Character string containing device attribute name
+ *
+ * @retval         != 0       sysfs path to specified attribute for this device
+ */
+char *path_get_vfio_ap_attr(const char *uuid, const char *attr)
+{
+	return util_path_sysfs("%s/%s/%s", VFIO_AP_PARENT_PATH, uuid, attr);
+}
+
+/**
+ * Return path to ap udev config file
+ * Note: caller is responsible for freeing the returned string
+ *
+ * @retval         != 0       path to the ap udev config file
+ */
+char *path_get_ap_udev(void)
+{
+	char *path;
+
+	util_asprintf(&path, "%s", AP_UDEV_FILE);
+	return path;
+}
+
+/**
+ * Take one line from the active 'matrix' attribute and parse it
+ * into a list of adapters and domains.  Each line of the 'matrix'
+ * attribute is presented as a "adapter.domain" (e.g. "03.0005") where the
+ * numeric values are always hexadecimal.
+ * In a case where no adapters are assigned, a valid string might be ".0005"
+ * In a case where no domains are assigned, a valid string might be "03."
+ *
+ * @param[in, out] dev        Vfio-ap struct that will be updated
+ * @param[in]      matrix     Character string to parse
+ */
+void vfio_ap_parse_matrix(struct vfio_ap_device *dev, char *matrix)
+{
+	char *curr;
+	int val;
+
+	if (!matrix)
+		return;
+
+	if (*matrix != '.') {
+		/* Handle a device with adapters */
+		curr = strtok(matrix, ".");
+		val = strtol(curr, NULL, 16);
+		vfio_ap_node_add_tail(dev->adapters, val);
+		curr = strtok(NULL, "\n");
+	} else {
+		/* Handle a device with no adapters */
+		curr = strtok(matrix + 1, "\n");
+	}
+
+	/* Leave now if the device has no domains */
+	if (!curr)
+		return;
+	/* Get the domain */
+	val = strtol(curr, NULL, 16);
+	vfio_ap_node_add_tail(dev->domains, val);
+}
+
+/**
+ * Function to sort the results of repeated vfio_ap_parse_matrix calls
+ *
+ * @param[in, out] dev        Vfio-ap struct whose lists will be sorted
+ */
+void vfio_ap_sort_matrix_results(struct vfio_ap_device *dev)
+{
+	/* Sort the lists for later use */
+	util_list_sort(dev->adapters, vfio_ap_node_cmp, NULL);
+	util_list_sort(dev->domains, vfio_ap_node_cmp, NULL);
+
+	/* Run the lists and delete duplicates */
+	vfio_ap_node_remove_dupes(dev->adapters);
+	vfio_ap_node_remove_dupes(dev->domains);
+}
+
+/**
+ * Take the string provided by the active 'control_domains' attribute and
+ * parse it into a list of control domains
+ *
+ * @param[in, out] dev        Vfio-ap struct that will be updated
+ * @param[in]      control    Character string to parse
+ */
+void vfio_ap_parse_control(struct vfio_ap_device *dev, char *control)
+{
+	char *curr;
+	int val;
+
+	curr = strtok(control, "\n");
+
+	while (curr != NULL) {
+		val = strtol(curr, NULL, 16);
+		vfio_ap_node_add_tail(dev->controls, val);
+		curr = strtok(NULL, "\n");
+	}
+}
+
+#ifdef HAVE_JSONC
+
+/**
+ * For a given path, read in the contents.  If no path is provided, get the
+ * input from stdin instead.
+ *
+ * @param[in]      path       Path to mdevctl config file
+ * @param[in, out] dev        Vfio-ap struct that will be updated
+ *
+ * @retval         0          Config read successfully, dev updated
+ * @retval         -1         Failed to read config, dev may have partial info
+ */
+int vfio_ap_read_device_config(const char *path, struct vfio_ap_device *dev)
+{
+	json_object *root, *type, *start, *attrs, *attr;
+	int i, len, rc = 0;
+	const char *val;
+
+	if (path == NULL)
+		root = json_object_from_fd(STDIN_FILENO);
+	else
+		root = json_object_from_file(path);
+	if (root == NULL)
+		return -1;
+
+	if (json_object_object_get_ex(root, "mdev_type", &type)) {
+		val = json_object_get_string(type);
+		if (!val)
+			goto err;
+		dev->type = util_strdup(val);
+		if (strcmp(val, "vfio_ap-passthrough") != 0)
+			goto err;
+	}
+
+	if (json_object_object_get_ex(root, "start", &start)) {
+		val = json_object_get_string(start);
+		if (!val)
+			goto err;
+		if (strcmp(val, "auto") == 0)
+			dev->manual = false;
+		else if (strcmp(val, "manual") == 0)
+			dev->manual = true;
+		else
+			goto err;
+	}
+
+	if (json_object_object_get_ex(root, "attrs", &attrs)) {
+		len = json_object_array_length(attrs);
+		for (i = 0; i < len; i++) {
+			attr = json_object_array_get_idx(attrs, i);
+			json_object_object_foreach(attr, key, setting) {
+				val = json_object_get_string(setting);
+				load_attr_to_device(dev, key, val);
+			}
+		}
+	}
+
+out:
+	json_object_put(root);
+	return rc;
+
+err:
+	rc = -1;
+	goto out;
+}
+
+#else
+int vfio_ap_read_device_config(const char *path, struct vfio_ap_device *dev)
+{
+	return -1;
+}
+#endif /* HAVE_JSONC */
+
+/**
+ * Allocate and initialize a vfio-ap device structure.
+ *
+ * @retval         !=0        Address of the new vfio-ap device structure
+ */
+struct vfio_ap_device *vfio_ap_device_new(void)
+{
+	struct vfio_ap_device *dev;
+
+	dev = util_zalloc(sizeof(struct vfio_ap_device));
+
+	dev->manual = false;
+	dev->type = NULL;
+	dev->uuid = NULL;
+
+	dev->adapters = util_list_new(struct vfio_ap_node, node);
+	dev->domains = util_list_new(struct vfio_ap_node, node);
+	dev->controls = util_list_new(struct vfio_ap_node, node);
+
+	return dev;
+}
+
+/**
+ * Re-initialize a vfio-ap device structure, leaving the structure allocated.
+ *
+ * @param[in, out] dev        Vfio-ap struct that will be re-initialized
+ */
+void vfio_ap_device_clear(struct vfio_ap_device *dev)
+{
+	if (dev == NULL)
+		return;
+
+	dev->manual = false;
+	if (dev->type) {
+		free(dev->type);
+		dev->type = NULL;
+	}
+	if (dev->uuid) {
+		free(dev->uuid);
+		dev->uuid = NULL;
+	}
+
+	ap_list_remove_all(dev->adapters);
+	ap_list_remove_all(dev->domains);
+	ap_list_remove_all(dev->controls);
+}
+
+/**
+ * Clear and release a vfio-ap device structure.
+ *
+ * @param[in, out] dev        Vfio-ap struct that will be freed
+ */
+void vfio_ap_device_free(struct vfio_ap_device *dev)
+{
+	if (dev == NULL)
+		return;
+
+	vfio_ap_device_clear(dev);
+	util_list_free(dev->adapters);
+	util_list_free(dev->domains);
+	util_list_free(dev->controls);
+	free(dev);
+}
+
+static int read_sysfs_mask(const char *path, char *mask, int size)
+{
+	return util_file_read_line(mask, size, "%s", path);
+}
+
+/**
+ * Get the apmask and aqmask from sysfs
+ *
+ * @param[in, out] ap         Buffer to hold apmask contents
+ * @param[in, out] aq         Buffer to hold aqmask contents
+ * @param[in]      size       Size of the mask buffers
+ *
+ * @retval         0          Both mask values read successfully
+ * @retval         != 0       Failed to read one or both mask values
+ */
+int ap_read_sysfs_masks(char *ap, char *aq, int size)
+{
+	char *path;
+	int rc = 0;
+
+	path = path_get_bus_attr("ap", "apmask");
+	rc = read_sysfs_mask(path, ap, size);
+	free(path);
+	if (rc != 0)
+		goto out;
+	path = path_get_bus_attr("ap", "aqmask");
+	rc = read_sysfs_mask(path, aq, size);
+	free(path);
+
+out:
+	return rc;
+}
+
+/**
+ * Get the apmask and aqmask from udev, falling back to sysfs if a udev rule
+ * is not available or does not provide values for both masks.
+ * The values in read_ap and read_aq tell the caller whether each mask was
+ * successfully loaded from udev or if the sysfs value was substituted
+ * Note: both ap and aq must point to a string that is at least AP_MASK_SIZE
+ * in length.
+ *
+ * @param[in]      path       Path to the ap udev file
+ * @param[in, out] ap         Buffer to hold apmask contents
+ * @param[in, out] aq         Buffer to hold aqmask contents
+ * @param[out]     read_ap    Specifies if an apmask value was read from udev
+ * @param[out]     read_aq    Specifies if an aqmask value was read from udev
+ *
+ * @retval         true       Udev file read successfully or did not exist
+ * @retval         false      Udev file exists but error was encountered
+ */
+bool ap_read_udev_masks(char *path, char *ap, char *aq, bool *read_ap,
+			bool *read_aq)
+{
+	struct util_udev_entry_node *entry;
+	struct util_udev_file *file = NULL;
+	struct util_udev_line_node *line;
+	char sysap[AP_MASK_SIZE];
+	char sysaq[AP_MASK_SIZE];
+	int rc;
+
+	/* Assume we fail to read both masks */
+	*read_ap = *read_aq = false;
+
+	/* If a udev file doesn't exist, quietly use the active masks */
+	if (!util_path_exists(path))
+		goto out;
+
+	rc = util_udev_read_file(path, &file);
+
+	/* If errors were encountered reading the udev file, exit now */
+	if (rc)
+		return false;
+
+	util_list_iterate(&file->lines, line) {
+		entry = util_list_start(&line->entries);
+
+		/* Skip comments and empty lines. */
+		if (!entry)
+			continue;
+
+		if (starts_with(entry->key, "ATTR{")) {
+			if (strstr(entry->key, "apmask")) {
+				util_strlcpy(ap, entry->value, AP_MASK_SIZE);
+				*read_ap = true;
+			} else if (strstr(entry->key, "aqmask")) {
+				util_strlcpy(aq, entry->value, AP_MASK_SIZE);
+				*read_aq = true;
+			}
+		}
+	}
+
+	util_udev_free_file(file);
+
+out:
+	/* If we didn't read in masks, use current sysfs values */
+	if ((!*read_ap) || (!*read_aq)) {
+		rc = ap_read_sysfs_masks(sysap, sysaq, AP_MASK_SIZE);
+		if (rc != 0)
+			return false;
+		if (!*read_ap)
+			strcpy(ap, sysap);
+		if (!*read_aq)
+			strcpy(aq, sysaq);
+	}
+
+	return true;
+}
+
+/**
+ * For a given bitmask, create a list of vfio_ap_node entries corresponding
+ * to the ON bits in the mask.
+ *
+ * @param[in]      mask       Character string of hex characters
+ * @param[in, out] list       List to be updated with entries for each ON bit
+ */
+void ap_mask_to_list(char *mask, struct util_list *list)
+{
+	int i;
+
+	if (mask == NULL || list == NULL)
+		return;
+
+	for (i = 0; i <= AP_MAX_MASK_VALUE; i++) {
+		if (ap_test_bit(i, mask))
+			vfio_ap_node_add_tail(list, i);
+	}
+
+	/* Could have duplicates if the input list was not empty */
+	util_list_sort(list, vfio_ap_node_cmp, NULL);
+	vfio_ap_node_remove_dupes(list);
+}
+
+/**
+ * For the specified list, remove all elements and free each node
+ *
+ * @param[in, out] list       List that will have all entries removed
+ */
+void ap_list_remove_all(struct util_list *list)
+{
+	struct ap_node *node;
+
+	while (!util_list_is_empty(list)) {
+		node = util_list_start(list);
+		util_list_remove(list, node);
+		free(node);
+	}
+}
+
+#ifdef HAVE_LOCKFILE
+static int ap_lockfile_create(int flags)
+{
+	return lockfile_create(AP_LOCKFILE, AP_LOCK_RETRIES, flags);
+}
+
+/**
+ * Acquire the ap config lock using this process PID (L_PID)
+ *
+ * @retval         0          Lock successfully acquired on behalf of L_PID
+ * @retval         != 0       Error, lock was not obtained
+ */
+int ap_get_lock(void)
+{
+	return ap_lockfile_create(L_PID);
+}
+
+/**
+ * Acquire the ap config lock using the parent process PID (L_PPID) -- intended
+ * for use by the mdevctl callout ap-check utility
+ *
+ * @retval         0          Lock successfully acquired on behalf of L_PPID
+ * @retval         != 0       Error, lock was not obtained
+ */
+int ap_get_lock_callout(void)
+{
+	return ap_lockfile_create(L_PPID);
+}
+
+/**
+ * Release the ap config lock
+ *
+ * @retval         0          Lock successfully released or file didn't exist
+ * @retval         != 0       Error removing the lockfile
+ */
+int ap_release_lock(void)
+{
+	return lockfile_remove(AP_LOCKFILE);
+}
+#else
+/* If no liblockfile, actions are performed without acquiring the file lock */
+int ap_get_lock(void)
+{
+	return 0;
+}
+
+int ap_get_lock_callout(void)
+{
+	return 0;
+}
+
+int ap_release_lock(void)
+{
+	return 0;
+}
+
+#endif /* HAVE_LOCKFILE */