From 1bcfcd3253caef143930699de12ab6879005efda Mon Sep 17 00:00:00 2001 From: Benjamin Block Date: Fri, 17 Sep 2021 12:44:52 +0200 Subject: [PATCH] chreipl-fcp-mpath: test if event subject represents re-IPL target MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Now that we know that the system uses s390x IPL and the current re-IPL target is from FCP, whenever we get a udev event that indicates a path state changed, we have to figure out whether it affects the path that is currently set as re-IPL target, or goes to the same volume (so we might use it as alternative re-IPL path). Add a new helper `chreipl-fcp-mpath-is-ipl-tgt` for this task. Based on the information provided in /sys/firmware/reipl/fcp/{device,wwpn,lun} it figures out whether the current event subject has the same triplet :: (in T10 SCSI: I_T_L nexus), or whether one of its parts (in case of dm-multipath) has. If true, we know for sure, that we deal with an event affecting the current re-IPL setting, and continue. When accessing the re-IPL firmware information a lock is taken via the `flock` utility (from util-linux). This is done so concurrent changes from the toolset, that are added in a later commit, don't result in inconsistent reads. Reviewed-by: Steffen Maier Signed-off-by: Benjamin Block Signed-off-by: Jan Höppner --- chreipl-fcp-mpath/.gitignore | 1 + chreipl-fcp-mpath/Makefile | 1 + .../chreipl-fcp-mpath-common.sh.in | 166 ++++++++++++++++++ .../chreipl-fcp-mpath-is-ipl-tgt.in | 56 ++++++ chreipl-fcp-mpath/chreipl-fcp-mpath.mak | 8 + .../udev/rules.d/70-chreipl-fcp-mpath.rules | 30 ++++ 6 files changed, 262 insertions(+) create mode 100644 chreipl-fcp-mpath/chreipl-fcp-mpath-is-ipl-tgt.in diff --git a/chreipl-fcp-mpath/.gitignore b/chreipl-fcp-mpath/.gitignore index e1121d76..a490115a 100644 --- a/chreipl-fcp-mpath/.gitignore +++ b/chreipl-fcp-mpath/.gitignore @@ -1,3 +1,4 @@ # build artifacts /chreipl-fcp-mpath-common.sh +/chreipl-fcp-mpath-is-ipl-tgt /chreipl-fcp-mpath-is-reipl-zfcp diff --git a/chreipl-fcp-mpath/Makefile b/chreipl-fcp-mpath/Makefile index 4ea3423f..8f2c8396 100644 --- a/chreipl-fcp-mpath/Makefile +++ b/chreipl-fcp-mpath/Makefile @@ -44,6 +44,7 @@ clean: chreipl-fcp-mpath-clean CHREIPL_FCP_MPATH_COMMON := \ chreipl-fcp-mpath-common.sh CHREIPL_FCP_MPATH_UDEV_HELPER := \ + chreipl-fcp-mpath-is-ipl-tgt \ chreipl-fcp-mpath-is-reipl-zfcp $(CHREIPL_FCP_MPATH_UDEV_HELPER) $(CHREIPL_FCP_MPATH_COMMON): $(MAKEFILE_LIST) diff --git a/chreipl-fcp-mpath/chreipl-fcp-mpath-common.sh.in b/chreipl-fcp-mpath/chreipl-fcp-mpath-common.sh.in index dd64cc5c..5eb0beb0 100644 --- a/chreipl-fcp-mpath/chreipl-fcp-mpath-common.sh.in +++ b/chreipl-fcp-mpath/chreipl-fcp-mpath-common.sh.in @@ -7,10 +7,15 @@ # Uses the following system-utilities (and shell-builtins): # GNU coreutils: # - mktemp +# - readlink +# util-linux: +# - flock # Makes use of udev event environment variables: # SEQNUM +# (1) expand failed globs to an empty string +shopt -s nullglob # (1) don't overwrite existing files using redirects (e.g.: `>`) set -o noclobber @@ -42,3 +47,164 @@ then else unset tlg fi + +declare -gr FW_LOCK_FILE='@chreiplzfcpmp-fwlock-file@' + +declare -gA TRAP_EXIT_FN=() +declare -gf trap_exit 1>/dev/null +function trap_exit() { + local fn + + for fn in "${TRAP_EXIT_FN[@]}"; do + "${fn}" + done + + trap - EXIT +} +trap trap_exit EXIT + +# Output variables: +# firmware_unlock_shared() - call to unlock when finished with critical section +# +# XXX: `firmware_lock_*` can't be taken recursively +function firmware_lock_shared() { + declare -g FIRMWARE_LOCK="" + + # Open the file defined in ${FW_LOCK_FILE} for reading, and store the + # corresponding file descriptor in ${FIRMWARE_LOCK} (it doesn't matter + # whether this is a normal file or directory). This file descriptor + # will only be used for locking - not for actual I/O. + { exec {FIRMWARE_LOCK}<"${FW_LOCK_FILE}"; } 2>/dev/null || return 1 + + declare -gf firmware_unlock_shared 1>/dev/null + function firmware_unlock_shared() { + if [ -v FIRMWARE_LOCK ]; then + # release file and implicitly the lock, if taken + exec {FIRMWARE_LOCK}<&- + unset FIRMWARE_LOCK + fi + + unset "TRAP_EXIT_FN[firmware_unlock_shared]" + } + TRAP_EXIT_FN+=([firmware_unlock_shared]=firmware_unlock_shared) + + flock --shared --timeout 5 "${FIRMWARE_LOCK}" || return 2 + + return 0 +} + +# Output variables: +# IPL_TYPE +# IPL_BUSID +# IPL_WWPN +# IPL_LUN +function firmware_get_ipl_information() { + declare -g IPL_TYPE="" IPL_BUSID="" IPL_WWPN="" IPL_LUN="" + + # Take lock so we don't see any intermediate state from other helpers + # running in parallel + firmware_lock_shared || return 5 + + { read -r IPL_TYPE _ < /sys/firmware/reipl/reipl_type; } 2>/dev/null \ + || return 1 + { read -r IPL_BUSID _ < /sys/firmware/reipl/fcp/device; } 2>/dev/null \ + || return 2 + { read -r IPL_WWPN _ < /sys/firmware/reipl/fcp/wwpn; } 2>/dev/null \ + || return 3 + { read -r IPL_LUN _ < /sys/firmware/reipl/fcp/lun; } 2>/dev/null \ + || return 4 + + firmware_unlock_shared + + # show read values in debug log if enabled + if '@DEBUG@'; then + declare -p IPL_TYPE IPL_BUSID IPL_WWPN IPL_LUN 1>&2 + fi + + return 0 +} + +# Input: +# 1: absolute canonical path to the scsi device in sysfs, e.g.: +# /sys/devices/css0/0.0.0014/0.0.1700/host1/rport-1:0-0/target1:0:0/1:0:0:1075789848 +# Output variables: +# SDEV_LUN +function sdev_get_lun() { + local sdev="${1}" sdev_lun_str + # bash uses `intmax_t` as width for integer variables, and glibc + # defines this either as `long int` on 64 bit systems, or + # `long long int` on other. + local -i sdev_lun=0 fcp_lun=0 + + sdev_lun_str="${sdev##*:}" + # e.g.: 1075789848 + [[ "${sdev_lun_str}" == +([[:digit:]]) ]] || return 1 + # "cast" to integer + sdev_lun="${sdev_lun_str}" + + # convert the Linux integer LUN format to the hexadecimal 64 bit T10 + # LUN representation format used by many s390x interfaces + (( fcp_lun = (((sdev_lun >> 0) & 0xffff) << 48) + | (((sdev_lun >> 16) & 0xffff) << 32) + | (((sdev_lun >> 32) & 0xffff) << 16) + | (((sdev_lun >> 48) & 0xffff) << 0) )) + # the '0x' prefix is part of the length + printf -v SDEV_LUN "%#018llx" "${fcp_lun}" + + # show read values in debug log if enabled + if '@DEBUG@'; then + declare -p SDEV_LUN 1>&2 + fi + + return 0 +} + +# Input: +# 1: path to the scsi device in sysfs, e.g.: +# /sys/devices/css0/0.0.0014/0.0.1700/host1/rport-1:0-0/target1:0:0/1:0:0:1075789848 +# , or a symlink pointing to the scsi device, e.g.: +# /sys/class/block/sds/device +# Output variables: +# SDEV_BUSID +# SDEV_WWPN +# SDEV_LUN +function sdev_get_fcp_addressing() { + local sdev="${1}" fcp_lun rport rport_wwpn zfcp_dev + + declare -g SDEV_BUSID="" SDEV_WWPN="" SDEV_LUN="" + + sdev="$(readlink -se "${sdev}")" || return 1 + + # get the LUN for this SDEV + # + # sets ${SDEV_LUN} + sdev_get_lun "${sdev}" || return 2 + + # get the WWPN of the remote port this SDEV is attached to + printf -v rport "%s" "${sdev}"/../../fc_remote_ports/rport-*:*-* + # e.g.: /sys/devices/css0/0.0.0016/0.0.1740/host0/rport-0:0-1/fc_remote_ports/rport-0:0-1 + [ "${rport}" != "" ] || return 3 + + # XXX: This works even if the rport is currently in a bad + # state, so e.g. when it has just gone down because of a + # cable pull. + { read -r rport_wwpn _ < "${rport}"/port_name; } 2>/dev/null \ + || return 4 + + # The Linux kernel doesn't guarantee the same format as in + # /sys/firmware/..., so make sure it is the one we expect. + [[ "${rport_wwpn}" =~ ^0x[[:xdigit:]]{1,16}$ ]] || return 5 + # the '0x' prefix is part of the length + printf -v rport_wwpn "%#018llx" "${rport_wwpn}" + + # get the Device Bus-ID of the device via which this SDEV is attached + zfcp_dev="$(readlink -se "${sdev}"/../../../..)" || return 6 + # e.g.: /sys/devices/css0/0.0.0016/0.0.1740 + zfcp_dev="${zfcp_dev##*/}" + + # shellcheck disable=2034 + SDEV_BUSID="${zfcp_dev}" + # shellcheck disable=2034 + SDEV_WWPN="${rport_wwpn}" + return 0 +} diff --git a/chreipl-fcp-mpath/chreipl-fcp-mpath-is-ipl-tgt.in b/chreipl-fcp-mpath/chreipl-fcp-mpath-is-ipl-tgt.in new file mode 100644 index 00000000..671e65da --- /dev/null +++ b/chreipl-fcp-mpath/chreipl-fcp-mpath-is-ipl-tgt.in @@ -0,0 +1,56 @@ +#!/bin/bash +# SPDX-License-Identifier: MIT +# +# chreipl-fcp-mpath: use multipath information to change FCP IPL target +# (C) Copyright IBM Corp. 2021 +# +# Uses the following system-utilities (and shell-builtins): +# Those necessary for sourced library: +# - chreipl-fcp-mpath-common.sh + +# Find out whether the device in udev event environment variable ${DEVPATH} +# represents the device we want to re-IPL from. We do this by comparing +# Device-Bus-ID/Target-WWPN/LUN of the individual SDEVs to the parameters set +# in `/sys/firmware/reipl/fcp/`. +# +# Makes use of udev event environment variables: +# DM_UUID +# SUBSYSTEM +# DEVPATH + +# shellcheck disable=SC2034 +declare -gr debug_trace_tag=05iilt +# shellcheck disable=SC1091 +source '@chreiplzfcpmp-lib@' || exit 127 + +firmware_get_ipl_information || exit 1 + +if [[ "${DM_UUID}" == mpath-* ]]; then + # Assume Multipath Device Mapper Device; + # e.g.: DEVPATH = /devices/virtual/block/dm-0 + declare sdev + + # depends on `nullglob` from `chreipl-fcp-mpath-common.sh` + for sdev in /sys/"${DEVPATH}"/slaves/sd*/device; do + sdev_get_fcp_addressing "${sdev}" || continue + [ "${SDEV_LUN}" = "${IPL_LUN}" ] || continue + [ "${SDEV_WWPN}" = "${IPL_WWPN}" ] || continue + [ "${SDEV_BUSID}" = "${IPL_BUSID}" ] || continue + + exit 0 + done + unset sdev + +elif [ "${SUBSYSTEM}" = block ]; then + # Assume SCSI Disk; + # e.g.: DEVPATH = /devices/css0/0.0.0014/0.0.1700/host0/rport-0:0-0/target0:0:0/0:0:0:1074806808/block/sds + + sdev_get_fcp_addressing /sys/"${DEVPATH}"/device || exit 2 + [ "${SDEV_LUN}" = "${IPL_LUN}" ] || exit 3 + [ "${SDEV_WWPN}" = "${IPL_WWPN}" ] || exit 4 + [ "${SDEV_BUSID}" = "${IPL_BUSID}" ] || exit 5 + + exit 0 +fi + +exit 6 diff --git a/chreipl-fcp-mpath/chreipl-fcp-mpath.mak b/chreipl-fcp-mpath/chreipl-fcp-mpath.mak index 642f3713..d5d49479 100644 --- a/chreipl-fcp-mpath/chreipl-fcp-mpath.mak +++ b/chreipl-fcp-mpath/chreipl-fcp-mpath.mak @@ -23,6 +23,13 @@ DEBUGOUTDIR = $(UDEVRUNDIR) INSTALL_EXEC = $(INSTALL) -g $(GROUP) -o $(OWNER) --preserve-timestamps INSTALL_DATA = $(INSTALL_EXEC) --mode=0644 +# file used to implement mutual exclusion when accessing firmware IPL info: +# - this should be something that is (practically) always available, so we +# dont have to worry about fallbacks or error-handling; +# - at the same time, it should not be used by anything else with flock(2) to +# hold a lock for long periods. +chreiplzfcpmp-fwlock-file = /sys/firmware/reipl + .DELETE_ON_ERROR: # export build-time definitions to the scripts/built-components @@ -30,6 +37,7 @@ define chreiplzfcpmp-sed-buildvar-replace = tmpout=$$(mktemp -p ./ .make.tmp.XXXXXXXXXXXXXXXX) && { \ $(SED) -E \ -e 's|@DEBUG@|$(if $(filter 1,$(D)),true,false)|g' \ + -e 's|@chreiplzfcpmp-fwlock-file@|$(chreiplzfcpmp-fwlock-file)|g' \ -e 's|@chreiplzfcpmp-lib@|$(CHREIPLZFCPMPDIR)/chreipl-fcp-mpath-common.sh|g' \ -e 's|@debugoutdir@|$(DEBUGOUTDIR)|g' \ $(1) > $${tmpout} \ diff --git a/chreipl-fcp-mpath/udev/rules.d/70-chreipl-fcp-mpath.rules b/chreipl-fcp-mpath/udev/rules.d/70-chreipl-fcp-mpath.rules index dbf78561..2084eadf 100644 --- a/chreipl-fcp-mpath/udev/rules.d/70-chreipl-fcp-mpath.rules +++ b/chreipl-fcp-mpath/udev/rules.d/70-chreipl-fcp-mpath.rules @@ -23,4 +23,34 @@ LABEL="chreipl_fcp_mpath_path_change" TEST!="/sys/firmware/ipl", GOTO="chreipl_fcp_mpath_end" PROGRAM!="chreipl-fcp-mpath-is-reipl-zfcp", GOTO="chreipl_fcp_mpath_end" +# Consider the following scenarios. +# Either: +# +# (A) We recognized a new SCSI Disk. This might represent: +# (a) the path we want to ReIPL from. +# +# Or: +# +# (B) We recognized a PATH_ event for a multipath device. This might represent: +# the path we want to ReIPL from: +# (a) went away; +# (b) came back online; +# an alternative path to the volume we want to ReIPL from: +# (c) went away; +# (d) came back online. + +# Test whether the affected device is, or contains, the current IPL target. +# +# This covers scenarios: +# (A) (a), +# (B) (a)/(b)/(c)/(d) +PROGRAM!="chreipl-fcp-mpath-is-ipl-tgt", \ + ENV{CHREIPL_FCP_MPATH_IS_TGT}="false", \ + GOTO="chreipl_fcp_mpath_not_direct_match" +ENV{CHREIPL_FCP_MPATH_IS_TGT}="true" + +# If the even subject is not a direct match (not the sdev that is the current +# ReIPL target, and not a mpath device that contains the current ReIPL target) +LABEL="chreipl_fcp_mpath_not_direct_match" + LABEL="chreipl_fcp_mpath_end"