From 04be704083a2369f0ad5db614204ae16cdd9b92a Mon Sep 17 00:00:00 2001 From: Benjamin Block Date: Fri, 17 Sep 2021 13:00:41 +0200 Subject: [PATCH] chreipl-fcp-mpath: record the event subject WWID if it repr. the tgt MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When we have identified the current event subject to represent the re-IPL target, we record its WWID for future identification in a stateful ID-file (per default: /run/udev/chreiplzfcpmp-ipl-volume-id). In addition to the WWID, we also record the current re-IPL triplet (::), so that when that changes - e.g. due to an operator manually changing the re-IPL target -, we know that the recorded WWID is stale. This record may be used in cases when the current re-IPL target is completely gone from the system, so we can't used it as comparison object for when events arrive for paths that go to the same volume, but don't have the same I_T_L nexus. They however have the same WWID. We may use these (new) paths as replacement for the one that is completely gone. The new helper `chreipl-fcp-mpath-record-volume-identifier` uses the kernel scsi-device attribute `wwid` as source for the WWID (verbatim). As with reading the re-IPL firmware information, when writing to the ID-file, a lock is taken via `flock`, to prevent overlapping writes/reads to the file. Reviewed-by: Steffen Maier Signed-off-by: Benjamin Block Signed-off-by: Jan Höppner --- chreipl-fcp-mpath/.gitignore | 1 + chreipl-fcp-mpath/Makefile | 3 +- .../chreipl-fcp-mpath-common.sh.in | 90 ++++++++++++++++++- ...eipl-fcp-mpath-record-volume-identifier.in | 73 +++++++++++++++ chreipl-fcp-mpath/chreipl-fcp-mpath.mak | 3 + .../udev/rules.d/70-chreipl-fcp-mpath.rules | 17 ++++ 6 files changed, 185 insertions(+), 2 deletions(-) create mode 100644 chreipl-fcp-mpath/chreipl-fcp-mpath-record-volume-identifier.in diff --git a/chreipl-fcp-mpath/.gitignore b/chreipl-fcp-mpath/.gitignore index a490115a..704bf5f5 100644 --- a/chreipl-fcp-mpath/.gitignore +++ b/chreipl-fcp-mpath/.gitignore @@ -2,3 +2,4 @@ /chreipl-fcp-mpath-common.sh /chreipl-fcp-mpath-is-ipl-tgt /chreipl-fcp-mpath-is-reipl-zfcp +/chreipl-fcp-mpath-record-volume-identifier diff --git a/chreipl-fcp-mpath/Makefile b/chreipl-fcp-mpath/Makefile index 8f2c8396..299ab389 100644 --- a/chreipl-fcp-mpath/Makefile +++ b/chreipl-fcp-mpath/Makefile @@ -45,7 +45,8 @@ CHREIPL_FCP_MPATH_COMMON := \ chreipl-fcp-mpath-common.sh CHREIPL_FCP_MPATH_UDEV_HELPER := \ chreipl-fcp-mpath-is-ipl-tgt \ - chreipl-fcp-mpath-is-reipl-zfcp + chreipl-fcp-mpath-is-reipl-zfcp \ + chreipl-fcp-mpath-record-volume-identifier $(CHREIPL_FCP_MPATH_UDEV_HELPER) $(CHREIPL_FCP_MPATH_COMMON): $(MAKEFILE_LIST) $(CHREIPL_FCP_MPATH_UDEV_HELPER) $(CHREIPL_FCP_MPATH_COMMON): % : %.in diff --git a/chreipl-fcp-mpath/chreipl-fcp-mpath-common.sh.in b/chreipl-fcp-mpath/chreipl-fcp-mpath-common.sh.in index 5eb0beb0..1ef9c25c 100644 --- a/chreipl-fcp-mpath/chreipl-fcp-mpath-common.sh.in +++ b/chreipl-fcp-mpath/chreipl-fcp-mpath-common.sh.in @@ -8,6 +8,7 @@ # GNU coreutils: # - mktemp # - readlink +# - sync # util-linux: # - flock @@ -15,7 +16,8 @@ # SEQNUM # (1) expand failed globs to an empty string -shopt -s nullglob +# (2) extended pattern matching to strip leading/trailing whitespaces +shopt -s nullglob extglob # (1) don't overwrite existing files using redirects (e.g.: `>`) set -o noclobber @@ -48,6 +50,7 @@ else unset tlg fi +declare -gr ID_FILE='@chreiplzfcpmp-id-file@' declare -gr FW_LOCK_FILE='@chreiplzfcpmp-fwlock-file@' declare -gA TRAP_EXIT_FN=() @@ -63,6 +66,52 @@ function trap_exit() { } trap trap_exit EXIT +# Output variables: +# id_file_unlock_exclusive_create() - call to unlock when finished with +# critical section +# +# XXX: `id_file_lock_*` can't be taken recursively +function id_file_lock_exclusive_create() { + declare -g ID_FILE_LOCK="" + + # prevent concurrent file creation + # + # First, open the file defined in ${ID_FILE} for writing; this will + # succeed and create the file only if it doesn't exist already. If the + # file already exist, the first open attempt will fail and we fall + # back to opening it only for reading; this will always succeed if the + # file already exists (the reason why the first attempty failed). In + # both cases store the corresponding file descriptor in + # ${ID_FILE_LOCK}. + # + # XXX: This should be race free. + # open() with O_EXCL... is atomic (we set `noclobber` as shell + # option); at least as long as we talk about a local FS. + if ! { exec {ID_FILE_LOCK}>"${ID_FILE}"; } 2>/dev/null; then + { exec {ID_FILE_LOCK}<"${ID_FILE}"; } 2>/dev/null \ + || return 1 + fi + + declare -gf id_file_unlock_exclusive_create 1>/dev/null + function id_file_unlock_exclusive_create() { + if [ -v ID_FILE_LOCK ]; then + sync "${ID_FILE}" 2>/dev/null + # release file and implicitly the lock, if taken + exec {ID_FILE_LOCK}>&- + unset ID_FILE_LOCK + fi + + unset "TRAP_EXIT_FN[id_file_unlock_exclusive_create]" + } + TRAP_EXIT_FN+=( + [id_file_unlock_exclusive_create]=id_file_unlock_exclusive_create + ) + + flock --exclusive --timeout 5 "${ID_FILE_LOCK}" || return 2 + + return 0 +} + # Output variables: # firmware_unlock_shared() - call to unlock when finished with critical section # @@ -208,3 +257,42 @@ function sdev_get_fcp_addressing() { SDEV_WWPN="${rport_wwpn}" return 0 } + +# Input: +# 1: path to the scsi device in sysfs +# Output variables: +# SDEV_WWID +function sdev_get_wwid() { + local sdev="${1}" + local -a wwid + + declare -g SDEV_WWID="" + + # read the volume identifier without stripping any content + # + # XXX: we can read the WWID file, even if the SDEV is currently not + # operational (e.g.: due to the path has gone away), as long as + # the VPD PG 83 is still cached in the kernel; and the page gets + # only released on SDEV device release. + { readarray -d "" -t wwid < "${sdev}"/wwid; } 2>/dev/null \ + || return 1 + if '@DEBUG@'; then declare -p wwid 1>&2; fi + + # test whether we read something + # + # This strips all leading spaces from the beginning of the read WWID + # (until the first non-space or NUL character), and checks whether the + # result is empty. + # Hence, we return early if the WWID consists of only whitespace. + # + # XXX: there could be unexpected characters in the returned ID. + # `scsi_id` from the udev helpers sanitizes the strings it reads + # from the devices, so they can be used in environment variables + # without much danger. + # But we don't export anything here, so it should be fine. + [ "${wwid[0]/#*([[:space:]])}" != "" ] || return 2 + + # shellcheck disable=2034 + SDEV_WWID="${wwid[0]}" + return 0 +} diff --git a/chreipl-fcp-mpath/chreipl-fcp-mpath-record-volume-identifier.in b/chreipl-fcp-mpath/chreipl-fcp-mpath-record-volume-identifier.in new file mode 100644 index 00000000..59eca8b2 --- /dev/null +++ b/chreipl-fcp-mpath/chreipl-fcp-mpath-record-volume-identifier.in @@ -0,0 +1,73 @@ +#!/bin/bash +# SPDX-License-Identifier: MIT +# +# chreipl-fcp-mpath: use multipath information to change FCP IPL target +# (C) Copyright IBM Corp. 2021 +# +# Uses the following system-utilities (and shell-builtins): +# Those necessary for sourced library: +# - chreipl-fcp-mpath-common.sh +# GNU coreutils: +# - truncate +# util-linux: +# - hexdump + +# Record the identification of the volume we want to re-IPL from +# +# Makes use of udev event environment variables: +# DM_UUID +# SUBSYSTEM +# DEVPATH + +# shellcheck disable=SC2034 +declare -gr debug_trace_tag=10rvid +# shellcheck disable=SC1091 +source '@chreiplzfcpmp-lib@' || exit 127 + +function id_file_record_ipl_information() { + local sdev_wwid="${1}" ipl_busid="${2}" ipl_wwpn="${3}" ipl_lun="${4}" + + # lock file before writing ID, so noone sees any intermediate state + id_file_lock_exclusive_create || return 1 + + # reset ID without removing the file (necessary for the locking to work + # properly, since the FD we use for locking is on this file/inode) + truncate --no-create --size=0 "${ID_FILE}" || return 3 + echo -ne "${sdev_wwid}\x00${ipl_busid}\x00${ipl_wwpn}\x00${ipl_lun}\x00" \ + >>"${ID_FILE}" || return 4 + + if '@DEBUG@'; then hexdump -vC "${ID_FILE}" 1>&2; fi + + id_file_unlock_exclusive_create + + return 0 +} + +if [[ "${DM_UUID}" == mpath-* ]]; then + # Assume Multipath Device Mapper Device; + # e.g.: DEVPATH = /devices/virtual/block/dm-0 + declare sdev + + for sdev in /sys/"${DEVPATH}"/slaves/sd*/device; do + if sdev_get_wwid "${sdev}"; then + break + fi + done + unset sdev + +elif [ "${SUBSYSTEM}" = block ]; then + # Assume SCSI Disk; + # e.g.: DEVPATH = /devices/css0/0.0.0014/0.0.1700/host0/rport-0:0-0/target0:0:0/0:0:0:1074806808/block/sds + + sdev_get_wwid /sys/"${DEVPATH}"/device +fi +# shellcheck disable=SC2153 +[ "${SDEV_WWID}" != "" ] || exit 1 + +firmware_get_ipl_information || exit 2 +# shellcheck disable=SC2153 +id_file_record_ipl_information \ + "${SDEV_WWID}" "${IPL_BUSID}" "${IPL_WWPN}" "${IPL_LUN}" \ + || exit 3 + +exit 0 diff --git a/chreipl-fcp-mpath/chreipl-fcp-mpath.mak b/chreipl-fcp-mpath/chreipl-fcp-mpath.mak index d5d49479..08bd03a0 100644 --- a/chreipl-fcp-mpath/chreipl-fcp-mpath.mak +++ b/chreipl-fcp-mpath/chreipl-fcp-mpath.mak @@ -23,6 +23,8 @@ DEBUGOUTDIR = $(UDEVRUNDIR) INSTALL_EXEC = $(INSTALL) -g $(GROUP) -o $(OWNER) --preserve-timestamps INSTALL_DATA = $(INSTALL_EXEC) --mode=0644 +# used for data exchange and synchronization across the different helpers +chreiplzfcpmp-id-file = $(UDEVRUNDIR)/chreiplzfcpmp-ipl-volume-id # file used to implement mutual exclusion when accessing firmware IPL info: # - this should be something that is (practically) always available, so we # dont have to worry about fallbacks or error-handling; @@ -37,6 +39,7 @@ define chreiplzfcpmp-sed-buildvar-replace = tmpout=$$(mktemp -p ./ .make.tmp.XXXXXXXXXXXXXXXX) && { \ $(SED) -E \ -e 's|@DEBUG@|$(if $(filter 1,$(D)),true,false)|g' \ + -e 's|@chreiplzfcpmp-id-file@|$(chreiplzfcpmp-id-file)|g' \ -e 's|@chreiplzfcpmp-fwlock-file@|$(chreiplzfcpmp-fwlock-file)|g' \ -e 's|@chreiplzfcpmp-lib@|$(CHREIPLZFCPMPDIR)/chreipl-fcp-mpath-common.sh|g' \ -e 's|@debugoutdir@|$(DEBUGOUTDIR)|g' \ diff --git a/chreipl-fcp-mpath/udev/rules.d/70-chreipl-fcp-mpath.rules b/chreipl-fcp-mpath/udev/rules.d/70-chreipl-fcp-mpath.rules index 2084eadf..682a88ba 100644 --- a/chreipl-fcp-mpath/udev/rules.d/70-chreipl-fcp-mpath.rules +++ b/chreipl-fcp-mpath/udev/rules.d/70-chreipl-fcp-mpath.rules @@ -49,8 +49,25 @@ PROGRAM!="chreipl-fcp-mpath-is-ipl-tgt", \ GOTO="chreipl_fcp_mpath_not_direct_match" ENV{CHREIPL_FCP_MPATH_IS_TGT}="true" +# Record the WWID, Device-Bus-ID, Remote WWPN, and LUN of the ReIPL target +# (see `chreipl-fcp-mpath-is-ipl-vol` for usecases). This information +# might change, depending on whether the machine operator changes the ReIPL +# target to a different volume. +# +# XXX: Because the kernel doesn't generate any events upon changing of +# the ReIPL target, the chreipl-fcp-mpath toolset can't take any +# actions until the next path event for the new target is +# generated. Following that, we assume that when the machine +# operator changes the ReIPL target, the new target is reachable +# and in a good state at this point in time. +PROGRAM!="chreipl-fcp-mpath-record-volume-identifier", \ + GOTO="chreipl_fcp_mpath_try_change_ipl_path" +GOTO="chreipl_fcp_mpath_try_change_ipl_path" + # If the even subject is not a direct match (not the sdev that is the current # ReIPL target, and not a mpath device that contains the current ReIPL target) LABEL="chreipl_fcp_mpath_not_direct_match" +LABEL="chreipl_fcp_mpath_try_change_ipl_path" + LABEL="chreipl_fcp_mpath_end"