mirror of
https://github.com/ibm-s390-linux/s390-tools.git
synced 2026-08-05 02:14:52 +00:00
The opticsmon tool started out as a tool for monitoring the health of optical modules in directly attached PCI NICs. In the future however it will also monitor the health of other PCI devices. In particular in a first step it will monitor the health of directly attached NVMe devices. To reflect this broadening of its scope rename opticsmon to zpcimon. Add zpcimon.service and install it both under the new name and symlinked as opticsmon.service for backwards compatibility. Since users are expected to mostly just enable the service this keeps old instructions just working. Reviewed-by: Jan Höppner <hoeppner@linux.ibm.com> Signed-off-by: Niklas Schnelle <schnelle@linux.ibm.com> Signed-off-by: Jan Höppner <hoeppner@linux.ibm.com>
1638 lines
45 KiB
Bash
Executable File
1638 lines
45 KiB
Bash
Executable File
#!/bin/sh
|
|
#
|
|
# dbginfo.sh - Tool to collect runtime, configuration, and trace information
|
|
#
|
|
# Copyright IBM Corp. 2002, 2026
|
|
#
|
|
# s390-tools is free software; you can redistribute it and/or modify
|
|
# it under the terms of the MIT license. See LICENSE for details.
|
|
#
|
|
|
|
# Switching to neutral locale
|
|
LC_ALL=C
|
|
export LC_ALL
|
|
|
|
########################################
|
|
# Global used variables
|
|
readonly SCRIPTNAME="${0##*/}" # current name of this script
|
|
readonly FULLPATHSCRIPT="$(readlink -f "${0}")"
|
|
#
|
|
readonly DATETIME="$(date +%Y-%m-%d-%H-%M-%S 2>/dev/null)"
|
|
readonly DOCKER=$(if type docker >/dev/null 2>&1; then echo "YES"; else echo "NO"; fi)
|
|
readonly DUMP2TAR_OK=$(if type dump2tar >/dev/null 2>&1; then echo "YES"; else echo "NO"; fi)
|
|
readonly HW="$(uname -m 2>/dev/null)"
|
|
readonly IFS_ORI="${IFS}"
|
|
# retrieve and split kernel version
|
|
readonly KERNEL_BASE="$(uname -r 2>/dev/null)"
|
|
readonly KERNEL_VERSION=$(echo ${KERNEL_BASE} | cut -d'.' -f1 )
|
|
readonly KERNEL_MAJOR_REVISION=$(echo ${KERNEL_BASE} | cut -d'.' -f2 )
|
|
readonly KERNEL_MINOR_REVISION=$(echo ${KERNEL_BASE} | cut -d'.' -f3 | sed 's/[^0-9].*//g')
|
|
readonly KERNEL_INFO=${KERNEL_VERSION}.${KERNEL_MAJOR_REVISION}.${KERNEL_MINOR_REVISION}
|
|
readonly KUBERNETES=$(if type kubectl >/dev/null 2>&1; then echo "YES"; else echo "NO"; fi)
|
|
readonly KUBERNETES_LOG_LINES=50
|
|
readonly KVM=$(if type virsh >/dev/null 2>&1; then echo "YES"; else echo "NO"; fi)
|
|
# The file to indicate that another instance of the script is already running
|
|
readonly LOCKFILE="/tmp/${SCRIPTNAME}.lock"
|
|
# check limits for logfiles like /var/log/messages
|
|
readonly LOG_FILE_SIZE_CHECK=50 # max logfile size in MB
|
|
readonly LOG_FILE_AGE_CHECK=7 # age in days to include for size checking
|
|
# Mount point of the debug file system
|
|
readonly MOUNT_POINT_DEBUGFS="/sys/kernel/debug"
|
|
# Network devices
|
|
readonly NETWORK_DEVS=$(cd /sys/class/net; ls -d */ 2>/dev/null | sed 's/\///g')
|
|
# distro info
|
|
readonly OSPRETTY="$(cat /etc/os* 2>/dev/null | grep -m1 PRETTY_NAME | sed 's/\"//g')"
|
|
readonly OS_NAME="${OSPRETTY##*=}"
|
|
readonly PODMAN=$(if type podman >/dev/null 2>&1; then echo "YES"; else echo "NO"; fi)
|
|
# The processor ID for the first processor
|
|
readonly PROCESSORID="$(grep -E ".*processor 0:.*" /proc/cpuinfo | \
|
|
sed 's/.*identification[[:space:]]*\=[[:space:]]*\([[:alnum:]]*\).*/\1/g')"
|
|
readonly PROCESSORVERSION="$(grep -E ".*processor 0:.*" /proc/cpuinfo | \
|
|
sed 's/.*version[[:space:]]*\=[[:space:]]*\([[:alnum:]]*\).*/\1/g')"
|
|
if test "x${PROCESSORVERSION}" = "xFF" || test "x${PROCESSORVERSION}" = "xff"; then
|
|
RUNTIME_ENVIRONMENT=$(grep -E "VM00.*Control Program.*" /proc/sysinfo | \
|
|
sed 's/.*:[[:space:]]*\([[:graph:]]*\).*/\1/g')
|
|
else
|
|
RUNTIME_ENVIRONMENT="LPAR"
|
|
fi
|
|
# check for Dynamic Partition Mode
|
|
readonly DPM_UUID=$(grep "LPAR UUID" /proc/sysinfo | \
|
|
sed 's/.*:[[:space:]]*\([[:graph:]]*\).*/\1/g')
|
|
if test "${#DPM_UUID}" -eq 0; then
|
|
DPM_MODE="NO"
|
|
else
|
|
DPM_MODE="YES"
|
|
fi
|
|
readonly SYSTEMHOSTNAME="$(hostname -s 2>/dev/null)" # hostname of system being analysed
|
|
readonly TERMINAL="$(tty 2>/dev/null)"
|
|
# timeout seconds TOS / kill timeout TOKS
|
|
readonly TOS=15
|
|
readonly TOKS=30
|
|
readonly TIMEOUT=$(if type timeout >/dev/null 2>&1; then echo "YES"; else echo "NO"; fi)
|
|
readonly TIMEOUT_OK=$(if test "x${TIMEOUT}" = "xYES"; then
|
|
if test $(timeout -k ${TOKS} ${TOS} uname 2>/dev/null) = $(uname); then
|
|
echo "YES"; else echo "WRONG_VERSION"; fi
|
|
else echo "NO"; fi)
|
|
readonly ZDEV_CONF=$(lszdev --configured 2>/dev/null | wc -l)
|
|
readonly ZDEV_OFF=$(lszdev --offline 2>/dev/null | wc -l)
|
|
readonly ZDEV_ONL=$(lszdev --online 2>/dev/null | wc -l)
|
|
|
|
paramWORKDIR_BASE="/tmp" # initial default path
|
|
|
|
########################################
|
|
# print dbginfo.sh version info
|
|
print_version() {
|
|
cat <<EOF
|
|
${SCRIPTNAME}: Debug information script version %S390_TOOLS_VERSION%
|
|
Copyright IBM Corp. 2002, 2026
|
|
EOF
|
|
}
|
|
|
|
########################################
|
|
# print how to use this script
|
|
print_usage() {
|
|
print_version
|
|
|
|
cat <<EOF
|
|
|
|
Usage: ${SCRIPTNAME} [OPTION]
|
|
|
|
This script collects runtime, configuration and trace information on
|
|
a Linux on IBM Z installation for debugging purposes.
|
|
|
|
It also traces information about z/VM if the Linux runs under z/VM.
|
|
Virtualization platform data is collected on a host serving this.
|
|
|
|
Default location for data collection and final tar file is "/tmp/".
|
|
The collected information is written to a TAR archive named
|
|
|
|
DBGINFO-[date]-[time]-[hostname]-[processorid].tgz
|
|
|
|
where [date] and [time] are the date and time when debug data is collected.
|
|
[hostname] indicates the hostname of the system the data was collected from.
|
|
The [processorid] is taken from the processor 0 and indicates the processor
|
|
identification.
|
|
|
|
Options:
|
|
-d|--directory specify the directory where the data collection
|
|
stores the temporary data and the final archive.
|
|
-h|--help print this help
|
|
-v|--version print version information
|
|
-c|--check online base system information (no data collection)
|
|
|
|
EOF
|
|
}
|
|
|
|
########################################
|
|
# check for oversized logfiles and missing rotation
|
|
logfile_checker() {
|
|
local counter
|
|
local logfile
|
|
local logfiles
|
|
|
|
# find files bigger than recommended
|
|
counter=$(find $1 -maxdepth 1 -type f -mtime -${LOG_FILE_AGE_CHECK} \
|
|
-size ${LOG_FILE_SIZE_CHECK}M | wc -l)
|
|
|
|
echo " ${counter} logfiles over ${LOG_FILE_SIZE_CHECK} MB"
|
|
# maybe check for rotation of base names
|
|
if [ ${counter} -ne 0 ]; then
|
|
for logfile in $(find $1 -maxdepth 1 -type f -mtime -${LOG_FILE_AGE_CHECK} \
|
|
-size ${LOG_FILE_SIZE_CHECK}M -print); do
|
|
# use a neutral separtor ':' as concat is different in some bash
|
|
# insert the 'blank' for later use in for loop
|
|
# add the base name before '.' or '-' only for checks
|
|
logfiles="${logfiles}: ${logfile%%[.-]*}"
|
|
done
|
|
# change separator to new line for sorting
|
|
logfiles=$(echo "${logfiles}" | sed s'/:/\n/g' | sort -u)
|
|
for logfile in ${logfiles}; do
|
|
counter=$(ls ${logfile}* 2>/dev/null | wc -l)
|
|
if [ ${counter} -eq 1 ]; then
|
|
echo " CHECK - ${logfile} may miss a rotation"
|
|
else
|
|
echo " OK - ${logfile}* may have a rotation "\
|
|
"in place: ${counter} files"
|
|
fi
|
|
done
|
|
fi
|
|
}
|
|
|
|
########################################
|
|
# print basic info and online checks
|
|
print_check() {
|
|
print_version
|
|
cat <<EOF
|
|
|
|
Hardware platform = ${HW}
|
|
Runtime environment = ${RUNTIME_ENVIRONMENT} - DPM: ${DPM_MODE}
|
|
$(cat /proc/sysinfo | grep 'Name')
|
|
Kernel version = ${KERNEL_INFO}
|
|
OS version / distro = ${OS_NAME}
|
|
KVM host = ${KVM}
|
|
container host Kubernetes: ${KUBERNETES} - docker: ${DOCKER} - podman: ${PODMAN}
|
|
|
|
Current user = $(whoami) (must be root for data collection)
|
|
Date and time = $(date)
|
|
Uptime =$(uptime)
|
|
Number of coredumps = $(corecumpctl 2>/dev/null | wc -l)
|
|
zdevice onl/conf/offl = ${ZDEV_ONL} / ${ZDEV_CONF} / ${ZDEV_OFF}
|
|
Log file check =$(logfile_checker "/var/log*")
|
|
|
|
Working directory = $(ls -d ${paramWORKDIR_BASE} 2>&1 && df -k ${paramWORKDIR_BASE})
|
|
$(ls -ltr ${paramWORKDIR_BASE}/DBGINFO*tgz 2>/dev/null | tail -2)
|
|
$(ls ${LOCKFILE} 2>/dev/null && echo " Warning: dbginfo running since: $(cat ${LOCKFILE})")
|
|
Tool/command dependency check successful:
|
|
timeout = ${TIMEOUT_OK}
|
|
dump2tar = ${DUMP2TAR_OK}
|
|
|
|
This is a console output only - no data was saved using option -c !
|
|
|
|
EOF
|
|
}
|
|
|
|
#######################################
|
|
# Parsing the command line and pre checks
|
|
while [ ${#} -gt 0 ]; do
|
|
case ${1} in
|
|
--help|-h)
|
|
print_usage
|
|
exit 0
|
|
;;
|
|
--version|-v)
|
|
print_version
|
|
exit 0
|
|
;;
|
|
--directory|-d)
|
|
paramWORKDIR_BASE=${2}
|
|
if test -z "${paramWORKDIR_BASE}"; then
|
|
echo "${SCRIPTNAME}: Error: No directory specified for data collection!"
|
|
echo
|
|
exit 1
|
|
elif test ! -d "${paramWORKDIR_BASE}"; then
|
|
echo "${SCRIPTNAME}: Error: The specified directory does not exist!"
|
|
echo
|
|
exit 1
|
|
else
|
|
# jump to next param
|
|
shift
|
|
fi
|
|
;;
|
|
--check|-c)
|
|
print_check
|
|
exit 0
|
|
;;
|
|
-*|--*|*)
|
|
echo
|
|
echo "${SCRIPTNAME}: invalid option \"${1}\""
|
|
echo "Try '${SCRIPTNAME} --help' for more information"
|
|
echo
|
|
exit 1
|
|
;;
|
|
esac
|
|
# next parameter, if already last the final shift will do termination
|
|
shift
|
|
done
|
|
|
|
# finally verification to run as root
|
|
if test "$(/usr/bin/id -u 2>/dev/null)" -ne 0; then
|
|
echo "${SCRIPTNAME}: Error: You must be user root to run data collection \"${SCRIPTNAME}\"!"
|
|
echo "${SCRIPTNAME}: Hint: for base system information you may run \"${SCRIPTNAME} -c\""
|
|
exit 1
|
|
fi
|
|
|
|
#########################################
|
|
# The base working directory and derieved path info
|
|
readonly WORKDIR_BASE="$(echo "${paramWORKDIR_BASE}" | sed -e 's#/$##')/"
|
|
# The current working directory for the actual script execution
|
|
if test -z "${PROCESSORID}"; then
|
|
readonly WORKDIR_CURRENT="DBGINFO-${DATETIME}-${SYSTEMHOSTNAME:-localhost}"
|
|
else
|
|
readonly WORKDIR_CURRENT="DBGINFO-${DATETIME}-${SYSTEMHOSTNAME:-localhost}-${PROCESSORID}"
|
|
fi
|
|
# The current path where the collected information is put together
|
|
readonly WORKPATH="${WORKDIR_BASE}${WORKDIR_CURRENT}/"
|
|
# The current TAR archive that finally includes all collected information
|
|
readonly WORKARCHIVE="${WORKDIR_BASE}${WORKDIR_CURRENT}.tgz"
|
|
# The log file of activities from this script execution
|
|
readonly LOGFILE="${WORKPATH}dbginfo.log"
|
|
# File names for output files per section (duplicates are ok)
|
|
readonly OUTPUT_FILE_BRIDGE="${WORKPATH}network.out"
|
|
readonly OUTPUT_FILE_CMD="${WORKPATH}runtime.out"
|
|
readonly OUTPUT_FILE_COREDUMPCTL="${WORKPATH}coredump.out" # separate file needed
|
|
readonly OUTPUT_FILE_DASD="${WORKPATH}dasd.out"
|
|
readonly OUTPUT_FILE_DOCKER="${WORKPATH}docker_runtime.out"
|
|
readonly OUTPUT_FILE_ETHTOOL="${WORKPATH}network.out"
|
|
readonly OUTPUT_FILE_HYPTOP="${WORKPATH}runtime.out"
|
|
readonly OUTPUT_FILE_JOURNALCTL="${WORKPATH}journalctl.out"
|
|
readonly OUTPUT_FILE_KVM="${WORKPATH}kvm_runtime.out"
|
|
readonly OUTPUT_FILE_LSOF="${WORKPATH}open_files.out"
|
|
readonly OUTPUT_FILE_NETWORK="${WORKPATH}network.out"
|
|
readonly OUTPUT_FILE_NVME="${WORKPATH}runtime.out"
|
|
readonly OUTPUT_FILE_OVS="${WORKPATH}network.out"
|
|
readonly OUTPUT_FILE_PODMAN="${WORKPATH}podman_runtime.out"
|
|
readonly OUTPUT_FILE_ISW="${WORKPATH}installed_sw.out"
|
|
readonly OUTPUT_FILE_TC="${WORKPATH}network.out"
|
|
readonly OUTPUT_FILE_VMCMD="${WORKPATH}zvm_runtime.out"
|
|
readonly OUTPUT_FILE_UDEVDB="${WORKPATH}udevdb.out"
|
|
# Base file names for different output files - no extension !
|
|
readonly OUTPUT_FILE_KUBERNETES="${WORKPATH}kubernetes"
|
|
readonly OUTPUT_FILE_OSAOAT="${WORKPATH}network"
|
|
readonly OUTPUT_FILE_SYSFS="${WORKPATH}sysfs"
|
|
|
|
# define order of collection steps
|
|
# - state/debug files are collected first, to avoid overwriting by command execution
|
|
ALL_STEPS="\
|
|
collect_sysfs\
|
|
collect_procfs\
|
|
collect_configfiles\
|
|
collect_initrd_configfiles\
|
|
collect_cmdsout\
|
|
collect_hyptop\
|
|
collect_vmcmdsout\
|
|
collect_network\
|
|
collect_osaoat\
|
|
collect_ethtool\
|
|
collect_tc\
|
|
collect_bridge\
|
|
collect_ovs\
|
|
collect_kvm\
|
|
collect_container\
|
|
collect_nvme\
|
|
collect_dasd\
|
|
collect_logfiles\
|
|
post_processing\
|
|
create_package\
|
|
environment_cleanup\
|
|
"
|
|
# The amount of steps running the whole collections, without last cleanup
|
|
readonly COLLECTION_COUNT=`expr $(echo ${ALL_STEPS} | wc -w) - 1`
|
|
|
|
########################################
|
|
# Collection of proc fs entries
|
|
PROCFILES="\
|
|
/proc/buddyinfo\
|
|
/proc/cio_ignore\
|
|
/proc/cmdline\
|
|
/proc/cpuinfo\
|
|
/proc/crypto\
|
|
/proc/dasd\
|
|
/proc/devices\
|
|
/proc/diskstats\
|
|
/proc/interrupts\
|
|
/proc/iomem\
|
|
/proc/kallsyms\
|
|
/proc/mdstat\
|
|
/proc/meminfo\
|
|
/proc/misc\
|
|
/proc/modules\
|
|
/proc/mounts\
|
|
/proc/net\
|
|
/proc/partitions\
|
|
/proc/qeth\
|
|
/proc/qeth_perf\
|
|
/proc/qeth_ipa_takeover\
|
|
/proc/sched_debug\
|
|
/proc/schedstat\
|
|
/proc/service_levels\
|
|
/proc/slabinfo\
|
|
/proc/softirqs\
|
|
/proc/stat\
|
|
/proc/swaps\
|
|
/proc/sys/kernel\
|
|
/proc/sys/vm\
|
|
/proc/sysinfo\
|
|
/proc/version\
|
|
/proc/zoneinfo\
|
|
"
|
|
|
|
# Adding files to PROCFILES in case scsi devices are available
|
|
if test -e /proc/scsi; then
|
|
PROCFILES="${PROCFILES}\
|
|
$(find /proc/scsi -type f -perm /444 2>/dev/null)\
|
|
"
|
|
fi
|
|
|
|
########################################
|
|
|
|
LOGFILES="\
|
|
/root/anaconda-ks.cfg\
|
|
/root/original-ks.cfg\
|
|
/run/containerd/events.log\
|
|
/run/docker/libcontainerd/containerd/events.log\
|
|
/run/udev/chreiplzfcpmp-[0-9][0-9][a-z][a-z][a-z][a-z]-*\
|
|
/var/log/anaconda\
|
|
/var/log/anaconda.*\
|
|
/var/log/audit\
|
|
/var/log/boot*\
|
|
/var/log/crash/*/vmcore-dmesg.txt\
|
|
/var/log/cron*\
|
|
/var/log/dmesg*\
|
|
/var/log/dnf.*\
|
|
/var/log/dracut.log*\
|
|
/var/log/IBMtape.trace\
|
|
/var/log/IBMtape.errorlog\
|
|
/var/log/kdump.log\
|
|
/var/log/libvirt\
|
|
/var/log/lin_tape.trace\
|
|
/var/log/lin_tape.errorlog\
|
|
/var/log/messages*\
|
|
/var/log/openvswitch/ovs-vswitchd.log\
|
|
/var/log/openvswitch/ovsdb-server.log\
|
|
/var/log/pbl.log\
|
|
/var/log/sa\
|
|
/var/log/syslog*\
|
|
/var/log/yum.log\
|
|
/var/log/YaST2\
|
|
/var/log/zypp*\
|
|
"
|
|
|
|
########################################
|
|
|
|
CONFIGFILES="\
|
|
/boot/grub2/grub.cfg\
|
|
/boot/loader/entries/*.conf\
|
|
/boot/zipl/active_devices.txt\
|
|
/boot/zipl/config\
|
|
/conf/\
|
|
/etc/*.conf\
|
|
/etc/*release\
|
|
/etc/anacrontab\
|
|
/etc/apparmor.d\
|
|
/etc/audit\
|
|
/etc/auto.*\
|
|
/etc/cmdline\
|
|
/etc/cmdline.d\
|
|
/etc/conf.d\
|
|
/etc/containers\
|
|
/etc/cron.*\
|
|
/etc/crontab\
|
|
/etc/crypttab\
|
|
/etc/dasd.*\
|
|
/etc/default\
|
|
/etc/depmod.d\
|
|
/etc/dnf/dnf.conf\
|
|
/etc/docker\
|
|
/etc/dracut.conf.d\
|
|
/etc/exports\
|
|
/etc/fpc*\
|
|
/etc/fstab\
|
|
/etc/groups\
|
|
/etc/grub.d\
|
|
/etc/hosts*\
|
|
/etc/iscsi\
|
|
/etc/inittab\
|
|
/etc/kdump\
|
|
/etc/libvirt\
|
|
/etc/logrotate.d\
|
|
/etc/lvm\
|
|
/etc/modprobe.conf*\
|
|
/etc/modprobe.d\
|
|
/etc/modules-load.d/*.conf\
|
|
/etc/mtab\
|
|
/etc/multipath\
|
|
/etc/netplan\
|
|
/etc/network\
|
|
/etc/networks\
|
|
/etc/NetworkManager\
|
|
/etc/opencryptoki/*.conf\
|
|
/etc/pam.d\
|
|
/etc/profile\
|
|
/etc/profile.d\
|
|
/etc/pki/tls/openssl.cnf\
|
|
/etc/rc.d\
|
|
/etc/rc.local\
|
|
/etc/resolv.*\
|
|
/etc/rsyslog.d\
|
|
/etc/selinux\
|
|
/etc/ssl/openssl.conf\
|
|
/etc/ssl/openssl.cnf\
|
|
/etc/sysconfig\
|
|
/etc/sysctl.d\
|
|
/etc/syslog*\
|
|
/etc/systemd\
|
|
/etc/udev*\
|
|
/etc/xinet.d\
|
|
/etc/zfcp.*\
|
|
/lib/modprobe.d\
|
|
$(find /lib/modules -name modules.dep 2>/dev/null)\
|
|
/lib/systemd/system/docker.service\
|
|
/lib/udev/rules.d\
|
|
/usr/lib/modprobe.d\
|
|
/usr/lib/modules-load.d/*.conf\
|
|
/usr/lib/systemd/system/\
|
|
/usr/local/lib/modprobe.d\
|
|
/usr/local/lib/modules-load.d/*.conf\
|
|
/run/modprobe.d\
|
|
/run/modules-load.d/*.conf\
|
|
/run/udev/chreiplzfcpmp-ipl-volume-id\
|
|
/run/udev/rules.d\
|
|
/run/zdev_id.env\
|
|
/run/zdev.*\
|
|
"
|
|
|
|
########################################
|
|
# system state commands (non alphabetical order here)
|
|
CMDS="uname -a\
|
|
:uptime\
|
|
:timedatectl\
|
|
:chronyc sources\
|
|
:chronyc sourcestats\
|
|
:chronyc ntpdata\
|
|
:chronyc tracking\
|
|
:zhypinfo\
|
|
:last\
|
|
:lscpu -ae\
|
|
:lscpu -ye\
|
|
:lscpumf -i\
|
|
:lsmem\
|
|
:zmemtopo -t\
|
|
:lsmod\
|
|
:lsshut\
|
|
:lsstp\
|
|
:runlevel\
|
|
:sysctl -a\
|
|
:systemctl --all --no-pager show\
|
|
:systemctl --all --no-pager list-units\
|
|
:systemctl --all --no-pager list-unit-files\
|
|
:systemd-delta\
|
|
:ulimit -a\
|
|
:env --null | sort --zero-terminated | tr '\000' '\n'\
|
|
:find /boot -print0 | sort -z | xargs -0 -n 10 ls -ld\
|
|
:find /var/crash -print0 | sort -z | xargs -0 -n 10 ls -ld\
|
|
:df -h\
|
|
:df -i\
|
|
:mount\
|
|
"
|
|
|
|
# Z device subsystem commands (first commands in non alphabetical order)
|
|
CMDS="${CMDS}\
|
|
:lschp\
|
|
:lscss --vpm\
|
|
:lszdev\
|
|
:find /dev -print0 | sort -z | xargs -0 -n 10 ls -ld\
|
|
:lspci -t\
|
|
:lspci -vvv\
|
|
:lstape\
|
|
:smc_dbg\
|
|
"
|
|
|
|
# block device and other scsi device commands (in non alphabetical order)
|
|
# command groups itself have an intentional topical grouping and order
|
|
CMDS="${CMDS}\
|
|
:lsblk\
|
|
:lsblk -o +FSTYPE,UUID\
|
|
:lsdasd\
|
|
:lsdasd -u\
|
|
:pvpath -qa\
|
|
:dmsetup ls\
|
|
:dmsetup ls --tree\
|
|
:dmsetup table\
|
|
:dmsetup table --target multipath\
|
|
:dmsetup status\
|
|
:multipathd -k'show config'\
|
|
:multipathd -k'show config local'\
|
|
:multipathd -k'show maps'\
|
|
:multipathd -k'show topo'\
|
|
:multipathd -k'show paths'\
|
|
:multipathd -k'show paths format %w|%a|%r|%L|%p|%i|%d|%D|%t|%T|%o|%0|%C' | sort\
|
|
:multipathd -k'show maps stats'\
|
|
:multipathd -k'show maps status'\
|
|
:multipathd -k'show status'\
|
|
:multipathd -k'show daemon'\
|
|
:multipathd -k'show blacklist'\
|
|
:multipathd -k'show devices'\
|
|
:multipath -v6 -ll\
|
|
:multipath -d\
|
|
:multipath -t\
|
|
:lszfcp\
|
|
:lszfcp -D\
|
|
:lszfcp -V\
|
|
:ziorep_config -ADM\
|
|
:lsscsi\
|
|
:fdisk -l\
|
|
:lvmconfig\
|
|
:lvdisplay -m\
|
|
:vgs -v\
|
|
:pvs -v\
|
|
:lvs -v\
|
|
"
|
|
|
|
# crypto specific commands
|
|
CMDS="${CMDS}\
|
|
:cpacfinfo # for MSA 13 info\
|
|
:cpacfinfo -m -f -a -n\
|
|
:ep11info -D\
|
|
:ep11info -H\
|
|
:ep11info -M\
|
|
:icainfo\
|
|
:icastats --all\
|
|
:ivp.e # IBM CCA package install check\
|
|
:ls -al /usr/lib64/opencryptoki/stdll\
|
|
:lszcrypt -VV\
|
|
:pkcsconf -i\
|
|
:pkcsconf -s\
|
|
:pkcsconf -l\
|
|
:pkcsconf -t\
|
|
:pkcsconf -m\
|
|
:systemctl status pkcsslotd\
|
|
"
|
|
|
|
# product / distro specific commands
|
|
CMDS="${CMDS}\
|
|
:java -version\
|
|
:qemu-ga -V\
|
|
:SPident # SLES service package\
|
|
"
|
|
|
|
# commands with long output at the end
|
|
CMDS="${CMDS}\
|
|
:cat /root/.bash_history\
|
|
:dmesg -s 1048576\
|
|
:ps -emo pid,tid,nlwp,policy,user,tname,ni,pri,psr,sgi_p,stat,wchan,start_time,time,pcpu,pmem,vsize,size,rss,share,command\
|
|
:ps -eHo pid,tid,nlwp,policy,user,tname,ni,pri,psr,sgi_p,stat,wchan,start_time,time,pcpu,pmem,vsize,size,rss,share,command\
|
|
:ps axX\
|
|
"
|
|
|
|
# commands with separate output file
|
|
CMDS="${CMDS}\
|
|
:apt list >> '${OUTPUT_FILE_ISW}'\
|
|
:snap list --all >> '${OUTPUT_FILE_ISW}'\
|
|
:coredumpctl && coredumpctl info -o ${OUTPUT_FILE_COREDUMPCTL}\
|
|
:journalctl --all --no-pager --lines=100000 --output=short-precise\
|
|
>> '${OUTPUT_FILE_JOURNALCTL}'\
|
|
:lsof >> '${OUTPUT_FILE_LSOF}'\
|
|
:rpm -qa | sort >> '${OUTPUT_FILE_ISW}'\
|
|
:udevadm info --export-db >> '${OUTPUT_FILE_UDEVDB}'\
|
|
"
|
|
|
|
########################################
|
|
# network comamnds for separate output file with ip -br a as header command
|
|
NETWORK_CMDS="ip -br a\
|
|
:firewall-cmd --list-all\
|
|
:ifconfig -a\
|
|
:ip a sh\
|
|
:ip -s -s link\
|
|
:ip link show\
|
|
:ip neigh list\
|
|
:ip ntable\
|
|
:ip route list\
|
|
:ip route list table all\
|
|
:ip rule list\
|
|
:ipcs -a\
|
|
:iptables -L # about to be replaced by nftables\
|
|
:lsqeth\
|
|
:netstat -pantu\
|
|
:netstat -s\
|
|
:nft list tables # is replacing iptables\
|
|
:nm-tool\
|
|
:nstat -az\
|
|
:openssl engine # deprecated in OpenSSL 3.n\
|
|
:openssl list --providers 2>/dev/null # redirect help on older systems\
|
|
:opticsmon --module-info\ # Now called zpcimon but kept for compatibility
|
|
:route -n\
|
|
"
|
|
|
|
########################################
|
|
DOCKER_CMDS="docker version\
|
|
:docker info\
|
|
:docker images\
|
|
:docker network ls\
|
|
:docker ps -a\
|
|
:docker stats --no-stream\
|
|
:systemctl status docker.service\
|
|
"
|
|
########################################
|
|
PODMAN_CMDS="podman version\
|
|
:podman info\
|
|
:podman images\
|
|
:podman network ls\
|
|
:podman ps -a\
|
|
:podman stats --no-stream\
|
|
:podman pod stats -a --no-stream\
|
|
:ls -l /var/lib/containers/storage/overlay/* # list of files is sufficent\
|
|
:ls -l /var/log/crio/pods/*/*\
|
|
"
|
|
|
|
########################################
|
|
KUBERNETES_CMDS="kubectl version\
|
|
:oc version # get also OCP version if installed\
|
|
:kubectl top pod\
|
|
:kubectl top node\
|
|
:kubectl describe pod -A\
|
|
:kubectl describe node -A\
|
|
:kubectl describe endpoints\
|
|
:ls -l /var/log/containers # list of logs is sufficent\
|
|
:kubectl cluster-info dump >>${OUTPUT_FILE_KUBERNETES}.dmp\
|
|
"
|
|
|
|
########################################
|
|
VM_CMDS="q userid\
|
|
:q users\
|
|
:q privclass\
|
|
:q cplevel\
|
|
:q cpservice\
|
|
:q cpprot user\
|
|
:q specex\
|
|
:q ssi\
|
|
:q cpus\
|
|
:q srm\
|
|
:q vtod\
|
|
:q time full\
|
|
:q timezone\
|
|
:q loaddev\
|
|
:q v osa\
|
|
:q v dasd\
|
|
:q v crypto\
|
|
:q v fcp\
|
|
:q v pav\
|
|
:q v sw\
|
|
:q v st\
|
|
:q v nic\
|
|
:q st\
|
|
:q xstore\
|
|
:q xstore user system\
|
|
:q sxspages\
|
|
:q vmlan\
|
|
:q vswitch\
|
|
:q vswitch details\
|
|
:q vswitch access\
|
|
:q vswitch active\
|
|
:q vswitch accesslist\
|
|
:q vswitch promiscuous\
|
|
:q vswitch controller\
|
|
:q port group all active details\
|
|
:q set\
|
|
:q comm\
|
|
:q controller all\
|
|
:q fcp\
|
|
:q frames\
|
|
:q lan\
|
|
:q lan all details\
|
|
:q lan all access\
|
|
:q memassist\
|
|
:q nic\
|
|
:q pav\
|
|
:q proc\
|
|
:q proc topology\
|
|
:q mt\
|
|
:q qioass\
|
|
:q spaces\
|
|
:q swch all\
|
|
:q trace\
|
|
:q mdcache\
|
|
:q alloc page\
|
|
:q alloc spool\
|
|
:q dump\
|
|
:q dumpdev\
|
|
:q reorder VMUSERID\
|
|
:q quickdsp VMUSERID\
|
|
:q pcifunction\
|
|
:q vmrelocate\
|
|
:q syscontrol\
|
|
:ind load\
|
|
:ind sp\
|
|
:ind user\
|
|
"
|
|
|
|
###############################################################################
|
|
KVM_CMDS="virsh version\
|
|
:virsh nodeinfo\
|
|
:virsh nodememstats\
|
|
:virsh nodecpustats\
|
|
:virsh list --all\
|
|
:virsh iface-list\
|
|
:virsh net-list\
|
|
:virsh nwfilter-list\
|
|
:virsh nodedev-list --tree\
|
|
:virsh pool-list\
|
|
:virt-host-validate\
|
|
:kvm_stat --once --debugfs --debugfs-include-past --tracepoints\
|
|
"
|
|
|
|
########################################
|
|
collect_cmdsout() {
|
|
local cmd
|
|
|
|
pr_collect_output "command"
|
|
|
|
IFS=:
|
|
for cmd in ${CMDS}; do
|
|
IFS=${IFS_ORI} call_run_command "${cmd}" "${OUTPUT_FILE_CMD}"
|
|
done
|
|
IFS="${IFS_ORI}"
|
|
}
|
|
|
|
########################################
|
|
collect_network() {
|
|
local cmd
|
|
|
|
pr_collect_output "network"
|
|
|
|
IFS=:
|
|
for cmd in ${NETWORK_CMDS}; do
|
|
IFS=${IFS_ORI} call_run_command "${cmd}" "${OUTPUT_FILE_NETWORK}"
|
|
done
|
|
IFS="${IFS_ORI}"
|
|
}
|
|
|
|
########################################
|
|
collect_vmcmdsout() {
|
|
local vm_command
|
|
local cp_command
|
|
local vm_cmds
|
|
local vm_userid
|
|
local module_loaded=1
|
|
local cp_buffer_size
|
|
local rc_buffer_check
|
|
|
|
if echo "${RUNTIME_ENVIRONMENT}" | grep -qi "z/VM" >/dev/null 2>&1; then
|
|
pr_collect_output "z/VM"
|
|
|
|
if type vmcp >/dev/null; then
|
|
cp_command="vmcp"
|
|
if ! lsmod 2>/dev/null | grep -q vmcp && modinfo vmcp >/dev/null 2>&1; then
|
|
modprobe vmcp && module_loaded=0 && sleep 2
|
|
fi
|
|
elif type hcp >/dev/null; then
|
|
cp_command="hcp"
|
|
if ! lsmod 2>/dev/null | grep -q cpint; then
|
|
modprobe cpint && module_loaded=0 && sleep 2
|
|
fi
|
|
else
|
|
pr_log_stdout "${SCRIPTNAME}: Warning: No program to communicate to z/VM CP"
|
|
pr_skip "z/VM: vmcp not available"
|
|
return 1
|
|
fi
|
|
vm_userid=$(${cp_command} q userid 2>/dev/null | sed -ne 's/^\([^[:space:]]*\).*$/\1/p')
|
|
vm_cmds=$(echo "${VM_CMDS}" | sed "s/VMUSERID/${vm_userid}/g")
|
|
|
|
IFS=:
|
|
for vm_command in ${vm_cmds}; do
|
|
IFS="${IFS_ORI}"
|
|
# initialize buffer values for automatic resizing
|
|
cp_buffer_size=2
|
|
rc_buffer_check=2
|
|
while test ${rc_buffer_check} -eq 2 && test ${cp_buffer_size} -lt 1024; do
|
|
cp_buffer_size=$(( cp_buffer_size * 2 ))
|
|
eval ${cp_command} -b ${cp_buffer_size}k "${vm_command}" \
|
|
>/dev/null 2>&1
|
|
rc_buffer_check=$?
|
|
done
|
|
call_run_command "${cp_command} -b ${cp_buffer_size}k ${vm_command}" \
|
|
"${OUTPUT_FILE_VMCMD}"
|
|
IFS=:
|
|
done
|
|
IFS="${IFS_ORI}"
|
|
|
|
if test ${module_loaded} -eq 0 && test "x${cp_command}" = "xhcp"; then
|
|
rmmod cpint
|
|
elif test ${module_loaded} -eq 0 && test "x${cp_command}" = "xvmcp"; then
|
|
rmmod vmcp
|
|
fi
|
|
else
|
|
pr_skip "z/VM: no z/VM environment"
|
|
fi
|
|
}
|
|
|
|
########################################
|
|
collect_hyptop() {
|
|
local param
|
|
local delay=1 # seconds
|
|
local iter=5
|
|
local sec=`expr ${delay} \\* ${iter}`
|
|
|
|
case ${RUNTIME_ENVIRONMENT} in
|
|
"z/VM")
|
|
param="\#,c,m,C:s,M:s,o" # z/VM guest fields
|
|
;;
|
|
"LPAR")
|
|
param="\#,T,c,e,m,C:s,E:s,M:s,o" # all LPAR fields
|
|
;;
|
|
*) # KVM guest
|
|
pr_skip "hyptop: not available for ${RUNTIME_ENVIRONMENT}"
|
|
return 1
|
|
;;
|
|
esac
|
|
pr_collect_output "hyptop for ${RUNTIME_ENVIRONMENT} - ${sec}s"
|
|
call_run_command "hyptop -b -d ${delay} -n ${iter} -f ${param} -S c" "${OUTPUT_FILE_HYPTOP}"
|
|
}
|
|
|
|
########################################
|
|
collect_procfs() {
|
|
local file_name
|
|
|
|
pr_collect "procfs"
|
|
|
|
for file_name in ${PROCFILES}; do
|
|
call_collect_file "${file_name}"
|
|
done
|
|
}
|
|
|
|
########################################
|
|
collect_sysfs() {
|
|
local debugfs_mounted=0
|
|
local dir_name
|
|
local file_name
|
|
local rc=0
|
|
|
|
pr_collect "sysfs"
|
|
if ! grep -qE "${MOUNT_POINT_DEBUGFS}.*debugfs" /proc/mounts 2>/dev/null; then
|
|
if mount -t debugfs debugfs "${MOUNT_POINT_DEBUGFS}" >/dev/null 2>&1; then
|
|
sleep 2
|
|
debugfs_mounted=1
|
|
else
|
|
pr_log_stdout "${SCRIPTNAME}: Warning: Unable to mount debugfs \
|
|
at \"${MOUNT_POINT_DEBUGFS}\""
|
|
fi
|
|
fi
|
|
|
|
# Collect sysfs files using multiple threads (-J 1) while excluding
|
|
# files known to block on read (-x). Stop reading a file that takes
|
|
# more than 5 seconds (-T 5) such as an active ftrace buffer.
|
|
# error messages are not written to the log
|
|
if test "x${DUMP2TAR_OK}" = "xYES"; then
|
|
dump2tar /sys -z -o "${OUTPUT_FILE_SYSFS}.tgz" \
|
|
-x '*/tracing/trace_pipe*' \
|
|
-x '*/page_idle/bitmap*' \
|
|
-x '*/tracing/per_cpu/*' \
|
|
-x '*/tracing/free_buffer' \
|
|
-x '*/devices/css[0-9]*/[0-9]*.*.*/[0-9]*.*.*/host[0-9]*/rport-[0-9]*:*-*/target[0-9]*:*:*/[0-9]*:*:*:*/block/sd*/mq/0/cpu[0-9]*' \
|
|
--ignore-failed-read -J 1 -T 5 2>>${OUTPUT_FILE_SYSFS}.err
|
|
rc=$?
|
|
else
|
|
echo "${SCRIPTNAME}: Warning: dump2tar is unavailable"
|
|
rc=255
|
|
fi
|
|
|
|
|
|
if [ ${rc} -eq 0 ] ; then
|
|
echo " all failed entries are logged to ${OUTPUT_FILE_SYSFS}.err"
|
|
else
|
|
if [ $rc -ne 255 ]; then
|
|
echo "${SCRIPTNAME}: Warning: dump2tar failed with $rc"
|
|
fi
|
|
pr_log_stdout " Warning: falling back to slow path"
|
|
call_run_command "find /sys -print0 | sort -z \
|
|
| xargs -0 -n 10 ls -ld" "${OUTPUT_FILE_SYSFS}.out"
|
|
|
|
find /sys -noleaf -type d 2>/dev/null | while IFS= read -r dir_name; do
|
|
mkdir -p "${WORKPATH}${dir_name}"
|
|
done
|
|
|
|
find /sys -noleaf -type f -perm /444 \
|
|
-a -not \( -path '*tracing*' -a -name "trace_pipe*" \) \
|
|
-a -not \( -path '*page_idle*' -a -name 'bitmap*' \) \
|
|
-a -not -path '*tracing/per_cpu*' \
|
|
-a -not -path '*/tracing/free_buffer' \
|
|
-a -not -path '*/devices/css[0-9]*/[0-9]*.*.*/[0-9]*.*.*/host[0-9]*/rport-[0-9]*:*-*/target[0-9]*:*:*/[0-9]*:*:*:*/block/sd*/mq/0/cpu[0-9]*' \
|
|
2>/dev/null | while IFS= read -r file_name; do
|
|
echo " ${file_name}"
|
|
if ! dd if="${file_name}" status=noxfer iflag=nonblock \
|
|
of="${WORKPATH}${file_name}" >/dev/null 2>&1; then
|
|
echo "${SCRIPTNAME}: Warning: failed to copy \"${file_name}\""
|
|
fi
|
|
done
|
|
fi
|
|
|
|
if test ${debugfs_mounted} -eq 1; then
|
|
umount "${MOUNT_POINT_DEBUGFS}"
|
|
fi
|
|
}
|
|
|
|
########################################
|
|
collect_logfiles() {
|
|
local file_name
|
|
|
|
pr_collect "log files"
|
|
|
|
for file_name in ${LOGFILES}; do
|
|
call_collect_file "${file_name}"
|
|
done
|
|
|
|
pr_log_stdout "$(logfile_checker "/var/log*")"
|
|
}
|
|
|
|
########################################
|
|
collect_configfiles() {
|
|
local file_name
|
|
|
|
pr_collect "config files"
|
|
|
|
for file_name in ${CONFIGFILES}; do
|
|
call_collect_file "${file_name}"
|
|
done
|
|
}
|
|
|
|
########################################
|
|
collect_initrd_configfiles() {
|
|
local file_name
|
|
local rd rdbase rdreldir rdextract mypopd noglob mycat patterns=""
|
|
|
|
pr_collect "initrd config files"
|
|
cd "${WORKPATH}" || return
|
|
mypopd="$OLDPWD" # (pushd alternative)
|
|
# prevent expansion of glob special characters such as '*'
|
|
noglob=$(set +o | grep -F noglob)
|
|
set -o noglob
|
|
# prepare relative configfiles paths
|
|
for file_name in ${CONFIGFILES}; do
|
|
# strip leading slash to make path relative as in initrd
|
|
file_name=${file_name#/}
|
|
# in contrast to call_collect_file(), cpio extract with patterns
|
|
# does not recurse, but cpio wildcards match slash, so add suffix;
|
|
# otherwise e.g. /lib/udev/rules.d or /etc/multipath only extracts
|
|
# just that directory without any of its content
|
|
file_name="${file_name}*"
|
|
patterns="$patterns ${file_name}"
|
|
done
|
|
patterns="$patterns squash-root.img"
|
|
$noglob
|
|
# iterate over initrds
|
|
for rd in /boot/initr* /boot/zipl/initr* /var/lib/kdump/initr*; do
|
|
[ -f "$rd" ] || continue # (nullglob alternative)
|
|
rdbase=${rd##*/} # basename
|
|
rdreldir=${rd%/*} # dirname
|
|
rdreldir=${rdreldir#/} # strip leading slash to make path relative
|
|
mkdir -p "${rdreldir}"
|
|
cd "${rdreldir}" || break # (pushd alternative)
|
|
if type lsinitrd >/dev/null 2>&1; then
|
|
lsinitrd "${rd}" > "${rdbase}.lsinitrd"
|
|
fi
|
|
if type lsinitramfs >/dev/null 2>&1; then
|
|
lsinitramfs -l "${rd}" > "${rdbase}.lsinitramfs"
|
|
fi
|
|
cd - || break # change to $OLDPWD (popd alternative)
|
|
rdextract=${rd#/}
|
|
mkdir -p "${rdextract}"
|
|
cd "${rdextract}" || break # (pushd alternative)
|
|
case $(file "${rd}") in
|
|
*cpio*) mycat="cat" ;;
|
|
*gzip*) mycat="zcat" ;;
|
|
*XZ*) mycat="xzcat" ;;
|
|
*Zstandard*) mycat="zstdcat" ;;
|
|
*bzip2*) mycat="bzcat" ;;
|
|
*LZMA*) mycat="lzcat" ;;
|
|
*)
|
|
# change to $OLDPWD (popd alternative)
|
|
cd - || { cd "$mypopd" || return; return; }
|
|
continue
|
|
;;
|
|
esac
|
|
# prevent expansion of glob special characters such as '*'
|
|
noglob=$(set +o | grep -F noglob)
|
|
set -o noglob
|
|
# we expect no early-initrd stuff on s390 so extract directly.
|
|
# globbing is turned off and we want word split on $patterns:
|
|
# shellcheck disable=SC2086
|
|
"$mycat" "${rd}" | cpio --extract --no-absolute-filenames \
|
|
--make-directories \
|
|
--preserve-modification-time $patterns
|
|
$noglob
|
|
if [ ! -f "squash-root.img" ]; then
|
|
cd - || break # change to $OLDPWD (popd alternative)
|
|
continue
|
|
fi
|
|
# prevent expansion of glob special characters such as '*'
|
|
noglob=$(set +o | grep -F noglob)
|
|
set -o noglob
|
|
# globbing is turned off and we want word split on $patterns:
|
|
# shellcheck disable=SC2086
|
|
unsquashfs -no-progress -dest "squashfs-root" "squash-root.img" \
|
|
$patterns
|
|
$noglob
|
|
rm -f "squash-root.img"
|
|
cd - || break # change to $OLDPWD (popd alternative)
|
|
done
|
|
cd "$mypopd" || return
|
|
}
|
|
|
|
########################################
|
|
collect_osaoat() {
|
|
local osa_devices
|
|
local osa_device
|
|
|
|
if type qethqoat >/dev/null; then
|
|
osa_devices=$(lsqeth 2>/dev/null | grep "Device name" \
|
|
| sed 's/D.*:[[:space:]]*\([^[:space:]]*\)[[:space:]]\+/\1/g' \
|
|
| sed 's/[()]//g' )
|
|
if test -n "${osa_devices}"; then
|
|
pr_collect_output "osa oat"
|
|
for osa_device in ${osa_devices}; do
|
|
call_run_command "qethqoat ${osa_device}" \
|
|
"${OUTPUT_FILE_OSAOAT}.out" &&
|
|
call_run_command "qethqoat -r ${osa_device}" \
|
|
"${OUTPUT_FILE_OSAOAT}_${osa_device}.raw"
|
|
done
|
|
else
|
|
pr_skip "osa oat: no devices"
|
|
fi
|
|
else
|
|
pr_skip "osa oat: qethqoat not available"
|
|
fi
|
|
}
|
|
|
|
########################################
|
|
collect_ethtool() {
|
|
local network_device
|
|
|
|
if type ethtool >/dev/null; then
|
|
if test -n "${NETWORK_DEVS}"; then
|
|
pr_collect_output "ethtool"
|
|
for network_device in ${NETWORK_DEVS}; do
|
|
call_run_command "ethtool ${network_device}" \
|
|
"${OUTPUT_FILE_ETHTOOL}"
|
|
call_run_command "ethtool -k ${network_device}" \
|
|
"${OUTPUT_FILE_ETHTOOL}"
|
|
call_run_command "ethtool -a ${network_device}" \
|
|
"${OUTPUT_FILE_ETHTOOL}"
|
|
call_run_command "ethtool -c ${network_device}" \
|
|
"${OUTPUT_FILE_ETHTOOL}"
|
|
call_run_command "ethtool --per-queue ${network_device} \
|
|
--show-coalesce" "${OUTPUT_FILE_ETHTOOL}"
|
|
call_run_command "ethtool -g ${network_device}" \
|
|
"${OUTPUT_FILE_ETHTOOL}"
|
|
call_run_command "ethtool -i ${network_device}" \
|
|
"${OUTPUT_FILE_ETHTOOL}"
|
|
call_run_command "ethtool -l ${network_device}" \
|
|
"${OUTPUT_FILE_ETHTOOL}"
|
|
call_run_command "ethtool -P ${network_device}" \
|
|
"${OUTPUT_FILE_ETHTOOL}"
|
|
call_run_command "ethtool -S ${network_device}" \
|
|
"${OUTPUT_FILE_ETHTOOL}"
|
|
call_run_command "ethtool -T ${network_device}" \
|
|
"${OUTPUT_FILE_ETHTOOL}"
|
|
call_run_command "ethtool --module-info ${network_device}" \
|
|
"${OUTPUT_FILE_ETHTOOL}"
|
|
done
|
|
else
|
|
pr_skip "ethtool: no devices"
|
|
fi
|
|
else
|
|
pr_skip "ethtool: not available"
|
|
fi
|
|
}
|
|
|
|
########################################
|
|
collect_tc() {
|
|
local network_device
|
|
|
|
if type tc >/dev/null; then
|
|
if test -n "${NETWORK_DEVS}"; then
|
|
pr_collect_output "Traffic Control"
|
|
for network_device in ${NETWORK_DEVS}; do
|
|
call_run_command "tc -s qdisc show dev ${network_device}" \
|
|
"${OUTPUT_FILE_TC}"
|
|
done
|
|
else
|
|
pr_skip "Traffic Control: no devices"
|
|
fi
|
|
else
|
|
pr_skip "Traffic Control: tc not available"
|
|
fi
|
|
}
|
|
|
|
########################################
|
|
collect_bridge() {
|
|
local network_device
|
|
|
|
if type bridge >/dev/null; then
|
|
if test -n "${NETWORK_DEVS}"; then
|
|
pr_collect_output "bridge"
|
|
for network_device in ${NETWORK_DEVS}; do
|
|
call_run_command "bridge -d link show dev ${network_device}" \
|
|
"${OUTPUT_FILE_BRIDGE}"
|
|
call_run_command "bridge -s fdb show dev ${network_device}" \
|
|
"${OUTPUT_FILE_BRIDGE}"
|
|
call_run_command "bridge -d mdb show dev ${network_device}" \
|
|
"${OUTPUT_FILE_BRIDGE}"
|
|
done
|
|
else
|
|
pr_skip "bridge: no devices"
|
|
fi
|
|
else
|
|
pr_skip "bridge: not available"
|
|
fi
|
|
}
|
|
|
|
########################################
|
|
# OpenVSwitch
|
|
collect_ovs() {
|
|
local ovscmd
|
|
local bridge
|
|
local ovscmds
|
|
local ovsbrcmd
|
|
local ovsbrcmds
|
|
|
|
ovscmds="ovs-dpctl -s show\
|
|
:ovs-vsctl -t 5 show\
|
|
:ovsdb-client dump\
|
|
"
|
|
if type ovs-vsctl >/dev/null; then
|
|
pr_collect_output "OpenVSwitch"
|
|
IFS=:
|
|
for ovscmd in ${ovscmds}; do
|
|
IFS=${IFS_ORI} call_run_command "${ovscmd}" "${OUTPUT_FILE_OVS}"
|
|
done
|
|
IFS="${IFS_ORI}"
|
|
|
|
for bridge in ${ovs-vsctl list-br}; do
|
|
ovsbrcmds="ovs-ofctl show ${bridge}\
|
|
:ovs-ofctl dump-flows ${bridge}\
|
|
:ovs-appctl fdb/show ${bridge}\
|
|
"
|
|
IFS=:
|
|
for ovsbrcmd in ${ovsbrcmds}; do
|
|
IFS=${IFS_ORI} call_run_command "${ovsbrcmd}" "${OUTPUT_FILE_OVS}"
|
|
done
|
|
IFS="${IFS_ORI}"
|
|
done
|
|
else
|
|
pr_skip "OpenVSwitch: ovs-vsctl not available"
|
|
fi
|
|
}
|
|
|
|
########################################
|
|
collect_container() {
|
|
local container_list
|
|
local item
|
|
|
|
# check if container environment exists
|
|
if test "x${DOCKER}" = "xYES" || test "x${KUBERNETES}" = "xYES" || test "x${PODMAN}" = "xYES"; then
|
|
pr_collect_output "container host"
|
|
else
|
|
pr_skip "container host: not found"
|
|
fi
|
|
|
|
# for docker command exists
|
|
if test "x${DOCKER}" = "xYES"; then
|
|
pr_syslog_stdout " docker ..."
|
|
IFS=:
|
|
for item in ${DOCKER_CMDS}; do
|
|
IFS=${IFS_ORI} call_run_command "${item}" "${OUTPUT_FILE_DOCKER}"
|
|
done
|
|
IFS="${IFS_ORI}"
|
|
fi
|
|
|
|
# for podman command exists
|
|
if test "x${PODMAN}" = "xYES"; then
|
|
pr_syslog_stdout " podman ..."
|
|
IFS=:
|
|
for item in ${PODMAN_CMDS}; do
|
|
IFS=${IFS_ORI} call_run_command "${item}" "${OUTPUT_FILE_PODMAN}"
|
|
done
|
|
IFS="${IFS_ORI}"
|
|
fi
|
|
|
|
# for kubectl command exists
|
|
if test "x${KUBERNETES}" = "xYES"; then
|
|
pr_syslog_stdout " Kubernetes ..."
|
|
container_list=$(kubectl top pod | grep -v "MEMORY(bytes)" | cut -d' ' -f1)
|
|
|
|
IFS=:
|
|
for item in ${KUBERNETES_CMDS}; do
|
|
IFS=${IFS_ORI} call_run_command "${item}" "${OUTPUT_FILE_KUBERNETES}.out"
|
|
done
|
|
IFS="${IFS_ORI}"
|
|
if test -n "${container_list}"; then
|
|
for item in ${container_list}; do
|
|
call_run_command "kubectl logs --tail=${KUBERNETES_LOG_LINES} ${item}"\
|
|
"${OUTPUT_FILE_KUBERNETES}.log"
|
|
done
|
|
fi
|
|
fi
|
|
}
|
|
|
|
########################################
|
|
collect_nvme() {
|
|
local device
|
|
|
|
if type nvme >/dev/null; then
|
|
pr_collect_output "NVME storage"
|
|
call_run_command "nvme list" "${OUTPUT_FILE_NVME}"
|
|
for device in /dev/nvme[0-9]*; do
|
|
if [ -c $device ]; then
|
|
call_run_command "smartctl -x $device" "${OUTPUT_FILE_NVME}"
|
|
call_run_command "nvme fw-log $device" "${OUTPUT_FILE_NVME}"
|
|
call_run_command "nvme smart-log $device" "${OUTPUT_FILE_NVME}"
|
|
call_run_command "nvme error-log $device" "${OUTPUT_FILE_NVME}"
|
|
fi
|
|
done
|
|
else
|
|
pr_skip "nvme: not available"
|
|
fi
|
|
}
|
|
|
|
########################################
|
|
collect_dasd() {
|
|
local device
|
|
|
|
if type dasdview >/dev/null; then
|
|
pr_collect_output "DASD storage"
|
|
call_run_command "lsdasd" "${OUTPUT_FILE_DASD}" # duplicate as file header
|
|
for device in /dev/dasd*; do
|
|
if [ -b $device ]; then
|
|
call_run_command "dasdview -i $device" "${OUTPUT_FILE_DASD}"
|
|
call_run_command "dasdview -t info $device" "${OUTPUT_FILE_DASD}"
|
|
else
|
|
echo "$device is no block device" >> ${OUTPUT_FILE_DASD}
|
|
fi
|
|
done
|
|
else
|
|
pr_skip "dasdview: not available"
|
|
fi
|
|
}
|
|
|
|
########################################
|
|
collect_kvm() {
|
|
local cmd
|
|
local domain_list
|
|
local domain
|
|
|
|
# check if KVM virsh command exists
|
|
if [ "x${KVM}" = "xYES" ]; then
|
|
pr_collect_output "KVM"
|
|
IFS=:
|
|
for cmd in ${KVM_CMDS}; do
|
|
IFS=${IFS_ORI} call_run_command "${cmd}" "${OUTPUT_FILE_KVM}"
|
|
done
|
|
IFS="${IFS_ORI}"
|
|
|
|
# domain/guest specific commands
|
|
domain_list=$(virsh list --all --name)
|
|
path="/machine/sclp/s390-sclp-event-facility/sclpcpi"
|
|
props="timestamp\
|
|
system_name\
|
|
system_type\
|
|
system_level\
|
|
sysplex_name"
|
|
if test -n "${domain_list}"; then
|
|
for domain in ${domain_list}; do
|
|
call_run_command "virsh dominfo ${domain}" "${OUTPUT_FILE_KVM}"
|
|
call_run_command "virsh domblklist ${domain}" "${OUTPUT_FILE_KVM}"
|
|
call_run_command "virsh domstats ${domain}" "${OUTPUT_FILE_KVM}"
|
|
for prop in ${props}; do
|
|
qm=$(printf '{
|
|
"execute": "qom-get",
|
|
"arguments": {
|
|
"path": "%s",
|
|
"property": "%s"
|
|
}
|
|
}' "${path}" "${prop}")
|
|
call_run_command "virsh qemu-monitor-command ${domain} \
|
|
--pretty '${qm}'" "${OUTPUT_FILE_KVM}"
|
|
done
|
|
done
|
|
else
|
|
echo "no KVM domains found" | tee -a ${OUTPUT_FILE_KVM}
|
|
fi
|
|
else
|
|
pr_skip "KVM: no virsh command"
|
|
fi
|
|
}
|
|
|
|
########################################
|
|
post_processing() {
|
|
local file_mtime
|
|
local file_mtime_epoche
|
|
local tmp_file
|
|
local file_name
|
|
local base_dir
|
|
local dir_list
|
|
|
|
pr_syslog_stdout ${step_num} "Postprocessing"
|
|
|
|
# wipe possible passwords
|
|
dir_list="${WORKPATH} \
|
|
${WORKPATH}etc/ssl/ \
|
|
${WORKPATH}etc/libvirt/"
|
|
for base_dir in ${dir_list}; do
|
|
find "${base_dir}" -maxdepth 2 -name "*xml" -o -name "*conf" -o -name "*cnf" \
|
|
2>/dev/null | while read -r file_name; do
|
|
file_mtime_epoche=$(stat --format=%Y "${file_name}")
|
|
file_mtime=$(date +%Y%m%d%H%M.%S --date="@${file_mtime_epoche}")
|
|
tmp_file=${file_name}.$$
|
|
echo " clean pw: ${file_name}"
|
|
if ! sed "s/\(.*[Pp]assw.*=\).*/\1********/g" "${file_name}" > "${tmp_file}"; then
|
|
echo "${SCRIPTNAME}: Warning: Postprocessing failed on ${file_name}"
|
|
fi
|
|
mv "${tmp_file}" "${file_name}"
|
|
touch --time=mtime -t "${file_mtime}" "${file_name}"
|
|
done
|
|
done
|
|
|
|
# compressing data folder to avoid full unpack for any DBGINFO
|
|
base_dir="${WORKPATH}proc/"
|
|
search4="kallsyms"
|
|
find "${base_dir}" -name ${search4} 2>/dev/null | while read -r file_name; do
|
|
tmp_file=${file_name}-${KERNEL_BASE}.tgz
|
|
echo " compress: ${file_name}"
|
|
if ! tar -czf "${tmp_file}" -C "${base_dir}" "${search4}"; then
|
|
echo "${SCRIPTNAME}: Warning: Postprocessing failed on ${file_name}"
|
|
echo
|
|
else
|
|
rm -f "${file_name}"
|
|
fi
|
|
done
|
|
|
|
echo "log work directory state after post processing:"
|
|
echo "du summary: $(du -ks ${WORKPATH})"
|
|
df -k ${WORKPATH}
|
|
}
|
|
|
|
########################################
|
|
# Be aware that this output must be
|
|
# redirected into a separate logfile
|
|
call_run_command() {
|
|
local rc=0
|
|
local cmd="${1}"
|
|
local logfile="${2}"
|
|
# extract the raw_command and set cmd_type, as some shell might split into
|
|
# several lines restrict the cmd echo to the first line only
|
|
local raw_cmd=$(echo "${cmd}" | head -1 | sed -ne 's/^\([^[:space:]]*\).*$/\1/p')
|
|
local cmd_type=$(type -t ${raw_cmd})
|
|
|
|
echo "#######################################################" >> "${logfile}"
|
|
echo "${SYSTEMHOSTNAME:-localhost}> ${cmd}" >> "${logfile}"
|
|
|
|
# check calling command type
|
|
if [ "X${cmd_type}" = "Xbuiltin" ]; then
|
|
# command is a builtin (use of timeout not possible)
|
|
eval "${cmd}" >> ${logfile} 2>&1
|
|
rc=$?
|
|
elif [ "X${cmd_type}" != "X" ]; then
|
|
# command was found and has a type
|
|
if [ "x${TIMEOUT_OK}" = "xYES" ]; then
|
|
eval timeout -k ${TOKS} ${TOS} "${cmd}" >> ${logfile} 2>&1
|
|
|
|
rc=$?
|
|
else
|
|
# fall back - call all existing commands without timeout
|
|
eval "${cmd}" >> ${logfile} 2>&1
|
|
rc=$?
|
|
fi
|
|
else
|
|
echo "Command \"${raw_cmd}\" not available" >> "${logfile}"
|
|
echo >> "${logfile}"
|
|
return 1
|
|
fi
|
|
|
|
# add return code or new line and log a warning if rc is not 0 and define return
|
|
if [ ${rc} -eq 0 ]; then
|
|
echo >> "${logfile}"
|
|
return 0
|
|
elif [ "x${TIMEOUT_OK}" = "xYES" ] && [ ${rc} -eq 124 ]; then
|
|
echo "Warning: \"${cmd}\" cancelled by timeout"
|
|
echo "-> command cancelled by timeout" >> "${logfile}"
|
|
return 1
|
|
else
|
|
echo "${SCRIPTNAME}: Warning: Command \"${cmd}\" failed" >> "${logfile}"
|
|
echo "rc=${rc}" >> "${logfile}"
|
|
echo "Warning: Command \"${cmd}\" ended with ${rc}"
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
########################################
|
|
call_collect_file() {
|
|
local file_name="${1}"
|
|
local directory_name=$(dirname "${file_name}" 2>/dev/null)
|
|
echo " ${file_name}"
|
|
|
|
if test ! -e "${WORKPATH}${directory_name}"; then
|
|
mkdir -p "${WORKPATH}${directory_name}" 2>&1
|
|
fi
|
|
if ! cp -r --preserve=mode,timestamps -dL --parents "${file_name}" "${WORKPATH}" 2>&1; then
|
|
return 1
|
|
else
|
|
return 0
|
|
fi
|
|
}
|
|
|
|
########################################
|
|
# print that an instance is already running
|
|
print_alreadyrunning() {
|
|
print_version
|
|
|
|
cat <<EOF
|
|
|
|
|
|
Please check the system if another instance of '${SCRIPTNAME}' is already
|
|
running. If this is not the case, please remove the lock file
|
|
'${LOCKFILE}'.
|
|
EOF
|
|
}
|
|
|
|
########################################
|
|
# Setup the environment
|
|
environment_setup() {
|
|
if test ! -e "${WORKDIR_BASE}"; then
|
|
mkdir -p "${WORKDIR_BASE}"
|
|
elif test ! -d "${WORKDIR_BASE}"; then
|
|
echo "${SCRIPTNAME}: Error: \"${WORKDIR_BASE}\" exists but this is a file!"
|
|
echo " Please make sure \"${WORKDIR_BASE}\" is a directory."
|
|
exit 1
|
|
fi
|
|
if test -e "${LOCKFILE}"; then
|
|
print_alreadyrunning
|
|
exit 1
|
|
else
|
|
touch "${LOCKFILE}"
|
|
echo "${DATETIME}" > "${LOCKFILE}"
|
|
fi
|
|
if ! mkdir "${WORKPATH}" 2>/dev/null; then
|
|
echo "${SCRIPTNAME}: Error: Target directory \"${WORKPATH}\" already exists or"
|
|
echo " \"${WORKDIR_BASE}\" does not exist!"
|
|
exit 1
|
|
fi
|
|
chmod 0700 "${WORKPATH}"
|
|
}
|
|
|
|
########################################
|
|
# create gzip-ped tar file
|
|
create_package() {
|
|
local rc_tar
|
|
pr_syslog_stdout ${step_num} "Finalizing: Creating archive with collected data"
|
|
# get a copy of the script used
|
|
cp -p "${FULLPATHSCRIPT}" "${WORKPATH}"
|
|
# create the archive
|
|
cd "${WORKDIR_BASE}"
|
|
touch "${WORKARCHIVE}"
|
|
chmod 0600 "${WORKARCHIVE}"
|
|
tar -czf "${WORKARCHIVE}" "${WORKDIR_CURRENT}"
|
|
rc_tar=$?
|
|
if [ $rc_tar -eq 0 ]; then
|
|
chmod 0600 "${WORKARCHIVE}"
|
|
pr_stdout " "
|
|
pr_stdout "Collected data was saved to:"
|
|
pr_stdout " >> ${WORKARCHIVE} <<"
|
|
pr_stdout " "
|
|
pr_stdout "Please review all collected data before sending to your" \
|
|
"service organization."
|
|
pr_stdout " "
|
|
elif [ $rc_tar -eq 127 ]; then
|
|
pr_stdout " "
|
|
pr_stdout "${SCRIPTNAME}: Error: tar command is not available!"
|
|
pr_stdout " Please install the corresponding package!"
|
|
else
|
|
pr_stdout " "
|
|
pr_stdout "${SCRIPTNAME}: Error: Collection of data failed!"
|
|
pr_stdout " The creation of \"${WORKARCHIVE}\" was not successful."
|
|
pr_stdout " Please check the directory \"${WORKDIR_BASE}\""
|
|
pr_stdout " to provide enough free available space."
|
|
fi
|
|
}
|
|
|
|
########################################
|
|
# Cleaning up the prepared/collected information
|
|
environment_cleanup() {
|
|
if ! rm -rf "${WORKPATH}" 2>/dev/null; then
|
|
pr_stdout " "
|
|
pr_stdout "${SCRIPTNAME}: Warning: Deletion of \"${WORKPATH}\" failed"
|
|
pr_stdout " Please remove the directory manually"
|
|
pr_stdout " "
|
|
fi
|
|
if ! rm -f "${LOCKFILE}" 2>/dev/null; then
|
|
pr_stdout " "
|
|
pr_stdout "${SCRIPTNAME}: Warning: Deletion of \"${LOCKFILE}\" failed"
|
|
pr_stdout " Please remove the file manually"
|
|
pr_stdout " "
|
|
fi
|
|
}
|
|
|
|
########################################
|
|
# Function to perform a cleanup in case of a received signal
|
|
emergency_exit() {
|
|
pr_stdout " "
|
|
pr_stdout "${SCRIPTNAME}: Info: Data collection has been interrupted"
|
|
pr_stdout " Cleanup of temporary collected data"
|
|
environment_cleanup
|
|
pr_stdout "${SCRIPTNAME}: Info: Emergency exit processed"
|
|
|
|
pr_stdout " "
|
|
logger -t "${SCRIPTNAME}" "Data collection interrupted"
|
|
exit
|
|
}
|
|
|
|
########################################
|
|
# Function to print to stdout when rediretion is active
|
|
pr_stdout() {
|
|
echo "${@}" >&8
|
|
}
|
|
|
|
########################################
|
|
# Function to print to stdout and into log file when rediretion is active
|
|
pr_log_stdout() {
|
|
echo "$@"
|
|
echo "$@" >&8
|
|
}
|
|
|
|
########################################
|
|
# Function to print to stdout and into log file when rediretion is active
|
|
pr_syslog_stdout() {
|
|
echo "$@" >&8
|
|
echo
|
|
echo "$(date +%H:%M:%S.%N) - " "$@"
|
|
logger -t "${SCRIPTNAME}" "$@"
|
|
}
|
|
|
|
########################################
|
|
# print "Collecting ... output"
|
|
pr_collect_output() {
|
|
pr_syslog_stdout ${step_num} "Collecting" $1 "output"
|
|
}
|
|
|
|
########################################
|
|
# print "Collecting ..." like fs
|
|
pr_collect() {
|
|
pr_syslog_stdout ${step_num} "Collecting" "$@"
|
|
}
|
|
|
|
########################################
|
|
# print "Skipping ..." info with reason
|
|
pr_skip() {
|
|
pr_syslog_stdout ${step_num} "Skip" "$@"
|
|
}
|
|
|
|
###############################################################################
|
|
# Running the script (main)
|
|
|
|
environment_setup
|
|
print_version
|
|
|
|
# saving stdout/stderr and redirecting stdout/stderr into log file
|
|
exec 8>&1 9>&2 >"${LOGFILE}" 2>&1
|
|
|
|
# run print_version again to be logged in dbginfo.log
|
|
print_version
|
|
|
|
# trap on SIGHUP=1 SIGINT=2 SIGTERM=15
|
|
trap emergency_exit SIGHUP SIGINT SIGTERM
|
|
|
|
pr_log_stdout ""
|
|
pr_log_stdout "Hardware platform = ${HW}"
|
|
pr_log_stdout "Runtime environment = ${RUNTIME_ENVIRONMENT} - DPM: ${DPM_MODE}"
|
|
pr_log_stdout "Kernel version = ${KERNEL_INFO} (${KERNEL_BASE})"
|
|
pr_log_stdout "OS version / distro = ${OS_NAME}"
|
|
pr_log_stdout "Date and time of info = ${DATETIME}"
|
|
pr_log_stdout ""
|
|
|
|
logger -t "${SCRIPTNAME}" "Starting data collection"
|
|
|
|
# step counter
|
|
current_step=1
|
|
# run all collection steps
|
|
for step in ${ALL_STEPS}; do
|
|
# generate step numbering
|
|
step_num="${current_step} of ${COLLECTION_COUNT}: "
|
|
# calling step procedure
|
|
${step}
|
|
current_step=`expr ${current_step} + 1`
|
|
done
|
|
|
|
logger -t "${SCRIPTNAME}" "Data collection completed"
|
|
|
|
exec 1>&8 2>&9 8>&- 9>&-
|
|
|
|
#EOF
|