mirror of
https://github.com/ibm-s390-linux/s390-tools.git
synced 2026-08-05 02:14:52 +00:00
Add procfs files to collect network socket data. Suggested-by: Eberhard Pasch <epasch@de.ibm.com> Reviewed-by: Mario Held <mario.held@de.ibm.com> Signed-off-by: Joern Siglen <siglen@de.ibm.com> Signed-off-by: Jan Höppner <hoeppner@linux.ibm.com>
1589 lines
44 KiB
Bash
Executable File
1589 lines
44 KiB
Bash
Executable File
#!/bin/sh
|
|
#
|
|
# dbginfo.sh - Tool to collect runtime, configuration, and trace information
|
|
#
|
|
# Copyright IBM Corp. 2002, 2024
|
|
#
|
|
# s390-tools is free software; you can redistribute it and/or modify
|
|
# it under the terms of the MIT license. See LICENSE for details.
|
|
#
|
|
|
|
# Switching to neutral locale
|
|
LC_ALL=C
|
|
export LC_ALL
|
|
|
|
########################################
|
|
# Global used variables
|
|
readonly SCRIPTNAME="${0##*/}" # general name of this script
|
|
readonly FULLPATHSCRIPT="$(readlink -f "${0}")"
|
|
#
|
|
readonly DATETIME="$(date +%Y-%m-%d-%H-%M-%S 2>/dev/null)"
|
|
readonly DOCKER=$(if type docker >/dev/null 2>&1; then echo "YES"; else echo "NO"; fi)
|
|
readonly DUMP2TAR_OK=$(if type dump2tar >/dev/null 2>&1; then echo "YES"; else echo "NO"; fi)
|
|
readonly HW="$(uname -m 2>/dev/null)"
|
|
readonly IFS_ORI="${IFS}"
|
|
# retrieve and split kernel version
|
|
readonly KERNEL_BASE="$(uname -r 2>/dev/null)"
|
|
readonly KERNEL_VERSION=$(echo ${KERNEL_BASE} | cut -d'.' -f1 )
|
|
readonly KERNEL_MAJOR_REVISION=$(echo ${KERNEL_BASE} | cut -d'.' -f2 )
|
|
readonly KERNEL_MINOR_REVISION=$(echo ${KERNEL_BASE} | cut -d'.' -f3 | sed 's/[^0-9].*//g')
|
|
readonly KERNEL_INFO=${KERNEL_VERSION}.${KERNEL_MAJOR_REVISION}.${KERNEL_MINOR_REVISION}
|
|
readonly KUBERNETES=$(if type kubectl >/dev/null 2>&1; then echo "YES"; else echo "NO"; fi)
|
|
readonly KUBERNETES_LOG_LINES=50
|
|
readonly KVM=$(if type virsh >/dev/null 2>&1; then echo "YES"; else echo "NO"; fi)
|
|
# The file to indicate that another instance of the script is already running
|
|
readonly LOCKFILE="/tmp/${SCRIPTNAME}.lock"
|
|
# check limits for logfiles like /var/log/messages
|
|
readonly LOG_FILE_SIZE_CHECK=50 # max logfile size in MB
|
|
readonly LOG_FILE_AGE_CHECK=7 # age in days to include for size checking
|
|
# Mount point of the debug file system
|
|
readonly MOUNT_POINT_DEBUGFS="/sys/kernel/debug"
|
|
# distro info
|
|
readonly OSPRETTY="$(cat /etc/os* 2>/dev/null | grep -m1 PRETTY_NAME | sed 's/\"//g')"
|
|
readonly OS_NAME="${OSPRETTY##*=}"
|
|
# The processor ID for the first processor
|
|
readonly PROCESSORID="$(grep -E ".*processor 0:.*" /proc/cpuinfo | \
|
|
sed 's/.*identification[[:space:]]*\=[[:space:]]*\([[:alnum:]]*\).*/\1/g')"
|
|
readonly PROCESSORVERSION="$(grep -E ".*processor 0:.*" /proc/cpuinfo | \
|
|
sed 's/.*version[[:space:]]*\=[[:space:]]*\([[:alnum:]]*\).*/\1/g')"
|
|
if test "x${PROCESSORVERSION}" = "xFF" || test "x${PROCESSORVERSION}" = "xff"; then
|
|
RUNTIME_ENVIRONMENT=$(grep -E "VM00.*Control Program.*" /proc/sysinfo | \
|
|
sed 's/.*:[[:space:]]*\([[:graph:]]*\).*/\1/g')
|
|
else
|
|
RUNTIME_ENVIRONMENT="LPAR"
|
|
fi
|
|
# check for Dynamic Partition Mode
|
|
readonly DPM_UUID=$(grep "LPAR UUID" /proc/sysinfo | \
|
|
sed 's/.*:[[:space:]]*\([[:graph:]]*\).*/\1/g')
|
|
if test "${#DPM_UUID}" -eq 0; then
|
|
DPM_MODE="NO"
|
|
else
|
|
DPM_MODE="YES"
|
|
fi
|
|
readonly SYSTEMHOSTNAME="$(hostname -s 2>/dev/null)" # hostname of system being analysed
|
|
readonly TERMINAL="$(tty 2>/dev/null)"
|
|
# timeout seconds TOS / kill timeout TOKS
|
|
readonly TOS=15
|
|
readonly TOKS=30
|
|
readonly TIMEOUT=$(if type timeout >/dev/null 2>&1; then echo "YES"; else echo "NO"; fi)
|
|
readonly TIMEOUT_OK=$(if test "x${TIMEOUT}" = "xYES"; then
|
|
if test $(timeout -k ${TOKS} ${TOS} uname 2>/dev/null) = $(uname); then
|
|
echo "YES"; else echo "WRONG_VERSION"; fi
|
|
else echo "NO"; fi)
|
|
readonly ZDEV_CONF=$(lszdev --configured 2>/dev/null | wc -l)
|
|
readonly ZDEV_OFF=$(lszdev --offline 2>/dev/null | wc -l)
|
|
readonly ZDEV_ONL=$(lszdev --online 2>/dev/null | wc -l)
|
|
|
|
paramWORKDIR_BASE="/tmp" # initial default path
|
|
|
|
########################################
|
|
# print dbginfo.sh version info
|
|
print_version() {
|
|
cat <<EOF
|
|
${SCRIPTNAME}: Debug information script version %S390_TOOLS_VERSION%
|
|
Copyright IBM Corp. 2002, 2024
|
|
EOF
|
|
}
|
|
|
|
########################################
|
|
# print how to use this script
|
|
print_usage() {
|
|
print_version
|
|
|
|
cat <<EOF
|
|
|
|
Usage: ${SCRIPTNAME} [OPTION]
|
|
|
|
This script collects runtime, configuration and trace information on
|
|
a Linux on IBM Z installation for debugging purposes.
|
|
|
|
It also traces information about z/VM if the Linux runs under z/VM.
|
|
Virtualization platform data is collected on a host serving this.
|
|
|
|
Default location for data collection and final tar file is "/tmp/".
|
|
The collected information is written to a TAR archive named
|
|
|
|
DBGINFO-[date]-[time]-[hostname]-[processorid].tgz
|
|
|
|
where [date] and [time] are the date and time when debug data is collected.
|
|
[hostname] indicates the hostname of the system the data was collected from.
|
|
The [processorid] is taken from the processor 0 and indicates the processor
|
|
identification.
|
|
|
|
Options:
|
|
-d|--directory specify the directory where the data collection
|
|
stores the temporary data and the final archive.
|
|
-h|--help print this help
|
|
-v|--version print version information
|
|
-c|--check online base system information (no data collection)
|
|
|
|
Please report bugs to: linux390@de.ibm.com
|
|
|
|
EOF
|
|
}
|
|
|
|
########################################
|
|
# check for oversize logfiles and missing rotation
|
|
logfile_checker() {
|
|
local counter
|
|
local logfile
|
|
local logfiles
|
|
|
|
# find files bigger than recommended
|
|
counter=$(find $1 -maxdepth 1 -type f -mtime -${LOG_FILE_AGE_CHECK} \
|
|
-size ${LOG_FILE_SIZE_CHECK}M | wc -l)
|
|
|
|
echo " ${counter} logfiles over ${LOG_FILE_SIZE_CHECK} MB"
|
|
# maybe check for rotation of base names
|
|
if [ ${counter} -ne 0 ]; then
|
|
for logfile in $(find $1 -maxdepth 1 -type f -mtime -${LOG_FILE_AGE_CHECK} \
|
|
-size ${LOG_FILE_SIZE_CHECK}M -print); do
|
|
# use a neutral separtor ':' as concat is different in some bash
|
|
# insert the 'blank' for later use in for loop
|
|
# add the base name before '.' or '-' only for checks
|
|
logfiles="${logfiles}: ${logfile%%[.-]*}"
|
|
done
|
|
# change separator to new line for sorting
|
|
logfiles=$(echo "${logfiles}" | sed s'/:/\n/g' | sort -u)
|
|
for logfile in ${logfiles}; do
|
|
counter=$(ls ${logfile}* 2>/dev/null | wc -l)
|
|
if [ ${counter} -eq 1 ]; then
|
|
echo " CHECK - ${logfile} may miss a rotation"
|
|
else
|
|
echo " OK - ${logfile}* may have a rotation "\
|
|
"in place: ${counter} files"
|
|
fi
|
|
done
|
|
fi
|
|
}
|
|
|
|
########################################
|
|
# print basic info and online checks
|
|
print_check() {
|
|
print_version
|
|
cat <<EOF
|
|
|
|
Hardware platform = ${HW}
|
|
Runtime environment = ${RUNTIME_ENVIRONMENT} - DPM: ${DPM_MODE}
|
|
$(cat /proc/sysinfo | grep 'Name')
|
|
Kernel version = ${KERNEL_INFO}
|
|
OS version / distro = ${OS_NAME}
|
|
KVM host = ${KVM}
|
|
container host Kubernetes: ${KUBERNETES} - docker: ${DOCKER}
|
|
|
|
Current user = $(whoami) (must be root for data collection)
|
|
Date and time = $(date)
|
|
Uptime =$(uptime)
|
|
Number of coredumps = $(corecumpctl 2>/dev/null | wc -l)
|
|
zdevice onl/conf/offl = ${ZDEV_ONL} / ${ZDEV_CONF} / ${ZDEV_OFF}
|
|
Log file check =$(logfile_checker "/var/log*")
|
|
|
|
Working directory = $(ls -d ${paramWORKDIR_BASE} 2>&1 && df -k ${paramWORKDIR_BASE})
|
|
$(ls -ltr ${paramWORKDIR_BASE}/DBGINFO*tgz 2>/dev/null | tail -2)
|
|
$(ls ${LOCKFILE} 2>/dev/null && echo " Warning: dbginfo running since: $(cat ${LOCKFILE})")
|
|
Tool/command dependency check successful:
|
|
timeout = ${TIMEOUT_OK}
|
|
dump2tar = ${DUMP2TAR_OK}
|
|
|
|
This is a console output only - no data was saved using option -c !
|
|
|
|
EOF
|
|
}
|
|
|
|
#######################################
|
|
# Parsing the command line and pre checks
|
|
while [ ${#} -gt 0 ]; do
|
|
case ${1} in
|
|
--help|-h)
|
|
print_usage
|
|
exit 0
|
|
;;
|
|
--version|-v)
|
|
print_version
|
|
exit 0
|
|
;;
|
|
--directory|-d)
|
|
paramWORKDIR_BASE=${2}
|
|
if test -z "${paramWORKDIR_BASE}"; then
|
|
echo "${SCRIPTNAME}: Error: No directory specified for data collection!"
|
|
echo
|
|
exit 1
|
|
elif test ! -d "${paramWORKDIR_BASE}"; then
|
|
echo "${SCRIPTNAME}: Error: The specified directory does not exist!"
|
|
echo
|
|
exit 1
|
|
else
|
|
# jump to next param
|
|
shift
|
|
fi
|
|
;;
|
|
--check|-c)
|
|
print_check
|
|
exit 0
|
|
;;
|
|
-*|--*|*)
|
|
echo
|
|
echo "${SCRIPTNAME}: invalid option \"${1}\""
|
|
echo "Try '${SCRIPTNAME} --help' for more information"
|
|
echo
|
|
exit 1
|
|
;;
|
|
esac
|
|
# next parameter, if already last the final shift will do termination
|
|
shift
|
|
done
|
|
|
|
# finally verification to run as root
|
|
if test "$(/usr/bin/id -u 2>/dev/null)" -ne 0; then
|
|
echo "${SCRIPTNAME}: Error: You must be user root to run data collection \"${SCRIPTNAME}\"!"
|
|
echo "${SCRIPTNAME}: Hint: for base system information you may run \"${SCRIPTNAME} -c\""
|
|
exit 1
|
|
fi
|
|
|
|
#########################################
|
|
# The base working directory and derieved path info
|
|
readonly WORKDIR_BASE="$(echo "${paramWORKDIR_BASE}" | sed -e 's#/$##')/"
|
|
# The current working directory for the actual script execution
|
|
if test -z "${PROCESSORID}"; then
|
|
readonly WORKDIR_CURRENT="DBGINFO-${DATETIME}-${SYSTEMHOSTNAME:-localhost}"
|
|
else
|
|
readonly WORKDIR_CURRENT="DBGINFO-${DATETIME}-${SYSTEMHOSTNAME:-localhost}-${PROCESSORID}"
|
|
fi
|
|
# The current path where the collected information is put together
|
|
readonly WORKPATH="${WORKDIR_BASE}${WORKDIR_CURRENT}/"
|
|
# The current TAR archive that finally includes all collected information
|
|
readonly WORKARCHIVE="${WORKDIR_BASE}${WORKDIR_CURRENT}.tgz"
|
|
# The log file of activities from this script execution
|
|
readonly LOGFILE="${WORKPATH}dbginfo.log"
|
|
# File names for output files per section (duplicates are ok)
|
|
readonly OUTPUT_FILE_BRIDGE="${WORKPATH}network.out"
|
|
readonly OUTPUT_FILE_CMD="${WORKPATH}runtime.out"
|
|
readonly OUTPUT_FILE_COREDUMPCTL="${WORKPATH}coredump.out" # separate file needed
|
|
readonly OUTPUT_FILE_DASD="${WORKPATH}dasd.out"
|
|
readonly OUTPUT_FILE_DOCKER="${WORKPATH}docker_runtime.out"
|
|
readonly OUTPUT_FILE_ETHTOOL="${WORKPATH}network.out"
|
|
readonly OUTPUT_FILE_HYPTOP="${WORKPATH}runtime.out"
|
|
readonly OUTPUT_FILE_JOURNALCTL="${WORKPATH}journalctl.out"
|
|
readonly OUTPUT_FILE_KVM="${WORKPATH}kvm_runtime.out"
|
|
readonly OUTPUT_FILE_LSOF="${WORKPATH}open_files.out"
|
|
readonly OUTPUT_FILE_NETWORK="${WORKPATH}network.out"
|
|
readonly OUTPUT_FILE_NVME="${WORKPATH}runtime.out"
|
|
readonly OUTPUT_FILE_OVS="${WORKPATH}network.out"
|
|
readonly OUTPUT_FILE_ISW="${WORKPATH}installed_sw.out"
|
|
readonly OUTPUT_FILE_TC="${WORKPATH}network.out"
|
|
readonly OUTPUT_FILE_VMCMD="${WORKPATH}zvm_runtime.out"
|
|
readonly OUTPUT_FILE_UDEVDB="${WORKPATH}udevdb.out"
|
|
# Base file names for different output files - no extension !
|
|
readonly OUTPUT_FILE_KUBERNETES="${WORKPATH}kubernetes"
|
|
readonly OUTPUT_FILE_OSAOAT="${WORKPATH}network"
|
|
readonly OUTPUT_FILE_SYSFS="${WORKPATH}sysfs"
|
|
|
|
# define order of collection steps
|
|
# - state/debug files are collected first, to avoid overwriting by command execution
|
|
ALL_STEPS="\
|
|
collect_sysfs\
|
|
collect_procfs\
|
|
collect_configfiles\
|
|
collect_initrd_configfiles\
|
|
collect_cmdsout\
|
|
collect_hyptop\
|
|
collect_vmcmdsout\
|
|
collect_network\
|
|
collect_osaoat\
|
|
collect_ethtool\
|
|
collect_tc\
|
|
collect_bridge\
|
|
collect_ovs\
|
|
collect_kvm\
|
|
collect_container\
|
|
collect_nvme\
|
|
collect_dasd\
|
|
collect_logfiles\
|
|
post_processing\
|
|
create_package\
|
|
environment_cleanup\
|
|
"
|
|
# The amount of steps running the whole collections, without last cleanup
|
|
readonly COLLECTION_COUNT=`expr $(echo ${ALL_STEPS} | wc -w) - 1`
|
|
|
|
########################################
|
|
# Collection of proc fs entries
|
|
PROCFILES="\
|
|
/proc/buddyinfo\
|
|
/proc/cio_ignore\
|
|
/proc/cmdline\
|
|
/proc/cpuinfo\
|
|
/proc/crypto\
|
|
/proc/dasd/devices\
|
|
/proc/dasd/statistics\
|
|
/proc/devices\
|
|
/proc/diskstats\
|
|
/proc/interrupts\
|
|
/proc/iomem\
|
|
/proc/kallsyms\
|
|
/proc/mdstat\
|
|
/proc/meminfo\
|
|
/proc/misc\
|
|
/proc/modules\
|
|
/proc/mounts\
|
|
/proc/net/vlan\
|
|
/proc/net/bonding\
|
|
/proc/net/sockstat\
|
|
/proc/net/sockstat6\
|
|
/proc/net/softnet_stat\
|
|
/proc/partitions\
|
|
/proc/qeth\
|
|
/proc/qeth_perf\
|
|
/proc/qeth_ipa_takeover\
|
|
/proc/sched_debug\
|
|
/proc/schedstat\
|
|
/proc/service_levels\
|
|
/proc/slabinfo\
|
|
/proc/softirqs\
|
|
/proc/stat\
|
|
/proc/swaps\
|
|
/proc/sys/kernel\
|
|
/proc/sys/vm\
|
|
/proc/sysinfo\
|
|
/proc/version\
|
|
/proc/zoneinfo\
|
|
"
|
|
|
|
# Adding files to PROCFILES in case scsi devices are available
|
|
if test -e /proc/scsi; then
|
|
PROCFILES="${PROCFILES}\
|
|
$(find /proc/scsi -type f -perm /444 2>/dev/null)\
|
|
"
|
|
fi
|
|
|
|
########################################
|
|
|
|
LOGFILES="\
|
|
/root/anaconda-ks.cfg\
|
|
/root/original-ks.cfg\
|
|
/run/containerd/events.log\
|
|
/run/docker/libcontainerd/containerd/events.log\
|
|
/run/udev/chreiplzfcpmp-[0-9][0-9][a-z][a-z][a-z][a-z]-*\
|
|
/var/log/anaconda\
|
|
/var/log/anaconda.*\
|
|
/var/log/audit\
|
|
/var/log/boot*\
|
|
/var/log/cron*\
|
|
/var/log/dmesg*\
|
|
/var/log/dnf.*\
|
|
/var/log/dracut.log*\
|
|
/var/log/IBMtape.trace\
|
|
/var/log/IBMtape.errorlog\
|
|
/var/log/libvirt\
|
|
/var/log/lin_tape.trace\
|
|
/var/log/lin_tape.errorlog\
|
|
/var/log/messages*\
|
|
/var/log/openvswitch/ovs-vswitchd.log\
|
|
/var/log/openvswitch/ovsdb-server.log\
|
|
/var/log/pbl.log\
|
|
/var/log/sa\
|
|
/var/log/syslog*\
|
|
/var/log/yum.log\
|
|
/var/log/YaST2\
|
|
/var/log/zypp*\
|
|
"
|
|
|
|
########################################
|
|
|
|
CONFIGFILES="\
|
|
/boot/grub2/grub.cfg\
|
|
/boot/loader/entries/*.conf\
|
|
/boot/zipl/active_devices.txt\
|
|
/boot/zipl/config\
|
|
/conf/\
|
|
/etc/*.conf\
|
|
/etc/*release\
|
|
/etc/anacrontab\
|
|
/etc/apparmor.d\
|
|
/etc/auto.*\
|
|
/etc/cmdline\
|
|
/etc/cmdline.d\
|
|
/etc/conf.d\
|
|
/etc/cron.*\
|
|
/etc/crontab\
|
|
/etc/crypttab\
|
|
/etc/dasd.*\
|
|
/etc/default\
|
|
/etc/depmod.d\
|
|
/etc/dnf/dnf.conf\
|
|
/etc/docker\
|
|
/etc/dracut.conf.d\
|
|
/etc/exports\
|
|
/etc/fstab\
|
|
/etc/groups\
|
|
/etc/grub.d\
|
|
/etc/hosts*\
|
|
/etc/iscsi\
|
|
/etc/inittab\
|
|
/etc/kdump\
|
|
/etc/libvirt\
|
|
/etc/logrotate.d\
|
|
/etc/lvm\
|
|
/etc/modprobe.conf*\
|
|
/etc/modprobe.d\
|
|
/etc/modules-load.d/*.conf\
|
|
/etc/mtab\
|
|
/etc/multipath\
|
|
/etc/network\
|
|
/etc/networks\
|
|
/etc/opencryptoki/*.conf\
|
|
/etc/pam.d\
|
|
/etc/profile\
|
|
/etc/profile.d\
|
|
/etc/pki/tls/openssl.cnf\
|
|
/etc/rc.d\
|
|
/etc/rc.local\
|
|
/etc/resolv.*\
|
|
/etc/rsyslog.d\
|
|
/etc/ssl/openssl.conf\
|
|
/etc/ssl/openssl.cnf\
|
|
/etc/sysconfig\
|
|
/etc/sysctl.d\
|
|
/etc/syslog*\
|
|
/etc/systemd\
|
|
/etc/udev*\
|
|
/etc/xinet.d\
|
|
/etc/zfcp.*\
|
|
/lib/modprobe.d\
|
|
$(find /lib/modules -name modules.dep 2>/dev/null)\
|
|
/lib/systemd/system/docker.service\
|
|
/lib/udev/rules.d\
|
|
/usr/lib/modprobe.d\
|
|
/usr/lib/modules-load.d/*.conf\
|
|
/usr/lib/systemd/system/\
|
|
/usr/local/lib/modprobe.d\
|
|
/usr/local/lib/modules-load.d/*.conf\
|
|
/run/modprobe.d\
|
|
/run/modules-load.d/*.conf\
|
|
/run/udev/chreiplzfcpmp-ipl-volume-id\
|
|
/run/udev/rules.d\
|
|
/run/zdev_id.env\
|
|
/run/zdev.*\
|
|
"
|
|
|
|
########################################
|
|
# system state commands (non alphabetical order here)
|
|
CMDS="uname -a\
|
|
:uptime\
|
|
:timedatectl\
|
|
:last\
|
|
:lscpu -ae\
|
|
:lscpu -ye\
|
|
:lsmem\
|
|
:lsmod\
|
|
:lsshut\
|
|
:runlevel\
|
|
:sysctl -a\
|
|
:systemctl --all --no-pager show\
|
|
:systemctl --all --no-pager list-units\
|
|
:systemctl --all --no-pager list-unit-files\
|
|
:systemd-delta\
|
|
:ulimit -a\
|
|
:env --null | sort --zero-terminated | tr '\000' '\n'\
|
|
:find /boot -print0 | sort -z | xargs -0 -n 10 ls -ld\
|
|
:find /var/crash -print0 | sort -z | xargs -0 -n 10 ls -ld\
|
|
:df -h\
|
|
:df -i\
|
|
:mount\
|
|
"
|
|
|
|
# Z device subsystem commands (first commands in non alphabetical order)
|
|
CMDS="${CMDS}\
|
|
:lschp\
|
|
:lscss --vpm\
|
|
:lszdev\
|
|
:find /dev -print0 | sort -z | xargs -0 -n 10 ls -ld\
|
|
:lspci -t\
|
|
:lspci -vvv\
|
|
:lstape\
|
|
:smc_dbg\
|
|
"
|
|
|
|
# block device and other scsi device commands (in non alphabetical order)
|
|
# command groups itself have an intentional topical grouping and order
|
|
CMDS="${CMDS}\
|
|
:lsdasd\
|
|
:lsdasd -u\
|
|
:lvdisplay\
|
|
:pvpath -qa\
|
|
:dmsetup ls\
|
|
:dmsetup ls --tree\
|
|
:dmsetup table\
|
|
:dmsetup table --target multipath\
|
|
:dmsetup status\
|
|
:multipathd -k'show config'\
|
|
:multipathd -k'show config local'\
|
|
:multipathd -k'show maps'\
|
|
:multipathd -k'show topo'\
|
|
:multipathd -k'show paths'\
|
|
:multipathd -k'show paths format %w|%a|%r|%L|%p|%i|%d|%D|%t|%T|%o|%0|%C' | sort\
|
|
:multipathd -k'show maps stats'\
|
|
:multipathd -k'show maps status'\
|
|
:multipathd -k'show status'\
|
|
:multipathd -k'show daemon'\
|
|
:multipathd -k'show blacklist'\
|
|
:multipathd -k'show devices'\
|
|
:multipath -v6 -ll\
|
|
:multipath -d\
|
|
:multipath -t\
|
|
:lszfcp\
|
|
:lszfcp -D\
|
|
:lszfcp -V\
|
|
:ziorep_config -ADM\
|
|
:lsscsi\
|
|
"
|
|
|
|
# crypto specific commands
|
|
CMDS="${CMDS}\
|
|
:ep11info -D\
|
|
:ep11info -H\
|
|
:ep11info -M\
|
|
:icainfo\
|
|
:icastats --all\
|
|
:ivp.e # IBM CCA package install check\
|
|
:ls -al /usr/lib64/opencryptoki/stdll\
|
|
:lszcrypt -VV\
|
|
:pkcsconf -i\
|
|
:pkcsconf -s\
|
|
:pkcsconf -l\
|
|
:pkcsconf -t\
|
|
:pkcsconf -m\
|
|
:systemctl status pkcsslotd\
|
|
"
|
|
|
|
# product / distro specific commands
|
|
CMDS="${CMDS}\
|
|
:java -version\
|
|
:qemu-ga -V\
|
|
:SPident # SLES service package\
|
|
"
|
|
|
|
# commands with long output at the end
|
|
CMDS="${CMDS}\
|
|
:cat /root/.bash_history\
|
|
:dmesg -s 1048576\
|
|
:ps -emo pid,tid,nlwp,policy,user,tname,ni,pri,psr,sgi_p,stat,wchan,start_time,time,pcpu,pmem,vsize,size,rss,share,command\
|
|
:ps -eHo pid,tid,nlwp,policy,user,tname,ni,pri,psr,sgi_p,stat,wchan,start_time,time,pcpu,pmem,vsize,size,rss,share,command\
|
|
:ps axX\
|
|
"
|
|
|
|
# commands with separate output file
|
|
CMDS="${CMDS}\
|
|
:apt list >> '${OUTPUT_FILE_ISW}'\
|
|
:coredumpctl && coredumpctl info -o ${OUTPUT_FILE_COREDUMPCTL}\
|
|
:journalctl --all --no-pager --lines=100000 --output=short-precise\
|
|
>> '${OUTPUT_FILE_JOURNALCTL}'\
|
|
:lsof >> '${OUTPUT_FILE_LSOF}'\
|
|
:rpm -qa | sort >> '${OUTPUT_FILE_ISW}'\
|
|
:udevadm info --export-db >> '${OUTPUT_FILE_UDEVDB}'\
|
|
"
|
|
|
|
########################################
|
|
# network comamnds for separate output file with ip -br a as header command
|
|
NETWORK_CMDS="ip -br a\
|
|
:firewall-cmd --list-all\
|
|
:ifconfig -a\
|
|
:ip a sh\
|
|
:ip -s -s link\
|
|
:ip link show\
|
|
:ip neigh list\
|
|
:ip ntable\
|
|
:ip route list\
|
|
:ip route list table all\
|
|
:ip rule list\
|
|
:ipcs -a\
|
|
:iptables -L\
|
|
:lsqeth\
|
|
:netstat -pantu\
|
|
:netstat -s\
|
|
:nm-tool\
|
|
:nstat -az\
|
|
:openssl engine\
|
|
:route -n\
|
|
"
|
|
|
|
########################################
|
|
DOCKER_CMDS="docker version\
|
|
:docker info\
|
|
:docker images\
|
|
:docker network ls\
|
|
:docker ps -a\
|
|
:docker stats --no-stream\
|
|
:systemctl status docker.service\
|
|
"
|
|
|
|
########################################
|
|
KUBERNETES_CMDS="kubectl version\
|
|
:oc version # get also OCP version if installed\
|
|
:kubectl top pod\
|
|
:kubectl top node\
|
|
:kubectl describe pod -A\
|
|
:kubectl describe node -A\
|
|
:kubectl describe endpoints\
|
|
:ls -l /var/log/containers # list of logs is sufficent\
|
|
:kubectl cluster-info dump >>${OUTPUT_FILE_KUBERNETES}.dmp\
|
|
"
|
|
|
|
########################################
|
|
VM_CMDS="q userid\
|
|
:q users\
|
|
:q privclass\
|
|
:q cplevel\
|
|
:q cpservice\
|
|
:q cpprot user\
|
|
:q specex\
|
|
:q ssi\
|
|
:q cpus\
|
|
:q srm\
|
|
:q vtod\
|
|
:q time full\
|
|
:q timezone\
|
|
:q loaddev\
|
|
:q v osa\
|
|
:q v dasd\
|
|
:q v crypto\
|
|
:q v fcp\
|
|
:q v pav\
|
|
:q v sw\
|
|
:q v st\
|
|
:q v nic\
|
|
:q st\
|
|
:q xstore\
|
|
:q xstore user system\
|
|
:q sxspages\
|
|
:q vmlan\
|
|
:q vswitch\
|
|
:q vswitch details\
|
|
:q vswitch access\
|
|
:q vswitch active\
|
|
:q vswitch accesslist\
|
|
:q vswitch promiscuous\
|
|
:q vswitch controller\
|
|
:q port group all active details\
|
|
:q set\
|
|
:q comm\
|
|
:q controller all\
|
|
:q fcp\
|
|
:q frames\
|
|
:q lan\
|
|
:q lan all details\
|
|
:q lan all access\
|
|
:q memassist\
|
|
:q nic\
|
|
:q pav\
|
|
:q proc\
|
|
:q proc topology\
|
|
:q mt\
|
|
:q qioass\
|
|
:q spaces\
|
|
:q swch all\
|
|
:q trace\
|
|
:q mdcache\
|
|
:q alloc page\
|
|
:q alloc spool\
|
|
:q dump\
|
|
:q dumpdev\
|
|
:q reorder VMUSERID\
|
|
:q quickdsp VMUSERID\
|
|
:q pcifunction\
|
|
:q vmrelocate\
|
|
:q syscontrol\
|
|
:ind load\
|
|
:ind sp\
|
|
:ind user\
|
|
"
|
|
|
|
###############################################################################
|
|
KVM_CMDS="virsh version\
|
|
:virsh nodeinfo\
|
|
:virsh nodememstats\
|
|
:virsh nodecpustats\
|
|
:virsh list --all\
|
|
:virsh iface-list\
|
|
:virsh net-list\
|
|
:virsh nwfilter-list\
|
|
:virsh nodedev-list --tree\
|
|
:virsh pool-list\
|
|
:virt-host-validate\
|
|
"
|
|
|
|
########################################
|
|
collect_cmdsout() {
|
|
local cmd
|
|
|
|
pr_collect_output "command"
|
|
|
|
IFS=:
|
|
for cmd in ${CMDS}; do
|
|
IFS=${IFS_ORI} call_run_command "${cmd}" "${OUTPUT_FILE_CMD}"
|
|
done
|
|
IFS="${IFS_ORI}"
|
|
}
|
|
|
|
########################################
|
|
collect_network() {
|
|
local cmd
|
|
|
|
pr_collect_output "network"
|
|
|
|
IFS=:
|
|
for cmd in ${NETWORK_CMDS}; do
|
|
IFS=${IFS_ORI} call_run_command "${cmd}" "${OUTPUT_FILE_NETWORK}"
|
|
done
|
|
IFS="${IFS_ORI}"
|
|
}
|
|
|
|
########################################
|
|
collect_vmcmdsout() {
|
|
local vm_command
|
|
local cp_command
|
|
local vm_cmds
|
|
local vm_userid
|
|
local module_loaded=1
|
|
local cp_buffer_size
|
|
local rc_buffer_check
|
|
|
|
if echo "${RUNTIME_ENVIRONMENT}" | grep -qi "z/VM" >/dev/null 2>&1; then
|
|
pr_collect_output "z/VM"
|
|
|
|
if type vmcp >/dev/null; then
|
|
cp_command="vmcp"
|
|
if ! lsmod 2>/dev/null | grep -q vmcp && modinfo vmcp >/dev/null 2>&1; then
|
|
modprobe vmcp && module_loaded=0 && sleep 2
|
|
fi
|
|
elif type hcp >/dev/null; then
|
|
cp_command="hcp"
|
|
if ! lsmod 2>/dev/null | grep -q cpint; then
|
|
modprobe cpint && module_loaded=0 && sleep 2
|
|
fi
|
|
else
|
|
pr_log_stdout "${SCRIPTNAME}: Warning: No program to communicate to z/VM CP"
|
|
pr_skip "z/VM: vmcp not available"
|
|
return 1
|
|
fi
|
|
vm_userid=$(${cp_command} q userid 2>/dev/null | sed -ne 's/^\([^[:space:]]*\).*$/\1/p')
|
|
vm_cmds=$(echo "${VM_CMDS}" | sed "s/VMUSERID/${vm_userid}/g")
|
|
|
|
IFS=:
|
|
for vm_command in ${vm_cmds}; do
|
|
IFS="${IFS_ORI}"
|
|
# initialize buffer values for automatic resizing
|
|
cp_buffer_size=2
|
|
rc_buffer_check=2
|
|
while test ${rc_buffer_check} -eq 2 && test ${cp_buffer_size} -lt 1024; do
|
|
cp_buffer_size=$(( cp_buffer_size * 2 ))
|
|
eval ${cp_command} -b ${cp_buffer_size}k "${vm_command}" \
|
|
>/dev/null 2>&1
|
|
rc_buffer_check=$?
|
|
done
|
|
call_run_command "${cp_command} -b ${cp_buffer_size}k ${vm_command}" \
|
|
"${OUTPUT_FILE_VMCMD}"
|
|
IFS=:
|
|
done
|
|
IFS="${IFS_ORI}"
|
|
|
|
if test ${module_loaded} -eq 0 && test "x${cp_command}" = "xhcp"; then
|
|
rmmod cpint
|
|
elif test ${module_loaded} -eq 0 && test "x${cp_command}" = "xvmcp"; then
|
|
rmmod vmcp
|
|
fi
|
|
else
|
|
pr_skip "z/VM: no z/VM environment"
|
|
fi
|
|
}
|
|
|
|
########################################
|
|
collect_hyptop() {
|
|
local param
|
|
local delay=1 # seconds
|
|
local iter=5
|
|
local sec=`expr ${delay} \\* ${iter}`
|
|
|
|
case ${RUNTIME_ENVIRONMENT} in
|
|
"z/VM")
|
|
param="\#,c,m,C:s,M:s,o" # z/VM guest fields
|
|
;;
|
|
"LPAR")
|
|
param="\#,T,c,e,m,C:s,E:s,M:s,o" # all LPAR fields
|
|
;;
|
|
*) # KVM guest
|
|
pr_skip "hyptop: not available for ${RUNTIME_ENVIRONMENT}"
|
|
return 1
|
|
;;
|
|
esac
|
|
pr_collect_output "hyptop for ${RUNTIME_ENVIRONMENT} - ${sec}s"
|
|
call_run_command "hyptop -b -d ${delay} -n ${iter} -f ${param} -S c" "${OUTPUT_FILE_HYPTOP}"
|
|
}
|
|
|
|
########################################
|
|
collect_procfs() {
|
|
local file_name
|
|
|
|
pr_collect "procfs"
|
|
|
|
for file_name in ${PROCFILES}; do
|
|
call_collect_file "${file_name}"
|
|
done
|
|
}
|
|
|
|
########################################
|
|
collect_sysfs() {
|
|
local debugfs_mounted=0
|
|
local dir_name
|
|
local file_name
|
|
local rc=0
|
|
|
|
pr_collect "sysfs"
|
|
if ! grep -qE "${MOUNT_POINT_DEBUGFS}.*debugfs" /proc/mounts 2>/dev/null; then
|
|
if mount -t debugfs debugfs "${MOUNT_POINT_DEBUGFS}" >/dev/null 2>&1; then
|
|
sleep 2
|
|
debugfs_mounted=1
|
|
else
|
|
pr_log_stdout "${SCRIPTNAME}: Warning: Unable to mount debugfs \
|
|
at \"${MOUNT_POINT_DEBUGFS}\""
|
|
fi
|
|
fi
|
|
|
|
# Collect sysfs files using multiple threads (-J 1) while excluding
|
|
# files known to block on read (-x). Stop reading a file that takes
|
|
# more than 5 seconds (-T 5) such as an active ftrace buffer.
|
|
# error messages are not written to the log
|
|
if test "x${DUMP2TAR_OK}" = "xYES"; then
|
|
dump2tar /sys -z -o "${OUTPUT_FILE_SYSFS}.tgz" \
|
|
-x '*/tracing/trace_pipe*' \
|
|
-x '*/page_idle/bitmap*' \
|
|
-x '*/tracing/per_cpu/*' \
|
|
-x '*/tracing/free_buffer' \
|
|
-x '*/devices/css[0-9]*/[0-9]*.*.*/[0-9]*.*.*/host[0-9]*/rport-[0-9]*:*-*/target[0-9]*:*:*/[0-9]*:*:*:*/block/sd*/mq/0/cpu[0-9]*' \
|
|
--ignore-failed-read -J 1 -T 5 2>>${OUTPUT_FILE_SYSFS}.err
|
|
rc=$?
|
|
else
|
|
echo "${SCRIPTNAME}: Warning: dump2tar is unavailable"
|
|
rc=255
|
|
fi
|
|
|
|
|
|
if [ ${rc} -eq 0 ] ; then
|
|
echo " all failed entries are logged to ${OUTPUT_FILE_SYSFS}.err"
|
|
else
|
|
if [ $rc -ne 255 ]; then
|
|
echo "${SCRIPTNAME}: Warning: dump2tar failed with $rc"
|
|
fi
|
|
pr_log_stdout " Warning: falling back to slow path"
|
|
call_run_command "find /sys -print0 | sort -z \
|
|
| xargs -0 -n 10 ls -ld" "${OUTPUT_FILE_SYSFS}.out"
|
|
|
|
find /sys -noleaf -type d 2>/dev/null | while IFS= read -r dir_name; do
|
|
mkdir -p "${WORKPATH}${dir_name}"
|
|
done
|
|
|
|
find /sys -noleaf -type f -perm /444 \
|
|
-a -not \( -path '*tracing*' -a -name "trace_pipe*" \) \
|
|
-a -not \( -path '*page_idle*' -a -name 'bitmap*' \) \
|
|
-a -not -path '*tracing/per_cpu*' \
|
|
-a -not -path '*/tracing/free_buffer' \
|
|
-a -not -path '*/devices/css[0-9]*/[0-9]*.*.*/[0-9]*.*.*/host[0-9]*/rport-[0-9]*:*-*/target[0-9]*:*:*/[0-9]*:*:*:*/block/sd*/mq/0/cpu[0-9]*' \
|
|
2>/dev/null | while IFS= read -r file_name; do
|
|
echo " ${file_name}"
|
|
if ! dd if="${file_name}" status=noxfer iflag=nonblock \
|
|
of="${WORKPATH}${file_name}" >/dev/null 2>&1; then
|
|
echo "${SCRIPTNAME}: Warning: failed to copy \"${file_name}\""
|
|
fi
|
|
done
|
|
fi
|
|
|
|
if test ${debugfs_mounted} -eq 1; then
|
|
umount "${MOUNT_POINT_DEBUGFS}"
|
|
fi
|
|
}
|
|
|
|
########################################
|
|
collect_logfiles() {
|
|
local file_name
|
|
|
|
pr_collect "log files"
|
|
|
|
for file_name in ${LOGFILES}; do
|
|
call_collect_file "${file_name}"
|
|
done
|
|
|
|
pr_log_stdout "$(logfile_checker "/var/log*")"
|
|
}
|
|
|
|
########################################
|
|
collect_configfiles() {
|
|
local file_name
|
|
|
|
pr_collect "config files"
|
|
|
|
for file_name in ${CONFIGFILES}; do
|
|
call_collect_file "${file_name}"
|
|
done
|
|
}
|
|
|
|
########################################
|
|
collect_initrd_configfiles() {
|
|
local file_name
|
|
local rd rdbase rdreldir rdextract mypopd noglob mycat patterns=""
|
|
|
|
pr_collect "initrd config files"
|
|
cd "${WORKPATH}" || return
|
|
mypopd="$OLDPWD" # (pushd alternative)
|
|
# prevent expansion of glob special characters such as '*'
|
|
noglob=$(set +o | grep -F noglob)
|
|
set -o noglob
|
|
# prepare relative configfiles paths
|
|
for file_name in ${CONFIGFILES}; do
|
|
# strip leading slash to make path relative as in initrd
|
|
file_name=${file_name#/}
|
|
# in contrast to call_collect_file(), cpio extract with patterns
|
|
# does not recurse, but cpio wildcards match slash, so add suffix;
|
|
# otherwise e.g. /lib/udev/rules.d or /etc/multipath only extracts
|
|
# just that directory without any of its content
|
|
file_name="${file_name}*"
|
|
patterns="$patterns ${file_name}"
|
|
done
|
|
patterns="$patterns squash-root.img"
|
|
$noglob
|
|
# iterate over initrds
|
|
for rd in /boot/initr* /boot/zipl/initr* /var/lib/kdump/initr*; do
|
|
[ -f "$rd" ] || continue # (nullglob alternative)
|
|
rdbase=${rd##*/} # basename
|
|
rdreldir=${rd%/*} # dirname
|
|
rdreldir=${rdreldir#/} # strip leading slash to make path relative
|
|
mkdir -p "${rdreldir}"
|
|
cd "${rdreldir}" || break # (pushd alternative)
|
|
if type lsinitrd >/dev/null 2>&1; then
|
|
lsinitrd "${rd}" > "${rdbase}.lsinitrd"
|
|
fi
|
|
if type lsinitramfs >/dev/null 2>&1; then
|
|
lsinitramfs -l "${rd}" > "${rdbase}.lsinitramfs"
|
|
fi
|
|
cd - || break # change to $OLDPWD (popd alternative)
|
|
rdextract=${rd#/}
|
|
mkdir -p "${rdextract}"
|
|
cd "${rdextract}" || break # (pushd alternative)
|
|
case $(file "${rd}") in
|
|
*cpio*) mycat="cat" ;;
|
|
*gzip*) mycat="zcat" ;;
|
|
*XZ*) mycat="xzcat" ;;
|
|
*Zstandard*) mycat="zstdcat" ;;
|
|
*bzip2*) mycat="bzcat" ;;
|
|
*LZMA*) mycat="lzcat" ;;
|
|
*)
|
|
# change to $OLDPWD (popd alternative)
|
|
cd - || { cd "$mypopd" || return; return; }
|
|
continue
|
|
;;
|
|
esac
|
|
# prevent expansion of glob special characters such as '*'
|
|
noglob=$(set +o | grep -F noglob)
|
|
set -o noglob
|
|
# we expect no early-initrd stuff on s390 so extract directly.
|
|
# globbing is turned off and we want word split on $patterns:
|
|
# shellcheck disable=SC2086
|
|
"$mycat" "${rd}" | cpio --extract --no-absolute-filenames \
|
|
--make-directories \
|
|
--preserve-modification-time $patterns
|
|
$noglob
|
|
if [ ! -f "squash-root.img" ]; then
|
|
cd - || break # change to $OLDPWD (popd alternative)
|
|
continue
|
|
fi
|
|
# prevent expansion of glob special characters such as '*'
|
|
noglob=$(set +o | grep -F noglob)
|
|
set -o noglob
|
|
# globbing is turned off and we want word split on $patterns:
|
|
# shellcheck disable=SC2086
|
|
unsquashfs -no-progress -dest "squashfs-root" "squash-root.img" \
|
|
$patterns
|
|
$noglob
|
|
rm -f "squash-root.img"
|
|
cd - || break # change to $OLDPWD (popd alternative)
|
|
done
|
|
cd "$mypopd" || return
|
|
}
|
|
|
|
########################################
|
|
collect_osaoat() {
|
|
local network_devices
|
|
local network_device
|
|
|
|
network_devices=$(lsqeth 2>/dev/null | grep "Device name" \
|
|
| sed 's/D.*:[[:space:]]*\([^[:space:]]*\)[[:space:]]\+/\1/g' \
|
|
| sed 's/[()]//g' )
|
|
if type qethqoat >/dev/null; then
|
|
if test -n "${network_devices}"; then
|
|
pr_collect_output "osa oat"
|
|
for network_device in ${network_devices}; do
|
|
call_run_command "qethqoat ${network_device}" \
|
|
"${OUTPUT_FILE_OSAOAT}.out" &&
|
|
call_run_command "qethqoat -r ${network_device}" \
|
|
"${OUTPUT_FILE_OSAOAT}_${network_device}.raw"
|
|
done
|
|
else
|
|
pr_skip "osa oat: no devices"
|
|
fi
|
|
else
|
|
pr_skip "osa oat: qethqoat not available"
|
|
fi
|
|
}
|
|
|
|
########################################
|
|
collect_ethtool() {
|
|
local network_devices
|
|
local network_device
|
|
|
|
network_devices=$(ls /sys/class/net 2>/dev/null)
|
|
if type ethtool >/dev/null; then
|
|
if test -n "${network_devices}"; then
|
|
pr_collect_output "ethtool"
|
|
for network_device in ${network_devices}; do
|
|
call_run_command "ethtool ${network_device}" \
|
|
"${OUTPUT_FILE_ETHTOOL}"
|
|
call_run_command "ethtool -k ${network_device}" \
|
|
"${OUTPUT_FILE_ETHTOOL}"
|
|
call_run_command "ethtool -a ${network_device}" \
|
|
"${OUTPUT_FILE_ETHTOOL}"
|
|
call_run_command "ethtool -c ${network_device}" \
|
|
"${OUTPUT_FILE_ETHTOOL}"
|
|
call_run_command "ethtool --per-queue ${network_device} \
|
|
--show-coalesce" "${OUTPUT_FILE_ETHTOOL}"
|
|
call_run_command "ethtool -g ${network_device}" \
|
|
"${OUTPUT_FILE_ETHTOOL}"
|
|
call_run_command "ethtool -i ${network_device}" \
|
|
"${OUTPUT_FILE_ETHTOOL}"
|
|
call_run_command "ethtool -l ${network_device}" \
|
|
"${OUTPUT_FILE_ETHTOOL}"
|
|
call_run_command "ethtool -P ${network_device}" \
|
|
"${OUTPUT_FILE_ETHTOOL}"
|
|
call_run_command "ethtool -S ${network_device}" \
|
|
"${OUTPUT_FILE_ETHTOOL}"
|
|
call_run_command "ethtool -T ${network_device}" \
|
|
"${OUTPUT_FILE_ETHTOOL}"
|
|
call_run_command "ethtool --module-info ${network_device}" \
|
|
"${OUTPUT_FILE_ETHTOOL}"
|
|
done
|
|
else
|
|
pr_skip "ethtool: no devices"
|
|
fi
|
|
else
|
|
pr_skip "ethtool: not available"
|
|
fi
|
|
}
|
|
|
|
########################################
|
|
collect_tc() {
|
|
local network_devices
|
|
local network_device
|
|
|
|
network_devices=$(ls /sys/class/net 2>/dev/null)
|
|
if type tc >/dev/null; then
|
|
if test -n "${network_devices}"; then
|
|
pr_collect_output "Trafic Control"
|
|
for network_device in ${network_devices}; do
|
|
call_run_command "tc -s qdisc show dev ${network_device}" \
|
|
"${OUTPUT_FILE_TC}"
|
|
done
|
|
else
|
|
pr_skip "Trafic Control: no devices"
|
|
fi
|
|
else
|
|
pr_skip "Trafic Control: tc not available"
|
|
fi
|
|
}
|
|
|
|
########################################
|
|
collect_bridge() {
|
|
local network_devices
|
|
local network_device
|
|
|
|
network_devices=$(ls /sys/class/net 2>/dev/null)
|
|
if type bridge >/dev/null; then
|
|
if test -n "${network_devices}"; then
|
|
pr_collect_output "bridge"
|
|
for network_device in ${network_devices}; do
|
|
call_run_command "bridge -d link show dev ${network_device}" \
|
|
"${OUTPUT_FILE_BRIDGE}"
|
|
call_run_command "bridge -s fdb show dev ${network_device}" \
|
|
"${OUTPUT_FILE_BRIDGE}"
|
|
call_run_command "bridge -d mdb show dev ${network_device}" \
|
|
"${OUTPUT_FILE_BRIDGE}"
|
|
done
|
|
else
|
|
pr_skip "bridge: no devices"
|
|
fi
|
|
else
|
|
pr_skip "bridge: not available"
|
|
fi
|
|
}
|
|
|
|
########################################
|
|
# OpenVSwitch
|
|
collect_ovs() {
|
|
local ovscmd
|
|
local bridge
|
|
local ovscmds
|
|
local ovsbrcmd
|
|
local ovsbrcmds
|
|
|
|
ovscmds="ovs-dpctl -s show\
|
|
:ovs-vsctl -t 5 show\
|
|
:ovsdb-client dump\
|
|
"
|
|
if type ovs-vsctl >/dev/null; then
|
|
pr_collect_output "OpenVSwitch"
|
|
IFS=:
|
|
for ovscmd in ${ovscmds}; do
|
|
IFS=${IFS_ORI} call_run_command "${ovscmd}" "${OUTPUT_FILE_OVS}"
|
|
done
|
|
IFS="${IFS_ORI}"
|
|
|
|
for bridge in ${ovs-vsctl list-br}; do
|
|
ovsbrcmds="ovs-ofctl show ${bridge}\
|
|
:ovs-ofctl dump-flows ${bridge}\
|
|
:ovs-appctl fdb/show ${bridge}\
|
|
"
|
|
IFS=:
|
|
for ovsbrcmd in ${ovsbrcmds}; do
|
|
IFS=${IFS_ORI} call_run_command "${ovsbrcmd}" "${OUTPUT_FILE_OVS}"
|
|
done
|
|
IFS="${IFS_ORI}"
|
|
done
|
|
else
|
|
pr_skip "OpenVSwitch: ovs-vsctl not available"
|
|
fi
|
|
}
|
|
|
|
########################################
|
|
collect_container() {
|
|
local container_list
|
|
local network_list
|
|
local item
|
|
|
|
# check if container environment exists
|
|
if test "x${DOCKER}" = "xYES" || test "x${KUBERNETES}" = "xYES"; then
|
|
pr_collect_output "container host"
|
|
else
|
|
pr_skip "container host: not found"
|
|
fi
|
|
|
|
# for docker command exists
|
|
if [ "x${DOCKER}" = "xYES" ]; then
|
|
pr_syslog_stdout " docker ..."
|
|
container_list=$(docker ps -qa)
|
|
network_list=$(docker network ls -q)
|
|
IFS=:
|
|
for item in ${DOCKER_CMDS}; do
|
|
IFS=${IFS_ORI} call_run_command "${item}" "${OUTPUT_FILE_DOCKER}"
|
|
done
|
|
IFS="${IFS_ORI}"
|
|
|
|
if test -n "${container_list}"; then
|
|
for item in ${container_list}; do
|
|
call_run_command "docker inspect ${item}" "${OUTPUT_FILE_DOCKER}"
|
|
done
|
|
fi
|
|
|
|
if test -n "${network_list}"; then
|
|
for item in ${network_list}; do
|
|
call_run_command "docker network inspect ${item}" \
|
|
"${OUTPUT_FILE_DOCKER}"
|
|
done
|
|
fi
|
|
fi
|
|
|
|
# for kubectl command exists
|
|
if [ "x${KUBERNETES}" = "xYES" ]; then
|
|
pr_syslog_stdout " Kubernetes ..."
|
|
container_list=$(kubectl top pod | grep -v "MEMORY(bytes)" | cut -d' ' -f1)
|
|
|
|
IFS=:
|
|
for item in ${KUBERNETES_CMDS}; do
|
|
IFS=${IFS_ORI} call_run_command "${item}" "${OUTPUT_FILE_KUBERNETES}.out"
|
|
done
|
|
IFS="${IFS_ORI}"
|
|
if test -n "${container_list}"; then
|
|
for item in ${container_list}; do
|
|
call_run_command "kubectl logs --tail=${KUBERNETES_LOG_LINES} ${item}"\
|
|
"${OUTPUT_FILE_KUBERNETES}.log"
|
|
done
|
|
fi
|
|
fi
|
|
}
|
|
|
|
########################################
|
|
collect_nvme() {
|
|
local device
|
|
|
|
if type nvme >/dev/null; then
|
|
pr_collect_output "NVME storage"
|
|
call_run_command "nvme list" "${OUTPUT_FILE_NVME}"
|
|
for device in /dev/nvme[0-9]*; do
|
|
if [ -c $device ]; then
|
|
call_run_command "smartctl -x $device" "${OUTPUT_FILE_NVME}"
|
|
call_run_command "nvme fw-log $device" "${OUTPUT_FILE_NVME}"
|
|
call_run_command "nvme smart-log $device" "${OUTPUT_FILE_NVME}"
|
|
call_run_command "nvme error-log $device" "${OUTPUT_FILE_NVME}"
|
|
fi
|
|
done
|
|
else
|
|
pr_skip "nvme: not available"
|
|
fi
|
|
}
|
|
|
|
########################################
|
|
collect_dasd() {
|
|
local device
|
|
|
|
if type dasdview >/dev/null; then
|
|
pr_collect_output "DASD storage"
|
|
call_run_command "lsdasd" "${OUTPUT_FILE_DASD}" # duplicate as file header
|
|
for device in /dev/dasd*; do
|
|
if [ -b $device ]; then
|
|
call_run_command "dasdview -i $device" "${OUTPUT_FILE_DASD}"
|
|
call_run_command "dasdview -t info $device" "${OUTPUT_FILE_DASD}"
|
|
else
|
|
echo "$device is no block device" >> ${OUTPUT_FILE_DASD}
|
|
fi
|
|
done
|
|
else
|
|
pr_skip "dasdview: not available"
|
|
fi
|
|
}
|
|
|
|
########################################
|
|
collect_kvm() {
|
|
local cmd
|
|
local domain_list
|
|
local domain
|
|
|
|
# check if KVM virsh command exists
|
|
if [ "x${KVM}" = "xYES" ]; then
|
|
pr_collect_output "KVM"
|
|
IFS=:
|
|
for cmd in ${KVM_CMDS}; do
|
|
IFS=${IFS_ORI} call_run_command "${cmd}" "${OUTPUT_FILE_KVM}"
|
|
done
|
|
IFS="${IFS_ORI}"
|
|
|
|
# domain/guest specific commands
|
|
domain_list=$(virsh list --all --name)
|
|
if test -n "${domain_list}"; then
|
|
for domain in ${domain_list}; do
|
|
call_run_command "virsh dominfo ${domain}" "${OUTPUT_FILE_KVM}"
|
|
call_run_command "virsh domblklist ${domain}" "${OUTPUT_FILE_KVM}"
|
|
call_run_command "virsh domstats ${domain}" "${OUTPUT_FILE_KVM}"
|
|
done
|
|
else
|
|
echo "no KVM domains found" | tee -a ${OUTPUT_FILE_KVM}
|
|
fi
|
|
else
|
|
pr_skip "KVM: no virsh command"
|
|
fi
|
|
}
|
|
|
|
########################################
|
|
post_processing() {
|
|
local file_mtime
|
|
local file_mtime_epoche
|
|
local tmp_file
|
|
local file_name
|
|
local base_dir
|
|
local dir_list
|
|
|
|
pr_syslog_stdout ${step_num} "Postprocessing"
|
|
|
|
# wipe possible passwords
|
|
dir_list="${WORKPATH} \
|
|
${WORKPATH}etc/ssl/ \
|
|
${WORKPATH}etc/libvirt/"
|
|
for base_dir in ${dir_list}; do
|
|
find "${base_dir}" -maxdepth 2 -name "*xml" -o -name "*conf" -o -name "*cnf" \
|
|
2>/dev/null | while read -r file_name; do
|
|
file_mtime_epoche=$(stat --format=%Y "${file_name}")
|
|
file_mtime=$(date +%Y%m%d%H%M.%S --date="@${file_mtime_epoche}")
|
|
tmp_file=${file_name}.$$
|
|
echo " clean pw: ${file_name}"
|
|
if ! sed "s/\(.*[Pp]assw.*=\).*/\1********/g" "${file_name}" > "${tmp_file}"; then
|
|
echo "${SCRIPTNAME}: Warning: Postprocessing failed on ${file_name}"
|
|
fi
|
|
mv "${tmp_file}" "${file_name}"
|
|
touch --time=mtime -t "${file_mtime}" "${file_name}"
|
|
done
|
|
done
|
|
|
|
# compressing data folder to avoid full unpack for any DBGINFO
|
|
base_dir="${WORKPATH}proc/"
|
|
search4="kallsyms"
|
|
find "${base_dir}" -name ${search4} 2>/dev/null | while read -r file_name; do
|
|
tmp_file=${file_name}-${KERNEL_BASE}.tgz
|
|
echo " compress: ${file_name}"
|
|
if ! tar -czf "${tmp_file}" -C "${base_dir}" "${search4}"; then
|
|
echo "${SCRIPTNAME}: Warning: Postprocessing failed on ${file_name}"
|
|
echo
|
|
else
|
|
rm -f "${file_name}"
|
|
fi
|
|
done
|
|
|
|
echo "log work directory state after post processing:"
|
|
echo "du summary: $(du -ks ${WORKPATH})"
|
|
df -k ${WORKPATH}
|
|
}
|
|
|
|
########################################
|
|
# Be aware that this output must be
|
|
# redirected into a separate logfile
|
|
call_run_command() {
|
|
local rc=0
|
|
local cmd="${1}"
|
|
local logfile="${2}"
|
|
# extract the raw_command and set cmd_type, as some shell might split into
|
|
# several lines restrict the cmd echo to the first line only
|
|
local raw_cmd=$(echo "${cmd}" | head -1 | sed -ne 's/^\([^[:space:]]*\).*$/\1/p')
|
|
local cmd_type=$(type ${raw_cmd} | cut -d' ' -sf4,5)
|
|
|
|
echo "#######################################################" >> "${logfile}"
|
|
echo "${USER}@${SYSTEMHOSTNAME:-localhost}> ${cmd}" >> "${logfile}"
|
|
|
|
# check calling command type
|
|
if [ "X${cmd_type}" = "Xshell builtin" ]; then
|
|
# command is a builtin (no use of timeout possible)
|
|
eval "${cmd}" >> ${logfile} 2>&1
|
|
rc=$?
|
|
elif [ "X${cmd_type}" != "Xnot found" ]; then
|
|
if [ "x${TIMEOUT_OK}" = "xYES" ]; then
|
|
eval timeout -k ${TOKS} ${TOS} "${cmd}" >> ${logfile} 2>&1
|
|
|
|
rc=$?
|
|
else
|
|
# fall back - call all existing commands without timeout
|
|
eval "${cmd}" >> ${logfile} 2>&1
|
|
rc=$?
|
|
fi
|
|
else
|
|
echo "${SCRIPTNAME}: Warning: Command \"${raw_cmd}\" not available" >> "${logfile}"
|
|
echo >> "${logfile}"
|
|
return 1
|
|
fi
|
|
|
|
# log a warning on rc not 0 and define return
|
|
if [ ${rc} ]; then
|
|
echo >> "${logfile}"
|
|
return 0
|
|
else
|
|
echo "${SCRIPTNAME}: Warning: Command \"${cmd}\" failed" >> "${logfile}"
|
|
echo >> "${logfile}"
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
########################################
|
|
call_collect_file() {
|
|
local file_name="${1}"
|
|
local directory_name=$(dirname "${file_name}" 2>/dev/null)
|
|
echo " ${file_name}"
|
|
|
|
if test ! -e "${WORKPATH}${directory_name}"; then
|
|
mkdir -p "${WORKPATH}${directory_name}" 2>&1
|
|
fi
|
|
if ! cp -r --preserve=mode,timestamps -dL --parents "${file_name}" "${WORKPATH}" 2>&1; then
|
|
return 1
|
|
else
|
|
return 0
|
|
fi
|
|
}
|
|
|
|
########################################
|
|
# print that an instance is already running
|
|
print_alreadyrunning() {
|
|
print_version
|
|
|
|
cat <<EOF
|
|
|
|
|
|
Please check the system if another instance of '${SCRIPTNAME}' is already
|
|
running. If this is not the case, please remove the lock file
|
|
'${LOCKFILE}'.
|
|
EOF
|
|
}
|
|
|
|
########################################
|
|
# Setup the environment
|
|
environment_setup() {
|
|
if test ! -e "${WORKDIR_BASE}"; then
|
|
mkdir -p "${WORKDIR_BASE}"
|
|
elif test ! -d "${WORKDIR_BASE}"; then
|
|
echo "${SCRIPTNAME}: Error: \"${WORKDIR_BASE}\" exists but this is a file!"
|
|
echo " Please make sure \"${WORKDIR_BASE}\" is a directory."
|
|
exit 1
|
|
fi
|
|
if test -e "${LOCKFILE}"; then
|
|
print_alreadyrunning
|
|
exit 1
|
|
else
|
|
touch "${LOCKFILE}"
|
|
echo "${DATETIME}" > "${LOCKFILE}"
|
|
fi
|
|
if ! mkdir "${WORKPATH}" 2>/dev/null; then
|
|
echo "${SCRIPTNAME}: Error: Target directory \"${WORKPATH}\" already exists or"
|
|
echo " \"${WORKDIR_BASE}\" does not exist!"
|
|
exit 1
|
|
fi
|
|
chmod 0700 "${WORKPATH}"
|
|
}
|
|
|
|
########################################
|
|
# create gzip-ped tar file
|
|
create_package() {
|
|
local rc_tar
|
|
pr_syslog_stdout ${step_num} "Finalizing: Creating archive with collected data"
|
|
# get a copy of the script used
|
|
cp -p "${FULLPATHSCRIPT}" "${WORKPATH}"
|
|
# create the archive
|
|
cd "${WORKDIR_BASE}"
|
|
touch "${WORKARCHIVE}"
|
|
chmod 0600 "${WORKARCHIVE}"
|
|
tar -czf "${WORKARCHIVE}" "${WORKDIR_CURRENT}"
|
|
rc_tar=$?
|
|
if [ $rc_tar -eq 0 ]; then
|
|
chmod 0600 "${WORKARCHIVE}"
|
|
pr_stdout " "
|
|
pr_stdout "Collected data was saved to:"
|
|
pr_stdout " >> ${WORKARCHIVE} <<"
|
|
pr_stdout " "
|
|
pr_stdout "Please review all collected data before sending to your" \
|
|
"service organization."
|
|
pr_stdout " "
|
|
elif [ $rc_tar -eq 127 ]; then
|
|
pr_stdout " "
|
|
pr_stdout "${SCRIPTNAME}: Error: tar command is not available!"
|
|
pr_stdout " Please install the corresponding package!"
|
|
else
|
|
pr_stdout " "
|
|
pr_stdout "${SCRIPTNAME}: Error: Collection of data failed!"
|
|
pr_stdout " The creation of \"${WORKARCHIVE}\" was not successful."
|
|
pr_stdout " Please check the directory \"${WORKDIR_BASE}\""
|
|
pr_stdout " to provide enough free available space."
|
|
fi
|
|
}
|
|
|
|
########################################
|
|
# Cleaning up the prepared/collected information
|
|
environment_cleanup() {
|
|
if ! rm -rf "${WORKPATH}" 2>/dev/null; then
|
|
pr_stdout " "
|
|
pr_stdout "${SCRIPTNAME}: Warning: Deletion of \"${WORKPATH}\" failed"
|
|
pr_stdout " Please remove the directory manually"
|
|
pr_stdout " "
|
|
fi
|
|
if ! rm -f "${LOCKFILE}" 2>/dev/null; then
|
|
pr_stdout " "
|
|
pr_stdout "${SCRIPTNAME}: Warning: Deletion of \"${LOCKFILE}\" failed"
|
|
pr_stdout " Please remove the file manually"
|
|
pr_stdout " "
|
|
fi
|
|
}
|
|
|
|
########################################
|
|
# Function to perform a cleanup in case of a received signal
|
|
emergency_exit() {
|
|
pr_stdout " "
|
|
pr_stdout "${SCRIPTNAME}: Info: Data collection has been interrupted"
|
|
pr_stdout " Cleanup of temporary collected data"
|
|
environment_cleanup
|
|
pr_stdout "${SCRIPTNAME}: Info: Emergency exit processed"
|
|
|
|
pr_stdout " "
|
|
logger -t "${SCRIPTNAME}" "Data collection interrupted"
|
|
exit
|
|
}
|
|
|
|
########################################
|
|
# Function to print to stdout when rediretion is active
|
|
pr_stdout() {
|
|
echo "${@}" >&8
|
|
}
|
|
|
|
########################################
|
|
# Function to print to stdout and into log file when rediretion is active
|
|
pr_log_stdout() {
|
|
echo "$@"
|
|
echo "$@" >&8
|
|
}
|
|
|
|
########################################
|
|
# Function to print to stdout and into log file when rediretion is active
|
|
pr_syslog_stdout() {
|
|
echo "$@" >&8
|
|
echo
|
|
echo "$(date +%H:%M:%S.%N) - " "$@"
|
|
logger -t "${SCRIPTNAME}" "$@"
|
|
}
|
|
|
|
########################################
|
|
# print "Collecting ... output"
|
|
pr_collect_output() {
|
|
pr_syslog_stdout ${step_num} "Collecting" $1 "output"
|
|
}
|
|
|
|
########################################
|
|
# print "Collecting ..." like fs
|
|
pr_collect() {
|
|
pr_syslog_stdout ${step_num} "Collecting" "$@"
|
|
}
|
|
|
|
########################################
|
|
# print "Skipping ..." info with reason
|
|
pr_skip() {
|
|
pr_syslog_stdout ${step_num} "Skip" "$@"
|
|
}
|
|
|
|
###############################################################################
|
|
# Running the script (main)
|
|
|
|
environment_setup
|
|
print_version
|
|
|
|
# saving stdout/stderr and redirecting stdout/stderr into log file
|
|
exec 8>&1 9>&2 >"${LOGFILE}" 2>&1
|
|
|
|
# run print_version again to be logged in dbginfo.log
|
|
print_version
|
|
|
|
# trap on SIGHUP=1 SIGINT=2 SIGTERM=15
|
|
trap emergency_exit SIGHUP SIGINT SIGTERM
|
|
|
|
pr_log_stdout ""
|
|
pr_log_stdout "Hardware platform = ${HW}"
|
|
pr_log_stdout "Runtime environment = ${RUNTIME_ENVIRONMENT} - DPM: ${DPM_MODE}"
|
|
pr_log_stdout "Kernel version = ${KERNEL_INFO} (${KERNEL_BASE})"
|
|
pr_log_stdout "OS version / distro = ${OS_NAME}"
|
|
pr_log_stdout "Date and time of info = ${DATETIME}"
|
|
pr_log_stdout ""
|
|
|
|
logger -t "${SCRIPTNAME}" "Starting data collection"
|
|
|
|
# step counter
|
|
current_step=1
|
|
# run all collection steps
|
|
for step in ${ALL_STEPS}; do
|
|
# generate step numbering
|
|
step_num="${current_step} of ${COLLECTION_COUNT}: "
|
|
# calling step procedure
|
|
${step}
|
|
current_step=`expr ${current_step} + 1`
|
|
done
|
|
|
|
logger -t "${SCRIPTNAME}" "Data collection completed"
|
|
|
|
exec 1>&8 2>&9 8>&- 9>&-
|
|
|
|
#EOF
|