dbginfo.sh: cleanup 2.x kernel & adding timeout

- clenaup outdated kernel - we no longer have 2.x kenrel in support
- add timeout on command excution - ensure to not miss all data for a
  single command to hang

Signed-off-by: Joern Siglen <siglen@de.ibm.com>
Signed-off-by: Jan Höppner <hoeppner@linux.ibm.com>
This commit is contained in:
Joern Siglen
2021-08-30 13:30:57 +02:00
committed by Jan Höppner
parent 5b96d86538
commit 68ff79e4c3
3 changed files with 184 additions and 141 deletions

View File

@@ -5,7 +5,9 @@ Release history for s390-tools (MIT version)
For Linux kernel version: 5.xx
Changes of existing tools:
- dbginfo.sh: cleanup of outdated sections and general code rework
- dbginfo.sh: Collect zipl boot menu entries from boot loader specification
- dbginfo.sh: add collection in area of timedate, coredump and --check option
Bug Fixes:

View File

@@ -12,12 +12,47 @@
LC_ALL=C
export LC_ALL
# The general name of this script
readonly SCRIPTNAME="${0##*/}"
########################################
# Global used variables
readonly SCRIPTNAME="${0##*/}" # general name of this script
#
readonly DOCKER=$(if which docker >/dev/null 2>&1; then echo "YES"; else echo "NO"; fi)
readonly HW="$(uname -i 2>/dev/null)"
# retrieve and split kernel version
readonly KERNEL_BASE="$(uname -r 2>/dev/null)"
readonly KERNEL_VERSION=$(echo ${KERNEL_BASE} | cut -d'.' -f1 )
readonly KERNEL_MAJOR_REVISION=$(echo ${KERNEL_BASE} | cut -d'.' -f2 )
readonly KERNEL_MINOR_REVISION=$(echo ${KERNEL_BASE} | cut -d'.' -f3 | sed 's/[^0-9].*//g')
readonly KERNEL_INFO=${KERNEL_VERSION}.${KERNEL_MAJOR_REVISION}.${KERNEL_MINOR_REVISION}
readonly KVM=$(if which virsh >/dev/null 2>&1; then echo "YES"; else echo "NO"; fi)
# The file to indicate that another instance of the script is already running
readonly LOCKFILE="/tmp/${SCRIPTNAME}.lock"
# check limits for logfiles like /var/log/messages
readonly LOG_FILE_SIZE_CHECK=50 # max logfile size in MB
readonly LOG_FILE_AGE_CHECK=7 # age in days to include for size checking
# distro info
readonly OSPRETTY="$(cat /etc/os* 2>/dev/null | grep -m1 PRETTY_NAME | sed 's/\"//g')"
readonly OS_NAME="${OSPRETTY##*=}"
# The processor ID for the first processor
readonly PROCESSORID="$(grep -E ".*processor 0:.*" /proc/cpuinfo | \
sed 's/.*identification[[:space:]]*\=[[:space:]]*\([[:alnum:]]*\).*/\1/g')"
readonly PROCESSORVERSION="$(grep -E ".*processor 0:.*" /proc/cpuinfo | \
sed 's/.*version[[:space:]]*\=[[:space:]]*\([[:alnum:]]*\).*/\1/g')"
if test "x${PROCESSORVERSION}" = "xFF" || test "x${PROCESSORVERSION}" = "xff"; then
RUNTIME_ENVIRONMENT=$(grep -E "VM00.*Control Program.*" /proc/sysinfo | \
sed 's/.*:[[:space:]]*\([[:graph:]]*\).*/\1/g')
else
RUNTIME_ENVIRONMENT="LPAR"
fi
readonly TOS=15 # timeout seconds for command execution
readonly ZDEV_CONF=$(lszdev --configured 2>/dev/null | wc -l)
readonly ZDEV_OFF=$(lszdev --offline 2>/dev/null | wc -l)
readonly ZDEV_ONL=$(lszdev --online 2>/dev/null | wc -l)
paramWORKDIR_BASE="/tmp/" # initial default path
########################################
# print version info
# print dbginfo.sh version info
print_version() {
cat <<EOF
${SCRIPTNAME}: Debug information script version %S390_TOOLS_VERSION%
@@ -25,11 +60,9 @@ Copyright IBM Corp. 2002, 2021
EOF
}
########################################
# print how to use this script
print_usage()
{
print_usage() {
print_version
cat <<EOF
@@ -41,11 +74,12 @@ This script collects runtime, configuration and trace information on
a Linux on IBM Z installation for debugging purposes.
It also traces information about z/VM if the Linux runs under z/VM.
KVM or DOCKER data ist collected on a host serving this.
Default location for data collection and final tar file is "/tmp/".
The collected information is written to a TAR archive named
/tmp/DBGINFO-[date]-[time]-[hostname]-[processorid].tgz
DBGINFO-[date]-[time]-[hostname]-[processorid].tgz
where [date] and [time] are the date and time when debug data is collected.
[hostname] indicates the hostname of the system the data was collected from.
@@ -53,24 +87,82 @@ The [processorid] is taken from the processor 0 and indicates the processor
identification.
Options:
-d|--directory specify the directory where the data collection
stores the temporary data and the final archive.
-h|--help print this help
-v|--version print version information
-c|--check online quick check (no data collection)
Please report bugs to: linux390@de.ibm.com
EOF
}
########################################
# check for oversize logfiles and missing rotation
logfile_checker() {
local counter
local logfile
local logfiles
# find files bigger than recommended
counter=$(find $1 -maxdepth 1 -type f -mtime -${LOG_FILE_AGE_CHECK} \
-size ${LOG_FILE_SIZE_CHECK}M | wc -l)
echo " ${counter} logfiles over ${LOG_FILE_SIZE_CHECK} MB"
# maybe check for rotation of base names
if [ ${counter} -ne 0 ]; then
for logfile in $(find $1 -maxdepth 1 -type f -mtime -${LOG_FILE_AGE_CHECK} \
-size ${LOG_FILE_SIZE_CHECK}M -print); do
# use a neutral separtor ':' as concat is different in some bash
# insert the 'blank' for later use in for loop
# add the base name before '.' or '-' only for checks
logfiles="${logfiles}: ${logfile%%[.-]*}"
done
# change separator to new line for sorting
logfiles=$(echo "${logfiles}" | sed s'/:/\n/g' | sort -u)
for logfile in ${logfiles}; do
counter=$(ls ${logfile}* 2>/dev/null | wc -l)
if [ ${counter} -eq 1 ]; then
echo " CHECK - ${logfile} may miss a rotation"
else
echo " OK - ${logfile}* may have a rotation in place: ${counter} files"
fi
done
fi
}
########################################
# print basic info and online checks
print_check() {
print_version
cat <<EOF
Hardware platform = ${HW}
Runtime environment = ${RUNTIME_ENVIRONMENT}
Kernel version = ${KERNEL_INFO}
OS version / distro = ${OS_NAME}
KVM host = ${KVM}
DOCKER host = ${DOCKER}
Current user = $(whoami) (must be root for data collection)
Date and time = $(date)
Uptime =$(uptime)
Number of coredumps = $(corecumpctl 2>/dev/null | wc -l)
zdevice onl/conf/offl = ${ZDEV_ONL} / ${ZDEV_CONF} / ${ZDEV_OFF}
Log file check =$(logfile_checker "/var/log*")
Working directory = $(ls -d ${paramWORKDIR_BASE} 2>&1 && df -k ${paramWORKDIR_BASE})
$(ls -ltr ${paramWORKDIR_BASE}/DBGINFO*tgz 2>/dev/null | tail -2)
$(ls ${LOCKFILE} 2>/dev/null && echo " WARNING: dbginfo running since: $(cat ${LOCKFILE})")
This is a console output only - no data was saved using option -c !
EOF
}
#######################################
# Parsing the command line and pre checks
#
paramWORKDIR_BASE="/tmp/"
while [ ${#} -gt 0 ]; do
case ${1} in
--help|-h)
@@ -87,11 +179,19 @@ while [ ${#} -gt 0 ]; do
echo "${SCRIPTNAME}: Error: No directory specified for data collection!"
echo
exit 1
elif test ! -d "${paramWORKDIR_BASE}"; then
echo "${SCRIPTNAME}: Error: The specified directory \"${paramWORKDIR_BASE}\" does not exist!"
echo
exit 1
else
# jump to next param, if already last the final shift can do termination
# jump to next param
shift
fi
;;
--check|-c)
print_check
exit 0
;;
-*|--*|*)
echo
echo "${SCRIPTNAME}: invalid option \"${1}\""
@@ -100,17 +200,10 @@ while [ ${#} -gt 0 ]; do
exit 1
;;
esac
# next parameter
# next parameter, if already last the final shift will do termination
shift
done
# check for a valid path
if test ! -d "${paramWORKDIR_BASE}"; then
echo "${SCRIPTNAME}: Error: The specified directory \"${paramWORKDIR_BASE}\" does not exist!"
echo
exit 1
fi
# finally verification to run as root
if test "$(/usr/bin/id -u 2>/dev/null)" -ne 0; then
echo "${SCRIPTNAME}: Error: You must be user root to run \"${SCRIPTNAME}\"!"
@@ -133,12 +226,6 @@ readonly SYSTEMHOSTNAME="$(hostname -s 2>/dev/null)"
# The kernel release version as delivered from uname -r
readonly KERNEL_RELEASE_VERSION="$(uname -r 2>/dev/null)"
# The processor ID for the first processor
readonly PROCESSORID="$(grep -E ".*processor 0:.*" /proc/cpuinfo | \
sed 's/.*identification[[:space:]]*\=[[:space:]]*\([[:alnum:]]*\).*/\1/g')"
# The processor version for the first processor
readonly PROCESSORVERSION="$(grep -E ".*processor 0:.*" /proc/cpuinfo | \
sed 's/.*version[[:space:]]*\=[[:space:]]*\([[:alnum:]]*\).*/\1/g')"
# The current date
readonly DATETIME="$(date +%Y-%m-%d-%H-%M-%S 2>/dev/null)"
@@ -158,9 +245,6 @@ readonly WORKARCHIVE="${WORKDIR_BASE}${WORKDIR_CURRENT}.tgz"
# The log file of activities from this script execution
readonly LOGFILE="${WORKPATH}dbginfo.log"
# The file to indicate that another instance of the script is already running
readonly LOCKFILE="/tmp/${SCRIPTNAME}.lock"
# File that includes output of Linux commands
readonly OUTPUT_FILE_CMD="${WORKPATH}runtime.out"
@@ -203,39 +287,6 @@ readonly OUTPUT_FILE_KVM="${WORKPATH}kvm_runtime.out"
# Mount point of the debug file system
readonly MOUNT_POINT_DEBUGFS="/sys/kernel/debug"
# The kernel version (e.g. '2' from 2.6.32 or '3' from 3.2.1)
readonly KERNEL_VERSION=$(uname -r 2>/dev/null | cut -d'.' -f1)
# The kernel major revision number (e.g. '6' from 2.6.32 or '2' from 3.2.1)
readonly KERNEL_MAJOR_REVISION=$(uname -r 2>/dev/null | cut -d'.' -f2)
# The kernel mainor revision number (e.g. '32' from 2.6.32 or '1' from 3.2.1)
readonly KERNEL_MINOR_REVISION=$(uname -r 2>/dev/null | cut -d'.' -f3 | sed 's/[^0-9].*//g')
# Is this kernel supporting sysfs - since 2.4 (0=yes, 1=no)
if test "${KERNEL_VERSION}" -lt 2 ||
( test "${KERNEL_VERSION}" -eq 2 && test "${KERNEL_MAJOR_REVISION}" -le 4 ); then
readonly LINUX_SUPPORT_SYSFS=1
else
readonly LINUX_SUPPORT_SYSFS=0
fi
# Is this kernel potentially using the /sys/kernel/debug feature - since 2.6.13 (0=yes, 1=no)
if test "${KERNEL_VERSION}" -lt 2 ||
( test "${KERNEL_VERSION}" -eq 2 &&
( test "${KERNEL_MAJOR_REVISION}" -lt 6 ||
( test "${KERNEL_MAJOR_REVISION}" -eq 6 && test "${KERNEL_MINOR_REVISION}" -lt 13 ))); then
readonly LINUX_SUPPORT_SYSFSDBF=1
else
readonly LINUX_SUPPORT_SYSFSDBF=0
fi
if test "x${PROCESSORVERSION}" = "xFF" || test "x${PROCESSORVERSION}" = "xff"; then
readonly RUNTIME_ENVIRONMENT=$(grep -E "VM00.*Control Program.*" /proc/sysinfo| sed 's/.*:[[:space:]]*\([[:graph:]]*\).*/\1/g')
else
readonly RUNTIME_ENVIRONMENT="LPAR"
fi
# define order of collection steps
ALL_STEPS="\
collect_cmdsout\
@@ -310,26 +361,6 @@ if test -e /proc/scsi; then
"
fi
# Adding files to PROCFILES in case we run on Kernel 2.4 or older
if test "${LINUX_SUPPORT_SYSFS}" -eq 1; then
PROCFILES="${PROCFILES}\
/proc/chpids\
/proc/chandev\
/proc/ksyms\
/proc/lvm/global\
/proc/subchannels\
"
fi
# Adding s390dbf files to PROCFILE in case we run on Kernel lower than 2.6.13
if test "${LINUX_SUPPORT_SYSFSDBF}" -eq 1; then
if test -e /proc/s390dbf; then
PROCFILES="${PROCFILES}\
$(find /proc/s390dbf -type f -not -path "*/raw" -not -path "*/flush" 2>/dev/null)\
"
fi
fi
########################################
LOGFILES="\
@@ -706,11 +737,7 @@ collect_sysfs() {
local file_name
debugfs_mounted=0
# Requires kernel version newer then 2.4
if test "${LINUX_SUPPORT_SYSFS}" -eq 0; then
pr_syslog_stdout "${step_num} Collecting sysfs"
# Requires kernel version of 2.6.13 or newer
if test "${LINUX_SUPPORT_SYSFSDBF}" -eq 0; then
if ! grep -qE "${MOUNT_POINT_DEBUGFS}.*debugfs" /proc/mounts 2>/dev/null; then
if mount -t debugfs debugfs "${MOUNT_POINT_DEBUGFS}" >/dev/null 2>&1; then
sleep 2
@@ -719,7 +746,6 @@ collect_sysfs() {
pr_log_stdout "${SCRIPTNAME}: Warning: Unable to mount debugfs at \"${MOUNT_POINT_DEBUGFS}\""
fi
fi
fi
# Collect sysfs files using multiple threads (-J 1) while excluding
# files known to block on read (-x). Stop reading a file that takes
@@ -748,9 +774,6 @@ collect_sysfs() {
if test ${debugfs_mounted} -eq 1; then
umount "${MOUNT_POINT_DEBUGFS}"
fi
else
pr_syslog_stdout "${step_num} Collecting sysfs skipped. Kernel $(uname -r) must be newer than 2.4"
fi
pr_log_stdout " "
}
@@ -766,7 +789,7 @@ collect_logfiles() {
call_collect_file "${file_name}"
done
pr_log_stdout " "
pr_log_stdout "$(logfile_checker "/var/log*")"
}
@@ -1091,34 +1114,36 @@ post_processing() {
# Be aware that this output must be
# redirected into a separate logfile
call_run_command() {
local cmd
local logfile
local raw_cmd
cmd="${1}"
logfile="${2}"
raw_cmd=$(echo "${cmd}" | sed -ne 's/^\([^[:space:]]*\).*$/\1/p')
local rc
local cmd="${1}"
local logfile="${2}"
local raw_cmd=$(echo "${cmd}" | sed -ne 's/^\([^[:space:]]*\).*$/\1/p')
echo "#######################################################" >> "${logfile}"
echo "${USER}@${SYSTEMHOSTNAME:-localhost}> ${cmd}" >> "${logfile}"
# check if command exists
if ! which "${raw_cmd}" >/dev/null 2>&1; then
# check if command is a builtin
if ! command -v "${raw_cmd}" >/dev/null 2>&1; then
echo "${SCRIPTNAME}: Warning: Command \"${raw_cmd}\" not available" >> "${logfile}"
echo >> "${logfile}"
return 1
fi
# check if calling command and timeout exist
if which "${raw_cmd}" >/dev/null 2>&1 && which timeout >/dev/null 2>&1; then
eval timeout ${TOS} "${cmd}" >> ${logfile} 2>&1
rc=$?
# check if command is a builtin (no use of timeout possible)
elif command -v "${raw_cmd}" >/dev/null 2>&1; then
eval "${cmd}" >> ${logfile} 2>&1
rc=$?
else
echo "${SCRIPTNAME}: Warning: Command \"${raw_cmd}\" not available" >> "${logfile}"
echo >> "${logfile}"
return 1
fi
if ! eval "${cmd}" >> "${logfile}" 2>&1; then
echo "${SCRIPTNAME}: Warning: Command \"${cmd}\" failed" >> "${logfile}"
echo >> "${logfile}"
return 1
# log a warning on rc not 0 and define return
if [ ${rc} ]; then
echo >> "${logfile}"
return 0
else
echo >> "${logfile}"
return 0
echo "${SCRIPTNAME}: Warning: Command \"${cmd}\" failed" >> "${logfile}"
echo >> "${logfile}"
return 1
fi
}
@@ -1285,9 +1310,8 @@ pr_syslog_stdout()
logger -t "${SCRIPTNAME}" "$@"
}
###############################################################################
# Running the script
# Running the script (main)
environment_setup
print_version
@@ -1299,9 +1323,10 @@ exec 8>&1 9>&2 >"${LOGFILE}" 2>&1
trap emergency_exit SIGHUP SIGINT SIGTERM
pr_log_stdout ""
pr_log_stdout "Hardware platform = $(uname -i)"
pr_log_stdout "Kernel version = ${KERNEL_VERSION}.${KERNEL_MAJOR_REVISION}.${KERNEL_MINOR_REVISION} ($(uname -r 2>/dev/null))"
pr_log_stdout "Hardware platform = ${HW}"
pr_log_stdout "Kernel version = ${KERNEL_INFO} (${KERNEL_BASE})"
pr_log_stdout "Runtime environment = ${RUNTIME_ENVIRONMENT}"
pr_log_stdout "OS version / distro = ${OS_NAME}"
pr_log_stdout ""
logger -t "${SCRIPTNAME}" "Starting data collection"

View File

@@ -14,8 +14,10 @@ for debugging Linux on IBM Z
This script collects runtime, configuration and trace information that can
be used to debug a Linux on IBM Z instance.
For Linux on z/VM, the script also traces information about the z/VM system.
KVM or DOCKER data ist collected on a host serving this.
The debug information is written to a file
/tmp/DBGINFO\-<date>\-<time>\-<hostname>\-<processorid>.tgz
/<DIRECTORY>/DBGINFO\-<date>\-<time>\-<hostname>\-<processorid>.tgz
where [date] and [time] are the date and time when the debug data was
collected. [hostname] indicates the hostname of the system the data was
collected from. The [processorid] is taken from the processor 0 and indicates
@@ -32,7 +34,12 @@ Print version information, then exit.
.TP
\fB\-d <DIRECTORY>\fP, \fB\-\-directory <DIRECTORY>\fP
Specify the DIRECTORY where the data collection stores the temporary data and the final archive. The specified directory must already exist. If this parameter is not specified, /tmp is used by default.
Specify the DIRECTORY where the data collection stores the temporary data and the final archive.
The specified directory must already exist. If this parameter is not specified, /tmp is used by default.
.TP
\fB\-c\fP, \fB\-\-check\fP
Print online quick check (not saving any data) - can be combind with a preceding -d option.
.SH FILES
A .tgz file of the form
@@ -56,48 +63,57 @@ Copyright IBM Corp. 2002, 2021
.PP
Hardware platform = s390x
.br
Kernel version = 5.4.0 (5.4.0-70-generic)
.br
Runtime environment = z/VM
.br
Kernel version = 4.18.0 (4.18.0-305.el8.s390x)
.br
OS version / distro = Red Hat Enterprise Linux 8.4 (Ootpa)
.br
Date and time of info = 2021-08-26-15-27-58
.PP
1 of 16: Collecting command output
1 of 18: Collecting command output
.PP
2 of 16: Collecting z/VM command output
2 of 18: Collecting hyptop for z/VM - 5s output
.PP
3 of 16: Collecting procfs
3 of 18: Collecting z/VM output
.PP
4 of 16: Collecting sysfs
4 of 18: Collecting procfs
.PP
5 of 16: Collecting log files
5 of 18: Collecting sysfs
.PP
6 of 16: Collecting config files
6 of 18: Collecting config files
.PP
7 of 16: Collecting osa oat output
7 of 18: Collecting network output
.PP
8 of 16: Collecting ethtool output
8 of 18: Collecting osa oat output
.PP
9 of 16: Collecting tc output
9 of 18: Collecting ethtool output
.PP
10 of 16: Collecting bridge output
10 of 18: Collecting Trafic Control output
.PP
11 of 16: Collecting OpenVSwitch output
11 of 18: Collecting bridge output
.PP
12a of 16: Collecting docker container
12b of 16: Collecting docker network
12 of 18: Skip OpenVSwitch: ovs-vsctl not available
.PP
13 of 16: Collecting nvme output
13 of 18: Skip KVM: no virsh command
.PP
14 of 16: Collecting KVM data
14 of 18: Collecting docker output
.PP
15 of 16: Postprocessing
15 of 18: Collecting NVME storage output
.PP
16 of 16: Finalizing: Creating archive with collected data
16 of 18: Collecting log files
.PP
0 logfiles over 50 MB
.PP
17 of 18: Postprocessing
.PP
18 of 18: Finalizing: Creating archive with collected data
.PP
Collected data was saved to:
.br
>> /data\-collection/DBGINFO\-2021\-04\-20\-14\-00\-07\-host\-012345.tgz <<
>> /data\-collection/DBGINFO\-2021\-08\-26\-15\-27\-58\-host\-012345.tgz <<
.br
Review the collected data before sending to your service organization.
Please review all collected data before sending to your service organization.
.SH HINTS
Run the script with root authority.
.br