diff --git a/hack/grab-profiles.sh b/hack/grab-profiles.sh index 04422f80b9f..d51b35577ee 100755 --- a/hack/grab-profiles.sh +++ b/hack/grab-profiles.sh @@ -18,17 +18,48 @@ set -o errexit set -o nounset set -o pipefail +function grab_profiles_from_component { + local requested_profiles=$1 + local mem_pprof_flags=$2 + local binary=$3 + local tunnel_port=$4 + local path=$5 + local output_prefix=$6 + local timestamp=$7 + + echo "binary: $binary" + + for profile in ${requested_profiles}; do + case ${profile} in + cpu) + go tool pprof "-pdf" "${binary}" "http://localhost:${tunnel_port}/${path}/profile" > "${output_prefix}-${profile}-profile-${timestamp}.pdf" + ;; + mem) + # There are different kinds of memory profiles that are available that + # had to be grabbed separately: --inuse-space, --inuse-objects, + # --alloc-space, --alloc-objects. We need to iterate over all requested + # kinds. + for flag in ${mem_pprof_flags}; do + go tool pprof "-${flag}" "-pdf" "${binary}" "http://localhost:${tunnel_port}/${path}/heap" > "${output_prefix}-${profile}-${flag}-profile-${timestamp}.pdf" + done + ;; + esac + done +} + KUBE_ROOT=$(dirname "${BASH_SOURCE}")/.. source "${KUBE_ROOT}/hack/lib/init.sh" server_addr="" +kubelet_addreses="" +kubelet_binary="" requested_profiles="" mem_pprof_flags="" profile_components="" output_dir="." tunnel_port="${tunnel_port:-1234}" -args=$(getopt -o s:mho: -l server:,master,heapster,output:,help,inuse-space,inuse-objects,alloc-space,alloc-objects,cpu -- "$@") +args=$(getopt -o s:mho:k: -l server:,master,heapster,output:,kubelet:,help,inuse-space,inuse-objects,alloc-space,alloc-objects,cpu,kubelet-binary: -- "$@") if [[ $? -ne 0 ]]; then >&2 echo "Error in getopt" exit 1 @@ -37,6 +68,7 @@ fi HEAPSTER_VERSION="v0.18.2" MASTER_PPROF_PATH="debug/pprof" HEAPSTER_PPROF_PATH="api/v1/proxy/namespaces/kube-system/services/monitoring-heapster/debug/pprof" +KUBELET_PPROF_PATH_PREFIX="api/v1/proxy/nodes" eval set -- "${args}" @@ -68,6 +100,25 @@ while true; do output_dir=$1 shift ;; + -k|--kubelet) + shift + profile_components="kubelet ${profile_components}" + if [ -z "$1" ]; then + >&2 echo "empty argumet to --kubelet flag" + exit 1 + fi + kubelet_addreses="$1 $kubelet_addreses" + shift + ;; + --kubelet-binary) + shift + if [ -z "$1" ]; then + >&2 echo "empty argumet to --kubelet-binary flag" + exit 1 + fi + kubelet_binary=$1 + shift + ;; --inuse-space) shift requested_profiles="mem ${requested_profiles}" @@ -139,6 +190,8 @@ kube::util::trap_add 'kill $SSH_PID' EXIT kube::util::trap_add 'kill $SSH_PID' SIGTERM requested_profiles=$(echo ${requested_profiles} | xargs -n1 | sort -u | xargs) +profile_components=$(echo ${profile_components} | xargs -n1 | sort -u | xargs) +kubelet_addreses=$(echo ${kubelet_addreses} | xargs -n1 | sort -u | xargs) echo "requested profiles: ${requested_profiles}" echo "flags for heap profile: ${mem_pprof_flags}" @@ -159,19 +212,21 @@ for component in ${profile_components}; do binary=heapster path=${HEAPSTER_PPROF_PATH} ;; + kubelet) + path="${KUBELET_PPROF_PATH_PREFIX}" + if [[ -z "${kubelet_binary}" ]]; then + binary="${KUBE_ROOT}/_output/local/bin/linux/amd64/kubelet" + else + binary=${kubelet_binary} + fi + ;; esac - for profile in ${requested_profiles}; do - case ${profile} in - cpu) - go tool pprof "-pdf" "${binary}" "http://localhost:${tunnel_port}/${path}/profile" > "${output_dir}/${component}-${profile}-profile-${timestamp}.pdf" - ;; - mem) - for flag in ${mem_pprof_flags}; do - go tool pprof "-${flag}" "-pdf" "${binary}" "http://localhost:${tunnel_port}/${path}/heap" > "${output_dir}/${component}-${profile}-${flag}-profile-${timestamp}.pdf" - done - ;; - esac - done + if [[ "${component}" == "kubelet" ]]; then + for node in $(echo ${kubelet_addreses} | sed 's/[,;]/\n/g'); do + grab_profiles_from_component "${requested_profiles}" "${mem_pprof_flags}" "${binary}" "${tunnel_port}" "${path}/${node}/debug/pprof" "${output_dir}/${component}" "${timestamp}" + done + else + grab_profiles_from_component "${requested_profiles}" "${mem_pprof_flags}" "${binary}" "${tunnel_port}" "${path}" "${output_dir}/${component}" "${timestamp}" + fi done -