From 8e879db9387f40f78ddba44c377eb8ddfb603af4 Mon Sep 17 00:00:00 2001 From: Maciej Borsz Date: Tue, 18 Dec 2018 11:57:03 +0100 Subject: [PATCH 1/2] Revert "Revert "Check for hostError and automaticRestart when test finishes."" This reverts commit 047aa25484e72427a9afad168630d75dbd44ed14. --- cluster/log-dump/log-dump.sh | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/cluster/log-dump/log-dump.sh b/cluster/log-dump/log-dump.sh index 60d962dc382..6609fd01292 100755 --- a/cluster/log-dump/log-dump.sh +++ b/cluster/log-dump/log-dump.sh @@ -429,6 +429,30 @@ function dump_nodes_with_logexporter() { fi } +function detect_node_failures() { + if ! [[ "${gcloud_supported_providers}" =~ "${KUBERNETES_PROVIDER}" ]]; then + return + fi + + detect-node-names + for group in "${INSTANCE_GROUPS[@]}"; do + local creation_timestamp=$(gcloud compute instance-groups managed describe \ + "${group}" \ + --project "${PROJECT}" \ + --zone "${ZONE}" \ + --format='value(creationTimestamp)') + echo "Failures for ${group}" + gcloud logging read --order=asc \ + --format='table(timestamp,jsonPayload.resource.name,jsonPayload.event_subtype)' \ + --project "${PROJECT}" \ + "resource.type=\"gce_instance\" + logName=\"projects/${PROJECT}/logs/compute.googleapis.com%2Factivity_log\" + (jsonPayload.event_subtype=\"compute.instances.hostError\" OR jsonPayload.event_subtype=\"compute.instances.automaticRestart\") + jsonPayload.resource.name:\"${group}\" + timestamp >= \"${creation_timestamp}\"" + done +} + function main() { setup # Copy master logs to artifacts dir locally (through SSH). @@ -447,6 +471,8 @@ function main() { echo "Dumping logs from nodes locally to '${report_dir}'" dump_nodes fi + + detect_node_failures } main From 325511d0ab993c34070a1ecac1f29572a96b6b97 Mon Sep 17 00:00:00 2001 From: Maciej Borsz Date: Tue, 18 Dec 2018 11:59:11 +0100 Subject: [PATCH 2/2] Check if INSTANCE_GROUPS is empty in detect_node_failures. --- cluster/log-dump/log-dump.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cluster/log-dump/log-dump.sh b/cluster/log-dump/log-dump.sh index 6609fd01292..c74afe11b4b 100755 --- a/cluster/log-dump/log-dump.sh +++ b/cluster/log-dump/log-dump.sh @@ -435,6 +435,9 @@ function detect_node_failures() { fi detect-node-names + if [ -z "$INSTANCE_GROUPS" ]; then + return + fi for group in "${INSTANCE_GROUPS[@]}"; do local creation_timestamp=$(gcloud compute instance-groups managed describe \ "${group}" \