From 004c3bc7ad8dcac5d10aeda4902203d26f05179b Mon Sep 17 00:00:00 2001 From: Jeff Grafton Date: Tue, 6 Oct 2015 14:53:38 -0700 Subject: [PATCH 1/2] Add script to use gcloud to print GCP resources used. --- cluster/gce/list-resources.sh | 68 +++++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100755 cluster/gce/list-resources.sh diff --git a/cluster/gce/list-resources.sh b/cluster/gce/list-resources.sh new file mode 100755 index 00000000000..9c0d9c0420b --- /dev/null +++ b/cluster/gce/list-resources.sh @@ -0,0 +1,68 @@ +#!/bin/bash + +# Copyright 2015 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Calls gcloud to print out a variety of Google Cloud Platform resources used by +# Kubernetes. Can be run before/after test runs and compared to track leaking +# resources. + +# PROJECT must be set in the environment. +# If ZONE, KUBE_GCE_INSTANCE_PREFIX, CLUSTER_NAME, KUBE_GCE_NETWORK, or +# KUBE_GKE_NETWORK is set, they will be used to filter the results. + +set -o errexit +set -o nounset +set -o pipefail + +ZONE=${ZONE:-} +REGION=${ZONE%-*} +INSTANCE_PREFIX=${KUBE_GCE_INSTANCE_PREFIX:-${CLUSTER_NAME:-}} +NETWORK=${KUBE_GCE_NETWORK:-${KUBE_GKE_NETWORK:-}} + +# Usage: gcloud-compute-list +# GREP_REGEX is applied to the output of gcloud if set +GREP_REGEX="" +function gcloud-compute-list() { + local -r resource=$1 + echo -e "\n\n[ ${resource} ]" + gcloud compute ${resource} list --project=${PROJECT} ${@:2} | grep "${GREP_REGEX}" +} + +echo "Project: ${PROJECT}" +echo "Region: ${REGION}" +echo "Zone: ${ZONE}" +echo "Instance prefix: ${INSTANCE_PREFIX:-}" +echo "Network: ${NETWORK:-}" + +# List resources related to instances, filtering by the instance prefix if +# provided. +gcloud-compute-list instance-templates --regexp="${INSTANCE_PREFIX}.*" +gcloud-compute-list instance-groups ${ZONE:+"--zone=${ZONE}"} --regexp="${INSTANCE_PREFIX}.*" +gcloud-compute-list instances ${ZONE:+"--zone=${ZONE}"} --regexp="${INSTANCE_PREFIX}.*" + +# List disk resources, filterying by instance prefix if provided. +gcloud-compute-list disks ${ZONE:+"--zone=${ZONE}"} --regexp="${INSTANCE_PREFIX}.*" + +# List network resources. We include names starting with "a", corresponding to +# those that Kubernetes creates. +gcloud-compute-list addresses ${REGION:+"--region=${REGION}"} --regexp="a.*|${INSTANCE_PREFIX}.*" +# Match either the header or a line with the specified e2e network. +# This assumes that the network name is the second field in the output. +GREP_REGEX="^NAME\|^[^\s]\+\s\+\(default\|${NETWORK}\)\s" +gcloud-compute-list routes --regexp="default.*|${INSTANCE_PREFIX}.*" +gcloud-compute-list firewall-rules --regexp="default.*|k8s-fw.*|${INSTANCE_PREFIX}.*" +GREP_REGEX="" +gcloud-compute-list forwarding-rules ${REGION:+"--region=${REGION}"} +gcloud-compute-list target-pools ${REGION:+"--region=${REGION}"} From 0024bfd1d5b1a3a52a9fd7048bfc0dd7fd667c44 Mon Sep 17 00:00:00 2001 From: Jeff Grafton Date: Tue, 6 Oct 2015 15:34:55 -0700 Subject: [PATCH 2/2] List and diff GCP resources before/after in Jenkins This currently won't fail runs if there is a diff. Additionally, clean up some unnecessary symlinking of JUnit test reports. --- hack/jenkins/e2e.sh | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/hack/jenkins/e2e.sh b/hack/jenkins/e2e.sh index 774896fbf2e..e96efd6f127 100755 --- a/hack/jenkins/e2e.sh +++ b/hack/jenkins/e2e.sh @@ -807,6 +807,9 @@ cd kubernetes ARTIFACTS=${WORKSPACE}/_artifacts mkdir -p ${ARTIFACTS} export E2E_REPORT_DIR=${ARTIFACTS} +declare -r gcp_resources_before="${ARTIFACTS}/gcp-resources-before.txt" +declare -r gcp_resources_cluster_up="${ARTIFACTS}/gcp-resources-cluster-up.txt" +declare -r gcp_resources_after="${ARTIFACTS}/gcp-resources-after.txt" ### Pre Set Up ### # Install gcloud from a custom path if provided. Used to test GKE with gcloud @@ -826,8 +829,14 @@ fi ### Set up ### if [[ "${E2E_UP,,}" == "true" ]]; then go run ./hack/e2e.go ${E2E_OPT} -v --down + if [[ ${KUBERNETES_PROVIDER} == "gce" || ${KUBERNETES_PROVIDER} == "gke" ]]; then + ./cluster/gce/list-resources.sh > "${gcp_resources_before}" + fi go run ./hack/e2e.go ${E2E_OPT} -v --up go run ./hack/e2e.go -v --ctl="version --match-server-version=false" + if [[ ${KUBERNETES_PROVIDER} == "gce" || ${KUBERNETES_PROVIDER} == "gke" ]]; then + ./cluster/gce/list-resources.sh > "${gcp_resources_cluster_up}" + fi fi ### Run tests ### @@ -862,14 +871,6 @@ if [[ "${USE_KUBEMARK:-}" == "true" ]]; then unset NUM_MINIONS_BKP fi -# TODO(zml): We have a bunch of legacy Jenkins configs that are -# expecting junit*.xml to be in ${WORKSPACE} root and it's Friday -# afternoon, so just put the junit report where it's expected. -# If link already exists, non-zero return code should not cause build to fail. -for junit in ${ARTIFACTS}/junit*.xml; do - ln -s -f ${junit} ${WORKSPACE} || true -done - ### Clean up ### if [[ "${E2E_DOWN,,}" == "true" ]]; then # Sleep before deleting the cluster to give the controller manager time to @@ -880,4 +881,11 @@ if [[ "${E2E_DOWN,,}" == "true" ]]; then # for the wait between attempts. sleep 30 go run ./hack/e2e.go ${E2E_OPT} -v --down + if [[ ${KUBERNETES_PROVIDER} == "gce" || ${KUBERNETES_PROVIDER} == "gke" ]]; then + ./cluster/gce/list-resources.sh > "${gcp_resources_after}" + fi +fi + +if [[ -f "${gcp_resources_before}" && -f "${gcp_resources_after}" ]]; then + diff -sw -U0 -F'^\[.*\]$' "${gcp_resources_before}" "${gcp_resources_after}" || true fi