2be0bd2569
add a script to enforce that all e2e tests have one and only one sig owner defined according to specific policies
224 lines
7.6 KiB
Bash
Executable File
224 lines
7.6 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
|
|
# Copyright 2014 The Kubernetes Authors.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
# This script verifies the following e2e test ownership policies
|
|
# - tests MUST start with [sig-foo]
|
|
# - tests MUST use top-level SIGDescribe
|
|
# - tests SHOULD NOT have multiple [sig-foo] tags
|
|
# - tests MUST NOT use nested SIGDescribe
|
|
# TODO: these two can be dropped if KubeDescribe is gone from codebase
|
|
# - tests MUST NOT have [k8s.io] in test names
|
|
# - tests MUST NOT use KubeDescribe
|
|
|
|
set -o errexit
|
|
set -o nounset
|
|
set -o pipefail
|
|
|
|
# This will canonicalize the path
|
|
KUBE_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. && pwd -P)
|
|
source "${KUBE_ROOT}/hack/lib/init.sh"
|
|
|
|
# Set REUSE_BUILD_OUTPUT=y to skip rebuilding dependencies if present
|
|
REUSE_BUILD_OUTPUT=${REUSE_BUILD_OUTPUT:-n}
|
|
# set VERBOSE_OUTPUT=y to output .jq files and shell commands
|
|
VERBOSE_OUTPUT=${VERBOSE_OUTPUT:-n}
|
|
|
|
if [[ ${VERBOSE_OUTPUT} =~ ^[yY]$ ]]; then
|
|
set -x
|
|
fi
|
|
|
|
pushd "${KUBE_ROOT}" > /dev/null
|
|
|
|
# Setup a tmpdir to hold generated scripts and results
|
|
readonly tmpdir=$(mktemp -d -t verify-e2e-test-ownership.XXXX)
|
|
trap 'rm -rf ${tmpdir}' EXIT
|
|
|
|
# input
|
|
spec_summaries="${KUBE_ROOT}/_output/specsummaries.json"
|
|
# output
|
|
results_json="${tmpdir}/results.json"
|
|
summary_json="${tmpdir}/summary.json"
|
|
failures_json="${tmpdir}/failures.json"
|
|
|
|
# rebuild dependencies if necessary
|
|
function ensure_dependencies() {
|
|
local -r ginkgo="${KUBE_ROOT}/_output/bin/ginkgo"
|
|
local -r e2e_test="${KUBE_ROOT}/_output/bin/e2e.test"
|
|
if ! { [ -f "${ginkgo}" ] && [[ "${REUSE_BUILD_OUTPUT}" =~ ^[yY]$ ]]; }; then
|
|
make ginkgo
|
|
fi
|
|
if ! { [ -f "${e2e_test}" ] && [[ "${REUSE_BUILD_OUTPUT}" =~ ^[yY]$ ]]; }; then
|
|
hack/make-rules/build.sh test/e2e/e2e.test
|
|
fi
|
|
if ! { [ -f "${spec_summaries}" ] && [[ "${REUSE_BUILD_OUTPUT}" =~ ^[yY]$ ]]; }; then
|
|
"${ginkgo}" --dryRun=true "${e2e_test}" -- --spec-dump "${spec_summaries}" > /dev/null
|
|
fi
|
|
}
|
|
|
|
# evaluate ginkgo spec summaries against e2e test ownership polices
|
|
# output to ${results_json}
|
|
function generate_results_json() {
|
|
readonly results_jq=${tmpdir}/results.jq
|
|
cat >"${results_jq}" <<EOS
|
|
[.[] | . as { ComponentTexts: \$text, ComponentCodeLocations: \$code } | {
|
|
calls: [ \$text | range(1;length) as \$i | {
|
|
sig: ((\$text[\$i] | match("\\\[(sig-[^\\\]]+)\\\]") | .captures[0].string) // "unknown"),
|
|
text: \$text[\$i],
|
|
# unused, but if we ever wanted to have policies based on other tags...
|
|
# tags: \$text[\$i] | [match("(\\\[[^\\\]]+\\\])"; "g").string],
|
|
line: \$code[\$i] | "\(.FileName):\(.LineNumber)",
|
|
} + (\$code[\$i] | .FullStackTrace | match("^(.*)\\\.(.+)\\\(.*\\n") | .captures | {
|
|
package: .[0].string,
|
|
func: .[1].string
|
|
})],
|
|
} | {
|
|
owner: .calls[0].sig,
|
|
testname: .calls | map(.text) | join(" "),
|
|
calls,
|
|
policies: [(
|
|
.calls[0] |
|
|
{
|
|
fail: (.sig == "unknown"),
|
|
level: "FAIL",
|
|
category: "unowned_test",
|
|
reason: "must start with [sig-foo]",
|
|
found: .,
|
|
}, {
|
|
fail: (.func != "SIGDescribe"),
|
|
level: "FAIL",
|
|
category: "no_sig_describe",
|
|
reason: "must use top-level SIGDescribe",
|
|
found: .,
|
|
}
|
|
), (
|
|
.calls[1:] |
|
|
(map(select(.sig != "unknown")) // [] | {
|
|
fail: . | any,
|
|
level: "WARN",
|
|
category: "too_many_sigs",
|
|
reason: "should not have multiple [sig-foo] tags",
|
|
found: .,
|
|
}),
|
|
(map(select(.func == "SIGDescribe")) // [] | {
|
|
fail: . | any,
|
|
level: "FAIL",
|
|
category: "nested_sig_describe",
|
|
reason: "must not use nested SIGDescribe",
|
|
found: .,
|
|
})
|
|
)
|
|
]
|
|
}]
|
|
EOS
|
|
if [[ ${VERBOSE_OUTPUT} =~ ^[yY]$ ]]; then
|
|
echo "about to ${results_jq}..."
|
|
cat -n "${results_jq}"
|
|
echo
|
|
fi
|
|
<"${spec_summaries}" jq --slurp --from-file "${results_jq}" > "${results_json}"
|
|
}
|
|
|
|
# summarize e2e test policy results
|
|
# output to ${summary_json}
|
|
function generate_summary_json() {
|
|
summary_jq=${tmpdir}/summary.jq
|
|
cat >"${summary_jq}" <<EOS
|
|
. as \$results |
|
|
# for each policy category
|
|
reduce \$results[0].policies[] as \$p ({}; . + {
|
|
# add a convenience .policy field containing that policy's result
|
|
(\$p.category): \$results | map(. + {policy: .policies[] | select(.category == \$p.category)}) | {
|
|
level: \$p.level,
|
|
reason: \$p.reason,
|
|
passing: map(select(.policy.fail | not)) | length,
|
|
failing: map(select(.policy.fail)) | length,
|
|
testnames: map(select(.policy.fail) | .testname),
|
|
}
|
|
})
|
|
# add a meta policy based on whether any policy failed
|
|
+ {
|
|
all_policies: \$results | {
|
|
level: "WARN",
|
|
reason: "should pass all policies",
|
|
passing: map(select(.policies | map(.fail) | any | not)) | length,
|
|
failing: map(select(.policies | map(.fail) | any)) | length,
|
|
testnames: map(select(.policies | map(.fail) | any) | .testname),
|
|
}
|
|
}
|
|
# if a policy has no failing tests, change its log output to PASS
|
|
| with_entries(.value += { log: (if (.value.failing == 0) then "PASS" else .value.level end) })
|
|
# sort by policies with the most failing tests first
|
|
| to_entries | sort_by(.value.failing) | reverse | from_entries
|
|
EOS
|
|
if [[ ${VERBOSE_OUTPUT} =~ ^[yY]$ ]]; then
|
|
echo "about to run ${results_jq}..."
|
|
cat -n "${summary_jq}"
|
|
echo
|
|
fi
|
|
<"${results_json}" jq --from-file "${summary_jq}" > "${summary_json}"
|
|
}
|
|
|
|
# filter e2e policy tests results to tests that failed, with the policies they failed
|
|
# output to ${failures_json}
|
|
function generate_failures_json() {
|
|
local -r failures_jq="${tmpdir}/failures.jq"
|
|
cat >"${failures_jq}" <<EOS
|
|
.
|
|
# for each test
|
|
| map(
|
|
# filter down to failing policies; trim category, .reason is more verbose
|
|
.policies |= map(select(.fail) | del(.category))
|
|
# trim the full callstack, .found will contain the relevant call
|
|
| del(.calls)
|
|
)
|
|
# filter down to tests that have failed policies
|
|
| map(select(.policies | map (.fail) | any))
|
|
EOS
|
|
if [[ ${VERBOSE_OUTPUT} =~ ^[yY]$ ]]; then
|
|
echo "about to run ${failures_jq}..."
|
|
cat -n "${failures_jq}"
|
|
echo
|
|
fi
|
|
<"${results_json}" jq --from-file "${failures_jq}" > "${failures_json}"
|
|
}
|
|
|
|
function output_results_and_exit_if_failed() {
|
|
local -r total_tests=$(<"${spec_summaries}" wc -l | awk '{print $1}')
|
|
|
|
# output results to console
|
|
(
|
|
echo "run at datetime: $(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
|
echo "based on commit: $(git log -n1 --date=iso-strict --pretty='%h - %cd - %s')"
|
|
echo
|
|
<"${failures_json}" cat
|
|
printf "%4s: e2e tests %-40s: %-4d\n" "INFO" "in total" "${total_tests}"
|
|
<"${summary_json}" jq -r 'to_entries[].value |
|
|
"printf \"%4s: ..failing %-40s: %-4d\\n\" \"\(.log)\" \"\(.reason)\" \"\(.failing)\""' | sh
|
|
) | tee "${tmpdir}/output.txt"
|
|
# if we said "FAIL" in that output, we should fail
|
|
if <"${tmpdir}/output.txt" grep -q "^FAIL"; then
|
|
echo "FAIL"
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
ensure_dependencies
|
|
generate_results_json
|
|
generate_failures_json
|
|
generate_summary_json
|
|
output_results_and_exit_if_failed
|
|
echo "PASS"
|