From 9cdf9f6c6cdaae7e91cec71c3a27a7af97573e88 Mon Sep 17 00:00:00 2001 From: Brian Goff Date: Fri, 22 Jul 2022 20:44:52 +0000 Subject: [PATCH] Use jq and only show failed tests on summary Signed-off-by: Brian Goff --- .github/workflows/ci.yml | 60 ++++++++++++--------- script/test/test2annotation.jq | 97 ++++++++++++++++++++++++++++++++++ script/test/test2annotation.sh | 19 +++++++ 3 files changed, 151 insertions(+), 25 deletions(-) create mode 100755 script/test/test2annotation.jq create mode 100755 script/test/test2annotation.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4247d2de6..3bd6553ef 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -328,24 +328,28 @@ jobs: env: CGO_ENABLED: 1 GOTESTSUM_JUNITFILE: ${{github.workspace}}/test-unit-root.xml - GOTESTSUM_JSONFILE: ${{github.workspace}}/test-unit-root.json + GOTESTSUM_JSONFILE: ${{github.workspace}}/test-unit-root-gotest.json run: mingw32-make.exe test root-test - - run: echo '# Root Test' >> $GITHUB_STEP_SUMMARY; teststat -markdown ${TESTFILE} >> $GITHUB_STEP_SUMMARY + - run: if [ -f *-gotest.json ]; then echo '# Root Test' >> $GITHUB_STEP_SUMMARY; teststat -markdown *-gotest.json >> $GITHUB_STEP_SUMMARY; fi if: always() + - run: script/test/test2annotation.sh ${TESTFILE} env: - TESTFILE: ${{github.workspace}}/test-unit-root.json + TESTFILE: ${{github.workspace}}/test-unit-root-gotest.json + if: always() - name: Integration 1 env: CGO_ENABLED: 1 ENABLE_CRI_SANDBOXES: ${{ matrix.enable_cri_sandboxes }} GOTESTSUM_JUNITFILE: ${{github.workspace}}/test-integration-serial-junit.xml - GOTESTSUM_JSONFILE: ${{github.workspace}}/test-integration-serial.json + GOTESTSUM_JSONFILE: ${{github.workspace}}/test-integration-serial-gotest.json run: mingw32-make.exe integration - - run: echo '# Integration 1' >> $GITHUB_STEP_SUMMARY; teststat -markdown ${TESTFILE} >> $GITHUB_STEP_SUMMARY + - run: if [ -f *-gotest.json ]; then echo '# Integration 1' >> $GITHUB_STEP_SUMMARY; teststat -markdown *-gotest.json >> $GITHUB_STEP_SUMMARY; fi if: always() + - run: script/test/test2annotation.sh ${TESTFILE} env: - TESTFILE: ${{github.workspace}}/test-integration-serial.json + TESTFILE: ${{github.workspace}}/test-integration-serial-gotest.json + if: always() # Run the integration suite a second time. See discussion in github.com/containerd/containerd/pull/1759 - name: Integration 2 @@ -355,12 +359,14 @@ jobs: CGO_ENABLED: 1 ENABLE_CRI_SANDBOXES: ${{ matrix.enable_cri_sandboxes }} GOTESTSUM_JUNITFILE: ${{github.workspace}}/test-integration-parallel-junit.xml - GOTESTSUM_JSONFILE: ${{github.workspace}}/test-integration-parallel.json + GOTESTSUM_JSONFILE: ${{github.workspace}}/test-integration-parallel-gotest.json run: mingw32-make.exe integration - - run: echo '# Integration 2' >> $GITHUB_STEP_SUMMARY; teststat -markdown ${TESTFILE} >> $GITHUB_STEP_SUMMARY + - run: if [ -f *-gotest.json ]; then echo '# Integration 2' >> $GITHUB_STEP_SUMMARY; teststat -markdown *-gotest.json >> $GITHUB_STEP_SUMMARY; fi if: always() + - run: script/test/test2annotation.sh ${TESTFILE} env: - TESTFILE: ${{github.workspace}}/test-integration-parallel.json + TESTFILE: ${{github.workspace}}/test-integration-parallel-gotest.json + if: always() - name: CRI Integration Test env: @@ -392,6 +398,7 @@ jobs: name: TestResults ${{ matrix.os }} path: | ${{github.workspace}}/*-junit.xml + ${{github.workspace}}/*-gotest.json integration-linux: name: Linux Integration @@ -454,14 +461,14 @@ jobs: - name: Tests env: GOTESTSUM_JUNITFILE: ${{github.workspace}}/test-unit-root-junit.xml - GOTESTSUM_JSONFILE: ${{github.workspace}}/test-unit-root.json + GOTESTSUM_JSONFILE: ${{github.workspace}}/test-unit-root-gotest.json run: | make test sudo -E PATH=$PATH make root-test - - run: echo '# Root Test' >> $GITHUB_STEP_SUMMARY; teststat -markdown ${TESTFILE} >> $GITHUB_STEP_SUMMARY + - run: if [ -f *-gotest.json ]; then echo '# Root Test' >> $GITHUB_STEP_SUMMARY; teststat -markdown *-gotest.json >> $GITHUB_STEP_SUMMARY; fi + if: always() + - run: script/test/test2annotation.sh *-gotest.json if: always() - env: - TESTFILE: ${{github.workspace}}/test-unit-root.json - name: Integration 1 env: @@ -469,17 +476,17 @@ jobs: RUNC_FLAVOR: ${{ matrix.runc }} ENABLE_CRI_SANDBOXES: ${{ matrix.enable_cri_sandboxes }} GOTESTSUM_JUNITFILE: ${{github.workspace}}/test-integration-serial-junit.xml - GOTESTSUM_JSONFILE: ${{github.workspace}}/test-integration-serial.json + GOTESTSUM_JSONFILE: ${{github.workspace}}/test-integration-serial-gotest.json run: | extraflags="" [ "${RUNC_FLAVOR}" == "crun" ] && { extraflags="EXTRA_TESTFLAGS=-no-criu"; } sudo -E PATH=$PATH make integration ${extraflags} TESTFLAGS_RACE=-race - - run: echo '# Integration 1' >> $GITHUB_STEP_SUMMARY; teststat -markdown ${TESTFILE} >> $GITHUB_STEP_SUMMARY + - run: if [ -f *-gotest.json ]; then echo '# Integration 1' >> $GITHUB_STEP_SUMMARY; teststat -markdown *-gotest.json >> $GITHUB_STEP_SUMMARY; fi + if: always() + - run: script/test/test2annotation.sh *-gotest.json if: always() - env: - TESTFILE: ${{github.workspace}}/test-integration-serial.json # Run the integration suite a second time. See discussion in github.com/containerd/containerd/pull/1759 - name: Integration 2 @@ -488,17 +495,17 @@ jobs: RUNC_FLAVOR: ${{ matrix.runc }} ENABLE_CRI_SANDBOXES: ${{ matrix.enable_cri_sandboxes }} GOTESTSUM_JUNITFILE: ${{github.workspace}}/test-integration-parallel-junit.xml - GOTESTSUM_JSONFILE: ${{github.workspace}}/test-integration-parallel.json + GOTESTSUM_JSONFILE: ${{github.workspace}}/test-integration-parallel-gotest.json run: | extraflags="" [ "${RUNC_FLAVOR}" == "crun" ] && { extraflags="EXTRA_TESTFLAGS=-no-criu"; } sudo -E PATH=$PATH TESTFLAGS_PARALLEL=1 make integration ${extraflags} - - run: echo '# Integration 2' >> $GITHUB_STEP_SUMMARY; teststat -markdown ${TESTFILE} >> $GITHUB_STEP_SUMMARY + - run: if [ -f *-gotest.json ]; then echo '# Integration 2' >> $GITHUB_STEP_SUMMARY; teststat -markdown *-gotest.json >> $GITHUB_STEP_SUMMARY; fi + if: always() + - run: script/test/test2annotation.sh *-gotest.json if: always() - env: - TESTFILE: ${{github.workspace}}/test-integration-parallel.json - name: CRI Integration Test env: @@ -547,6 +554,7 @@ jobs: name: TestResults ${{ matrix.runtime }} ${{matrix.runc}} path: | *-junit.xml + *-gotest.json ${{github.workspace}}/critestreport/*.xml tests-mac-os: @@ -563,18 +571,20 @@ jobs: go-version: ${{ env.GO_VERSION }} - uses: actions/checkout@v2 - run: script/setup/install-gotestsum + - run: script/setup/install-teststat - name: Tests env: GOTESTSUM_JUNITFILE: "${{ github.workspace }}/macos-test-junit.xml" - GOTESTSUM_JSONFILE: "${{ github.workspace }}/macos-test.json" + GOTESTSUM_JSONFILE: "${{ github.workspace }}/macos-test-gotest.json" run: make test - - run: script/setup/install-teststat && teststat -markdown ${TESTFILE} >> $GITHUB_STEP_SUMMARY + - run: if [ -f *-gotest.json ]; then echo '# Unit Tests' >> $GITHUB_STEP_SUMMARY; teststat -markdown *-gotest.json >> $GITHUB_STEP_SUMMARY; fi + if: always() + - run: script/test/test2annotation.sh *-gotest.json if: always() - env: - TESTFILE: "${{ github.workspace }}/macos-test.json" - uses: actions/upload-artifact@v2 if: always() with: name: TestResults MacOS path: | *-junit.xml + *-gotest.json diff --git a/script/test/test2annotation.jq b/script/test/test2annotation.jq new file mode 100755 index 000000000..aaf0fe4ad --- /dev/null +++ b/script/test/test2annotation.jq @@ -0,0 +1,97 @@ +#!/usr/bin/env -S jq -s -r -f + +# Copyright The containerd Authors. + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Takes the raw test2json as input and groups each item by package name + test name +# +# Example +# Input: +# {...} +# {...} +# {...} +# Output: +# [[{...},{...},{...}]] +def group_tests: reduce (.[] | select(.Test != null)) as $item ( + {}; .["\($item.Package).\($item.Test)"] += [$item] +); + +# Filter a group of tests by action. +# The way this works is for a group of tests, +# if none of the tests have an action matching the provided value, the group is +# dropped. +def filter_group_action(a): map(select(.[].Action == a)); + +# Filter a group of tests to just the tests that failed +def only_failed: filter_group_action("fail"); + +def merge_strings(s1; s2): if s1 == null then s2 else "\(s1)\(s2)" end; + +# Matches an output string to find the test file and line number. +# The actual string looks something like: +# "test_foo.go:12: some failure message\n" +# There may be arbitrary whitespace at the beginning of the string. +def capture_fail_details: capture("^\\s*(?\\w+_test\\.go):(?\\d+): "; "x"); + +# Filter out all the undesirable test line outputs +def filter_test_item: + select(.Output != null) + | select(.Output | test("^\\s*=== RUN")| not) + | select(.Output | test("^\\s*=== PAUSE")| not) + | select(.Output | test("^\\s*=== CONT")| not) + | select(.Output | test("^\\s*--- FAIL:")| not) +; + +# Take a list of test groups and make a map of tests keyed on the package name + +# test name with all the output concatenated. +# +# This uses the last test log message to get the the file/line number for the test (as .Details) +def merge_output: reduce (.[][] | filter_test_item) as $item ( + {}; ( + "\($item.Package).\($item.Test)" as $key + | merge_strings(.[$key].Output; $item.Output) as $merged + | .[$key] += { + Output: $merged | sub("^\\s*"; ""; "x"), + Test: $item.Test, + Package: $item.Package, + Details: (($item.Output | capture_fail_details) // .[$key].Details), # "//" is an operator that returns the first true/non-null value + } + ) // . +); + +# Used as the "title" field for error annotations +def err_title: "Failed: \(.Package).\(.Test)"; + +# Split the containerd root package path to get the directory name that a test belongs to. +def file_dir: .Package | split("github.com/containerd/containerd/")[-1]; + +def err_file: if .Details != null and .Details.file != null then "\(file_dir)/\(.Details.file)" else "" end; + +# Some cases there may not be any extra details because they got filtered out. +# This can happen, for instance, if the failure log was create from a file that is not a `_test.go` file. +# In this case we'd still want the annotation but we just can't get the file/line number (without allowing non-test files to be the source of the failure). +def details_to_string: if .Details != null and .Details.file != null then ",file=\(err_file),line=\(.Details.line)" else "" end; + +# Replace all occurrences of "\n" with the url-encoded version +# This allows multi-line strings to work with github annotations. +# https://github.com/actions/toolkit/issues/193#issuecomment-605394935 +def encode_output: gsub("\n"; "%0A"); + +# Creates github actions error annotations for each input +# It is expected that the input is the output of merge_output. +def to_error: reduce .[] as $item ( + ""; . += "::error title=\($item | err_title)\($item | details_to_string)::\($item.Output | encode_output)\n" +); + +group_tests | only_failed | merge_output | to_error \ No newline at end of file diff --git a/script/test/test2annotation.sh b/script/test/test2annotation.sh new file mode 100755 index 000000000..36c0a18aa --- /dev/null +++ b/script/test/test2annotation.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env sh + +# Copyright The containerd Authors. + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +f=$(dirname $0)/test2annotation.jq + +exec jq -f "${f}" -s -r $@