Use jq and only show failed tests on summary
Signed-off-by: Brian Goff <cpuguy83@gmail.com>
This commit is contained in:
Brian Goff
parent
34ad96babd
commit
9cdf9f6c6c
60
.github/workflows/ci.yml
vendored
60
.github/workflows/ci.yml
vendored
@ -328,24 +328,28 @@ jobs:
|
||||
env:
|
||||
CGO_ENABLED: 1
|
||||
GOTESTSUM_JUNITFILE: ${{github.workspace}}/test-unit-root.xml
|
||||
GOTESTSUM_JSONFILE: ${{github.workspace}}/test-unit-root.json
|
||||
GOTESTSUM_JSONFILE: ${{github.workspace}}/test-unit-root-gotest.json
|
||||
run: mingw32-make.exe test root-test
|
||||
- run: echo '# Root Test' >> $GITHUB_STEP_SUMMARY; teststat -markdown ${TESTFILE} >> $GITHUB_STEP_SUMMARY
|
||||
- run: if [ -f *-gotest.json ]; then echo '# Root Test' >> $GITHUB_STEP_SUMMARY; teststat -markdown *-gotest.json >> $GITHUB_STEP_SUMMARY; fi
|
||||
if: always()
|
||||
- run: script/test/test2annotation.sh ${TESTFILE}
|
||||
env:
|
||||
TESTFILE: ${{github.workspace}}/test-unit-root.json
|
||||
TESTFILE: ${{github.workspace}}/test-unit-root-gotest.json
|
||||
if: always()
|
||||
|
||||
- name: Integration 1
|
||||
env:
|
||||
CGO_ENABLED: 1
|
||||
ENABLE_CRI_SANDBOXES: ${{ matrix.enable_cri_sandboxes }}
|
||||
GOTESTSUM_JUNITFILE: ${{github.workspace}}/test-integration-serial-junit.xml
|
||||
GOTESTSUM_JSONFILE: ${{github.workspace}}/test-integration-serial.json
|
||||
GOTESTSUM_JSONFILE: ${{github.workspace}}/test-integration-serial-gotest.json
|
||||
run: mingw32-make.exe integration
|
||||
- run: echo '# Integration 1' >> $GITHUB_STEP_SUMMARY; teststat -markdown ${TESTFILE} >> $GITHUB_STEP_SUMMARY
|
||||
- run: if [ -f *-gotest.json ]; then echo '# Integration 1' >> $GITHUB_STEP_SUMMARY; teststat -markdown *-gotest.json >> $GITHUB_STEP_SUMMARY; fi
|
||||
if: always()
|
||||
- run: script/test/test2annotation.sh ${TESTFILE}
|
||||
env:
|
||||
TESTFILE: ${{github.workspace}}/test-integration-serial.json
|
||||
TESTFILE: ${{github.workspace}}/test-integration-serial-gotest.json
|
||||
if: always()
|
||||
|
||||
# Run the integration suite a second time. See discussion in github.com/containerd/containerd/pull/1759
|
||||
- name: Integration 2
|
||||
@ -355,12 +359,14 @@ jobs:
|
||||
CGO_ENABLED: 1
|
||||
ENABLE_CRI_SANDBOXES: ${{ matrix.enable_cri_sandboxes }}
|
||||
GOTESTSUM_JUNITFILE: ${{github.workspace}}/test-integration-parallel-junit.xml
|
||||
GOTESTSUM_JSONFILE: ${{github.workspace}}/test-integration-parallel.json
|
||||
GOTESTSUM_JSONFILE: ${{github.workspace}}/test-integration-parallel-gotest.json
|
||||
run: mingw32-make.exe integration
|
||||
- run: echo '# Integration 2' >> $GITHUB_STEP_SUMMARY; teststat -markdown ${TESTFILE} >> $GITHUB_STEP_SUMMARY
|
||||
- run: if [ -f *-gotest.json ]; then echo '# Integration 2' >> $GITHUB_STEP_SUMMARY; teststat -markdown *-gotest.json >> $GITHUB_STEP_SUMMARY; fi
|
||||
if: always()
|
||||
- run: script/test/test2annotation.sh ${TESTFILE}
|
||||
env:
|
||||
TESTFILE: ${{github.workspace}}/test-integration-parallel.json
|
||||
TESTFILE: ${{github.workspace}}/test-integration-parallel-gotest.json
|
||||
if: always()
|
||||
|
||||
- name: CRI Integration Test
|
||||
env:
|
||||
@ -392,6 +398,7 @@ jobs:
|
||||
name: TestResults ${{ matrix.os }}
|
||||
path: |
|
||||
${{github.workspace}}/*-junit.xml
|
||||
${{github.workspace}}/*-gotest.json
|
||||
|
||||
integration-linux:
|
||||
name: Linux Integration
|
||||
@ -454,14 +461,14 @@ jobs:
|
||||
- name: Tests
|
||||
env:
|
||||
GOTESTSUM_JUNITFILE: ${{github.workspace}}/test-unit-root-junit.xml
|
||||
GOTESTSUM_JSONFILE: ${{github.workspace}}/test-unit-root.json
|
||||
GOTESTSUM_JSONFILE: ${{github.workspace}}/test-unit-root-gotest.json
|
||||
run: |
|
||||
make test
|
||||
sudo -E PATH=$PATH make root-test
|
||||
- run: echo '# Root Test' >> $GITHUB_STEP_SUMMARY; teststat -markdown ${TESTFILE} >> $GITHUB_STEP_SUMMARY
|
||||
- run: if [ -f *-gotest.json ]; then echo '# Root Test' >> $GITHUB_STEP_SUMMARY; teststat -markdown *-gotest.json >> $GITHUB_STEP_SUMMARY; fi
|
||||
if: always()
|
||||
- run: script/test/test2annotation.sh *-gotest.json
|
||||
if: always()
|
||||
env:
|
||||
TESTFILE: ${{github.workspace}}/test-unit-root.json
|
||||
|
||||
- name: Integration 1
|
||||
env:
|
||||
@ -469,17 +476,17 @@ jobs:
|
||||
RUNC_FLAVOR: ${{ matrix.runc }}
|
||||
ENABLE_CRI_SANDBOXES: ${{ matrix.enable_cri_sandboxes }}
|
||||
GOTESTSUM_JUNITFILE: ${{github.workspace}}/test-integration-serial-junit.xml
|
||||
GOTESTSUM_JSONFILE: ${{github.workspace}}/test-integration-serial.json
|
||||
GOTESTSUM_JSONFILE: ${{github.workspace}}/test-integration-serial-gotest.json
|
||||
run: |
|
||||
extraflags=""
|
||||
[ "${RUNC_FLAVOR}" == "crun" ] && {
|
||||
extraflags="EXTRA_TESTFLAGS=-no-criu";
|
||||
}
|
||||
sudo -E PATH=$PATH make integration ${extraflags} TESTFLAGS_RACE=-race
|
||||
- run: echo '# Integration 1' >> $GITHUB_STEP_SUMMARY; teststat -markdown ${TESTFILE} >> $GITHUB_STEP_SUMMARY
|
||||
- run: if [ -f *-gotest.json ]; then echo '# Integration 1' >> $GITHUB_STEP_SUMMARY; teststat -markdown *-gotest.json >> $GITHUB_STEP_SUMMARY; fi
|
||||
if: always()
|
||||
- run: script/test/test2annotation.sh *-gotest.json
|
||||
if: always()
|
||||
env:
|
||||
TESTFILE: ${{github.workspace}}/test-integration-serial.json
|
||||
|
||||
# Run the integration suite a second time. See discussion in github.com/containerd/containerd/pull/1759
|
||||
- name: Integration 2
|
||||
@ -488,17 +495,17 @@ jobs:
|
||||
RUNC_FLAVOR: ${{ matrix.runc }}
|
||||
ENABLE_CRI_SANDBOXES: ${{ matrix.enable_cri_sandboxes }}
|
||||
GOTESTSUM_JUNITFILE: ${{github.workspace}}/test-integration-parallel-junit.xml
|
||||
GOTESTSUM_JSONFILE: ${{github.workspace}}/test-integration-parallel.json
|
||||
GOTESTSUM_JSONFILE: ${{github.workspace}}/test-integration-parallel-gotest.json
|
||||
run: |
|
||||
extraflags=""
|
||||
[ "${RUNC_FLAVOR}" == "crun" ] && {
|
||||
extraflags="EXTRA_TESTFLAGS=-no-criu";
|
||||
}
|
||||
sudo -E PATH=$PATH TESTFLAGS_PARALLEL=1 make integration ${extraflags}
|
||||
- run: echo '# Integration 2' >> $GITHUB_STEP_SUMMARY; teststat -markdown ${TESTFILE} >> $GITHUB_STEP_SUMMARY
|
||||
- run: if [ -f *-gotest.json ]; then echo '# Integration 2' >> $GITHUB_STEP_SUMMARY; teststat -markdown *-gotest.json >> $GITHUB_STEP_SUMMARY; fi
|
||||
if: always()
|
||||
- run: script/test/test2annotation.sh *-gotest.json
|
||||
if: always()
|
||||
env:
|
||||
TESTFILE: ${{github.workspace}}/test-integration-parallel.json
|
||||
|
||||
- name: CRI Integration Test
|
||||
env:
|
||||
@ -547,6 +554,7 @@ jobs:
|
||||
name: TestResults ${{ matrix.runtime }} ${{matrix.runc}}
|
||||
path: |
|
||||
*-junit.xml
|
||||
*-gotest.json
|
||||
${{github.workspace}}/critestreport/*.xml
|
||||
|
||||
tests-mac-os:
|
||||
@ -563,18 +571,20 @@ jobs:
|
||||
go-version: ${{ env.GO_VERSION }}
|
||||
- uses: actions/checkout@v2
|
||||
- run: script/setup/install-gotestsum
|
||||
- run: script/setup/install-teststat
|
||||
- name: Tests
|
||||
env:
|
||||
GOTESTSUM_JUNITFILE: "${{ github.workspace }}/macos-test-junit.xml"
|
||||
GOTESTSUM_JSONFILE: "${{ github.workspace }}/macos-test.json"
|
||||
GOTESTSUM_JSONFILE: "${{ github.workspace }}/macos-test-gotest.json"
|
||||
run: make test
|
||||
- run: script/setup/install-teststat && teststat -markdown ${TESTFILE} >> $GITHUB_STEP_SUMMARY
|
||||
- run: if [ -f *-gotest.json ]; then echo '# Unit Tests' >> $GITHUB_STEP_SUMMARY; teststat -markdown *-gotest.json >> $GITHUB_STEP_SUMMARY; fi
|
||||
if: always()
|
||||
- run: script/test/test2annotation.sh *-gotest.json
|
||||
if: always()
|
||||
env:
|
||||
TESTFILE: "${{ github.workspace }}/macos-test.json"
|
||||
- uses: actions/upload-artifact@v2
|
||||
if: always()
|
||||
with:
|
||||
name: TestResults MacOS
|
||||
path: |
|
||||
*-junit.xml
|
||||
*-gotest.json
|
||||
|
||||
97
script/test/test2annotation.jq
Executable file
97
script/test/test2annotation.jq
Executable file
@ -0,0 +1,97 @@
|
||||
#!/usr/bin/env -S jq -s -r -f
|
||||
|
||||
# Copyright The containerd Authors.
|
||||
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# Takes the raw test2json as input and groups each item by package name + test name
|
||||
#
|
||||
# Example
|
||||
# Input:
|
||||
# {...}
|
||||
# {...}
|
||||
# {...}
|
||||
# Output:
|
||||
# [[{...},{...},{...}]]
|
||||
def group_tests: reduce (.[] | select(.Test != null)) as $item (
|
||||
{}; .["\($item.Package).\($item.Test)"] += [$item]
|
||||
);
|
||||
|
||||
# Filter a group of tests by action.
|
||||
# The way this works is for a group of tests,
|
||||
# if none of the tests have an action matching the provided value, the group is
|
||||
# dropped.
|
||||
def filter_group_action(a): map(select(.[].Action == a));
|
||||
|
||||
# Filter a group of tests to just the tests that failed
|
||||
def only_failed: filter_group_action("fail");
|
||||
|
||||
def merge_strings(s1; s2): if s1 == null then s2 else "\(s1)\(s2)" end;
|
||||
|
||||
# Matches an output string to find the test file and line number.
|
||||
# The actual string looks something like:
|
||||
# "test_foo.go:12: some failure message\n"
|
||||
# There may be arbitrary whitespace at the beginning of the string.
|
||||
def capture_fail_details: capture("^\\s*(?<file>\\w+_test\\.go):(?<line>\\d+): "; "x");
|
||||
|
||||
# Filter out all the undesirable test line outputs
|
||||
def filter_test_item:
|
||||
select(.Output != null)
|
||||
| select(.Output | test("^\\s*=== RUN")| not)
|
||||
| select(.Output | test("^\\s*=== PAUSE")| not)
|
||||
| select(.Output | test("^\\s*=== CONT")| not)
|
||||
| select(.Output | test("^\\s*--- FAIL:")| not)
|
||||
;
|
||||
|
||||
# Take a list of test groups and make a map of tests keyed on the package name +
|
||||
# test name with all the output concatenated.
|
||||
#
|
||||
# This uses the last test log message to get the the file/line number for the test (as .Details)
|
||||
def merge_output: reduce (.[][] | filter_test_item) as $item (
|
||||
{}; (
|
||||
"\($item.Package).\($item.Test)" as $key
|
||||
| merge_strings(.[$key].Output; $item.Output) as $merged
|
||||
| .[$key] += {
|
||||
Output: $merged | sub("^\\s*"; ""; "x"),
|
||||
Test: $item.Test,
|
||||
Package: $item.Package,
|
||||
Details: (($item.Output | capture_fail_details) // .[$key].Details), # "//" is an operator that returns the first true/non-null value
|
||||
}
|
||||
) // .
|
||||
);
|
||||
|
||||
# Used as the "title" field for error annotations
|
||||
def err_title: "Failed: \(.Package).\(.Test)";
|
||||
|
||||
# Split the containerd root package path to get the directory name that a test belongs to.
|
||||
def file_dir: .Package | split("github.com/containerd/containerd/")[-1];
|
||||
|
||||
def err_file: if .Details != null and .Details.file != null then "\(file_dir)/\(.Details.file)" else "" end;
|
||||
|
||||
# Some cases there may not be any extra details because they got filtered out.
|
||||
# This can happen, for instance, if the failure log was create from a file that is not a `_test.go` file.
|
||||
# In this case we'd still want the annotation but we just can't get the file/line number (without allowing non-test files to be the source of the failure).
|
||||
def details_to_string: if .Details != null and .Details.file != null then ",file=\(err_file),line=\(.Details.line)" else "" end;
|
||||
|
||||
# Replace all occurrences of "\n" with the url-encoded version
|
||||
# This allows multi-line strings to work with github annotations.
|
||||
# https://github.com/actions/toolkit/issues/193#issuecomment-605394935
|
||||
def encode_output: gsub("\n"; "%0A");
|
||||
|
||||
# Creates github actions error annotations for each input
|
||||
# It is expected that the input is the output of merge_output.
|
||||
def to_error: reduce .[] as $item (
|
||||
""; . += "::error title=\($item | err_title)\($item | details_to_string)::\($item.Output | encode_output)\n"
|
||||
);
|
||||
|
||||
group_tests | only_failed | merge_output | to_error
|
||||
19
script/test/test2annotation.sh
Executable file
19
script/test/test2annotation.sh
Executable file
@ -0,0 +1,19 @@
|
||||
#!/usr/bin/env sh
|
||||
|
||||
# Copyright The containerd Authors.
|
||||
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
f=$(dirname $0)/test2annotation.jq
|
||||
|
||||
exec jq -f "${f}" -s -r $@
|
||||
Loading…
Reference in New Issue
Block a user