github/containerd
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 2 Wiki Activity

Merge pull request #7192 from cpuguy83/test_summary

Browse Source
This commit is contained in:
Samuel Karp 2022-09-26 15:28:33 -07:00 committed by GitHub
commit 34d078e99f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 201 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

82
.github/workflows/ci.yml vendored
View File

@ -3,16 +3,16 @@ on:
push:
branches:
- main
- 'release/**'
- "release/**"
pull_request:
branches:
- main
- 'release/**'
- "release/**"
env:
# Go version we currently use to build containerd across all CI.
# Note: don't forget to update `Binaries` step, as it contains the matrix of all supported Go versions.
GO_VERSION: '1.19.1'
GO_VERSION: "1.19.1"
jobs:
#
@ -219,7 +219,7 @@ jobs:
strategy:
matrix:
os: [ubuntu-18.04, macos-12, windows-2019, windows-2022]
go-version: ['1.19.1', '1.18.6']
go-version: ["1.19.1", "1.18.6"]
steps:
- uses: actions/setup-go@v2
with:
@ -256,7 +256,7 @@ jobs:
fail-fast: false
matrix:
os: [windows-2019, windows-2022]
enable_cri_sandboxes: [ "", "sandboxed"]
enable_cri_sandboxes: ["", "sandboxed"]
defaults:
run:
@ -304,14 +304,14 @@ jobs:
RESOURCE_CONSUMER_TESTING_IMAGE_REF: "registry.k8s.io/e2e-test-images/resource-consumer:1.10"
WEBSERVER_TESTING_IMAGE_REF: "registry.k8s.io/e2e-test-images/nginx:1.14-2"
run: |
cat > "${{ env.TEST_IMAGE_LIST }}" << EOF
busybox = "${{ env.BUSYBOX_TESTING_IMAGE_REF }}"
ResourceConsumer = "${{ env.RESOURCE_CONSUMER_TESTING_IMAGE_REF }}"
EOF
cat > "${{ env.CRI_TEST_IMAGES }}" << EOF
defaultTestContainerImage: ${{ env.BUSYBOX_TESTING_IMAGE_REF }}
webServerTestImage: ${{ env.WEBSERVER_TESTING_IMAGE_REF }}
EOF
cat > "${{ env.TEST_IMAGE_LIST }}" << EOF
busybox = "${{ env.BUSYBOX_TESTING_IMAGE_REF }}"
ResourceConsumer = "${{ env.RESOURCE_CONSUMER_TESTING_IMAGE_REF }}"
EOF
cat > "${{ env.CRI_TEST_IMAGES }}" << EOF
defaultTestContainerImage: ${{ env.BUSYBOX_TESTING_IMAGE_REF }}
webServerTestImage: ${{ env.WEBSERVER_TESTING_IMAGE_REF }}
EOF
- name: Get crictl tool
shell: powershell
@ -323,18 +323,33 @@ jobs:
mv crictl.exe "${{ github.workspace }}/bin/crictl.exe" # Move crictl somewhere in path
- run: script/setup/install-gotestsum
- run: script/setup/install-teststat
- name: Tests
env:
CGO_ENABLED: 1
GOTESTSUM_JUNITFILE: ${{github.workspace}}/test-unit-root.xml
GOTESTSUM_JSONFILE: ${{github.workspace}}/test-unit-root-gotest.json
run: mingw32-make.exe test root-test
- run: if [ -f *-gotest.json ]; then echo '# Root Test' >> $GITHUB_STEP_SUMMARY; teststat -markdown *-gotest.json >> $GITHUB_STEP_SUMMARY; fi
if: always()
- run: script/test/test2annotation.sh ${TESTFILE}
env:
TESTFILE: ${{github.workspace}}/test-unit-root-gotest.json
if: always()
- name: Integration 1
env:
CGO_ENABLED: 1
ENABLE_CRI_SANDBOXES: ${{ matrix.enable_cri_sandboxes }}
GOTESTSUM_JUNITFILE: ${{github.workspace}}/test-integration-serial-junit.xml
GOTESTSUM_JSONFILE: ${{github.workspace}}/test-integration-serial-gotest.json
run: mingw32-make.exe integration
- run: if [ -f *-gotest.json ]; then echo '# Integration 1' >> $GITHUB_STEP_SUMMARY; teststat -markdown *-gotest.json >> $GITHUB_STEP_SUMMARY; fi
if: always()
- run: script/test/test2annotation.sh ${TESTFILE}
env:
TESTFILE: ${{github.workspace}}/test-integration-serial-gotest.json
if: always()
# Run the integration suite a second time. See discussion in github.com/containerd/containerd/pull/1759
- name: Integration 2
@ -344,7 +359,14 @@ jobs:
CGO_ENABLED: 1
ENABLE_CRI_SANDBOXES: ${{ matrix.enable_cri_sandboxes }}
GOTESTSUM_JUNITFILE: ${{github.workspace}}/test-integration-parallel-junit.xml
GOTESTSUM_JSONFILE: ${{github.workspace}}/test-integration-parallel-gotest.json
run: mingw32-make.exe integration
- run: if [ -f *-gotest.json ]; then echo '# Integration 2' >> $GITHUB_STEP_SUMMARY; teststat -markdown *-gotest.json >> $GITHUB_STEP_SUMMARY; fi
if: always()
- run: script/test/test2annotation.sh ${TESTFILE}
env:
TESTFILE: ${{github.workspace}}/test-integration-parallel-gotest.json
if: always()
- name: CRI Integration Test
env:
@ -376,6 +398,7 @@ jobs:
name: TestResults ${{ matrix.os }}
path: |
${{github.workspace}}/*-junit.xml
${{github.workspace}}/*-gotest.json
integration-linux:
name: Linux Integration
@ -386,9 +409,14 @@ jobs:
strategy:
fail-fast: false
matrix:
runtime: [io.containerd.runtime.v1.linux, io.containerd.runc.v1, io.containerd.runc.v2]
runtime:
[
io.containerd.runtime.v1.linux,
io.containerd.runc.v1,
io.containerd.runc.v2,
]
runc: [runc, crun]
enable_cri_sandboxes: [ "", "sandboxed"]
enable_cri_sandboxes: ["", "sandboxed"]
exclude:
- runtime: io.containerd.runc.v1
runc: crun
@ -429,12 +457,18 @@ jobs:
sudo -E PATH=$PATH make install
- run: script/setup/install-gotestsum
- run: script/setup/install-teststat
- name: Tests
env:
GOTESTSUM_JUNITFILE: ${{github.workspace}}/test-unit-root-junit.xml
GOTESTSUM_JSONFILE: ${{github.workspace}}/test-unit-root-gotest.json
run: |
make test
sudo -E PATH=$PATH make root-test
- run: if [ -f *-gotest.json ]; then echo '# Root Test' >> $GITHUB_STEP_SUMMARY; teststat -markdown *-gotest.json >> $GITHUB_STEP_SUMMARY; fi
if: always()
- run: script/test/test2annotation.sh *-gotest.json
if: always()
- name: Integration 1
env:
@ -442,12 +476,17 @@ jobs:
RUNC_FLAVOR: ${{ matrix.runc }}
ENABLE_CRI_SANDBOXES: ${{ matrix.enable_cri_sandboxes }}
GOTESTSUM_JUNITFILE: ${{github.workspace}}/test-integration-serial-junit.xml
GOTESTSUM_JSONFILE: ${{github.workspace}}/test-integration-serial-gotest.json
run: |
extraflags=""
[ "${RUNC_FLAVOR}" == "crun" ] && {
extraflags="EXTRA_TESTFLAGS=-no-criu";
}
sudo -E PATH=$PATH make integration ${extraflags} TESTFLAGS_RACE=-race
- run: if [ -f *-gotest.json ]; then echo '# Integration 1' >> $GITHUB_STEP_SUMMARY; teststat -markdown *-gotest.json >> $GITHUB_STEP_SUMMARY; fi
if: always()
- run: script/test/test2annotation.sh *-gotest.json
if: always()
# Run the integration suite a second time. See discussion in github.com/containerd/containerd/pull/1759
- name: Integration 2
@ -456,12 +495,17 @@ jobs:
RUNC_FLAVOR: ${{ matrix.runc }}
ENABLE_CRI_SANDBOXES: ${{ matrix.enable_cri_sandboxes }}
GOTESTSUM_JUNITFILE: ${{github.workspace}}/test-integration-parallel-junit.xml
GOTESTSUM_JSONFILE: ${{github.workspace}}/test-integration-parallel-gotest.json
run: |
extraflags=""
[ "${RUNC_FLAVOR}" == "crun" ] && {
extraflags="EXTRA_TESTFLAGS=-no-criu";
}
sudo -E PATH=$PATH TESTFLAGS_PARALLEL=1 make integration ${extraflags}
- run: if [ -f *-gotest.json ]; then echo '# Integration 2' >> $GITHUB_STEP_SUMMARY; teststat -markdown *-gotest.json >> $GITHUB_STEP_SUMMARY; fi
if: always()
- run: script/test/test2annotation.sh *-gotest.json
if: always()
- name: CRI Integration Test
env:
@ -510,6 +554,7 @@ jobs:
name: TestResults ${{ matrix.runtime }} ${{matrix.runc}}
path: |
*-junit.xml
*-gotest.json
${{github.workspace}}/critestreport/*.xml
tests-mac-os:
@ -526,13 +571,20 @@ jobs:
go-version: ${{ env.GO_VERSION }}
- uses: actions/checkout@v2
- run: script/setup/install-gotestsum
- run: script/setup/install-teststat
- name: Tests
env:
GOTESTSUM_JUNITFILE: "${{ github.workspace }}/macos-test-junit.xml"
GOTESTSUM_JSONFILE: "${{ github.workspace }}/macos-test-gotest.json"
run: make test
- run: if [ -f *-gotest.json ]; then echo '# Unit Tests' >> $GITHUB_STEP_SUMMARY; teststat -markdown *-gotest.json >> $GITHUB_STEP_SUMMARY; fi
if: always()
- run: script/test/test2annotation.sh *-gotest.json
if: always()
- uses: actions/upload-artifact@v2
if: always()
with:
name: TestResults MacOS
path: |
*-junit.xml
*-gotest.json

1
Vagrantfile vendored
View File

@ -227,6 +227,7 @@ EOF
'RUNC_FLAVOR': ENV['RUNC_FLAVOR'] || "runc",
'GOTEST': ENV['GOTEST'] || "go test",
'GOTESTSUM_JUNITFILE': ENV['GOTESTSUM_JUNITFILE'],
'GOTESTSUM_JSONFILE': ENV['GOTESTSUM_JSONFILE'],
}
sh.inline = <<~SHELL
#!/usr/bin/env bash

17
script/setup/install-teststat Executable file
View File

@ -0,0 +1,17 @@
#!/usr/bin/env bash
# Copyright The containerd Authors.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# http://www.apache.org/licenses/LICENSE-2.0
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
GO111MODULE=on go install github.com/vearutop/teststat@v0.1.3

97
script/test/test2annotation.jq Executable file
View File

@ -0,0 +1,97 @@
#!/usr/bin/env -S jq -s -r -f
# Copyright The containerd Authors.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# http://www.apache.org/licenses/LICENSE-2.0
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Takes the raw test2json as input and groups each item by package name + test name
#
# Example
# Input:
# {...}
# {...}
# {...}
# Output:
# [[{...},{...},{...}]]
def group_tests: reduce (.[] | select(.Test != null)) as $item (
{}; .["\($item.Package).\($item.Test)"] += [$item]
);
# Filter a group of tests by action.
# The way this works is for a group of tests,
# if none of the tests have an action matching the provided value, the group is
# dropped.
def filter_group_action(a): map(select(.[].Action == a));
# Filter a group of tests to just the tests that failed
def only_failed: filter_group_action("fail");
def merge_strings(s1; s2): if s1 == null then s2 else "\(s1)\(s2)" end;
# Matches an output string to find the test file and line number.
# The actual string looks something like:
# "test_foo.go:12: some failure message\n"
# There may be arbitrary whitespace at the beginning of the string.
def capture_fail_details: capture("^\\s*(?<file>\\w+_test\\.go):(?<line>\\d+): "; "x");
# Filter out all the undesirable test line outputs
def filter_test_item:
select(.Output != null)
| select(.Output | test("^\\s*=== RUN")| not)
| select(.Output | test("^\\s*=== PAUSE")| not)
| select(.Output | test("^\\s*=== CONT")| not)
| select(.Output | test("^\\s*--- FAIL:")| not)
;
# Take a list of test groups and make a map of tests keyed on the package name +
# test name with all the output concatenated.
#
# This uses the last test log message to get the the file/line number for the test (as .Details)
def merge_output: reduce (.[][] | filter_test_item) as $item (
{}; (
"\($item.Package).\($item.Test)" as $key
| merge_strings(.[$key].Output; $item.Output) as $merged
| .[$key] += {
Output: $merged | sub("^\\s*"; ""; "x"),
Test: $item.Test,
Package: $item.Package,
Details: (($item.Output | capture_fail_details) // .[$key].Details), # "//" is an operator that returns the first true/non-null value
}
) // .
);
# Used as the "title" field for error annotations
def err_title: "Failed: \(.Package).\(.Test)";
# Split the containerd root package path to get the directory name that a test belongs to.
def file_dir: .Package | split("github.com/containerd/containerd/")[-1];
def err_file: if .Details != null and .Details.file != null then "\(file_dir)/\(.Details.file)" else "" end;
# Some cases there may not be any extra details because they got filtered out.
# This can happen, for instance, if the failure log was create from a file that is not a `_test.go` file.
# In this case we'd still want the annotation but we just can't get the file/line number (without allowing non-test files to be the source of the failure).
def details_to_string: if .Details != null and .Details.file != null then ",file=\(err_file),line=\(.Details.line)" else "" end;
# Replace all occurrences of "\n" with the url-encoded version
# This allows multi-line strings to work with github annotations.
# https://github.com/actions/toolkit/issues/193#issuecomment-605394935
def encode_output: gsub("\n"; "%0A");
# Creates github actions error annotations for each input
# It is expected that the input is the output of merge_output.
def to_error: reduce .[] as $item (
""; . += "::error title=\($item | err_title)\($item | details_to_string)::\($item.Output | encode_output)\n"
);
group_tests | only_failed | merge_output | to_error

19
script/test/test2annotation.sh Executable file
View File

@ -0,0 +1,19 @@
#!/usr/bin/env sh
# Copyright The containerd Authors.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# http://www.apache.org/licenses/LICENSE-2.0
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
f=$(dirname $0)/test2annotation.jq
exec jq -f "${f}" -s -r $@