diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 817e7a160..3bd6553ef 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -3,16 +3,16 @@ on: push: branches: - main - - 'release/**' + - "release/**" pull_request: branches: - main - - 'release/**' + - "release/**" env: # Go version we currently use to build containerd across all CI. # Note: don't forget to update `Binaries` step, as it contains the matrix of all supported Go versions. - GO_VERSION: '1.19.1' + GO_VERSION: "1.19.1" jobs: # @@ -219,7 +219,7 @@ jobs: strategy: matrix: os: [ubuntu-18.04, macos-12, windows-2019, windows-2022] - go-version: ['1.19.1', '1.18.6'] + go-version: ["1.19.1", "1.18.6"] steps: - uses: actions/setup-go@v2 with: @@ -256,7 +256,7 @@ jobs: fail-fast: false matrix: os: [windows-2019, windows-2022] - enable_cri_sandboxes: [ "", "sandboxed"] + enable_cri_sandboxes: ["", "sandboxed"] defaults: run: @@ -304,14 +304,14 @@ jobs: RESOURCE_CONSUMER_TESTING_IMAGE_REF: "registry.k8s.io/e2e-test-images/resource-consumer:1.10" WEBSERVER_TESTING_IMAGE_REF: "registry.k8s.io/e2e-test-images/nginx:1.14-2" run: | - cat > "${{ env.TEST_IMAGE_LIST }}" << EOF - busybox = "${{ env.BUSYBOX_TESTING_IMAGE_REF }}" - ResourceConsumer = "${{ env.RESOURCE_CONSUMER_TESTING_IMAGE_REF }}" - EOF - cat > "${{ env.CRI_TEST_IMAGES }}" << EOF - defaultTestContainerImage: ${{ env.BUSYBOX_TESTING_IMAGE_REF }} - webServerTestImage: ${{ env.WEBSERVER_TESTING_IMAGE_REF }} - EOF + cat > "${{ env.TEST_IMAGE_LIST }}" << EOF + busybox = "${{ env.BUSYBOX_TESTING_IMAGE_REF }}" + ResourceConsumer = "${{ env.RESOURCE_CONSUMER_TESTING_IMAGE_REF }}" + EOF + cat > "${{ env.CRI_TEST_IMAGES }}" << EOF + defaultTestContainerImage: ${{ env.BUSYBOX_TESTING_IMAGE_REF }} + webServerTestImage: ${{ env.WEBSERVER_TESTING_IMAGE_REF }} + EOF - name: Get crictl tool shell: powershell @@ -323,18 +323,33 @@ jobs: mv crictl.exe "${{ github.workspace }}/bin/crictl.exe" # Move crictl somewhere in path - run: script/setup/install-gotestsum + - run: script/setup/install-teststat - name: Tests env: CGO_ENABLED: 1 GOTESTSUM_JUNITFILE: ${{github.workspace}}/test-unit-root.xml + GOTESTSUM_JSONFILE: ${{github.workspace}}/test-unit-root-gotest.json run: mingw32-make.exe test root-test + - run: if [ -f *-gotest.json ]; then echo '# Root Test' >> $GITHUB_STEP_SUMMARY; teststat -markdown *-gotest.json >> $GITHUB_STEP_SUMMARY; fi + if: always() + - run: script/test/test2annotation.sh ${TESTFILE} + env: + TESTFILE: ${{github.workspace}}/test-unit-root-gotest.json + if: always() - name: Integration 1 env: CGO_ENABLED: 1 ENABLE_CRI_SANDBOXES: ${{ matrix.enable_cri_sandboxes }} GOTESTSUM_JUNITFILE: ${{github.workspace}}/test-integration-serial-junit.xml + GOTESTSUM_JSONFILE: ${{github.workspace}}/test-integration-serial-gotest.json run: mingw32-make.exe integration + - run: if [ -f *-gotest.json ]; then echo '# Integration 1' >> $GITHUB_STEP_SUMMARY; teststat -markdown *-gotest.json >> $GITHUB_STEP_SUMMARY; fi + if: always() + - run: script/test/test2annotation.sh ${TESTFILE} + env: + TESTFILE: ${{github.workspace}}/test-integration-serial-gotest.json + if: always() # Run the integration suite a second time. See discussion in github.com/containerd/containerd/pull/1759 - name: Integration 2 @@ -344,7 +359,14 @@ jobs: CGO_ENABLED: 1 ENABLE_CRI_SANDBOXES: ${{ matrix.enable_cri_sandboxes }} GOTESTSUM_JUNITFILE: ${{github.workspace}}/test-integration-parallel-junit.xml + GOTESTSUM_JSONFILE: ${{github.workspace}}/test-integration-parallel-gotest.json run: mingw32-make.exe integration + - run: if [ -f *-gotest.json ]; then echo '# Integration 2' >> $GITHUB_STEP_SUMMARY; teststat -markdown *-gotest.json >> $GITHUB_STEP_SUMMARY; fi + if: always() + - run: script/test/test2annotation.sh ${TESTFILE} + env: + TESTFILE: ${{github.workspace}}/test-integration-parallel-gotest.json + if: always() - name: CRI Integration Test env: @@ -376,6 +398,7 @@ jobs: name: TestResults ${{ matrix.os }} path: | ${{github.workspace}}/*-junit.xml + ${{github.workspace}}/*-gotest.json integration-linux: name: Linux Integration @@ -386,9 +409,14 @@ jobs: strategy: fail-fast: false matrix: - runtime: [io.containerd.runtime.v1.linux, io.containerd.runc.v1, io.containerd.runc.v2] + runtime: + [ + io.containerd.runtime.v1.linux, + io.containerd.runc.v1, + io.containerd.runc.v2, + ] runc: [runc, crun] - enable_cri_sandboxes: [ "", "sandboxed"] + enable_cri_sandboxes: ["", "sandboxed"] exclude: - runtime: io.containerd.runc.v1 runc: crun @@ -429,12 +457,18 @@ jobs: sudo -E PATH=$PATH make install - run: script/setup/install-gotestsum + - run: script/setup/install-teststat - name: Tests env: GOTESTSUM_JUNITFILE: ${{github.workspace}}/test-unit-root-junit.xml + GOTESTSUM_JSONFILE: ${{github.workspace}}/test-unit-root-gotest.json run: | make test sudo -E PATH=$PATH make root-test + - run: if [ -f *-gotest.json ]; then echo '# Root Test' >> $GITHUB_STEP_SUMMARY; teststat -markdown *-gotest.json >> $GITHUB_STEP_SUMMARY; fi + if: always() + - run: script/test/test2annotation.sh *-gotest.json + if: always() - name: Integration 1 env: @@ -442,12 +476,17 @@ jobs: RUNC_FLAVOR: ${{ matrix.runc }} ENABLE_CRI_SANDBOXES: ${{ matrix.enable_cri_sandboxes }} GOTESTSUM_JUNITFILE: ${{github.workspace}}/test-integration-serial-junit.xml + GOTESTSUM_JSONFILE: ${{github.workspace}}/test-integration-serial-gotest.json run: | extraflags="" [ "${RUNC_FLAVOR}" == "crun" ] && { extraflags="EXTRA_TESTFLAGS=-no-criu"; } sudo -E PATH=$PATH make integration ${extraflags} TESTFLAGS_RACE=-race + - run: if [ -f *-gotest.json ]; then echo '# Integration 1' >> $GITHUB_STEP_SUMMARY; teststat -markdown *-gotest.json >> $GITHUB_STEP_SUMMARY; fi + if: always() + - run: script/test/test2annotation.sh *-gotest.json + if: always() # Run the integration suite a second time. See discussion in github.com/containerd/containerd/pull/1759 - name: Integration 2 @@ -456,12 +495,17 @@ jobs: RUNC_FLAVOR: ${{ matrix.runc }} ENABLE_CRI_SANDBOXES: ${{ matrix.enable_cri_sandboxes }} GOTESTSUM_JUNITFILE: ${{github.workspace}}/test-integration-parallel-junit.xml + GOTESTSUM_JSONFILE: ${{github.workspace}}/test-integration-parallel-gotest.json run: | extraflags="" [ "${RUNC_FLAVOR}" == "crun" ] && { extraflags="EXTRA_TESTFLAGS=-no-criu"; } sudo -E PATH=$PATH TESTFLAGS_PARALLEL=1 make integration ${extraflags} + - run: if [ -f *-gotest.json ]; then echo '# Integration 2' >> $GITHUB_STEP_SUMMARY; teststat -markdown *-gotest.json >> $GITHUB_STEP_SUMMARY; fi + if: always() + - run: script/test/test2annotation.sh *-gotest.json + if: always() - name: CRI Integration Test env: @@ -510,6 +554,7 @@ jobs: name: TestResults ${{ matrix.runtime }} ${{matrix.runc}} path: | *-junit.xml + *-gotest.json ${{github.workspace}}/critestreport/*.xml tests-mac-os: @@ -526,13 +571,20 @@ jobs: go-version: ${{ env.GO_VERSION }} - uses: actions/checkout@v2 - run: script/setup/install-gotestsum + - run: script/setup/install-teststat - name: Tests env: GOTESTSUM_JUNITFILE: "${{ github.workspace }}/macos-test-junit.xml" + GOTESTSUM_JSONFILE: "${{ github.workspace }}/macos-test-gotest.json" run: make test + - run: if [ -f *-gotest.json ]; then echo '# Unit Tests' >> $GITHUB_STEP_SUMMARY; teststat -markdown *-gotest.json >> $GITHUB_STEP_SUMMARY; fi + if: always() + - run: script/test/test2annotation.sh *-gotest.json + if: always() - uses: actions/upload-artifact@v2 if: always() with: name: TestResults MacOS path: | *-junit.xml + *-gotest.json diff --git a/Vagrantfile b/Vagrantfile index b96def522..d802b4cdb 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -227,6 +227,7 @@ EOF 'RUNC_FLAVOR': ENV['RUNC_FLAVOR'] || "runc", 'GOTEST': ENV['GOTEST'] || "go test", 'GOTESTSUM_JUNITFILE': ENV['GOTESTSUM_JUNITFILE'], + 'GOTESTSUM_JSONFILE': ENV['GOTESTSUM_JSONFILE'], } sh.inline = <<~SHELL #!/usr/bin/env bash diff --git a/script/setup/install-teststat b/script/setup/install-teststat new file mode 100755 index 000000000..07b9eba04 --- /dev/null +++ b/script/setup/install-teststat @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + +# Copyright The containerd Authors. + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +GO111MODULE=on go install github.com/vearutop/teststat@v0.1.3 diff --git a/script/test/test2annotation.jq b/script/test/test2annotation.jq new file mode 100755 index 000000000..aaf0fe4ad --- /dev/null +++ b/script/test/test2annotation.jq @@ -0,0 +1,97 @@ +#!/usr/bin/env -S jq -s -r -f + +# Copyright The containerd Authors. + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Takes the raw test2json as input and groups each item by package name + test name +# +# Example +# Input: +# {...} +# {...} +# {...} +# Output: +# [[{...},{...},{...}]] +def group_tests: reduce (.[] | select(.Test != null)) as $item ( + {}; .["\($item.Package).\($item.Test)"] += [$item] +); + +# Filter a group of tests by action. +# The way this works is for a group of tests, +# if none of the tests have an action matching the provided value, the group is +# dropped. +def filter_group_action(a): map(select(.[].Action == a)); + +# Filter a group of tests to just the tests that failed +def only_failed: filter_group_action("fail"); + +def merge_strings(s1; s2): if s1 == null then s2 else "\(s1)\(s2)" end; + +# Matches an output string to find the test file and line number. +# The actual string looks something like: +# "test_foo.go:12: some failure message\n" +# There may be arbitrary whitespace at the beginning of the string. +def capture_fail_details: capture("^\\s*(?\\w+_test\\.go):(?\\d+): "; "x"); + +# Filter out all the undesirable test line outputs +def filter_test_item: + select(.Output != null) + | select(.Output | test("^\\s*=== RUN")| not) + | select(.Output | test("^\\s*=== PAUSE")| not) + | select(.Output | test("^\\s*=== CONT")| not) + | select(.Output | test("^\\s*--- FAIL:")| not) +; + +# Take a list of test groups and make a map of tests keyed on the package name + +# test name with all the output concatenated. +# +# This uses the last test log message to get the the file/line number for the test (as .Details) +def merge_output: reduce (.[][] | filter_test_item) as $item ( + {}; ( + "\($item.Package).\($item.Test)" as $key + | merge_strings(.[$key].Output; $item.Output) as $merged + | .[$key] += { + Output: $merged | sub("^\\s*"; ""; "x"), + Test: $item.Test, + Package: $item.Package, + Details: (($item.Output | capture_fail_details) // .[$key].Details), # "//" is an operator that returns the first true/non-null value + } + ) // . +); + +# Used as the "title" field for error annotations +def err_title: "Failed: \(.Package).\(.Test)"; + +# Split the containerd root package path to get the directory name that a test belongs to. +def file_dir: .Package | split("github.com/containerd/containerd/")[-1]; + +def err_file: if .Details != null and .Details.file != null then "\(file_dir)/\(.Details.file)" else "" end; + +# Some cases there may not be any extra details because they got filtered out. +# This can happen, for instance, if the failure log was create from a file that is not a `_test.go` file. +# In this case we'd still want the annotation but we just can't get the file/line number (without allowing non-test files to be the source of the failure). +def details_to_string: if .Details != null and .Details.file != null then ",file=\(err_file),line=\(.Details.line)" else "" end; + +# Replace all occurrences of "\n" with the url-encoded version +# This allows multi-line strings to work with github annotations. +# https://github.com/actions/toolkit/issues/193#issuecomment-605394935 +def encode_output: gsub("\n"; "%0A"); + +# Creates github actions error annotations for each input +# It is expected that the input is the output of merge_output. +def to_error: reduce .[] as $item ( + ""; . += "::error title=\($item | err_title)\($item | details_to_string)::\($item.Output | encode_output)\n" +); + +group_tests | only_failed | merge_output | to_error \ No newline at end of file diff --git a/script/test/test2annotation.sh b/script/test/test2annotation.sh new file mode 100755 index 000000000..36c0a18aa --- /dev/null +++ b/script/test/test2annotation.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env sh + +# Copyright The containerd Authors. + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +f=$(dirname $0)/test2annotation.jq + +exec jq -f "${f}" -s -r $@