From af1e2af72012aa9d5bc71d032c0bc682a4b307ab Mon Sep 17 00:00:00 2001 From: Brian Goff Date: Thu, 25 Mar 2021 19:11:29 +0000 Subject: [PATCH] ci: upload junit formatted test results This allows us to dig more details out of test runs and maintain a better history. For this we can use `gotestsum`, which is a utility that wraps `go test` so that it outputs test2json (go's format) and output junit (a format more easily imported into other systems). The PR makes it possible to override the Makefile's use of `go test` to use any other command tto executet the test. For CI we'll use `gotestsum --`, where `gotestsum` expects everything after the `--` to be flags for `go test`. We then use environment variables to configure `gotestsum` (e.g. `GOTESTSUM_JUNITFILE` is an env var accepted by `gotestsum`). For cri tests, the test suite supports outputing test results to a directory, these are in junit format already. The file is not named properly just because the code that creates it (in ginkgo) is not configured well. We can fix that upstream to give us a better name... until then I'm keeping those results in a separate dir. A second workflow is also added so the test results can be summed up and a report added to the workflow run. The 2nd workflow is required for this since PR runs do not have access to do some of this due to safety reasons (https://securitylab.github.com/research/github-actions-preventing-pwn-requests/) Signed-off-by: Brian Goff --- .github/workflows/ci.yml | 66 ++++++++++++++++++++-- .github/workflows/publish_test_results.yml | 61 ++++++++++++++++++++ Makefile | 9 ++- Vagrantfile | 20 ++++++- script/setup/install-gotestsum | 17 ++++++ 5 files changed, 165 insertions(+), 8 deletions(-) create mode 100644 .github/workflows/publish_test_results.yml create mode 100755 script/setup/install-gotestsum diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 864fe7cc3..6de541eba 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -247,6 +247,8 @@ jobs: runs-on: windows-2019 timeout-minutes: 30 needs: [project, linters, protos, man] + env: + GOTEST: gotestsum -- defaults: run: @@ -270,6 +272,7 @@ jobs: - name: Set env run: | echo "GOPATH=${{ github.workspace }}" >> $GITHUB_ENV + echo "${{ github.workspace }}/bin" >> $GITHUB_PATH echo "${{ github.workspace }}/src/github.com/containerd/containerd/bin" >> $GITHUB_PATH - run: script/setup/install-dev-tools @@ -287,22 +290,32 @@ jobs: git checkout "${SHIM_COMMIT}" GO111MODULE=on go build -mod=vendor -o "${bindir}/integration/client/containerd-shim-runhcs-v1.exe" ./cmd/containerd-shim-runhcs-v1 + - run: script/setup/install-gotestsum - name: Tests env: CGO_ENABLED: 1 + GOTESTSUM_JUNITFILE: ${{github.workspace}}/test-unit-root.xml run: mingw32-make.exe test root-test - name: Integration 1 env: CGO_ENABLED: 1 + GOTESTSUM_JUNITFILE: ${{github.workspace}}/test-integration-serial-junit.xml run: mingw32-make.exe integration - # Run the integration suite a second time. See discussion in github.com/containerd/containerd/pull/175 + # Run the integration suite a second time. See discussion in github.com/containerd/containerd/pull/1759 - name: Integration 2 env: TESTFLAGS_PARALLEL: 1 CGO_ENABLED: 1 + GOTESTSUM_JUNITFILE: ${{github.workspace}}/test-integration-parallel-junit.xml run: mingw32-make.exe integration + - uses: actions/upload-artifact@v2 + if: always() + with: + name: TestResults Windows + path: | + ${{github.workspace}}/*-junit.xml integration-linux: name: Linux Integration @@ -321,6 +334,8 @@ jobs: - runtime: io.containerd.runtime.v1.linux runc: crun + env: + GOTEST: gotestsum -- steps: - uses: actions/setup-go@v2 with: @@ -370,9 +385,12 @@ jobs: sudo make install working-directory: src/github.com/containerd/containerd + - run: sudo -E PATH=$PATH script/setup/install-gotestsum + working-directory: src/github.com/containerd/containerd - name: Tests env: GOPROXY: direct + GOTESTSUM_JUNITFILE: ${{github.workspace}}/test-unit-root-junit.xml run: | make test sudo -E PATH=$PATH GOPATH=$GOPATH GOPROXY=$GOPROXY make root-test @@ -383,6 +401,7 @@ jobs: GOPROXY: direct TEST_RUNTIME: ${{ matrix.runtime }} RUNC_FLAVOR: ${{ matrix.runc }} + GOTESTSUM_JUNITFILE: ${{github.workspace}}/test-integration-serial-junit.xml run: | sudo GOPATH=$GOPATH GOPROXY=$GOPROXY TEST_RUNTIME=$TEST_RUNTIME RUNC_FLAVOR=$RUNC_FLAVOR make integration EXTRA_TESTFLAGS=-no-criu TESTFLAGS_RACE=-race working-directory: src/github.com/containerd/containerd @@ -393,6 +412,7 @@ jobs: GOPROXY: direct TEST_RUNTIME: ${{ matrix.runtime }} RUNC_FLAVOR: ${{ matrix.runc }} + GOTESTSUM_JUNITFILE: ${{github.workspace}}/test-integration-parallel-junit.xml run: | sudo GOPATH=$GOPATH GOPROXY=$GOPROXY TEST_RUNTIME=$TEST_RUNTIME RUNC_FLAVOR=$RUNC_FLAVOR TESTFLAGS_PARALLEL=1 make integration EXTRA_TESTFLAGS=-no-criu working-directory: src/github.com/containerd/containerd @@ -404,6 +424,7 @@ jobs: GOPROXY: direct TEST_RUNTIME: ${{ matrix.runtime }} RUNC_FLAVOR: ${{ matrix.runc }} + GOTESTSUM_JUNITFILE: ${{github.workspace}}/test-integration-criu-junit.xml # crun doesn't have "checkpoint" command. if: ${{ matrix.runc == 'runc' }} run: | @@ -433,7 +454,7 @@ jobs: sudo ls /etc/cni/net.d sudo PATH=$PATH BDIR=$BDIR /usr/local/bin/containerd -a ${BDIR}/c.sock --config ${BDIR}/config.toml --root ${BDIR}/root --state ${BDIR}/state --log-level debug &> ${BDIR}/containerd-cri.log & sudo PATH=$PATH BDIR=$BDIR /usr/local/bin/ctr -a ${BDIR}/c.sock version - sudo PATH=$PATH BDIR=$BDIR GOPATH=$GOPATH critest --runtime-endpoint=unix:///${BDIR}/c.sock --parallel=8 + sudo PATH=$PATH BDIR=$BDIR GOPATH=$GOPATH critest --report-dir "${{github.workspace}}/critestreport" --runtime-endpoint=unix:///${BDIR}/c.sock --parallel=8 TEST_RC=$? test $TEST_RC -ne 0 && cat ${BDIR}/containerd-cri.log sudo pkill containerd @@ -449,12 +470,21 @@ jobs: mount df losetup -l + - uses: actions/upload-artifact@v2 + if: always() + with: + name: TestResults ${{ matrix.runtime }} ${{matrix.runc}} + path: | + *-junit.xml + ${{github.workspace}}/critestreport/*.xml tests-mac-os: name: MacOS unit tests runs-on: macos-10.15 timeout-minutes: 10 needs: [project, linters, protos, man] + env: + GOTEST: gotestsum -- steps: - uses: actions/setup-go@v2 @@ -470,12 +500,21 @@ jobs: echo "GOPATH=${{ github.workspace }}" >> $GITHUB_ENV echo "${{ github.workspace }}/bin" >> $GITHUB_PATH + - run: sudo -E PATH=$PATH script/setup/install-gotestsum + working-directory: src/github.com/containerd/containerd - name: Tests env: GOPROXY: direct + GOTESTSUM_JUNITFILE: "${{ github.workspace }}/macos-test-junit.xml" run: | make test working-directory: src/github.com/containerd/containerd + - uses: actions/upload-artifact@v2 + if: always() + with: + name: TestResults MacOS + path: | + *-junit.xml cgroup2: name: CGroupsV2 and SELinux Integration @@ -488,6 +527,8 @@ jobs: # Currently crun is disabled to decrease CI flakiness. # We can enable crun again when we get a better CI infra. runc: [runc] + env: + GOTEST: gotestsum -- steps: - uses: actions/checkout@v2 @@ -506,10 +547,27 @@ jobs: env: RUNC_FLAVOR: ${{ matrix.runc }} SELINUX: Enforcing - run: vagrant up --provision-with=selinux,install-runc,test-integration + GOTESTSUM_JUNITFILE: /tmp/test-integration-junit.xml + run: vagrant up --provision-with=selinux,install-runc,install-gotestsum,test-integration - name: CRI test env: RUNC_FLAVOR: ${{ matrix.runc }} SELINUX: Enforcing - run: vagrant up --provision-with=selinux,install-runc,test-cri + REPORT_DIR: /tmp/critestreport + run: vagrant up --provision-with=selinux,install-runc,install-gotestsum,test-cri + - name: Get test reports + if: always() + run: | + set -e + vagrant plugin install vagrant-vbguest + vagrant plugin install vagrant-scp + vagrant scp :/tmp/test-integration-junit.xml "${{ github.workspace }}/" + vagrant scp :/tmp/critestreport "${{ github.workspace }}/critestreport" + - uses: actions/upload-artifact@v2 + if: always() + with: + name: TestResults cgroup2 ${{ matrix.runtime }} ${{matrix.runc}} + path: | + ${{github.workspace}}/*-junit.xml + ${{github.workspace}}/critestreport/* \ No newline at end of file diff --git a/.github/workflows/publish_test_results.yml b/.github/workflows/publish_test_results.yml new file mode 100644 index 000000000..31ba7769f --- /dev/null +++ b/.github/workflows/publish_test_results.yml @@ -0,0 +1,61 @@ +name: Publish Test Results + +on: + workflow_run: + workflows: ["CI"] + types: + - completed + +jobs: + test-results: + runs-on: ubuntu-latest + name: Publish Test Results + if: > + github.event.workflow_run.conclusion != 'skipped' && ( + github.event.workflow_run.head_repository.full_name != github.repository + ) + steps: + - name: Download Artifacts + uses: actions/github-script@v3.1.0 + with: + script: | + var fs = require('fs'); + var path = require('path'); + var artifacts_path = path.join('${{github.workspace}}', 'artifacts') + fs.mkdirSync(artifacts_path, { recursive: true }) + + var artifacts = await github.actions.listWorkflowRunArtifacts({ + owner: context.repo.owner, + repo: context.repo.repo, + run_id: ${{ github.event.workflow_run.id }}, + }); + + for (const artifact of artifacts.data.artifacts) { + var download = await github.actions.downloadArtifact({ + owner: context.repo.owner, + repo: context.repo.repo, + artifact_id: artifact.id, + archive_format: 'zip', + }); + var artifact_path = path.join(artifacts_path, `${artifact.name}.zip`) + fs.writeFileSync(artifact_path, Buffer.from(download.data)); + console.log(`Downloaded ${artifact_path}`); + } + - name: Extract Artifacts + run: | + for file in artifacts/*.zip + do + if [ -f "$file" ] + then + dir="${file/%.zip/}" + mkdir -p "$dir" + unzip -d "$dir" "$file" + fi + done + - name: Publish Test Results + uses: EnricoMi/publish-unit-test-result-action@v1 + with: + files: artifacts/*/**/*.xml + check_name: Test Results + comment_on_pr: true + commit: ${{ github.event.workflow_run.head_sha }} \ No newline at end of file diff --git a/Makefile b/Makefile index cc4ed89ac..bfb2a784c 100644 --- a/Makefile +++ b/Makefile @@ -120,6 +120,9 @@ BINARIES=$(addprefix bin/,$(COMMANDS)) TESTFLAGS ?= $(TESTFLAGS_RACE) $(EXTRA_TESTFLAGS) TESTFLAGS_PARALLEL ?= 8 +# Use this to replace `go test` with, for instance, `gotestsum` +GOTEST ?= go test + OUTPUTDIR = $(join $(ROOTDIR), _output) CRIDIR=$(OUTPUTDIR)/cri @@ -170,15 +173,15 @@ build: ## build the go packages test: ## run tests, except integration tests and tests that require root @echo "$(WHALE) $@" - @go test ${TESTFLAGS} ${PACKAGES} + @$(GOTEST) ${TESTFLAGS} ${PACKAGES} root-test: ## run tests, except integration tests @echo "$(WHALE) $@" - @go test ${TESTFLAGS} ${TEST_REQUIRES_ROOT_PACKAGES} -test.root + @$(GOTEST) ${TESTFLAGS} ${TEST_REQUIRES_ROOT_PACKAGES} -test.root integration: ## run integration tests @echo "$(WHALE) $@" - @cd "${ROOTDIR}/integration/client" && go mod download && go test -v ${TESTFLAGS} -test.root -parallel ${TESTFLAGS_PARALLEL} . + @cd "${ROOTDIR}/integration/client" && go mod download && $(GOTEST) -v ${TESTFLAGS} -test.root -parallel ${TESTFLAGS_PARALLEL} . # TODO integrate cri integration bucket with coverage bin/cri-integration.test: diff --git a/Vagrantfile b/Vagrantfile index 35c6f6ace..49d748f6d 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -173,6 +173,18 @@ EOF SHELL end + config.vm.provision "install-gotestsum", type: "shell", run: "once" do |sh| + sh.upload_path = "/tmp/vagrant-install-gotestsum" + sh.inline = <<~SHELL + #!/usr/bin/env bash + source /etc/environment + source /etc/profile.d/sh.local + set -eux -o pipefail + ${GOPATH}/src/github.com/containerd/containerd/script/setup/install-gotestsum + sudo cp ${GOPATH}/bin/gotestsum /usr/local/bin/ + SHELL + end + # SELinux is Enforcing by default. # To set SELinux as Disabled on a VM that has already been provisioned: # SELINUX=Disabled vagrant up --provision-with=selinux @@ -196,6 +208,8 @@ EOF sh.upload_path = "/tmp/test-integration" sh.env = { 'RUNC_FLAVOR': ENV['RUNC_FLAVOR'] || "runc", + 'GOTEST': ENV['GOTEST'] || "go test", + 'GOTESTSUM_JUNITFILE': ENV['GOTESTSUM_JUNITFILE'], } sh.inline = <<~SHELL #!/usr/bin/env bash @@ -213,6 +227,10 @@ EOF # config.vm.provision "test-cri", type: "shell", run: "never" do |sh| sh.upload_path = "/tmp/test-cri" + sh.env = { + 'GOTEST': ENV['GOTEST'] || "go test", + 'REPORT_DIR': ENV['REPORT_DIR'], + } sh.inline = <<~SHELL #!/usr/bin/env bash source /etc/environment @@ -235,7 +253,7 @@ EOF fi trap cleanup EXIT ctr version - critest --parallel=$(nproc) --ginkgo.skip='HostIpc is true' + critest --parallel=$(nproc) --report-dir="${REPORT_DIR}" --ginkgo.skip='HostIpc is true' SHELL end diff --git a/script/setup/install-gotestsum b/script/setup/install-gotestsum new file mode 100755 index 000000000..90c4a3cb8 --- /dev/null +++ b/script/setup/install-gotestsum @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + +# Copyright The containerd Authors. + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +GO111MODULE=on go install gotest.tools/gotestsum@v1.6.2 \ No newline at end of file