diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 864fe7cc3..6de541eba 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -247,6 +247,8 @@ jobs: runs-on: windows-2019 timeout-minutes: 30 needs: [project, linters, protos, man] + env: + GOTEST: gotestsum -- defaults: run: @@ -270,6 +272,7 @@ jobs: - name: Set env run: | echo "GOPATH=${{ github.workspace }}" >> $GITHUB_ENV + echo "${{ github.workspace }}/bin" >> $GITHUB_PATH echo "${{ github.workspace }}/src/github.com/containerd/containerd/bin" >> $GITHUB_PATH - run: script/setup/install-dev-tools @@ -287,22 +290,32 @@ jobs: git checkout "${SHIM_COMMIT}" GO111MODULE=on go build -mod=vendor -o "${bindir}/integration/client/containerd-shim-runhcs-v1.exe" ./cmd/containerd-shim-runhcs-v1 + - run: script/setup/install-gotestsum - name: Tests env: CGO_ENABLED: 1 + GOTESTSUM_JUNITFILE: ${{github.workspace}}/test-unit-root.xml run: mingw32-make.exe test root-test - name: Integration 1 env: CGO_ENABLED: 1 + GOTESTSUM_JUNITFILE: ${{github.workspace}}/test-integration-serial-junit.xml run: mingw32-make.exe integration - # Run the integration suite a second time. See discussion in github.com/containerd/containerd/pull/175 + # Run the integration suite a second time. See discussion in github.com/containerd/containerd/pull/1759 - name: Integration 2 env: TESTFLAGS_PARALLEL: 1 CGO_ENABLED: 1 + GOTESTSUM_JUNITFILE: ${{github.workspace}}/test-integration-parallel-junit.xml run: mingw32-make.exe integration + - uses: actions/upload-artifact@v2 + if: always() + with: + name: TestResults Windows + path: | + ${{github.workspace}}/*-junit.xml integration-linux: name: Linux Integration @@ -321,6 +334,8 @@ jobs: - runtime: io.containerd.runtime.v1.linux runc: crun + env: + GOTEST: gotestsum -- steps: - uses: actions/setup-go@v2 with: @@ -370,9 +385,12 @@ jobs: sudo make install working-directory: src/github.com/containerd/containerd + - run: sudo -E PATH=$PATH script/setup/install-gotestsum + working-directory: src/github.com/containerd/containerd - name: Tests env: GOPROXY: direct + GOTESTSUM_JUNITFILE: ${{github.workspace}}/test-unit-root-junit.xml run: | make test sudo -E PATH=$PATH GOPATH=$GOPATH GOPROXY=$GOPROXY make root-test @@ -383,6 +401,7 @@ jobs: GOPROXY: direct TEST_RUNTIME: ${{ matrix.runtime }} RUNC_FLAVOR: ${{ matrix.runc }} + GOTESTSUM_JUNITFILE: ${{github.workspace}}/test-integration-serial-junit.xml run: | sudo GOPATH=$GOPATH GOPROXY=$GOPROXY TEST_RUNTIME=$TEST_RUNTIME RUNC_FLAVOR=$RUNC_FLAVOR make integration EXTRA_TESTFLAGS=-no-criu TESTFLAGS_RACE=-race working-directory: src/github.com/containerd/containerd @@ -393,6 +412,7 @@ jobs: GOPROXY: direct TEST_RUNTIME: ${{ matrix.runtime }} RUNC_FLAVOR: ${{ matrix.runc }} + GOTESTSUM_JUNITFILE: ${{github.workspace}}/test-integration-parallel-junit.xml run: | sudo GOPATH=$GOPATH GOPROXY=$GOPROXY TEST_RUNTIME=$TEST_RUNTIME RUNC_FLAVOR=$RUNC_FLAVOR TESTFLAGS_PARALLEL=1 make integration EXTRA_TESTFLAGS=-no-criu working-directory: src/github.com/containerd/containerd @@ -404,6 +424,7 @@ jobs: GOPROXY: direct TEST_RUNTIME: ${{ matrix.runtime }} RUNC_FLAVOR: ${{ matrix.runc }} + GOTESTSUM_JUNITFILE: ${{github.workspace}}/test-integration-criu-junit.xml # crun doesn't have "checkpoint" command. if: ${{ matrix.runc == 'runc' }} run: | @@ -433,7 +454,7 @@ jobs: sudo ls /etc/cni/net.d sudo PATH=$PATH BDIR=$BDIR /usr/local/bin/containerd -a ${BDIR}/c.sock --config ${BDIR}/config.toml --root ${BDIR}/root --state ${BDIR}/state --log-level debug &> ${BDIR}/containerd-cri.log & sudo PATH=$PATH BDIR=$BDIR /usr/local/bin/ctr -a ${BDIR}/c.sock version - sudo PATH=$PATH BDIR=$BDIR GOPATH=$GOPATH critest --runtime-endpoint=unix:///${BDIR}/c.sock --parallel=8 + sudo PATH=$PATH BDIR=$BDIR GOPATH=$GOPATH critest --report-dir "${{github.workspace}}/critestreport" --runtime-endpoint=unix:///${BDIR}/c.sock --parallel=8 TEST_RC=$? test $TEST_RC -ne 0 && cat ${BDIR}/containerd-cri.log sudo pkill containerd @@ -449,12 +470,21 @@ jobs: mount df losetup -l + - uses: actions/upload-artifact@v2 + if: always() + with: + name: TestResults ${{ matrix.runtime }} ${{matrix.runc}} + path: | + *-junit.xml + ${{github.workspace}}/critestreport/*.xml tests-mac-os: name: MacOS unit tests runs-on: macos-10.15 timeout-minutes: 10 needs: [project, linters, protos, man] + env: + GOTEST: gotestsum -- steps: - uses: actions/setup-go@v2 @@ -470,12 +500,21 @@ jobs: echo "GOPATH=${{ github.workspace }}" >> $GITHUB_ENV echo "${{ github.workspace }}/bin" >> $GITHUB_PATH + - run: sudo -E PATH=$PATH script/setup/install-gotestsum + working-directory: src/github.com/containerd/containerd - name: Tests env: GOPROXY: direct + GOTESTSUM_JUNITFILE: "${{ github.workspace }}/macos-test-junit.xml" run: | make test working-directory: src/github.com/containerd/containerd + - uses: actions/upload-artifact@v2 + if: always() + with: + name: TestResults MacOS + path: | + *-junit.xml cgroup2: name: CGroupsV2 and SELinux Integration @@ -488,6 +527,8 @@ jobs: # Currently crun is disabled to decrease CI flakiness. # We can enable crun again when we get a better CI infra. runc: [runc] + env: + GOTEST: gotestsum -- steps: - uses: actions/checkout@v2 @@ -506,10 +547,27 @@ jobs: env: RUNC_FLAVOR: ${{ matrix.runc }} SELINUX: Enforcing - run: vagrant up --provision-with=selinux,install-runc,test-integration + GOTESTSUM_JUNITFILE: /tmp/test-integration-junit.xml + run: vagrant up --provision-with=selinux,install-runc,install-gotestsum,test-integration - name: CRI test env: RUNC_FLAVOR: ${{ matrix.runc }} SELINUX: Enforcing - run: vagrant up --provision-with=selinux,install-runc,test-cri + REPORT_DIR: /tmp/critestreport + run: vagrant up --provision-with=selinux,install-runc,install-gotestsum,test-cri + - name: Get test reports + if: always() + run: | + set -e + vagrant plugin install vagrant-vbguest + vagrant plugin install vagrant-scp + vagrant scp :/tmp/test-integration-junit.xml "${{ github.workspace }}/" + vagrant scp :/tmp/critestreport "${{ github.workspace }}/critestreport" + - uses: actions/upload-artifact@v2 + if: always() + with: + name: TestResults cgroup2 ${{ matrix.runtime }} ${{matrix.runc}} + path: | + ${{github.workspace}}/*-junit.xml + ${{github.workspace}}/critestreport/* \ No newline at end of file diff --git a/.github/workflows/publish_test_results.yml b/.github/workflows/publish_test_results.yml new file mode 100644 index 000000000..31ba7769f --- /dev/null +++ b/.github/workflows/publish_test_results.yml @@ -0,0 +1,61 @@ +name: Publish Test Results + +on: + workflow_run: + workflows: ["CI"] + types: + - completed + +jobs: + test-results: + runs-on: ubuntu-latest + name: Publish Test Results + if: > + github.event.workflow_run.conclusion != 'skipped' && ( + github.event.workflow_run.head_repository.full_name != github.repository + ) + steps: + - name: Download Artifacts + uses: actions/github-script@v3.1.0 + with: + script: | + var fs = require('fs'); + var path = require('path'); + var artifacts_path = path.join('${{github.workspace}}', 'artifacts') + fs.mkdirSync(artifacts_path, { recursive: true }) + + var artifacts = await github.actions.listWorkflowRunArtifacts({ + owner: context.repo.owner, + repo: context.repo.repo, + run_id: ${{ github.event.workflow_run.id }}, + }); + + for (const artifact of artifacts.data.artifacts) { + var download = await github.actions.downloadArtifact({ + owner: context.repo.owner, + repo: context.repo.repo, + artifact_id: artifact.id, + archive_format: 'zip', + }); + var artifact_path = path.join(artifacts_path, `${artifact.name}.zip`) + fs.writeFileSync(artifact_path, Buffer.from(download.data)); + console.log(`Downloaded ${artifact_path}`); + } + - name: Extract Artifacts + run: | + for file in artifacts/*.zip + do + if [ -f "$file" ] + then + dir="${file/%.zip/}" + mkdir -p "$dir" + unzip -d "$dir" "$file" + fi + done + - name: Publish Test Results + uses: EnricoMi/publish-unit-test-result-action@v1 + with: + files: artifacts/*/**/*.xml + check_name: Test Results + comment_on_pr: true + commit: ${{ github.event.workflow_run.head_sha }} \ No newline at end of file diff --git a/Makefile b/Makefile index cc4ed89ac..bfb2a784c 100644 --- a/Makefile +++ b/Makefile @@ -120,6 +120,9 @@ BINARIES=$(addprefix bin/,$(COMMANDS)) TESTFLAGS ?= $(TESTFLAGS_RACE) $(EXTRA_TESTFLAGS) TESTFLAGS_PARALLEL ?= 8 +# Use this to replace `go test` with, for instance, `gotestsum` +GOTEST ?= go test + OUTPUTDIR = $(join $(ROOTDIR), _output) CRIDIR=$(OUTPUTDIR)/cri @@ -170,15 +173,15 @@ build: ## build the go packages test: ## run tests, except integration tests and tests that require root @echo "$(WHALE) $@" - @go test ${TESTFLAGS} ${PACKAGES} + @$(GOTEST) ${TESTFLAGS} ${PACKAGES} root-test: ## run tests, except integration tests @echo "$(WHALE) $@" - @go test ${TESTFLAGS} ${TEST_REQUIRES_ROOT_PACKAGES} -test.root + @$(GOTEST) ${TESTFLAGS} ${TEST_REQUIRES_ROOT_PACKAGES} -test.root integration: ## run integration tests @echo "$(WHALE) $@" - @cd "${ROOTDIR}/integration/client" && go mod download && go test -v ${TESTFLAGS} -test.root -parallel ${TESTFLAGS_PARALLEL} . + @cd "${ROOTDIR}/integration/client" && go mod download && $(GOTEST) -v ${TESTFLAGS} -test.root -parallel ${TESTFLAGS_PARALLEL} . # TODO integrate cri integration bucket with coverage bin/cri-integration.test: diff --git a/Vagrantfile b/Vagrantfile index 35c6f6ace..49d748f6d 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -173,6 +173,18 @@ EOF SHELL end + config.vm.provision "install-gotestsum", type: "shell", run: "once" do |sh| + sh.upload_path = "/tmp/vagrant-install-gotestsum" + sh.inline = <<~SHELL + #!/usr/bin/env bash + source /etc/environment + source /etc/profile.d/sh.local + set -eux -o pipefail + ${GOPATH}/src/github.com/containerd/containerd/script/setup/install-gotestsum + sudo cp ${GOPATH}/bin/gotestsum /usr/local/bin/ + SHELL + end + # SELinux is Enforcing by default. # To set SELinux as Disabled on a VM that has already been provisioned: # SELINUX=Disabled vagrant up --provision-with=selinux @@ -196,6 +208,8 @@ EOF sh.upload_path = "/tmp/test-integration" sh.env = { 'RUNC_FLAVOR': ENV['RUNC_FLAVOR'] || "runc", + 'GOTEST': ENV['GOTEST'] || "go test", + 'GOTESTSUM_JUNITFILE': ENV['GOTESTSUM_JUNITFILE'], } sh.inline = <<~SHELL #!/usr/bin/env bash @@ -213,6 +227,10 @@ EOF # config.vm.provision "test-cri", type: "shell", run: "never" do |sh| sh.upload_path = "/tmp/test-cri" + sh.env = { + 'GOTEST': ENV['GOTEST'] || "go test", + 'REPORT_DIR': ENV['REPORT_DIR'], + } sh.inline = <<~SHELL #!/usr/bin/env bash source /etc/environment @@ -235,7 +253,7 @@ EOF fi trap cleanup EXIT ctr version - critest --parallel=$(nproc) --ginkgo.skip='HostIpc is true' + critest --parallel=$(nproc) --report-dir="${REPORT_DIR}" --ginkgo.skip='HostIpc is true' SHELL end diff --git a/script/setup/install-gotestsum b/script/setup/install-gotestsum new file mode 100755 index 000000000..90c4a3cb8 --- /dev/null +++ b/script/setup/install-gotestsum @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + +# Copyright The containerd Authors. + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +GO111MODULE=on go install gotest.tools/gotestsum@v1.6.2 \ No newline at end of file