Merge pull request #5271 from cpuguy83/ci_output_junit

ci: upload junit formatted test results

.github/workflows/ci.yml

@@ -247,6 +247,8 @@ jobs:
     runs-on: windows-2019
     timeout-minutes: 30
     needs: [project, linters, protos, man]
+    env:
+      GOTEST: gotestsum --
 
     defaults:
       run:
@@ -270,6 +272,7 @@ jobs:
       - name: Set env
         run: |
           echo "GOPATH=${{ github.workspace }}" >> $GITHUB_ENV
+          echo "${{ github.workspace }}/bin" >> $GITHUB_PATH
           echo "${{ github.workspace }}/src/github.com/containerd/containerd/bin" >> $GITHUB_PATH
 
       - run: script/setup/install-dev-tools
@@ -287,22 +290,32 @@ jobs:
           git checkout "${SHIM_COMMIT}"
           GO111MODULE=on go build -mod=vendor -o "${bindir}/integration/client/containerd-shim-runhcs-v1.exe" ./cmd/containerd-shim-runhcs-v1
 
+      - run: script/setup/install-gotestsum
       - name: Tests
         env:
           CGO_ENABLED: 1
+          GOTESTSUM_JUNITFILE: ${{github.workspace}}/test-unit-root.xml
         run: mingw32-make.exe test root-test
 
       - name: Integration 1
         env:
           CGO_ENABLED: 1
+          GOTESTSUM_JUNITFILE: ${{github.workspace}}/test-integration-serial-junit.xml
         run: mingw32-make.exe integration
 
-      # Run the integration suite a second time. See discussion in github.com/containerd/containerd/pull/175
+      # Run the integration suite a second time. See discussion in github.com/containerd/containerd/pull/1759
       - name: Integration 2
         env:
           TESTFLAGS_PARALLEL: 1
           CGO_ENABLED: 1
+          GOTESTSUM_JUNITFILE: ${{github.workspace}}/test-integration-parallel-junit.xml
         run: mingw32-make.exe integration
+      - uses: actions/upload-artifact@v2
+        if: always()
+        with:
+          name: TestResults Windows
+          path: |
+            ${{github.workspace}}/*-junit.xml
 
   integration-linux:
     name: Linux Integration
@@ -321,6 +334,8 @@ jobs:
           - runtime: io.containerd.runtime.v1.linux
             runc: crun
 
+    env:
+      GOTEST: gotestsum --
     steps:
       - uses: actions/setup-go@v2
         with:
@@ -370,9 +385,12 @@ jobs:
           sudo make install
         working-directory: src/github.com/containerd/containerd
 
+      - run: sudo -E PATH=$PATH script/setup/install-gotestsum
+        working-directory: src/github.com/containerd/containerd
       - name: Tests
         env:
           GOPROXY: direct
+          GOTESTSUM_JUNITFILE: ${{github.workspace}}/test-unit-root-junit.xml
         run: |
           make test
           sudo -E PATH=$PATH GOPATH=$GOPATH GOPROXY=$GOPROXY make root-test
@@ -383,6 +401,7 @@ jobs:
           GOPROXY: direct
           TEST_RUNTIME: ${{ matrix.runtime }}
           RUNC_FLAVOR: ${{ matrix.runc }}
+          GOTESTSUM_JUNITFILE: ${{github.workspace}}/test-integration-serial-junit.xml
         run: |
           sudo GOPATH=$GOPATH GOPROXY=$GOPROXY TEST_RUNTIME=$TEST_RUNTIME RUNC_FLAVOR=$RUNC_FLAVOR make integration EXTRA_TESTFLAGS=-no-criu TESTFLAGS_RACE=-race
         working-directory: src/github.com/containerd/containerd
@@ -393,6 +412,7 @@ jobs:
           GOPROXY: direct
           TEST_RUNTIME: ${{ matrix.runtime }}
           RUNC_FLAVOR: ${{ matrix.runc }}
+          GOTESTSUM_JUNITFILE: ${{github.workspace}}/test-integration-parallel-junit.xml
         run: |
           sudo GOPATH=$GOPATH GOPROXY=$GOPROXY TEST_RUNTIME=$TEST_RUNTIME RUNC_FLAVOR=$RUNC_FLAVOR TESTFLAGS_PARALLEL=1 make integration EXTRA_TESTFLAGS=-no-criu
         working-directory: src/github.com/containerd/containerd
@@ -404,6 +424,7 @@ jobs:
           GOPROXY: direct
           TEST_RUNTIME: ${{ matrix.runtime }}
           RUNC_FLAVOR: ${{ matrix.runc }}
+          GOTESTSUM_JUNITFILE: ${{github.workspace}}/test-integration-criu-junit.xml
         # crun doesn't have "checkpoint" command.
         if: ${{ matrix.runc == 'runc' }}
         run: |
@@ -434,7 +455,7 @@ jobs:
           sudo ls /etc/cni/net.d
           sudo PATH=$PATH BDIR=$BDIR /usr/local/bin/containerd -a ${BDIR}/c.sock --config ${BDIR}/config.toml --root ${BDIR}/root --state ${BDIR}/state --log-level debug &> ${BDIR}/containerd-cri.log &
           sudo PATH=$PATH BDIR=$BDIR /usr/local/bin/ctr -a ${BDIR}/c.sock version
-          sudo PATH=$PATH BDIR=$BDIR GOPATH=$GOPATH critest --runtime-endpoint=unix:///${BDIR}/c.sock --parallel=8
+          sudo PATH=$PATH BDIR=$BDIR GOPATH=$GOPATH critest --report-dir "${{github.workspace}}/critestreport" --runtime-endpoint=unix:///${BDIR}/c.sock --parallel=8
           TEST_RC=$?
           test $TEST_RC -ne 0 && cat ${BDIR}/containerd-cri.log
           sudo pkill containerd
@@ -450,12 +471,21 @@ jobs:
           mount
           df
           losetup -l
+      - uses: actions/upload-artifact@v2
+        if: always()
+        with:
+          name: TestResults ${{ matrix.runtime }} ${{matrix.runc}}
+          path: |
+            *-junit.xml
+            ${{github.workspace}}/critestreport/*.xml
 
   tests-mac-os:
     name: MacOS unit tests
     runs-on: macos-10.15
     timeout-minutes: 10
     needs: [project, linters, protos, man]
+    env:
+      GOTEST: gotestsum --
 
     steps:
       - uses: actions/setup-go@v2
@@ -471,12 +501,21 @@ jobs:
           echo "GOPATH=${{ github.workspace }}" >> $GITHUB_ENV
           echo "${{ github.workspace }}/bin" >> $GITHUB_PATH
 
+      - run: sudo -E PATH=$PATH script/setup/install-gotestsum
+        working-directory: src/github.com/containerd/containerd
       - name: Tests
         env:
           GOPROXY: direct
+          GOTESTSUM_JUNITFILE: "${{ github.workspace }}/macos-test-junit.xml"
         run: |
           make test
         working-directory: src/github.com/containerd/containerd
+      - uses: actions/upload-artifact@v2
+        if: always()
+        with:
+          name: TestResults MacOS
+          path: |
+            *-junit.xml
 
   cgroup2:
     name: CGroupsV2 and SELinux Integration
@@ -489,6 +528,8 @@ jobs:
         # Currently crun is disabled to decrease CI flakiness.
         # We can enable crun again when we get a better CI infra.
         runc: [runc]
+    env:
+      GOTEST: gotestsum --
     steps:
       - uses: actions/checkout@v2
 
@@ -507,10 +548,27 @@ jobs:
         env:
           RUNC_FLAVOR: ${{ matrix.runc }}
           SELINUX: Enforcing
-        run: vagrant up --provision-with=selinux,install-runc,test-integration
+          GOTESTSUM_JUNITFILE: /tmp/test-integration-junit.xml
+        run: vagrant up --provision-with=selinux,install-runc,install-gotestsum,test-integration
 
       - name: CRI test
         env:
           RUNC_FLAVOR: ${{ matrix.runc }}
           SELINUX: Enforcing
-        run: vagrant up --provision-with=selinux,install-runc,test-cri
+          REPORT_DIR: /tmp/critestreport
+        run: vagrant up --provision-with=selinux,install-runc,install-gotestsum,test-cri
+      - name: Get test reports
+        if: always()
+        run: |
+          set -e
+          vagrant plugin install vagrant-vbguest
+          vagrant plugin install vagrant-scp
+          vagrant scp :/tmp/test-integration-junit.xml "${{ github.workspace }}/"
+          vagrant scp :/tmp/critestreport "${{ github.workspace }}/critestreport"
+      - uses: actions/upload-artifact@v2
+        if: always()
+        with:
+          name:  TestResults cgroup2 ${{ matrix.runtime }} ${{matrix.runc}}
+          path: |
+            ${{github.workspace}}/*-junit.xml
+            ${{github.workspace}}/critestreport/*

.github/workflows/publish_test_results.yml

@@ -0,0 +1,61 @@
+name: Publish Test Results
+
+on:
+   workflow_run:
+      workflows: ["CI"]
+      types:
+         - completed
+
+jobs:
+  test-results:
+    runs-on: ubuntu-latest
+    name: Publish Test Results
+    if: >
+        github.event.workflow_run.conclusion != 'skipped' && (
+          github.event.workflow_run.head_repository.full_name != github.repository
+        )
+    steps:
+      - name: Download Artifacts
+        uses: actions/github-script@v3.1.0
+        with:
+          script: |
+              var fs = require('fs');
+              var path = require('path');
+              var artifacts_path = path.join('${{github.workspace}}', 'artifacts')
+              fs.mkdirSync(artifacts_path, { recursive: true })
+
+              var artifacts = await github.actions.listWorkflowRunArtifacts({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                run_id: ${{ github.event.workflow_run.id }},
+              });
+
+              for (const artifact of artifacts.data.artifacts) {
+                var download = await github.actions.downloadArtifact({
+                    owner: context.repo.owner,
+                    repo: context.repo.repo,
+                    artifact_id: artifact.id,
+                    archive_format: 'zip',
+                });
+                var artifact_path = path.join(artifacts_path, `${artifact.name}.zip`)
+                fs.writeFileSync(artifact_path, Buffer.from(download.data));
+                console.log(`Downloaded ${artifact_path}`);
+              }
+      - name: Extract Artifacts
+        run: |
+          for file in artifacts/*.zip
+          do
+            if [ -f "$file" ]
+            then
+              dir="${file/%.zip/}"
+              mkdir -p "$dir"
+              unzip -d "$dir" "$file"
+            fi
+          done
+      - name: Publish Test Results
+        uses: EnricoMi/publish-unit-test-result-action@v1
+        with:
+          files: artifacts/*/**/*.xml
+          check_name: Test Results
+          comment_on_pr: true
+          commit: ${{ github.event.workflow_run.head_sha }}

Makefile

@@ -120,6 +120,9 @@ BINARIES=$(addprefix bin/,$(COMMANDS))
 TESTFLAGS ?= $(TESTFLAGS_RACE) $(EXTRA_TESTFLAGS)
 TESTFLAGS_PARALLEL ?= 8
 
+# Use this to replace `go test` with, for instance, `gotestsum`
+GOTEST ?= go test
+
 OUTPUTDIR = $(join $(ROOTDIR), _output)
 CRIDIR=$(OUTPUTDIR)/cri
 
@@ -170,15 +173,15 @@ build: ## build the go packages
 
 test: ## run tests, except integration tests and tests that require root
 	@echo "$(WHALE) $@"
-	@go test ${TESTFLAGS} ${PACKAGES}
+	@$(GOTEST) ${TESTFLAGS} ${PACKAGES}
 
 root-test: ## run tests, except integration tests
 	@echo "$(WHALE) $@"
-	@go test ${TESTFLAGS} ${TEST_REQUIRES_ROOT_PACKAGES} -test.root
+	@$(GOTEST) ${TESTFLAGS} ${TEST_REQUIRES_ROOT_PACKAGES} -test.root
 
 integration: ## run integration tests
 	@echo "$(WHALE) $@"
-	@cd "${ROOTDIR}/integration/client" && go mod download && go test -v ${TESTFLAGS} -test.root -parallel ${TESTFLAGS_PARALLEL} .
+	@cd "${ROOTDIR}/integration/client" && go mod download && $(GOTEST) -v ${TESTFLAGS} -test.root -parallel ${TESTFLAGS_PARALLEL} .
 
 # TODO integrate cri integration bucket with coverage
 bin/cri-integration.test:

Vagrantfile

@@ -173,6 +173,18 @@ EOF
     SHELL
   end
 
+  config.vm.provision "install-gotestsum", type: "shell",  run: "once" do |sh|
+      sh.upload_path = "/tmp/vagrant-install-gotestsum"
+      sh.inline = <<~SHELL
+        #!/usr/bin/env bash
+        source /etc/environment
+        source /etc/profile.d/sh.local
+        set -eux -o pipefail
+        ${GOPATH}/src/github.com/containerd/containerd/script/setup/install-gotestsum
+        sudo cp ${GOPATH}/bin/gotestsum /usr/local/bin/
+      SHELL
+  end
+
   # SELinux is Enforcing by default.
   # To set SELinux as Disabled on a VM that has already been provisioned:
   #   SELINUX=Disabled vagrant up --provision-with=selinux
@@ -196,6 +208,8 @@ EOF
     sh.upload_path = "/tmp/test-integration"
     sh.env = {
         'RUNC_FLAVOR': ENV['RUNC_FLAVOR'] || "runc",
+        'GOTEST': ENV['GOTEST'] || "go test",
+        'GOTESTSUM_JUNITFILE': ENV['GOTESTSUM_JUNITFILE'],
     }
     sh.inline = <<~SHELL
         #!/usr/bin/env bash
@@ -213,6 +227,10 @@ EOF
   #
   config.vm.provision "test-cri", type: "shell", run: "never" do |sh|
     sh.upload_path = "/tmp/test-cri"
+    sh.env = {
+        'GOTEST': ENV['GOTEST'] || "go test",
+        'REPORT_DIR': ENV['REPORT_DIR'],
+    }
     sh.inline = <<~SHELL
         #!/usr/bin/env bash
         source /etc/environment
@@ -235,7 +253,7 @@ EOF
         fi
         trap cleanup EXIT
         ctr version
-        critest --parallel=$(nproc) --ginkgo.skip='HostIpc is true'
+        critest --parallel=$(nproc) --report-dir="${REPORT_DIR}" --ginkgo.skip='HostIpc is true'
     SHELL
   end
 

script/setup/install-gotestsum

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+
+#   Copyright The containerd Authors.
+
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+
+#       http://www.apache.org/licenses/LICENSE-2.0
+
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+
+GO111MODULE=on go install gotest.tools/gotestsum@v1.6.2