ci: upload junit formatted test results

This allows us to dig more details out of test runs and maintain a
better history.

For this we can use `gotestsum`, which is a utility that wraps `go test`
so that it outputs test2json (go's format) and output junit (a format
more easily imported into other systems).

The PR makes it possible to override the Makefile's use of `go test` to
use any other command tto executet the test. For CI we'll use `gotestsum
--`, where `gotestsum` expects everything after the `--` to be flags for
`go test`.
We then use environment variables to configure `gotestsum` (e.g.
`GOTESTSUM_JUNITFILE` is an env var accepted by `gotestsum`).

For cri tests, the test suite supports outputing test results to a
directory, these are in junit format already. The file is not named
properly just because the code that creates it (in ginkgo) is not
configured well. We can fix that upstream to give us a better name...
until then I'm keeping those results in a separate dir.

A second workflow is also added so the test results can be summed up and
a report added to the workflow run. The 2nd workflow is required for
this since PR runs do not have access to do some of this due to safety
reasons
(https://securitylab.github.com/research/github-actions-preventing-pwn-requests/)

Signed-off-by: Brian Goff <cpuguy83@gmail.com>

.github/workflows/ci.yml

@@ -247,6 +247,8 @@ jobs:
     runs-on: windows-2019
     timeout-minutes: 30
     needs: [project, linters, protos, man]
+    env:
+      GOTEST: gotestsum --
 
     defaults:
       run:
@@ -270,6 +272,7 @@ jobs:
       - name: Set env
         run: |
           echo "GOPATH=${{ github.workspace }}" >> $GITHUB_ENV
+          echo "${{ github.workspace }}/bin" >> $GITHUB_PATH
           echo "${{ github.workspace }}/src/github.com/containerd/containerd/bin" >> $GITHUB_PATH
 
       - run: script/setup/install-dev-tools
@@ -287,22 +290,32 @@ jobs:
           git checkout "${SHIM_COMMIT}"
           GO111MODULE=on go build -mod=vendor -o "${bindir}/integration/client/containerd-shim-runhcs-v1.exe" ./cmd/containerd-shim-runhcs-v1
 
+      - run: script/setup/install-gotestsum
       - name: Tests
         env:
           CGO_ENABLED: 1
+          GOTESTSUM_JUNITFILE: ${{github.workspace}}/test-unit-root.xml
         run: mingw32-make.exe test root-test
 
       - name: Integration 1
         env:
           CGO_ENABLED: 1
+          GOTESTSUM_JUNITFILE: ${{github.workspace}}/test-integration-serial-junit.xml
         run: mingw32-make.exe integration
 
-      # Run the integration suite a second time. See discussion in github.com/containerd/containerd/pull/175
+      # Run the integration suite a second time. See discussion in github.com/containerd/containerd/pull/1759
       - name: Integration 2
         env:
           TESTFLAGS_PARALLEL: 1
           CGO_ENABLED: 1
+          GOTESTSUM_JUNITFILE: ${{github.workspace}}/test-integration-parallel-junit.xml
         run: mingw32-make.exe integration
+      - uses: actions/upload-artifact@v2
+        if: always()
+        with:
+          name: TestResults Windows
+          path: |
+            ${{github.workspace}}/*-junit.xml
 
   integration-linux:
     name: Linux Integration
@@ -321,6 +334,8 @@ jobs:
           - runtime: io.containerd.runtime.v1.linux
             runc: crun
 
+    env:
+      GOTEST: gotestsum --
     steps:
       - uses: actions/setup-go@v2
         with:
@@ -370,9 +385,12 @@ jobs:
           sudo make install
         working-directory: src/github.com/containerd/containerd
 
+      - run: sudo -E PATH=$PATH script/setup/install-gotestsum
+        working-directory: src/github.com/containerd/containerd
       - name: Tests
         env:
           GOPROXY: direct
+          GOTESTSUM_JUNITFILE: ${{github.workspace}}/test-unit-root-junit.xml
         run: |
           make test
           sudo -E PATH=$PATH GOPATH=$GOPATH GOPROXY=$GOPROXY make root-test
@@ -383,6 +401,7 @@ jobs:
           GOPROXY: direct
           TEST_RUNTIME: ${{ matrix.runtime }}
           RUNC_FLAVOR: ${{ matrix.runc }}
+          GOTESTSUM_JUNITFILE: ${{github.workspace}}/test-integration-serial-junit.xml
         run: |
           sudo GOPATH=$GOPATH GOPROXY=$GOPROXY TEST_RUNTIME=$TEST_RUNTIME RUNC_FLAVOR=$RUNC_FLAVOR make integration EXTRA_TESTFLAGS=-no-criu TESTFLAGS_RACE=-race
         working-directory: src/github.com/containerd/containerd
@@ -393,6 +412,7 @@ jobs:
           GOPROXY: direct
           TEST_RUNTIME: ${{ matrix.runtime }}
           RUNC_FLAVOR: ${{ matrix.runc }}
+          GOTESTSUM_JUNITFILE: ${{github.workspace}}/test-integration-parallel-junit.xml
         run: |
           sudo GOPATH=$GOPATH GOPROXY=$GOPROXY TEST_RUNTIME=$TEST_RUNTIME RUNC_FLAVOR=$RUNC_FLAVOR TESTFLAGS_PARALLEL=1 make integration EXTRA_TESTFLAGS=-no-criu
         working-directory: src/github.com/containerd/containerd
@@ -404,6 +424,7 @@ jobs:
           GOPROXY: direct
           TEST_RUNTIME: ${{ matrix.runtime }}
           RUNC_FLAVOR: ${{ matrix.runc }}
+          GOTESTSUM_JUNITFILE: ${{github.workspace}}/test-integration-criu-junit.xml
         # crun doesn't have "checkpoint" command.
         if: ${{ matrix.runc == 'runc' }}
         run: |
@@ -433,7 +454,7 @@ jobs:
           sudo ls /etc/cni/net.d
           sudo PATH=$PATH BDIR=$BDIR /usr/local/bin/containerd -a ${BDIR}/c.sock --config ${BDIR}/config.toml --root ${BDIR}/root --state ${BDIR}/state --log-level debug &> ${BDIR}/containerd-cri.log &
           sudo PATH=$PATH BDIR=$BDIR /usr/local/bin/ctr -a ${BDIR}/c.sock version
-          sudo PATH=$PATH BDIR=$BDIR GOPATH=$GOPATH critest --runtime-endpoint=unix:///${BDIR}/c.sock --parallel=8
+          sudo PATH=$PATH BDIR=$BDIR GOPATH=$GOPATH critest --report-dir "${{github.workspace}}/critestreport" --runtime-endpoint=unix:///${BDIR}/c.sock --parallel=8
           TEST_RC=$?
           test $TEST_RC -ne 0 && cat ${BDIR}/containerd-cri.log
           sudo pkill containerd
@@ -449,12 +470,21 @@ jobs:
           mount
           df
           losetup -l
+      - uses: actions/upload-artifact@v2
+        if: always()
+        with:
+          name: TestResults ${{ matrix.runtime }} ${{matrix.runc}}
+          path: |
+            *-junit.xml
+            ${{github.workspace}}/critestreport/*.xml
 
   tests-mac-os:
     name: MacOS unit tests
     runs-on: macos-10.15
     timeout-minutes: 10
     needs: [project, linters, protos, man]
+    env:
+      GOTEST: gotestsum --
 
     steps:
       - uses: actions/setup-go@v2
@@ -470,12 +500,21 @@ jobs:
           echo "GOPATH=${{ github.workspace }}" >> $GITHUB_ENV
           echo "${{ github.workspace }}/bin" >> $GITHUB_PATH
 
+      - run: sudo -E PATH=$PATH script/setup/install-gotestsum
+        working-directory: src/github.com/containerd/containerd
       - name: Tests
         env:
           GOPROXY: direct
+          GOTESTSUM_JUNITFILE: "${{ github.workspace }}/macos-test-junit.xml"
         run: |
           make test
         working-directory: src/github.com/containerd/containerd
+      - uses: actions/upload-artifact@v2
+        if: always()
+        with:
+          name: TestResults MacOS
+          path: |
+            *-junit.xml
 
   cgroup2:
     name: CGroupsV2 and SELinux Integration
@@ -488,6 +527,8 @@ jobs:
         # Currently crun is disabled to decrease CI flakiness.
         # We can enable crun again when we get a better CI infra.
         runc: [runc]
+    env:
+      GOTEST: gotestsum --
     steps:
       - uses: actions/checkout@v2
 
@@ -506,10 +547,27 @@ jobs:
         env:
           RUNC_FLAVOR: ${{ matrix.runc }}
           SELINUX: Enforcing
-        run: vagrant up --provision-with=selinux,install-runc,test-integration
+          GOTESTSUM_JUNITFILE: /tmp/test-integration-junit.xml
+        run: vagrant up --provision-with=selinux,install-runc,install-gotestsum,test-integration
 
       - name: CRI test
         env:
           RUNC_FLAVOR: ${{ matrix.runc }}
           SELINUX: Enforcing
-        run: vagrant up --provision-with=selinux,install-runc,test-cri
+          REPORT_DIR: /tmp/critestreport
+        run: vagrant up --provision-with=selinux,install-runc,install-gotestsum,test-cri
+      - name: Get test reports
+        if: always()
+        run: |
+          set -e
+          vagrant plugin install vagrant-vbguest
+          vagrant plugin install vagrant-scp
+          vagrant scp :/tmp/test-integration-junit.xml "${{ github.workspace }}/"
+          vagrant scp :/tmp/critestreport "${{ github.workspace }}/critestreport"
+      - uses: actions/upload-artifact@v2
+        if: always()
+        with:
+          name:  TestResults cgroup2 ${{ matrix.runtime }} ${{matrix.runc}}
+          path: |
+            ${{github.workspace}}/*-junit.xml
+            ${{github.workspace}}/critestreport/*

.github/workflows/publish_test_results.yml

@@ -0,0 +1,61 @@
+name: Publish Test Results
+
+on:
+   workflow_run:
+      workflows: ["CI"]
+      types:
+         - completed
+
+jobs:
+  test-results:
+    runs-on: ubuntu-latest
+    name: Publish Test Results
+    if: >
+        github.event.workflow_run.conclusion != 'skipped' && (
+          github.event.workflow_run.head_repository.full_name != github.repository
+        )
+    steps:
+      - name: Download Artifacts
+        uses: actions/github-script@v3.1.0
+        with:
+          script: |
+              var fs = require('fs');
+              var path = require('path');
+              var artifacts_path = path.join('${{github.workspace}}', 'artifacts')
+              fs.mkdirSync(artifacts_path, { recursive: true })
+
+              var artifacts = await github.actions.listWorkflowRunArtifacts({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                run_id: ${{ github.event.workflow_run.id }},
+              });
+
+              for (const artifact of artifacts.data.artifacts) {
+                var download = await github.actions.downloadArtifact({
+                    owner: context.repo.owner,
+                    repo: context.repo.repo,
+                    artifact_id: artifact.id,
+                    archive_format: 'zip',
+                });
+                var artifact_path = path.join(artifacts_path, `${artifact.name}.zip`)
+                fs.writeFileSync(artifact_path, Buffer.from(download.data));
+                console.log(`Downloaded ${artifact_path}`);
+              }
+      - name: Extract Artifacts
+        run: |
+          for file in artifacts/*.zip
+          do
+            if [ -f "$file" ]
+            then
+              dir="${file/%.zip/}"
+              mkdir -p "$dir"
+              unzip -d "$dir" "$file"
+            fi
+          done
+      - name: Publish Test Results
+        uses: EnricoMi/publish-unit-test-result-action@v1
+        with:
+          files: artifacts/*/**/*.xml
+          check_name: Test Results
+          comment_on_pr: true
+          commit: ${{ github.event.workflow_run.head_sha }}

Makefile

@@ -120,6 +120,9 @@ BINARIES=$(addprefix bin/,$(COMMANDS))
 TESTFLAGS ?= $(TESTFLAGS_RACE) $(EXTRA_TESTFLAGS)
 TESTFLAGS_PARALLEL ?= 8
 
+# Use this to replace `go test` with, for instance, `gotestsum`
+GOTEST ?= go test
+
 OUTPUTDIR = $(join $(ROOTDIR), _output)
 CRIDIR=$(OUTPUTDIR)/cri
 
@@ -170,15 +173,15 @@ build: ## build the go packages
 
 test: ## run tests, except integration tests and tests that require root
 	@echo "$(WHALE) $@"
-	@go test ${TESTFLAGS} ${PACKAGES}
+	@$(GOTEST) ${TESTFLAGS} ${PACKAGES}
 
 root-test: ## run tests, except integration tests
 	@echo "$(WHALE) $@"
-	@go test ${TESTFLAGS} ${TEST_REQUIRES_ROOT_PACKAGES} -test.root
+	@$(GOTEST) ${TESTFLAGS} ${TEST_REQUIRES_ROOT_PACKAGES} -test.root
 
 integration: ## run integration tests
 	@echo "$(WHALE) $@"
-	@cd "${ROOTDIR}/integration/client" && go mod download && go test -v ${TESTFLAGS} -test.root -parallel ${TESTFLAGS_PARALLEL} .
+	@cd "${ROOTDIR}/integration/client" && go mod download && $(GOTEST) -v ${TESTFLAGS} -test.root -parallel ${TESTFLAGS_PARALLEL} .
 
 # TODO integrate cri integration bucket with coverage
 bin/cri-integration.test:

Vagrantfile

@@ -173,6 +173,18 @@ EOF
     SHELL
   end
 
+  config.vm.provision "install-gotestsum", type: "shell",  run: "once" do |sh|
+      sh.upload_path = "/tmp/vagrant-install-gotestsum"
+      sh.inline = <<~SHELL
+        #!/usr/bin/env bash
+        source /etc/environment
+        source /etc/profile.d/sh.local
+        set -eux -o pipefail
+        ${GOPATH}/src/github.com/containerd/containerd/script/setup/install-gotestsum
+        sudo cp ${GOPATH}/bin/gotestsum /usr/local/bin/
+      SHELL
+  end
+
   # SELinux is Enforcing by default.
   # To set SELinux as Disabled on a VM that has already been provisioned:
   #   SELINUX=Disabled vagrant up --provision-with=selinux
@@ -196,6 +208,8 @@ EOF
     sh.upload_path = "/tmp/test-integration"
     sh.env = {
         'RUNC_FLAVOR': ENV['RUNC_FLAVOR'] || "runc",
+        'GOTEST': ENV['GOTEST'] || "go test",
+        'GOTESTSUM_JUNITFILE': ENV['GOTESTSUM_JUNITFILE'],
     }
     sh.inline = <<~SHELL
         #!/usr/bin/env bash
@@ -213,6 +227,10 @@ EOF
   #
   config.vm.provision "test-cri", type: "shell", run: "never" do |sh|
     sh.upload_path = "/tmp/test-cri"
+    sh.env = {
+        'GOTEST': ENV['GOTEST'] || "go test",
+        'REPORT_DIR': ENV['REPORT_DIR'],
+    }
     sh.inline = <<~SHELL
         #!/usr/bin/env bash
         source /etc/environment
@@ -235,7 +253,7 @@ EOF
         fi
         trap cleanup EXIT
         ctr version
-        critest --parallel=$(nproc) --ginkgo.skip='HostIpc is true'
+        critest --parallel=$(nproc) --report-dir="${REPORT_DIR}" --ginkgo.skip='HostIpc is true'
     SHELL
   end
 

script/setup/install-gotestsum

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+
+#   Copyright The containerd Authors.
+
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+
+#       http://www.apache.org/licenses/LICENSE-2.0
+
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+
+GO111MODULE=on go install gotest.tools/gotestsum@v1.6.2