CI: Output a summary using GITHUB_SUMMARY

Uses teststat to parse the go test json and output markdown which will
be posted as a summary to the github action run.

Signed-off-by: Brian Goff <cpuguy83@gmail.com>

.github/workflows/ci.yml

@@ -3,16 +3,16 @@ on:
   push:
     branches:
       - main
-      - 'release/**'
+      - "release/**"
   pull_request:
     branches:
       - main
-      - 'release/**'
+      - "release/**"
 
 env:
   # Go version we currently use to build containerd across all CI.
   # Note: don't forget to update `Binaries` step, as it contains the matrix of all supported Go versions.
-  GO_VERSION: '1.19.1'
+  GO_VERSION: "1.19.1"
 
 jobs:
   #
@@ -219,7 +219,7 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-18.04, macos-12, windows-2019, windows-2022]
-        go-version: ['1.19.1', '1.18.6']
+        go-version: ["1.19.1", "1.18.6"]
     steps:
       - uses: actions/setup-go@v2
         with:
@@ -256,7 +256,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [windows-2019, windows-2022]
-        enable_cri_sandboxes: [ "", "sandboxed"]
+        enable_cri_sandboxes: ["", "sandboxed"]
 
     defaults:
       run:
@@ -304,14 +304,14 @@ jobs:
           RESOURCE_CONSUMER_TESTING_IMAGE_REF: "registry.k8s.io/e2e-test-images/resource-consumer:1.10"
           WEBSERVER_TESTING_IMAGE_REF: "registry.k8s.io/e2e-test-images/nginx:1.14-2"
         run: |
-            cat > "${{ env.TEST_IMAGE_LIST }}" << EOF
+          cat > "${{ env.TEST_IMAGE_LIST }}" << EOF
-            busybox = "${{ env.BUSYBOX_TESTING_IMAGE_REF }}"
+          busybox = "${{ env.BUSYBOX_TESTING_IMAGE_REF }}"
-            ResourceConsumer = "${{ env.RESOURCE_CONSUMER_TESTING_IMAGE_REF }}"
+          ResourceConsumer = "${{ env.RESOURCE_CONSUMER_TESTING_IMAGE_REF }}"
-            EOF
+          EOF
-            cat > "${{ env.CRI_TEST_IMAGES }}" << EOF
+          cat > "${{ env.CRI_TEST_IMAGES }}" << EOF
-            defaultTestContainerImage: ${{ env.BUSYBOX_TESTING_IMAGE_REF }}
+          defaultTestContainerImage: ${{ env.BUSYBOX_TESTING_IMAGE_REF }}
-            webServerTestImage: ${{ env.WEBSERVER_TESTING_IMAGE_REF }}
+          webServerTestImage: ${{ env.WEBSERVER_TESTING_IMAGE_REF }}
-            EOF
+          EOF
 
       - name: Get crictl tool
         shell: powershell
@@ -323,18 +323,29 @@ jobs:
           mv crictl.exe "${{ github.workspace }}/bin/crictl.exe" # Move crictl somewhere in path
 
       - run: script/setup/install-gotestsum
+      - run: script/setup/install-teststat
       - name: Tests
         env:
           CGO_ENABLED: 1
           GOTESTSUM_JUNITFILE: ${{github.workspace}}/test-unit-root.xml
+          GOTESTSUM_JSONFILE: ${{github.workspace}}/test-unit-root.json
         run: mingw32-make.exe test root-test
+      - run: echo '# Root Test' >> $GITHUB_STEP_SUMMARY; teststat -markdown ${TESTFILE} >> $GITHUB_STEP_SUMMARY
+        if: always()
+        env:
+          TESTFILE: ${{github.workspace}}/test-unit-root.json
 
       - name: Integration 1
         env:
           CGO_ENABLED: 1
           ENABLE_CRI_SANDBOXES: ${{ matrix.enable_cri_sandboxes }}
           GOTESTSUM_JUNITFILE: ${{github.workspace}}/test-integration-serial-junit.xml
+          GOTESTSUM_JSONFILE: ${{github.workspace}}/test-integration-serial.json
         run: mingw32-make.exe integration
+      - run: echo '# Integration 1' >> $GITHUB_STEP_SUMMARY; teststat -markdown ${TESTFILE} >> $GITHUB_STEP_SUMMARY
+        if: always()
+        env:
+          TESTFILE: ${{github.workspace}}/test-integration-serial.json
 
       # Run the integration suite a second time. See discussion in github.com/containerd/containerd/pull/1759
       - name: Integration 2
@@ -344,7 +355,12 @@ jobs:
           CGO_ENABLED: 1
           ENABLE_CRI_SANDBOXES: ${{ matrix.enable_cri_sandboxes }}
           GOTESTSUM_JUNITFILE: ${{github.workspace}}/test-integration-parallel-junit.xml
+          GOTESTSUM_JSONFILE: ${{github.workspace}}/test-integration-parallel.json
         run: mingw32-make.exe integration
+      - run: echo '# Integration 2' >> $GITHUB_STEP_SUMMARY; teststat -markdown ${TESTFILE} >> $GITHUB_STEP_SUMMARY
+        if: always()
+        env:
+          TESTFILE: ${{github.workspace}}/test-integration-parallel.json
 
       - name: CRI Integration Test
         env:
@@ -386,9 +402,14 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        runtime: [io.containerd.runtime.v1.linux, io.containerd.runc.v1, io.containerd.runc.v2]
+        runtime:
+          [
+            io.containerd.runtime.v1.linux,
+            io.containerd.runc.v1,
+            io.containerd.runc.v2,
+          ]
         runc: [runc, crun]
-        enable_cri_sandboxes: [ "", "sandboxed"]
+        enable_cri_sandboxes: ["", "sandboxed"]
         exclude:
           - runtime: io.containerd.runc.v1
             runc: crun
@@ -429,12 +450,18 @@ jobs:
           sudo -E PATH=$PATH make install
 
       - run: script/setup/install-gotestsum
+      - run: script/setup/install-teststat
       - name: Tests
         env:
           GOTESTSUM_JUNITFILE: ${{github.workspace}}/test-unit-root-junit.xml
+          GOTESTSUM_JSONFILE: ${{github.workspace}}/test-unit-root.json
         run: |
           make test
           sudo -E PATH=$PATH make root-test
+      - run: echo '# Root Test' >> $GITHUB_STEP_SUMMARY; teststat -markdown ${TESTFILE} >> $GITHUB_STEP_SUMMARY
+        if: always()
+        env:
+          TESTFILE: ${{github.workspace}}/test-unit-root.json
 
       - name: Integration 1
         env:
@@ -442,12 +469,17 @@ jobs:
           RUNC_FLAVOR: ${{ matrix.runc }}
           ENABLE_CRI_SANDBOXES: ${{ matrix.enable_cri_sandboxes }}
           GOTESTSUM_JUNITFILE: ${{github.workspace}}/test-integration-serial-junit.xml
+          GOTESTSUM_JSONFILE: ${{github.workspace}}/test-integration-serial.json
         run: |
           extraflags=""
           [ "${RUNC_FLAVOR}" == "crun" ] && {
                   extraflags="EXTRA_TESTFLAGS=-no-criu";
           }
           sudo -E PATH=$PATH make integration ${extraflags} TESTFLAGS_RACE=-race
+      - run: echo '# Integration 1' >> $GITHUB_STEP_SUMMARY; teststat -markdown ${TESTFILE} >> $GITHUB_STEP_SUMMARY
+        if: always()
+        env:
+          TESTFILE: ${{github.workspace}}/test-integration-serial.json
 
       # Run the integration suite a second time. See discussion in github.com/containerd/containerd/pull/1759
       - name: Integration 2
@@ -456,12 +488,17 @@ jobs:
           RUNC_FLAVOR: ${{ matrix.runc }}
           ENABLE_CRI_SANDBOXES: ${{ matrix.enable_cri_sandboxes }}
           GOTESTSUM_JUNITFILE: ${{github.workspace}}/test-integration-parallel-junit.xml
+          GOTESTSUM_JSONFILE: ${{github.workspace}}/test-integration-parallel.json
         run: |
           extraflags=""
           [ "${RUNC_FLAVOR}" == "crun" ] && {
                   extraflags="EXTRA_TESTFLAGS=-no-criu";
           }
           sudo -E PATH=$PATH TESTFLAGS_PARALLEL=1 make integration ${extraflags}
+      - run: echo '# Integration 2' >> $GITHUB_STEP_SUMMARY; teststat -markdown ${TESTFILE} >> $GITHUB_STEP_SUMMARY
+        if: always()
+        env:
+          TESTFILE: ${{github.workspace}}/test-integration-parallel.json
 
       - name: CRI Integration Test
         env:
@@ -529,7 +566,12 @@ jobs:
       - name: Tests
         env:
           GOTESTSUM_JUNITFILE: "${{ github.workspace }}/macos-test-junit.xml"
+          GOTESTSUM_JSONFILE: "${{ github.workspace }}/macos-test.json"
         run: make test
+      - run: script/setup/install-teststat && teststat -markdown ${TESTFILE} >> $GITHUB_STEP_SUMMARY
+        if: always()
+        env:
+          TESTFILE: "${{ github.workspace }}/macos-test.json"
       - uses: actions/upload-artifact@v2
         if: always()
         with:

Vagrantfile

@@ -227,6 +227,7 @@ EOF
         'RUNC_FLAVOR': ENV['RUNC_FLAVOR'] || "runc",
         'GOTEST': ENV['GOTEST'] || "go test",
         'GOTESTSUM_JUNITFILE': ENV['GOTESTSUM_JUNITFILE'],
+        'GOTESTSUM_JSONFILE': ENV['GOTESTSUM_JSONFILE'],
     }
     sh.inline = <<~SHELL
         #!/usr/bin/env bash

script/setup/install-teststat

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+
+#   Copyright The containerd Authors.
+
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+
+#       http://www.apache.org/licenses/LICENSE-2.0
+
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+
+GO111MODULE=on go install github.com/vearutop/teststat@v0.1.3