Merge pull request #7192 from cpuguy83/test_summary

2022-09-26 15:28:33 -07:00
parent 5018b6aa01 9cdf9f6c6c
commit 34d078e99f
5 changed files with 201 additions and 15 deletions
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -3,16 +3,16 @@ on:
  push:
    branches:
      - main
-      - 'release/**'
+      - "release/**"
  pull_request:
    branches:
      - main
-      - 'release/**'
+      - "release/**"

 env:
  # Go version we currently use to build containerd across all CI.
  # Note: don't forget to update `Binaries` step, as it contains the matrix of all supported Go versions.
-  GO_VERSION: '1.19.1'
+  GO_VERSION: "1.19.1"

 jobs:
  #
@@ -219,7 +219,7 @@ jobs:
    strategy:
      matrix:
        os: [ubuntu-18.04, macos-12, windows-2019, windows-2022]
-        go-version: ['1.19.1', '1.18.6']
+        go-version: ["1.19.1", "1.18.6"]
    steps:
      - uses: actions/setup-go@v2
        with:
@@ -256,7 +256,7 @@ jobs:
      fail-fast: false
      matrix:
        os: [windows-2019, windows-2022]
-        enable_cri_sandboxes: [ "", "sandboxed"]
+        enable_cri_sandboxes: ["", "sandboxed"]

    defaults:
      run:
@@ -304,14 +304,14 @@ jobs:
          RESOURCE_CONSUMER_TESTING_IMAGE_REF: "registry.k8s.io/e2e-test-images/resource-consumer:1.10"
          WEBSERVER_TESTING_IMAGE_REF: "registry.k8s.io/e2e-test-images/nginx:1.14-2"
        run: |
-            cat > "${{ env.TEST_IMAGE_LIST }}" << EOF
-            busybox = "${{ env.BUSYBOX_TESTING_IMAGE_REF }}"
-            ResourceConsumer = "${{ env.RESOURCE_CONSUMER_TESTING_IMAGE_REF }}"
-            EOF
-            cat > "${{ env.CRI_TEST_IMAGES }}" << EOF
-            defaultTestContainerImage: ${{ env.BUSYBOX_TESTING_IMAGE_REF }}
-            webServerTestImage: ${{ env.WEBSERVER_TESTING_IMAGE_REF }}
-            EOF
+          cat > "${{ env.TEST_IMAGE_LIST }}" << EOF
+          busybox = "${{ env.BUSYBOX_TESTING_IMAGE_REF }}"
+          ResourceConsumer = "${{ env.RESOURCE_CONSUMER_TESTING_IMAGE_REF }}"
+          EOF
+          cat > "${{ env.CRI_TEST_IMAGES }}" << EOF
+          defaultTestContainerImage: ${{ env.BUSYBOX_TESTING_IMAGE_REF }}
+          webServerTestImage: ${{ env.WEBSERVER_TESTING_IMAGE_REF }}
+          EOF

      - name: Get crictl tool
        shell: powershell
@@ -323,18 +323,33 @@ jobs:
          mv crictl.exe "${{ github.workspace }}/bin/crictl.exe" # Move crictl somewhere in path

      - run: script/setup/install-gotestsum
+      - run: script/setup/install-teststat
      - name: Tests
        env:
          CGO_ENABLED: 1
          GOTESTSUM_JUNITFILE: ${{github.workspace}}/test-unit-root.xml
+          GOTESTSUM_JSONFILE: ${{github.workspace}}/test-unit-root-gotest.json
        run: mingw32-make.exe test root-test
+      - run: if [ -f *-gotest.json ]; then echo '# Root Test' >> $GITHUB_STEP_SUMMARY; teststat -markdown *-gotest.json >> $GITHUB_STEP_SUMMARY; fi
+        if: always()
+      - run: script/test/test2annotation.sh ${TESTFILE}
+        env:
+          TESTFILE: ${{github.workspace}}/test-unit-root-gotest.json
+        if: always()

      - name: Integration 1
        env:
          CGO_ENABLED: 1
          ENABLE_CRI_SANDBOXES: ${{ matrix.enable_cri_sandboxes }}
          GOTESTSUM_JUNITFILE: ${{github.workspace}}/test-integration-serial-junit.xml
+          GOTESTSUM_JSONFILE: ${{github.workspace}}/test-integration-serial-gotest.json
        run: mingw32-make.exe integration
+      - run: if [ -f *-gotest.json ]; then echo '# Integration 1' >> $GITHUB_STEP_SUMMARY; teststat -markdown *-gotest.json >> $GITHUB_STEP_SUMMARY; fi
+        if: always()
+      - run: script/test/test2annotation.sh ${TESTFILE}
+        env:
+          TESTFILE: ${{github.workspace}}/test-integration-serial-gotest.json
+        if: always()

      # Run the integration suite a second time. See discussion in github.com/containerd/containerd/pull/1759
      - name: Integration 2
@@ -344,7 +359,14 @@ jobs:
          CGO_ENABLED: 1
          ENABLE_CRI_SANDBOXES: ${{ matrix.enable_cri_sandboxes }}
          GOTESTSUM_JUNITFILE: ${{github.workspace}}/test-integration-parallel-junit.xml
+          GOTESTSUM_JSONFILE: ${{github.workspace}}/test-integration-parallel-gotest.json
        run: mingw32-make.exe integration
+      - run: if [ -f *-gotest.json ]; then echo '# Integration 2' >> $GITHUB_STEP_SUMMARY; teststat -markdown *-gotest.json >> $GITHUB_STEP_SUMMARY; fi
+        if: always()
+      - run: script/test/test2annotation.sh ${TESTFILE}
+        env:
+          TESTFILE: ${{github.workspace}}/test-integration-parallel-gotest.json
+        if: always()

      - name: CRI Integration Test
        env:
@@ -376,6 +398,7 @@ jobs:
          name: TestResults ${{ matrix.os }}
          path: |
            ${{github.workspace}}/*-junit.xml
+            ${{github.workspace}}/*-gotest.json

  integration-linux:
    name: Linux Integration
@@ -386,9 +409,14 @@ jobs:
    strategy:
      fail-fast: false
      matrix:
-        runtime: [io.containerd.runtime.v1.linux, io.containerd.runc.v1, io.containerd.runc.v2]
+        runtime:
+          [
+            io.containerd.runtime.v1.linux,
+            io.containerd.runc.v1,
+            io.containerd.runc.v2,
+          ]
        runc: [runc, crun]
-        enable_cri_sandboxes: [ "", "sandboxed"]
+        enable_cri_sandboxes: ["", "sandboxed"]
        exclude:
          - runtime: io.containerd.runc.v1
            runc: crun
@@ -429,12 +457,18 @@ jobs:
          sudo -E PATH=$PATH make install

      - run: script/setup/install-gotestsum
+      - run: script/setup/install-teststat
      - name: Tests
        env:
          GOTESTSUM_JUNITFILE: ${{github.workspace}}/test-unit-root-junit.xml
+          GOTESTSUM_JSONFILE: ${{github.workspace}}/test-unit-root-gotest.json
        run: |
          make test
          sudo -E PATH=$PATH make root-test
+      - run: if [ -f *-gotest.json ]; then echo '# Root Test' >> $GITHUB_STEP_SUMMARY; teststat -markdown *-gotest.json >> $GITHUB_STEP_SUMMARY; fi
+        if: always()
+      - run: script/test/test2annotation.sh *-gotest.json
+        if: always()

      - name: Integration 1
        env:
@@ -442,12 +476,17 @@ jobs:
          RUNC_FLAVOR: ${{ matrix.runc }}
          ENABLE_CRI_SANDBOXES: ${{ matrix.enable_cri_sandboxes }}
          GOTESTSUM_JUNITFILE: ${{github.workspace}}/test-integration-serial-junit.xml
+          GOTESTSUM_JSONFILE: ${{github.workspace}}/test-integration-serial-gotest.json
        run: |
          extraflags=""
          [ "${RUNC_FLAVOR}" == "crun" ] && {
                  extraflags="EXTRA_TESTFLAGS=-no-criu";
          }
          sudo -E PATH=$PATH make integration ${extraflags} TESTFLAGS_RACE=-race
+      - run: if [ -f *-gotest.json ]; then echo '# Integration 1' >> $GITHUB_STEP_SUMMARY; teststat -markdown *-gotest.json >> $GITHUB_STEP_SUMMARY; fi
+        if: always()
+      - run: script/test/test2annotation.sh *-gotest.json
+        if: always()

      # Run the integration suite a second time. See discussion in github.com/containerd/containerd/pull/1759
      - name: Integration 2
@@ -456,12 +495,17 @@ jobs:
          RUNC_FLAVOR: ${{ matrix.runc }}
          ENABLE_CRI_SANDBOXES: ${{ matrix.enable_cri_sandboxes }}
          GOTESTSUM_JUNITFILE: ${{github.workspace}}/test-integration-parallel-junit.xml
+          GOTESTSUM_JSONFILE: ${{github.workspace}}/test-integration-parallel-gotest.json
        run: |
          extraflags=""
          [ "${RUNC_FLAVOR}" == "crun" ] && {
                  extraflags="EXTRA_TESTFLAGS=-no-criu";
          }
          sudo -E PATH=$PATH TESTFLAGS_PARALLEL=1 make integration ${extraflags}
+      - run: if [ -f *-gotest.json ]; then echo '# Integration 2' >> $GITHUB_STEP_SUMMARY; teststat -markdown *-gotest.json >> $GITHUB_STEP_SUMMARY; fi
+        if: always()
+      - run: script/test/test2annotation.sh *-gotest.json
+        if: always()

      - name: CRI Integration Test
        env:
@@ -510,6 +554,7 @@ jobs:
          name: TestResults ${{ matrix.runtime }} ${{matrix.runc}}
          path: |
            *-junit.xml
+            *-gotest.json
            ${{github.workspace}}/critestreport/*.xml

  tests-mac-os:
@@ -526,13 +571,20 @@ jobs:
          go-version: ${{ env.GO_VERSION }}
      - uses: actions/checkout@v2
      - run: script/setup/install-gotestsum
+      - run: script/setup/install-teststat
      - name: Tests
        env:
          GOTESTSUM_JUNITFILE: "${{ github.workspace }}/macos-test-junit.xml"
+          GOTESTSUM_JSONFILE: "${{ github.workspace }}/macos-test-gotest.json"
        run: make test
+      - run: if [ -f *-gotest.json ]; then echo '# Unit Tests' >> $GITHUB_STEP_SUMMARY; teststat -markdown *-gotest.json >> $GITHUB_STEP_SUMMARY; fi
+        if: always()
+      - run: script/test/test2annotation.sh *-gotest.json
+        if: always()
      - uses: actions/upload-artifact@v2
        if: always()
        with:
          name: TestResults MacOS
          path: |
            *-junit.xml
+            *-gotest.json