ci: upload junit formatted test results
This allows us to dig more details out of test runs and maintain a better history. For this we can use `gotestsum`, which is a utility that wraps `go test` so that it outputs test2json (go's format) and output junit (a format more easily imported into other systems). The PR makes it possible to override the Makefile's use of `go test` to use any other command tto executet the test. For CI we'll use `gotestsum --`, where `gotestsum` expects everything after the `--` to be flags for `go test`. We then use environment variables to configure `gotestsum` (e.g. `GOTESTSUM_JUNITFILE` is an env var accepted by `gotestsum`). For cri tests, the test suite supports outputing test results to a directory, these are in junit format already. The file is not named properly just because the code that creates it (in ginkgo) is not configured well. We can fix that upstream to give us a better name... until then I'm keeping those results in a separate dir. A second workflow is also added so the test results can be summed up and a report added to the workflow run. The 2nd workflow is required for this since PR runs do not have access to do some of this due to safety reasons (https://securitylab.github.com/research/github-actions-preventing-pwn-requests/) Signed-off-by: Brian Goff <cpuguy83@gmail.com>
This commit is contained in:
Brian Goff
parent
1c05317a47
commit
af1e2af720
66
.github/workflows/ci.yml
vendored
66
.github/workflows/ci.yml
vendored
@ -247,6 +247,8 @@ jobs:
|
||||
runs-on: windows-2019
|
||||
timeout-minutes: 30
|
||||
needs: [project, linters, protos, man]
|
||||
env:
|
||||
GOTEST: gotestsum --
|
||||
|
||||
defaults:
|
||||
run:
|
||||
@ -270,6 +272,7 @@ jobs:
|
||||
- name: Set env
|
||||
run: |
|
||||
echo "GOPATH=${{ github.workspace }}" >> $GITHUB_ENV
|
||||
echo "${{ github.workspace }}/bin" >> $GITHUB_PATH
|
||||
echo "${{ github.workspace }}/src/github.com/containerd/containerd/bin" >> $GITHUB_PATH
|
||||
|
||||
- run: script/setup/install-dev-tools
|
||||
@ -287,22 +290,32 @@ jobs:
|
||||
git checkout "${SHIM_COMMIT}"
|
||||
GO111MODULE=on go build -mod=vendor -o "${bindir}/integration/client/containerd-shim-runhcs-v1.exe" ./cmd/containerd-shim-runhcs-v1
|
||||
|
||||
- run: script/setup/install-gotestsum
|
||||
- name: Tests
|
||||
env:
|
||||
CGO_ENABLED: 1
|
||||
GOTESTSUM_JUNITFILE: ${{github.workspace}}/test-unit-root.xml
|
||||
run: mingw32-make.exe test root-test
|
||||
|
||||
- name: Integration 1
|
||||
env:
|
||||
CGO_ENABLED: 1
|
||||
GOTESTSUM_JUNITFILE: ${{github.workspace}}/test-integration-serial-junit.xml
|
||||
run: mingw32-make.exe integration
|
||||
|
||||
# Run the integration suite a second time. See discussion in github.com/containerd/containerd/pull/175
|
||||
# Run the integration suite a second time. See discussion in github.com/containerd/containerd/pull/1759
|
||||
- name: Integration 2
|
||||
env:
|
||||
TESTFLAGS_PARALLEL: 1
|
||||
CGO_ENABLED: 1
|
||||
GOTESTSUM_JUNITFILE: ${{github.workspace}}/test-integration-parallel-junit.xml
|
||||
run: mingw32-make.exe integration
|
||||
- uses: actions/upload-artifact@v2
|
||||
if: always()
|
||||
with:
|
||||
name: TestResults Windows
|
||||
path: |
|
||||
${{github.workspace}}/*-junit.xml
|
||||
|
||||
integration-linux:
|
||||
name: Linux Integration
|
||||
@ -321,6 +334,8 @@ jobs:
|
||||
- runtime: io.containerd.runtime.v1.linux
|
||||
runc: crun
|
||||
|
||||
env:
|
||||
GOTEST: gotestsum --
|
||||
steps:
|
||||
- uses: actions/setup-go@v2
|
||||
with:
|
||||
@ -370,9 +385,12 @@ jobs:
|
||||
sudo make install
|
||||
working-directory: src/github.com/containerd/containerd
|
||||
|
||||
- run: sudo -E PATH=$PATH script/setup/install-gotestsum
|
||||
working-directory: src/github.com/containerd/containerd
|
||||
- name: Tests
|
||||
env:
|
||||
GOPROXY: direct
|
||||
GOTESTSUM_JUNITFILE: ${{github.workspace}}/test-unit-root-junit.xml
|
||||
run: |
|
||||
make test
|
||||
sudo -E PATH=$PATH GOPATH=$GOPATH GOPROXY=$GOPROXY make root-test
|
||||
@ -383,6 +401,7 @@ jobs:
|
||||
GOPROXY: direct
|
||||
TEST_RUNTIME: ${{ matrix.runtime }}
|
||||
RUNC_FLAVOR: ${{ matrix.runc }}
|
||||
GOTESTSUM_JUNITFILE: ${{github.workspace}}/test-integration-serial-junit.xml
|
||||
run: |
|
||||
sudo GOPATH=$GOPATH GOPROXY=$GOPROXY TEST_RUNTIME=$TEST_RUNTIME RUNC_FLAVOR=$RUNC_FLAVOR make integration EXTRA_TESTFLAGS=-no-criu TESTFLAGS_RACE=-race
|
||||
working-directory: src/github.com/containerd/containerd
|
||||
@ -393,6 +412,7 @@ jobs:
|
||||
GOPROXY: direct
|
||||
TEST_RUNTIME: ${{ matrix.runtime }}
|
||||
RUNC_FLAVOR: ${{ matrix.runc }}
|
||||
GOTESTSUM_JUNITFILE: ${{github.workspace}}/test-integration-parallel-junit.xml
|
||||
run: |
|
||||
sudo GOPATH=$GOPATH GOPROXY=$GOPROXY TEST_RUNTIME=$TEST_RUNTIME RUNC_FLAVOR=$RUNC_FLAVOR TESTFLAGS_PARALLEL=1 make integration EXTRA_TESTFLAGS=-no-criu
|
||||
working-directory: src/github.com/containerd/containerd
|
||||
@ -404,6 +424,7 @@ jobs:
|
||||
GOPROXY: direct
|
||||
TEST_RUNTIME: ${{ matrix.runtime }}
|
||||
RUNC_FLAVOR: ${{ matrix.runc }}
|
||||
GOTESTSUM_JUNITFILE: ${{github.workspace}}/test-integration-criu-junit.xml
|
||||
# crun doesn't have "checkpoint" command.
|
||||
if: ${{ matrix.runc == 'runc' }}
|
||||
run: |
|
||||
@ -433,7 +454,7 @@ jobs:
|
||||
sudo ls /etc/cni/net.d
|
||||
sudo PATH=$PATH BDIR=$BDIR /usr/local/bin/containerd -a ${BDIR}/c.sock --config ${BDIR}/config.toml --root ${BDIR}/root --state ${BDIR}/state --log-level debug &> ${BDIR}/containerd-cri.log &
|
||||
sudo PATH=$PATH BDIR=$BDIR /usr/local/bin/ctr -a ${BDIR}/c.sock version
|
||||
sudo PATH=$PATH BDIR=$BDIR GOPATH=$GOPATH critest --runtime-endpoint=unix:///${BDIR}/c.sock --parallel=8
|
||||
sudo PATH=$PATH BDIR=$BDIR GOPATH=$GOPATH critest --report-dir "${{github.workspace}}/critestreport" --runtime-endpoint=unix:///${BDIR}/c.sock --parallel=8
|
||||
TEST_RC=$?
|
||||
test $TEST_RC -ne 0 && cat ${BDIR}/containerd-cri.log
|
||||
sudo pkill containerd
|
||||
@ -449,12 +470,21 @@ jobs:
|
||||
mount
|
||||
df
|
||||
losetup -l
|
||||
- uses: actions/upload-artifact@v2
|
||||
if: always()
|
||||
with:
|
||||
name: TestResults ${{ matrix.runtime }} ${{matrix.runc}}
|
||||
path: |
|
||||
*-junit.xml
|
||||
${{github.workspace}}/critestreport/*.xml
|
||||
|
||||
tests-mac-os:
|
||||
name: MacOS unit tests
|
||||
runs-on: macos-10.15
|
||||
timeout-minutes: 10
|
||||
needs: [project, linters, protos, man]
|
||||
env:
|
||||
GOTEST: gotestsum --
|
||||
|
||||
steps:
|
||||
- uses: actions/setup-go@v2
|
||||
@ -470,12 +500,21 @@ jobs:
|
||||
echo "GOPATH=${{ github.workspace }}" >> $GITHUB_ENV
|
||||
echo "${{ github.workspace }}/bin" >> $GITHUB_PATH
|
||||
|
||||
- run: sudo -E PATH=$PATH script/setup/install-gotestsum
|
||||
working-directory: src/github.com/containerd/containerd
|
||||
- name: Tests
|
||||
env:
|
||||
GOPROXY: direct
|
||||
GOTESTSUM_JUNITFILE: "${{ github.workspace }}/macos-test-junit.xml"
|
||||
run: |
|
||||
make test
|
||||
working-directory: src/github.com/containerd/containerd
|
||||
- uses: actions/upload-artifact@v2
|
||||
if: always()
|
||||
with:
|
||||
name: TestResults MacOS
|
||||
path: |
|
||||
*-junit.xml
|
||||
|
||||
cgroup2:
|
||||
name: CGroupsV2 and SELinux Integration
|
||||
@ -488,6 +527,8 @@ jobs:
|
||||
# Currently crun is disabled to decrease CI flakiness.
|
||||
# We can enable crun again when we get a better CI infra.
|
||||
runc: [runc]
|
||||
env:
|
||||
GOTEST: gotestsum --
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
|
||||
@ -506,10 +547,27 @@ jobs:
|
||||
env:
|
||||
RUNC_FLAVOR: ${{ matrix.runc }}
|
||||
SELINUX: Enforcing
|
||||
run: vagrant up --provision-with=selinux,install-runc,test-integration
|
||||
GOTESTSUM_JUNITFILE: /tmp/test-integration-junit.xml
|
||||
run: vagrant up --provision-with=selinux,install-runc,install-gotestsum,test-integration
|
||||
|
||||
- name: CRI test
|
||||
env:
|
||||
RUNC_FLAVOR: ${{ matrix.runc }}
|
||||
SELINUX: Enforcing
|
||||
run: vagrant up --provision-with=selinux,install-runc,test-cri
|
||||
REPORT_DIR: /tmp/critestreport
|
||||
run: vagrant up --provision-with=selinux,install-runc,install-gotestsum,test-cri
|
||||
- name: Get test reports
|
||||
if: always()
|
||||
run: |
|
||||
set -e
|
||||
vagrant plugin install vagrant-vbguest
|
||||
vagrant plugin install vagrant-scp
|
||||
vagrant scp :/tmp/test-integration-junit.xml "${{ github.workspace }}/"
|
||||
vagrant scp :/tmp/critestreport "${{ github.workspace }}/critestreport"
|
||||
- uses: actions/upload-artifact@v2
|
||||
if: always()
|
||||
with:
|
||||
name: TestResults cgroup2 ${{ matrix.runtime }} ${{matrix.runc}}
|
||||
path: |
|
||||
${{github.workspace}}/*-junit.xml
|
||||
${{github.workspace}}/critestreport/*
|
||||
61
.github/workflows/publish_test_results.yml
vendored
Normal file
61
.github/workflows/publish_test_results.yml
vendored
Normal file
@ -0,0 +1,61 @@
|
||||
name: Publish Test Results
|
||||
|
||||
on:
|
||||
workflow_run:
|
||||
workflows: ["CI"]
|
||||
types:
|
||||
- completed
|
||||
|
||||
jobs:
|
||||
test-results:
|
||||
runs-on: ubuntu-latest
|
||||
name: Publish Test Results
|
||||
if: >
|
||||
github.event.workflow_run.conclusion != 'skipped' && (
|
||||
github.event.workflow_run.head_repository.full_name != github.repository
|
||||
)
|
||||
steps:
|
||||
- name: Download Artifacts
|
||||
uses: actions/github-script@v3.1.0
|
||||
with:
|
||||
script: |
|
||||
var fs = require('fs');
|
||||
var path = require('path');
|
||||
var artifacts_path = path.join('${{github.workspace}}', 'artifacts')
|
||||
fs.mkdirSync(artifacts_path, { recursive: true })
|
||||
|
||||
var artifacts = await github.actions.listWorkflowRunArtifacts({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
run_id: ${{ github.event.workflow_run.id }},
|
||||
});
|
||||
|
||||
for (const artifact of artifacts.data.artifacts) {
|
||||
var download = await github.actions.downloadArtifact({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
artifact_id: artifact.id,
|
||||
archive_format: 'zip',
|
||||
});
|
||||
var artifact_path = path.join(artifacts_path, `${artifact.name}.zip`)
|
||||
fs.writeFileSync(artifact_path, Buffer.from(download.data));
|
||||
console.log(`Downloaded ${artifact_path}`);
|
||||
}
|
||||
- name: Extract Artifacts
|
||||
run: |
|
||||
for file in artifacts/*.zip
|
||||
do
|
||||
if [ -f "$file" ]
|
||||
then
|
||||
dir="${file/%.zip/}"
|
||||
mkdir -p "$dir"
|
||||
unzip -d "$dir" "$file"
|
||||
fi
|
||||
done
|
||||
- name: Publish Test Results
|
||||
uses: EnricoMi/publish-unit-test-result-action@v1
|
||||
with:
|
||||
files: artifacts/*/**/*.xml
|
||||
check_name: Test Results
|
||||
comment_on_pr: true
|
||||
commit: ${{ github.event.workflow_run.head_sha }}
|
||||
9
Makefile
9
Makefile
@ -120,6 +120,9 @@ BINARIES=$(addprefix bin/,$(COMMANDS))
|
||||
TESTFLAGS ?= $(TESTFLAGS_RACE) $(EXTRA_TESTFLAGS)
|
||||
TESTFLAGS_PARALLEL ?= 8
|
||||
|
||||
# Use this to replace `go test` with, for instance, `gotestsum`
|
||||
GOTEST ?= go test
|
||||
|
||||
OUTPUTDIR = $(join $(ROOTDIR), _output)
|
||||
CRIDIR=$(OUTPUTDIR)/cri
|
||||
|
||||
@ -170,15 +173,15 @@ build: ## build the go packages
|
||||
|
||||
test: ## run tests, except integration tests and tests that require root
|
||||
@echo "$(WHALE) $@"
|
||||
@go test ${TESTFLAGS} ${PACKAGES}
|
||||
@$(GOTEST) ${TESTFLAGS} ${PACKAGES}
|
||||
|
||||
root-test: ## run tests, except integration tests
|
||||
@echo "$(WHALE) $@"
|
||||
@go test ${TESTFLAGS} ${TEST_REQUIRES_ROOT_PACKAGES} -test.root
|
||||
@$(GOTEST) ${TESTFLAGS} ${TEST_REQUIRES_ROOT_PACKAGES} -test.root
|
||||
|
||||
integration: ## run integration tests
|
||||
@echo "$(WHALE) $@"
|
||||
@cd "${ROOTDIR}/integration/client" && go mod download && go test -v ${TESTFLAGS} -test.root -parallel ${TESTFLAGS_PARALLEL} .
|
||||
@cd "${ROOTDIR}/integration/client" && go mod download && $(GOTEST) -v ${TESTFLAGS} -test.root -parallel ${TESTFLAGS_PARALLEL} .
|
||||
|
||||
# TODO integrate cri integration bucket with coverage
|
||||
bin/cri-integration.test:
|
||||
|
||||
20
Vagrantfile
vendored
20
Vagrantfile
vendored
@ -173,6 +173,18 @@ EOF
|
||||
SHELL
|
||||
end
|
||||
|
||||
config.vm.provision "install-gotestsum", type: "shell", run: "once" do |sh|
|
||||
sh.upload_path = "/tmp/vagrant-install-gotestsum"
|
||||
sh.inline = <<~SHELL
|
||||
#!/usr/bin/env bash
|
||||
source /etc/environment
|
||||
source /etc/profile.d/sh.local
|
||||
set -eux -o pipefail
|
||||
${GOPATH}/src/github.com/containerd/containerd/script/setup/install-gotestsum
|
||||
sudo cp ${GOPATH}/bin/gotestsum /usr/local/bin/
|
||||
SHELL
|
||||
end
|
||||
|
||||
# SELinux is Enforcing by default.
|
||||
# To set SELinux as Disabled on a VM that has already been provisioned:
|
||||
# SELINUX=Disabled vagrant up --provision-with=selinux
|
||||
@ -196,6 +208,8 @@ EOF
|
||||
sh.upload_path = "/tmp/test-integration"
|
||||
sh.env = {
|
||||
'RUNC_FLAVOR': ENV['RUNC_FLAVOR'] || "runc",
|
||||
'GOTEST': ENV['GOTEST'] || "go test",
|
||||
'GOTESTSUM_JUNITFILE': ENV['GOTESTSUM_JUNITFILE'],
|
||||
}
|
||||
sh.inline = <<~SHELL
|
||||
#!/usr/bin/env bash
|
||||
@ -213,6 +227,10 @@ EOF
|
||||
#
|
||||
config.vm.provision "test-cri", type: "shell", run: "never" do |sh|
|
||||
sh.upload_path = "/tmp/test-cri"
|
||||
sh.env = {
|
||||
'GOTEST': ENV['GOTEST'] || "go test",
|
||||
'REPORT_DIR': ENV['REPORT_DIR'],
|
||||
}
|
||||
sh.inline = <<~SHELL
|
||||
#!/usr/bin/env bash
|
||||
source /etc/environment
|
||||
@ -235,7 +253,7 @@ EOF
|
||||
fi
|
||||
trap cleanup EXIT
|
||||
ctr version
|
||||
critest --parallel=$(nproc) --ginkgo.skip='HostIpc is true'
|
||||
critest --parallel=$(nproc) --report-dir="${REPORT_DIR}" --ginkgo.skip='HostIpc is true'
|
||||
SHELL
|
||||
end
|
||||
|
||||
|
||||
17
script/setup/install-gotestsum
Executable file
17
script/setup/install-gotestsum
Executable file
@ -0,0 +1,17 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# Copyright The containerd Authors.
|
||||
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
GO111MODULE=on go install gotest.tools/gotestsum@v1.6.2
|
||||
Loading…
Reference in New Issue
Block a user