From 6c471c6c3aa8d56f6ef8022840feede01f61c263 Mon Sep 17 00:00:00 2001 From: Andreea Florescu Date: Tue, 4 Jun 2019 18:56:11 +0300 Subject: [PATCH] added Buildkite pipeline and integration tests This pipeline contains the tests that are currently run for kvm-ioctls. The pipeline also has a step that runs the coverage test defined in integration_tests/test_coverage.py. The purpose of this pipeline is to be used by all rust-vmm repositories to ensure we keep the same quality bar for all crates. Signed-off-by: Andreea Florescu --- .buildkite/pipeline.yml | 190 ++++++++++++++++++++++++++ CODEOWNERS | 3 + README.md | 209 ++++++++++++++++++++++++++++- coverage_config.json.sample | 5 + integration_tests/conftest.py | 30 +++++ integration_tests/test_coverage.py | 120 +++++++++++++++++ integration_tests/utils.py | 11 ++ 7 files changed, 567 insertions(+), 1 deletion(-) create mode 100644 .buildkite/pipeline.yml create mode 100644 CODEOWNERS create mode 100644 coverage_config.json.sample create mode 100644 integration_tests/conftest.py create mode 100644 integration_tests/test_coverage.py create mode 100644 integration_tests/utils.py diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml new file mode 100644 index 0000000..333c349 --- /dev/null +++ b/.buildkite/pipeline.yml @@ -0,0 +1,190 @@ +# Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +steps: + - label: "build-gnu-x86" + commands: + - cargo build --release + retry: + automatic: false + agents: + platform: x86_64.metal + os: linux + plugins: + - docker#v3.0.1: + image: "rustvmm/dev:v2" + always-pull: true + + - label: "build-gnu-arm" + commands: + - cargo build --release + retry: + automatic: false + agents: + platform: arm.metal + os: linux + plugins: + - docker#v3.0.1: + image: "rustvmm/dev:v2" + always-pull: true + + - label: "build-musl-x86" + commands: + - cargo build --release --target x86_64-unknown-linux-musl + retry: + automatic: false + agents: + platform: x86_64.metal + os: linux + plugins: + - docker#v3.0.1: + image: "rustvmm/dev:v2" + always-pull: true + + - label: "build-musl-arm" + commands: + - cargo build --release --target aarch64-unknown-linux-musl + retry: + automatic: false + agents: + platform: arm.metal + os: linux + plugins: + - docker#v3.0.1: + image: "rustvmm/dev:v2" + always-pull: true + + - label: "style" + command: cargo fmt --all -- --check + retry: + automatic: false + agents: + platform: x86_64.metal + os: linux + plugins: + - docker#v3.0.1: + image: "rustvmm/dev:v2" + always-pull: true + + - label: "unittests-gnu-x86" + commands: + - cargo test --all-features + retry: + automatic: false + agents: + platform: x86_64.metal + os: linux + plugins: + - docker#v3.0.1: + privileged: true + image: "rustvmm/dev:v2" + always-pull: true + + - label: "unittests-gnu-arm" + commands: + - cargo test --all-features + retry: + automatic: false + agents: + platform: arm.metal + os: linux + plugins: + - docker#v3.0.1: + privileged: true + image: "rustvmm/dev:v2" + always-pull: true + + - label: "unittests-musl-x86" + command: + - cargo test --all-features --target x86_64-unknown-linux-musl + retry: + automatic: false + agents: + platform: x86_64.metal + os: linux + plugins: + - docker#v3.0.1: + privileged: true + image: "rustvmm/dev:v2" + always-pull: true + + - label: "unittests-musl-arm" + command: + - cargo test --all-features --target aarch64-unknown-linux-musl + retry: + automatic: false + agents: + platform: arm.metal + os: linux + plugins: + - docker#v3.0.1: + privileged: true + image: "rustvmm/dev:v2" + always-pull: true + + - label: "clippy-x86" + commands: + - cargo clippy --all -- -D warnings + retry: + automatic: false + agents: + platform: x86_64.metal + os: linux + plugins: + - docker#v3.0.1: + image: "rustvmm/dev:v2" + always-pull: true + + - label: "clippy-arm" + commands: + - cargo clippy --all -- -D warnings + retry: + automatic: false + agents: + platform: arm.metal + os: linux + plugins: + - docker#v3.0.1: + image: "rustvmm/dev:v2" + always-pull: true + + - label: "check-warnings-x86" + commands: + - RUSTFLAGS="-D warnings" cargo check --all-targets + retry: + automatic: false + agents: + platform: x86_64.metal + os: linux + plugins: + - docker#v3.0.1: + privileged: true + image: "rustvmm/dev:v2" + always-pull: true + + - label: "check-warnings-arm" + command: + - RUSTFLAGS="-D warnings" cargo check --all-targets + retry: + automatic: false + agents: + platform: arm.metal + os: linux + plugins: + - docker#v3.0.1: + privileged: true + image: "rustvmm/dev:v2" + always-pull: true + + - label: "coverage-x86" + commands: + - pytest rust-vmm-ci/integration_tests/test_coverage.py + retry: + automatic: false + agents: + platform: x86_64.metal + os: linux + plugins: + - docker#v3.0.1: + privileged: true + image: "rustvmm/dev:v2" + always-pull: true diff --git a/CODEOWNERS b/CODEOWNERS new file mode 100644 index 0000000..e8b307b --- /dev/null +++ b/CODEOWNERS @@ -0,0 +1,3 @@ +# Buildkite rust-vmm organization admins: +* @andreeaflorescu +* @sameo diff --git a/README.md b/README.md index 4a10925..5e9c62e 100644 --- a/README.md +++ b/README.md @@ -1 +1,208 @@ -# rust-vmm-ci \ No newline at end of file +# rust-vmm-ci + +The `rust-vmm-ci` repository contains [integration tests](#integration-tests) +and [Buildkite pipeline](#buildkite-pipeline) definitions that are used for +running the CI for all rust-vmm crates. + +Having a centralized place for the tests is one of the enablers for keeping the +same quality standard for all crates in rust-vmm. + +## Getting Started with rust-vmm-ci + +To run the integration tests defined in the pipeline as part of the CI: + +1. Add rust-vmm-ci as a git submodule to your repository + +```bash +# Add rust-vmm-ci as a submodule. This will point to the latest rust-vmm-ci +# commit from the master branch. The following command will also add a +# `.gitmodules` file and the `rust-vmm-ci` to the index. +git submodule add git@github.com:rust-vmm/rust-vmm-ci.git +# Commit the changes to your repository so that the CI can run using the +# rust-vmm-ci pipeline and tests. +git commit -s -m "Added rust-vmm-ci as submodule" +``` + +2. Create the coverage test configuration file named coverage_config.json in +the root of the repository. One example of the configuration file can be found +in [coverage_config.json.sample](coverage_config.json.sample). + +The json must have the following fields: +- *coverage_score*: The coverage of the repository. +- *exclude_path*: This field is used for excluding files from the report. It + should be used to exclude autogenerated files. Files in *exclude_path* are + separated by one coma. If the repository does not have any autogenerated + files, *exclude_path* should be an empty string. +- *crate_features*: `cargo kcov` does not build crate features by default. To + get the coverage report including optional features these need to be + specified in *crate_features* separated by coma. If the crate does not have + any features, this field should be empty. + +This file is required for the coverage integration so it needs to be added +to the repository as well. + +3. Create a new pipeline definition in Buildkite. For this step ask one of the +rust-vmm Buildkite [admins](CODEOWNERS) to create one for you. Add a pipeline +step that is uploading the rust-vmm-ci pipeline: + +```bash +buildkite-agent pipeline upload rust-vmm-ci/.buildkite/pipeline.yml +``` + +4. The code owner of the repository will have to setup a WebHook for triggering +the CI on Pull Requests and pushes. + +## Buildkite Pipeline + +The [Buildkite](https://buildkite.com) pipeline is the definition of tests to +be run as part of the CI. It includes steps for running unit tests and linters +(including coding style checks), and computing the coverage. + + Currently the tests can run on Linux x86_64 and aarch64 hosts. + +Example of step that checks the build: + +```yaml +steps: + - label: "build-gnu-x86" + commands: + - cargo build --release + retry: + automatic: false + agents: + platform: x86_64.metal + plugins: + - docker#v3.0.1: + image: "rustvmm/dev:v2" + always-pull: true +``` + +To see all steps in the pipeline check the +[.buikite/pipeline.yml](.buildkite/pipeline.yml) file. + +### Custom Pipeline + +Some crates might need to test functionality that is specific to that +particular component and thus cannot be added to the common pipeline. + +In this situation, the repositories need to create a custom pipeline (besides +the rust-vmm-ci pipeline) and add it in the repository. The preferred path for +the custom pipeline is `.buildkite/pipeline.yml`. + +For example to test the build with one non-default +[feature](https://doc.rust-lang.org/1.19.0/book/first-edition/conditional-compilation.html) +enabled, the following step can be added in the custom pipeline under +`.buildkite/pipeline.yml`. + +```yaml +steps: + - label: "build-gnu-x86-bzimage" + commands: + - cargo build --release --features bzimage + retry: + automatic: false + agents: + platform: x86_64.metal + plugins: + - docker#v3.0.1: + image: "rustvmm/dev:v2" +always-pull: true +``` + +### Custom Repository Hooks + +The integration tests of some repositories have dependencies on external +resources. One example is +[linux-loader](https://github.com/rust-vmm/linux-loader/) which needs to +download a bzImage before running the unit tests. Because this is specific +to the linux-loader crate, the logic for downloading the required resources +cannot be part of the common pipeline. The mechanism used here is +[Repository Hooks](https://buildkite.com/docs/agent/v3/hooks#repository-hooks). +The hooks are defined per repository and live in the crate repository under +`.buildkite/hooks`. + +Example of post-checkout hook that downloads and extracts a bzImage: + +```bash +#!/bin/bash + +DEB_NAME="linux-image-4.9.0-9-amd64_4.9.168-1_amd64.deb" +DEB_URL="http://ftp.debian.org/debian/pool/main/l/linux/${DEB_NAME}" + +REPO_PATH="${BUILDKITE_BUILD_CHECKOUT_PATH}" +DEB_PATH="${REPO_PATH}/${DEB_NAME}" +EXTRACT_PATH="${REPO_PATH}/src/bzimage-archive" +BZIMAGE_PATH="${EXTRACT_PATH}/boot/vmlinuz-4.9.0-9-amd64" + +mkdir -p ${EXTRACT_PATH} + +wget ${DEB_URL} -P ${REPO_PATH} +dpkg-deb -x ${DEB_PATH} ${EXTRACT_PATH} + +mv ${BZIMAGE_PATH} ${REPO_PATH}/src/bzimage +rm -r ${EXTRACT_PATH} +rm -f ${DEB_PATH} +``` + +In this example the post-checkout hook downloads a deb image, extracts the +contents of it and places it in `linux-loader/src/bzimage`. The unit tests +will use the relative path `src/bzimage` which does not depend on the image +being downloaded. + +## Integration Tests + +In addition to the one-liner tests defined in the +[Builkite Pipeline](#buildkite-pipeline), the rust-vmm-ci also has more +complex tests defined in [integration_tests](integration_tests). + +### Test Profiles + +The integration tests support two test profiles: +- **devel**: this is the recommended profile for running the integration tests + on a local development machine. +- **ci** (default option): this is the profile used when running the + integration tests as part of the the Continuous Integration (CI). + +The test profiles are applicable to tests that run using pytest. Currently only +the [coverage test](tests/test_coverage.py) follows this model as all the other +integration tests are run using the +[Buildkite pipeline](https://buildkite.com/rust-vmm/kvm-ioctls-ci). + +The difference between is declaring tests as passed or failed: +- with the **devel** profile the coverage test passes if the current coverage + is equal or higher than the upstream coverage value. In case the current + coverage is higher, the coverage file is updated to the new coverage value. +- with the **ci** profile the coverage test passes only if the current coverage + is equal to the upstream coverage value. + +Further details about the coverage test can be found in the +[Adaptive Coverage](#adaptive-coverage) section. + +### Adaptive Coverage + +The line coverage is saved in [tests/coverage](tests/coverage). To update the +coverage before submitting a PR, run the coverage test: + +```bash +docker run --device=/dev/kvm \ + -it \ + --security-opt seccomp=unconfined \ + --volume $(pwd)/kvm-ioctls:/kvm-ioctls \ + rustvmm/dev:v2 +cd kvm-ioctls/ +pytest --profile=devel tests/test_coverage.py +``` + +If the PR coverage is higher than the upstream coverage, the coverage file +needs to be manually added to the commit before submitting the PR: + +```bash +git add tests/coverage +``` + +Failing to do so will generate a fail on the CI pipeline when publishing the +PR. + +**NOTE:** The coverage file is only updated in the `devel` test profile. In +the `ci` profile the coverage test will fail if the current coverage is higher +than the coverage reported in [tests/coverage](tests/coverage) diff --git a/coverage_config.json.sample b/coverage_config.json.sample new file mode 100644 index 0000000..60dcb24 --- /dev/null +++ b/coverage_config.json.sample @@ -0,0 +1,5 @@ +{ + "coverage_score": 90, + "exclude_path": "my_autogenerated_file.rs,second_file.rs", + "crate_features": "my_dummy_feature,dummy_feature2" +} diff --git a/integration_tests/conftest.py b/integration_tests/conftest.py new file mode 100644 index 0000000..78c9c95 --- /dev/null +++ b/integration_tests/conftest.py @@ -0,0 +1,30 @@ +# Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +import pytest + + +PROFILE_CI="ci" +PROFILE_DEVEL="devel" + + +def pytest_addoption(parser): + parser.addoption( + "--profile", + default=PROFILE_CI, + choices=[PROFILE_CI, PROFILE_DEVEL], + help="Profile for running the test: {} or {}".format( + PROFILE_CI, + PROFILE_DEVEL + ) + ) + + +@pytest.fixture +def profile(request): + return request.config.getoption("--profile") + + +# This is used for defining global variables in pytest. +def pytest_configure(): + pytest.profile_ci = PROFILE_CI + pytest.profile_devel = PROFILE_DEVEL diff --git a/integration_tests/test_coverage.py b/integration_tests/test_coverage.py new file mode 100644 index 0000000..0750fd3 --- /dev/null +++ b/integration_tests/test_coverage.py @@ -0,0 +1,120 @@ +# Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Test the coverage and update the threshold when coverage is increased.""" + +import json, os, re, shutil, subprocess +import pytest + +from utils import get_repo_root_path + +REPO_ROOT_PATH = get_repo_root_path() +COVERAGE_CONFIG_PATH = os.path.join(REPO_ROOT_PATH, "coverage_config.json") + + +def _read_test_config(): + """ + Reads the config of the coverage for the repository being tested. + + Returns a JSON object with the configuration. + """ + coverage_config = {} + with open(COVERAGE_CONFIG_PATH) as config_file: + coverage_config = json.load(config_file) + + assert "coverage_score" in coverage_config + assert "exclude_path" in coverage_config + assert "crate_features" in coverage_config + + return coverage_config + + +def _write_coverage_config(coverage_config): + """Updates the coverage config file as per `coverage_config`""" + with open(COVERAGE_CONFIG_PATH, 'w') as outfile: + json.dump(coverage_config, outfile) + + +def _get_current_coverage(coverage_config): + """Helper function that returns the coverage computed with kcov.""" + kcov_ouput_dir = os.path.join(REPO_ROOT_PATH, "kcov_output") + + # By default the build output for kcov and unit tests are both in the debug + # directory. This causes some linker errors that I haven't investigated. + # Error: error: linking with `cc` failed: exit code: 1 + # An easy fix is to have separate build directories for kcov & unit tests. + kcov_build_dir = os.path.join(REPO_ROOT_PATH, "kcov_build") + + # Remove kcov output and build directory to be sure we are always working + # on a clean environment. + shutil.rmtree(kcov_ouput_dir, ignore_errors=True) + shutil.rmtree(kcov_build_dir, ignore_errors=True) + + exclude_pattern = ( + '${CARGO_HOME:-$HOME/.cargo/},' + 'usr/lib/,' + 'lib/' + ) + exclude_region = "'mod tests {'" + additional_exclude_path = coverage_config["exclude_path"] + if additional_exclude_path: + exclude_pattern += ',' + additional_exclude_path + + additional_kcov_param = '' + crate_features = coverage_config["crate_features"] + if crate_features: + additional_kcov_param += '--features=' + crate_features + + kcov_cmd = "CARGO_TARGET_DIR={} cargo kcov {} --all " \ + "--output {} -- " \ + "--exclude-region={} " \ + "--exclude-pattern={} " \ + "--verify".format( + kcov_build_dir, + additional_kcov_param, + kcov_ouput_dir, + exclude_region, + exclude_pattern + ) + + subprocess.run(kcov_cmd, shell=True, check=True) + + # Read the coverage reported by kcov. + coverage_file = os.path.join(kcov_ouput_dir, 'index.js') + with open(coverage_file) as cov_output: + coverage = float(re.findall( + r'"covered":"(\d+\.\d)"', + cov_output.read() + )[0]) + + # Remove coverage related directories. + shutil.rmtree(kcov_ouput_dir, ignore_errors=True) + shutil.rmtree(kcov_build_dir, ignore_errors=True) + + return coverage + + +def test_coverage(profile): + coverage_config = _read_test_config() + current_coverage = _get_current_coverage(coverage_config) + previous_coverage = coverage_config["coverage_score"] + if previous_coverage < current_coverage: + if profile == pytest.profile_ci: + # In the CI Profile we expect the coverage to be manually updated. + assert False, "Coverage is increased from {} to {}. " \ + "Please update the coverage in " \ + "tests/coverage.".format( + previous_coverage, + current_coverage + ) + elif profile == pytest.profile_devel: + coverage_config["coverage_score"] = current_coverage + _write_coverage_config(coverage_config) + else: + # This should never happen because pytest should only accept + # the valid test profiles specified with `choices` in + # `pytest_addoption`. + assert False, "Invalid test profile." + elif previous_coverage > current_coverage: + diff = float(previous_coverage - current_coverage) + assert False, "Coverage drops by {:.2f}%. Please add unit tests for" \ + "the uncovered lines.".format(diff) diff --git a/integration_tests/utils.py b/integration_tests/utils.py new file mode 100644 index 0000000..1e50257 --- /dev/null +++ b/integration_tests/utils.py @@ -0,0 +1,11 @@ +# Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +import os + + +def get_repo_root_path(): + """Terrible hack to get the root path of the repository.""" + integration_tests_path = os.path.dirname(os.path.realpath(__file__)) + rust_vmm_ci_path = os.path.dirname(integration_tests_path) + + return os.path.dirname(rust_vmm_ci_path)