mirror of
https://github.com/rust-vmm/rust-vmm-ci.git
synced 2026-08-05 03:08:31 +00:00
ae7db2d98a
Updated occurrences of "master" with "main". Signed-off-by: Catalin Dumitru <catdum@amazon.com>
117 lines
3.8 KiB
Python
117 lines
3.8 KiB
Python
# Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.
|
|
# SPDX-License-Identifier: Apache-2.0
|
|
"""
|
|
Compare benchmark results before and after a pull request.
|
|
|
|
This test works properly on the local machine only when the environment
|
|
variables REMOTE and BASE_BRANCH are set. Otherwise the default values
|
|
are "origin" for the remote name of the upstream repository and "main"
|
|
for the name of the base branch, and this test may not work as expected.
|
|
"""
|
|
|
|
import os
|
|
import subprocess
|
|
|
|
from utils import get_repo_root_path
|
|
|
|
REMOTE = \
|
|
os.environ.get('BUILDKITE_REPO') or \
|
|
os.environ.get('REMOTE') or \
|
|
"origin"
|
|
BASE_BRANCH = \
|
|
os.environ.get('BUILDKITE_PULL_REQUEST_BASE_BRANCH') or \
|
|
os.environ.get('BASE_BRANCH') or \
|
|
"main"
|
|
# File used for saving the results of cargo bench
|
|
# when running on the PR branch.
|
|
PR_BENCH_RESULTS_FILE = "pr_bench_results"
|
|
# File used for saving the results of cargo bench
|
|
# when running on the upstream branch.
|
|
UPSTREAM_BENCH_RESULTS_FILE = "upstream_bench_results"
|
|
|
|
|
|
def test_bench():
|
|
"""Runs benchmarks before and after and compares the results."""
|
|
os.chdir(get_repo_root_path())
|
|
|
|
# Get numbers for current HEAD.
|
|
return_code, stdout, stderr = _run_cargo_bench(PR_BENCH_RESULTS_FILE)
|
|
# Even if it is the first time this test is run, the benchmark tests should
|
|
# pass. For this purpose, we need to explicitly check the return code.
|
|
assert return_code == 0, "stdout: {}\n stderr: {}".format(stdout, stderr)
|
|
|
|
# Get numbers from upstream tip, without the changes from the current PR.
|
|
_git_checkout_upstream_branch()
|
|
return_code, stdout, stderr = _run_cargo_bench(UPSTREAM_BENCH_RESULTS_FILE)
|
|
|
|
# Before checking any results, let's just go back to the PR branch.
|
|
# This way we make sure that the cleanup always happens even if the test
|
|
# fails.
|
|
_git_checkout_pr_branch()
|
|
|
|
if return_code == 0:
|
|
# In case this benchmark also ran successfully, we can call critcmp and
|
|
# compare the results.
|
|
_run_critcmp()
|
|
else:
|
|
# The benchmark did not run successfully, but it might be that it is
|
|
# because a benchmark does not exist. In this case, we do not want to
|
|
# fail the test.
|
|
if "error: no bench target named `main`" in stderr:
|
|
# This is a bit of a &*%^ way of checking if the benchmark does not
|
|
# exist. Hopefully it will be possible to check it in another way
|
|
# ...soon
|
|
print(
|
|
"There are no benchmarks in main. No comparison can happen."
|
|
)
|
|
else:
|
|
assert return_code == 0, "stdout: {}\n stderr: {}".format(
|
|
stdout, stderr)
|
|
|
|
|
|
def _run_cargo_bench(baseline):
|
|
"""Runs `cargo bench` and tags the baseline."""
|
|
process = subprocess.run(
|
|
"cargo bench --bench main --all-features -- --noplot "
|
|
"--save-baseline {}".format(baseline),
|
|
shell=True, stderr=subprocess.PIPE, stdout=subprocess.PIPE
|
|
)
|
|
|
|
return process.returncode, process.stdout.decode('utf-8'),\
|
|
process.stderr.decode('utf-8')
|
|
|
|
|
|
def _run_critcmp():
|
|
p = subprocess.run(
|
|
"critcmp {} {}".format(
|
|
UPSTREAM_BENCH_RESULTS_FILE, PR_BENCH_RESULTS_FILE
|
|
),
|
|
shell=True, check=True,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE
|
|
)
|
|
|
|
print(p.stdout.decode('utf-8'))
|
|
print('ERRORS')
|
|
print(p.stderr.decode('utf-8'))
|
|
|
|
|
|
def _git_checkout_upstream_branch():
|
|
subprocess.run(
|
|
"git fetch {} {}".format(REMOTE, BASE_BRANCH),
|
|
shell=True, check=True
|
|
)
|
|
subprocess.run(
|
|
"git checkout FETCH_HEAD",
|
|
shell=True, check=True
|
|
)
|
|
|
|
|
|
def _git_checkout_pr_branch():
|
|
subprocess.run(
|
|
"git checkout -",
|
|
shell=True, check=True,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE
|
|
)
|