From 645a5c3a74818384a98e388e8f85dbf28e4c637e Mon Sep 17 00:00:00 2001
From: Andreea Florescu <fandree@amazon.com>
Date: Wed, 29 Jul 2020 18:20:44 +0300
Subject: [PATCH] test_bench: don't crash when no bench on master

When there are no benchmarks on the master branch, instead of
crashing first check if the error. Pass the test in case
cargo bench failed due to main bench not being present.

Fixes: https://github.com/rust-vmm/rust-vmm-ci/issues/30

Signed-off-by: Andreea Florescu <fandree@amazon.com>
---
 README.md                           | 13 ++++++++-
 integration_tests/test_benchmark.py | 42 ++++++++++++++++++++---------
 2 files changed, 42 insertions(+), 13 deletions(-)

diff --git a/README.md b/README.md
index 521b8b1..ac5fe07 100644
--- a/README.md
+++ b/README.md
@@ -238,7 +238,18 @@ steps:
 ```
 
 The test requires [`criterion`](https://github.com/bheisler/criterion.rs)
-benchmarks to be exported by the crate. `criterion` collects performance
+benchmarks to be exported by the crate. The test expects the entry point
+into the performance benchmarks to be named `main`. In other words, the
+following configuration is expected in `Cargo.toml`:
+```toml
+[[bench]]
+name = "main"
+```
+
+All benchmarks need to be collected in a main.rs file placed in `benches/`.
+
+
+`criterion` collects performance
 results by running a function for a user-configured number of iterations,
 timing the runs, and applying statistics. The individual benchmark tests must
 be added in the crate. They can be run outside the CI with:
diff --git a/integration_tests/test_benchmark.py b/integration_tests/test_benchmark.py
index 41e432a..5c18631 100644
--- a/integration_tests/test_benchmark.py
+++ b/integration_tests/test_benchmark.py
@@ -18,7 +18,10 @@ def test_bench():
     os.chdir(get_repo_root_path())
 
     # Get numbers for current HEAD.
-    _run_cargo_bench("after")
+    return_code, stdout, stderr = _run_cargo_bench("after")
+    # Even if it is the first time this test is run, the benchmark tests should pass.
+    # For this purpose, we need to explicitly check the return code.
+    assert return_code == 0, "stdout: {}\n stderr: {}".format(stdout, stderr)
 
     # Move to upstream tip.
     subprocess.run(
@@ -31,9 +34,33 @@ def test_bench():
     )
 
     # Get numbers from upstream tip, without the changes from the current PR.
-    _run_cargo_bench("before")
+    return_code, stdout, stderr = _run_cargo_bench("before")
+    if return_code == 0:
+        # In case this benchmark also ran successfully, we can call critcmp and compare the results.
+        _run_critcmp()
+    else:
+        # The benchmark did not run successfully, but it might be that it is because a benchmark does not exist.
+        # In this case, we do not want to fail the test.
+        if "error: no bench target named `main`" in stderr:
+            # This is a bit of a &*%^ way of checking if the benchmark does not exist.
+            # Hopefully it will be possible to check it in another way...soon
+            print("There are no benchmarks in master. No comparison can happen.")
+        else:
+            assert return_code == 0, "stdout: {}\n stderr: {}".format(stdout, stderr)
 
-    # Compare.
+
+def _run_cargo_bench(baseline):
+    """Runs `cargo bench` and tags the baseline."""
+    process = subprocess.run(
+        "cargo bench --bench main --all-features -- --noplot --save-baseline {}"
+        .format(baseline),
+        shell=True, stderr=subprocess.PIPE, stdout=subprocess.PIPE
+    )
+
+    return process.returncode, process.stdout.decode('utf-8'), process.stderr.decode('utf-8')
+
+
+def _run_critcmp():
     p = subprocess.run(
         "critcmp before after",
         shell=True, check=True,
@@ -44,12 +71,3 @@ def test_bench():
     print(p.stdout.decode('utf-8'))
     print('ERRORS')
     print(p.stderr.decode('utf-8'))
-
-
-def _run_cargo_bench(baseline):
-    """Runs `cargo bench` and tags the baseline."""
-    subprocess.run(
-        "cargo bench --all-features -- --noplot --save-baseline {}"
-        .format(baseline),
-        shell=True, check=True
-    )