test_bench: don't crash when no bench on master

When there are no benchmarks on the master branch, instead of
crashing first check if the error. Pass the test in case
cargo bench failed due to main bench not being present.

Fixes: https://github.com/rust-vmm/rust-vmm-ci/issues/30

Signed-off-by: Andreea Florescu <fandree@amazon.com>

integration_tests/test_benchmark.py

@@ -18,7 +18,10 @@ def test_bench():
     os.chdir(get_repo_root_path())
 
     # Get numbers for current HEAD.
-    _run_cargo_bench("after")
+    return_code, stdout, stderr = _run_cargo_bench("after")
+    # Even if it is the first time this test is run, the benchmark tests should pass.
+    # For this purpose, we need to explicitly check the return code.
+    assert return_code == 0, "stdout: {}\n stderr: {}".format(stdout, stderr)
 
     # Move to upstream tip.
     subprocess.run(
@@ -31,9 +34,33 @@ def test_bench():
     )
 
     # Get numbers from upstream tip, without the changes from the current PR.
-    _run_cargo_bench("before")
+    return_code, stdout, stderr = _run_cargo_bench("before")
+    if return_code == 0:
+        # In case this benchmark also ran successfully, we can call critcmp and compare the results.
+        _run_critcmp()
+    else:
+        # The benchmark did not run successfully, but it might be that it is because a benchmark does not exist.
+        # In this case, we do not want to fail the test.
+        if "error: no bench target named `main`" in stderr:
+            # This is a bit of a &*%^ way of checking if the benchmark does not exist.
+            # Hopefully it will be possible to check it in another way...soon
+            print("There are no benchmarks in master. No comparison can happen.")
+        else:
+            assert return_code == 0, "stdout: {}\n stderr: {}".format(stdout, stderr)
 
-    # Compare.
+
+def _run_cargo_bench(baseline):
+    """Runs `cargo bench` and tags the baseline."""
+    process = subprocess.run(
+        "cargo bench --bench main --all-features -- --noplot --save-baseline {}"
+        .format(baseline),
+        shell=True, stderr=subprocess.PIPE, stdout=subprocess.PIPE
+    )
+
+    return process.returncode, process.stdout.decode('utf-8'), process.stderr.decode('utf-8')
+
+
+def _run_critcmp():
     p = subprocess.run(
         "critcmp before after",
         shell=True, check=True,
@@ -44,12 +71,3 @@ def test_bench():
     print(p.stdout.decode('utf-8'))
     print('ERRORS')
     print(p.stderr.decode('utf-8'))
-
-
-def _run_cargo_bench(baseline):
-    """Runs `cargo bench` and tags the baseline."""
-    subprocess.run(
-        "cargo bench --all-features -- --noplot --save-baseline {}"
-        .format(baseline),
-        shell=True, check=True
-    )