diff --git a/performance-metrics/src/main.rs b/performance-metrics/src/main.rs index 2f6b8bdb5..622d8793c 100644 --- a/performance-metrics/src/main.rs +++ b/performance-metrics/src/main.rs @@ -28,6 +28,14 @@ enum Error { TestFailed, } +#[derive(Deserialize, Serialize)] +enum TestStatus { + #[serde(rename = "PASSED")] + Passed, + #[serde(rename = "FAILED")] + Failed, +} + #[derive(Deserialize, Serialize)] pub struct PerformanceTestResult { name: String, @@ -35,6 +43,31 @@ pub struct PerformanceTestResult { std_dev: f64, max: f64, min: f64, + status: TestStatus, +} + +impl PerformanceTestResult { + fn passed(name: &str, mean: f64, std_dev: f64, max: f64, min: f64) -> Self { + Self { + name: name.to_string(), + mean, + std_dev, + max, + min, + status: TestStatus::Passed, + } + } + + fn failed(name: &str) -> Self { + Self { + name: name.to_string(), + mean: 0.0, + std_dev: 0.0, + max: 0.0, + min: 0.0, + status: TestStatus::Failed, + } + } } #[derive(Deserialize, Serialize)] @@ -280,13 +313,7 @@ impl PerformanceTest { let max = (self.unit_adjuster)(metrics.clone().into_iter().reduce(f64::max).unwrap()); let min = (self.unit_adjuster)(metrics.clone().into_iter().reduce(f64::min).unwrap()); - PerformanceTestResult { - name: self.name.to_string(), - mean, - std_dev, - max, - min, - } + PerformanceTestResult::passed(self.name, mean, std_dev, max, min) } // Calculate the timeout for each test @@ -1316,6 +1343,14 @@ fn main() { .action(ArgAction::SetTrue) .required(false), ) + .arg( + Arg::new("continue-on-failure") + .long("continue-on-failure") + .help("Continue running remaining tests after a test failure") + .num_args(0) + .action(ArgAction::SetTrue) + .required(false), + ) .arg( Arg::new("report-file") .long("report-file") @@ -1406,6 +1441,9 @@ fn main() { init_tests(&overrides); } + let continue_on_failure = cmd_arguments.get_flag("continue-on-failure"); + let mut has_failure = false; + for test in tests_to_run { settle_host(); match run_test_with_timeout(test, &overrides) { @@ -1413,8 +1451,17 @@ fn main() { metrics_report.results.push(r); } Err(e) => { - eprintln!("Aborting test due to error: '{e:?}'"); - std::process::exit(1); + if continue_on_failure { + eprintln!("Test '{}' failed: '{e:?}'. Continuing.", test.name); + has_failure = true; + metrics_report + .results + .push(PerformanceTestResult::failed(test.name)); + cleanup_stale_processes(); + } else { + eprintln!("Aborting test due to error: '{e:?}'"); + std::process::exit(1); + } } } } @@ -1448,4 +1495,8 @@ fn main() { std::process::exit(1); }) .unwrap(); + + if has_failure { + std::process::exit(1); + } }