performance-metrics: Add --continue-on-failure flag and status tracking

Add a --continue-on-failure CLI flag that allows the test harness to
continue executing remaining tests after encountering a failure, instead
of aborting immediately. When set, failed tests are recorded with zeroed
metrics and a "FAILED" status, the report file is always generated, and
the process exits with a non-zero code if any test failed.

Without the flag, the existing fail-fast behavior is preserved.

Also add a "status" field ("PASSED"/"FAILED") to PerformanceTestResult
so report consumers can distinguish successful tests from failed ones.

Signed-off-by: Anirudh Rayabharam <anrayabh@microsoft.com>

performance-metrics/src/main.rs

@@ -28,6 +28,14 @@ enum Error {
     TestFailed,
 }
 
+#[derive(Deserialize, Serialize)]
+enum TestStatus {
+    #[serde(rename = "PASSED")]
+    Passed,
+    #[serde(rename = "FAILED")]
+    Failed,
+}
+
 #[derive(Deserialize, Serialize)]
 pub struct PerformanceTestResult {
     name: String,
@@ -35,6 +43,31 @@ pub struct PerformanceTestResult {
     std_dev: f64,
     max: f64,
     min: f64,
+    status: TestStatus,
+}
+
+impl PerformanceTestResult {
+    fn passed(name: &str, mean: f64, std_dev: f64, max: f64, min: f64) -> Self {
+        Self {
+            name: name.to_string(),
+            mean,
+            std_dev,
+            max,
+            min,
+            status: TestStatus::Passed,
+        }
+    }
+
+    fn failed(name: &str) -> Self {
+        Self {
+            name: name.to_string(),
+            mean: 0.0,
+            std_dev: 0.0,
+            max: 0.0,
+            min: 0.0,
+            status: TestStatus::Failed,
+        }
+    }
 }
 
 #[derive(Deserialize, Serialize)]
@@ -280,13 +313,7 @@ impl PerformanceTest {
         let max = (self.unit_adjuster)(metrics.clone().into_iter().reduce(f64::max).unwrap());
         let min = (self.unit_adjuster)(metrics.clone().into_iter().reduce(f64::min).unwrap());
 
-        PerformanceTestResult {
-            name: self.name.to_string(),
-            mean,
-            std_dev,
-            max,
-            min,
-        }
+        PerformanceTestResult::passed(self.name, mean, std_dev, max, min)
     }
 
     // Calculate the timeout for each test
@@ -1316,6 +1343,14 @@ fn main() {
                 .action(ArgAction::SetTrue)
                 .required(false),
         )
+        .arg(
+            Arg::new("continue-on-failure")
+                .long("continue-on-failure")
+                .help("Continue running remaining tests after a test failure")
+                .num_args(0)
+                .action(ArgAction::SetTrue)
+                .required(false),
+        )
         .arg(
             Arg::new("report-file")
                 .long("report-file")
@@ -1406,6 +1441,9 @@ fn main() {
         init_tests(&overrides);
     }
 
+    let continue_on_failure = cmd_arguments.get_flag("continue-on-failure");
+    let mut has_failure = false;
+
     for test in tests_to_run {
         settle_host();
         match run_test_with_timeout(test, &overrides) {
@@ -1413,8 +1451,17 @@ fn main() {
                 metrics_report.results.push(r);
             }
             Err(e) => {
-                eprintln!("Aborting test due to error: '{e:?}'");
-                std::process::exit(1);
+                if continue_on_failure {
+                    eprintln!("Test '{}' failed: '{e:?}'. Continuing.", test.name);
+                    has_failure = true;
+                    metrics_report
+                        .results
+                        .push(PerformanceTestResult::failed(test.name));
+                    cleanup_stale_processes();
+                } else {
+                    eprintln!("Aborting test due to error: '{e:?}'");
+                    std::process::exit(1);
+                }
             }
         }
     }
@@ -1448,4 +1495,8 @@ fn main() {
             std::process::exit(1);
         })
         .unwrap();
+
+    if has_failure {
+        std::process::exit(1);
+    }
 }