Add tests for kata containers policies (#221)

closes #220

Signed-off-by: Anand Krishnamoorthi <anakrish@microsoft.com>

.github/workflows/pr.yml

@@ -30,6 +30,8 @@ jobs:
       run: cargo test -r --verbose
     - name: Run tests (ACI)
       run: cargo test -r --test aci
+    - name: Run tests (KATA)
+      run: cargo test -r --test kata
     - name: Run tests (OPA Conformance)
       run: >-
         cargo test -r --test opa --features opa-testutil,serde_json/arbitrary_precision  -- $(tr '\n' ' ' < tests/opa.passing)
@@ -39,6 +41,8 @@ jobs:
       run: cargo test -r --verbose --target x86_64-unknown-linux-musl
     - name: Run tests (MUSL ACI)
       run: cargo test -r --test aci --target x86_64-unknown-linux-musl
+    - name: Run tests (KATA ACI)
+      run: cargo test -r --test kata --target x86_64-unknown-linux-musl
     - name: Run tests (MUSL OPA Conformance)
       run: >-
         cargo test -r --test opa --features opa-testutil,serde_json/arbitrary_precision --target x86_64-unknown-linux-musl  -- $(tr '\n' ' ' < tests/opa.passing)

Cargo.toml

@@ -124,6 +124,11 @@ name="aci"
 harness=false
 test=false
 
+[[test]]
+name="kata"
+harness=false
+test=false
+
 [package.metadata.docs.rs]
 # To build locally:
 # RUSTDOCFLAGS="--cfg docsrs" cargo +nightly doc --all-features --no-deps

scripts/pre-push

@@ -15,6 +15,7 @@ if [ -f Cargo.toml ]; then
    # Ensure that all tests pass
    cargo test -r
    cargo test -r --test aci
+   cargo test -r --test kata
 
    # Ensure that OPA conformance tests don't regress.
    cargo test -r --features opa-testutil,serde_json/arbitrary_precision --test opa -- $(tr '\n' ' ' < tests/opa.passing)

tests/kata/data/k8s-policy-job/output.json

@@ -0,0 +1,19 @@
+[
+  false,
+  true,
+  true,
+  true,
+  true,
+  true,
+  true,
+  true,
+  true,
+  true,
+  true,
+  true,
+  true,
+  true,
+  true,
+  true,
+  true
+]

tests/kata/data/k8s-policy-pod/output.json

@@ -0,0 +1,19 @@
+[
+  false,
+  true,
+  true,
+  true,
+  true,
+  true,
+  true,
+  true,
+  true,
+  true,
+  true,
+  true,
+  true,
+  true,
+  true,
+  true,
+  true
+]

tests/kata/data/k8s-policy-rc/output.json

@@ -0,0 +1,19 @@
+[
+  false,
+  true,
+  true,
+  true,
+  true,
+  true,
+  true,
+  true,
+  true,
+  true,
+  true,
+  true,
+  true,
+  true,
+  true,
+  true,
+  true
+]

tests/kata/main.rs

@@ -0,0 +1,115 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+use regorus::*;
+
+use std::path::Path;
+
+use anyhow::Result;
+use clap::Parser;
+use walkdir::WalkDir;
+
+fn run_kata_tests(tests_dir: &Path, generate: bool) -> Result<()> {
+    for entry in WalkDir::new(tests_dir)
+        .max_depth(1) // Do not recurse
+        .sort_by_file_name()
+        .into_iter()
+        .filter_map(|e| e.ok())
+    {
+        let path = entry.path();
+        if path == tests_dir || !path.is_dir() {
+            continue;
+        }
+
+        let policy_file = path.join("policy.rego");
+        let inputs_file = path.join("inputs.txt");
+        let outputs_file = path.join("output.json");
+
+        let mut engine = Engine::new();
+        engine.add_policy_from_file(&policy_file)?;
+        engine.set_gather_prints(true);
+        engine.set_strict_builtin_errors(false);
+
+        #[cfg(feature = "coverage")]
+        engine.set_enable_coverage(true);
+
+        // Keep a copy of the engine.
+        let engine_base = engine.clone();
+        let mut results = if generate {
+            vec![]
+        } else {
+            Value::from_json_str(&std::fs::read_to_string(&outputs_file)?)?
+                .as_array()?
+                .iter()
+                .cloned()
+                .rev()
+                .collect()
+        };
+
+        let inputs = std::fs::read_to_string(&inputs_file)?;
+        for (lineno, line) in inputs.split('\n').enumerate() {
+            let line = line.trim();
+            if line.is_empty() {
+                continue;
+            }
+            // Remove "ep":
+            let line = line.replace("\"ep\":", "");
+            // Remove trailing ,
+            let line = &line[0..line.len() - 1];
+
+            let request = Value::from_json_str(line)?;
+
+            let rule = format!("data.agent_policy.{}", request[0].as_string()?.as_ref());
+            let input = request[1].clone();
+
+            // Evaluate using engine.
+            engine.set_input(input.clone());
+            let r = engine.eval_rule(rule.clone())?;
+
+            // Evaluate using fresh engine.
+            let mut new_engine = engine_base.clone();
+            new_engine.set_input(input);
+            let r_new = new_engine.eval_rule(rule)?;
+
+            // Ensure that both evaluations produced the same result.
+            assert_eq!(r, r_new);
+
+            if generate {
+                results.push(r);
+            } else {
+                let expected = results.pop().unwrap();
+                assert_eq!(r, expected, "{lineno} failed in {}", inputs_file.display());
+            }
+        }
+
+        if generate {
+            std::fs::write(outputs_file, Value::from(results).to_json_str()?)?;
+        }
+        #[cfg(feature = "coverage")]
+        {
+            let report = engine.get_coverage_report()?;
+            println!("{}", report.to_colored_string()?);
+        }
+    }
+
+    println!("kata tests passed");
+    Ok(())
+}
+
+#[derive(clap::Parser)]
+#[command(author, version, about, long_about = None)]
+struct Cli {
+    /// Path to Kata test suite.
+    #[arg(long, short)]
+    #[clap(default_value = "tests/kata/data")]
+    test_dir: String,
+
+    /// Generate outputs instead of testing.
+    #[arg(long, short)]
+    #[clap(default_value = "false")]
+    generate: bool,
+}
+
+fn main() -> Result<()> {
+    let cli = Cli::parse();
+    run_kata_tests(&Path::new(&cli.test_dir), cli.generate)
+}

tests/lexer/mod.rs

@@ -226,7 +226,7 @@ fn invalid_line() -> Result<()> {
 
 #[test]
 fn file_more_than_64_kb_size() -> Result<()> {
-    let source = Source::from_file("tests/coco/data/large.rego")?;
+    let source = Source::from_file("tests/kata/data/large.rego")?;
     let mut lexer = Lexer::new(&source);
 
     let mut count = 0;