OPA conformance tests (#45)

Run regorus against OPA test suite.

To run full test suite:
  cargo test --test opa

To run specific folder (e.g semver):
  cargo test --test opa -- semver

Signed-off-by: Anand Krishnamoorthi <anakrish@microsoft.com>

Cargo.toml

@@ -28,3 +28,7 @@ anyhow = "1.0.66"
 [profile.release]
 debug = true
 
+[[test]]
+name="opa"
+harness=false
+test=false

src/interpreter.rs

@@ -1567,9 +1567,7 @@ impl Interpreter {
                     }
                     return r;
                 }
-                return Err(span
-                    .source
-                    .error(span.line, span.col, "could not find function"));
+                bail!(span.error(format!("could not find function {fcn_path}").as_str()));
             }
         };
 

tests/opa.rs

@@ -0,0 +1,250 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+use regorus::*;
+
+use std::collections::{BTreeMap, BTreeSet};
+use std::io::{self, Write};
+use std::path::Path;
+use std::process::Command;
+
+use anyhow::{bail, Result};
+use clap::Parser;
+use serde::{Deserialize, Serialize};
+use walkdir::WalkDir;
+
+const OPA_REPO: &str = "https://github.com/open-policy-agent/opa";
+const OPA_BRANCH: &str = "v0.58.0";
+
+#[derive(Serialize, Deserialize, PartialEq, Debug)]
+struct TestCase {
+    data: Option<Value>,
+    input: Option<Value>,
+    modules: Option<Vec<String>>,
+    note: String,
+    query: String,
+    sort_bindings: Option<bool>,
+    want_result: Option<Value>,
+    skip: Option<bool>,
+    error: Option<String>,
+    traces: Option<bool>,
+    want_error: Option<String>,
+    want_error_code: Option<String>,
+}
+
+#[derive(Serialize, Deserialize, PartialEq, Debug)]
+struct YamlTest {
+    cases: Vec<TestCase>,
+}
+
+fn eval_test_case(case: &TestCase) -> Result<Value> {
+    let mut engine = Engine::new();
+
+    if let Some(data) = &case.data {
+        engine.add_data(data.clone())?;
+    }
+    if let Some(input) = &case.input {
+        engine.set_input(input.clone());
+    }
+    if let Some(modules) = &case.modules {
+        for (idx, rego) in modules.iter().enumerate() {
+            engine.add_policy(format!("rego{idx}.rego"), rego.clone())?;
+        }
+    }
+    let query_results = engine.eval_query(case.query.clone(), true)?;
+
+    let mut values = vec![];
+    for qr in query_results.result {
+        values.push(if !qr.bindings.is_empty_object() {
+            qr.bindings.clone()
+        } else if let Some(v) = qr.expressions.last() {
+            v["value"].clone()
+        } else {
+            Value::Undefined
+        });
+    }
+    let result = Value::from_array(values);
+    // Make result json compatible. (E.g: avoid sets).
+    Value::from_json_str(&result.to_string())
+}
+
+fn run_opa_tests(opa_tests_dir: String, folders: &[String]) -> Result<()> {
+    println!("OPA TESTSUITE: {opa_tests_dir}");
+    let tests_path = Path::new(&opa_tests_dir);
+    let mut status = BTreeMap::<String, (u32, u32)>::new();
+    let mut n = 0;
+    let mut missing_functions = BTreeSet::new();
+    for entry in WalkDir::new(&opa_tests_dir)
+        .sort_by_file_name()
+        .into_iter()
+        .filter_map(|e| e.ok())
+    {
+        let path_str = entry.path().to_string_lossy().to_string();
+        if path_str == opa_tests_dir {
+            continue;
+        }
+        let path = Path::new(&path_str);
+        let path_dir = path.strip_prefix(tests_path)?.parent().unwrap();
+        let path_dir_str = path_dir.to_string_lossy().to_string();
+
+        if path.is_dir() {
+            n = 0;
+            continue;
+        } else if !path.is_file() || !path_str.ends_with(".yaml") {
+            continue;
+        }
+
+        let run_test = folders.is_empty() || folders.iter().any(|f| path_dir_str.contains(f));
+        if !run_test {
+            continue;
+        }
+
+        let entry = status.entry(path_dir_str).or_insert((0, 0));
+
+        let yaml_str = std::fs::read_to_string(&path_str)?;
+        let test: YamlTest = serde_yaml::from_str(&yaml_str)?;
+
+        for case in &test.cases {
+            match (eval_test_case(case), &case.want_result) {
+                (Ok(actual), Some(expected)) if &actual == expected => {
+                    entry.0 += 1;
+                }
+                (Err(_), None) if case.want_error.is_some() => {
+                    // Expected failure.
+                    entry.0 += 1;
+                }
+                (r, _) => {
+                    print!("\n{} failed.", case.note);
+                    if let Err(e) = r {
+                        let msg = e.to_string();
+                        let pat = "could not find function ";
+                        if let Some(pos) = msg.find(pat) {
+                            let fcn = &msg[pos + pat.len()..];
+                            missing_functions.insert(fcn.to_string());
+                        }
+                    }
+                    let path = Path::new("target/opa/failures").join(path_dir);
+                    std::fs::create_dir_all(path.clone())?;
+
+                    let mut cmd = "target/debug/examples/regorus eval".to_string();
+                    if let Some(data) = &case.data {
+                        let json_path = path.join(format!("data{n}.json"));
+                        cmd += format!(" -d {}", json_path.display()).as_str();
+                        std::fs::write(json_path, data.to_json_str()?.as_bytes())?;
+                    };
+                    if let Some(input) = &case.input {
+                        let input_path = path.join(format!("data{n}.json"));
+                        cmd += format!(" -i {}", input_path.display()).as_str();
+                        std::fs::write(input_path, input.to_json_str()?.as_bytes())?;
+                    };
+
+                    if let Some(modules) = &case.modules {
+                        if modules.len() == 1 {
+                            let rego_path = path.join(format!("rego{n}.rego"));
+                            cmd += format!(" -d {}", rego_path.display()).as_str();
+                            std::fs::write(rego_path, modules[0].as_bytes())?;
+                        } else {
+                            for (i, m) in modules.iter().enumerate() {
+                                let rego_path = path.join(format!("rego{n}_{i}.rego"));
+                                cmd += format!(" -d {}", rego_path.display()).as_str();
+                                std::fs::write(rego_path, m.as_bytes())?;
+                            }
+                        }
+                    }
+
+                    std::fs::write(path.join(format!("query{n}.text")), case.query.as_bytes())?;
+                    cmd += format!(" \"{}\"", &case.query).as_str();
+
+                    println!(" To debug, run:\n\x1b[31m{cmd}\x1b[0m");
+                    entry.1 += 1;
+                    n += 1;
+                    continue;
+                }
+            };
+        }
+    }
+
+    println!("\nTESTSUITE STATUS");
+    println!("    {:40}  {:4} {:4}", "FOLDER", "PASS", "FAIL");
+    let (mut npass, mut nfail) = (0, 0);
+    for (dir, (pass, fail)) in status {
+        if fail == 0 {
+            println!("\x1b[32m    {dir:40}: {pass:4} {fail:4}\x1b[0m");
+        } else {
+            println!("\x1b[31m    {dir:40}: {pass:4} {fail:4}\x1b[0m");
+        }
+        npass += pass;
+        nfail += fail;
+    }
+    println!();
+
+    if npass == 0 && nfail == 0 {
+        bail!("no matching tests found.");
+    } else if nfail == 0 {
+        println!("\x1b[32m    {:40}: {npass:4} {nfail:4}\x1b[0m", "TOTAL");
+    } else {
+        println!("\x1b[31m    {:40}: {npass:4} {nfail:4}\x1b[0m", "TOTAL");
+    }
+
+    if !missing_functions.is_empty() {
+        println!("\nMISSING FUNCTIONS");
+        for (idx, fcn) in missing_functions.iter().enumerate() {
+            println!("\x1b[31m    {:4}: {fcn}\x1b[0m", idx + 1);
+        }
+    }
+
+    if nfail != 0 {
+        bail!("OPA tests failed");
+    }
+
+    Ok(())
+}
+
+#[derive(clap::Parser)]
+#[command(author, version, about, long_about = None)]
+struct Cli {
+    /// Path to OPA test suite.
+    #[arg(long, short)]
+    test_suite_path: Option<String>,
+
+    /// Specific test folder to run.
+    folders: Vec<String>,
+}
+
+fn main() -> Result<()> {
+    let cli = Cli::parse();
+
+    let opa_tests_dir = match cli.test_suite_path {
+        Some(p) => p,
+        None => {
+            let branch_dir = format!("target/opa/branch/{OPA_BRANCH}");
+            std::fs::create_dir_all(&branch_dir)?;
+            if !std::path::Path::exists(Path::new(format!("{branch_dir}/.git").as_str())) {
+                let output = match Command::new("git")
+                    .arg("clone")
+                    .arg(OPA_REPO)
+                    .arg("--depth")
+                    .arg("1")
+                    .arg("--single-branch")
+                    .arg("--branch")
+                    .arg(OPA_BRANCH)
+                    .arg(&branch_dir)
+                    .output()
+                {
+                    Ok(o) => o,
+                    Err(e) => {
+                        bail!("failed to execute git clone. {e}")
+                    }
+                };
+                println!("status: {}", output.status);
+                io::stdout().write_all(&output.stdout).unwrap();
+                io::stderr().write_all(&output.stderr).unwrap();
+                if !output.status.success() {
+                    bail!("failed to clone OPA repository");
+                }
+            }
+            format!("{branch_dir}/test/cases/testdata")
+        }
+    };
+
+    run_opa_tests(opa_tests_dir, &cli.folders)
+}

tests/opa/mod.rs

@@ -1,169 +0,0 @@
-// Copyright (c) Microsoft Corporation.
-// Licensed under the MIT License.
-
-use regorus::*;
-
-use std::collections::BTreeMap;
-use std::path::Path;
-
-use anyhow::Result;
-use walkdir::WalkDir;
-
-use serde::{Deserialize, Serialize};
-
-#[derive(Serialize, Deserialize, PartialEq, Debug)]
-struct TestCase {
-    data: Option<Value>,
-    input: Option<Value>,
-    modules: Option<Vec<String>>,
-    note: String,
-    query: String,
-    sort_bindings: Option<bool>,
-    want_result: Option<Value>,
-    skip: Option<bool>,
-    error: Option<String>,
-    traces: Option<bool>,
-    want_error: Option<String>,
-    want_error_code: Option<String>,
-}
-
-#[derive(Serialize, Deserialize, PartialEq, Debug)]
-struct YamlTest {
-    cases: Vec<TestCase>,
-}
-
-fn eval_test_case(case: &TestCase) -> Result<Value> {
-    let mut engine = Engine::new();
-
-    if let Some(data) = &case.data {
-        engine.add_data(data.clone())?;
-    }
-    if let Some(input) = &case.input {
-        engine.set_input(input.clone());
-    }
-    if let Some(modules) = &case.modules {
-        for (idx, rego) in modules.iter().enumerate() {
-            engine.add_policy(format!("rego{idx}.rego"), rego.clone())?;
-        }
-    }
-    let query_results = engine.eval_query(case.query.clone(), true)?;
-
-    let mut values = vec![];
-    for qr in query_results.result {
-        values.push(if !qr.bindings.is_empty_object() {
-            qr.bindings.clone()
-        } else if let Some(v) = qr.expressions.last() {
-            v["value"].clone()
-        } else {
-            Value::Undefined
-        });
-    }
-    let result = Value::from_array(values);
-    // Make result json compatible. (E.g: avoid sets).
-    Value::from_json_str(&result.to_string())
-}
-
-#[test]
-fn run_opa_tests() -> Result<()> {
-    let opa_tests_dir = match std::env::var("OPA_TESTS_DIR") {
-        Ok(v) => v,
-        _ => {
-            println!("OPA_TESTS_DIR environment vairable not defined.");
-            return Ok(());
-        }
-    };
-
-    let tests_path = Path::new(&opa_tests_dir);
-    let mut status = BTreeMap::<String, (u32, u32)>::new();
-    let mut n = 0;
-    for entry in WalkDir::new(&opa_tests_dir)
-        .sort_by_file_name()
-        .into_iter()
-        .filter_map(|e| e.ok())
-    {
-        let path_str = entry.path().to_string_lossy().to_string();
-        let path = Path::new(&path_str);
-        if !path.is_file() || !path_str.ends_with(".yaml") {
-            continue;
-        }
-
-        let path_dir = path.strip_prefix(tests_path)?.parent().unwrap();
-
-        let path_dir_str = path_dir.to_string_lossy().to_string();
-        let entry = status.entry(path_dir_str).or_insert((0, 0));
-
-        let yaml_str = std::fs::read_to_string(&path_str)?;
-        let test: YamlTest = serde_yaml::from_str(&yaml_str)?;
-
-        for case in &test.cases {
-            print!("{} ...", case.note);
-            match (eval_test_case(case), &case.want_result) {
-                (Ok(actual), Some(expected)) if &actual == expected => {
-                    println!("passed");
-                    entry.0 += 1;
-                }
-                (Err(_), None) if case.want_error.is_some() => {
-                    // Expected failure.
-                    println!("passed");
-                    entry.0 += 1;
-                }
-                _ => {
-                    let path = Path::new("target/opa").join(path_dir);
-                    std::fs::create_dir_all(path.clone())?;
-
-                    if let Some(data) = &case.data {
-                        std::fs::write(
-                            path.join(format!("data{n}.json")),
-                            data.to_json_str()?.as_bytes(),
-                        )?;
-                    };
-                    if let Some(input) = &case.input {
-                        std::fs::write(
-                            path.join(format!("input{n}.json")),
-                            input.to_json_str()?.as_bytes(),
-                        )?;
-                    };
-
-                    if let Some(modules) = &case.modules {
-                        if modules.len() == 1 {
-                            std::fs::write(
-                                path.join(format!("rego{n}.rego")),
-                                modules[0].as_bytes(),
-                            )?;
-                        } else {
-                            for (i, m) in modules.iter().enumerate() {
-                                std::fs::write(
-                                    path.join(format!("rego{n}_{i}.rego")),
-                                    m.as_bytes(),
-                                )?;
-                            }
-                        }
-                    }
-
-                    std::fs::write(path.join(format!("query{n}.text")), case.query.as_bytes())?;
-
-                    println!("failed");
-                    entry.1 += 1;
-                    n += 1;
-                    continue;
-                }
-            };
-        }
-    }
-
-    println!("TESTSUITE STATUS");
-    println!("    {:40}  {:4} {:4}", "FOLDER", "PASS", "FAIL");
-    let (mut npass, mut nfail) = (0, 0);
-    for (dir, (pass, fail)) in status {
-        if fail == 0 {
-            println!("\x1b[32m    {dir:40}: {pass:4} {fail:4}\x1b[0m");
-        } else {
-            println!("\x1b[31m    {dir:40}: {pass:4} {fail:4}\x1b[0m");
-        }
-        npass += pass;
-        nfail += fail;
-    }
-    println!();
-    println!("\x1b[31m    {:40}: {npass:4} {nfail:4}\x1b[0m", "TOTAL");
-    Ok(())
-}

tests/tests.rs

@@ -3,7 +3,6 @@
 
 mod interpreter;
 mod lexer;
-mod opa;
 mod parser;
 mod scheduler;
 mod value;