kata-containers/regorus
1
0
Fork 0
mirror of https://github.com/microsoft/regorus.git synced 2026-08-05 02:16:11 +00:00
Code Issues Packages Projects Releases Wiki Activity

Ability to run the OPA testsuite (#39)

Browse Source 
`OPA_TESTS_DIR=path/to/testsuite cargo test opa -- --shot-output` to run opa tests.
Failing test cases are saved in target/opa/folder for investigation.

Signed-off-by: Anand Krishnamoorthi <anakrish@microsoft.com>
This commit is contained in:
Anand Krishnamoorthi
2023-11-09 15:11:33 -08:00
committed by GitHub
parent d69b413c8e
commit 0af8b6ea12
9 changed files with 186 additions and 193 deletions
Download Patch File Download Diff File Expand all files Collapse all files

159
tests/interpreter/mod.rs
View File

@@ -1,18 +1,12 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.
#![cfg(test)]
use std::env;
use std::path::Path;
use anyhow::{bail, Result};
use regorus::*;
use serde::{ser::SerializeMap, Deserialize, Deserializer, Serialize, Serializer};
use test_generator::test_resources;
use walkdir::WalkDir;
mod cases;
// Process test value specified in json/yaml to interpret special encodings.
pub fn process_value(v: &Value) -> Result<Value> {
@@ -191,91 +185,6 @@ fn push_query_results(query_results: QueryResults, results: &mut Vec<Value>) {
}
}
pub fn eval_file_first_rule(
regos: &[String],
data_opt: Option<Value>,
input_opt: Option<ValueOrVec>,
query: &str,
enable_tracing: bool,
) -> Result<Vec<Value>> {
let mut results = vec![];
let mut files = vec![];
let mut sources = vec![];
let mut modules = vec![];
let mut modules_ref = vec![];
for (idx, _) in regos.iter().enumerate() {
files.push(format!("rego_{idx}"));
}
for (idx, file) in files.iter().enumerate() {
let contents = regos[idx].as_str();
sources.push(Source::new(file.to_string(), contents.to_string()));
}
for source in &sources {
let mut parser = Parser::new(source)?;
modules.push(parser.parse()?);
}
for m in &modules {
modules_ref.push(m);
}
let query_source = regorus::Source::new("<query.rego>".to_string(), query.to_string());
let query_span = regorus::Span {
source: query_source.clone(),
line: 1,
col: 1,
start: 0,
end: query.len() as u16,
};
let mut parser = regorus::Parser::new(&query_source)?;
let query_node = parser.parse_query(query_span, "")?;
let query_schedule =
regorus::Analyzer::new().analyze_query_snippet(&modules_ref, &query_node)?;
let analyzer = Analyzer::new();
let schedule = analyzer.analyze(&modules_ref)?;
let mut interpreter = interpreter::Interpreter::new(&modules_ref)?;
if let Some(input) = input_opt {
// if inputs are defined then first the evaluation if prepared
interpreter.prepare_for_eval(Some(schedule), &data_opt)?;
// then all modules are evaluated for each input
let mut inputs = vec![];
match input {
ValueOrVec::Single(single_input) => inputs.push(single_input),
ValueOrVec::Many(mut many_input) => inputs.append(&mut many_input),
}
for input in inputs {
if let Some(module) = &modules.get(0) {
if let Some(rule) = &module.policy.get(0) {
interpreter.eval_rule_with_input(module, rule, &Some(input), enable_tracing)?;
}
}
// Now eval the query.
push_query_results(
interpreter.eval_user_query(&query_node, &query_schedule, enable_tracing)?,
&mut results,
);
}
} else {
// it no input is defined then one evaluation of all modules is performed
interpreter.eval(&data_opt, &None, enable_tracing, Some(schedule))?;
// Now eval the query
push_query_results(
interpreter.eval_user_query(&query_node, &query_schedule, enable_tracing)?,
&mut results,
);
}
Ok(results)
}
pub fn eval_file(
regos: &[String],
data_opt: Option<Value>,
@@ -415,7 +324,7 @@ struct YamlTest {
cases: Vec<TestCase>,
}
fn yaml_test_impl(file: &str, is_opa_test: bool) -> Result<()> {
fn yaml_test_impl(file: &str) -> Result<()> {
let yaml_str = std::fs::read_to_string(file)?;
let test: YamlTest = serde_yaml::from_str(&yaml_str)?;
@@ -430,7 +339,6 @@ fn yaml_test_impl(file: &str, is_opa_test: bool) -> Result<()> {
match (&case.want_result, &case.error) {
(Some(_), None) | (None, Some(_)) => (),
_ if is_opa_test => (),
_ => panic!("either want_result or error must be specified in test case."),
}
@@ -453,23 +361,10 @@ fn yaml_test_impl(file: &str, is_opa_test: bool) -> Result<()> {
}
}
if is_opa_test {
// Convert value to json compatible representation.
let results =
Value::from_json_str(serde_json::to_string(&results)?.as_str())?;
match_values(&results, &expected_results[0])?;
} else {
check_output(&results, &expected_results)?;
}
check_output(&results, &expected_results)?;
}
_ => bail!("eval succeeded and did not produce any errors"),
},
Err(actual) if is_opa_test => {
if case.want_error.is_none() && case.want_error_code.is_none() {
return Err(actual);
}
// opa test expects execution to fail and it did.
}
Err(actual) => match &case.error {
Some(expected) => {
let actual = actual.to_string();
@@ -492,8 +387,8 @@ fn yaml_test_impl(file: &str, is_opa_test: bool) -> Result<()> {
Ok(())
}
fn yaml_test(file: &str, is_opa_test: bool) -> Result<()> {
match yaml_test_impl(file, is_opa_test) {
fn yaml_test(file: &str) -> Result<()> {
match yaml_test_impl(file) {
Ok(_) => Ok(()),
Err(e) => {
// If Err is returned, it doesn't always get printed by cargo test.
@@ -505,20 +400,17 @@ fn yaml_test(file: &str, is_opa_test: bool) -> Result<()> {
#[test]
fn yaml_test_basic() -> Result<()> {
yaml_test("tests/interpreter/cases/basic_001.yaml", false)
yaml_test("tests/interpreter/cases/basic_001.yaml")
}
#[test]
#[ignore = "intended for use by scripts/yaml-test-eval"]
fn one_yaml() -> Result<()> {
let mut file = String::default();
let mut is_opa_test = false;
for a in env::args() {
if a.ends_with(".yaml") {
file = a;
} else if a == "opa-test" {
is_opa_test = true;
}
}
@@ -526,47 +418,10 @@ fn one_yaml() -> Result<()> {
bail!("missing <yaml-file>");
}
yaml_test(file.as_str(), is_opa_test)
yaml_test(file.as_str())
}
#[test_resources("tests/interpreter/**/*.yaml")]
fn run(path: &str) {
yaml_test(path, false).unwrap()
}
#[test]
#[ignore = "intended for running opa test suite"]
fn run_opa_tests() -> Result<()> {
let mut failures = vec![];
for a in env::args() {
if !Path::new(&a).is_dir() {
continue;
}
for entry in WalkDir::new(a)
.sort_by_file_name()
.into_iter()
.filter_map(|e| e.ok())
{
let path = entry.path().to_string_lossy().to_string();
if !Path::new(&path).is_file() || !path.ends_with(".yaml") {
continue;
}
let yaml = path;
match yaml_test_impl(yaml.as_str(), true) {
Ok(_) => (),
Err(e) => {
failures.push((yaml, e));
}
}
}
}
if !failures.is_empty() {
for (f, e) in failures {
println!("{f} failed.\n{e}");
}
panic!("failed");
}
Ok(())
yaml_test(path).unwrap()
}

19
tests/lexer/mod.rs
View File

@@ -4,7 +4,6 @@
use anyhow::{bail, Result};
use regorus::*;
use serde::{Deserialize, Serialize};
use std::env;
use test_generator::test_resources;
fn get_tokens(source: &Source) -> Result<Vec<Token>> {
@@ -155,24 +154,6 @@ fn yaml_test(file: &str) -> Result<()> {
}
}
#[test]
#[ignore = "intended for use by scripts/yaml-test-lex"]
fn one_yaml() -> Result<()> {
let mut file = String::default();
for a in env::args() {
if a.ends_with(".yaml") {
file = a;
break;
}
}
if file.is_empty() {
bail!("missing yaml test file");
}
yaml_test(file.as_str())
}
#[test_resources("tests/lexer/**/*.yaml")]
fn run(path: &str) {
yaml_test(path).unwrap()

163
tests/opa/mod.rs Normal file
View File

@@ -0,0 +1,163 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.
use regorus::*;
use std::collections::BTreeMap;
use std::path::Path;
use anyhow::Result;
use walkdir::WalkDir;
use serde::{Deserialize, Serialize};
#[derive(Serialize, Deserialize, PartialEq, Debug)]
struct TestCase {
data: Option<Value>,
input: Option<Value>,
modules: Option<Vec<String>>,
note: String,
query: String,
sort_bindings: Option<bool>,
want_result: Option<Value>,
skip: Option<bool>,
error: Option<String>,
traces: Option<bool>,
want_error: Option<String>,
want_error_code: Option<String>,
}
#[derive(Serialize, Deserialize, PartialEq, Debug)]
struct YamlTest {
cases: Vec<TestCase>,
}
fn eval_test_case(case: &TestCase) -> Result<Value> {
let mut engine = Engine::new();
if let Some(data) = &case.data {
engine.add_data(data.clone())?;
}
if let Some(input) = &case.input {
engine.set_input(input.clone());
}
if let Some(modules) = &case.modules {
for (idx, rego) in modules.iter().enumerate() {
engine.add_policy(format!("rego{idx}.rego"), rego.clone())?;
}
}
let query_results = engine.eval_query(case.query.clone(), true)?;
let mut values = vec![];
for qr in query_results.result {
values.push(if !qr.bindings.is_empty_object() {
qr.bindings.clone()
} else if let Some(v) = qr.expressions.last() {
v["value"].clone()
} else {
Value::Undefined
});
}
let result = Value::from_array(values);
// Make result json compatible. (E.g: avoid sets).
Value::from_json_str(&result.to_string())
}
#[test]
fn run_opa_tests() -> Result<()> {
let opa_tests_dir = match std::env::var("OPA_TESTS_DIR") {
Ok(v) => v,
_ => {
println!("OPA_TESTS_DIR environment vairable not defined.");
return Ok(());
}
};
dbg!(&opa_tests_dir);
let tests_path = Path::new(&opa_tests_dir);
let mut status = BTreeMap::<String, (u32, u32)>::new();
let mut n = 0;
for entry in WalkDir::new(&opa_tests_dir)
.sort_by_file_name()
.into_iter()
.filter_map(|e| e.ok())
{
let path_str = entry.path().to_string_lossy().to_string();
let path = Path::new(&path_str);
if !path.is_file() || !path_str.ends_with(".yaml") {
continue;
}
let path_dir = path.strip_prefix(tests_path)?.parent().unwrap();
let path_dir_str = path_dir.to_string_lossy().to_string();
let entry = status.entry(path_dir_str).or_insert((0, 0));
let yaml_str = std::fs::read_to_string(&path_str)?;
let test: YamlTest = serde_yaml::from_str(&yaml_str)?;
for case in &test.cases {
print!("{} ...", case.note);
match (eval_test_case(case), &case.want_result) {
(Ok(actual), Some(expected)) if &actual == expected => {
println!("passed");
entry.0 += 1;
}
(Err(_), None) if case.want_error.is_some() => {
// Expected failure.
println!("passed");
entry.0 += 1;
}
_ => {
let path = Path::new("target/opa").join(path_dir);
std::fs::create_dir_all(path.clone())?;
if let Some(data) = &case.data {
std::fs::write(
path.join(format!("data{n}.json")),
data.to_json_str()?.as_bytes(),
)?;
};
if let Some(input) = &case.input {
std::fs::write(
path.join(format!("input{n}.json")),
input.to_json_str()?.as_bytes(),
)?;
};
if let Some(modules) = &case.modules {
if modules.len() == 1 {
std::fs::write(
path.join(format!("rego{n}.rego")),
modules[0].as_bytes(),
)?;
} else {
for (i, m) in modules.iter().enumerate() {
std::fs::write(
path.join(format!("rego{n}_{i}.json")),
m.as_bytes(),
)?;
}
}
}
println!("failed");
entry.1 += 1;
n += 1;
continue;
}
};
}
}
println!("TESTSUITE STATUS");
println!(" {:30} {:4} {:4}", "FOLDER", "PASS", "FAIL");
for (dir, (pass, fail)) in status {
if fail == 0 {
println!("\x1b[32m {dir:40}: {pass:4} {fail:4}\x1b[0m");
} else {
println!("\x1b[31m {dir:40}: {pass:4} {fail:4}\x1b[0m");
}
}
Ok(())
}

19
tests/parser/mod.rs
View File

@@ -4,7 +4,6 @@
use anyhow::{anyhow, bail, Result};
use regorus::*;
use serde::{Deserialize, Serialize};
use std::env;
use test_generator::test_resources;
macro_rules! my_assert_eq {
@@ -700,24 +699,6 @@ fn yaml_test(file: &str) -> Result<()> {
}
}
#[test]
#[ignore = "intended for use by scripts/yaml-test-parse"]
fn one_yaml() -> Result<()> {
let mut file = String::default();
for a in env::args() {
if a.ends_with(".yaml") {
file = a;
break;
}
}
if file.is_empty() {
bail!("missing yaml test file");
}
yaml_test(file.as_str())
}
#[test_resources("tests/parser/**/*.yaml")]
fn run(path: &str) {
yaml_test(path).unwrap()

1
tests/tests.rs
View File

@@ -3,6 +3,7 @@
mod interpreter;
mod lexer;
mod opa;
mod parser;
mod scheduler;
mod value;