diff --git a/.gitignore b/.gitignore index d5d3532..33384ff 100644 --- a/.gitignore +++ b/.gitignore @@ -10,4 +10,7 @@ Cargo.lock **/*.rs.bk # vscode files -.vscode/ \ No newline at end of file +.vscode/ + +# worktrees +worktrees/ \ No newline at end of file diff --git a/Cargo.toml b/Cargo.toml index 89f3cc3..88f93f1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,9 +13,12 @@ base64url = ["dep:data-encoding"] crypto = ["dep:constant_time_eq", "dep:hmac", "dep:hex", "dep:md-5", "dep:sha1", "dep:sha2"] deprecated = [] hex = ["dep:data-encoding"] +http = [] +jwt = [] glob = ["dep:wax"] graph = [] jsonschema = ["dep:jsonschema"] +opa-runtime = [] regex = ["dep:regex"] semver = ["dep:semver"] uuid = ["dep:uuid"] @@ -30,12 +33,15 @@ full-opa = [ "glob", "graph", "hex", + "http", + "jwt", "jsonschema", + "opa-runtime", "regex", "semver", + "time", "uuid", "urlquery", - "time", "yaml" ] diff --git a/build.rs b/build.rs index f2d9c3f..ad7214c 100644 --- a/build.rs +++ b/build.rs @@ -3,6 +3,7 @@ use anyhow::Result; use std::path::Path; +use std::process::Command; fn main() -> Result<()> { // Copy hooks to appropriate location so that git will run them. @@ -11,5 +12,14 @@ fn main() -> Result<()> { std::fs::copy("./scripts/pre-commit", "./.git/hooks/pre-commit")?; std::fs::copy("./scripts/pre-push", "./.git/hooks/pre-push")?; } + + // Supply information as compile-time environment variables. + let output = Command::new("git") + .args(["rev-parse", "HEAD"]) + .output() + .unwrap(); + let git_hash = String::from_utf8(output.stdout).unwrap(); + println!("cargo:rustc-env=GIT_HASH={}", git_hash); + Ok(()) } diff --git a/src/builtins/aggregates.rs b/src/builtins/aggregates.rs index 0d6c226..c610172 100644 --- a/src/builtins/aggregates.rs +++ b/src/builtins/aggregates.rs @@ -21,7 +21,7 @@ pub fn register(m: &mut HashMap<&'static str, builtins::BuiltinFcn>) { m.insert("sum", (sum, 1)); } -fn count(span: &Span, params: &[Ref], args: &[Value], _strict: bool) -> Result { +fn count(span: &Span, params: &[Ref], args: &[Value], strict: bool) -> Result { ensure_args_count(span, "count", params, args, 1)?; Ok(Value::from(Number::from(match &args[0] { @@ -29,12 +29,13 @@ fn count(span: &Span, params: &[Ref], args: &[Value], _strict: bool) -> Re Value::Set(a) => a.len(), Value::Object(a) => a.len(), Value::String(a) => a.encode_utf16().count(), - a => { + a if strict => { let span = params[0].span(); bail!(span.error( format!("`count` requires array/object/set/string argument. Got `{a}`.").as_str() )) } + _ => return Ok(Value::Undefined), }))) } diff --git a/src/builtins/http.rs b/src/builtins/http.rs new file mode 100644 index 0000000..c7549ac --- /dev/null +++ b/src/builtins/http.rs @@ -0,0 +1,23 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use crate::ast::{Expr, Ref}; +use crate::builtins; +use crate::builtins::utils::ensure_args_count; + +use crate::lexer::Span; +use crate::value::Value; + +use std::collections::HashMap; + +use anyhow::Result; + +pub fn register(m: &mut HashMap<&'static str, builtins::BuiltinFcn>) { + m.insert("http.send", (send, 1)); +} + +fn send(span: &Span, params: &[Ref], args: &[Value], _strict: bool) -> Result { + let name = "http.send"; + ensure_args_count(span, name, params, args, 1)?; + Ok(Value::Undefined) +} diff --git a/src/builtins/jwt.rs b/src/builtins/jwt.rs new file mode 100644 index 0000000..30d4f7c --- /dev/null +++ b/src/builtins/jwt.rs @@ -0,0 +1,28 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use crate::ast::{Expr, Ref}; +use crate::builtins; +use crate::builtins::utils::ensure_args_count; + +use crate::lexer::Span; +use crate::value::Value; + +use std::collections::HashMap; + +use anyhow::Result; + +pub fn register(m: &mut HashMap<&'static str, builtins::BuiltinFcn>) { + m.insert("io.jwt.decode_verify", (jwt_decode_verify, 2)); +} + +fn jwt_decode_verify( + span: &Span, + params: &[Ref], + args: &[Value], + _strict: bool, +) -> Result { + let name = "io.jwt.decode_verify"; + ensure_args_count(span, name, params, args, 2)?; + Ok(Value::Undefined) +} diff --git a/src/builtins/mod.rs b/src/builtins/mod.rs index d8a03db..7fbb9c7 100644 --- a/src/builtins/mod.rs +++ b/src/builtins/mod.rs @@ -17,14 +17,21 @@ mod encoding; mod glob; #[cfg(feature = "graph")] mod graph; +#[cfg(feature = "http")] +mod http; +#[cfg(feature = "jwt")] +mod jwt; pub mod numbers; mod objects; +#[cfg(feature = "opa-runtime")] +mod opa; #[cfg(feature = "regex")] mod regex; #[cfg(feature = "semver")] mod semver; pub mod sets; mod strings; +#[cfg(feature = "time")] mod time; mod tracing; pub mod types; @@ -77,22 +84,24 @@ lazy_static! { //units::register(&mut m); types::register(&mut m); encoding::register(&mut m); - //token_signing::register(&mut m); - //token_verification::register(&mut m); + #[cfg(feature = "jwt")] + jwt::register(&mut m); #[cfg(feature = "time")] time::register(&mut m); #[cfg(feature = "crypto")] crypto::register(&mut m); //graphql::register(&mut m); - //http::register(&mut m); + #[cfg(feature = "http")] + http::register(&mut m); //net::register(&mut m); #[cfg(feature = "uuid")] uuid::register(&mut m); #[cfg(feature = "semver")] semver::register(&mut m); //rego::register(&mut m); - //opa::register(&mut m); + #[cfg(feature = "opa-runtime")] + opa::register(&mut m); debugging::register(&mut m); tracing::register(&mut m); units::register(&mut m); @@ -106,9 +115,10 @@ lazy_static! { pub fn must_cache(path: &str) -> Option<&'static str> { match path { + "opa.runtime" => Some("opa.runtime"), "rand.intn" => Some("rand.intn"), - "uuid.rfc4122" => Some("uuid.rfc4122"), "time.now_ns" => Some("time.now_ns"), + "uuid.rfc4122" => Some("uuid.rfc4122"), _ => None, } } diff --git a/src/builtins/opa.rs b/src/builtins/opa.rs new file mode 100644 index 0000000..363db4b --- /dev/null +++ b/src/builtins/opa.rs @@ -0,0 +1,130 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use crate::ast::{Expr, Ref}; +use crate::builtins; +use crate::builtins::utils::ensure_args_count; + +use crate::lexer::Span; +use crate::value::Value; + +use std::collections::{BTreeMap, HashMap}; + +use anyhow::Result; + +pub fn register(m: &mut HashMap<&'static str, builtins::BuiltinFcn>) { + m.insert("opa.runtime", (opa_runtime, 0)); +} + +fn opa_runtime(span: &Span, params: &[Ref], args: &[Value], _strict: bool) -> Result { + let name = "opa.runtime"; + ensure_args_count(span, name, params, args, 0)?; + let mut obj = BTreeMap::new(); + + obj.insert( + Value::String("commit".into()), + Value::String(env!("GIT_HASH").into()), + ); + + obj.insert( + Value::String("regorus-version".into()), + Value::String(env!("CARGO_PKG_VERSION").into()), + ); + + obj.insert( + Value::String("version".into()), + Value::String("0.60.0".into()), + ); + + // Emitting environment variables could lead to confidential data being leaked. + if false { + obj.insert( + Value::String("env".into()), + Value::from_map( + std::env::vars() + .map(|(k, v)| (Value::String(k.into()), Value::String(v.into()))) + .collect(), + ), + ); + } + + let features = [ + #[cfg(feature = "base64")] + "base64", + #[cfg(feature = "base64url")] + "base64url", + #[cfg(feature = "crypto")] + "crypto", + #[cfg(feature = "deprecated")] + "deprecated", + #[cfg(feature = "glob")] + "glob", + #[cfg(feature = "graph")] + "graph", + #[cfg(feature = "hex")] + "hex", + #[cfg(feature = "http")] + "http", + #[cfg(feature = "jwt")] + "jwt", + #[cfg(feature = "jsonschema")] + "jsonschema", + #[cfg(feature = "opa-runtime")] + "opa-runtime", + #[cfg(feature = "regex")] + "regex", + #[cfg(feature = "semver")] + "semver", + #[cfg(feature = "time")] + "time", + #[cfg(feature = "uuid")] + "uuid", + #[cfg(feature = "urlquery")] + "urlquery", + #[cfg(feature = "yaml")] + "yaml", + "", + ]; + + let features = &features[..features.len() - 1]; + obj.insert( + Value::String("features".into()), + Value::from_array( + features + .iter() + .map(|f| Value::String(f.to_string().into())) + .collect(), + ), + ); + + let mut builtins: Vec<&&str> = builtins::BUILTINS.keys().collect(); + builtins.sort(); + + obj.insert( + Value::String("builtins".into()), + Value::from_array( + builtins + .iter() + .map(|f| Value::String(f.to_string().into())) + .collect(), + ), + ); + + #[cfg(feature = "deprecated")] + { + let mut deprecated: Vec<&&str> = builtins::deprecated::DEPRECATED.keys().collect(); + deprecated.sort(); + + obj.insert( + Value::String("deprecated".into()), + Value::from_array( + deprecated + .iter() + .map(|f| Value::String(f.to_string().into())) + .collect(), + ), + ); + } + + Ok(Value::from_map(obj)) +} diff --git a/src/builtins/time.rs b/src/builtins/time.rs index fcbc0d9..b45471e 100644 --- a/src/builtins/time.rs +++ b/src/builtins/time.rs @@ -10,6 +10,7 @@ use crate::value::Value; use std::collections::HashMap; use anyhow::{anyhow, bail, Result}; + use chrono::{ DateTime, Datelike, Days, FixedOffset, Local, Months, NaiveDateTime, SecondsFormat, TimeZone, Timelike, Utc, Weekday, diff --git a/src/engine.rs b/src/engine.rs index 3d979a0..c0cb6fa 100644 --- a/src/engine.rs +++ b/src/engine.rs @@ -84,7 +84,6 @@ impl Engine { let analyzer = Analyzer::new(); let schedule = analyzer.analyze(&self.modules)?; - self.interpreter.init_with_document()?; self.interpreter.set_schedule(Some(schedule)); self.interpreter.set_modules(&self.modules); @@ -94,6 +93,10 @@ impl Engine { let init_data = self.interpreter.get_data_mut().clone(); self.interpreter.set_init_data(init_data); + // Initialize the with-document with initial data values. + // with-modifiers will be applied to this document. + self.interpreter.init_with_document()?; + self.interpreter .set_functions(gather_functions(&self.modules)?); self.interpreter.gather_rules()?; diff --git a/src/interpreter.rs b/src/interpreter.rs index 2829a0f..e165b41 100644 --- a/src/interpreter.rs +++ b/src/interpreter.rs @@ -21,6 +21,19 @@ type Scope = BTreeMap; type DefaultRuleInfo = (Ref, Option); type ContextExprs = (Option>, Option>); +type State = ( + Value, + Value, + Value, + BTreeSet>, + BTreeMap, +); + +#[derive(Debug, Clone)] +enum FunctionModifier { + Function(String), + Value(Value), +} #[derive(Clone)] pub struct Interpreter { @@ -32,7 +45,7 @@ pub struct Interpreter { data: Value, init_data: Value, with_document: Value, - with_functions: BTreeMap, + with_functions: BTreeMap, scopes: Vec, // TODO: handle recursive calls where same expr could have different values. loop_var_values: BTreeMap, @@ -1162,6 +1175,127 @@ impl Interpreter { }) } + fn apply_with_modifiers(&mut self, stmt: &LiteralStmt) -> Result<(Option, bool)> { + if !stmt.with_mods.is_empty() { + // Save state; + let with_document = self.with_document.clone(); + let input = self.input.clone(); + let data = self.data.clone(); + let processed = self.processed.clone(); + let with_functions = self.with_functions.clone(); + + self.processed.clear(); + + let mut skip_exec = false; + // Apply with modifiers. + for wm in &stmt.with_mods { + let path = Parser::get_path_ref_components(&wm.refr)?; + let path: Vec<&str> = path.iter().map(|s| s.text()).collect(); + let mut target = path.join("."); + + let mut target_is_function = self.lookup_function_by_name(&target).is_some() + || matches!(self.lookup_builtin(wm.refr.span(), &target), Ok(Some(_))); + + if !target_is_function + && !target.starts_with("data.") + && !target.starts_with("input.") + && target != "input" + { + // target must be a function. + if self.lookup_function_by_name(&target).is_none() + && !matches!(self.lookup_builtin(wm.refr.span(), &target), Ok(Some(_))) + { + // Prefix target with current module path. + target = self.current_module_path.clone() + "." + ⌖ + if self.lookup_function_by_name(&target).is_none() { + bail!(wm.refr.span().error("undefined rule")); + } + target_is_function = true; + } + } + + if target_is_function { + match self.eval_expr(&wm.r#as) { + Ok(v) if v != Value::Undefined => { + // Function replaced by value. + self.with_functions + .insert(target, FunctionModifier::Value(v)); + } + _ => { + // Function replaced by another function. + // Lookup by with current module path prefixed. + let mut function_path = + get_path_string(&wm.r#as, Some(&self.current_module_path))?; + if self.lookup_function_by_name(&function_path).is_none() { + // Lookup without current module path prefixed. + function_path = get_path_string(&wm.r#as, None)?; + if self.lookup_function_by_name(&function_path).is_none() + && !matches!( + self.lookup_builtin(wm.r#as.span(), &function_path), + Ok(Some(_)) + ) + { + // bail!(wm.r#as.span().error("could not evaluate expression")); + skip_exec = true; + } + } + self.with_functions + .insert(target, FunctionModifier::Function(function_path)); + } + } + } else { + let value = self.eval_expr(&wm.r#as)?; + skip_exec = value == Value::Undefined; + if path[0] == "input" || path[0] == "data" { + // Override existing values in case of conflict. + let mut obj = &mut self.with_document; + for p in &path[0..path.len()] { + if !matches!(obj, Value::Object(_)) { + *obj = Value::new_object(); + } + + obj = obj + .as_object_mut()? + .entry(Value::String(p.to_string().into())) + .or_insert(Value::new_object()); + } + *obj = value; + // Mark modified rules as processed. + if let Some(rules) = self.rules.get(&target) { + for r in rules { + self.processed.insert(r.clone()); + } + } + } else { + bail!(wm.refr.span().error("not a valid target for with modifier")); + } + } + } + + self.data = self.with_document["data"].clone(); + self.input = self.with_document["input"].clone(); + Ok(( + Some((with_document, input, data, processed, with_functions)), + skip_exec, + )) + } else { + Ok((None, false)) + } + } + + fn restore_state(&mut self, saved_state: Option) -> Result<()> { + if let Some(s) = saved_state { + ( + self.with_document, + self.input, + self.data, + self.processed, + self.with_functions, + ) = s; + } + Ok(()) + } + fn eval_stmt(&mut self, stmt: &LiteralStmt, stmts: &[&LiteralStmt]) -> Result { debug_new_group!( "eval_stmt {}:{} {}", @@ -1170,94 +1304,14 @@ impl Interpreter { stmt.span.text() ); - let mut skip_exec = false; - let saved_state = if !stmt.with_mods.is_empty() { - // Save state; - let with_document = self.with_document.clone(); - let input = self.input.clone(); - let data = self.data.clone(); - let processed = self.processed.clone(); - let with_functions = self.with_functions.clone(); - - // Apply with modifiers. - for wm in &stmt.with_mods { - let path = Parser::get_path_ref_components(&wm.refr)?; - let path: Vec<&str> = path.iter().map(|s| s.text()).collect(); - let target = path.join("."); - - let value = match self.eval_expr(&wm.r#as) { - Ok(Value::Undefined) => { - if let Ok(fcn_path) = get_path_string(&wm.r#as, None) { - let span = wm.r#as.span(); - if self.lookup_function_by_name(&fcn_path).is_some() { - let mut fcn_path = get_path_string(&wm.r#as, None)?; - if !fcn_path.starts_with("data.") { - fcn_path = self.current_module_path.clone() + "." + &fcn_path; - } - self.with_functions.insert(target, fcn_path); - continue; - } else if self.lookup_builtin(span, &fcn_path).is_ok() { - self.with_functions.insert(target, fcn_path); - continue; - } - } - skip_exec = true; - continue; - } - Ok(v) => v, - Err(e) => return Err(e), - }; - - if path[0] == "input" || path[0] == "data" { - if path.len() > 1 { - let vref = - Self::make_or_get_value_mut(&mut self.with_document, &path[0..1])?; - match *vref { - Value::Object(_) => (), - _ => *vref = Value::new_object(), - } - } - - *Self::make_or_get_value_mut(&mut self.with_document, &path[..])? = value; - } - - /* else if path.len() == 1 { - // TODO: handle var in current module. - } else { - // TODO: error about input, data - } */ - } - - self.data = self.with_document["data"].clone(); - self.input = self.with_document["input"].clone(); - self.processed.clear(); - (with_document, input, data, processed, with_functions) - } else { - ( - Value::Undefined, - Value::Undefined, - Value::Undefined, - BTreeSet::new(), - BTreeMap::new(), - ) - }; - + let (saved_state, skip_exec) = self.apply_with_modifiers(stmt)?; let r = if !skip_exec { self.eval_stmt_impl(stmt, stmts) } else { Ok(false) }; - // Restore state. - if saved_state.0 != Value::Undefined { - ( - self.with_document, - self.input, - self.data, - self.processed, - self.with_functions, - ) = saved_state; - } + self.restore_state(saved_state)?; r } @@ -1282,6 +1336,9 @@ impl Interpreter { let loop_expr = &loops[0]; let mut result = false; + // Apply with modifiers before evaluating the loop expression. + let (saved_state, _) = self.apply_with_modifiers(stmts[0])?; + let loop_expr_value = loop_expr.value(); let loop_expr_value = if let Expr::Call { span, fcn, params } = loop_expr_value.as_ref() { @@ -1302,7 +1359,11 @@ impl Interpreter { self.eval_expr(&loop_expr_value)? }; - // If the loop's index variable has already been assigned a value + // Restore with modifiers. + // TODO: Delay this restore so that the stmt doesn't have to apply with modifiers again. + self.restore_state(saved_state)?; + + // If the loop's index variable h bail!(span.error("invalid function expression")), }; + let mut param_values = Vec::with_capacity(params.len()); + let mut error = None; + for p in params { + match self.eval_expr(p) { + Ok(v) => param_values.push(v), + Err(e) => { + error = Some(Err(e)); + break; + } + } + } + let orig_fcn_path = fcn_path; - let fcn_path = match self.with_functions.remove(&orig_fcn_path) { - Some(p) => p, + let mut with_functions_saved = None; + let fcn_path = match self.with_functions.get(&orig_fcn_path) { + Some(FunctionModifier::Function(p)) => { + let p = p.clone(); + with_functions_saved = Some(self.with_functions.clone()); + self.with_functions.clear(); + p + } + Some(FunctionModifier::Value(v)) => { + if param_values.iter().any(|v| v == &Value::Undefined) { + return Ok(Value::Undefined); + } + if let Some(err) = error { + err?; + }; + return Ok(v.clone()); + } _ => orig_fcn_path.clone(), }; @@ -1896,8 +1984,8 @@ impl Interpreter { // Look up builtin function. else if let Ok(Some(builtin)) = self.lookup_builtin(span, &fcn_path) { let r = self.eval_builtin_call(span, &fcn_path.clone(), *builtin, params); - if orig_fcn_path != fcn_path { - self.with_functions.insert(orig_fcn_path, fcn_path); + if let Some(with_functions) = with_functions_saved { + self.with_functions = with_functions; } return r; } else { @@ -1905,19 +1993,17 @@ impl Interpreter { } } }; + if param_values.iter().any(|v| v == &Value::Undefined) { + if let Some(with_functions) = with_functions_saved { + self.with_functions = with_functions; + } + return Ok(Value::Undefined); + } let fcns = fcns_rules.clone(); let mut results: Vec = Vec::new(); let mut errors: Vec = Vec::new(); - let mut param_values = Vec::with_capacity(params.len()); - for p in params { - let v = self.eval_expr(p)?; - if v == Value::Undefined { - return Ok(v); - } - param_values.push(v); - } 'outer: for fcn_rule in fcns { let (args, output_expr, bodies) = match fcn_rule.as_ref() { @@ -2000,6 +2086,8 @@ impl Interpreter { // If the function successfully executed, but did not return any value, then return true. Value::Set(s) if s.is_empty() && output_expr.is_none() => Value::Bool(true), + Value::Set(s) if s.is_empty() => Value::Undefined, + // If the function execution resulted in undefined, then propagate it. Value::Undefined => Value::Undefined, @@ -2015,6 +2103,10 @@ impl Interpreter { } } + if self.strict_builtin_errors && !errors.is_empty() { + return Err(anyhow!(errors[0].to_string())); + } + if results.is_empty() { // Back up local variables of current function and empty // the local variables of callee function. @@ -2044,6 +2136,10 @@ impl Interpreter { self.scopes = scopes; } + if let Some(with_functions) = with_functions_saved { + self.with_functions = with_functions; + } + if results.is_empty() { if errors.is_empty() { return Ok(Value::Undefined); @@ -2061,10 +2157,6 @@ impl Interpreter { )); } - if orig_fcn_path != fcn_path { - self.with_functions.insert(orig_fcn_path, fcn_path); - } - Ok(results[0].clone()) } @@ -2122,6 +2214,10 @@ impl Interpreter { fn ensure_module_evaluated(&mut self, path: String) -> Result<()> { for module in self.modules.clone() { + if Some(&module) == self.module.as_ref() { + // Prevent cyclic evaluation. + continue; + } let module_path = get_path_string(&module.package.refr, Some("data"))?; if module_path.starts_with(&path) && (module_path.len() == path.len() @@ -2138,7 +2234,9 @@ impl Interpreter { } for rule in &module.policy { - self.eval_rule(&module, rule)?; + if !self.processed.contains(rule) { + self.eval_rule(&module, rule)?; + } } } } @@ -2165,7 +2263,6 @@ impl Interpreter { } } } - Ok(()) } @@ -2193,19 +2290,20 @@ impl Interpreter { // Ensure that rules are evaluated if name.text() == "data" { - let v = Self::get_value_chained(self.data.clone(), fields); + // With modifiers may be used to specify part of a module that that not yet been + // evaluated. Therefore ensure that module is evaluated first. + let path = "data.".to_owned() + &fields.join("."); + self.ensure_module_evaluated(path)?; // If the rule has already been evaluated or specified via a with modifier, // use that value. + let v = Self::get_value_chained(self.data.clone(), fields); + if v != Value::Undefined { debug!("returning v = {v}"); return Ok(v); } - if fields.is_empty() { - bail!(span.error("this results in recursive evaluation of data.")) - } - // Find the rule to which the var being looked up corresponds to. This is the prefix for // which rules exist. let mut found = false; @@ -2781,7 +2879,9 @@ impl Interpreter { self.processed.insert(rule.clone()); } - RuleHead::Func { refr, .. } => { + RuleHead::Func { + refr, args, assign, .. + } => { let mut path = Parser::get_path_ref_components(&self.current_module()?.package.refr)?; @@ -2798,6 +2898,20 @@ impl Interpreter { Value::new_object(), )?; } + + if args.is_empty() { + let ctx = Context { + key_expr: None, + output_expr: assign.as_ref().map(|a| a.value.clone()), + value: Value::new_array(), + result: None, + results: QueryResults::default(), + is_compr: false, + }; + + let value = self.eval_rule_bodies(ctx, span, rule_body)?; + self.update_data(refr.span(), refr, &path[..], value)?; + } } } } diff --git a/src/scheduler.rs b/src/scheduler.rs index 762d46c..9f677bc 100644 --- a/src/scheduler.rs +++ b/src/scheduler.rs @@ -3,6 +3,7 @@ use crate::ast::Expr::*; use crate::ast::*; +use crate::builtins; use crate::lexer::*; use crate::utils::*; @@ -555,6 +556,10 @@ impl Analyzer { ) -> Result<()> { // First process assign, some expressions and gather local vars. for stmt in &query.stmts { + for wm in &stmt.with_mods { + gather_input_vars(&wm.r#as, &self.scopes, scope)?; + gather_loop_vars(&wm.r#as, &self.scopes, scope)?; + } match &stmt.literal { Literal::SomeVars { vars, .. } => vars.iter().for_each(|v| { scope.locals.insert(v.source_str(), v.clone()); @@ -622,6 +627,7 @@ impl Analyzer { ) -> Result<(Vec, Vec>)> { let mut used_vars = vec![]; let mut comprs = vec![]; + let full_expr = expr; traverse(expr, &mut |e| match e.as_ref() { Var(v) if !matches!(v.text(), "_" | "input" | "data") => { let name = v.source_str(); @@ -638,7 +644,15 @@ impl Analyzer { first_use.entry(name).or_insert(v.clone()); } } else if !scope.inputs.contains(&name) { - bail!(v.error(format!("use of undefined variable `{name}` is unsafe").as_str())); + match get_path_string(full_expr, None) { + Ok(path) + if builtins::BUILTINS.contains_key(path.as_str()) + || builtins::deprecated::DEPRECATED.contains_key(path.as_str()) => { + } + _ => bail!(v.error( + format!("use of undefined variable `{name}` is unsafe").as_str() + )), + } } Ok(false) } @@ -758,6 +772,7 @@ impl Analyzer { Ok(vars) } + #[allow(clippy::too_many_arguments)] fn process_assign_expr( &mut self, op: &AssignOp, @@ -766,6 +781,8 @@ impl Analyzer { scope: &mut Scope, first_use: &mut BTreeMap, definitions: &mut Vec>, + mut with_mods_used_vars: Vec, + mut with_mods_comprs: Vec>, ) -> Result<()> { let empty_str = lhs.span().source_str().clone_empty(); match (lhs.as_ref(), rhs.as_ref()) { @@ -790,6 +807,8 @@ impl Analyzer { scope, first_use, definitions, + with_mods_used_vars.clone(), + with_mods_comprs.clone(), )?; } return Ok(()); @@ -797,13 +816,15 @@ impl Analyzer { // TODO: object _ => { { - let (mut used_vars, comprs) = Self::gather_used_vars_comprs_index_vars( + let (mut used_vars, mut comprs) = Self::gather_used_vars_comprs_index_vars( rhs, scope, first_use, definitions, &None, )?; + used_vars.append(&mut with_mods_used_vars.clone()); + comprs.append(&mut with_mods_comprs.clone()); self.process_comprs(&comprs[..], scope, first_use, &mut used_vars)?; let check_first_use = *op == AssignOp::ColEq; let assigned_vars = @@ -824,13 +845,15 @@ impl Analyzer { } } { - let (mut used_vars, comprs) = Self::gather_used_vars_comprs_index_vars( + let (mut used_vars, mut comprs) = Self::gather_used_vars_comprs_index_vars( lhs, scope, first_use, definitions, &None, )?; + used_vars.append(&mut with_mods_used_vars); + comprs.append(&mut with_mods_comprs); let check_first_use = false; self.process_comprs(&comprs[..], scope, first_use, &mut used_vars)?; let assigned_vars = @@ -861,20 +884,32 @@ impl Analyzer { scope: &mut Scope, first_use: &mut BTreeMap, definitions: &mut Vec>, + mut with_mods_used_vars: Vec, + mut with_mods_comprs: Vec>, ) -> Result<()> { match expr.as_ref() { - AssignExpr { op, lhs, rhs, .. } => { - self.process_assign_expr(op, lhs, rhs, scope, first_use, definitions) - } + AssignExpr { op, lhs, rhs, .. } => self.process_assign_expr( + op, + lhs, + rhs, + scope, + first_use, + definitions, + with_mods_used_vars, + with_mods_comprs, + ), _ => { - let (mut used_vars, comprs) = Self::gather_used_vars_comprs_index_vars( + let (mut used_vars, mut comprs) = Self::gather_used_vars_comprs_index_vars( expr, scope, first_use, definitions, &None, )?; + comprs.append(&mut with_mods_comprs); + used_vars.append(&mut with_mods_used_vars); self.process_comprs(&comprs[..], scope, first_use, &mut used_vars)?; + definitions.push(Definition { var: expr.span().source_str().clone_empty(), used_vars, @@ -935,6 +970,20 @@ impl Analyzer { let mut first_use = BTreeMap::new(); for stmt in &query.stmts { let mut definitions = vec![]; + let mut with_mods_used_vars = vec![]; + let mut with_mods_comprs = vec![]; + for wm in &stmt.with_mods { + let (mut used_vars, mut comprs) = Self::gather_used_vars_comprs_index_vars( + &wm.r#as, + &mut scope, + &mut first_use, + &mut definitions, + &None, + )?; + + with_mods_used_vars.append(&mut used_vars); + with_mods_comprs.append(&mut comprs); + } match &stmt.literal { Literal::SomeVars { vars, .. } => { for v in vars { @@ -968,13 +1017,16 @@ impl Analyzer { )?; let mut col_definitions = vec![]; - let (mut col_used_vars, col_comprs) = Self::gather_used_vars_comprs_index_vars( - collection, - &mut scope, - &mut first_use, - &mut col_definitions, - &None, - )?; + let (mut col_used_vars, mut col_comprs) = + Self::gather_used_vars_comprs_index_vars( + collection, + &mut scope, + &mut first_use, + &mut col_definitions, + &None, + )?; + col_used_vars.append(&mut with_mods_used_vars); + col_comprs.append(&mut with_mods_comprs.clone()); definitions.append(&mut col_definitions); self.process_comprs( @@ -995,13 +1047,16 @@ impl Analyzer { let mut used_vars = vec![]; for e in non_vars { let mut definitions = vec![]; - let (uv, comprs) = Self::gather_used_vars_comprs_index_vars( + let (mut uv, mut comprs) = Self::gather_used_vars_comprs_index_vars( &e, &mut scope, &mut first_use, &mut definitions, &None, )?; + + uv.append(&mut with_mods_used_vars.clone()); + comprs.append(&mut with_mods_comprs.clone()); if !definitions.is_empty() { bail!("internal error: non empty definitions"); } @@ -1035,13 +1090,15 @@ impl Analyzer { } // Gather vars being used. - let (mut used_vars, comprs) = Self::gather_used_vars_comprs_index_vars( + let (mut used_vars, mut comprs) = Self::gather_used_vars_comprs_index_vars( expr, &mut scope, &mut first_use, &mut definitions, &Some(&extras_scope.unscoped), )?; + used_vars.append(&mut with_mods_used_vars); + comprs.append(&mut with_mods_comprs); self.process_comprs( &comprs[..], @@ -1064,11 +1121,25 @@ impl Analyzer { }); } } else { - self.process_expr(expr, &mut scope, &mut first_use, &mut definitions)?; + self.process_expr( + expr, + &mut scope, + &mut first_use, + &mut definitions, + with_mods_used_vars, + with_mods_comprs, + )?; } } Literal::NotExpr { expr, .. } => { - self.process_expr(expr, &mut scope, &mut first_use, &mut definitions)?; + self.process_expr( + expr, + &mut scope, + &mut first_use, + &mut definitions, + with_mods_used_vars, + with_mods_comprs, + )?; } Literal::Every { key, @@ -1078,13 +1149,15 @@ impl Analyzer { .. } => { // Create dependencies for vars used in domain. - let (mut uv, comprs) = Self::gather_used_vars_comprs_index_vars( + let (mut uv, mut comprs) = Self::gather_used_vars_comprs_index_vars( domain, &mut scope, &mut first_use, &mut definitions, &None, )?; + uv.append(&mut with_mods_used_vars); + comprs.append(&mut with_mods_comprs); self.process_comprs(&comprs[..], &mut scope, &mut first_use, &mut uv)?; definitions.push(Definition { var: empty_str.clone(), diff --git a/tests/interpreter/cases/call/or.yaml b/tests/interpreter/cases/call/or.yaml index 42982ad..62c4570 100644 --- a/tests/interpreter/cases/call/or.yaml +++ b/tests/interpreter/cases/call/or.yaml @@ -132,6 +132,7 @@ cases: want_result: a1: "hello world" a2: 6 + strict: false - note: or-all-error data: {} diff --git a/tests/interpreter/mod.rs b/tests/interpreter/mod.rs index 2047810..61310c5 100644 --- a/tests/interpreter/mod.rs +++ b/tests/interpreter/mod.rs @@ -138,8 +138,10 @@ pub fn eval_file( input_opt: Option, query: &str, enable_tracing: bool, + strict: bool, ) -> Result> { let mut engine: Engine = engine::Engine::new(); + engine.set_strict_builtin_errors(strict); let mut results = vec![]; let mut files = vec![]; @@ -237,6 +239,12 @@ struct TestCase { traces: Option, want_error: Option, want_error_code: Option, + #[serde(default = "default_strict")] + strict: bool, +} + +fn default_strict() -> bool { + true } #[derive(Serialize, Deserialize, PartialEq, Debug)] @@ -270,6 +278,7 @@ fn yaml_test_impl(file: &str) -> Result<()> { case.input, case.query.as_str(), enable_tracing, + case.strict, ) { Ok(results) => match case.want_result { Some(want_result) => { diff --git a/tests/opa.passing b/tests/opa.passing index 97ed1b7..fdd3682 100644 --- a/tests/opa.passing +++ b/tests/opa.passing @@ -6,6 +6,7 @@ array assignments base64builtins base64urlbuiltins +baseandvirtualdocs bitsand bitsnegate bitsor @@ -47,7 +48,6 @@ indexing indirectreferences inputvalues intersection -invalidkeyerror jsonbuiltins jsonfilter jsonfilteridempotent @@ -110,4 +110,5 @@ urlbuiltins uuid varreferences virtualdocs -walkbuiltin \ No newline at end of file +walkbuiltin +withkeyword \ No newline at end of file diff --git a/tests/opa.rs b/tests/opa.rs index 05d3599..9fba8e7 100644 --- a/tests/opa.rs +++ b/tests/opa.rs @@ -13,7 +13,7 @@ use serde::{Deserialize, Serialize}; use walkdir::WalkDir; const OPA_REPO: &str = "https://github.com/open-policy-agent/opa"; -const OPA_BRANCH: &str = "v0.58.0"; +const OPA_BRANCH: &str = "v0.60.0"; #[derive(Serialize, Deserialize, PartialEq, Debug)] #[serde(deny_unknown_fields)] @@ -85,7 +85,19 @@ fn eval_test_case(case: &TestCase) -> Result { let mut values = vec![]; for qr in query_results.result { values.push(if !qr.bindings.is_empty_object() { - qr.bindings.clone() + if case.sort_bindings == Some(true) { + let mut v = qr.bindings.clone(); + let bindings = v.as_object_mut()?; + for (_, v) in bindings.iter_mut() { + match v { + Value::Array(_) => v.as_array_mut()?.sort(), + _ => (), + } + } + v + } else { + qr.bindings.clone() + } } else if let Some(v) = qr.expressions.last() { v.value.clone() } else { @@ -168,7 +180,14 @@ fn run_opa_tests(opa_tests_dir: String, folders: &[String]) -> Result<()> { ] }]"#, )?; - }; + } else if case.note == "withkeyword/builtin-builtin: arity 0" { + // The test expects empty object to be returned by opa.runtime. + // This cannot happen. + // Skip the test. + println!("skipping impossible test: {}", case.note); + continue; + } + match (eval_test_case(&case), &case.want_result) { (Ok(actual), Some(expected)) if is_json_schema_test && json_schema_tests_check(&actual, &expected) =>