From ab968c23866c2682656f1274c6ca5da92020af1b Mon Sep 17 00:00:00 2001 From: Anand Krishnamoorthi <35780660+anakrish@users.noreply.github.com> Date: Sun, 3 Dec 2023 12:08:17 -0800 Subject: [PATCH] Regex and Glob builtins (#56) Signed-off-by: Anand Krishnamoorthi --- Cargo.toml | 5 +- src/builtins/glob.rs | 112 ++++++++++++++++++++++++++++++ src/builtins/mod.rs | 3 +- src/builtins/regex.rs | 155 ++++++++++++++++++++++++++++++++++++++++-- src/parser.rs | 150 +++++++++++++++++++++++----------------- tests/opa.passing | 5 ++ tests/opa.rs | 2 +- 7 files changed, 362 insertions(+), 70 deletions(-) create mode 100644 src/builtins/glob.rs diff --git a/Cargo.toml b/Cargo.toml index 1f7593c..f9e0443 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,7 +18,10 @@ rand = "0.8.5" data-encoding = "2.4.0" regex = "1.10.2" num = "0.4.1" -rust_decimal = { version = "1.33.1", features = ["serde-with-arbitrary-precision", "serde_json", "maths"] } +rust_decimal = { version = "1.33.1", features = ["serde-with-arbitrary-precision"] } +glob = "0.3.1" +wax = { version = "0.6.0", features = [], default-features = false } + [dev-dependencies] clap = { version = "4.4.7", features = ["derive"] } diff --git a/src/builtins/glob.rs b/src/builtins/glob.rs new file mode 100644 index 0000000..b55b2a3 --- /dev/null +++ b/src/builtins/glob.rs @@ -0,0 +1,112 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use crate::ast::{Expr, Ref}; +use crate::builtins; +use crate::builtins::utils::{ensure_args_count, ensure_string, ensure_string_collection}; +use crate::lexer::Span; +use crate::value::Value; + +use std::collections::HashMap; + +use anyhow::{bail, Result}; +//use glob::{Pattern, MatchOptions}; +use wax::{Glob, Pattern}; + +pub fn register(m: &mut HashMap<&'static str, builtins::BuiltinFcn>) { + m.insert("glob.match", (glob_match, 3)); + m.insert("glob.quote_meta", (quote_meta, 1)); +} + +const PLACE_HOLDER: &str = "\0"; + +fn suppress_unix_style_delimiter(s: &str) -> Result { + // Replace unix-style delimiter with placeholder so that no delimiter is + // encountered during glob matching. + Ok(s.replace('/', PLACE_HOLDER)) +} + +fn make_delimiters_unix_style(s: &str, delimiters: &[char]) -> Result { + if s.contains(PLACE_HOLDER) { + bail!("string contains internal glob placeholder"); + } + + let has_unix_style = delimiters.contains(&'/'); + + let mut s = if !has_unix_style { + suppress_unix_style_delimiter(s)? + } else { + s.to_string() + }; + + for d in delimiters { + if *d == ':' { + s = s.replace(*d, PLACE_HOLDER); + } else if *d != '/' { + // Insert / before occurances of delimiter. + s = s.replace(*d, format!("/{}/", *d).as_str()); + } + } + + Ok(s) +} + +fn make_glob<'a>(pattern: &'a str, span: &'a Span) -> Result> { + Glob::new(pattern).or_else(|_| bail!(span.error("invalid glob"))) +} + +fn glob_match(span: &Span, params: &[Ref], args: &[Value]) -> Result { + let name = "glob.match"; + ensure_args_count(span, name, params, args, 3)?; + + let pattern = ensure_string(name, ¶ms[0], &args[0])?; + let value = ensure_string(name, ¶ms[2], &args[2])?; + + let pattern = pattern.as_ref(); + let value = value.as_ref(); + + if let Value::Null = &args[1] { + // Ensure that / is not treated as a delimiter. + let value = suppress_unix_style_delimiter(value)?; + let pattern = suppress_unix_style_delimiter(pattern)?; + + let glob = make_glob(&pattern, params[0].span())?; + return Ok(Value::Bool(glob.is_match(&value[..]))); + } + + let delimiters = if let Value::Array(_) = &args[1] { + ensure_string_collection(name, ¶ms[1], &args[1])? + } else { + bail!(params[1] + .span() + .error(format!("{name} requires string array").as_str())); + }; + + if delimiters.iter().any(|d| d.len() > 1) { + bail!(params[1] + .span() + .error("delimiters must be single character")); + } + + let mut delimiters: Vec = delimiters.iter().filter_map(|d| d.chars().next()).collect(); + + if delimiters.is_empty() { + delimiters.push('.'); + } + + let pattern = make_delimiters_unix_style(pattern, &delimiters)?; + let value = make_delimiters_unix_style(value, &delimiters)?; + + let glob = make_glob(&pattern, params[0].span())?; + Ok(Value::Bool(glob.is_match(&value[..]))) +} + +fn quote_meta(span: &Span, params: &[Ref], args: &[Value]) -> Result { + let name = "glob.quote_meta"; + ensure_args_count(span, name, params, args, 1)?; + + let pattern = ensure_string(name, ¶ms[0], &args[0])?; + // Ensure that the glob is valid. + let _ = make_glob(&pattern, params[0].span())?; + Ok(Value::String(pattern.as_ref().replace('*', "\\*").into())) +} diff --git a/src/builtins/mod.rs b/src/builtins/mod.rs index afd2133..ed7c04f 100644 --- a/src/builtins/mod.rs +++ b/src/builtins/mod.rs @@ -10,6 +10,7 @@ mod debugging; pub mod deprecated; mod encoding; +mod glob; pub mod numbers; mod objects; mod regex; @@ -48,7 +49,7 @@ lazy_static! { objects::register(&mut m); strings::register(&mut m); regex::register(&mut m); - //glob::register(&mut m); + glob::register(&mut m); bitwise::register(&mut m); conversions::register(&mut m); //units::register(&mut m); diff --git a/src/builtins/regex.rs b/src/builtins/regex.rs index 110b5a9..6e5de98 100644 --- a/src/builtins/regex.rs +++ b/src/builtins/regex.rs @@ -3,7 +3,7 @@ use crate::ast::{Expr, Ref}; use crate::builtins; -use crate::builtins::utils::{ensure_args_count, ensure_string}; +use crate::builtins::utils::{ensure_args_count, ensure_numeric, ensure_string}; use crate::lexer::Span; use crate::value::Value; @@ -13,9 +13,89 @@ use anyhow::{bail, Result}; use regex::Regex; pub fn register(m: &mut HashMap<&'static str, builtins::BuiltinFcn>) { + m.insert( + "regex.find_all_string_submatch_n", + (find_all_string_submatch_n, 3), + ); + m.insert("regex.find_n", (find_n, 3)); + // TODO: m.insert("regex.globs_match", (globs_match, 2)); m.insert("regex.is_valid", (is_valid, 1)); m.insert("regex.match", (regex_match, 2)); + m.insert("regex.replace", (regex_replace, 3)); m.insert("regex.split", (regex_split, 2)); + m.insert("regex.template_match", (regex_template_match, 4)); +} + +fn find_all_string_submatch_n(span: &Span, params: &[Ref], args: &[Value]) -> Result { + let name = "regex.find_all_string_submatch_n"; + ensure_args_count(span, name, params, args, 3)?; + + let pattern = ensure_string(name, ¶ms[0], &args[0])?; + let value = ensure_string(name, ¶ms[1], &args[1])?; + let n = ensure_numeric(name, ¶ms[2], &args[2])?; + + let pattern = + Regex::new(&pattern).or_else(|_| bail!(params[0].span().error("invalid regex")))?; + + if !n.is_integer() { + bail!(params[2].span().error("n must be an integer")); + } + + let n = match n.as_i64() { + Some(n) if n < 0 => usize::MAX, + Some(n) => n as usize, + None => usize::MAX, + }; + + Ok(Value::from_array( + pattern + .captures_iter(&value) + .map(|capture| { + Value::from_array( + capture + .iter() + .map(|group| { + Value::String(match group { + Some(s) => s.as_str().into(), + _ => "".into(), + }) + }) + .collect(), + ) + }) + .take(n) + .collect(), + )) +} + +fn find_n(span: &Span, params: &[Ref], args: &[Value]) -> Result { + let name = "regex.find_n"; + ensure_args_count(span, name, params, args, 3)?; + + let pattern = ensure_string(name, ¶ms[0], &args[0])?; + let value = ensure_string(name, ¶ms[1], &args[1])?; + let n = ensure_numeric(name, ¶ms[2], &args[2])?; + + let pattern = + Regex::new(&pattern).or_else(|_| bail!(params[0].span().error("invalid regex")))?; + + if !n.is_integer() { + bail!(params[2].span().error("n must be an integer")); + } + + let n = match n.as_i64() { + Some(n) if n < 0 => usize::MAX, + Some(n) => n as usize, + None => usize::MAX, + }; + + Ok(Value::from_array( + pattern + .find_iter(&value) + .map(|m| Value::String(m.as_str().into())) + .take(n) + .collect(), + )) } fn is_valid(span: &Span, params: &[Ref], args: &[Value]) -> Result { @@ -31,17 +111,38 @@ pub fn regex_match(span: &Span, params: &[Ref], args: &[Value]) -> Result< let pattern = ensure_string(name, ¶ms[0], &args[0])?; let value = ensure_string(name, ¶ms[1], &args[1])?; - let pattern = Regex::new(&pattern).or_else(|_| bail!(span.error("invalid regex")))?; + let pattern = + Regex::new(&pattern).or_else(|_| bail!(params[0].span().error("invalid regex")))?; Ok(Value::Bool(pattern.is_match(&value))) } -pub fn regex_split(span: &Span, params: &[Ref], args: &[Value]) -> Result { +fn regex_replace(span: &Span, params: &[Ref], args: &[Value]) -> Result { + let name = "regex.replace"; + ensure_args_count(span, name, params, args, 3)?; + + let s = ensure_string(name, ¶ms[0], &args[0])?; + let pattern = ensure_string(name, ¶ms[1], &args[1])?; + let value = ensure_string(name, ¶ms[2], &args[2])?; + + let pattern = match Regex::new(&pattern) { + Ok(p) => p, + // TODO: This behavior is due to OPA test not raising error. Should we raise error? + _ => return Ok(Value::Undefined), + }; + + Ok(Value::String( + pattern.replace_all(&s, value.as_ref()).into(), + )) +} + +fn regex_split(span: &Span, params: &[Ref], args: &[Value]) -> Result { let name = "regex.split"; ensure_args_count(span, name, params, args, 2)?; let pattern = ensure_string(name, ¶ms[0], &args[0])?; let value = ensure_string(name, ¶ms[1], &args[1])?; - let pattern = Regex::new(&pattern).or_else(|_| bail!(span.error("invalid regex")))?; + let pattern = + Regex::new(&pattern).or_else(|_| bail!(params[0].span().error("invalid regex")))?; Ok(Value::from_array( pattern .split(&value) @@ -49,3 +150,49 @@ pub fn regex_split(span: &Span, params: &[Ref], args: &[Value]) -> Result< .collect::>(), )) } + +fn regex_template_match(span: &Span, params: &[Ref], args: &[Value]) -> Result { + let name = "regex.template_match"; + ensure_args_count(span, name, params, args, 4)?; + let template = ensure_string(name, ¶ms[0], &args[0])?; + let value = ensure_string(name, ¶ms[1], &args[1])?; + let delimiter_start = ensure_string(name, ¶ms[2], &args[2])?; + let delimiter_end = ensure_string(name, ¶ms[3], &args[3])?; + + let delimiter_start = delimiter_start.as_ref(); + let delimiter_end = delimiter_end.as_ref(); + let mut template = template.as_ref(); + let mut value = value.as_ref(); + + while let (Some(start), Some(end)) = + (template.find(delimiter_start), template.find(delimiter_end)) + { + if start >= end { + return Ok(Value::Undefined); + } + // Match precesing literal (if any) + if template[0..start] != value[0..start] { + return Ok(Value::Bool(false)); + } + + // Fetch pattern, excluding delimiters. + let pattern = Regex::new(&template[start + delimiter_start.len()..end]) + .or_else(|_| bail!(params[0].span().error("invalid regex")))?; + + // Skip preceding literal in value. + value = &value[start..]; + + let m = match pattern.find(value) { + Some(m) if m.start() == 0 => m, + _ => return Ok(Value::Bool(false)), + }; + // Skip match in string. + value = &value[m.len()..]; + + // Skip regex and delimiter in template. + template = &template[end + delimiter_end.len()..]; + } + + // Ensure that ending literal matches. + Ok(Value::Bool(template == value)) +} diff --git a/src/parser.rs b/src/parser.rs index 07b303a..e66b0ca 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -33,6 +33,15 @@ impl<'source> Parser<'source> { }) } + pub fn token_text(&self) -> std::rc::Rc<&str> { + match self.tok.0 { + TokenKind::Symbol | TokenKind::Number | TokenKind::Ident | TokenKind::Eof => { + self.tok.1.text() + } + TokenKind::String | TokenKind::RawString => "".into(), + } + } + pub fn next_token(&mut self) -> Result<()> { self.line = self.tok.1.line; self.end = self.tok.1.end; @@ -41,7 +50,7 @@ impl<'source> Parser<'source> { } fn expect(&mut self, text: &str, context: &str) -> Result<()> { - if *self.tok.1.text() == text { + if *self.token_text() == text { self.next_token() } else { let msg = format!("expecting `{text}` {context}"); @@ -54,7 +63,7 @@ impl<'source> Parser<'source> { } pub fn warn_future_keyword(&self) { - let kw = self.tok.1.text(); + let kw = self.token_text(); let msg = format!( "`{kw}` will be treated as identifier due to missing `import future.keywords.{kw}`" ); @@ -141,7 +150,7 @@ impl<'source> Parser<'source> { is_optional: bool, context: &str, ) -> Result<()> { - if *self.tok.1.text() == kw { + if *self.token_text() == kw { match &self.future_keywords.get(kw) { Some(_) => self.next_token(), None => { @@ -221,7 +230,7 @@ impl<'source> Parser<'source> { TokenKind::Number => Expr::Number(span), TokenKind::String => Expr::String(span), TokenKind::RawString => Expr::RawString(span), - TokenKind::Ident => match *self.tok.1.text() { + TokenKind::Ident => match *self.token_text() { "null" => Expr::Null(span), "true" => Expr::True(span), "false" => Expr::False(span), @@ -246,7 +255,7 @@ impl<'source> Parser<'source> { // Parse the first expression as a ref. let term = match self.parse_ref() { - Ok(e) if *self.tok.1.text() == "|" => e, + Ok(e) if *self.token_text() == "|" => e, _ => { // Not a comprehension. Restore state. *self = state; @@ -291,11 +300,11 @@ impl<'source> Parser<'source> { // No progress was made in parsing comprehension. // Parse as array. let mut items = vec![]; - if *self.tok.1.text() != "]" { + if *self.token_text() != "]" { items.push(Ref::new(self.parse_in_expr()?)); - while *self.tok.1.text() == "," { + while *self.token_text() == "," { self.next_token()?; - match *self.tok.1.text() { + match *self.token_text() { "]" => break, "" if self.tok.0 == TokenKind::Eof => break, _ => items.push(Ref::new(self.parse_in_expr()?)), @@ -334,7 +343,7 @@ impl<'source> Parser<'source> { // It could be a set, object or object comprehension. // In all the cases, the first expression must parse successfully. - if *self.tok.1.text() == "}" { + if *self.token_text() == "}" { self.next_token()?; span.end = self.end; return Ok(Expr::Object { @@ -346,12 +355,12 @@ impl<'source> Parser<'source> { let mut item_span = self.tok.1.clone(); let first = self.parse_in_expr()?; - if *self.tok.1.text() != ":" { + if *self.token_text() != ":" { // Parse as set. let mut items = vec![Ref::new(first)]; - while *self.tok.1.text() == "," { + while *self.token_text() == "," { self.next_token()?; - match *self.tok.1.text() { + match *self.token_text() { "}" => break, "" if self.tok.0 == TokenKind::Eof => break, _ => items.push(Ref::new(self.parse_in_expr()?)), @@ -391,10 +400,10 @@ impl<'source> Parser<'source> { item_span.end = self.end; items.push((item_span, Ref::new(first), Ref::new(value))); - while *self.tok.1.text() == "," { + while *self.token_text() == "," { self.next_token()?; let item_start = self.tok.1.start; - let key = match *self.tok.1.text() { + let key = match *self.token_text() { "}" => break, "" if self.tok.0 == TokenKind::Eof => break, _ => self.parse_in_expr()?, @@ -450,8 +459,8 @@ impl<'source> Parser<'source> { fn parse_ref(&mut self) -> Result { let start = self.tok.1.start; - let mut term = match *self.tok.1.text() { - "[" => self.parse_compr_or_array()?, + let mut term = match *self.token_text() { + "[" if self.tok.0 == TokenKind::Symbol => self.parse_compr_or_array()?, "{" => self.parse_compr_set_or_object()?, "set(" => self.parse_empty_set()?, "(" => return self.parse_parens_expr(), @@ -459,13 +468,28 @@ impl<'source> Parser<'source> { _ => self.parse_scalar_or_var()?, }; - let mut possible_fcn = matches!(&term, Expr::Var(_)); + let mut possible_fcn = true; + let mut expr = &term; + while possible_fcn { + match expr { + Expr::Var(_) => break, + Expr::RefDot { refr, .. } => expr = &refr, + Expr::RefBrack { refr, index, .. } => { + expr = &refr; + possible_fcn = matches!(index.as_ref(), Expr::String(_)); + } + _ => { + possible_fcn = false; + } + } + } + matches!(&term, Expr::Var(_)); loop { let mut span = self.tok.1.clone(); let sep_pos = span.start; span.start = start; - match *self.tok.1.text() { + match *self.token_text() { "." | "[" if self.tok.1.start != self.end => { if self.line != self.tok.1.line { // Newline encountered. This could be a separate @@ -477,7 +501,7 @@ impl<'source> Parser<'source> { self.source.error( self.tok.1.line, self.tok.1.col, - format!("invalid whitespace before {}", self.tok.1.text()).as_str() + format!("invalid whitespace before {}", self.token_text()).as_str() ) ); } @@ -523,11 +547,11 @@ impl<'source> Parser<'source> { "(" if possible_fcn => { self.next_token()?; let mut args = vec![]; - if *self.tok.1.text() != ")" { + if *self.token_text() != ")" { args.push(Ref::new(self.parse_in_expr()?)); - while *self.tok.1.text() == "," { + while *self.token_text() == "," { self.next_token()?; - match *self.tok.1.text() { + match *self.token_text() { ")" => break, "" if self.tok.0 == TokenKind::Eof => break, _ => args.push(Ref::new(self.parse_in_expr()?)), @@ -563,7 +587,7 @@ impl<'source> Parser<'source> { loop { let mut span = self.tok.1.clone(); span.start = start; - let op = match *self.tok.1.text() { + let op = match *self.token_text() { "*" => ArithOp::Mul, "/" => ArithOp::Div, "%" => ArithOp::Mod, @@ -588,7 +612,7 @@ impl<'source> Parser<'source> { loop { let mut span = self.tok.1.clone(); span.start = start; - let op = match *self.tok.1.text() { + let op = match *self.token_text() { "+" => ArithOp::Add, "-" => ArithOp::Sub, _ => return Ok(expr), @@ -609,7 +633,7 @@ impl<'source> Parser<'source> { let start = self.tok.1.start; let mut expr = self.parse_arith_expr()?; - while *self.tok.1.text() == "&" { + while *self.token_text() == "&" { let mut span = self.tok.1.clone(); span.start = start; self.next_token()?; @@ -629,7 +653,7 @@ impl<'source> Parser<'source> { let start = self.tok.1.start; let mut expr = self.parse_and_expr()?; - while *self.tok.1.text() == "|" { + while *self.token_text() == "|" { let mut span = self.tok.1.clone(); span.start = start; self.next_token()?; @@ -651,7 +675,7 @@ impl<'source> Parser<'source> { loop { let mut span = self.tok.1.clone(); span.start = start; - let op = match *self.tok.1.text() { + let op = match *self.token_text() { "<" => BoolOp::Lt, "<=" => BoolOp::Le, "==" => BoolOp::Eq, @@ -697,7 +721,7 @@ impl<'source> Parser<'source> { }; expr2 = None; - if *self.tok.1.text() != "in" { + if *self.token_text() != "in" { break; } } @@ -709,7 +733,7 @@ impl<'source> Parser<'source> { let start = self.tok.1.start; let mut expr = self.parse_bool_expr()?; - while *self.tok.1.text() == "in" && self.future_keywords.get("in").is_some() { + while *self.token_text() == "in" && self.future_keywords.get("in").is_some() { expr = self.parse_membership_tail(start, expr, None)?; } @@ -720,13 +744,13 @@ impl<'source> Parser<'source> { let start = self.tok.1.start; let mut expr = self.parse_bool_expr()?; - if *self.tok.1.text() == "," { + if *self.token_text() == "," { self.next_token()?; let value = self.parse_bool_expr()?; expr = self.parse_membership_tail(start, expr, Some(value))?; } - while *self.tok.1.text() == "in" && self.is_imported_future_keyword("in") { + while *self.token_text() == "in" && self.is_imported_future_keyword("in") { expr = self.parse_membership_tail(start, expr, None)?; } @@ -740,7 +764,7 @@ impl<'source> Parser<'source> { let mut span = self.tok.1.clone(); span.start = start; - let op = match *self.tok.1.text() { + let op = match *self.token_text() { "=" => AssignOp::Eq, ":=" => AssignOp::ColEq, _ => { @@ -762,7 +786,7 @@ impl<'source> Parser<'source> { fn parse_with_modifiers(&mut self) -> Result> { let mut modifiers = vec![]; - while *self.tok.1.text() == "with" { + while *self.token_text() == "with" { let mut span = self.tok.1.clone(); self.next_token()?; let refr = self.parse_path_ref()?; @@ -784,7 +808,7 @@ impl<'source> Parser<'source> { self.parse_future_keyword("every", false, context)?; let ident = self.parse_var()?; - let (key, value) = match *self.tok.1.text() { + let (key, value) = match *self.token_text() { "," => { self.next_token()?; match self.parse_var() { @@ -825,7 +849,7 @@ impl<'source> Parser<'source> { let mut vars = vec![self.tok.1.clone()]; let mut refs = vec![Ref::new(self.parse_ref()?)]; - while *self.tok.1.text() == "," { + while *self.token_text() == "," { self.next_token()?; let mut span = self.tok.1.clone(); refs.push(Ref::new(self.parse_ref()?)); @@ -833,8 +857,8 @@ impl<'source> Parser<'source> { vars.push(span); } - if *self.tok.1.text() != "in" || !self.is_imported_future_keyword("in") { - if *self.tok.1.text() == "in" { + if *self.token_text() != "in" || !self.is_imported_future_keyword("in") { + if *self.token_text() == "in" { self.warn_future_keyword(); } // All the refs must be identifiers @@ -884,7 +908,7 @@ impl<'source> Parser<'source> { } fn parse_literal(&mut self) -> Result { - match *self.tok.1.text() { + match *self.token_text() { "some" => return self.parse_some_stmt(), "every" => { if self.future_keywords.get("every").is_some() { @@ -895,7 +919,7 @@ impl<'source> Parser<'source> { _ => (), } let mut span = self.tok.1.clone(); - let not_expr = if *self.tok.1.text() == "not" { + let not_expr = if *self.token_text() == "not" { self.next_token()?; true } else { @@ -926,7 +950,7 @@ impl<'source> Parser<'source> { pub fn parse_query(&mut self, mut span: Span, end_delim: &str) -> Result { let state = self.clone(); - let is_definite_query = matches!(*self.tok.1.text(), "some" | "every"); + let is_definite_query = matches!(*self.token_text(), "some" | "every"); // TODO: empty query? let mut literals = vec![]; @@ -942,7 +966,7 @@ impl<'source> Parser<'source> { } }; - if *self.tok.1.text() == "," { + if *self.token_text() == "," { // This is likely an array or set. // Restore the state. *self = state; @@ -952,7 +976,7 @@ impl<'source> Parser<'source> { literals.push(stmt); loop { - match *self.tok.1.text() { + match *self.token_text() { t if t == end_delim => break, "" if self.tok.0 == TokenKind::Eof => break, ";" => self.next_token()?, @@ -980,7 +1004,7 @@ impl<'source> Parser<'source> { pub fn parse_rule_assign(&mut self) -> Result> { let mut span = self.tok.1.clone(); - let op = match *self.tok.1.text() { + let op = match *self.token_text() { "=" => { self.next_token()?; AssignOp::Eq @@ -1010,14 +1034,14 @@ impl<'source> Parser<'source> { let mut span = self.tok.1.clone(); let sep_pos = span.start; span.start = start; - match *self.tok.1.text() { + match *self.token_text() { "." | "[" if self.tok.1.start != self.end => { bail!( "{}", self.source.error( self.tok.1.line, self.tok.1.col - 1, - format!("invalid whitespace before {}", *self.tok.1.text()).as_str() + format!("invalid whitespace before {}", *self.token_text()).as_str() ) ); } @@ -1110,7 +1134,7 @@ impl<'source> Parser<'source> { loop { let mut span = self.tok.1.clone(); span.start = start; - match *self.tok.1.text() { + match *self.token_text() { // . and [ must not have any space between the previous token. "." | "[" if self.tok.1.start != self.end => { bail!( @@ -1118,7 +1142,7 @@ impl<'source> Parser<'source> { self.source.error( self.tok.1.line, self.tok.1.col - 1, - format!("invalid whitespace before {}", *self.tok.1.text()).as_str() + format!("invalid whitespace before {}", *self.token_text()).as_str() ) ); } @@ -1167,16 +1191,16 @@ impl<'source> Parser<'source> { let mut span = self.tok.1.clone(); let rule_ref = Ref::new(self.parse_rule_ref()?); - match *self.tok.1.text() { + match *self.token_text() { "(" => { self.check_rule_ref(&rule_ref)?; self.next_token()?; let mut args = vec![]; - if *self.tok.1.text() != ")" { + if *self.token_text() != ")" { args.push(Ref::new(self.parse_term()?)); - while *self.tok.1.text() == "," { + while *self.token_text() == "," { self.next_token()?; - match *self.tok.1.text() { + match *self.token_text() { ")" => break, "" if self.tok.0 == TokenKind::Eof => break, _ => args.push(Ref::new(self.parse_term()?)), @@ -1217,8 +1241,8 @@ impl<'source> Parser<'source> { } // Determine whether to create a set or a compr - let is_set_follower = !self.is_keyword(*self.tok.1.text()) - && !self.is_imported_future_keyword(*self.tok.1.text()); + let is_set_follower = !self.is_keyword(*self.token_text()) + && !self.is_imported_future_keyword(*self.token_text()); if assign.is_none() && is_set_follower { match rule_ref.as_ref() { Expr::RefBrack { refr, index, .. } @@ -1259,7 +1283,7 @@ impl<'source> Parser<'source> { let state = self.clone(); let mut span = self.tok.1.clone(); - if *self.tok.1.text() == "{" { + if *self.token_text() == "{" { self.next_token()?; let pos = self.end; match self.parse_query(span.clone(), "}") { @@ -1284,7 +1308,7 @@ impl<'source> Parser<'source> { let mut bodies = vec![]; let assign = None; - let has_query = match *self.tok.1.text() { + let has_query = match *self.token_text() { "if" if self.if_is_keyword() => { self.next_token()?; let query = Ref::new(self.parse_query_or_literal_stmt()?); @@ -1314,7 +1338,7 @@ impl<'source> Parser<'source> { _ => false, }; - match *self.tok.1.text() { + match *self.token_text() { "{" if has_query => self.parse_query_blocks(&mut bodies)?, "else" if has_query => self.parse_else_blocks(&mut bodies)?, _ => (), @@ -1324,7 +1348,7 @@ impl<'source> Parser<'source> { } pub fn parse_query_blocks(&mut self, bodies: &mut Vec) -> Result<()> { - while *self.tok.1.text() == "{" { + while *self.token_text() == "{" { let mut span = self.tok.1.clone(); self.next_token()?; let query = Ref::new(self.parse_query(span.clone(), "}")?); @@ -1342,7 +1366,7 @@ impl<'source> Parser<'source> { loop { let mut span = self.tok.1.clone(); - match *self.tok.1.text() { + match *self.token_text() { "{" => { return Err(self.source.error( self.tok.1.line, @@ -1356,7 +1380,7 @@ impl<'source> Parser<'source> { let assign = self.parse_rule_assign()?; - match *self.tok.1.text() { + match *self.token_text() { "if" if self.if_is_keyword() => { self.next_token()?; let query = Ref::new(self.parse_query_or_literal_stmt()?); @@ -1378,7 +1402,7 @@ impl<'source> Parser<'source> { }); } _ if assign.is_none() => { - if *self.tok.1.text() == "if" { + if *self.token_text() == "if" { self.warn_future_keyword(); } return Err(self.source.error( @@ -1398,7 +1422,7 @@ impl<'source> Parser<'source> { self.expect("default", "while parsing default rule")?; let rule_ref = Ref::new(self.parse_rule_ref()?); - let op = match *self.tok.1.text() { + let op = match *self.token_text() { "=" => AssignOp::Eq, ":=" => AssignOp::ColEq, _ => { @@ -1485,7 +1509,7 @@ impl<'source> Parser<'source> { fn parse_imports(&mut self) -> Result> { let mut imports = vec![]; - while *self.tok.1.text() == "import" { + while *self.token_text() == "import" { let mut span = self.tok.1.clone(); self.next_token()?; let refr = Ref::new(self.parse_path_ref()?); @@ -1501,7 +1525,7 @@ impl<'source> Parser<'source> { let is_future_kw = self.handle_import_future_keywords(&comps)?; - let var = if *self.tok.1.text() == "as" { + let var = if *self.token_text() == "as" { if is_future_kw { return Err(self.source.error( self.tok.1.line, diff --git a/tests/opa.passing b/tests/opa.passing index 6269d6a..d1722d2 100644 --- a/tests/opa.passing +++ b/tests/opa.passing @@ -18,6 +18,7 @@ embeddedvirtualdoc evaltermexpr example fix1863 +globmatch helloworld indexing intersection @@ -30,8 +31,12 @@ objectremoveidempotent objectremovenonstringkey partialsetdoc rand +regexfind +regexfindallstringsubmatch regexisvalid regexmatch +regexmatchtemplate +regexreplace regexsplit replacen semvercompare diff --git a/tests/opa.rs b/tests/opa.rs index 3b38393..047ef47 100644 --- a/tests/opa.rs +++ b/tests/opa.rs @@ -120,7 +120,7 @@ fn run_opa_tests(opa_tests_dir: String, folders: &[String]) -> Result<()> { entry.0 += 1; } (r, _) => { - print!("\n{} failed.", case.note); + println!("\n{} failed.", case.note); println!("{}", serde_yaml::to_string(&case)?); match &r { Ok(actual) => println!("GOT\n{}", serde_yaml::to_string(&actual)?),