kata-containers/regorus
1
0
Fork 0
mirror of https://github.com/microsoft/regorus.git synced 2026-08-05 02:16:11 +00:00
Code Issues Packages Projects Releases Wiki Activity

Engine (#38)

Browse Source 
- Avoid lifetime parameter for Source, Span. Use Rc instead.
- Engine for simplified API

Signed-off-by: Anand Krishnamoorthi <anakrish@microsoft.com>
This commit is contained in:
Anand Krishnamoorthi
2023-11-07 23:15:28 -08:00
committed by GitHub
parent a751cd69e1
commit d69b413c8e
15 changed files with 721 additions and 690 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
Cargo.toml
View File

@@ -10,6 +10,7 @@ anyhow = "1.0.66"
ordered-float = "3.4.0"
serde = {version = "1.0.150", features = ["derive", "rc"] }
serde_json = "1.0.89"
serde_yaml = "0.9.16"
log = "0.4.17"
env_logger="0.10.0"
lazy_static = "1.4.0"
@@ -18,7 +19,6 @@ data-encoding = "2.4.0"
[dev-dependencies]
clap = { version = "4.4.7", features = ["derive"] }
serde_yaml = "0.9.16"
test-generator = "0.3.1"
walkdir = "2.3.2"

116
examples/regorus.rs
View File

@@ -1,7 +1,7 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.
use anyhow::{bail, Context, Result};
use anyhow::{bail, Result};
use clap::{Parser, Subcommand};
fn rego_eval(
@@ -10,111 +10,50 @@ fn rego_eval(
query: String,
enable_tracing: bool,
) -> Result<()> {
// User specified data.
let mut data = regorus::Value::new_object();
// Create engine.
let mut engine = regorus::Engine::new();
// Read all policy files.
let mut policies = vec![];
// Load given files.
for file in files.iter() {
let contents =
std::fs::read_to_string(file).with_context(|| format!("Failed to read {file}"))?;
if file.ends_with(".rego") {
policies.push(contents);
// Read policy file.
engine.add_policy_from_file(file.to_string())?;
} else {
let value: regorus::Value = if file.ends_with(".json") {
serde_json::from_str(&contents)?
// Read data file.
let data = if file.ends_with(".json") {
regorus::Value::from_json_file(file)?
} else if file.ends_with(".yaml") {
serde_yaml::from_str(&contents)?
regorus::Value::from_yaml_file(file)?
} else {
bail!("Unsupported data file `{file}`. Must be rego, json or yaml.")
};
if let Err(err) = data.merge(value) {
bail!("Error processing {file}. {err}");
}
// Merge given data.
engine.add_data(data)?;
}
}
// Create source objects.
let mut sources = vec![];
for (idx, rego) in policies.iter().enumerate() {
sources.push(regorus::Source {
file: &files[idx],
contents: rego.as_str(),
lines: rego.split('\n').collect(),
});
}
// Parse the policy files.
let mut modules = vec![];
for source in &sources {
let mut parser = regorus::Parser::new(source)?;
modules.push(parser.parse()?);
}
// Parse input file.
let input = if let Some(file) = input {
let input_contents = std::fs::read_to_string(file.clone())
.with_context(|| format!("Failed to read {file}"))?;
Some(if file.ends_with(".json") {
serde_json::from_str(&input_contents)?
if let Some(file) = input {
let input = if file.ends_with(".json") {
regorus::Value::from_json_file(&file)?
} else if file.ends_with(".yaml") {
serde_yaml::from_str(&input_contents)?
regorus::Value::from_yaml_file(&file)?
} else {
bail!("invalid input file {file}");
})
} else {
None
};
bail!("Unsupported input file `{file}`. Must be json or yaml.")
};
engine.set_input(input);
}
let modules_ref: Vec<&regorus::Module> = modules.iter().collect();
// Analyze the modules and determine how statements must be schedules.
let analyzer = regorus::Analyzer::new();
let schedule = analyzer.analyze(&modules_ref)?;
// Create interpreter object.
let mut interpreter = regorus::Interpreter::new(&modules_ref)?;
// Evaluate all the modules.
interpreter.eval(&Some(data), &input, false, Some(schedule))?;
// Parse the query.
let query_source = regorus::Source {
file: "<query.rego>",
contents: &query,
lines: query.split('\n').collect(),
};
let query_span = regorus::Span {
source: &query_source,
line: 1,
col: 1,
start: 0,
end: query.len() as u16,
};
let mut parser = regorus::Parser::new(&query_source)?;
let query_node = parser.parse_query(query_span, "")?;
let query_schedule =
regorus::Analyzer::new().analyze_query_snippet(&modules_ref, &query_node)?;
let results = interpreter.eval_user_query(&query_node, &query_schedule, enable_tracing)?;
// Evaluate query.
let results = engine.eval_query(query, enable_tracing)?;
println!("{}", serde_json::to_string_pretty(&results)?);
Ok(())
}
fn rego_lex(file: String, verbose: bool) -> Result<()> {
let contents =
std::fs::read_to_string(file.clone()).with_context(|| format!("Failed to read {file}"))?;
// Create source.
let source = regorus::Source {
file: file.as_str(),
contents: contents.as_str(),
lines: contents.split('\n').collect(),
};
let source = regorus::Source::from_file(file)?;
// Create lexer.
let mut lexer = regorus::Lexer::new(&source);
@@ -138,15 +77,8 @@ fn rego_lex(file: String, verbose: bool) -> Result<()> {
}
fn rego_parse(file: String) -> Result<()> {
let contents =
std::fs::read_to_string(file.clone()).with_context(|| format!("Failed to read {file}"))?;
// Create source.
let source = regorus::Source {
file: file.as_str(),
contents: contents.as_str(),
lines: contents.split('\n').collect(),
};
let source = regorus::Source::from_file(file)?;
// Create a parser and parse the source.
let mut parser = regorus::Parser::new(&source)?;

233
src/ast.rs
View File

@@ -35,115 +35,115 @@ pub enum AssignOp {
}
#[derive(Debug, Clone)]
pub enum Expr<'source> {
pub enum Expr {
// Simple items that only have a span as content.
String(Span<'source>),
RawString(Span<'source>),
Number(Span<'source>),
True(Span<'source>),
False(Span<'source>),
Null(Span<'source>),
Var(Span<'source>),
String(Span),
RawString(Span),
Number(Span),
True(Span),
False(Span),
Null(Span),
Var(Span),
// array
Array {
span: Span<'source>,
items: Vec<Expr<'source>>,
span: Span,
items: Vec<Expr>,
},
// set
Set {
span: Span<'source>,
items: Vec<Expr<'source>>,
span: Span,
items: Vec<Expr>,
},
Object {
span: Span<'source>,
fields: Vec<(Span<'source>, Expr<'source>, Expr<'source>)>,
span: Span,
fields: Vec<(Span, Expr, Expr)>,
},
// Comprehensions
ArrayCompr {
span: Span<'source>,
term: Box<Expr<'source>>,
query: Query<'source>,
span: Span,
term: Box<Expr>,
query: Query,
},
SetCompr {
span: Span<'source>,
term: Box<Expr<'source>>,
query: Query<'source>,
span: Span,
term: Box<Expr>,
query: Query,
},
ObjectCompr {
span: Span<'source>,
key: Box<Expr<'source>>,
value: Box<Expr<'source>>,
query: Query<'source>,
span: Span,
key: Box<Expr>,
value: Box<Expr>,
query: Query,
},
Call {
span: Span<'source>,
fcn: Box<Expr<'source>>,
params: Vec<Expr<'source>>,
span: Span,
fcn: Box<Expr>,
params: Vec<Expr>,
},
UnaryExpr {
span: Span<'source>,
expr: Box<Expr<'source>>,
span: Span,
expr: Box<Expr>,
},
// ref
RefDot {
span: Span<'source>,
refr: Box<Expr<'source>>,
field: Span<'source>,
span: Span,
refr: Box<Expr>,
field: Span,
},
RefBrack {
span: Span<'source>,
refr: Box<Expr<'source>>,
index: Box<Expr<'source>>,
span: Span,
refr: Box<Expr>,
index: Box<Expr>,
},
// Infix expressions
BinExpr {
span: Span<'source>,
span: Span,
op: BinOp,
lhs: Box<Expr<'source>>,
rhs: Box<Expr<'source>>,
lhs: Box<Expr>,
rhs: Box<Expr>,
},
BoolExpr {
span: Span<'source>,
span: Span,
op: BoolOp,
lhs: Box<Expr<'source>>,
rhs: Box<Expr<'source>>,
lhs: Box<Expr>,
rhs: Box<Expr>,
},
ArithExpr {
span: Span<'source>,
span: Span,
op: ArithOp,
lhs: Box<Expr<'source>>,
rhs: Box<Expr<'source>>,
lhs: Box<Expr>,
rhs: Box<Expr>,
},
AssignExpr {
span: Span<'source>,
span: Span,
op: AssignOp,
lhs: Box<Expr<'source>>,
rhs: Box<Expr<'source>>,
lhs: Box<Expr>,
rhs: Box<Expr>,
},
Membership {
span: Span<'source>,
key: Box<Option<Expr<'source>>>,
value: Box<Expr<'source>>,
collection: Box<Expr<'source>>,
span: Span,
key: Box<Option<Expr>>,
value: Box<Expr>,
collection: Box<Expr>,
},
}
impl<'source> Expr<'source> {
pub fn span(&self) -> &Span<'source> {
impl Expr {
pub fn span(&self) -> &Span {
use Expr::*;
match self {
String(s) | RawString(s) | Number(s) | True(s) | False(s) | Null(s) | Var(s) => s,
@@ -167,121 +167,121 @@ impl<'source> Expr<'source> {
}
#[derive(Debug, Clone)]
pub enum Literal<'source> {
pub enum Literal {
SomeVars {
span: Span<'source>,
vars: Vec<Span<'source>>,
span: Span,
vars: Vec<Span>,
},
SomeIn {
span: Span<'source>,
key: Option<Expr<'source>>,
value: Expr<'source>,
collection: Expr<'source>,
span: Span,
key: Option<Expr>,
value: Expr,
collection: Expr,
},
Expr {
span: Span<'source>,
expr: Expr<'source>,
span: Span,
expr: Expr,
},
NotExpr {
span: Span<'source>,
expr: Expr<'source>,
span: Span,
expr: Expr,
},
Every {
span: Span<'source>,
key: Option<Span<'source>>,
value: Span<'source>,
domain: Expr<'source>,
query: Query<'source>,
span: Span,
key: Option<Span>,
value: Span,
domain: Expr,
query: Query,
},
}
#[derive(Debug, Clone)]
pub struct WithModifier<'source> {
pub span: Span<'source>,
pub refr: Expr<'source>,
pub r#as: Expr<'source>,
pub struct WithModifier {
pub span: Span,
pub refr: Expr,
pub r#as: Expr,
}
#[derive(Debug, Clone)]
pub struct LiteralStmt<'source> {
pub span: Span<'source>,
pub literal: Literal<'source>,
pub with_mods: Vec<WithModifier<'source>>,
pub struct LiteralStmt {
pub span: Span,
pub literal: Literal,
pub with_mods: Vec<WithModifier>,
}
#[derive(Debug, Clone)]
pub struct Query<'source> {
pub span: Span<'source>,
pub stmts: Vec<LiteralStmt<'source>>,
pub struct Query {
pub span: Span,
pub stmts: Vec<LiteralStmt>,
}
#[derive(Debug, Clone)]
pub struct RuleAssign<'source> {
pub span: Span<'source>,
pub struct RuleAssign {
pub span: Span,
pub op: AssignOp,
pub value: Expr<'source>,
pub value: Expr,
}
#[derive(Debug, Clone)]
pub struct RuleBody<'source> {
pub span: Span<'source>,
pub assign: Option<RuleAssign<'source>>,
pub query: Query<'source>,
pub struct RuleBody {
pub span: Span,
pub assign: Option<RuleAssign>,
pub query: Query,
}
#[derive(Debug, Clone)]
pub enum RuleHead<'source> {
pub enum RuleHead {
Compr {
span: Span<'source>,
refr: Expr<'source>,
assign: Option<RuleAssign<'source>>,
span: Span,
refr: Expr,
assign: Option<RuleAssign>,
},
Set {
span: Span<'source>,
refr: Expr<'source>,
key: Option<Expr<'source>>,
span: Span,
refr: Expr,
key: Option<Expr>,
},
Func {
span: Span<'source>,
refr: Expr<'source>,
args: Vec<Expr<'source>>,
assign: Option<RuleAssign<'source>>,
span: Span,
refr: Expr,
args: Vec<Expr>,
assign: Option<RuleAssign>,
},
}
#[derive(Debug, Clone)]
pub enum Rule<'source> {
pub enum Rule {
Spec {
span: Span<'source>,
head: RuleHead<'source>,
bodies: Vec<RuleBody<'source>>,
span: Span,
head: RuleHead,
bodies: Vec<RuleBody>,
},
Default {
span: Span<'source>,
refr: Expr<'source>,
span: Span,
refr: Expr,
op: AssignOp,
value: Expr<'source>,
value: Expr,
},
}
#[derive(Debug, Clone)]
pub struct Package<'source> {
pub span: Span<'source>,
pub refr: Expr<'source>,
pub struct Package {
pub span: Span,
pub refr: Expr,
}
#[derive(Debug, Clone)]
pub struct Import<'source> {
pub span: Span<'source>,
pub refr: Expr<'source>,
pub r#as: Option<Span<'source>>,
pub struct Import {
pub span: Span,
pub refr: Expr,
pub r#as: Option<Span>,
}
#[derive(Debug, Clone)]
pub struct Module<'source> {
pub package: Package<'source>,
pub imports: Vec<Import<'source>>,
pub policy: Vec<Rule<'source>>,
pub struct Module {
pub package: Package,
pub imports: Vec<Import>,
pub policy: Vec<Rule>,
}
#[derive(Debug, Clone)]
@@ -315,7 +315,6 @@ impl<'a, T> PartialOrd for Ref<'a, T> {
impl<'a, T> Ord for Ref<'a, T> {
fn cmp(&self, other: &Self) -> std::cmp::Ordering {
//std::ptr::from_ref(self.r).partial_cmp(std::ptr::from_ref(other.r))
(self.r as *const T).cmp(&(other.r as *const T))
}
}

96
src/engine.rs Normal file
View File

@@ -0,0 +1,96 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.
use crate::ast::*;
use crate::interpreter::*;
use crate::lexer::*;
use crate::parser::*;
use crate::scheduler::*;
use crate::value::*;
use anyhow::Result;
#[derive(Clone)]
pub struct Engine {
modules: Vec<std::rc::Rc<Module>>,
input: Value,
data: Value,
}
impl Default for Engine {
fn default() -> Self {
Self::new()
}
}
impl Engine {
pub fn new() -> Self {
Self {
modules: vec![],
input: Value::new_object(),
data: Value::new_object(),
}
}
pub fn add_policy(&mut self, path: String, rego: String) -> Result<()> {
let source = Source::new(path, rego);
let mut parser = Parser::new(&source)?;
self.modules.push(std::rc::Rc::new(parser.parse()?));
Ok(())
}
pub fn add_policy_from_file(&mut self, path: String) -> Result<()> {
let source = Source::from_file(path)?;
let mut parser = Parser::new(&source)?;
self.modules.push(std::rc::Rc::new(parser.parse()?));
Ok(())
}
pub fn set_input(&mut self, input: Value) {
self.input = input;
}
pub fn clear_data(&mut self) {
self.data = Value::new_object();
}
pub fn add_data(&mut self, data: Value) -> Result<()> {
self.data.merge(data)
}
pub fn eval_query(&self, query: String, enable_tracing: bool) -> Result<QueryResults> {
let modules_ref: Vec<&Module> = self.modules.iter().map(|m| &**m).collect();
// Analyze the modules and determine how statements must be scheduled.
let analyzer = Analyzer::new();
let schedule = analyzer.analyze(&modules_ref)?;
// Create interpreter object.
let mut interpreter = Interpreter::new(&modules_ref)?;
// Evaluate all the modules.
interpreter.eval(
&Some(self.data.clone()),
&Some(self.input.clone()),
false,
Some(schedule),
)?;
// Parse the query.
let query_len = query.len();
let query_source = Source::new("<query.rego>".to_string(), query);
let query_span = Span {
source: query_source.clone(),
line: 1,
col: 1,
start: 0,
end: query_len as u16,
};
let mut parser = Parser::new(&query_source)?;
let query_node = parser.parse_query(query_span, "")?;
let query_schedule = Analyzer::new().analyze_query_snippet(&modules_ref, &query_node)?;
let results = interpreter.eval_user_query(&query_node, &query_schedule, enable_tracing)?;
Ok(results)
}
}

300
src/interpreter.rs
View File

@@ -18,8 +18,8 @@ use std::rc::Rc;
type Scope = BTreeMap<String, Value>;
pub struct Interpreter<'source> {
modules: Vec<&'source Module<'source>>,
module: Option<&'source Module<'source>>,
modules: Vec<&'source Module>,
module: Option<&'source Module>,
schedule: Option<Schedule<'source>>,
current_module_path: String,
prepared: bool,
@@ -29,13 +29,13 @@ pub struct Interpreter<'source> {
with_document: Value,
scopes: Vec<Scope>,
// TODO: handle recursive calls where same expr could have different values.
loop_var_values: BTreeMap<Ref<'source, Expr<'source>>, Value>,
loop_var_values: BTreeMap<Ref<'source, Expr>, Value>,
contexts: Vec<Context<'source>>,
functions: FunctionTable<'source>,
rules: HashMap<String, Vec<&'source Rule<'source>>>,
default_rules: HashMap<String, Vec<(&'source Rule<'source>, Option<String>)>>,
processed: BTreeSet<Ref<'source, Rule<'source>>>,
active_rules: Vec<&'source Rule<'source>>,
rules: HashMap<String, Vec<&'source Rule>>,
default_rules: HashMap<String, Vec<(&'source Rule, Option<String>)>>,
processed: BTreeSet<Ref<'source, Rule>>,
active_rules: Vec<&'source Rule>,
builtins_cache: BTreeMap<(&'static str, Vec<Value>), Value>,
no_rules_lookup: bool,
traces: Option<Vec<String>>,
@@ -66,8 +66,8 @@ pub struct QueryResults {
#[derive(Debug, Clone)]
struct Context<'source> {
key_expr: Option<&'source Expr<'source>>,
output_expr: Option<&'source Expr<'source>>,
key_expr: Option<&'source Expr>,
output_expr: Option<&'source Expr>,
value: Value,
result: Option<QueryResult>,
results: QueryResults,
@@ -75,14 +75,14 @@ struct Context<'source> {
#[derive(Debug)]
struct LoopExpr<'source> {
span: &'source Span<'source>,
expr: &'source Expr<'source>,
value: &'source Expr<'source>,
span: &'source Span,
expr: &'source Expr,
value: &'source Expr,
index: &'source str,
}
impl<'source> Interpreter<'source> {
pub fn new(modules: &[&'source Module<'source>]) -> Result<Interpreter<'source>> {
pub fn new(modules: &[&'source Module]) -> Result<Interpreter<'source>> {
let mut with_document = Value::new_object();
*Self::make_or_get_value_mut(&mut with_document, &["data"])? = Value::new_object();
*Self::make_or_get_value_mut(&mut with_document, &["input"])? = Value::new_object();
@@ -111,7 +111,7 @@ impl<'source> Interpreter<'source> {
})
}
pub fn get_modules(&mut self) -> &mut Vec<&'source Module<'source>> {
pub fn get_modules(&mut self) -> &mut Vec<&'source Module> {
&mut self.modules
}
@@ -149,7 +149,7 @@ impl<'source> Interpreter<'source> {
Ok(())
}
fn current_module(&self) -> Result<&'source Module<'source>> {
fn current_module(&self) -> Result<&'source Module> {
self.module
.ok_or_else(|| anyhow!("internal error: current module not set"))
}
@@ -201,7 +201,7 @@ impl<'source> Interpreter<'source> {
}
}
fn eval_chained_ref_dot_or_brack(&mut self, mut expr: &'source Expr<'source>) -> Result<Value> {
fn eval_chained_ref_dot_or_brack(&mut self, mut expr: &'source Expr) -> Result<Value> {
// Collect a chaing of '.field' or '["field"]'
let mut path = vec![];
loop {
@@ -218,13 +218,13 @@ impl<'source> Interpreter<'source> {
// Accumulate chained . field accesses.
Expr::RefDot { refr, field, .. } => {
expr = refr;
path.push(field.text());
path.push(*field.text());
}
Expr::RefBrack { refr, index, .. } => match index.as_ref() {
// refr["field"] is the same as refr.field
Expr::String(s) => {
expr = refr;
path.push(s.text());
path.push(*s.text());
}
// Handle other forms of refr.
// Note, we have the choice to evaluate a non-string index
@@ -263,7 +263,7 @@ impl<'source> Interpreter<'source> {
}
}
fn hoist_loops_impl(&self, expr: &'source Expr<'source>, loops: &mut Vec<LoopExpr<'source>>) {
fn hoist_loops_impl(&self, expr: &'source Expr, loops: &mut Vec<LoopExpr<'source>>) {
use Expr::*;
match expr {
RefBrack { refr, index, span } => {
@@ -272,11 +272,11 @@ impl<'source> Interpreter<'source> {
// Then hoist the current bracket operation.
match index.as_ref() {
Var(ident) if self.is_loop_index_var(ident.text()) => loops.push(LoopExpr {
Var(ident) if self.is_loop_index_var(*ident.text()) => loops.push(LoopExpr {
span,
expr,
value: refr,
index: ident.text(),
index: *ident.text(),
}),
_ => {
// hoist any loops in index expression.
@@ -334,7 +334,7 @@ impl<'source> Interpreter<'source> {
}
}
fn hoist_loops(&self, literal: &'source Literal<'source>) -> Vec<LoopExpr<'source>> {
fn hoist_loops(&self, literal: &'source Literal) -> Vec<LoopExpr<'source>> {
let mut loops = vec![];
use Literal::*;
match literal {
@@ -362,8 +362,8 @@ impl<'source> Interpreter<'source> {
fn eval_bool_expr(
&mut self,
op: &BoolOp,
lhs_expr: &'source Expr<'source>,
rhs_expr: &'source Expr<'source>,
lhs_expr: &'source Expr,
rhs_expr: &'source Expr,
) -> Result<Value> {
let lhs = self.eval_expr(lhs_expr)?;
let rhs = self.eval_expr(rhs_expr)?;
@@ -378,8 +378,8 @@ impl<'source> Interpreter<'source> {
fn eval_bin_expr(
&mut self,
op: &BinOp,
lhs: &'source Expr<'source>,
rhs: &'source Expr<'source>,
lhs: &'source Expr,
rhs: &'source Expr,
) -> Result<Value> {
let lhs_value = self.eval_expr(lhs)?;
let rhs_value = self.eval_expr(rhs)?;
@@ -397,8 +397,8 @@ impl<'source> Interpreter<'source> {
fn eval_arith_expr(
&mut self,
op: &ArithOp,
lhs: &'source Expr<'source>,
rhs: &'source Expr<'source>,
lhs: &'source Expr,
rhs: &'source Expr,
) -> Result<Value> {
let lhs_value = self.eval_expr(lhs)?;
let rhs_value = self.eval_expr(rhs)?;
@@ -418,8 +418,8 @@ impl<'source> Interpreter<'source> {
fn eval_assign_expr(
&mut self,
op: &AssignOp,
lhs: &'source Expr<'source>,
rhs: &'source Expr<'source>,
lhs: &'source Expr,
rhs: &'source Expr,
) -> Result<Value> {
let (name, value) = match op {
AssignOp::Eq => {
@@ -477,7 +477,7 @@ impl<'source> Interpreter<'source> {
// TODO: Check this
// Allow variable overwritten inside a loop
if self.lookup_local_var(name).is_some()
if self.lookup_local_var(*name).is_some()
&& self.loop_var_values.get(&Ref::make(rhs)).is_none()
{
bail!(rhs
@@ -494,10 +494,10 @@ impl<'source> Interpreter<'source> {
return Ok(Value::Bool(false));
}
self.add_variable_or(name)?;
self.add_variable_or(*name)?;
// TODO: optimize this
self.variables_assignment(name, &value)?;
self.variables_assignment(*name, &value)?;
info!(
"eval_assign_expr before, op: {:?}, lhs: {:?}, rhs: {:?}",
@@ -509,11 +509,11 @@ impl<'source> Interpreter<'source> {
fn eval_every(
&mut self,
_span: &'source Span<'source>,
key: &'source Option<Span<'source>>,
value: &'source Span<'source>,
domain: &'source Expr<'source>,
query: &'source Query<'source>,
_span: &'source Span,
key: &'source Option<Span>,
value: &'source Span,
domain: &'source Expr,
query: &'source Query,
) -> Result<bool> {
let domain = self.eval_expr(domain)?;
@@ -529,9 +529,9 @@ impl<'source> Interpreter<'source> {
match domain {
Value::Array(a) => {
for (idx, v) in a.iter().enumerate() {
self.add_variable(value.text(), v.clone())?;
self.add_variable(*value.text(), v.clone())?;
if let Some(key) = key {
self.add_variable(key.text(), Value::from_float(idx as Float))?;
self.add_variable(*key.text(), Value::from_float(idx as Float))?;
}
if !self.eval_query(query)? {
r = false;
@@ -541,9 +541,9 @@ impl<'source> Interpreter<'source> {
}
Value::Set(s) => {
for v in s.iter() {
self.add_variable(value.text(), v.clone())?;
self.add_variable(*value.text(), v.clone())?;
if let Some(key) = key {
self.add_variable(key.text(), v.clone())?;
self.add_variable(*key.text(), v.clone())?;
}
if !self.eval_query(query)? {
r = false;
@@ -553,9 +553,9 @@ impl<'source> Interpreter<'source> {
}
Value::Object(o) => {
for (k, v) in o.iter() {
self.add_variable(value.text(), v.clone())?;
self.add_variable(*value.text(), v.clone())?;
if let Some(key) = key {
self.add_variable(key.text(), k.clone())?;
self.add_variable(*key.text(), k.clone())?;
}
if !self.eval_query(query)? {
r = false;
@@ -574,8 +574,8 @@ impl<'source> Interpreter<'source> {
fn lookup_or_eval_expr(
&mut self,
cache: &mut BTreeMap<Ref<'source, Expr<'source>>, Value>,
expr: &'source Expr<'source>,
cache: &mut BTreeMap<Ref<'source, Expr>, Value>,
expr: &'source Expr,
) -> Result<Value> {
match cache.get(&Ref::make(expr)) {
Some(v) => Ok(v.clone()),
@@ -590,9 +590,9 @@ impl<'source> Interpreter<'source> {
fn make_bindings_impl(
&mut self,
is_last: bool,
type_match: &mut BTreeSet<Ref<'source, Expr<'source>>>,
cache: &mut BTreeMap<Ref<'source, Expr<'source>>, Value>,
expr: &'source Expr<'source>,
type_match: &mut BTreeSet<Ref<'source, Expr>>,
cache: &mut BTreeMap<Ref<'source, Expr>, Value>,
expr: &'source Expr,
value: &Value,
) -> Result<bool> {
// Propagate undefined.
@@ -604,7 +604,7 @@ impl<'source> Interpreter<'source> {
match (expr, value) {
(Expr::Var(ident), _) => {
self.add_variable(ident.text(), value.clone())?;
self.add_variable(*ident.text(), value.clone())?;
Ok(true)
}
@@ -688,9 +688,9 @@ impl<'source> Interpreter<'source> {
fn make_bindings(
&mut self,
is_last: bool,
type_match: &mut BTreeSet<Ref<'source, Expr<'source>>>,
cache: &mut BTreeMap<Ref<'source, Expr<'source>>, Value>,
expr: &'source Expr<'source>,
type_match: &mut BTreeSet<Ref<'source, Expr>>,
cache: &mut BTreeMap<Ref<'source, Expr>, Value>,
expr: &'source Expr,
value: &Value,
) -> Result<bool> {
let prev = self.no_rules_lookup;
@@ -703,9 +703,9 @@ impl<'source> Interpreter<'source> {
fn make_key_value_bindings(
&mut self,
is_last: bool,
type_match: &mut BTreeSet<Ref<'source, Expr<'source>>>,
cache: &mut BTreeMap<Ref<'source, Expr<'source>>, Value>,
exprs: (&'source Option<Expr<'source>>, &'source Expr<'source>),
type_match: &mut BTreeSet<Ref<'source, Expr>>,
cache: &mut BTreeMap<Ref<'source, Expr>, Value>,
exprs: (&'source Option<Expr>, &'source Expr),
values: (&Value, &Value),
) -> Result<bool> {
let (key_expr, value_expr) = exprs;
@@ -720,11 +720,11 @@ impl<'source> Interpreter<'source> {
fn eval_some_in(
&mut self,
_span: &'source Span<'source>,
key_expr: &'source Option<Expr<'source>>,
value_expr: &'source Expr<'source>,
collection: &'source Expr<'source>,
stmts: &[&'source LiteralStmt<'source>],
_span: &'source Span,
key_expr: &'source Option<Expr>,
value_expr: &'source Expr,
collection: &'source Expr,
stmts: &[&'source LiteralStmt],
) -> Result<bool> {
let scope_saved = self.current_scope()?.clone();
let mut type_match = BTreeSet::new();
@@ -821,8 +821,8 @@ impl<'source> Interpreter<'source> {
fn eval_stmt_impl(
&mut self,
stmt: &'source LiteralStmt<'source>,
stmts: &[&'source LiteralStmt<'source>],
stmt: &'source LiteralStmt,
stmts: &[&'source LiteralStmt],
) -> Result<bool> {
Ok(match &stmt.literal {
Literal::Expr { span, expr, .. } => {
@@ -879,7 +879,7 @@ impl<'source> Interpreter<'source> {
Literal::SomeVars { span, vars, .. } => {
for var in vars {
let name = var.text();
if let Ok(variable) = self.add_variable_or(name) {
if let Ok(variable) = self.add_variable_or(*name) {
if variable != Value::Undefined {
return Err(anyhow!(
"duplicated definition of local variable {}",
@@ -933,8 +933,8 @@ impl<'source> Interpreter<'source> {
fn eval_stmt(
&mut self,
stmt: &'source LiteralStmt<'source>,
stmts: &[&'source LiteralStmt<'source>],
stmt: &'source LiteralStmt,
stmts: &[&'source LiteralStmt],
) -> Result<bool> {
let saved_state = if !stmt.with_mods.is_empty() {
// Save state;
@@ -948,7 +948,7 @@ impl<'source> Interpreter<'source> {
// Evaluate value and ref
let value = self.eval_expr(&wm.r#as)?;
let path = Parser::get_path_ref_components(&wm.refr)?;
let path: Vec<&str> = path.iter().map(|s| s.text()).collect();
let path: Vec<&str> = path.iter().map(|s| *s.text()).collect();
if path[0] == "input" || path[0] == "data" {
*Self::make_or_get_value_mut(&mut self.with_document, &path[..])? = value;
@@ -986,7 +986,7 @@ impl<'source> Interpreter<'source> {
fn eval_stmts_in_loop(
&mut self,
stmts: &[&'source LiteralStmt<'source>],
stmts: &[&'source LiteralStmt],
loops: &[LoopExpr<'source>],
) -> Result<bool> {
if loops.is_empty() {
@@ -1208,12 +1208,7 @@ impl<'source> Interpreter<'source> {
}
}
fn get_exprs_from_context(
&self,
) -> Result<(
Option<&'source Expr<'source>>,
Option<&'source Expr<'source>>,
)> {
fn get_exprs_from_context(&self) -> Result<(Option<&'source Expr>, Option<&'source Expr>)> {
let ctx = self.get_current_context()?;
Ok((ctx.key_expr, ctx.output_expr))
}
@@ -1242,7 +1237,7 @@ impl<'source> Interpreter<'source> {
}
}
fn eval_stmts(&mut self, stmts: &[&'source LiteralStmt<'source>]) -> Result<bool> {
fn eval_stmts(&mut self, stmts: &[&'source LiteralStmt]) -> Result<bool> {
let mut result = true;
for (idx, stmt) in stmts.iter().enumerate() {
@@ -1270,28 +1265,27 @@ impl<'source> Interpreter<'source> {
Ok(result)
}
fn eval_query(&mut self, query: &'source Query<'source>) -> Result<bool> {
fn eval_query(&mut self, query: &'source Query) -> Result<bool> {
// Execute the query in a new scope
self.scopes.push(Scope::new());
let ordered_stmts: Vec<&'source LiteralStmt<'source>> =
if let Some(schedule) = &self.schedule {
match schedule.order.get(&Ref::make(query)) {
Some(ord) => ord.iter().map(|i| &query.stmts[*i as usize]).collect(),
// TODO
_ => bail!(query
.span
.error("statements not scheduled in query {query:?}")),
}
} else {
query.stmts.iter().collect()
};
let ordered_stmts: Vec<&'source LiteralStmt> = if let Some(schedule) = &self.schedule {
match schedule.order.get(&Ref::make(query)) {
Some(ord) => ord.iter().map(|i| &query.stmts[*i as usize]).collect(),
// TODO
_ => bail!(query
.span
.error("statements not scheduled in query {query:?}")),
}
} else {
query.stmts.iter().collect()
};
let r = self.eval_stmts(&ordered_stmts);
self.scopes.pop();
r
}
fn eval_array(&mut self, items: &'source Vec<Expr<'source>>) -> Result<Value> {
fn eval_array(&mut self, items: &'source Vec<Expr>) -> Result<Value> {
let mut array = Vec::new();
for item in items {
@@ -1322,7 +1316,7 @@ impl<'source> Interpreter<'source> {
Ok(Value::from_map(object))
}
fn eval_set(&mut self, items: &'source Vec<Expr<'source>>) -> Result<Value> {
fn eval_set(&mut self, items: &'source Vec<Expr>) -> Result<Value> {
let mut set = BTreeSet::new();
for item in items {
@@ -1338,9 +1332,9 @@ impl<'source> Interpreter<'source> {
fn eval_membership(
&mut self,
key: &'source Option<Expr<'source>>,
value: &'source Expr<'source>,
collection: &'source Expr<'source>,
key: &'source Option<Expr>,
value: &'source Expr,
collection: &'source Expr,
) -> Result<Value> {
let value = self.eval_expr(value)?;
let collection = self.eval_expr(collection)?;
@@ -1378,11 +1372,7 @@ impl<'source> Interpreter<'source> {
Ok(Value::Bool(result))
}
fn eval_array_compr(
&mut self,
term: &'source Expr<'source>,
query: &'source Query<'source>,
) -> Result<Value> {
fn eval_array_compr(&mut self, term: &'source Expr, query: &'source Query) -> Result<Value> {
// Push new context
self.contexts.push(Context {
key_expr: None,
@@ -1401,11 +1391,7 @@ impl<'source> Interpreter<'source> {
}
}
fn eval_set_compr(
&mut self,
term: &'source Expr<'source>,
query: &'source Query<'source>,
) -> Result<Value> {
fn eval_set_compr(&mut self, term: &'source Expr, query: &'source Query) -> Result<Value> {
// Push new context
self.contexts.push(Context {
key_expr: None,
@@ -1425,9 +1411,9 @@ impl<'source> Interpreter<'source> {
fn eval_object_compr(
&mut self,
key: &'source Expr<'source>,
value: &'source Expr<'source>,
query: &'source Query<'source>,
key: &'source Expr,
value: &'source Expr,
query: &'source Query,
) -> Result<Value> {
// Push new context
self.contexts.push(Context {
@@ -1446,7 +1432,7 @@ impl<'source> Interpreter<'source> {
}
}
fn lookup_function(&self, fcn: &'source Expr<'source>) -> Result<&Vec<&'source Rule<'source>>> {
fn lookup_function(&self, fcn: &'source Expr) -> Result<&Vec<&'source Rule>> {
let mut path = Self::get_path_string(fcn, None)?;
if !path.starts_with("data.") {
path = self.current_module_path.clone() + "." + &path;
@@ -1462,10 +1448,10 @@ impl<'source> Interpreter<'source> {
fn eval_builtin_call(
&mut self,
span: &'source Span<'source>,
span: &'source Span,
name: String,
builtin: builtins::BuiltinFcn,
params: &'source [Expr<'source>],
params: &'source [Expr],
) -> Result<Value> {
let mut args = vec![];
let allow_undefined = name == "print"; // TODO: with modifier
@@ -1501,9 +1487,9 @@ impl<'source> Interpreter<'source> {
fn eval_call_impl(
&mut self,
span: &'source Span<'source>,
fcn: &'source Expr<'source>,
params: &'source [Expr<'source>],
span: &'source Span,
fcn: &'source Expr,
params: &'source [Expr],
) -> Result<Value> {
let fcns_rules = match self.lookup_function(fcn) {
Ok(r) => r,
@@ -1637,20 +1623,20 @@ impl<'source> Interpreter<'source> {
fn eval_call(
&mut self,
span: &'source Span<'source>,
fcn: &'source Expr<'source>,
params: &'source Vec<Expr<'source>>,
extra_arg: Option<&'source Expr<'source>>,
span: &'source Span,
fcn: &'source Expr,
params: &'source Vec<Expr>,
extra_arg: Option<&'source Expr>,
allow_return_arg: bool,
) -> Result<Value> {
// TODO: global var check; interop with `some var`
match extra_arg {
Some(Expr::Var(var))
if allow_return_arg && self.lookup_local_var(var.text()).is_none() =>
if allow_return_arg && self.lookup_local_var(*var.text()).is_none() =>
{
let value = self.eval_call_impl(span, fcn, &params[..params.len() - 1])?;
if var.text() != "_" {
self.add_variable(var.text(), value)?;
if *var.text() != "_" {
self.add_variable(*var.text(), value)?;
}
Ok(Value::Bool(true))
}
@@ -1696,16 +1682,16 @@ impl<'source> Interpreter<'source> {
Ok(())
}
fn lookup_var(&mut self, span: &'source Span<'source>, fields: &[&str]) -> Result<Value> {
fn lookup_var(&mut self, span: &'source Span, fields: &[&str]) -> Result<Value> {
let name = span.text();
// Return local variable/argument.
if let Some(v) = self.lookup_local_var(name) {
if let Some(v) = self.lookup_local_var(*name) {
return Ok(Self::get_value_chained(v, fields));
}
// Handle input.
if name == "input" {
if *name == "input" {
return Ok(Self::get_value_chained(self.input.clone(), fields));
}
@@ -1715,7 +1701,7 @@ impl<'source> Interpreter<'source> {
}
// Ensure that rules are evaluated
if name == "data" {
if *name == "data" {
let v = Self::get_value_chained(self.data.clone(), fields);
// If the rule has already been evaluated or specified via a with modifier,
@@ -1735,12 +1721,9 @@ impl<'source> Interpreter<'source> {
Ok(Self::get_value_chained(self.data.clone(), fields))
} else if !self.modules.is_empty() {
let mut path: Vec<&str> =
Parser::get_path_ref_components(&self.module.unwrap().package.refr)?
.iter()
.map(|s| s.text())
.collect();
path.push(name);
let path = Parser::get_path_ref_components(&self.module.unwrap().package.refr)?;
let mut path: Vec<&str> = path.iter().map(|s| *s.text()).collect();
path.push(*name);
let v = Self::get_value_chained(self.data.clone(), &path);
@@ -1753,7 +1736,7 @@ impl<'source> Interpreter<'source> {
// Add module prefix and ensure that any matching rule is evaluated.
let module_path =
Self::get_path_string(&self.current_module()?.package.refr, Some("data"))?;
let rule_path = module_path + "." + name;
let rule_path = module_path + "." + *name;
self.ensure_rule_evaluated(rule_path)?;
@@ -1764,12 +1747,12 @@ impl<'source> Interpreter<'source> {
}
}
fn eval_expr(&mut self, expr: &'source Expr<'source>) -> Result<Value> {
fn eval_expr(&mut self, expr: &'source Expr) -> Result<Value> {
match expr {
Expr::Null(_) => Ok(Value::Null),
Expr::True(_) => Ok(Value::Bool(true)),
Expr::False(_) => Ok(Value::Bool(false)),
Expr::Number(span) => match serde_json::from_str::<Value>(span.text()) {
Expr::Number(span) => match serde_json::from_str::<Value>(*span.text()) {
Ok(v) => Ok(v),
Err(e) => Err(span.source.error(
span.line,
@@ -1814,10 +1797,7 @@ impl<'source> Interpreter<'source> {
}
}
fn make_rule_context(
&self,
head: &'source RuleHead<'source>,
) -> Result<(Context<'source>, Vec<Span<'source>>)> {
fn make_rule_context(&self, head: &'source RuleHead) -> Result<(Context<'source>, Vec<Span>)> {
//TODO: include "data" ?
let mut path = Parser::get_path_ref_components(&self.module.unwrap().package.refr)?;
@@ -1861,7 +1841,7 @@ impl<'source> Interpreter<'source> {
}
}
fn get_rule_module(&self, rule: &'source Rule<'source>) -> Result<&'source Module<'source>> {
fn get_rule_module(&self, rule: &'source Rule) -> Result<&'source Module> {
for m in &self.modules {
if m.policy.iter().any(|r| Ref::make(r) == Ref::make(rule)) {
return Ok(m);
@@ -1873,8 +1853,8 @@ impl<'source> Interpreter<'source> {
fn eval_rule_bodies(
&mut self,
ctx: Context<'source>,
span: &'source Span<'source>,
bodies: &'source Vec<RuleBody<'source>>,
span: &'source Span,
bodies: &'source Vec<RuleBody>,
) -> Result<Value> {
let n_scopes = self.scopes.len();
let result = if bodies.is_empty() {
@@ -1974,7 +1954,7 @@ impl<'source> Interpreter<'source> {
}
}
pub fn merge_value(span: &Span<'source>, value: &mut Value, new: Value) -> Result<()> {
pub fn merge_value(span: &Span, value: &mut Value, new: Value) -> Result<()> {
match value.merge(new) {
Ok(()) => Ok(()),
Err(err) => return Err(span.error(format!("{err}").as_str())),
@@ -1987,19 +1967,19 @@ impl<'source> Interpreter<'source> {
while expr.is_some() {
match expr {
Some(Expr::RefDot { refr, field, .. }) => {
comps.push(field.text());
comps.push(*field.text());
expr = Some(refr);
}
Some(Expr::RefBrack { refr, index, .. })
if matches!(index.as_ref(), Expr::String(_)) =>
{
if let Expr::String(s) = index.as_ref() {
comps.push(s.text());
comps.push(*s.text());
expr = Some(refr);
}
}
Some(Expr::Var(v)) => {
comps.push(v.text());
comps.push(*v.text());
expr = None;
}
_ => bail!("internal error: not a simple ref"),
@@ -2014,8 +1994,8 @@ impl<'source> Interpreter<'source> {
fn set_current_module(
&mut self,
module: Option<&'source Module<'source>>,
) -> Result<Option<&'source Module<'source>>> {
module: Option<&'source Module>,
) -> Result<Option<&'source Module>> {
let m = self.module;
if let Some(m) = module {
self.current_module_path = Self::get_path_string(&m.package.refr, Some("data"))?;
@@ -2024,7 +2004,7 @@ impl<'source> Interpreter<'source> {
Ok(m)
}
fn get_rule_refr(rule: &'source Rule<'source>) -> &'source Expr<'source> {
fn get_rule_refr(rule: &'source Rule) -> &'source Expr {
match rule {
Rule::Spec { head, .. } => match &head {
RuleHead::Compr { refr, .. }
@@ -2035,7 +2015,7 @@ impl<'source> Interpreter<'source> {
}
}
fn check_default_value(expr: &'source Expr<'source>) -> Result<()> {
fn check_default_value(expr: &'source Expr) -> Result<()> {
use Expr::*;
let (kind, span) = match expr {
// Scalars are supported
@@ -2091,7 +2071,7 @@ impl<'source> Interpreter<'source> {
Ok(())
}
fn eval_default_rule(&mut self, rule: &'source Rule<'source>) -> Result<()> {
fn eval_default_rule(&mut self, rule: &'source Rule) -> Result<()> {
// Skip reprocessing rule.
if self.processed.contains(&Ref::make(rule)) {
return Ok(());
@@ -2113,7 +2093,7 @@ impl<'source> Interpreter<'source> {
};
Parser::get_path_ref_components_into(refr, &mut path)?;
let paths: Vec<&str> = path.iter().map(|s| s.text()).collect();
let paths: Vec<&str> = path.iter().map(|s| *s.text()).collect();
Self::check_default_value(value)?;
let value = self.eval_expr(value)?;
@@ -2149,11 +2129,7 @@ impl<'source> Interpreter<'source> {
Ok(())
}
fn eval_rule(
&mut self,
module: &'source Module<'source>,
rule: &'source Rule<'source>,
) -> Result<()> {
fn eval_rule(&mut self, module: &'source Module, rule: &'source Rule) -> Result<()> {
// Skip reprocessing rule
if self.processed.contains(&Ref::make(rule)) {
return Ok(());
@@ -2207,14 +2183,14 @@ impl<'source> Interpreter<'source> {
Value::Set(_) if special_set => {
let entry = path[path.len() - 1].text();
let mut s = BTreeSet::new();
s.insert(Value::String(entry.to_owned()));
s.insert(Value::String(entry.to_owned().to_string()));
path = path[0..path.len() - 1].to_vec();
Value::from_set(s)
}
v => v,
};
if value != Value::Undefined {
let paths: Vec<&str> = path.iter().map(|s| s.text()).collect();
let paths: Vec<&str> = path.iter().map(|s| *s.text()).collect();
let vref = Self::make_or_get_value_mut(&mut self.data, &paths[..])?;
Self::merge_value(span, vref, value)?;
}
@@ -2225,7 +2201,7 @@ impl<'source> Interpreter<'source> {
Parser::get_path_ref_components(&self.current_module()?.package.refr)?;
Parser::get_path_ref_components_into(refr, &mut path)?;
let path: Vec<&str> = path.iter().map(|s| s.text()).collect();
let path: Vec<&str> = path.iter().map(|s| *s.text()).collect();
// Ensure that for functions with a nesting level (e.g: a.foo),
// `a` is created as an empty object.
@@ -2249,8 +2225,8 @@ impl<'source> Interpreter<'source> {
pub fn eval_rule_with_input(
&mut self,
module: &'source Module<'source>,
rule: &'source Rule<'source>,
module: &'source Module,
rule: &'source Rule,
input: &Option<Value>,
enable_tracing: bool,
) -> Result<Value> {
@@ -2277,7 +2253,7 @@ impl<'source> Interpreter<'source> {
// Ensure that each module has an empty object
for m in &self.modules {
let path = Parser::get_path_ref_components(&m.package.refr)?;
let path: Vec<&str> = path.iter().map(|s| s.text()).collect();
let path: Vec<&str> = path.iter().map(|s| *s.text()).collect();
let vref = Self::make_or_get_value_mut(&mut self.data, &path[..])?;
if *vref == Value::Undefined {
*vref = Value::new_object();
@@ -2297,7 +2273,7 @@ impl<'source> Interpreter<'source> {
pub fn eval_module(
&mut self,
module: &'source Module<'source>,
module: &'source Module,
input: &Option<Value>,
enable_tracing: bool,
) -> Result<Value> {
@@ -2352,7 +2328,7 @@ impl<'source> Interpreter<'source> {
pub fn eval_user_query(
&mut self,
query: &'source Query<'source>,
query: &'source Query,
schedule: &Schedule<'source>,
enable_tracing: bool,
) -> Result<QueryResults> {

144
src/lexer.rs
View File

@@ -9,16 +9,80 @@ use crate::value::Value;
use anyhow::{anyhow, bail, Result};
#[derive(Clone)]
pub struct Source<'source> {
pub file: &'source str,
pub contents: &'source str,
pub lines: Vec<&'source str>,
struct SourceInternal {
pub file: String,
pub contents: String,
pub lines: Vec<(u16, u16)>,
}
impl<'source> Source<'source> {
#[derive(Clone)]
pub struct Source {
src: std::rc::Rc<SourceInternal>,
}
impl Source {
pub fn new(file: String, contents: String) -> Source {
let mut lines = vec![];
let mut prev_ch = ' ';
let mut prev_pos = 0u16;
let mut start = 0u16;
for (i, ch) in contents.char_indices() {
if ch == '\n' {
let end = match prev_ch {
'\r' => prev_pos,
_ => i as u16,
};
lines.push((start, end));
start = i as u16 + 1;
}
prev_ch = ch;
prev_pos = i as u16;
}
if (start as usize) < contents.len() {
lines.push((start, contents.len() as u16));
} else if contents.is_empty() {
lines.push((0, 0));
} else {
let s = (contents.len() - 1) as u16;
lines.push((s, s));
}
Self {
src: std::rc::Rc::new(SourceInternal {
file,
contents,
lines,
}),
}
}
pub fn from_file(path: String) -> Result<Source> {
let contents = match std::fs::read_to_string(&path) {
Ok(c) => c,
Err(e) => bail!("Failed to read {path}. {e}"),
};
Ok(Self::new(path, contents))
}
pub fn file(&self) -> &String {
&self.src.file
}
pub fn contents(&self) -> &String {
&self.src.contents
}
pub fn line(&self, idx: u16) -> &str {
let idx = idx as usize;
if idx < self.src.lines.len() {
let (start, end) = self.src.lines[idx];
&self.src.contents[start as usize..end as usize]
} else {
""
}
}
pub fn message(&self, line: u16, col: u16, kind: &str, msg: &str) -> String {
if line as usize > self.lines.len() {
return format!("{}: invalid line {} specified", self.file, line);
if line as usize > self.src.lines.len() {
return format!("{}: invalid line {} specified", self.src.file, line);
}
let line_str = format!("{line}");
@@ -30,12 +94,12 @@ impl<'source> Source<'source> {
{:<line_num_width$}| {}\n\
{:<line_num_width$}| {:<col_spaces$}^\n\
{}: {}",
self.file,
self.src.file,
line,
col,
"",
line,
self.lines[line as usize - 1],
self.line(line - 1),
"",
"",
kind,
@@ -49,17 +113,17 @@ impl<'source> Source<'source> {
}
#[derive(Clone)]
pub struct Span<'source> {
pub source: &'source Source<'source>,
pub struct Span {
pub source: Source,
pub line: u16,
pub col: u16,
pub start: u16,
pub end: u16,
}
impl<'source> Span<'source> {
pub fn text(&self) -> &'source str {
&self.source.contents[self.start as usize..self.end as usize]
impl Span {
pub fn text(&self) -> std::rc::Rc<&str> {
std::rc::Rc::new(&self.source.contents()[self.start as usize..self.end as usize])
}
pub fn message(&self, kind: &str, msg: &str) -> String {
@@ -71,7 +135,7 @@ impl<'source> Span<'source> {
}
}
impl<'source> Debug for Span<'source> {
impl Debug for Span {
fn fmt(&self, f: &mut Formatter<'_>) -> Result<(), std::fmt::Error> {
let t = self.text().escape_debug().to_string();
let max = 32;
@@ -99,21 +163,21 @@ pub enum TokenKind {
}
#[derive(Debug, Clone)]
pub struct Token<'source>(pub TokenKind, pub Span<'source>);
pub struct Token(pub TokenKind, pub Span);
#[derive(Clone)]
pub struct Lexer<'source> {
source: &'source Source<'source>,
source: Source,
iter: Peekable<CharIndices<'source>>,
line: u16,
col: u16,
}
impl<'source> Lexer<'source> {
pub fn new(source: &'source Source<'source>) -> Self {
pub fn new(source: &'source Source) -> Self {
Self {
source,
iter: source.contents.char_indices().peekable(),
source: source.clone(),
iter: source.contents().char_indices().peekable(),
line: 1,
col: 1,
}
@@ -122,18 +186,18 @@ impl<'source> Lexer<'source> {
fn peek(&mut self) -> (usize, char) {
match self.iter.peek() {
Some((index, chr)) => (*index, *chr),
_ => (self.source.contents.len(), '\x00'),
_ => (self.source.contents().len(), '\x00'),
}
}
fn peekahead(&mut self, n: usize) -> (usize, char) {
match self.iter.clone().nth(n) {
Some((index, chr)) => (index, chr),
_ => (self.source.contents.len(), '\x00'),
_ => (self.source.contents().len(), '\x00'),
}
}
fn read_ident(&mut self) -> Result<Token<'source>> {
fn read_ident(&mut self) -> Result<Token> {
let start = self.peek().0;
let col = self.col;
loop {
@@ -149,7 +213,7 @@ impl<'source> Lexer<'source> {
Ok(Token(
TokenKind::Ident,
Span {
source: self.source,
source: self.source.clone(),
line: self.line,
col,
start: start as u16,
@@ -165,7 +229,7 @@ impl<'source> Lexer<'source> {
}
// See https://www.json.org/json-en.html for number's grammar
fn read_number(&mut self) -> Result<Token<'source>> {
fn read_number(&mut self) -> Result<Token> {
let (start, chr) = self.peek();
let col = self.col;
self.iter.next();
@@ -206,7 +270,7 @@ impl<'source> Lexer<'source> {
}
// Ensure that the number is parsable in Rust.
match serde_json::from_str::<'source, Value>(&self.source.contents[start..end]) {
match serde_json::from_str::<Value>(&self.source.contents()[start..end]) {
Ok(_) => (),
Err(e) => {
let serde_msg = &e.to_string();
@@ -233,7 +297,7 @@ impl<'source> Lexer<'source> {
Ok(Token(
TokenKind::Number,
Span {
source: self.source,
source: self.source.clone(),
line: self.line,
col,
start: start as u16,
@@ -242,7 +306,7 @@ impl<'source> Lexer<'source> {
))
}
fn read_raw_string(&mut self) -> Result<Token<'source>> {
fn read_raw_string(&mut self) -> Result<Token> {
self.iter.next();
self.col += 1;
let (start, _) = self.peek();
@@ -270,7 +334,7 @@ impl<'source> Lexer<'source> {
Ok(Token(
TokenKind::RawString,
Span {
source: self.source,
source: self.source.clone(),
line,
col,
start: start as u16,
@@ -279,7 +343,7 @@ impl<'source> Lexer<'source> {
))
}
fn read_string(&mut self) -> Result<Token<'source>> {
fn read_string(&mut self) -> Result<Token> {
let (line, col) = (self.line, self.col);
self.iter.next();
self.col += 1;
@@ -335,7 +399,7 @@ impl<'source> Lexer<'source> {
self.col += (end - start) as u16;
// Ensure that the string is parsable in Rust.
match serde_json::from_str::<'source, String>(&self.source.contents[start - 1..end]) {
match serde_json::from_str::<String>(&self.source.contents()[start - 1..end]) {
Ok(_) => (),
Err(e) => {
let serde_msg = &e.to_string();
@@ -352,7 +416,7 @@ impl<'source> Lexer<'source> {
Ok(Token(
TokenKind::String,
Span {
source: self.source,
source: self.source.clone(),
line,
col: col + 1,
start: start as u16,
@@ -399,7 +463,7 @@ impl<'source> Lexer<'source> {
Ok(())
}
pub fn next_token(&mut self) -> Result<Token<'source>> {
pub fn next_token(&mut self) -> Result<Token> {
self.skip_ws()?;
let (start, chr) = self.peek();
@@ -423,7 +487,7 @@ impl<'source> Lexer<'source> {
self.col += 1;
self.iter.next();
Ok(Token(TokenKind::Symbol, Span {
source: self.source,
source: self.source.clone(),
line: self.line,
col,
start: start as u16,
@@ -440,7 +504,7 @@ impl<'source> Lexer<'source> {
end += 1;
}
Ok(Token(TokenKind::Symbol, Span {
source: self.source,
source: self.source.clone(),
line: self.line,
col,
start: start as u16,
@@ -456,7 +520,7 @@ impl<'source> Lexer<'source> {
self.iter.next();
};
Ok(Token(TokenKind::Symbol, Span {
source: self.source,
source: self.source.clone(),
line: self.line,
col,
start: start as u16,
@@ -468,7 +532,7 @@ impl<'source> Lexer<'source> {
self.iter.next();
self.iter.next();
Ok(Token(TokenKind::Symbol, Span {
source: self.source,
source: self.source.clone(),
line: self.line,
col,
start: start as u16,
@@ -478,7 +542,7 @@ impl<'source> Lexer<'source> {
'"' => self.read_string(),
'`' => self.read_raw_string(),
'\x00' => Ok(Token(TokenKind::Eof, Span {
source: self.source,
source: self.source.clone(),
line:self.line,
col,
start: start as u16,
@@ -487,7 +551,7 @@ impl<'source> Lexer<'source> {
_ if chr.is_ascii_digit() => self.read_number(),
_ if chr.is_ascii_alphabetic() || chr == '_' => {
let mut ident = self.read_ident()?;
if ident.1.text() == "set" && self.peek().1 == '(' {
if *ident.1.text() == "set" && self.peek().1 == '(' {
// set immediately followed by ( is treated as set( if
// the next token is ).
let state = (self.iter.clone(), self.line, self.col);
@@ -495,7 +559,7 @@ impl<'source> Lexer<'source> {
// Check it next token is ).
let next_tok = self.next_token()?;
let is_setp = next_tok.1.text() == ")";
let is_setp = *next_tok.1.text() == ")";
// Restore state
(self.iter, self.line, self.col) = state;

2
src/lib.rs
View File

@@ -3,6 +3,7 @@
pub mod ast;
pub mod builtins;
pub mod engine;
pub mod interpreter;
pub mod lexer;
pub mod parser;
@@ -11,6 +12,7 @@ mod utils;
pub mod value;
pub use ast::*;
pub use engine::*;
pub use interpreter::*;
pub use lexer::*;
pub use parser::*;

264
src/parser.rs
View File

@@ -9,22 +9,22 @@ use anyhow::{anyhow, bail, Result};
#[derive(Clone)]
pub struct Parser<'source> {
source: &'source Source<'source>,
source: Source,
lexer: Lexer<'source>,
tok: Token<'source>,
tok: Token,
line: u16,
end: u16,
future_keywords: BTreeMap<&'source str, Span<'source>>,
future_keywords: BTreeMap<String, Span>,
}
const FUTURE_KEYWORDS: [&str; 4] = ["contains", "every", "if", "in"];
impl<'source> Parser<'source> {
pub fn new(source: &'source Source<'source>) -> Result<Self> {
pub fn new(source: &'source Source) -> Result<Self> {
let mut lexer = Lexer::new(source);
let tok = lexer.next_token()?;
Ok(Self {
source,
source: source.clone(),
lexer,
tok,
line: 0,
@@ -41,7 +41,7 @@ impl<'source> Parser<'source> {
}
fn expect(&mut self, text: &str, context: &str) -> Result<()> {
if self.tok.1.text() == text {
if *self.tok.1.text() == text {
self.next_token()
} else {
let msg = format!("expecting `{text}` {context}");
@@ -65,7 +65,7 @@ impl<'source> Parser<'source> {
);
}
pub fn set_future_keyword(&mut self, kw: &'source str, span: &Span<'source>) -> Result<()> {
pub fn set_future_keyword(&mut self, kw: &str, span: &Span) -> Result<()> {
match &self.future_keywords.get(kw) {
Some(s) => Err(self.source.error(
span.line,
@@ -78,16 +78,13 @@ impl<'source> Parser<'source> {
.as_str(),
)),
None => {
self.future_keywords.insert(kw, span.clone());
self.future_keywords.insert(kw.to_string(), span.clone());
Ok(())
}
}
}
pub fn get_path_ref_components_into(
refr: &Expr<'source>,
comps: &mut Vec<Span<'source>>,
) -> Result<()> {
pub fn get_path_ref_components_into(refr: &Expr, comps: &mut Vec<Span>) -> Result<()> {
match refr {
Expr::RefDot { refr, field, .. } => {
Self::get_path_ref_components_into(refr, comps)?;
@@ -104,16 +101,16 @@ impl<'source> Parser<'source> {
Ok(())
}
pub fn get_path_ref_components(refr: &Expr<'source>) -> Result<Vec<Span<'source>>> {
pub fn get_path_ref_components(refr: &Expr) -> Result<Vec<Span>> {
let mut comps = vec![];
Self::get_path_ref_components_into(refr, &mut comps)?;
Ok(comps)
}
fn handle_import_future_keywords(&mut self, comps: &Vec<Span<'source>>) -> Result<bool> {
if comps.len() >= 2 && comps[0].text() == "future" && comps[1].text() == "keywords" {
fn handle_import_future_keywords(&mut self, comps: &Vec<Span>) -> Result<bool> {
if comps.len() >= 2 && *comps[0].text() == "future" && *comps[1].text() == "keywords" {
match comps.len() - 2 {
1 => self.set_future_keyword(comps[2].text(), &comps[2])?,
1 => self.set_future_keyword(&comps[2].text(), &comps[2])?,
0 => {
let span = &comps[1];
for kw in FUTURE_KEYWORDS.iter() {
@@ -128,7 +125,7 @@ impl<'source> Parser<'source> {
}
}
Ok(true)
} else if !comps.is_empty() && comps[0].text() == "future" {
} else if !comps.is_empty() && *comps[0].text() == "future" {
let s = &comps[0];
Err(self
.source
@@ -144,7 +141,7 @@ impl<'source> Parser<'source> {
is_optional: bool,
context: &str,
) -> Result<()> {
if self.tok.1.text() == kw {
if *self.tok.1.text() == kw {
match &self.future_keywords.get(kw) {
Some(_) => self.next_token(),
None => {
@@ -161,7 +158,7 @@ impl<'source> Parser<'source> {
}
}
fn is_keyword(&self, ident: &'source str) -> bool {
fn is_keyword(&self, ident: &str) -> bool {
matches!(
ident,
"as" | "default"
@@ -177,10 +174,10 @@ impl<'source> Parser<'source> {
)
}
fn parse_ident(&mut self) -> Result<Span<'source>> {
fn parse_ident(&mut self) -> Result<Span> {
let span = self.tok.1.clone();
match self.tok.0 {
TokenKind::Ident if self.is_keyword(span.text()) => Err(self.source.error(
TokenKind::Ident if self.is_keyword(*span.text()) => Err(self.source.error(
self.tok.1.line,
self.tok.1.col,
&format!("unexpected keyword `{}`", span.text()),
@@ -195,11 +192,12 @@ impl<'source> Parser<'source> {
}
}
fn parse_var(&mut self) -> Result<Span<'source>> {
fn parse_var(&mut self) -> Result<Span> {
let span = self.tok.1.clone();
match self.tok.0 {
TokenKind::Ident
if self.is_keyword(span.text()) || self.is_imported_future_keyword(span.text()) =>
if self.is_keyword(*span.text())
|| self.is_imported_future_keyword(*span.text()) =>
{
Err(self.source.error(
self.tok.1.line,
@@ -217,13 +215,13 @@ impl<'source> Parser<'source> {
}
}
fn parse_scalar_or_var(&mut self) -> Result<Expr<'source>> {
fn parse_scalar_or_var(&mut self) -> Result<Expr> {
let span = self.tok.1.clone();
let node = match &self.tok.0 {
TokenKind::Number => Expr::Number(span),
TokenKind::String => Expr::String(span),
TokenKind::RawString => Expr::RawString(span),
TokenKind::Ident => match self.tok.1.text() {
TokenKind::Ident => match *self.tok.1.text() {
"null" => Expr::Null(span),
"true" => Expr::True(span),
"false" => Expr::False(span),
@@ -241,14 +239,14 @@ impl<'source> Parser<'source> {
Ok(node)
}
fn parse_compr(&mut self, delim: &str) -> Result<(Expr<'source>, Query<'source>)> {
fn parse_compr(&mut self, delim: &str) -> Result<(Expr, Query)> {
// Save the state.
let state = self.clone();
let mut span = self.tok.1.clone();
// Parse the first expression as a ref.
let term = match self.parse_ref() {
Ok(e) if self.tok.1.text() == "|" => e,
Ok(e) if *self.tok.1.text() == "|" => e,
_ => {
// Not a comprehension. Restore state.
*self = state;
@@ -274,7 +272,7 @@ impl<'source> Parser<'source> {
}
}
fn parse_compr_or_array(&mut self) -> Result<Expr<'source>> {
fn parse_compr_or_array(&mut self) -> Result<Expr> {
// Save the state.
let mut span = self.tok.1.clone();
self.expect("[", "while parsing array comprehension or array")?;
@@ -293,11 +291,11 @@ impl<'source> Parser<'source> {
// No progress was made in parsing comprehension.
// Parse as array.
let mut items = vec![];
if self.tok.1.text() != "]" {
if *self.tok.1.text() != "]" {
items.push(self.parse_in_expr()?);
while self.tok.1.text() == "," {
while *self.tok.1.text() == "," {
self.next_token()?;
match self.tok.1.text() {
match *self.tok.1.text() {
"]" => break,
"" if self.tok.0 == TokenKind::Eof => break,
_ => items.push(self.parse_in_expr()?),
@@ -312,7 +310,7 @@ impl<'source> Parser<'source> {
}
}
fn parse_compr_set_or_object(&mut self) -> Result<Expr<'source>> {
fn parse_compr_set_or_object(&mut self) -> Result<Expr> {
let mut span = self.tok.1.clone();
self.expect("{", "while parsing set, object or comprehension")?;
@@ -336,7 +334,7 @@ impl<'source> Parser<'source> {
// It could be a set, object or object comprehension.
// In all the cases, the first expressoin must parse successfully.
if self.tok.1.text() == "}" {
if *self.tok.1.text() == "}" {
self.next_token()?;
span.end = self.end;
return Ok(Expr::Object {
@@ -348,12 +346,12 @@ impl<'source> Parser<'source> {
let mut item_span = self.tok.1.clone();
let first = self.parse_in_expr()?;
if self.tok.1.text() != ":" {
if *self.tok.1.text() != ":" {
// Parse as set.
let mut items = vec![first];
while self.tok.1.text() == "," {
while *self.tok.1.text() == "," {
self.next_token()?;
match self.tok.1.text() {
match *self.tok.1.text() {
"}" => break,
"" if self.tok.0 == TokenKind::Eof => break,
_ => items.push(self.parse_in_expr()?),
@@ -393,10 +391,10 @@ impl<'source> Parser<'source> {
item_span.end = self.end;
items.push((item_span, first, value));
while self.tok.1.text() == "," {
while *self.tok.1.text() == "," {
self.next_token()?;
let item_start = self.tok.1.start;
let key = match self.tok.1.text() {
let key = match *self.tok.1.text() {
"}" => break,
"" if self.tok.0 == TokenKind::Eof => break,
_ => self.parse_in_expr()?,
@@ -420,7 +418,7 @@ impl<'source> Parser<'source> {
})
}
fn parse_empty_set(&mut self) -> Result<Expr<'source>> {
fn parse_empty_set(&mut self) -> Result<Expr> {
let mut span = self.tok.1.clone();
self.expect("set(", "while parsing empty set")?;
self.expect(")", "while parsing empty set")?;
@@ -431,7 +429,7 @@ impl<'source> Parser<'source> {
})
}
fn parse_parens_expr(&mut self) -> Result<Expr<'source>> {
fn parse_parens_expr(&mut self) -> Result<Expr> {
self.next_token()?;
let expr = self.parse_membership_expr()?;
self.expect(")", "while parsing parenthesized expression")?;
@@ -439,7 +437,7 @@ impl<'source> Parser<'source> {
Ok(expr)
}
fn parse_unary_expr(&mut self) -> Result<Expr<'source>> {
fn parse_unary_expr(&mut self) -> Result<Expr> {
let mut span = self.tok.1.clone();
self.next_token()?;
let expr = self.parse_in_expr()?;
@@ -450,9 +448,9 @@ impl<'source> Parser<'source> {
})
}
fn parse_ref(&mut self) -> Result<Expr<'source>> {
fn parse_ref(&mut self) -> Result<Expr> {
let start = self.tok.1.start;
let mut term = match self.tok.1.text() {
let mut term = match *self.tok.1.text() {
"[" => self.parse_compr_or_array()?,
"{" => self.parse_compr_set_or_object()?,
"set(" => self.parse_empty_set()?,
@@ -467,7 +465,7 @@ impl<'source> Parser<'source> {
let mut span = self.tok.1.clone();
let sep_pos = span.start;
span.start = start;
match self.tok.1.text() {
match *self.tok.1.text() {
"." | "[" if self.tok.1.start != self.end => {
if self.line != self.tok.1.line {
// Newline encountered. This could be a separate
@@ -524,16 +522,16 @@ impl<'source> Parser<'source> {
}
"(" if possible_fcn => {
self.next_token()?;
if self.tok.1.text() == ")" {
if *self.tok.1.text() == ")" {
return Err(self
.tok
.1
.error("at least one argument required for function calls"));
}
let mut args = vec![self.parse_in_expr()?];
while self.tok.1.text() == "," {
while *self.tok.1.text() == "," {
self.next_token()?;
match self.tok.1.text() {
match *self.tok.1.text() {
")" => break,
"" if self.tok.0 == TokenKind::Eof => break,
_ => args.push(self.parse_in_expr()?),
@@ -557,18 +555,18 @@ impl<'source> Parser<'source> {
Ok(term)
}
fn parse_term(&mut self) -> Result<Expr<'source>> {
fn parse_term(&mut self) -> Result<Expr> {
self.parse_ref()
}
fn parse_mul_div_mod_expr(&mut self) -> Result<Expr<'source>> {
fn parse_mul_div_mod_expr(&mut self) -> Result<Expr> {
let start = self.tok.1.start;
let mut expr = self.parse_term()?;
loop {
let mut span = self.tok.1.clone();
span.start = start;
let op = match self.tok.1.text() {
let op = match *self.tok.1.text() {
"*" => ArithOp::Mul,
"/" => ArithOp::Div,
"%" => ArithOp::Mod,
@@ -586,14 +584,14 @@ impl<'source> Parser<'source> {
}
}
fn parse_arith_expr(&mut self) -> Result<Expr<'source>> {
fn parse_arith_expr(&mut self) -> Result<Expr> {
let start = self.tok.1.start;
let mut expr = self.parse_mul_div_mod_expr()?;
loop {
let mut span = self.tok.1.clone();
span.start = start;
let op = match self.tok.1.text() {
let op = match *self.tok.1.text() {
"+" => ArithOp::Add,
"-" => ArithOp::Sub,
_ => return Ok(expr),
@@ -610,11 +608,11 @@ impl<'source> Parser<'source> {
}
}
fn parse_and_expr(&mut self) -> Result<Expr<'source>> {
fn parse_and_expr(&mut self) -> Result<Expr> {
let start = self.tok.1.start;
let mut expr = self.parse_arith_expr()?;
while self.tok.1.text() == "&" {
while *self.tok.1.text() == "&" {
let mut span = self.tok.1.clone();
span.start = start;
self.next_token()?;
@@ -630,11 +628,11 @@ impl<'source> Parser<'source> {
Ok(expr)
}
fn parse_or_expr(&mut self) -> Result<Expr<'source>> {
fn parse_or_expr(&mut self) -> Result<Expr> {
let start = self.tok.1.start;
let mut expr = self.parse_and_expr()?;
while self.tok.1.text() == "|" {
while *self.tok.1.text() == "|" {
let mut span = self.tok.1.clone();
span.start = start;
self.next_token()?;
@@ -650,13 +648,13 @@ impl<'source> Parser<'source> {
Ok(expr)
}
fn parse_bool_expr(&mut self) -> Result<Expr<'source>> {
fn parse_bool_expr(&mut self) -> Result<Expr> {
let start = self.tok.1.start;
let mut expr = self.parse_or_expr()?;
loop {
let mut span = self.tok.1.clone();
span.start = start;
let op = match self.tok.1.text() {
let op = match *self.tok.1.text() {
"<" => BoolOp::Lt,
"<=" => BoolOp::Le,
"==" => BoolOp::Eq,
@@ -681,9 +679,9 @@ impl<'source> Parser<'source> {
fn parse_membership_tail(
&mut self,
start: u16,
mut expr1: Expr<'source>,
mut expr2: Option<Expr<'source>>,
) -> Result<Expr<'source>> {
mut expr1: Expr,
mut expr2: Option<Expr>,
) -> Result<Expr> {
loop {
let mut span = self.tok.1.clone();
span.start = start;
@@ -702,7 +700,7 @@ impl<'source> Parser<'source> {
};
expr2 = None;
if self.tok.1.text() != "in" {
if *self.tok.1.text() != "in" {
break;
}
}
@@ -710,42 +708,42 @@ impl<'source> Parser<'source> {
Ok(expr1)
}
fn parse_in_expr(&mut self) -> Result<Expr<'source>> {
fn parse_in_expr(&mut self) -> Result<Expr> {
let start = self.tok.1.start;
let mut expr = self.parse_bool_expr()?;
while self.tok.1.text() == "in" && self.future_keywords.get("in").is_some() {
while *self.tok.1.text() == "in" && self.future_keywords.get("in").is_some() {
expr = self.parse_membership_tail(start, expr, None)?;
}
Ok(expr)
}
pub fn parse_membership_expr(&mut self) -> Result<Expr<'source>> {
pub fn parse_membership_expr(&mut self) -> Result<Expr> {
let start = self.tok.1.start;
let mut expr = self.parse_bool_expr()?;
if self.tok.1.text() == "," {
if *self.tok.1.text() == "," {
self.next_token()?;
let value = self.parse_bool_expr()?;
expr = self.parse_membership_tail(start, expr, Some(value))?;
}
while self.tok.1.text() == "in" && self.is_imported_future_keyword("in") {
while *self.tok.1.text() == "in" && self.is_imported_future_keyword("in") {
expr = self.parse_membership_tail(start, expr, None)?;
}
Ok(expr)
}
pub fn parse_assign_expr(&mut self) -> Result<Expr<'source>> {
pub fn parse_assign_expr(&mut self) -> Result<Expr> {
let state = self.clone();
let start = self.tok.1.start;
let expr = self.parse_ref()?;
let mut span = self.tok.1.clone();
span.start = start;
let op = match self.tok.1.text() {
let op = match *self.tok.1.text() {
"=" => AssignOp::Eq,
":=" => AssignOp::ColEq,
_ => {
@@ -765,9 +763,9 @@ impl<'source> Parser<'source> {
})
}
fn parse_with_modifiers(&mut self) -> Result<Vec<WithModifier<'source>>> {
fn parse_with_modifiers(&mut self) -> Result<Vec<WithModifier>> {
let mut modifiers = vec![];
while self.tok.1.text() == "with" {
while *self.tok.1.text() == "with" {
let mut span = self.tok.1.clone();
self.next_token()?;
let refr = self.parse_path_ref()?;
@@ -779,13 +777,13 @@ impl<'source> Parser<'source> {
Ok(modifiers)
}
fn parse_every_stmt(&mut self) -> Result<Literal<'source>> {
fn parse_every_stmt(&mut self) -> Result<Literal> {
let mut span = self.tok.1.clone();
let context = "Failed to parse `every` statement.";
self.parse_future_keyword("every", false, context)?;
let ident = self.parse_var()?;
let (key, value) = match self.tok.1.text() {
let (key, value) = match *self.tok.1.text() {
"," => {
self.next_token()?;
match self.parse_var() {
@@ -818,7 +816,7 @@ impl<'source> Parser<'source> {
})
}
fn parse_some_stmt(&mut self) -> Result<Literal<'source>> {
fn parse_some_stmt(&mut self) -> Result<Literal> {
let mut span = self.tok.1.clone();
self.expect("some", "while parsing some-decl")?;
@@ -826,7 +824,7 @@ impl<'source> Parser<'source> {
let mut vars = vec![self.tok.1.clone()];
let mut refs = vec![self.parse_ref()?];
while self.tok.1.text() == "," {
while *self.tok.1.text() == "," {
self.next_token()?;
let mut span = self.tok.1.clone();
refs.push(self.parse_ref()?);
@@ -834,8 +832,8 @@ impl<'source> Parser<'source> {
vars.push(span);
}
if self.tok.1.text() != "in" || !self.is_imported_future_keyword("in") {
if self.tok.1.text() == "in" {
if *self.tok.1.text() != "in" || !self.is_imported_future_keyword("in") {
if *self.tok.1.text() == "in" {
self.warn_future_keyword();
}
// All the refs must be identifiers
@@ -846,7 +844,7 @@ impl<'source> Parser<'source> {
_ => {
return Err(anyhow!(
"{}:{}:{} error: encountered `{}` while expecting identifier",
span.source.file,
span.source.file(),
span.line,
span.col,
span.text()
@@ -866,7 +864,7 @@ impl<'source> Parser<'source> {
let span = &vars[2];
return Err(anyhow!(
"{}:{}:{} error: encountered `{}` while expecting `in`",
span.source.file,
span.source.file(),
span.line,
span.col,
span.text()
@@ -884,8 +882,8 @@ impl<'source> Parser<'source> {
})
}
fn parse_literal(&mut self) -> Result<Literal<'source>> {
match self.tok.1.text() {
fn parse_literal(&mut self) -> Result<Literal> {
match *self.tok.1.text() {
"some" => return self.parse_some_stmt(),
"every" => {
if self.future_keywords.get("every").is_some() {
@@ -896,7 +894,7 @@ impl<'source> Parser<'source> {
_ => (),
}
let mut span = self.tok.1.clone();
let not_expr = if self.tok.1.text() == "not" {
let not_expr = if *self.tok.1.text() == "not" {
self.next_token()?;
true
} else {
@@ -912,7 +910,7 @@ impl<'source> Parser<'source> {
}
}
pub fn parse_literal_stmt(&mut self) -> Result<LiteralStmt<'source>> {
pub fn parse_literal_stmt(&mut self) -> Result<LiteralStmt> {
let mut span = self.tok.1.clone();
let literal = self.parse_literal()?;
let with_mods = self.parse_with_modifiers()?;
@@ -925,13 +923,9 @@ impl<'source> Parser<'source> {
})
}
pub fn parse_query(
&mut self,
mut span: Span<'source>,
end_delim: &str,
) -> Result<Query<'source>> {
pub fn parse_query(&mut self, mut span: Span, end_delim: &str) -> Result<Query> {
let state = self.clone();
let _is_definite_query = matches!(self.tok.1.text(), "some" | "every");
let _is_definite_query = matches!(*self.tok.1.text(), "some" | "every");
// TODO: empty query?
let mut literals = vec![];
@@ -947,7 +941,7 @@ impl<'source> Parser<'source> {
}
};
if self.tok.1.text() == "," {
if *self.tok.1.text() == "," {
// This is likely an array or set.
// Restore the state.
*self = state;
@@ -957,7 +951,7 @@ impl<'source> Parser<'source> {
literals.push(stmt);
loop {
match self.tok.1.text() {
match *self.tok.1.text() {
t if t == end_delim => break,
"" if self.tok.0 == TokenKind::Eof => break,
";" => self.next_token()?,
@@ -982,10 +976,10 @@ impl<'source> Parser<'source> {
})
}
pub fn parse_rule_assign(&mut self) -> Result<Option<RuleAssign<'source>>> {
pub fn parse_rule_assign(&mut self) -> Result<Option<RuleAssign>> {
let mut span = self.tok.1.clone();
let op = match self.tok.1.text() {
let op = match *self.tok.1.text() {
"=" => {
self.next_token()?;
AssignOp::Eq
@@ -1006,7 +1000,7 @@ impl<'source> Parser<'source> {
}))
}
fn parse_path_ref(&mut self) -> Result<Expr<'source>> {
fn parse_path_ref(&mut self) -> Result<Expr> {
let start = self.tok.1.start;
let var = self.parse_var()?;
@@ -1015,14 +1009,14 @@ impl<'source> Parser<'source> {
let mut span = self.tok.1.clone();
let sep_pos = span.start;
span.start = start;
match self.tok.1.text() {
match *self.tok.1.text() {
"." | "[" if self.tok.1.start != self.end => {
bail!(
"{}",
self.source.error(
self.tok.1.line,
self.tok.1.col - 1,
format!("invalid whitespace before {}", self.tok.1.text()).as_str()
format!("invalid whitespace before {}", *self.tok.1.text()).as_str()
)
);
}
@@ -1098,7 +1092,7 @@ impl<'source> Parser<'source> {
}
}
fn parse_rule_ref(&mut self) -> Result<Expr<'source>> {
fn parse_rule_ref(&mut self) -> Result<Expr> {
let start = self.tok.1.start;
let span = self.tok.1.clone();
@@ -1115,7 +1109,7 @@ impl<'source> Parser<'source> {
loop {
let mut span = self.tok.1.clone();
span.start = start;
match self.tok.1.text() {
match *self.tok.1.text() {
// . and [ must not have any space between the previous token.
"." | "[" if self.tok.1.start != self.end => {
bail!(
@@ -1123,7 +1117,7 @@ impl<'source> Parser<'source> {
self.source.error(
self.tok.1.line,
self.tok.1.col - 1,
format!("invalid whitespace before {}", self.tok.1.text()).as_str()
format!("invalid whitespace before {}", *self.tok.1.text()).as_str()
)
);
}
@@ -1168,18 +1162,18 @@ impl<'source> Parser<'source> {
Ok(term)
}
pub fn parse_rule_head(&mut self) -> Result<RuleHead<'source>> {
pub fn parse_rule_head(&mut self) -> Result<RuleHead> {
let mut span = self.tok.1.clone();
let rule_ref = self.parse_rule_ref()?;
match self.tok.1.text() {
match *self.tok.1.text() {
"(" => {
self.check_rule_ref(&rule_ref)?;
self.next_token()?;
let mut args = vec![self.parse_term()?];
while self.tok.1.text() == "," {
while *self.tok.1.text() == "," {
self.next_token()?;
match self.tok.1.text() {
match *self.tok.1.text() {
")" => break,
"" if self.tok.0 == TokenKind::Eof => break,
_ => args.push(self.parse_term()?),
@@ -1219,8 +1213,8 @@ impl<'source> Parser<'source> {
}
// Determine whether to create a set or a compr
let is_set_follower = !self.is_keyword(self.tok.1.text())
&& !self.is_imported_future_keyword(self.tok.1.text());
let is_set_follower = !self.is_keyword(*self.tok.1.text())
&& !self.is_imported_future_keyword(*self.tok.1.text());
if assign.is_none() && is_set_follower {
match &rule_ref {
Expr::RefBrack { refr, index, .. }
@@ -1257,11 +1251,11 @@ impl<'source> Parser<'source> {
self.future_keywords.get("if").is_some()
}
pub fn parse_query_or_literal_stmt(&mut self) -> Result<Query<'source>> {
pub fn parse_query_or_literal_stmt(&mut self) -> Result<Query> {
let state = self.clone();
let mut span = self.tok.1.clone();
if self.tok.1.text() == "{" {
if *self.tok.1.text() == "{" {
self.next_token()?;
let pos = self.end;
match self.parse_query(span.clone(), "}") {
@@ -1281,12 +1275,12 @@ impl<'source> Parser<'source> {
Ok(Query { span, stmts })
}
pub fn parse_rule_bodies(&mut self) -> Result<Vec<RuleBody<'source>>> {
pub fn parse_rule_bodies(&mut self) -> Result<Vec<RuleBody>> {
let mut span = self.tok.1.clone();
let mut bodies = vec![];
let assign = None;
let has_query = match self.tok.1.text() {
let has_query = match *self.tok.1.text() {
"if" if self.if_is_keyword() => {
self.next_token()?;
let query = self.parse_query_or_literal_stmt()?;
@@ -1316,7 +1310,7 @@ impl<'source> Parser<'source> {
_ => false,
};
match self.tok.1.text() {
match *self.tok.1.text() {
"{" if has_query => self.parse_query_blocks(&mut bodies)?,
"else" if has_query => self.parse_else_blocks(&mut bodies)?,
_ => (),
@@ -1325,8 +1319,8 @@ impl<'source> Parser<'source> {
Ok(bodies)
}
pub fn parse_query_blocks(&mut self, bodies: &mut Vec<RuleBody<'source>>) -> Result<()> {
while self.tok.1.text() == "{" {
pub fn parse_query_blocks(&mut self, bodies: &mut Vec<RuleBody>) -> Result<()> {
while *self.tok.1.text() == "{" {
let mut span = self.tok.1.clone();
self.next_token()?;
let query = self.parse_query(span.clone(), "}")?;
@@ -1340,11 +1334,11 @@ impl<'source> Parser<'source> {
Ok(())
}
pub fn parse_else_blocks(&mut self, bodies: &mut Vec<RuleBody<'source>>) -> Result<()> {
pub fn parse_else_blocks(&mut self, bodies: &mut Vec<RuleBody>) -> Result<()> {
loop {
let mut span = self.tok.1.clone();
match self.tok.1.text() {
match *self.tok.1.text() {
"{" => {
return Err(self.source.error(
self.tok.1.line,
@@ -1358,7 +1352,7 @@ impl<'source> Parser<'source> {
let assign = self.parse_rule_assign()?;
match self.tok.1.text() {
match *self.tok.1.text() {
"if" if self.if_is_keyword() => {
self.next_token()?;
let query = self.parse_query_or_literal_stmt()?;
@@ -1380,7 +1374,7 @@ impl<'source> Parser<'source> {
});
}
_ if assign.is_none() => {
if self.tok.1.text() == "if" {
if *self.tok.1.text() == "if" {
self.warn_future_keyword();
}
return Err(self.source.error(
@@ -1395,12 +1389,12 @@ impl<'source> Parser<'source> {
Ok(())
}
pub fn parse_default_rule(&mut self) -> Result<Rule<'source>> {
pub fn parse_default_rule(&mut self) -> Result<Rule> {
let mut span = self.tok.1.clone();
self.expect("default", "while parsing default rule")?;
let rule_ref = self.parse_rule_ref()?;
let op = match self.tok.1.text() {
let op = match *self.tok.1.text() {
"=" => AssignOp::Eq,
":=" => AssignOp::ColEq,
_ => {
@@ -1423,7 +1417,7 @@ impl<'source> Parser<'source> {
})
}
pub fn parse_rule(&mut self) -> Result<Rule<'source>> {
pub fn parse_rule(&mut self) -> Result<Rule> {
let pos = self.end;
match self.parse_default_rule() {
Ok(r) => return Ok(r),
@@ -1438,7 +1432,7 @@ impl<'source> Parser<'source> {
Ok(Rule::Spec { span, head, bodies })
}
fn parse_package(&mut self) -> Result<Package<'source>> {
fn parse_package(&mut self) -> Result<Package> {
let mut span = self.tok.1.clone();
self.expect("package", "Missing package declaration.")?;
let name = self.parse_path_ref()?;
@@ -1446,21 +1440,13 @@ impl<'source> Parser<'source> {
Ok(Package { span, refr: name })
}
fn check_and_add_import(
&self,
import: Import<'source>,
imports: &mut Vec<Import<'source>>,
) -> Result<()> {
let comps: Vec<&str> = Self::get_path_ref_components(&import.refr)?
.iter()
.map(|s| s.text())
.collect();
fn check_and_add_import(&self, import: Import, imports: &mut Vec<Import>) -> Result<()> {
let ref_comps = Self::get_path_ref_components(&import.refr)?;
let comps: Vec<std::rc::Rc<&str>> = ref_comps.iter().map(|s| s.text()).collect();
for imp in imports.iter() {
let imp_comps: Vec<&str> = Self::get_path_ref_components(&imp.refr)?
.iter()
.map(|s| s.text())
.collect();
let imp_comps = Self::get_path_ref_components(&imp.refr)?;
let imp_comps: Vec<std::rc::Rc<&str>> = imp_comps.iter().map(|s| s.text()).collect();
let shadow = match (&imp.r#as, &import.r#as) {
(Some(i1), Some(i2)) if i1.text() == i2.text() => true,
@@ -1490,15 +1476,15 @@ impl<'source> Parser<'source> {
Ok(())
}
fn parse_imports(&mut self) -> Result<Vec<Import<'source>>> {
fn parse_imports(&mut self) -> Result<Vec<Import>> {
let mut imports = vec![];
while self.tok.1.text() == "import" {
while *self.tok.1.text() == "import" {
let mut span = self.tok.1.clone();
self.next_token()?;
let refr = self.parse_path_ref()?;
let comps = Self::get_path_ref_components(&refr)?;
if !matches!(comps[0].text(), "data" | "future" | "input") {
if !matches!(*comps[0].text(), "data" | "future" | "input") {
return Err(self.source.error(
comps[0].line,
comps[0].col,
@@ -1508,7 +1494,7 @@ impl<'source> Parser<'source> {
let is_future_kw = self.handle_import_future_keywords(&comps)?;
let var = if self.tok.1.text() == "as" {
let var = if *self.tok.1.text() == "as" {
if is_future_kw {
return Err(self.source.error(
self.tok.1.line,
@@ -1519,7 +1505,7 @@ impl<'source> Parser<'source> {
self.next_token()?;
let var = self.parse_var()?;
if var.text() == "_" {
if *var.text() == "_" {
return Err(self.source.error(
var.line,
var.col,
@@ -1547,7 +1533,7 @@ impl<'source> Parser<'source> {
Ok(imports)
}
pub fn parse(&mut self) -> Result<Module<'source>> {
pub fn parse(&mut self) -> Result<Module> {
let package = self.parse_package()?;
let imports = self.parse_imports()?;

139
src/scheduler.rs
View File

@@ -200,7 +200,7 @@ pub struct Scope<'a> {
pub inputs: BTreeSet<&'a str>,
}
fn traverse<'a>(expr: &'a Expr<'a>, f: &mut dyn FnMut(&'a Expr<'a>) -> Result<bool>) -> Result<()> {
fn traverse<'a>(expr: &'a Expr, f: &mut dyn FnMut(&'a Expr) -> Result<bool>) -> Result<()> {
if !f(expr)? {
return Ok(());
}
@@ -267,30 +267,30 @@ fn var_exists<'a>(name: &'a str, parent_scopes: &[Scope<'a>]) -> bool {
}
fn gather_assigned_vars<'a>(
expr: &'a Expr<'a>,
expr: &'a Expr,
can_shadow: bool,
parent_scopes: &[Scope<'a>],
scope: &mut Scope<'a>,
) -> Result<()> {
traverse(expr, &mut |e| match e {
// Ignore _, input, data.
Var(v) if matches!(v.text(), "_" | "input" | "data") => Ok(false),
Var(v) if matches!(*v.text(), "_" | "input" | "data") => Ok(false),
// Record local var that can shadow input var.
Var(v) if can_shadow => {
scope.locals.insert(v.text());
scope.locals.insert(*v.text());
Ok(false)
}
// Record input vars.
Var(v) if var_exists(v.text(), parent_scopes) => {
scope.inputs.insert(v.text());
Var(v) if var_exists(*v.text(), parent_scopes) => {
scope.inputs.insert(*v.text());
Ok(false)
}
// Record local var.
Var(v) => {
scope.locals.insert(v.text());
scope.locals.insert(*v.text());
Ok(false)
}
@@ -301,15 +301,15 @@ fn gather_assigned_vars<'a>(
}
fn gather_input_vars<'a>(
expr: &'a Expr<'a>,
expr: &'a Expr,
parent_scopes: &[Scope<'a>],
scope: &mut Scope<'a>,
) -> Result<()> {
traverse(expr, &mut |e| match e {
Var(v) if var_exists(v.text(), parent_scopes) => {
Var(v) if var_exists(*v.text(), parent_scopes) => {
let var = v.text();
if !scope.locals.contains(var) {
scope.inputs.insert(var);
if !scope.locals.contains(&*var) {
scope.inputs.insert(*var);
}
Ok(false)
}
@@ -318,19 +318,19 @@ fn gather_input_vars<'a>(
}
fn gather_loop_vars<'a>(
expr: &'a Expr<'a>,
expr: &'a Expr,
parent_scopes: &[Scope<'a>],
scope: &mut Scope<'a>,
) -> Result<()> {
traverse(expr, &mut |e| match e {
Var(v) if var_exists(v.text(), parent_scopes) => Ok(false),
Var(v) if var_exists(*v.text(), parent_scopes) => Ok(false),
RefBrack { index, .. } => {
if let Var(v) = index.as_ref() {
if !matches!(v.text(), "_" | "input" | "data")
&& !var_exists(v.text(), parent_scopes)
if !matches!(*v.text(), "_" | "input" | "data")
&& !var_exists(*v.text(), parent_scopes)
{
// Treat this as an index var.
scope.locals.insert(v.text());
scope.locals.insert(*v.text());
}
}
Ok(true)
@@ -346,7 +346,7 @@ fn gather_loop_vars<'a>(
// {k:y} = t
// Try inlining value of t
fn gather_vars<'a>(
expr: &'a Expr<'a>,
expr: &'a Expr,
can_shadow: bool,
parent_scopes: &[Scope<'a>],
scope: &mut Scope<'a>,
@@ -366,9 +366,9 @@ fn gather_vars<'a>(
gather_loop_vars(expr, parent_scopes, scope)
}
fn get_rule_prefix<'a>(expr: &Expr<'a>) -> Result<&'a str> {
fn get_rule_prefix(expr: &Expr) -> Result<&str> {
match expr {
Expr::Var(v) => Ok(v.text()),
Expr::Var(v) => Ok(*v.text()),
Expr::RefDot { refr, .. } => get_rule_prefix(refr),
Expr::RefBrack { refr, .. } => get_rule_prefix(refr),
_ => bail!("internal error: analyzer: could not get rule prefix"),
@@ -377,16 +377,16 @@ fn get_rule_prefix<'a>(expr: &Expr<'a>) -> Result<&'a str> {
pub struct Analyzer<'a> {
packages: BTreeMap<String, Scope<'a>>,
locals: BTreeMap<Ref<'a, Query<'a>>, Scope<'a>>,
locals: BTreeMap<Ref<'a, Query>, Scope<'a>>,
scopes: Vec<Scope<'a>>,
order: BTreeMap<Ref<'a, Query<'a>>, Vec<u16>>,
order: BTreeMap<Ref<'a, Query>, Vec<u16>>,
functions: FunctionTable<'a>,
}
#[derive(Clone)]
pub struct Schedule<'a> {
pub scopes: BTreeMap<Ref<'a, Query<'a>>, Scope<'a>>,
pub order: BTreeMap<Ref<'a, Query<'a>>, Vec<u16>>,
pub scopes: BTreeMap<Ref<'a, Query>, Scope<'a>>,
pub order: BTreeMap<Ref<'a, Query>, Vec<u16>>,
}
impl<'a> Default for Analyzer<'a> {
@@ -406,7 +406,7 @@ impl<'a> Analyzer<'a> {
}
}
pub fn analyze(mut self, modules: &'a [&'a Module<'a>]) -> Result<Schedule> {
pub fn analyze(mut self, modules: &'a [&'a Module]) -> Result<Schedule> {
self.add_rules(modules)?;
self.functions = gather_functions(modules)?;
@@ -422,8 +422,8 @@ impl<'a> Analyzer<'a> {
pub fn analyze_query_snippet(
mut self,
modules: &'a [&'a Module<'a>],
query: &'a Query<'a>,
modules: &'a [&'a Module],
query: &'a Query,
) -> Result<Schedule<'a>> {
self.add_rules(modules)?;
self.analyze_query(None, None, query, Scope::default())?;
@@ -434,7 +434,7 @@ impl<'a> Analyzer<'a> {
})
}
fn add_rules(&mut self, modules: &'a [&'a Module<'a>]) -> Result<()> {
fn add_rules(&mut self, modules: &'a [&'a Module]) -> Result<()> {
for m in modules {
let path = get_path_string(&m.package.refr, Some("data"))?;
let scope: &mut Scope = self.packages.entry(path).or_default();
@@ -456,7 +456,7 @@ impl<'a> Analyzer<'a> {
Ok(())
}
fn analyze_module(&mut self, m: &'a Module<'a>) -> Result<()> {
fn analyze_module(&mut self, m: &'a Module) -> Result<()> {
let path = get_path_string(&m.package.refr, Some("data"))?;
let scope = match self.packages.get(&path) {
Some(s) => s,
@@ -472,7 +472,7 @@ impl<'a> Analyzer<'a> {
Ok(())
}
fn analyze_rule(&mut self, r: &'a Rule<'a>) -> Result<()> {
fn analyze_rule(&mut self, r: &'a Rule) -> Result<()> {
match r {
Rule::Spec { head, bodies, .. } => {
let (key, value, scope) = self.analyze_rule_head(head)?;
@@ -497,7 +497,7 @@ impl<'a> Analyzer<'a> {
}
}
fn analyze_value_expr(&mut self, expr: &'a Expr<'a>) -> Result<()> {
fn analyze_value_expr(&mut self, expr: &'a Expr) -> Result<()> {
let mut comprs = vec![];
traverse(expr, &mut |e| match e {
ArrayCompr { .. } | SetCompr { .. } | ObjectCompr { .. } => {
@@ -527,8 +527,8 @@ impl<'a> Analyzer<'a> {
fn analyze_rule_head(
&mut self,
head: &'a RuleHead<'a>,
) -> Result<(Option<&'a Expr<'a>>, Option<&'a Expr<'a>>, Scope<'a>)> {
head: &'a RuleHead,
) -> Result<(Option<&'a Expr>, Option<&'a Expr>, Scope<'a>)> {
let mut scope = Scope::default();
Ok(match head {
RuleHead::Compr { assign, .. } => (None, assign.as_ref().map(|a| &a.value), scope),
@@ -536,7 +536,7 @@ impl<'a> Analyzer<'a> {
RuleHead::Func { args, assign, .. } => {
for a in args.iter() {
if let Var(v) = a {
scope.locals.insert(v.text());
scope.locals.insert(*v.text());
}
}
(None, assign.as_ref().map(|a| &a.value), scope)
@@ -546,16 +546,16 @@ impl<'a> Analyzer<'a> {
fn gather_local_vars(
&mut self,
key: Option<&'a Expr<'a>>,
value: Option<&'a Expr<'a>>,
query: &'a Query<'a>,
key: Option<&'a Expr>,
value: Option<&'a Expr>,
query: &'a Query,
scope: &mut Scope<'a>,
) -> Result<()> {
// First process assign, some expressions and gather local vars.
for stmt in &query.stmts {
match &stmt.literal {
Literal::SomeVars { vars, .. } => vars.iter().for_each(|v| {
scope.locals.insert(v.text());
scope.locals.insert(*v.text());
}),
Literal::SomeIn {
key,
@@ -602,16 +602,16 @@ impl<'a> Analyzer<'a> {
}
fn gather_used_vars_comprs_index_vars(
expr: &'a Expr<'a>,
expr: &'a Expr,
scope: &mut Scope<'a>,
first_use: &mut BTreeMap<&'a str, Span<'a>>,
first_use: &mut BTreeMap<&'a str, Span>,
definitions: &mut Vec<Definition<'a>>,
) -> Result<(Vec<&'a str>, Vec<&'a Expr<'a>>)> {
) -> Result<(Vec<&'a str>, Vec<&'a Expr>)> {
let mut used_vars = vec![];
let mut comprs = vec![];
traverse(expr, &mut |e| match e {
Var(v) if !matches!(v.text(), "_" | "input" | "data") => {
let name = v.text();
Var(v) if !matches!(*v.text(), "_" | "input" | "data") => {
let name = *v.text();
if scope.locals.contains(name)
/*|| scope.inputs.contains(name) */
{
@@ -625,7 +625,7 @@ impl<'a> Analyzer<'a> {
RefBrack { refr, index, .. } => {
if let Var(v) = index.as_ref() {
let var = v.text();
let var = *v.text();
if scope.locals.contains(var) {
let (rb_used_vars, rb_comprs) = Self::gather_used_vars_comprs_index_vars(
refr,
@@ -658,9 +658,9 @@ impl<'a> Analyzer<'a> {
fn process_comprs(
&mut self,
comprs: &[&'a Expr<'a>],
comprs: &[&'a Expr],
scope: &mut Scope<'a>,
first_use: &mut BTreeMap<&'a str, Span<'a>>,
first_use: &mut BTreeMap<&'a str, Span>,
used_vars: &mut Vec<&'a str>,
) -> Result<()> {
self.scopes.push(scope.clone());
@@ -706,15 +706,15 @@ impl<'a> Analyzer<'a> {
fn gather_assigned_vars(
&self,
expr: &'a Expr<'a>,
expr: &'a Expr,
scope: &Scope<'a>,
check_first_use: bool,
first_use: &BTreeMap<&'a str, Span<'a>>,
first_use: &BTreeMap<&'a str, Span>,
) -> Result<Vec<&'a str>> {
let mut vars = vec![];
traverse(expr, &mut |e| match e {
Var(v) => {
let var = v.text();
let var = *v.text();
if scope.locals.contains(var) {
if check_first_use {
Self::check_first_use(v, first_use)?;
@@ -733,10 +733,10 @@ impl<'a> Analyzer<'a> {
fn process_assign_expr(
&mut self,
op: &AssignOp,
lhs: &'a Expr<'a>,
rhs: &'a Expr<'a>,
lhs: &'a Expr,
rhs: &'a Expr,
scope: &mut Scope<'a>,
first_use: &mut BTreeMap<&'a str, Span<'a>>,
first_use: &mut BTreeMap<&'a str, Span>,
definitions: &mut Vec<Definition<'a>>,
) -> Result<()> {
match (lhs, rhs) {
@@ -807,9 +807,9 @@ impl<'a> Analyzer<'a> {
fn process_expr(
&mut self,
expr: &'a Expr<'a>,
expr: &'a Expr,
scope: &mut Scope<'a>,
first_use: &mut BTreeMap<&'a str, Span<'a>>,
first_use: &mut BTreeMap<&'a str, Span>,
definitions: &mut Vec<Definition<'a>>,
) -> Result<()> {
match expr {
@@ -826,8 +826,8 @@ impl<'a> Analyzer<'a> {
}
}
fn check_first_use(var: &Span<'a>, first_use: &BTreeMap<&'a str, Span<'a>>) -> Result<()> {
let name = var.text();
fn check_first_use(var: &Span, first_use: &BTreeMap<&'a str, Span>) -> Result<()> {
let name = *var.text();
if let Some(r#use) = first_use.get(name) {
if r#use.line < var.line || (r#use.line == var.line && r#use.col < var.col) {
bail!(r#use.error(
@@ -843,15 +843,15 @@ impl<'a> Analyzer<'a> {
}
fn gather_some_vars(
expr: &'a Expr<'a>,
expr: &'a Expr,
scope: &Scope<'a>,
_first_use: &BTreeMap<&'a str, Span<'a>>,
_first_use: &BTreeMap<&'a str, Span>,
vars: &mut Vec<&'a str>,
non_vars: &mut Vec<&'a Expr<'a>>,
non_vars: &mut Vec<&'a Expr>,
) -> Result<()> {
traverse(expr, &mut |e| match e {
Var(v) if scope.locals.contains(v.text()) => {
vars.push(v.text());
Var(v) if scope.locals.contains(*v.text()) => {
vars.push(*v.text());
Ok(false)
}
// TODO: Object key/value
@@ -865,9 +865,9 @@ impl<'a> Analyzer<'a> {
fn analyze_query(
&mut self,
key: Option<&'a Expr<'a>>,
value: Option<&'a Expr<'a>>,
query: &'a Query<'a>,
key: Option<&'a Expr>,
value: Option<&'a Expr>,
query: &'a Query,
mut scope: Scope<'a>,
) -> Result<()> {
self.gather_local_vars(key, value, query, &mut scope)?;
@@ -963,14 +963,14 @@ impl<'a> Analyzer<'a> {
&mut first_use,
&mut definitions,
)?;
let var = if return_arg.text() != "_" {
let var = if *return_arg.text() != "_" {
// The var in the return argument slot would have been processed as
// an used var. Remove it from used vars and add it as the variable being
// defined.
used_vars.pop();
return_arg.text()
} else {
""
std::rc::Rc::new("")
};
self.process_comprs(
&comprs[..],
@@ -978,7 +978,10 @@ impl<'a> Analyzer<'a> {
&mut first_use,
&mut used_vars,
)?;
definitions.push(Definition { var, used_vars });
definitions.push(Definition {
var: *var,
used_vars,
});
} else {
self.process_expr(expr, &mut scope, &mut first_use, &mut definitions)?;
}
@@ -1009,9 +1012,9 @@ impl<'a> Analyzer<'a> {
self.scopes.push(scope.clone());
let mut e_scope = Scope::default();
if let Some(key) = key {
e_scope.locals.insert(key.text());
e_scope.locals.insert(*key.text());
}
e_scope.locals.insert(value.text());
e_scope.locals.insert(*value.text());
self.scopes.push(e_scope);
// TODO: mark first use of key, value so that they cannot be := assigned

14
src/utils.rs
View File

@@ -9,24 +9,24 @@ use std::collections::BTreeMap;
use anyhow::{bail, Result};
pub fn get_path_string(refr: &Expr, document: Option<&str>) -> Result<String> {
let mut comps = vec![];
let mut comps: Vec<&str> = vec![];
let mut expr = Some(refr);
while expr.is_some() {
match expr {
Some(Expr::RefDot { refr, field, .. }) => {
comps.push(field.text());
comps.push(&field.text());
expr = Some(refr);
}
Some(Expr::RefBrack { refr, index, .. })
if matches!(index.as_ref(), Expr::String(_)) =>
{
if let Expr::String(s) = index.as_ref() {
comps.push(s.text());
comps.push(&s.text());
expr = Some(refr);
}
}
Some(Expr::Var(v)) => {
comps.push(v.text());
comps.push(&v.text());
expr = None;
}
_ => bail!("internal error: not a simple ref"),
@@ -39,9 +39,9 @@ pub fn get_path_string(refr: &Expr, document: Option<&str>) -> Result<String> {
Ok(comps.join("."))
}
pub type FunctionTable<'a> = BTreeMap<String, (Vec<&'a Rule<'a>>, u8)>;
pub type FunctionTable<'a> = BTreeMap<String, (Vec<&'a Rule>, u8)>;
pub fn get_extra_arg<'a>(expr: &'a Expr, functions: &FunctionTable) -> Option<&'a Expr<'a>> {
pub fn get_extra_arg<'a>(expr: &'a Expr, functions: &FunctionTable) -> Option<&'a Expr> {
if let Expr::Call { fcn, params, .. } = expr {
if let Ok(path) = get_path_string(fcn, None) {
let n_args = if let Some((_, n_args)) = functions.get(&path) {
@@ -62,7 +62,7 @@ pub fn get_extra_arg<'a>(expr: &'a Expr, functions: &FunctionTable) -> Option<&'
None
}
pub fn gather_functions<'a>(modules: &[&'a Module<'a>]) -> Result<FunctionTable<'a>> {
pub fn gather_functions<'a>(modules: &[&'a Module]) -> Result<FunctionTable<'a>> {
let mut table = FunctionTable::new();
for module in modules {

18
src/value.rs
View File

@@ -165,6 +165,24 @@ impl Value {
pub fn to_json_str(&self) -> Result<String> {
Ok(serde_json::to_string_pretty(self)?)
}
pub fn from_json_file(path: &String) -> Result<Value> {
match std::fs::read_to_string(path) {
Ok(c) => Self::from_json_str(c.as_str()),
Err(e) => bail!("Failed to read {path}. {e}"),
}
}
pub fn from_yaml_str(yaml: &str) -> Result<Value> {
Ok(serde_yaml::from_str(yaml)?)
}
pub fn from_yaml_file(path: &String) -> Result<Value> {
match std::fs::read_to_string(path) {
Ok(c) => Self::from_yaml_str(c.as_str()),
Err(e) => bail!("Failed to read {path}. {e}"),
}
}
}
impl Value {

28
tests/interpreter/mod.rs
View File

@@ -210,11 +210,7 @@ pub fn eval_file_first_rule(
for (idx, file) in files.iter().enumerate() {
let contents = regos[idx].as_str();
sources.push(Source {
file,
contents,
lines: contents.split('\n').collect(),
});
sources.push(Source::new(file.to_string(), contents.to_string()));
}
for source in &sources {
@@ -226,13 +222,9 @@ pub fn eval_file_first_rule(
modules_ref.push(m);
}
let query_source = regorus::Source {
file: "<query.rego>",
contents: query,
lines: query.split('\n').collect(),
};
let query_source = regorus::Source::new("<query.rego>".to_string(), query.to_string());
let query_span = regorus::Span {
source: &query_source,
source: query_source.clone(),
line: 1,
col: 1,
start: 0,
@@ -303,11 +295,7 @@ pub fn eval_file(
for (idx, file) in files.iter().enumerate() {
let contents = regos[idx].as_str();
sources.push(Source {
file,
contents,
lines: contents.split('\n').collect(),
});
sources.push(Source::new(file.to_string(), contents.to_string()));
}
for source in &sources {
@@ -319,13 +307,9 @@ pub fn eval_file(
modules_ref.push(m);
}
let query_source = regorus::Source {
file: "<query.rego>",
contents: query,
lines: query.split('\n').collect(),
};
let query_source = regorus::Source::new("<query.rego".to_string(), query.to_string());
let query_span = regorus::Span {
source: &query_source,
source: query_source.clone(),
line: 1,
col: 1,
start: 0,

33
tests/lexer/mod.rs
View File

@@ -7,7 +7,7 @@ use serde::{Deserialize, Serialize};
use std::env;
use test_generator::test_resources;
fn get_tokens<'source>(source: &'source Source<'source>) -> Result<Vec<Token<'source>>> {
fn get_tokens(source: &Source) -> Result<Vec<Token>> {
let mut tokens = vec![];
let mut lex = Lexer::new(source);
loop {
@@ -35,7 +35,9 @@ fn check_loc(tok: &Token) -> Result<()> {
false if tok.0 == TokenKind::Eof && source_idx >= source_line.len() => return Ok(()),
// Handle case where a raw string's first char is a newline.
false if tok.0 == TokenKind::RawString && &tok.1.text()[0..1] == "\n" => return Ok(()),
_ => bail!("could not find caret for {tok:#?} {msg}"),
_ => {
bail!("could not find caret for {tok:#?} {msg}");
}
}
match &caret_line[idx..idx + 1] {
"^" => {
@@ -76,12 +78,7 @@ fn yaml_test_impl(file: &str) -> Result<()> {
let test: Test = serde_yaml::from_str(&yaml)?;
for case in &test.cases {
let source = Source {
file: "case.rego",
contents: case.rego.as_str(),
lines: case.rego.as_str().split('\n').collect(),
};
let source = Source::new("case.rego".to_string(), case.rego.clone());
print!("case {} ", &case.note);
match get_tokens(&source) {
@@ -91,7 +88,7 @@ fn yaml_test_impl(file: &str) -> Result<()> {
break;
}
assert_eq!(
tok.1.text(),
*tok.1.text(),
case.tokens[idx],
"{} Expected token `{}` not found",
source.message(tok.1.line, tok.1.col, "mismatch-error", &case.tokens[idx]),
@@ -184,11 +181,7 @@ fn run(path: &str) {
#[test]
fn debug() -> Result<()> {
let rego = "\"This string is 35 characters long.\"\"short string\"";
let source = Source {
file: "case.rego",
contents: rego,
lines: rego.split('\n').collect(),
};
let source = Source::new("case.rego".to_string(), rego.to_string());
let mut lexer = Lexer::new(&source);
let tok = lexer.next_token()?;
@@ -210,11 +203,7 @@ fn debug() -> Result<()> {
#[test]
fn tab() -> Result<()> {
let rego = r#" "This string is 35 characters long."`raw string`p"#;
let source = Source {
file: "case.rego",
contents: rego,
lines: rego.split('\n').collect(),
};
let source = Source::new("case.rego".to_string(), rego.to_string());
let mut lexer = Lexer::new(&source);
@@ -244,11 +233,7 @@ fn tab() -> Result<()> {
#[test]
fn invalid_line() -> Result<()> {
let rego = "";
let source = Source {
file: "case.rego",
contents: rego,
lines: rego.split('\n').collect(),
};
let source = Source::new("case.rego".to_string(), rego.to_string());
assert_eq!(
source.message(2, 0, "", ""),

10
tests/parser/mod.rs
View File

@@ -28,7 +28,7 @@ fn match_span(s: &Span, v: &Value) -> Result<()> {
match &v {
Value::String(vs) => {
my_assert_eq!(
s.text(),
*s.text(),
vs,
"{}",
s.source
@@ -37,7 +37,7 @@ fn match_span(s: &Span, v: &Value) -> Result<()> {
}
_ => {
my_assert_eq!(
s.text(),
*s.text(),
serde_json::to_string_pretty(v)?,
"{}",
s.source
@@ -629,11 +629,7 @@ fn yaml_test_impl(file: &str) -> Result<()> {
for case in &test.cases {
print!("\ncase {} ", case.note);
let source = Source {
file: "case.rego",
contents: case.rego.as_str(),
lines: case.rego.split('\n').collect(),
};
let source = Source::new("case.rego".to_string(), case.rego.clone());
let mut parser = Parser::new(&source)?;
match parser.parse() {
Ok(module) => {

12
tests/scheduler/analyzer/mod.rs
View File

@@ -32,20 +32,10 @@ fn to_string_set(s: &BTreeSet<&str>) -> BTreeSet<String> {
}
fn analyze_file(regos: &[String], expected_scopes: &[Scope]) -> Result<()> {
let mut files = vec![];
let mut sources = vec![];
let mut modules = vec![];
for (idx, _) in regos.iter().enumerate() {
files.push(format!("rego_{idx}"));
}
for (idx, file) in files.iter().enumerate() {
let contents = regos[idx].as_str();
sources.push(Source {
file,
contents,
lines: contents.split('\n').collect(),
});
sources.push(Source::new(format!("rego_{idx}"), regos[idx].clone()));
}
for source in &sources {