More library functions (#51)

- units.parse, units.parse_bytes
- json.is_valid, json.marshal, json.unmarshal
- yaml.is_valid, yaml.marshal, yaml.unmarshal
- object.subset
- set_diff

* Also print number of errors due to each missing function
* Also lock down fully passing OPA suites

Signed-off-by: Anand Krishnamoorthi <anakrish@microsoft.com>

src/builtins/deprecated.rs

@@ -2,7 +2,7 @@
 // Licensed under the MIT License.
 
 use crate::ast::{Expr, Ref};
-use crate::builtins::utils::ensure_args_count;
+use crate::builtins::utils::{ensure_args_count, ensure_set};
 use crate::builtins::BuiltinFcn;
 use crate::lexer::Span;
 use crate::value::Value;
@@ -19,7 +19,7 @@ lazy_static! {
 	
 	m.insert("all", (all, 1));
 	m.insert("any", (any, 1));	
-	
+	m.insert("set_diff", (set_diff, 2));
 	m
     };
 }
@@ -49,3 +49,11 @@ fn any(span: &Span, params: &[Ref<Expr>], args: &[Value]) -> Result<Value> {
         }
     }))
 }
+
+fn set_diff(span: &Span, params: &[Ref<Expr>], args: &[Value]) -> Result<Value> {
+    let name = "set_diff";
+    ensure_args_count(span, name, params, args, 2)?;
+    let s1 = ensure_set(name, &params[0], args[0].clone())?;
+    let s2 = ensure_set(name, &params[1], args[1].clone())?;
+    Ok(Value::from_set(s1.difference(&s2).cloned().collect()))
+}

src/builtins/encoding.rs

@@ -9,11 +9,17 @@ use crate::value::Value;
 
 use std::collections::HashMap;
 
-use anyhow::Result;
+use anyhow::{Context, Result};
 use data_encoding::BASE64;
 
 pub fn register(m: &mut HashMap<&'static str, builtins::BuiltinFcn>) {
     m.insert("base64.decode", (base64_decode, 1));
+    m.insert("json.is_valid", (json_is_valid, 1));
+    m.insert("json.marshal", (json_marshal, 1));
+    m.insert("jsonunmarshal", (json_unmarshal, 1));
+    m.insert("yaml.is_valid", (yaml_is_valid, 1));
+    m.insert("yaml.marshal", (yaml_marshal, 1));
+    m.insert("yaml.unmarshal", (yaml_unmarshal, 1));
 }
 
 fn base64_decode(span: &Span, params: &[Ref<Expr>], args: &[Value]) -> Result<Value> {
@@ -26,3 +32,51 @@ fn base64_decode(span: &Span, params: &[Ref<Expr>], args: &[Value]) -> Result<Va
         String::from_utf8_lossy(&decoded_bytes).to_string(),
     ))
 }
+
+fn yaml_is_valid(span: &Span, params: &[Ref<Expr>], args: &[Value]) -> Result<Value> {
+    let name = "yaml.is_valid";
+    ensure_args_count(span, name, params, args, 1)?;
+
+    let yaml_str = ensure_string(name, &params[0], &args[0])?;
+    Ok(Value::Bool(Value::from_yaml_str(&yaml_str).is_ok()))
+}
+
+fn yaml_marshal(span: &Span, params: &[Ref<Expr>], args: &[Value]) -> Result<Value> {
+    let name = "yaml.marshal";
+    ensure_args_count(span, name, params, args, 1)?;
+    Ok(Value::String(
+        serde_yaml::to_string(&args[0])
+            .with_context(|| span.error("could not serialize to yaml"))?,
+    ))
+}
+
+fn yaml_unmarshal(span: &Span, params: &[Ref<Expr>], args: &[Value]) -> Result<Value> {
+    let name = "yaml.unmarshal";
+    ensure_args_count(span, name, params, args, 1)?;
+    let yaml_str = ensure_string(name, &params[0], &args[0])?;
+    Value::from_yaml_str(&yaml_str).with_context(|| span.error("could not deserialize yaml."))
+}
+
+fn json_is_valid(span: &Span, params: &[Ref<Expr>], args: &[Value]) -> Result<Value> {
+    let name = "json.is_valid";
+    ensure_args_count(span, name, params, args, 1)?;
+
+    let json_str = ensure_string(name, &params[0], &args[0])?;
+    Ok(Value::Bool(Value::from_json_str(&json_str).is_ok()))
+}
+
+fn json_marshal(span: &Span, params: &[Ref<Expr>], args: &[Value]) -> Result<Value> {
+    let name = "json.marshal";
+    ensure_args_count(span, name, params, args, 1)?;
+    Ok(Value::String(
+        serde_json::to_string(&args[0])
+            .with_context(|| span.error("could not serialize to json"))?,
+    ))
+}
+
+fn json_unmarshal(span: &Span, params: &[Ref<Expr>], args: &[Value]) -> Result<Value> {
+    let name = "json.unmarshal";
+    ensure_args_count(span, name, params, args, 1)?;
+    let json_str = ensure_string(name, &params[0], &args[0])?;
+    Value::from_json_str(&json_str).with_context(|| span.error("could not deserialize json."))
+}

src/builtins/mod.rs

@@ -17,6 +17,7 @@ mod strings;
 mod time;
 mod tracing;
 pub mod types;
+mod units;
 mod utils;
 
 use crate::ast::{Expr, Ref};
@@ -65,7 +66,7 @@ lazy_static! {
 	//opa::register(&mut m);
 	debugging::register(&mut m);
 	tracing::register(&mut m);
-	
+	units::register(&mut m);
 	m
     };
 }

src/builtins/objects.rs

@@ -20,6 +20,7 @@ pub fn register(m: &mut HashMap<&'static str, builtins::BuiltinFcn>) {
     m.insert("object.get", (get, 3));
     m.insert("object.keys", (keys, 1));
     m.insert("object.remove", (remove, 2));
+    m.insert("object.subset", (subset, 2));
 }
 
 fn json_filter_impl(v: &Value, filter: &Value) -> Value {
@@ -202,3 +203,31 @@ fn remove(span: &Span, params: &[Ref<Expr>], args: &[Value]) -> Result<Value> {
 
     Ok(Value::Object(obj))
 }
+
+fn is_subset(sup: &Value, sub: &Value) -> bool {
+    match (sup, sub) {
+        (Value::Object(sup), Value::Object(sub)) => {
+            sub.iter().all(|(k, vsub)| {
+                match sup.get(k) {
+                    //		    Some(vsup @ Value::Object(_)) => is_subset(vsup, vsub),
+                    Some(vsup) => is_subset(vsup, vsub),
+                    _ => false,
+                }
+            })
+        }
+        (Value::Set(sup), Value::Set(sub)) => sub.is_subset(sup),
+        (Value::Array(sup), Value::Array(sub)) => sup.windows(sub.len()).any(|w| w == &sub[..]),
+        (Value::Array(sup), Value::Set(_)) => {
+            let sup = Value::from_set(sup.iter().cloned().collect());
+            is_subset(&sup, sub)
+        }
+        (sup, sub) => sup == sub,
+    }
+}
+
+fn subset(span: &Span, params: &[Ref<Expr>], args: &[Value]) -> Result<Value> {
+    let name = "object.subset";
+    ensure_args_count(span, name, params, args, 2)?;
+
+    Ok(Value::Bool(is_subset(&args[0], &args[1])))
+}

src/builtins/units.rs

@@ -0,0 +1,162 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+use crate::ast::{Expr, Ref};
+use crate::builtins;
+use crate::builtins::utils::{ensure_args_count, ensure_string};
+use crate::lexer::Span;
+use crate::value::{Float, Number, Value};
+
+use std::collections::HashMap;
+
+use anyhow::{bail, Context, Result};
+use ordered_float::OrderedFloat;
+
+pub fn register(m: &mut HashMap<&'static str, builtins::BuiltinFcn>) {
+    m.insert("units.parse", (parse, 1));
+    m.insert("units.parse_bytes", (parse_bytes, 1));
+}
+
+fn parse(span: &Span, params: &[Ref<Expr>], args: &[Value]) -> Result<Value> {
+    let name = "units.parse";
+    ensure_args_count(span, name, params, args, 1)?;
+    let string = ensure_string(name, &params[0], &args[0])?;
+    let string = string.as_str();
+
+    // Remove quotes.
+    let string = if string.starts_with('"') && string.ends_with('"') && string.len() >= 2 {
+        &string[1..string.len() - 1]
+    } else {
+        string
+    };
+
+    // Disallow whitespace.
+    if string.chars().any(char::is_whitespace) {
+        bail!(span.error("spaces not allowed in resource strings"));
+    }
+
+    let (number_part, suffix) = match string.find(|c: char| c.is_alphabetic()) {
+        Some(p) => (&string[0..p], &string[p..]),
+        _ => (string, ""),
+    };
+
+    let n: Float = if number_part.starts_with('.') {
+        serde_json::from_str(format!("0{number_part}").as_str())
+    } else {
+        serde_json::from_str(number_part)
+    }
+    .with_context(|| span.error("could not parse number"))?;
+
+    Ok(Value::Number(Number(OrderedFloat(
+        n * 10f64.powf(match suffix {
+            "E" | "e" => 18,
+            "P" | "p" => 15,
+            "T" | "t" => 12,
+            "G" | "g" => 9,
+            "M" => 6,
+            "K" | "k" => 3,
+            "m" => -3,
+
+            // The following are not supported by OPA
+            "Q" => 30,
+            "R" => 27,
+            "Y" => 24,
+            "Z" => 21,
+            "h" => 2,
+            "da" => 1,
+            "d" => -1,
+            "c" => -2,
+
+            "μ" => -6,
+            "n" => -9,
+            "f" => -15,
+            "a" => -18,
+            "z" => -21,
+            "y" => -24,
+            "r" => -27,
+            "q" => -30,
+
+            // No suffix specified.
+            "" => 0,
+            _ => {
+                return Ok(Value::Number(Number(OrderedFloat(
+                    n * 2f64.powf(match suffix.to_ascii_lowercase().as_str() {
+                        "ki" => 10,
+                        "mi" => 20,
+                        "gi" => 30,
+                        "ti" => 40,
+                        "pi" => 50,
+                        "ei" => 60,
+                        "zi" => 70,
+                        "yi" => 80,
+                        _ => return Ok(Value::Undefined),
+                    } as f64),
+                ))));
+            }
+        } as f64),
+    ))))
+}
+
+fn parse_bytes(span: &Span, params: &[Ref<Expr>], args: &[Value]) -> Result<Value> {
+    let name = "units.parse_bytes";
+    ensure_args_count(span, name, params, args, 1)?;
+    let string = ensure_string(name, &params[0], &args[0])?;
+    let string = string.as_str();
+
+    // Remove quotes.
+    let string = if string.starts_with('"') && string.ends_with('"') && string.len() >= 2 {
+        &string[1..string.len() - 1]
+    } else {
+        string
+    };
+
+    // Disallow whitespace.
+    if string.chars().any(char::is_whitespace) {
+        bail!(span.error("spaces not allowed in resource strings"));
+    }
+
+    let (number_part, suffix) = match string.find(|c: char| c.is_alphabetic()) {
+        Some(p) => (&string[0..p], &string[p..]),
+        _ => (string, ""),
+    };
+
+    let n: Float = if number_part.starts_with('.') {
+        serde_json::from_str(format!("0{number_part}").as_str())
+    } else {
+        serde_json::from_str(number_part)
+    }
+    .with_context(|| span.error("could not parse number"))?;
+
+    Ok(Value::Number(Number(OrderedFloat(f64::round(
+        n * 2f64.powf(match suffix.to_ascii_lowercase().as_str() {
+            "yi" | "yib" => 80,
+            "zi" | "zib" => 70,
+            "ei" | "eib" => 60,
+            "pi" | "pib" => 50,
+            "ti" | "tib" => 40,
+            "gi" | "gib" => 30,
+            "mi" | "mib" => 20,
+            "ki" | "kib" => 10,
+            "" => 0,
+            _ => {
+                return Ok(Value::Number(Number(OrderedFloat(
+                    n * 10f64.powf(match suffix.to_ascii_lowercase().as_str() {
+                        "q" | "qb" => 30,
+                        "r" | "rb" => 27,
+                        "y" | "yb" => 24,
+                        "z" | "zb" => 21,
+                        "e" | "eb" => 18,
+                        "p" | "pb" => 15,
+                        "t" | "tb" => 12,
+                        "g" | "gb" => 9,
+                        "m" | "mb" => 6,
+                        "k" | "kb" => 3,
+                        _ => {
+                            return Ok(Value::Undefined);
+                        }
+                    } as f64),
+                ))))
+            }
+        } as f64),
+    )))))
+}

src/interpreter.rs

@@ -65,6 +65,7 @@ impl Default for QueryResult {
 
 #[derive(Debug, Clone, Default, Serialize)]
 pub struct QueryResults {
+    #[serde(skip_serializing_if = "Vec::is_empty")]
     pub result: Vec<QueryResult>,
 }
 
@@ -75,6 +76,7 @@ struct Context {
     value: Value,
     result: Option<QueryResult>,
     results: QueryResults,
+    is_compr: bool,
 }
 
 #[derive(Debug)]
@@ -494,7 +496,7 @@ impl Interpreter {
 
         // Omit recording undefined values.
         if value == Value::Undefined {
-            return Ok(Value::Bool(false));
+            return Ok(value); //Ok(Value::Bool(false));
         }
 
         self.add_variable_or(&name)?;
@@ -527,6 +529,7 @@ impl Interpreter {
             value: Value::new_set(),
             result: None,
             results: QueryResults::default(),
+            is_compr: false,
         });
         let mut r = true;
         match domain {
@@ -842,9 +845,14 @@ impl Interpreter {
 
                 if let Some(ctx) = self.contexts.last_mut() {
                     if let Some(result) = &mut ctx.result {
-                        result
-                            .expressions
-                            .push(Self::make_expression_result(span, &value))
+                        if value != Value::Undefined {
+                            result
+                                .expressions
+                                .push(Self::make_expression_result(span, &value))
+                        } else {
+                            result.bindings = Value::new_object();
+                            result.expressions.clear();
+                        }
                     }
                 }
 
@@ -1165,9 +1173,9 @@ impl Interpreter {
                             Value::Set(ref mut s) => {
                                 Rc::make_mut(s).insert(output);
                             }
-                            _ => bail!("internal error: invalid context value"),
+                            a => bail!("internal error: invalid context value {a}"),
                         }
-                    } else {
+                    } else if !ctx.is_compr {
                         match &ctx.value {
                             Value::Set(_) => (),
                             _ => ctx.value = Value::Undefined,
@@ -1191,7 +1199,11 @@ impl Interpreter {
                             .insert(Value::String(name.to_string()), value.clone());
                     }
                 }
-                ctx.results.result.push(result);
+                if result.expressions.iter().all(|v| v != &Value::Undefined)
+                    && !result.expressions.is_empty()
+                {
+                    ctx.results.result.push(result);
+                }
             }
 
             return Ok(true);
@@ -1306,7 +1318,11 @@ impl Interpreter {
                             .insert(Value::String(name.to_string()), value.clone());
                     }
                 }
-                ctx.results.result.push(result);
+                if result.expressions.iter().all(|v| v != &Value::Undefined)
+                    && !result.expressions.is_empty()
+                {
+                    ctx.results.result.push(result);
+                }
             }
         }
 
@@ -1428,6 +1444,7 @@ impl Interpreter {
             value: Value::new_array(),
             result: None,
             results: QueryResults::default(),
+            is_compr: true,
         });
 
         // Evaluate body first.
@@ -1447,6 +1464,7 @@ impl Interpreter {
             value: Value::new_set(),
             result: None,
             results: QueryResults::default(),
+            is_compr: true,
         });
 
         self.eval_query(query)?;
@@ -1470,6 +1488,7 @@ impl Interpreter {
             value: Value::new_object(),
             result: None,
             results: QueryResults::default(),
+            is_compr: true,
         });
 
         self.eval_query(query)?;
@@ -1615,6 +1634,7 @@ impl Interpreter {
                 value: Value::new_set(),
                 result: None,
                 results: QueryResults::default(),
+                is_compr: false,
             };
 
             // Back up local variables of current function and empty
@@ -1734,7 +1754,6 @@ impl Interpreter {
         }
         // Evaluate the associated default rules after non-default rules
         if let Some(rules) = self.default_rules.get(&path) {
-            dbg!(&path);
             for (r, _) in rules.clone() {
                 if !self.processed.contains(&r) {
                     let module = self.get_rule_module(&r)?;
@@ -1826,7 +1845,12 @@ impl Interpreter {
                 )),
             },
             // TODO: Handle string vs rawstring
-            Expr::String(span) => Ok(Value::String(span.text().to_string())),
+            Expr::String(span) => {
+                match serde_json::from_str::<Value>(format!("\"{}\"", span.text()).as_str()) {
+                    Ok(s) => Ok(s),
+                    Err(e) => bail!(span.error(format!("invalid string literal. {e}").as_str())),
+                }
+            }
             Expr::RawString(span) => Ok(Value::String(span.text().to_string())),
 
             // TODO: Handle undefined variables
@@ -1884,6 +1908,7 @@ impl Interpreter {
                         value,
                         result: None,
                         results: QueryResults::default(),
+                        is_compr: false,
                     },
                     path,
                 ))
@@ -1897,6 +1922,7 @@ impl Interpreter {
                         value: Value::new_set(),
                         result: None,
                         results: QueryResults::default(),
+                        is_compr: false,
                     },
                     path,
                 ))
@@ -1938,6 +1964,7 @@ impl Interpreter {
                         value: Value::new_array(),
                         result: None,
                         results: QueryResults::default(),
+                        is_compr: false,
                     });
                 }
                 result = self.eval_query(&body.query);
@@ -2018,10 +2045,10 @@ impl Interpreter {
         }
     }
 
-    pub fn merge_value(span: &Span, value: &mut Value, new: Value) -> Result<()> {
+    pub fn merge_rule_value(span: &Span, value: &mut Value, new: Value) -> Result<()> {
         match value.merge(new) {
             Ok(()) => Ok(()),
-            Err(err) => return Err(span.error(format!("{err}").as_str())),
+            Err(_) => Err(span.error("rules should not produce multiple outputs.")),
         }
     }
 
@@ -2173,15 +2200,15 @@ impl Interpreter {
 
                 if let Value::Object(btree) = &vref {
                     if !btree.contains_key(&index) {
-                        Self::merge_value(span, vref, object)?;
+                        Self::merge_rule_value(span, vref, object)?;
                     }
                 } else if let Value::Undefined = vref {
-                    Self::merge_value(span, vref, object)?;
+                    Self::merge_rule_value(span, vref, object)?;
                 }
             } else {
                 let vref = Self::make_or_get_value_mut(&mut self.data, &paths)?;
                 if let Value::Undefined = &vref {
-                    Self::merge_value(span, vref, value)?;
+                    Self::merge_rule_value(span, vref, value)?;
                 }
             };
 
@@ -2204,7 +2231,7 @@ impl Interpreter {
         // Ensure that path is created.
         let vref = Self::make_or_get_value_mut(&mut self.data, path)?;
         if Self::get_value_chained(self.init_data.clone(), path) == Value::Undefined {
-            Self::merge_value(span, vref, value)
+            Self::merge_rule_value(span, vref, value)
         } else {
             Err(span.error("value for rule has already been specified in data document"))
         }
@@ -2408,6 +2435,7 @@ impl Interpreter {
             // Request that results be gathered.
             result: Some(QueryResult::default()),
             results: QueryResults::default(),
+            is_compr: false,
         });
 
         let prev_module = self.set_current_module(self.modules.last().cloned())?;
@@ -2431,7 +2459,9 @@ impl Interpreter {
                             let orig_idx = ord[expr_idx] as usize;
                             ordered_expressions[orig_idx] = value.clone();
                         }
-                        results.result[idx].expressions = ordered_expressions;
+                        if !ordered_expressions.iter().any(|v| v == &Value::Undefined) {
+                            results.result[idx].expressions = ordered_expressions;
+                        }
                     }
                 }
                 self_schedule.order.remove(k);

src/scheduler.rs

@@ -43,6 +43,7 @@ pub fn schedule<Str: Clone + std::cmp::Ord + std::fmt::Debug>(
     empty: &Str,
 ) -> Result<SortResult> {
     let num_statements = infos.len();
+    let orig_infos: Vec<&StmtInfo<Str>> = infos.iter().collect();
 
     // Mapping from each var to the list of statements that define it.
     let mut defining_stmts: BTreeMap<Str, Vec<usize>> = BTreeMap::new();
@@ -191,7 +192,10 @@ pub fn schedule<Str: Clone + std::cmp::Ord + std::fmt::Debug>(
     }
 
     if order.len() != num_statements {
-        bail!("could not schedule all statements {order:?} {num_statements}");
+        eprintln!("could not schedule all statements {order:?} {orig_infos:?}");
+        return Ok(SortResult::Order(
+            (0..num_statements).map(|i| i as u16).collect(),
+        ));
     }
 
     // TODO: determine cycles.