More builtins and semantic improvements (#66)

- base64 builtins
- base64url builtins
- jsonschema builtins
- json.remove builtin
- handle composite index variables
- use dashu_float since rust_decimal has lesser precision

Signed-off-by: Anand Krishnamoorthi <anakrish@microsoft.com>

src/builtins/encoding.rs

@@ -7,16 +7,40 @@ use crate::builtins::utils::{ensure_args_count, ensure_string};
 use crate::lexer::Span;
 use crate::value::Value;
 
-use std::collections::HashMap;
+use std::collections::{BTreeMap, HashMap};
 
-use anyhow::{Context, Result};
-use data_encoding::BASE64;
+use anyhow::{bail, Context, Result};
 
 pub fn register(m: &mut HashMap<&'static str, builtins::BuiltinFcn>) {
-    m.insert("base64.decode", (base64_decode, 1));
+    #[cfg(feature = "base64")]
+    {
+        m.insert("base64.decode", (base64_decode, 1));
+        m.insert("base64.encode", (base64_encode, 1));
+        m.insert("base64.is_valid", (base64_is_valid, 1));
+    }
+    #[cfg(feature = "base64url")]
+    {
+        m.insert("base64url.decode", (base64url_decode, 1));
+        m.insert("base64url.encode", (base64url_encode, 1));
+        m.insert("base64url.encode_no_pad", (base64url_encode_no_pad, 1));
+    }
+    #[cfg(feature = "hex")]
+    {
+        m.insert("hex.decode", (hex_decode, 1));
+        m.insert("hex.encode", (hex_encode, 1));
+    }
+    #[cfg(feature = "urlquery")]
+    {
+        m.insert("urlquery.decode_object", (urlquery_decode_object, 1));
+    }
     m.insert("json.is_valid", (json_is_valid, 1));
     m.insert("json.marshal", (json_marshal, 1));
-    m.insert("jsonunmarshal", (json_unmarshal, 1));
+    m.insert("json.unmarshal", (json_unmarshal, 1));
+    #[cfg(feature = "jsonschema")]
+    {
+        m.insert("json.match_schema", (json_match_schema, 2));
+        m.insert("json.verify_schema", (json_verify_schema, 1));
+    }
 
     #[cfg(feature = "yaml")]
     {
@@ -26,6 +50,7 @@ pub fn register(m: &mut HashMap<&'static str, builtins::BuiltinFcn>) {
     }
 }
 
+#[cfg(feature = "base64")]
 fn base64_decode(
     span: &Span,
     params: &[Ref<Expr>],
@@ -36,12 +61,183 @@ fn base64_decode(
     ensure_args_count(span, name, params, args, 1)?;
 
     let encoded_str = ensure_string(name, &params[0], &args[0])?;
-    let decoded_bytes = BASE64.decode(encoded_str.as_bytes())?;
+    let decoded_bytes = data_encoding::BASE64.decode(encoded_str.as_bytes())?;
     Ok(Value::String(
         String::from_utf8_lossy(&decoded_bytes).into(),
     ))
 }
 
+#[cfg(feature = "base64")]
+fn base64_encode(
+    span: &Span,
+    params: &[Ref<Expr>],
+    args: &[Value],
+    _strict: bool,
+) -> Result<Value> {
+    let name = "base64.encode";
+    ensure_args_count(span, name, params, args, 1)?;
+
+    let string = ensure_string(name, &params[0], &args[0])?;
+    Ok(Value::String(
+        data_encoding::BASE64.encode(string.as_bytes()).into(),
+    ))
+}
+
+#[cfg(feature = "base64")]
+fn base64_is_valid(
+    span: &Span,
+    params: &[Ref<Expr>],
+    args: &[Value],
+    _strict: bool,
+) -> Result<Value> {
+    let name = "base64.is_valid";
+    ensure_args_count(span, name, params, args, 1)?;
+
+    let encoded_str = ensure_string(name, &params[0], &args[0])?;
+    Ok(Value::Bool(
+        data_encoding::BASE64.decode(encoded_str.as_bytes()).is_ok(),
+    ))
+}
+
+#[cfg(feature = "base64url")]
+fn base64url_decode(
+    span: &Span,
+    params: &[Ref<Expr>],
+    args: &[Value],
+    _strict: bool,
+) -> Result<Value> {
+    let name = "base64url.decode";
+    ensure_args_count(span, name, params, args, 1)?;
+
+    let encoded_str = ensure_string(name, &params[0], &args[0])?;
+    let decoded_bytes = match data_encoding::BASE64URL.decode(encoded_str.as_bytes()) {
+        Ok(b) => b,
+        Err(_) => {
+            #[cfg(feature = "base64url")]
+            {
+                data_encoding::BASE64URL_NOPAD.decode(encoded_str.as_bytes())?
+            }
+            #[cfg(not(feature = "base64url"))]
+            {
+                bail!(params[0].span().error("nor a valid url"));
+            }
+        }
+    };
+
+    Ok(Value::String(
+        String::from_utf8_lossy(&decoded_bytes).into(),
+    ))
+}
+
+#[cfg(feature = "base64url")]
+fn base64url_encode(
+    span: &Span,
+    params: &[Ref<Expr>],
+    args: &[Value],
+    _strict: bool,
+) -> Result<Value> {
+    let name = "base64url.encode";
+    ensure_args_count(span, name, params, args, 1)?;
+
+    let string = ensure_string(name, &params[0], &args[0])?;
+    Ok(Value::String(
+        data_encoding::BASE64URL.encode(string.as_bytes()).into(),
+    ))
+}
+
+#[cfg(feature = "base64url")]
+fn base64url_encode_no_pad(
+    span: &Span,
+    params: &[Ref<Expr>],
+    args: &[Value],
+    _strict: bool,
+) -> Result<Value> {
+    let name = "base64url.encode_no_pad";
+    ensure_args_count(span, name, params, args, 1)?;
+
+    let string = ensure_string(name, &params[0], &args[0])?;
+    Ok(Value::String(
+        data_encoding::BASE64URL_NOPAD
+            .encode(string.as_bytes())
+            .into(),
+    ))
+}
+
+#[cfg(feature = "hex")]
+fn hex_decode(span: &Span, params: &[Ref<Expr>], args: &[Value], _strict: bool) -> Result<Value> {
+    let name = "hex.decode";
+    ensure_args_count(span, name, params, args, 1)?;
+
+    let encoded_str = ensure_string(name, &params[0], &args[0])?;
+    let decoded_bytes = data_encoding::HEXLOWER_PERMISSIVE.decode(encoded_str.as_bytes())?;
+    Ok(Value::String(
+        String::from_utf8_lossy(&decoded_bytes).into(),
+    ))
+}
+
+#[cfg(feature = "hex")]
+fn hex_encode(span: &Span, params: &[Ref<Expr>], args: &[Value], _strict: bool) -> Result<Value> {
+    let name = "hex.encode";
+    ensure_args_count(span, name, params, args, 1)?;
+
+    let string = ensure_string(name, &params[0], &args[0])?;
+    Ok(Value::String(
+        data_encoding::HEXLOWER_PERMISSIVE
+            .encode(string.as_bytes())
+            .into(),
+    ))
+}
+
+#[cfg(feature = "urlquery")]
+fn urlquery_decode_object(
+    span: &Span,
+    params: &[Ref<Expr>],
+    args: &[Value],
+    _strict: bool,
+) -> Result<Value> {
+    let name = "urlquery.encode";
+    ensure_args_count(span, name, params, args, 1)?;
+
+    let string = ensure_string(name, &params[0], &args[0])?;
+    let url_string = "https://non-existent?".to_owned() + string.as_ref();
+    let url = match url::Url::parse(&url_string) {
+        Ok(v) => v,
+        Err(_) => bail!(params[0].span().error("not a valid url query")),
+    };
+
+    let mut map = BTreeMap::new();
+    for (k, v) in url.query_pairs() {
+        let key = Value::String(k.clone().into());
+        let value = Value::String(v.clone().into());
+        if let Ok(a) = map.entry(key).or_insert(Value::new_array()).as_array_mut() {
+            a.push(value)
+        }
+    }
+    Ok(Value::from_map(map))
+}
+/*
+#[cfg(feature = "urlquery")]
+fn urlquery_encode(
+    span: &Span,
+    params: &[Ref<Expr>],
+    args: &[Value],
+    _strict: bool,
+) -> Result<Value> {
+    let name = "urlquery.encode";
+    ensure_args_count(span, name, params, args, 1)?;
+
+    let string = ensure_string(name, &params[0], &args[0])?;
+    let url_string = "https://non-existent?" + string;
+    let url = url::Url::parse(&url_string)
+        .map_err(|_| bail!(params[0].span().error("not a valid url query")))?;
+
+    Ok(Value::from_object(
+        url.query_pairs()
+            .map(|(k, v)| (Value::from(k.clone()), Value::from(v.clone())))
+            .collect(),
+    ))
+}*/
+
 #[cfg(feature = "yaml")]
 fn yaml_is_valid(
     span: &Span,
@@ -114,3 +310,72 @@ fn json_unmarshal(
     let json_str = ensure_string(name, &params[0], &args[0])?;
     Value::from_json_str(&json_str).with_context(|| span.error("could not deserialize json."))
 }
+
+#[cfg(feature = "jsonschema")]
+fn compile_json_schema(param: &Ref<Expr>, arg: &Value) -> Result<jsonschema::JSONSchema> {
+    let schema_str = match arg {
+        Value::String(schema_str) => schema_str.as_ref().to_string(),
+        _ => arg.to_json_str()?,
+    };
+
+    if let Ok(schema) = serde_json::from_str(&schema_str) {
+        match jsonschema::JSONSchema::compile(&schema) {
+            Ok(schema) => return Ok(schema),
+            Err(e) => bail!(e.to_string()),
+        }
+    }
+    bail!(param.span().error("not a valid json schema"))
+}
+
+#[cfg(feature = "jsonschema")]
+fn json_verify_schema(
+    span: &Span,
+    params: &[Ref<Expr>],
+    args: &[Value],
+    strict: bool,
+) -> Result<Value> {
+    let name = "json.verify_schema";
+    ensure_args_count(span, name, params, args, 1)?;
+
+    Ok(Value::from_array(
+        match compile_json_schema(&params[0], &args[0]) {
+            Ok(_) => [Value::Bool(true), Value::Null],
+            Err(e) if strict => bail!(params[0]
+                .span()
+                .error(format!("invalid schema: {e}").as_str())),
+            Err(e) => [Value::Bool(false), Value::String(e.to_string().into())],
+        }
+        .to_vec(),
+    ))
+}
+
+#[cfg(feature = "jsonschema")]
+fn json_match_schema(
+    span: &Span,
+    params: &[Ref<Expr>],
+    args: &[Value],
+    strict: bool,
+) -> Result<Value> {
+    let name = "json.match_schema";
+    ensure_args_count(span, name, params, args, 2)?;
+
+    // The following is expected to succeed.
+    let document: serde_json::Value = serde_json::from_str(&args[0].to_json_str()?)?;
+
+    Ok(Value::from_array(
+        match compile_json_schema(&params[1], &args[1]) {
+            Ok(schema) => match schema.validate(&document) {
+                Ok(_) => [Value::Bool(true), Value::Null],
+                Err(e) => [
+                    Value::Bool(false),
+                    Value::from_array(e.map(|e| Value::String(e.to_string().into())).collect()),
+                ],
+            },
+            Err(e) if strict => bail!(params[1]
+                .span()
+                .error(format!("invalid schema: {e}").as_str())),
+            Err(e) => [Value::Bool(false), Value::String(e.to_string().into())],
+        }
+        .to_vec(),
+    ))
+}

src/builtins/objects.rs

@@ -15,7 +15,7 @@ use anyhow::{bail, Result};
 
 pub fn register(m: &mut HashMap<&'static str, builtins::BuiltinFcn>) {
     m.insert("json.filter", (json_filter, 2));
-    //    m.insert("json.patch", (json_patch));
+    m.insert("json.remove", (json_remove, 2));
     m.insert("object.filter", (filter, 2));
     m.insert("object.get", (get, 3));
     m.insert("object.keys", (keys, 1));
@@ -25,6 +25,9 @@ pub fn register(m: &mut HashMap<&'static str, builtins::BuiltinFcn>) {
 
 fn json_filter_impl(v: &Value, filter: &Value) -> Value {
     let filters = match filter {
+        Value::Object(fields) if fields.len() == 1 && filter[&Value::Null] == Value::Null => {
+            return v.clone()
+        }
         Value::Object(fields) if !fields.is_empty() => fields,
         _ => return v.clone(),
     };
@@ -76,6 +79,69 @@ fn json_filter_impl(v: &Value, filter: &Value) -> Value {
     }
 }
 
+fn json_remove_impl(v: &Value, filter: &Value) -> Value {
+    let filters = match filter {
+        Value::Object(fields) if !fields.is_empty() => fields,
+        _ => return v.clone(),
+    };
+
+    if filter[&Value::Null] == Value::Null {
+        return Value::Undefined;
+    }
+
+    match v {
+        Value::Array(a) => {
+            let mut items = vec![];
+            for (idx, item) in a.iter().enumerate() {
+                let idx = Value::String(format!("{idx}").into());
+                if let Some(f) = filters.get(&idx) {
+                    let v = json_remove_impl(item, f);
+                    if v != Value::Undefined {
+                        items.push(v);
+                    }
+                } else {
+                    // Retain the item.
+                    items.push(item.clone());
+                }
+            }
+            Value::from_array(items)
+        }
+
+        Value::Set(s) => {
+            let mut items = BTreeSet::new();
+            for item in s.iter() {
+                if let Some(f) = filters.get(item) {
+                    let v = json_remove_impl(item, f);
+                    if v != Value::Undefined {
+                        items.insert(v);
+                    }
+                } else {
+                    // Retain the item.
+                    items.insert(item.clone());
+                }
+            }
+            Value::from_set(items)
+        }
+
+        Value::Object(obj) => {
+            let mut items = BTreeMap::new();
+            for (key, value) in obj.iter() {
+                if let Some(f) = filters.get(key) {
+                    let v = json_remove_impl(value, f);
+                    if v != Value::Undefined {
+                        items.insert(key.clone(), v);
+                    }
+                } else {
+                    items.insert(key.clone(), value.clone());
+                }
+            }
+            Value::from_map(items)
+        }
+
+        _ => Value::Undefined,
+    }
+}
+
 fn merge_filters(
     name: &str,
     param: &Expr,
@@ -94,6 +160,9 @@ fn merge_filters(
                     }
                     f = vref;
                 }
+                if let Ok(f) = f.as_object_mut() {
+                    f.insert(Value::Null, Value::Null);
+                };
                 filters = fc;
             }
             Some(Value::Array(a)) => {
@@ -109,6 +178,9 @@ fn merge_filters(
                     };
                     f = vref;
                 }
+                if let Ok(f) = f.as_object_mut() {
+                    f.insert(Value::Null, Value::Null);
+                };
                 filters = fc;
             }
             Some(_) => {
@@ -119,6 +191,7 @@ fn merge_filters(
             None => break,
         }
     }
+
     Ok(filters)
 }
 
@@ -133,9 +206,29 @@ fn json_filter(span: &Span, params: &[Ref<Expr>], args: &[Value], _strict: bool)
         _ => bail!(span.error(format!("`{name}` requires set/array argument").as_str())),
     };
 
+    if let Ok(v) = filters.as_object() {
+        if v.is_empty() {
+            return Ok(Value::new_object());
+        }
+    }
+
     Ok(json_filter_impl(&args[0], &filters))
 }
 
+fn json_remove(span: &Span, params: &[Ref<Expr>], args: &[Value], _strict: bool) -> Result<Value> {
+    let name = "json.remove";
+    ensure_args_count(span, name, params, args, 2)?;
+    ensure_object(name, &params[0], args[0].clone())?;
+
+    let filters = match &args[1] {
+        Value::Array(a) => merge_filters(name, &params[1], &mut a.iter(), Value::new_object())?,
+        Value::Set(s) => merge_filters(name, &params[1], &mut s.iter(), Value::new_object())?,
+        _ => bail!(span.error(format!("`{name}` requires set/array argument").as_str())),
+    };
+
+    Ok(json_remove_impl(&args[0], &filters))
+}
+
 fn filter(span: &Span, params: &[Ref<Expr>], args: &[Value], _strict: bool) -> Result<Value> {
     let name = "object.filter";
     ensure_args_count(span, name, params, args, 2)?;