json.filter, object.filter, object.get, object.keys, object.remove

Signed-off-by: Anand Krishnamoorthi <anakrish@microsoft.com>

src/builtins/mod.rs

@@ -8,6 +8,7 @@ pub mod comparison;
 mod conversions;
 mod debugging;
 pub mod numbers;
+mod objects;
 pub mod sets;
 mod strings;
 mod tracing;
@@ -35,7 +36,7 @@ lazy_static! {
 	aggregates::register(&mut m);
 	arrays::register(&mut m);
 	sets::register(&mut m);
-	//objects::register(&mut m);
+	objects::register(&mut m);
 	strings::register(&mut m);
 	//regex::register(&mut m);
 	//glob::register(&mut m);
@@ -51,7 +52,6 @@ lazy_static! {
 	//graphs::register(&mut m);
 	//graphql::register(&mut m);
 	//http::register(&mut m);
-	//cryptography::register(&mut m);
 	//net::register(&mut m);
 	//uuid::register(&mut m);
 	//semantic_versions::register(&mut m);

src/builtins/objects.rs

@@ -0,0 +1,204 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+use crate::ast::Expr;
+use crate::builtins;
+use crate::builtins::utils::{ensure_args_count, ensure_object};
+use crate::lexer::Span;
+use crate::value::Value;
+
+use std::collections::{BTreeMap, BTreeSet, HashMap};
+use std::iter::Iterator;
+use std::rc::Rc;
+
+use anyhow::{bail, Result};
+
+pub fn register(m: &mut HashMap<&'static str, builtins::BuiltinFcn>) {
+    m.insert("json.filter", json_filter);
+    //    m.insert("json.patch", json_patch);
+    m.insert("object.filter", filter);
+    m.insert("object.get", get);
+    m.insert("object.keys", keys);
+    m.insert("object.remove", remove);
+}
+
+fn json_filter_impl(v: &Value, filter: &Value) -> Value {
+    let filters = match filter {
+        Value::Object(fields) if !fields.is_empty() => fields,
+        _ => return v.clone(),
+    };
+
+    match v {
+        Value::Array(_) => {
+            let mut items = vec![];
+            for (idx, filter) in filters.iter() {
+                // The string index must be parseable as a number.
+                // TODO: support integer indexes?
+                if let Value::String(idx) = idx {
+                    if let Ok(idx) = Value::from_json_str(idx) {
+                        let item = json_filter_impl(&v[&idx], filter);
+                        if item != Value::Undefined {
+                            items.push(item);
+                        }
+                    }
+                }
+            }
+            Value::from_array(items)
+        }
+
+        Value::Set(s) => {
+            let mut items = BTreeSet::new();
+            for (item, filter) in filters.iter() {
+                if s.contains(item) {
+                    let item = json_filter_impl(item, filter);
+                    if item != Value::Undefined {
+                        items.insert(item);
+                    }
+                }
+            }
+            Value::from_set(items)
+        }
+
+        Value::Object(_) => {
+            let mut items = BTreeMap::new();
+            for (key, filter) in filters.iter() {
+                let item = json_filter_impl(&v[key], filter);
+                if item != Value::Undefined {
+                    items.insert(key.clone(), item);
+                }
+            }
+
+            Value::from_map(items)
+        }
+
+        _ => Value::Undefined,
+    }
+}
+
+fn merge_filters<'a>(
+    name: &str,
+    param: &Expr,
+    itr: &mut dyn Iterator<Item = &'a Value>,
+    mut filters: Value,
+) -> Result<Value> {
+    loop {
+        match itr.next() {
+            Some(Value::String(s)) => {
+                let mut fc = filters;
+                let mut f = &mut fc;
+                for p in s.split('/') {
+                    let vref = f.make_or_get_value_mut(&[p])?;
+                    if *vref == Value::Undefined {
+                        *vref = Value::new_object();
+                    }
+                    f = vref;
+                }
+                filters = fc;
+            }
+            Some(Value::Array(a)) => {
+                let mut fc = filters;
+                let mut f = &mut fc;
+                for p in a.iter() {
+                    let vref = match f {
+                        Value::Object(obj) => {
+                            let obj = Rc::make_mut(obj);
+                            obj.entry(p.clone()).or_insert_with(Value::new_object)
+                        }
+                        _ => break,
+                    };
+                    f = vref;
+                }
+                filters = fc;
+            }
+            Some(_) => {
+                let span = param.span();
+                bail!(span.error(
+		    format!("`{name}` requires path to be '/' separated string or array of path components.").as_str()));
+            }
+            None => break,
+        }
+    }
+    Ok(filters)
+}
+
+fn json_filter(span: &Span, params: &[Expr], args: &[Value]) -> Result<Value> {
+    let name = "json.filter";
+    ensure_args_count(span, name, params, args, 2)?;
+    ensure_object(name, &params[0], args[0].clone())?;
+
+    let filters = match &args[1] {
+        Value::Array(a) => merge_filters(name, &params[1], &mut a.iter(), Value::new_object())?,
+        Value::Set(s) => merge_filters(name, &params[1], &mut s.iter(), Value::new_object())?,
+        _ => bail!(span.error(format!("`{name}` requires set/array argument").as_str())),
+    };
+
+    Ok(json_filter_impl(&args[0], &filters))
+}
+
+fn filter(span: &Span, params: &[Expr], args: &[Value]) -> Result<Value> {
+    let name = "object.filter";
+    ensure_args_count(span, name, params, args, 2)?;
+    let mut obj = ensure_object(name, &params[0], args[0].clone())?;
+    let obj_ref = Rc::make_mut(&mut obj);
+    match &args[1] {
+        Value::Array(a) => {
+            let keys: BTreeSet<&Value> = a.iter().collect();
+            obj_ref.retain(|k, _| keys.contains(k))
+        }
+        Value::Set(s) => obj_ref.retain(|k, _| s.contains(k)),
+        Value::Object(o) => obj_ref.retain(|k, _| o.contains_key(k)),
+        _ => bail!(span.error(format!("`{name}` requires array/object/set argument").as_str())),
+    };
+
+    Ok(Value::Object(obj))
+}
+
+fn get(span: &Span, params: &[Expr], args: &[Value]) -> Result<Value> {
+    let name = "object.get";
+    ensure_args_count(span, name, params, args, 3)?;
+    let obj = ensure_object(name, &params[0], args[0].clone())?;
+    let default = &args[2];
+
+    Ok(match &args[1] {
+        Value::Array(keys) => {
+            let mut v = &args[0];
+            for a in keys.iter() {
+                v = &v[a];
+                if v == &Value::Undefined {
+                    v = default;
+                    break;
+                }
+            }
+            v.clone()
+        }
+        key => match obj.get(key) {
+            Some(v) => v.clone(),
+            _ => default.clone(),
+        },
+    })
+}
+
+fn keys(span: &Span, params: &[Expr], args: &[Value]) -> Result<Value> {
+    let name = "object.keys";
+    ensure_args_count(span, name, params, args, 1)?;
+    let obj = ensure_object(name, &params[0], args[0].clone())?;
+    Ok(Value::from_set(obj.keys().cloned().collect()))
+}
+
+fn remove(span: &Span, params: &[Expr], args: &[Value]) -> Result<Value> {
+    let name = "object.remove";
+    ensure_args_count(span, name, params, args, 2)?;
+    let mut obj = ensure_object(name, &params[0], args[0].clone())?;
+    let obj_ref = Rc::make_mut(&mut obj);
+    match &args[1] {
+        Value::Array(a) => {
+            let keys: BTreeSet<&Value> = a.iter().collect();
+            obj_ref.retain(|k, _| !keys.contains(k))
+        }
+        Value::Set(s) => obj_ref.retain(|k, _| !s.contains(k)),
+        Value::Object(o) => obj_ref.retain(|k, _| !o.contains_key(k)),
+        _ => bail!(span.error(format!("`{name}` requires array/object/set argument").as_str())),
+    };
+
+    Ok(Value::Object(obj))
+}

src/value.rs

@@ -6,7 +6,7 @@ use std::collections::{BTreeMap, BTreeSet};
 use std::ops;
 use std::rc::Rc;
 
-use anyhow::{anyhow, Result};
+use anyhow::{anyhow, bail, Result};
 use ordered_float::OrderedFloat;
 use serde::de::{self, Deserializer};
 use serde::ser::{SerializeMap, Serializer};
@@ -277,6 +277,37 @@ impl Value {
     }
 }
 
+impl Value {
+    pub fn make_or_get_value_mut<'a>(&'a mut self, paths: &[&str]) -> Result<&'a mut Value> {
+        if paths.is_empty() {
+            return Ok(self);
+        }
+
+        let key = Value::String(paths[0].to_owned());
+        if self == &Value::Undefined {
+            *self = Value::new_object();
+        }
+        if let Value::Object(map) = self {
+            if map.get(&key).is_none() {
+                Rc::make_mut(map).insert(key.clone(), Value::Undefined);
+            }
+        }
+
+        match self {
+            Value::Object(map) => match Rc::make_mut(map).get_mut(&key) {
+                Some(v) if paths.len() == 1 => Ok(v),
+                Some(v) => Self::make_or_get_value_mut(v, &paths[1..]),
+                _ => bail!("internal error: unexpected"),
+            },
+            Value::Undefined if paths.len() > 1 => {
+                *self = Value::new_object();
+                Self::make_or_get_value_mut(self, paths)
+            }
+            Value::Undefined => Ok(self),
+            _ => bail!("make: not an selfect {self:?}"),
+        }
+    }
+}
 impl ops::Index<usize> for Value {
     type Output = Value;
 

tests/interpreter/cases/builtins/objects/filter.yaml

@@ -0,0 +1,39 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+cases:
+  - note: all
+    data: {}
+    modules:
+      - |
+        package test
+
+        obj = {
+          "a": 5,
+          "b": 6,
+          "c": 7,
+          {1}: 8,
+          {"a":5}: 9,
+        }
+
+        results = {
+          "o1": object.filter(obj, ["a", {1}]),
+          "o2": object.filter(obj, {"a":1, "c": 0}),
+          "o3": object.filter(obj, {"a", "c"}),         
+        }
+    query: data.test.results
+    want_result:
+      o1:
+        object!:
+          - key: a
+            value: 5
+          - key:
+              set!: [1]
+            value: 8
+      o2:
+        a: 5
+        c: 7
+      o3:
+        a: 5
+        c: 7
+        

tests/interpreter/cases/builtins/objects/get.yaml

@@ -0,0 +1,30 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+cases:
+  - note: all
+    data: {}
+    modules:
+      - |
+        package test
+
+        obj = {
+          "a": 5,
+          "b": {"c":8},
+          "c": 7,
+          {1}: 8,
+          {"a":5}: 9,
+        }
+
+        results = {
+          "v1": object.get(obj, "a", "default"),
+          "v2": object.get(obj, ["b", "c"], "default"),
+          "v3": object.get(obj, ["a", "b"], "default"),         
+          "v4": object.get(obj, {"a":5}, "default"),         
+        }
+    query: data.test.results
+    want_result:
+      v1: 5
+      v2: 8
+      v3: "default"
+      v4: 9

tests/interpreter/cases/builtins/objects/json.filter.yaml

@@ -0,0 +1,60 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+cases:
+  - note: all
+    data: {}
+    modules:
+      - |
+        package test
+
+        obj = {
+          "a" : 5,
+          "b" : {
+            "c" : {
+              "d" : 6
+            },
+            "d" : 7
+          },
+          "c" : 8
+        }
+        s = { "a": [[1, [7,8,9], 2], {"a":1, "b":2, "c":3}]}
+        t = { "a": { 1, 2, {"x"}, {8, 9, 10} }}
+        results = {
+          "o1": json.filter(obj, ["a", "b/c"]),
+          "o2": json.filter(obj, {"c", ["b", "d"]}),
+          "o3": json.filter(s, [["a", 1]]),
+          "o4": json.filter(s, ["a/0/1/2", ["a", "0", "2"], "a/0/1/1"]),
+          "o5": json.filter(t, [["a", 1]]),
+          "o6": json.filter(t, [["a", {"x"}]]),
+          "o7": json.filter(t, [["a", {8, 9, 10}, 9]]),
+        }
+
+        #TODO: more filters
+    query: data.test.results
+    want_result:
+      o1:
+        a: 5
+        b:
+          c:
+            d: 6
+      o2:
+        b:
+          d: 7
+        c: 8
+      o3:
+        a: []
+      o4:
+        a: [[[8, 9], 2]]
+      o5:
+        a:
+          set!: [1]
+      o6:
+        a:
+          set!:
+            - set!: ["x"]
+      o7:
+        a:
+          set!:
+            - set!: [9]
+#TODO: Extensive tests

tests/interpreter/cases/builtins/objects/keys.yaml

@@ -0,0 +1,37 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+cases:
+  - note: all
+    data: {}
+    modules:
+      - |
+        package test
+
+        obj = {
+          "a": 5,
+          "b": 6,
+          "c": 7,
+          {1}: 8,
+          {"a":5}: 9,
+          true: false,
+          5 : 10,
+          null: 5,
+        }
+
+        results = {
+          "v1": object.keys(obj),
+        }
+    query: data.test.results
+    want_result:
+      v1:
+        set!:
+          - "a"
+          - "b"
+          - "c"
+          - set!: [1]
+          - a: 5
+          - true
+          - 5
+          - null
+        

tests/interpreter/cases/builtins/objects/remove.yaml

@@ -0,0 +1,45 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+cases:
+  - note: all
+    data: {}
+    modules:
+      - |
+        package test
+
+        obj = {
+          "a": 5,
+          "b": 6,
+          "c": 7,
+          {1}: 8,
+          {"a":5}: 9,
+          true: false,
+          5 : 10,
+          null: 5,
+        }
+
+        r1 = object.remove(obj, ["a", "b", {1}])
+
+        results = {
+          "v1": r1,
+          "v2": r1 == object.remove(obj, {"a", "b", {1}}),
+          "v3": r1 ==  object.remove(obj, {"a":1, "b":1, {1}:1}),
+        }
+    query: data.test.results
+    want_result:
+      v1:
+        object!:
+          - key: c
+            value: 7
+          - key:
+              a: 5
+            value: 9
+          - key: true
+            value: false
+          - key: 5
+            value: 10
+          - key: null
+            value: 5
+      v2: true
+      v3: true