From cba0af0d3ffa09aafe9e9ceecd83b8993e8d1eb5 Mon Sep 17 00:00:00 2001 From: Anand Krishnamoorthi Date: Tue, 7 Mar 2023 07:26:49 -0800 Subject: [PATCH] json.filter, object.filter, object.get, object.keys, object.remove Signed-off-by: Anand Krishnamoorthi --- src/builtins/mod.rs | 4 +- src/builtins/objects.rs | 204 ++++++++++++++++++ src/value.rs | 33 ++- .../cases/builtins/objects/filter.yaml | 39 ++++ .../cases/builtins/objects/get.yaml | 30 +++ .../cases/builtins/objects/json.filter.yaml | 60 ++++++ .../cases/builtins/objects/keys.yaml | 37 ++++ .../cases/builtins/objects/remove.yaml | 45 ++++ 8 files changed, 449 insertions(+), 3 deletions(-) create mode 100644 src/builtins/objects.rs create mode 100644 tests/interpreter/cases/builtins/objects/filter.yaml create mode 100644 tests/interpreter/cases/builtins/objects/get.yaml create mode 100644 tests/interpreter/cases/builtins/objects/json.filter.yaml create mode 100644 tests/interpreter/cases/builtins/objects/keys.yaml create mode 100644 tests/interpreter/cases/builtins/objects/remove.yaml diff --git a/src/builtins/mod.rs b/src/builtins/mod.rs index 0b3a2db..e328970 100644 --- a/src/builtins/mod.rs +++ b/src/builtins/mod.rs @@ -8,6 +8,7 @@ pub mod comparison; mod conversions; mod debugging; pub mod numbers; +mod objects; pub mod sets; mod strings; mod tracing; @@ -35,7 +36,7 @@ lazy_static! { aggregates::register(&mut m); arrays::register(&mut m); sets::register(&mut m); - //objects::register(&mut m); + objects::register(&mut m); strings::register(&mut m); //regex::register(&mut m); //glob::register(&mut m); @@ -51,7 +52,6 @@ lazy_static! { //graphs::register(&mut m); //graphql::register(&mut m); //http::register(&mut m); - //cryptography::register(&mut m); //net::register(&mut m); //uuid::register(&mut m); //semantic_versions::register(&mut m); diff --git a/src/builtins/objects.rs b/src/builtins/objects.rs new file mode 100644 index 0000000..099e6f7 --- /dev/null +++ b/src/builtins/objects.rs @@ -0,0 +1,204 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use crate::ast::Expr; +use crate::builtins; +use crate::builtins::utils::{ensure_args_count, ensure_object}; +use crate::lexer::Span; +use crate::value::Value; + +use std::collections::{BTreeMap, BTreeSet, HashMap}; +use std::iter::Iterator; +use std::rc::Rc; + +use anyhow::{bail, Result}; + +pub fn register(m: &mut HashMap<&'static str, builtins::BuiltinFcn>) { + m.insert("json.filter", json_filter); + // m.insert("json.patch", json_patch); + m.insert("object.filter", filter); + m.insert("object.get", get); + m.insert("object.keys", keys); + m.insert("object.remove", remove); +} + +fn json_filter_impl(v: &Value, filter: &Value) -> Value { + let filters = match filter { + Value::Object(fields) if !fields.is_empty() => fields, + _ => return v.clone(), + }; + + match v { + Value::Array(_) => { + let mut items = vec![]; + for (idx, filter) in filters.iter() { + // The string index must be parseable as a number. + // TODO: support integer indexes? + if let Value::String(idx) = idx { + if let Ok(idx) = Value::from_json_str(idx) { + let item = json_filter_impl(&v[&idx], filter); + if item != Value::Undefined { + items.push(item); + } + } + } + } + Value::from_array(items) + } + + Value::Set(s) => { + let mut items = BTreeSet::new(); + for (item, filter) in filters.iter() { + if s.contains(item) { + let item = json_filter_impl(item, filter); + if item != Value::Undefined { + items.insert(item); + } + } + } + Value::from_set(items) + } + + Value::Object(_) => { + let mut items = BTreeMap::new(); + for (key, filter) in filters.iter() { + let item = json_filter_impl(&v[key], filter); + if item != Value::Undefined { + items.insert(key.clone(), item); + } + } + + Value::from_map(items) + } + + _ => Value::Undefined, + } +} + +fn merge_filters<'a>( + name: &str, + param: &Expr, + itr: &mut dyn Iterator, + mut filters: Value, +) -> Result { + loop { + match itr.next() { + Some(Value::String(s)) => { + let mut fc = filters; + let mut f = &mut fc; + for p in s.split('/') { + let vref = f.make_or_get_value_mut(&[p])?; + if *vref == Value::Undefined { + *vref = Value::new_object(); + } + f = vref; + } + filters = fc; + } + Some(Value::Array(a)) => { + let mut fc = filters; + let mut f = &mut fc; + for p in a.iter() { + let vref = match f { + Value::Object(obj) => { + let obj = Rc::make_mut(obj); + obj.entry(p.clone()).or_insert_with(Value::new_object) + } + _ => break, + }; + f = vref; + } + filters = fc; + } + Some(_) => { + let span = param.span(); + bail!(span.error( + format!("`{name}` requires path to be '/' separated string or array of path components.").as_str())); + } + None => break, + } + } + Ok(filters) +} + +fn json_filter(span: &Span, params: &[Expr], args: &[Value]) -> Result { + let name = "json.filter"; + ensure_args_count(span, name, params, args, 2)?; + ensure_object(name, ¶ms[0], args[0].clone())?; + + let filters = match &args[1] { + Value::Array(a) => merge_filters(name, ¶ms[1], &mut a.iter(), Value::new_object())?, + Value::Set(s) => merge_filters(name, ¶ms[1], &mut s.iter(), Value::new_object())?, + _ => bail!(span.error(format!("`{name}` requires set/array argument").as_str())), + }; + + Ok(json_filter_impl(&args[0], &filters)) +} + +fn filter(span: &Span, params: &[Expr], args: &[Value]) -> Result { + let name = "object.filter"; + ensure_args_count(span, name, params, args, 2)?; + let mut obj = ensure_object(name, ¶ms[0], args[0].clone())?; + let obj_ref = Rc::make_mut(&mut obj); + match &args[1] { + Value::Array(a) => { + let keys: BTreeSet<&Value> = a.iter().collect(); + obj_ref.retain(|k, _| keys.contains(k)) + } + Value::Set(s) => obj_ref.retain(|k, _| s.contains(k)), + Value::Object(o) => obj_ref.retain(|k, _| o.contains_key(k)), + _ => bail!(span.error(format!("`{name}` requires array/object/set argument").as_str())), + }; + + Ok(Value::Object(obj)) +} + +fn get(span: &Span, params: &[Expr], args: &[Value]) -> Result { + let name = "object.get"; + ensure_args_count(span, name, params, args, 3)?; + let obj = ensure_object(name, ¶ms[0], args[0].clone())?; + let default = &args[2]; + + Ok(match &args[1] { + Value::Array(keys) => { + let mut v = &args[0]; + for a in keys.iter() { + v = &v[a]; + if v == &Value::Undefined { + v = default; + break; + } + } + v.clone() + } + key => match obj.get(key) { + Some(v) => v.clone(), + _ => default.clone(), + }, + }) +} + +fn keys(span: &Span, params: &[Expr], args: &[Value]) -> Result { + let name = "object.keys"; + ensure_args_count(span, name, params, args, 1)?; + let obj = ensure_object(name, ¶ms[0], args[0].clone())?; + Ok(Value::from_set(obj.keys().cloned().collect())) +} + +fn remove(span: &Span, params: &[Expr], args: &[Value]) -> Result { + let name = "object.remove"; + ensure_args_count(span, name, params, args, 2)?; + let mut obj = ensure_object(name, ¶ms[0], args[0].clone())?; + let obj_ref = Rc::make_mut(&mut obj); + match &args[1] { + Value::Array(a) => { + let keys: BTreeSet<&Value> = a.iter().collect(); + obj_ref.retain(|k, _| !keys.contains(k)) + } + Value::Set(s) => obj_ref.retain(|k, _| !s.contains(k)), + Value::Object(o) => obj_ref.retain(|k, _| !o.contains_key(k)), + _ => bail!(span.error(format!("`{name}` requires array/object/set argument").as_str())), + }; + + Ok(Value::Object(obj)) +} diff --git a/src/value.rs b/src/value.rs index 7f1c847..3b2252d 100644 --- a/src/value.rs +++ b/src/value.rs @@ -6,7 +6,7 @@ use std::collections::{BTreeMap, BTreeSet}; use std::ops; use std::rc::Rc; -use anyhow::{anyhow, Result}; +use anyhow::{anyhow, bail, Result}; use ordered_float::OrderedFloat; use serde::de::{self, Deserializer}; use serde::ser::{SerializeMap, Serializer}; @@ -277,6 +277,37 @@ impl Value { } } +impl Value { + pub fn make_or_get_value_mut<'a>(&'a mut self, paths: &[&str]) -> Result<&'a mut Value> { + if paths.is_empty() { + return Ok(self); + } + + let key = Value::String(paths[0].to_owned()); + if self == &Value::Undefined { + *self = Value::new_object(); + } + if let Value::Object(map) = self { + if map.get(&key).is_none() { + Rc::make_mut(map).insert(key.clone(), Value::Undefined); + } + } + + match self { + Value::Object(map) => match Rc::make_mut(map).get_mut(&key) { + Some(v) if paths.len() == 1 => Ok(v), + Some(v) => Self::make_or_get_value_mut(v, &paths[1..]), + _ => bail!("internal error: unexpected"), + }, + Value::Undefined if paths.len() > 1 => { + *self = Value::new_object(); + Self::make_or_get_value_mut(self, paths) + } + Value::Undefined => Ok(self), + _ => bail!("make: not an selfect {self:?}"), + } + } +} impl ops::Index for Value { type Output = Value; diff --git a/tests/interpreter/cases/builtins/objects/filter.yaml b/tests/interpreter/cases/builtins/objects/filter.yaml new file mode 100644 index 0000000..5d05a6c --- /dev/null +++ b/tests/interpreter/cases/builtins/objects/filter.yaml @@ -0,0 +1,39 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +cases: + - note: all + data: {} + modules: + - | + package test + + obj = { + "a": 5, + "b": 6, + "c": 7, + {1}: 8, + {"a":5}: 9, + } + + results = { + "o1": object.filter(obj, ["a", {1}]), + "o2": object.filter(obj, {"a":1, "c": 0}), + "o3": object.filter(obj, {"a", "c"}), + } + query: data.test.results + want_result: + o1: + object!: + - key: a + value: 5 + - key: + set!: [1] + value: 8 + o2: + a: 5 + c: 7 + o3: + a: 5 + c: 7 + diff --git a/tests/interpreter/cases/builtins/objects/get.yaml b/tests/interpreter/cases/builtins/objects/get.yaml new file mode 100644 index 0000000..c6d673e --- /dev/null +++ b/tests/interpreter/cases/builtins/objects/get.yaml @@ -0,0 +1,30 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +cases: + - note: all + data: {} + modules: + - | + package test + + obj = { + "a": 5, + "b": {"c":8}, + "c": 7, + {1}: 8, + {"a":5}: 9, + } + + results = { + "v1": object.get(obj, "a", "default"), + "v2": object.get(obj, ["b", "c"], "default"), + "v3": object.get(obj, ["a", "b"], "default"), + "v4": object.get(obj, {"a":5}, "default"), + } + query: data.test.results + want_result: + v1: 5 + v2: 8 + v3: "default" + v4: 9 diff --git a/tests/interpreter/cases/builtins/objects/json.filter.yaml b/tests/interpreter/cases/builtins/objects/json.filter.yaml new file mode 100644 index 0000000..5f6aaf2 --- /dev/null +++ b/tests/interpreter/cases/builtins/objects/json.filter.yaml @@ -0,0 +1,60 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +cases: + - note: all + data: {} + modules: + - | + package test + + obj = { + "a" : 5, + "b" : { + "c" : { + "d" : 6 + }, + "d" : 7 + }, + "c" : 8 + } + s = { "a": [[1, [7,8,9], 2], {"a":1, "b":2, "c":3}]} + t = { "a": { 1, 2, {"x"}, {8, 9, 10} }} + results = { + "o1": json.filter(obj, ["a", "b/c"]), + "o2": json.filter(obj, {"c", ["b", "d"]}), + "o3": json.filter(s, [["a", 1]]), + "o4": json.filter(s, ["a/0/1/2", ["a", "0", "2"], "a/0/1/1"]), + "o5": json.filter(t, [["a", 1]]), + "o6": json.filter(t, [["a", {"x"}]]), + "o7": json.filter(t, [["a", {8, 9, 10}, 9]]), + } + + #TODO: more filters + query: data.test.results + want_result: + o1: + a: 5 + b: + c: + d: 6 + o2: + b: + d: 7 + c: 8 + o3: + a: [] + o4: + a: [[[8, 9], 2]] + o5: + a: + set!: [1] + o6: + a: + set!: + - set!: ["x"] + o7: + a: + set!: + - set!: [9] +#TODO: Extensive tests diff --git a/tests/interpreter/cases/builtins/objects/keys.yaml b/tests/interpreter/cases/builtins/objects/keys.yaml new file mode 100644 index 0000000..ec95b0a --- /dev/null +++ b/tests/interpreter/cases/builtins/objects/keys.yaml @@ -0,0 +1,37 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +cases: + - note: all + data: {} + modules: + - | + package test + + obj = { + "a": 5, + "b": 6, + "c": 7, + {1}: 8, + {"a":5}: 9, + true: false, + 5 : 10, + null: 5, + } + + results = { + "v1": object.keys(obj), + } + query: data.test.results + want_result: + v1: + set!: + - "a" + - "b" + - "c" + - set!: [1] + - a: 5 + - true + - 5 + - null + diff --git a/tests/interpreter/cases/builtins/objects/remove.yaml b/tests/interpreter/cases/builtins/objects/remove.yaml new file mode 100644 index 0000000..b43ab50 --- /dev/null +++ b/tests/interpreter/cases/builtins/objects/remove.yaml @@ -0,0 +1,45 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +cases: + - note: all + data: {} + modules: + - | + package test + + obj = { + "a": 5, + "b": 6, + "c": 7, + {1}: 8, + {"a":5}: 9, + true: false, + 5 : 10, + null: 5, + } + + r1 = object.remove(obj, ["a", "b", {1}]) + + results = { + "v1": r1, + "v2": r1 == object.remove(obj, {"a", "b", {1}}), + "v3": r1 == object.remove(obj, {"a":1, "b":1, {1}:1}), + } + query: data.test.results + want_result: + v1: + object!: + - key: c + value: 7 + - key: + a: 5 + value: 9 + - key: true + value: false + - key: 5 + value: 10 + - key: null + value: 5 + v2: true + v3: true