chore: release (#298)

Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>

.github/workflows/test-c-cpp.yml

@@ -18,6 +18,11 @@ jobs:
       - name: Setup gcc, g++, cmake, ninja
         run: sudo apt update && sudo apt install -y gcc g++ cmake ninja-build
 
+      - name: Workaround to ensure that regorus.h is generated
+        run: |
+          cargo build -r
+        working-directory: ./bindings/ffi
+        
       - name: Test c binding
         run: |
           mkdir bindings/c/build

.github/workflows/test-csharp.yml

@@ -19,8 +19,8 @@ jobs:
 
       - name: Build
         run: dotnet build
-        working-directory: ./bindings/csharp
+        working-directory: ./bindings/csharp/net8.0
 
       - name: Run
         run: LD_LIBRARY_PATH=. dotnet run
-        working-directory: ./bindings/csharp
+        working-directory: ./bindings/csharp/net8.0

.github/workflows/test-csharp40.yml

@@ -0,0 +1,28 @@
+name: bindings/csharp40
+
+on:
+  push:
+    branches: [ "main" ]
+  pull_request:
+    branches: [ "main" ]
+
+jobs:
+  test:
+    runs-on: windows-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - uses: actions/setup-dotnet@v1
+        with:
+            dotnet-version: "5.0.x"
+
+      - name: Build
+        run: dotnet build
+        working-directory: ./bindings/csharp/net40
+
+      - name: Run
+        run: dotnet run
+        working-directory: ./bindings/csharp/net40

.github/workflows/test-ffi.yml

@@ -1,4 +1,4 @@
-name: bindings/c-cpp
+name: bindings/ffi
 
 on:
   push:

.github/workflows/test-java.yml

@@ -32,6 +32,6 @@ jobs:
 
       - name: Test jar
         run: |
-          javac -cp target/regorus-java-0.1.5.jar Test.java
-          java -Djava.library.path=../../target/release -cp target/regorus-java-0.1.5.jar:. Test
+          javac -cp target/regorus-java-0.2.2.jar Test.java
+          java -Djava.library.path=../../target/release -cp target/regorus-java-0.2.2.jar:. Test
         working-directory: ./bindings/java

.gitignore

@@ -1,6 +1,8 @@
 # Generated by Cargo
 # will have compiled files and executables
 /target/
+**/wheels/
+**/__pycache__/
 
 # Remove Cargo.lock from gitignore if creating an executable, leave it for libraries
 # More information here https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html

CHANGELOG.md

@@ -6,6 +6,48 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [0.2.3](https://github.com/microsoft/regorus/compare/regorus-v0.2.2...regorus-v0.2.3) - 2024-08-16
+
+### Fixed
+- Match OPA behavior for split ([#295](https://github.com/microsoft/regorus/pull/295))
+- Merge data to init document ([#293](https://github.com/microsoft/regorus/pull/293))
+
+### Other
+- Update cbindgen requirement from 0.26.0 to 0.27.0 ([#296](https://github.com/microsoft/regorus/pull/296))
+- Bump rexml in /bindings/ruby in the bundler group across 1 directory ([#294](https://github.com/microsoft/regorus/pull/294))
+- Update csbindgen requirement from =1.9.0 to =1.9.3 ([#292](https://github.com/microsoft/regorus/pull/292))
+
+## [0.2.2](https://github.com/microsoft/regorus/compare/regorus-v0.2.1...regorus-v0.2.2) - 2024-07-28
+
+### Added
+- Update to opa v0.67.0 ([#286](https://github.com/microsoft/regorus/pull/286))
+
+### Fixed
+- Handle aliases in scheduler ([#285](https://github.com/microsoft/regorus/pull/285))
+
+### Other
+- Update readme ([#288](https://github.com/microsoft/regorus/pull/288))
+- Update binding versions ([#287](https://github.com/microsoft/regorus/pull/287))
+- build.rs create hooks dir if not exists ([#283](https://github.com/microsoft/regorus/pull/283))
+- add extension_list example ([#281](https://github.com/microsoft/regorus/pull/281))
+- Fix build break ([#278](https://github.com/microsoft/regorus/pull/278))
+- Update pyo3 requirement from 0.21.0 to 0.22.0 ([#275](https://github.com/microsoft/regorus/pull/275))
+- Update to OPA v0.66.0 ([#274](https://github.com/microsoft/regorus/pull/274))
+
+## [0.2.1](https://github.com/microsoft/regorus/compare/regorus-v0.2.0...regorus-v0.2.1) - 2024-06-19
+
+### Added
+- get_policies: Way to obtain policy files and content ([#267](https://github.com/microsoft/regorus/pull/267))
+
+### Other
+- Fix c,cpp,no-std binding examples ([#272](https://github.com/microsoft/regorus/pull/272))
+- Update binding versions for next release ([#270](https://github.com/microsoft/regorus/pull/270))
+- rename method from 'Clone' to 'clone' in 'Engine' class to match the java naming convention and definiont in the of java.lang.Object. ([#268](https://github.com/microsoft/regorus/pull/268))
+- Suppress clippy unused warning ([#269](https://github.com/microsoft/regorus/pull/269))
+- Provide ability to get JSON representation of policy AST ([#266](https://github.com/microsoft/regorus/pull/266))
+- Update OPA tests to v0.65.0 ([#264](https://github.com/microsoft/regorus/pull/264))
+- Allow lexer to be used for other policy languages ([#262](https://github.com/microsoft/regorus/pull/262))
+
 ## [0.2.0](https://github.com/microsoft/regorus/compare/regorus-v0.1.5...regorus-v0.2.0) - 2024-05-30
 
 ### Other
@@ -340,3 +382,4 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - LICENSE committed
 - CODE_OF_CONDUCT.md committed
 - Initial commit
+

Cargo.toml

@@ -12,7 +12,7 @@ members = [
 [package]
 name = "regorus"
 description = "A fast, lightweight Rego (OPA policy language) interpreter"
-version = "0.2.0"
+version = "0.2.3"
 edition = "2021"
 license-file = "LICENSE"
 repository = "https://github.com/microsoft/regorus"
@@ -27,6 +27,7 @@ doctest = false
 default = ["full-opa", "arc"]
 
 arc = ["scientific/arc"]
+ast = []
 base64 = ["dep:data-encoding"]
 base64url = ["dep:data-encoding"]
 coverage = []

README.md

@@ -10,7 +10,7 @@ Regorus is also
   - *cross-platform* - Written in platform-agnostic Rust.
   - *no_std compatible* - Regorus can be used in `no_std` environments too. Most of the builtins are supported.
   - *current* - We strive to keep Regorus up to date with latest OPA release. Regorus supports `import rego.v1`.
-  - *compliant* - Regorus is mostly compliant with the latest [OPA release v0.64.0](https://github.com/open-policy-agent/opa/releases/tag/v0.64.0). See [OPA Conformance](#opa-conformance) for details. Note that while we behaviorally produce the same results, we don't yet support all the builtins.
+  - *compliant* - Regorus is mostly compliant with the latest [OPA release v0.67.0](https://github.com/open-policy-agent/opa/releases/tag/v0.67.0). See [OPA Conformance](#opa-conformance) for details. Note that while we behaviorally produce the same results, we don't yet support all the builtins.
   - *extensible* - Extend the Rego language by implementing custom stateful builtins in Rust.
     See [add_extension](https://github.com/microsoft/regorus/blob/fc68bf9c8bea36427dae9401a7d1f6ada771f7ab/src/engine.rs#L352).
     Support for extensibility using other languages coming soon.
@@ -99,7 +99,7 @@ $ cargo build -r --example regorus --no-default-features; strip target/release/e
 -rwxr-xr-x  1 anand  staff   1.9M May 11 22:04 target/release/examples/regorus*
 ```
 
-Regorus passes the [OPA v0.64.0 test-suite](https://www.openpolicyagent.org/docs/latest/ir/#test-suite) barring a few
+Regorus passes the [OPA v0.67.0 test-suite](https://www.openpolicyagent.org/docs/latest/ir/#test-suite) barring a few
 builtins. See [OPA Conformance](#opa-conformance) below.
 
 ## Bindings
@@ -276,7 +276,7 @@ Benchmark 1: opa eval -b tests/aci -d tests/aci/data.json -i tests/aci/input.jso
 ```
 ## OPA Conformance
 
-Regorus has been verified to be compliant with [OPA v0.64.0](https://github.com/open-policy-agent/opa/releases/tag/v0.64.0)
+Regorus has been verified to be compliant with [OPA v0.67.0](https://github.com/open-policy-agent/opa/releases/tag/v0.67.0)
 using a [test driver](https://github.com/microsoft/regorus/blob/main/tests/opa.rs) that loads and runs the OPA testsuite using Regorus, and verifies that expected outputs are produced.
 
 The test driver can be invoked by running:

bindings/c-nostd/CMakeLists.txt

@@ -38,4 +38,4 @@ corrosion_import_crate(
 add_executable(regorus_test main.c)
 # Add path to <regorus-source-folder>/bindings/ffi
 target_include_directories(regorus_test PRIVATE "../ffi")
-target_link_libraries(regorus_test regorus-ffi)
+target_link_libraries(regorus_test regorus_ffi)

bindings/c/CMakeLists.txt

@@ -31,4 +31,4 @@ corrosion_import_crate(
 add_executable(regorus_test main.c)
 # Add path to <regorus-source-folder>/bindings/ffi
 target_include_directories(regorus_test PRIVATE "../ffi")
-target_link_libraries(regorus_test regorus-ffi)
+target_link_libraries(regorus_test regorus_ffi)

bindings/cpp/CMakeLists.txt

@@ -31,4 +31,4 @@ corrosion_import_crate(
 add_executable(regorus_test main.cpp)
 # Add path to <regorus-source-folder>/bindings/ffi
 target_include_directories(regorus_test PRIVATE "../ffi")
-target_link_libraries(regorus_test regorus-ffi)
+target_link_libraries(regorus_test regorus_ffi)

bindings/csharp/net40/Program.cs

@@ -0,0 +1,65 @@
+//-----------------------------------------------------------------------
+// <copyright file="Program.cs" company="Microsoft">
+//    Copyright (c)2012 Microsoft. All rights reserved.
+// </copyright>
+// <summary>
+//    Contains code to test the Regorus Policy Engine base class for C#
+// and .NET4.0 bindings. It can be built and tested in Windows only. 
+// </summary>
+//-----------------------------------------------------------------------
+using System;
+using System.Text;
+
+using System.Diagnostics;
+using Microsoft.WindowsAzure.Regorus.IaaS;
+
+namespace regoregorus_test
+{
+    class Program
+    {
+        static void Main(string[] args)
+        {
+            long nanosecPerTick = (1000L * 1000L * 1000L) / Stopwatch.Frequency;
+            var w = new Stopwatch();
+            w.Restart();
+
+            var engine = new RegorusPolicyEngine();
+
+            w.Stop();
+            var newEngineTicks = w.ElapsedTicks;
+
+
+            w.Restart();
+
+            // Load policies and data.
+            engine.AddPolicyFromFile("../../../examples/extension_list/agent_extension_policy.rego");
+            engine.AddDataFromJsonFile("../../../examples/extension_list/agent-extension-data-allow-only.json");
+
+
+            w.Stop();
+            var loadPoliciesTicks = w.ElapsedTicks;
+
+
+            w.Restart();
+
+            // Set input and eval query.
+            engine.SetInputFromJsonFile("../../../examples/extension_list/agent-extension-input.json");
+            var results = engine.EvalQuery("data.agent_extension_policy.extensions_to_download=x");
+            Console.WriteLine("Download query test: \n {0}", results);
+
+            results = engine.EvalQuery("data.agent_extension_policy.extensions_validated");
+        
+            Console.WriteLine("Signing validation test: \n {0}", results);
+
+            engine.Dispose();
+
+            w.Stop();
+            var evalTicks = w.ElapsedTicks;
+
+            Console.WriteLine("Engine creation took {0} msecs", (newEngineTicks * nanosecPerTick) / (1000.0 * 1000.0));
+            Console.WriteLine("Load policies and data took {0} msecs", (loadPoliciesTicks * nanosecPerTick) / (1000.0 * 1000.0));
+            Console.WriteLine("EvalQuery and print results took {0} msecs", (evalTicks * nanosecPerTick) / (1000.0 * 1000.0));
+        }
+    }
+}
+

bindings/csharp/net40/README

@@ -0,0 +1,4 @@
+The Regorus C# binding library can be built via command "dotnet build". We can use the Regorus C# binding library built from this 
+directory to create a Nuget. This Nuget will contain the Regorus C# binding library with definitions that 
+work for .NET framework 4.0 (net40) and above. Note the Nuget can only be created after the binding library has been built.
+RegorusCsharp-Lib-x64.nuspec is built for x64 architecture. 

bindings/csharp/net40/Regorus.cs

@@ -0,0 +1,203 @@
+//-----------------------------------------------------------------------
+// <copyright file="Regorus.cs" company="Microsoft">
+//    Copyright (c)2012 Microsoft. All rights reserved.
+// </copyright>
+// <summary>
+//    Contains code for the Regorus Policy Engine base class for C# and
+// .NET4.0 bindings. Currently this base class is not thread-safe. Make 
+// sure we use it in a signle-threaded environment or add additional 
+// protection when using it. 
+// </summary>
+//-----------------------------------------------------------------------
+
+
+using System;
+using System.Text;
+using System.IO;
+using System.Threading;
+
+namespace Microsoft.WindowsAzure.Regorus.IaaS
+{
+
+	public class RegorusPolicyEngine : ICloneable, IDisposable
+	{
+	unsafe private RegorusFFI.RegorusEngine* E;
+	
+	public RegorusPolicyEngine()
+	{
+		unsafe
+	    {
+		E = RegorusFFI.API.regorus_engine_new();
+	    }
+	}
+
+
+	public void Dispose()
+	{
+		unsafe
+		{
+			if (E != null)
+			{
+				RegorusFFI.API.regorus_engine_drop(E);
+				// to avoid Dispose() being called multiple times by mistake.
+				E = null;
+			}
+
+		}
+
+	}
+
+	public object Clone()
+	{
+	    var clone = (RegorusPolicyEngine)this.MemberwiseClone();
+	    unsafe
+	    {
+		clone.E = RegorusFFI.API.regorus_engine_clone(E);
+	    }
+	    return clone;
+
+	}
+
+	public void AddPolicy(string path, string rego)
+	{
+	    var pathBytes = Encoding.UTF8.GetBytes(path);
+	    var regoBytes = Encoding.UTF8.GetBytes(rego);
+	    
+	    unsafe
+	    {
+		fixed (byte* pathPtr = pathBytes)
+		{
+		    fixed(byte* regoPtr = regoBytes)
+		    {
+			CheckAndDropResult(RegorusFFI.API.regorus_engine_add_policy(E, pathPtr, regoPtr));
+		    }
+		}		
+	    }
+	}
+
+	public void AddPolicyFromFile(string path)
+	{
+	    var pathBytes = Encoding.UTF8.GetBytes(path);
+	    
+	    unsafe
+	    {
+		fixed (byte* pathPtr = pathBytes)
+		{
+		    CheckAndDropResult(RegorusFFI.API.regorus_engine_add_policy_from_file(E, pathPtr));
+		}		
+	    }
+	}
+	
+	public void AddPolicyFromPath(string path)
+	{
+		if (!Directory.Exists(path))
+		{
+			return;
+		}
+
+		string[] regoFiles = Directory.GetFiles(path, "*.rego", SearchOption.AllDirectories);
+		foreach (string file in regoFiles)
+		{
+			AddPolicyFromFile(file);
+		}
+	}
+
+	public void AddDataJson(string data)
+	{
+	    var dataBytes = Encoding.UTF8.GetBytes(data);
+	    
+	    unsafe
+	    {
+		fixed (byte* dataPtr = dataBytes)
+		{
+		    CheckAndDropResult(RegorusFFI.API.regorus_engine_add_data_json(E, dataPtr));
+
+		}		
+	    }
+	}
+	
+	public void AddDataFromJsonFile(string path)
+	{
+	    var pathBytes = Encoding.UTF8.GetBytes(path);
+	    
+	    unsafe
+	    {
+		fixed (byte* pathPtr = pathBytes)
+		{
+		    CheckAndDropResult(RegorusFFI.API.regorus_engine_add_data_from_json_file(E, pathPtr));
+
+		}		
+	    }
+	}
+	
+	public void SetInputJson(string input)
+	{
+	    var inputBytes = Encoding.UTF8.GetBytes(input);
+	    
+	    unsafe
+	    {
+		fixed (byte* inputPtr = inputBytes)
+		{
+		    CheckAndDropResult(RegorusFFI.API.regorus_engine_set_input_json(E, inputPtr));
+
+		}		
+	    }
+	}
+	
+	public void SetInputFromJsonFile(string path)
+	{
+	    var pathBytes = Encoding.UTF8.GetBytes(path);
+	    
+	    unsafe
+	    {
+		fixed (byte* pathPtr = pathBytes)
+		{
+		    CheckAndDropResult(RegorusFFI.API.regorus_engine_set_input_from_json_file(E, pathPtr));
+
+		}		
+	    }
+	}
+	
+	public string EvalQuery(string query)
+	{
+	    var queryBytes = Encoding.UTF8.GetBytes(query);
+
+	    var resultJson = "";
+	    unsafe
+	    {
+		fixed (byte* queryPtr = queryBytes)
+		{
+		    var result = RegorusFFI.API.regorus_engine_eval_query(E, queryPtr);
+		    if (result.status == RegorusFFI.RegorusStatus.RegorusStatusOk) {
+			if (result.output != null) {
+			    resultJson = System.Runtime.InteropServices.Marshal.PtrToStringAnsi((IntPtr)result.output);
+			}
+			RegorusFFI.API.regorus_result_drop(result);
+		    } else {
+			CheckAndDropResult(result);
+		    }
+		    
+		}		
+	    }
+	    if (resultJson != null) {
+		return resultJson;
+	    } else {
+		return "";
+	    }
+	}
+	
+	void CheckAndDropResult(RegorusFFI.RegorusResult result)
+	{
+	    if (result.status != RegorusFFI.RegorusStatus.RegorusStatusOk) {
+		unsafe {
+		    var message = System.Runtime.InteropServices.Marshal.PtrToStringAnsi((IntPtr)result.error_message);
+		    var ex = new Exception(message);
+		    RegorusFFI.API.regorus_result_drop(result);
+		    throw ex;
+		}
+	    }
+	    RegorusFFI.API.regorus_result_drop(result);		    
+	}
+
+    }
+}

bindings/csharp/net40/RegorusCsharp-Lib-x64.nuspec

@@ -0,0 +1,22 @@
+<?xml version="1.0" encoding="utf-8" ?>
+<package>
+  <metadata>
+    <id>RegorusCsharp-Lib-x64</id>
+    <version>0.2.1</version>
+    <title>RegorusCsharp-Lib-x64</title>
+    <authors>yangjie@microsoft.com</authors>
+    <owners>yangjie@microsoft.com</owners>
+    <projectUrl>https://www.microsoft.com</projectUrl>
+    <requireLicenseAcceptance>false</requireLicenseAcceptance>
+    <description>Regorus C# library for x64</description>
+    <releaseNotes>remove Regorus.cs from Nuget</releaseNotes>
+    <copyright>Copyright (C) Microsoft Corp</copyright>
+    <summary></summary>
+  </metadata>
+  <files>
+    <file src="RegorusFFI.g.cs" target="RegorusFFI.g.cs"/>
+    <file src="regorus_ffi.dll" target="lib\regorusc.dll" />
+    <file src="README" target="README" />
+    <file src="..\..\..\LICENSE" target="LICENSE" />
+  </files>
+</package>

bindings/csharp/net40/regorus-test.csproj

@@ -0,0 +1,24 @@
+<Project Sdk="Microsoft.NET.Sdk" InitialTargets="BuildRegorusFFI"> 
+
+  <Target Name="BuildRegorusFFI">
+    <Exec Command="cargo build -r --manifest-path ../../ffi/Cargo.toml" />
+    <Copy SourceFiles="../../ffi/RegorusFFI.g.cs" DestinationFolder="." />
+    <ItemGroup>
+        <RegorusDylib Include="..\..\..\target\release\*regorus_ffi*" />
+    </ItemGroup>    
+    <Copy SourceFiles="@(RegorusDylib)" DestinationFolder="." />
+  </Target>
+
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net40</TargetFramework>
+    <RootNamespace>regorus_test</RootNamespace>
+    <StartupObject>regoregorus_test.Program</StartupObject>
+    <ImplicitUsings>enable</ImplicitUsings> 
+    <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
+  </PropertyGroup>
+
+
+
+</Project>

bindings/csharp/net8.0/Program.cs

@@ -1,4 +1,14 @@
-using System.Diagnostics;
+//-----------------------------------------------------------------------
+// <copyright file="Program.cs" company="Microsoft">
+//    Copyright (c)2012 Microsoft. All rights reserved.
+// </copyright>
+// <summary>
+//    Contains code to test the Regorus class for C#
+// and .NET 8.0 bindings. 
+// </summary>
+//-----------------------------------------------------------------------
+
+using System.Diagnostics;
 
 long nanosecPerTick = (1000L*1000L*1000L) / Stopwatch.Frequency;
 var w = new Stopwatch();
@@ -21,10 +31,10 @@ var newEngineTicks = w.ElapsedTicks;
 w.Restart();
 
 // Load policies and data.
-engine.AddPolicyFromFile("../../tests/aci/framework.rego");
-engine.AddPolicyFromFile("../../tests/aci/api.rego");
-engine.AddPolicyFromFile("../../tests/aci/policy.rego");
-engine.AddDataFromJsonFile("../../tests/aci/data.json");
+engine.AddPolicyFromFile("../../../tests/aci/framework.rego");
+engine.AddPolicyFromFile("../../../tests/aci/api.rego");
+engine.AddPolicyFromFile("../../../tests/aci/policy.rego");
+engine.AddDataFromJsonFile("../../../tests/aci/data.json");
 
 
 w.Stop();
@@ -34,7 +44,7 @@ var loadPoliciesTicks = w.ElapsedTicks;
 w.Restart();
 
 // Set input and eval rule.
-engine.SetInputFromJsonFile("../../tests/aci/input.json");
+engine.SetInputFromJsonFile("../../../tests/aci/input.json");
 var value = engine.EvalQuery("data.framework.mount_overlay");
 var valueDoc = System.Text.Json.JsonDocument.Parse(value);
 

bindings/csharp/net8.0/regorus-test.csproj

@@ -1,10 +1,10 @@
 <Project Sdk="Microsoft.NET.Sdk" InitialTargets="BuildRegorusFFI"> 
 
   <Target Name="BuildRegorusFFI">
-    <Exec Command="cargo build -r --manifest-path ../ffi/Cargo.toml" />
-    <Copy SourceFiles="../ffi/RegorusFFI.g.cs" DestinationFolder="." />
+    <Exec Command="cargo build -r --manifest-path ../../ffi/Cargo.toml" />
+    <Copy SourceFiles="../../ffi/RegorusFFI.g.cs" DestinationFolder="." />
     <ItemGroup>
-        <RegorusDylib Include="..\..\target\release\*regorus_ffi*" />
+        <RegorusDylib Include="..\..\..\target\release\*regorus_ffi*" />
     </ItemGroup>    
     <Copy SourceFiles="@(RegorusDylib)" DestinationFolder="." />
   </Target>

bindings/ffi/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "regorus-ffi"
-version = "0.1.5"
+version = "0.2.2"
 edition = "2021"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
@@ -13,11 +13,12 @@ regorus = { path  = "../..", default-features = false }
 serde_json = "1.0.113"
 
 [features]
-default = ["std", "coverage", "regorus/arc", "regorus/full-opa"]
+default = ["ast", "std", "coverage", "regorus/arc", "regorus/full-opa"]
+ast = ["regorus/ast"]
 std = ["regorus/std"]
 coverage = ["regorus/coverage"]
 custom_allocator = []
 
 [build-dependencies]
-cbindgen = "0.26.0"
-csbindgen = "1.9.0"
+cbindgen = "0.27.0"
+csbindgen = "=1.9.3"

bindings/ffi/src/lib.rs

@@ -189,7 +189,6 @@ pub extern "C" fn regorus_engine_add_data_json(
 /// Get list of loaded Rego packages as JSON.
 ///
 /// See https://docs.rs/regorus/latest/regorus/struct.Engine.html#method.get_packages
-/// * `data`: JSON encoded value to be used as policy data.
 #[no_mangle]
 pub extern "C" fn regorus_engine_get_packages(engine: *mut RegorusEngine) -> RegorusResult {
     to_regorus_string_result(|| -> Result<String> {
@@ -198,6 +197,16 @@ pub extern "C" fn regorus_engine_get_packages(engine: *mut RegorusEngine) -> Reg
     }())
 }
 
+/// Get list of policies as JSON.
+///
+/// See https://docs.rs/regorus/latest/regorus/struct.Engine.html#method.get_policies
+#[no_mangle]
+pub extern "C" fn regorus_engine_get_policies(engine: *mut RegorusEngine) -> RegorusResult {
+    to_regorus_string_result(|| -> Result<String> {
+        to_ref(&engine)?.engine.get_policies_as_json()
+    }())
+}
+
 #[cfg(feature = "std")]
 #[no_mangle]
 pub extern "C" fn regorus_engine_add_data_from_json_file(
@@ -411,6 +420,23 @@ pub extern "C" fn regorus_engine_take_prints(engine: *mut RegorusEngine) -> Rego
     }
 }
 
+/// Get AST of policies.
+///
+/// See https://docs.rs/regorus/latest/regorus/coverage/struct.Engine.html#method.get_ast_as_json
+#[no_mangle]
+#[cfg(feature = "ast")]
+pub extern "C" fn regorus_engine_get_ast_as_json(engine: *mut RegorusEngine) -> RegorusResult {
+    let output = || -> Result<String> { to_ref(&engine)?.engine.get_ast_as_json() }();
+    match output {
+        Ok(out) => RegorusResult {
+            status: RegorusStatus::RegorusStatusOk,
+            output: to_c_str(out),
+            error_message: std::ptr::null_mut(),
+        },
+        Err(e) => to_regorus_result(Err(e)),
+    }
+}
+
 #[cfg(feature = "custom_allocator")]
 extern "C" {
     fn regorus_aligned_alloc(alignment: usize, size: usize) -> *mut u8;

bindings/go/main.go

@@ -82,6 +82,19 @@ func main() {
 		fmt.Fprintf(os.Stderr, "error: %v\n", err)
 		os.Exit(1)
 	}
+	fmt.Printf("%s\n", output)
 
+	// Print packages
+	if output, err = engine1.GetPackages(); err != nil {
+		fmt.Fprintf(os.Stderr, "error: %v\n", err)
+		os.Exit(1)
+	}
+	fmt.Printf("%s\n", output)
+
+	// Print policies
+	if output, err = engine1.GetPolicies(); err != nil {
+		fmt.Fprintf(os.Stderr, "error: %v\n", err)
+		os.Exit(1)
+	}
 	fmt.Printf("%s\n", output)
 }

bindings/go/pkg/regorus/mod.go

@@ -55,6 +55,25 @@ func (e *Engine) AddPolicyFromFile(path string) (string, error) {
 	return C.GoString(result.output), nil
 }
 
+func (e *Engine) GetPackages() (string, error) {
+	result := C.regorus_engine_get_packages(e.e)
+	defer C.regorus_result_drop(result)
+	if result.status != C.RegorusStatusOk {
+		return "", fmt.Errorf("%s", C.GoString(result.error_message))
+	}
+	return C.GoString(result.output), nil
+}
+
+func (e *Engine) GetPolicies() (string, error) {
+	result := C.regorus_engine_get_policies(e.e)
+	defer C.regorus_result_drop(result)
+	if result.status != C.RegorusStatusOk {
+		return "", fmt.Errorf("%s", C.GoString(result.error_message))
+	}
+	return C.GoString(result.output), nil
+}
+
+
 func (e *Engine) AddDataJson(data string) error {
 	data_c := C.CString(data)
 	defer C.free(unsafe.Pointer(data_c))

bindings/java/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "regorus-java"
-version = "0.1.5"
+version = "0.2.2"
 edition = "2021"
 repository = "https://github.com/microsoft/regorus/bindings/java"
 description = "Java bindings for Regorus - a fast, lightweight Rego interpreter written in Rust"
@@ -11,7 +11,9 @@ keywords = ["interpreter", "opa", "policy-as-code", "rego"]
 crate-type = ["cdylib"]
 
 [features]
-default = ["regorus/std", "regorus/full-opa"]
+default = ["ast", "coverage", "regorus/std", "regorus/full-opa"]
+coverage = ["regorus/coverage"]
+ast = ["regorus/ast"]
 
 [dependencies]
 anyhow = "1.0"

bindings/java/Test.java

@@ -31,6 +31,12 @@ public class Test {
 	    System.out.println(coverageJson);
 
 	    System.out.println(engine.getCoverageReportPretty());
+
+	    String packagesJson = engine.getPackages();
+	    System.out.println(packagesJson);
+
+	    String policiesJson = engine.getPolicies();
+	    System.out.println(policiesJson);
         }
     }
 }

bindings/java/pom.xml

@@ -9,7 +9,7 @@
 
   <groupId>com.microsoft.regorus</groupId>
   <artifactId>regorus-java</artifactId>
-  <version>0.1.5</version>
+  <version>0.2.2</version>
 
   <name>Regorus Java</name>
   <description>Java bindings for Regorus - a fast, lightweight Rego interpreter written in Rust</description>

bindings/java/src/lib.rs

@@ -89,6 +89,25 @@ pub extern "system" fn Java_com_microsoft_regorus_Engine_nativeGetPackages(
     }
 }
 
+#[no_mangle]
+pub extern "system" fn Java_com_microsoft_regorus_Engine_nativeGetPolicies(
+    env: JNIEnv,
+    _class: JClass,
+    engine_ptr: jlong,
+) -> jstring {
+    let res = throw_err(env, |env| {
+        let engine = unsafe { &mut *(engine_ptr as *mut Engine) };
+        let policies = engine.get_policies_as_json()?;
+        let policies_json = env.new_string(&policies)?;
+        Ok(policies_json.into_raw())
+    });
+
+    match res {
+        Ok(val) => val,
+        Err(_) => JObject::null().into_raw(),
+    }
+}
+
 #[no_mangle]
 pub extern "system" fn Java_com_microsoft_regorus_Engine_nativeClearData(
     env: JNIEnv,
@@ -205,6 +224,7 @@ pub extern "system" fn Java_com_microsoft_regorus_Engine_nativeEvalRule(
 }
 
 #[no_mangle]
+#[cfg(feature = "coverage")]
 pub extern "system" fn Java_com_microsoft_regorus_Engine_nativeSetEnableCoverage(
     env: JNIEnv,
     _class: JClass,
@@ -219,6 +239,7 @@ pub extern "system" fn Java_com_microsoft_regorus_Engine_nativeSetEnableCoverage
 }
 
 #[no_mangle]
+#[cfg(feature = "coverage")]
 pub extern "system" fn Java_com_microsoft_regorus_Engine_nativeGetCoverageReport(
     env: JNIEnv,
     _class: JClass,
@@ -238,6 +259,7 @@ pub extern "system" fn Java_com_microsoft_regorus_Engine_nativeGetCoverageReport
 }
 
 #[no_mangle]
+#[cfg(feature = "coverage")]
 pub extern "system" fn Java_com_microsoft_regorus_Engine_nativeGetCoverageReportPretty(
     env: JNIEnv,
     _class: JClass,
@@ -257,6 +279,7 @@ pub extern "system" fn Java_com_microsoft_regorus_Engine_nativeGetCoverageReport
 }
 
 #[no_mangle]
+#[cfg(feature = "coverage")]
 pub extern "system" fn Java_com_microsoft_regorus_Engine_nativeClearCoverageData(
     env: JNIEnv,
     _class: JClass,
@@ -302,6 +325,26 @@ pub extern "system" fn Java_com_microsoft_regorus_Engine_nativeTakePrints(
     }
 }
 
+#[no_mangle]
+#[cfg(feature = "ast")]
+pub extern "system" fn Java_com_microsoft_regorus_Engine_getAstAsJson(
+    env: JNIEnv,
+    _class: JClass,
+    engine_ptr: jlong,
+) -> jstring {
+    let res = throw_err(env, |env| {
+        let engine = unsafe { &mut *(engine_ptr as *mut Engine) };
+        let ast = engine.get_ast_as_json()?;
+        let output = env.new_string(&ast)?;
+        Ok(output.into_raw())
+    });
+
+    match res {
+        Ok(val) => val,
+        Err(_) => JObject::null().into_raw(),
+    }
+}
+
 #[no_mangle]
 pub extern "system" fn Java_com_microsoft_regorus_Engine_nativeDestroyEngine(
     _env: JNIEnv,

bindings/java/src/main/java/com/microsoft/regorus/Engine.java

@@ -26,6 +26,7 @@ public class Engine implements AutoCloseable, Cloneable {
     private static native String nativeAddPolicy(long enginePtr, String path, String rego);
     private static native String nativeAddPolicyFromFile(long enginePtr, String path);
     private static native String nativeGetPackages(long enginePtr);
+    private static native String nativeGetPolicies(long enginePtr);
     private static native void nativeClearData(long enginePtr);
     private static native void nativeAddDataJson(long enginePtr, String data);
     private static native void nativeAddDataJsonFromFile(long enginePtr, String path);
@@ -60,7 +61,7 @@ public class Engine implements AutoCloseable, Cloneable {
     /**
      * Efficiently clones an Engine.
      */
-    public Engine Clone() {
+    public Engine clone() {
 	return new Engine(nativeClone(enginePtr));
     }
     
@@ -96,6 +97,15 @@ public class Engine implements AutoCloseable, Cloneable {
         return nativeGetPackages(enginePtr);
     }
     
+    /**
+     * Get list of loaded policies.
+     * 
+     * @return List of Rego policies as a JSON array of sources.
+     */
+    public String getPolicies() {
+        return nativeGetPolicies(enginePtr);
+    }
+    
     /**
      * Clears the data  document.
      */

bindings/python/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "regoruspy"
-version = "0.1.5"
+version = "0.2.2"
 edition = "2021"
 repository = "https://github.com/microsoft/regorus/bindings/python"
 description = "Python bindings for Regorus - a fast, lightweight Rego interpreter written in Rust"
@@ -12,12 +12,14 @@ keywords = ["interpreter", "opa", "policy-as-code", "rego"]
 crate-type = ["cdylib"]
 
 [features]
-default = ["regorus/std", "regorus/full-opa"]
+default = ["ast", "coverage", "regorus/std", "regorus/full-opa"]
+ast = ["regorus/ast"]
+coverage = ["regorus/coverage"]
 
 [dependencies]
 anyhow = "1.0"
 ordered-float = "4.2.0"
-pyo3 = {version = "0.21.0", features = ["anyhow", "extension-module"] }
+pyo3 = {version = "0.22.0", features = ["anyhow", "extension-module"] }
 regorus = { path  = "../..", default-features = false, features = ["arc"] }
 serde_json = "1.0.112"
 

bindings/python/src/lib.rs

@@ -184,10 +184,18 @@ impl Engine {
 
     /// Get the list of packages defined by loaded policies.
     ///
-    pub fn get_packages(&mut self) -> Result<Vec<String>> {
+    pub fn get_packages(&self) -> Result<Vec<String>> {
         self.engine.get_packages()
     }
 
+    /// Get the list of policies.
+    ///
+    pub fn get_policies(&self) -> Result<String> {
+        Ok(serde_json::to_string_pretty(
+            &self.engine.get_policies_as_json()?,
+        )?)
+    }
+
     /// Add policy data.
     ///
     /// * `data`: Rego value. A Rego value is a number, bool, string, None
@@ -312,6 +320,7 @@ impl Engine {
 
     /// Get coverage report as json.
     ///
+    #[cfg(feature = "coverage")]
     pub fn get_coverage_report_as_json(&self) -> Result<String> {
         let report = self.engine.get_coverage_report()?;
         serde_json::to_string_pretty(&report).map_err(|e| anyhow!("{e}"))
@@ -319,12 +328,14 @@ impl Engine {
 
     /// Get coverage report as pretty printable string.
     ///
+    #[cfg(feature = "coverage")]
     pub fn get_coverage_report_pretty(&self) -> Result<String> {
         self.engine.get_coverage_report()?.to_string_pretty()
     }
 
     /// Clear coverage data.
     ///
+    #[cfg(feature = "coverage")]
     pub fn clear_coverage_data(&mut self) {
         self.engine.clear_coverage_data();
     }
@@ -350,6 +361,13 @@ impl Engine {
             engine: self.engine.clone(),
         }
     }
+
+    /// Get AST of policies.
+    ///
+    #[cfg(feature = "ast")]
+    pub fn get_ast_as_json(&self) -> Result<String> {
+        self.engine.get_ast_as_json()
+    }
 }
 
 #[pymodule]

bindings/python/test_extensions.py

@@ -0,0 +1,214 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+import json
+import pytest
+import regorus
+
+TEST_EXT_NAME = "Microsoft.Azure.ActiveDirectory.AADSSHLoginForLinux"
+
+
+@pytest.fixture(name="engine", scope="function")
+def engine_fixture():
+    """
+    Fixture to handle creation and cleanup of a default policy engine.
+    New engine is created for each test case.
+    """
+    engine = regorus.Engine()
+    engine.add_policy_from_file('../../examples/extension_list/agent_extension_policy.rego')
+    yield engine
+
+
+@pytest.fixture(name="input_data")
+def input_data_fixture():
+    """
+    Fixture to handle creation and cleanup of a default input data.
+    New input data is created for each test case.
+    """
+    input_data = {
+        "extensions": {
+            TEST_EXT_NAME: {
+                "signingInfo": {
+                    "extensionSigned": False
+                }
+            }
+        }
+    }
+    input_json = json.dumps(input_data)
+    yield input_json
+
+
+@pytest.fixture(name="default_data")
+def default_data_fixture():
+    """Fixture for default data"""
+    data_json = {
+        "azureGuestAgentPolicy": {
+            "policyVersion": "0.1.0",
+            "signingRules": {
+                "extensionSigned": False
+            },
+            "allowListOnly": False
+        }
+    }
+    data_json = json.dumps(data_json)
+    yield data_json
+
+
+def test_default_data_json(engine, input_data):
+    """Test the default data in json format for extension policy."""
+    data_json = {
+        "azureGuestAgentPolicy": {
+            "policyVersion": "0.1.0",
+            "signingRules": {
+                "extensionSigned": False
+            },
+            "allowListOnly": False
+        }
+    }
+    data_json = json.dumps(data_json)
+    engine.add_data_json(data_json)
+    engine.set_input_json(input_data)
+    # Eval query
+    results = engine.eval_query('data.agent_extension_policy')
+    assert results['result'][0]['expressions'][0]['value']['extensions_to_download'][TEST_EXT_NAME]['downloadAllowed']
+
+
+def test_default_data_file(engine, input_data):
+    """Test the default data in file format for extension policy."""
+    data_default_path = "../../examples/extension_list/agent-extension-default-data.json"
+    engine.add_data_from_json_file(data_default_path)
+    engine.set_input_json(input_data)
+    # Eval query
+    results = engine.eval_query('data.agent_extension_policy')
+    assert results['result'][0]['expressions'][0]['value']['extensions_to_download'][TEST_EXT_NAME]['downloadAllowed']
+
+
+def test_allow_all(engine, input_data):
+    """Test the policy engine with allow all policy."""
+    data_json = {
+        "azureGuestAgentPolicy": {
+            "policyVersion": "0.1.0",
+            "signingRules": {
+                "extensionSigned": False
+            },
+            "allowListOnly": False
+        }
+    }
+    data_json = json.dumps(data_json)
+    engine.add_data_json(data_json)
+    engine.set_input_json(input_data)
+    # Eval query
+    results = engine.eval_query('data.agent_extension_policy')
+    assert results['result'][0]['expressions'][0]['value']['extensions_to_download'][TEST_EXT_NAME]['downloadAllowed']
+
+
+def test_name_only_input(engine, default_data):
+    """Test input with only the extension name."""
+    input_data = {
+        "extensions": {
+            TEST_EXT_NAME: {
+            }
+        }
+    }
+    input_json = json.dumps(input_data)
+    engine.add_data_json(default_data)
+    engine.set_input_json(input_json)
+    # Eval query
+    results = engine.eval_query('data.agent_extension_policy')
+    assert results['result'][0]['expressions'][0]['value']['extensions_to_download'][TEST_EXT_NAME]['downloadAllowed']
+
+
+@pytest.mark.parametrize("input_signed, extension_signed", [
+    (True, True),
+    (True, False),
+    (False, True),
+    (False, False)
+])
+def test_extension_signed_rule(engine, input_signed, extension_signed):
+    """
+    Test extension signing rule. Engine should be able to handle
+    both signed and unsigned extensions, with extensionSigned rule set
+    to either true or false.
+    """
+    data_json = {
+        "azureGuestAgentPolicy": {
+            "policyVersion": "0.1.0",
+            "signingRules": {
+                "extensionSigned": extension_signed
+            },
+            "allowListOnly": False
+        }
+    }
+    input_data = {
+        "extensions": {
+            TEST_EXT_NAME: {
+                "signingInfo": {
+                    "extensionSigned": input_signed
+                }
+            }
+        }
+    }
+    data_json = json.dumps(data_json)
+    input_data = json.dumps(input_data)
+    engine.add_data_json(data_json)
+    engine.set_input_json(input_data)
+    # Eval query
+    results = engine.eval_query('data.agent_extension_policy')
+
+    # assert results
+    if extension_signed:
+        assert results['result'][0]['expressions'][0]['value']['extensions_validated'][TEST_EXT_NAME]['signingValidated'] == input_signed
+    else:
+        assert results['result'][0]['expressions'][0]['value']['extensions_validated'][TEST_EXT_NAME]['signingValidated']
+    assert results['result'][0]['expressions'][0]['value']['extensions_to_download'][TEST_EXT_NAME]['downloadAllowed']
+
+
+@pytest.mark.parametrize("ext_allowed, allow_rule", [
+    (True, True),
+    (True, False),
+    (False, True),
+    (False, False)
+])
+def test_allowlist_rule(engine, ext_allowed, allow_rule):
+    """
+    Test allowListOnly rule. Engine should be able to handle
+    both allowed and disallowed extensions, with allowListOnly rule
+    set to either true or false.
+    """
+    if ext_allowed:
+        ext_name = TEST_EXT_NAME
+    else:
+        ext_name = "random_disallowed_extension"
+
+    input_json = {
+        "extensions": {
+            ext_name: {
+                "signingInfo": {
+                    "extensionSigned": False
+                }
+            }
+        }
+    }
+    data_json = {
+        "azureGuestAgentPolicy": {
+            "signingRules": {
+                "extensionSigned": False
+            },
+            "allowListOnly": allow_rule
+        },
+        "azureGuestExtensionsPolicy": {
+            "Microsoft.CPlat.Core.RunCommandLinux": {
+            },
+            TEST_EXT_NAME: {
+            }
+        }
+    }
+    input_json = json.dumps(input_json)
+    data_json = json.dumps(data_json)
+    engine.add_data_json(data_json)
+    engine.set_input_json(input_json)
+    # Eval query
+    results = engine.eval_query('data.agent_extension_policy')
+    if allow_rule:
+        assert results['result'][0]['expressions'][0]['value']['extensions_to_download'][ext_name]['downloadAllowed'] == ext_allowed
+    else:
+        assert results['result'][0]['expressions'][0]['value']['extensions_to_download'][ext_name]['downloadAllowed']

bindings/ruby/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    regorusrb (0.1.5)
+    regorusrb (0.2.1)
       rb_sys (~> 0.9.97)
 
 GEM
@@ -10,9 +10,9 @@ GEM
     ast (2.4.2)
     json (2.7.2)
     language_server-protocol (3.17.0.3)
-    minitest (5.23.1)
-    parallel (1.24.0)
-    parser (3.3.1.0)
+    minitest (5.24.1)
+    parallel (1.25.1)
+    parser (3.3.4.0)
       ast (~> 2.4.1)
       racc
     racc (1.8.0)
@@ -20,25 +20,25 @@ GEM
     rake (13.2.1)
     rake-compiler (1.2.7)
       rake
-    rake-compiler-dock (1.5.0)
-    rb_sys (0.9.97)
+    rake-compiler-dock (1.5.1)
+    rb_sys (0.9.99)
     regexp_parser (2.9.2)
-    rexml (3.2.8)
-      strscan (>= 3.0.9)
-    rubocop (1.64.0)
+    rexml (3.3.3)
+      strscan
+    rubocop (1.65.0)
       json (~> 2.3)
       language_server-protocol (>= 3.17.0)
       parallel (~> 1.10)
       parser (>= 3.3.0.2)
       rainbow (>= 2.2.2, < 4.0)
-      regexp_parser (>= 1.8, < 3.0)
+      regexp_parser (>= 2.4, < 3.0)
       rexml (>= 3.2.5, < 4.0)
       rubocop-ast (>= 1.31.1, < 2.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (>= 2.4.0, < 3.0)
     rubocop-ast (1.31.3)
       parser (>= 3.3.1.0)
-    rubocop-minitest (0.35.0)
+    rubocop-minitest (0.35.1)
       rubocop (>= 1.61, < 2.0)
       rubocop-ast (>= 1.31.1, < 2.0)
     rubocop-rake (0.6.0)
@@ -62,4 +62,4 @@ DEPENDENCIES
   rubocop-rake
 
 BUNDLED WITH
-   2.5.10
+   2.5.13

bindings/ruby/ext/regorusrb/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "regorusrb"
-version = "0.1.5"
+version = "0.2.2"
 edition = "2021"
 description = "Ruby bindings for Regorus - a fast, lightweight Rego interpreter written in Rust"
 publish = false
@@ -10,10 +10,12 @@ crate-type = ["cdylib"]
 path = "src/lib.rs"
 
 [features]
-default = ["regorus/std", "regorus/full-opa"]
+default = ["ast", "coverage", "regorus/std", "regorus/full-opa"]
+ast = ["regorus/ast"]
+coverage = ["regorus/coverage"]
 
 [dependencies]
 magnus = { version = "0.6.4" }
-regorus = { git = "https://github.com/microsoft/regorus", default-features = false, features = ["arc"] }
+regorus = { path = "../../../..", default-features = false, features = ["arc"] }
 serde_json = "1.0.117"
 serde_magnus = "0.8.1"

bindings/ruby/ext/regorusrb/src/lib.rs

@@ -92,6 +92,20 @@ impl Engine {
         Ok(())
     }
 
+    fn get_packages(&self) -> Result<Vec<String>, Error> {
+        self.engine
+            .borrow()
+            .get_packages()
+            .map_err(|e| Error::new(runtime_error(), format!("Failed to get packages: {e}")))
+    }
+
+    fn get_policies(&self) -> Result<String, Error> {
+        self.engine
+            .borrow()
+            .get_policies_as_json()
+            .map_err(|e| Error::new(runtime_error(), format!("Failed to get policies: {e}")))
+    }
+
     fn set_input(&self, ruby_hash: magnus::RHash) -> Result<(), Error> {
         let input_value: regorus::Value = serde_magnus::deserialize(ruby_hash).map_err(|e| {
             Error::new(
@@ -192,11 +206,13 @@ impl Engine {
         Ok(self.engine.borrow_mut().eval_deny_query(query, false))
     }
 
+    #[cfg(feature = "coverage")]
     fn set_enable_coverage(&self, enable: bool) -> Result<(), Error> {
         self.engine.borrow_mut().set_enable_coverage(enable);
         Ok(())
     }
 
+    #[cfg(feature = "coverage")]
     fn get_coverage_report_as_json(&self) -> Result<String, Error> {
         let report = self
             .engine
@@ -217,6 +233,7 @@ impl Engine {
         })
     }
 
+    #[cfg(feature = "coverage")]
     fn get_coverage_report_pretty(&self) -> Result<String, Error> {
         let report = self
             .engine
@@ -237,6 +254,7 @@ impl Engine {
         })
     }
 
+    #[cfg(feature = "coverage")]
     fn clear_coverage_data(&self) -> Result<(), Error> {
         self.engine.borrow_mut().clear_coverage_data();
         Ok(())
@@ -256,6 +274,14 @@ impl Engine {
             )
         })
     }
+
+    #[cfg(feature = "ast")]
+    fn get_ast_as_json(&self) -> Result<String, Error> {
+        self.engine
+            .borrow()
+            .get_ast_as_json()
+            .map_err(|e| Error::new(runtime_error(), format!("Failed to get ast: {e}")))
+    }
 }
 
 #[magnus::init]
@@ -277,6 +303,8 @@ fn init(ruby: &Ruby) -> Result<(), Error> {
         "add_policy_from_file",
         method!(Engine::add_policy_from_file, 1),
     )?;
+    engine_class.define_method("get_packages", method!(Engine::get_packages, 0))?;
+    engine_class.define_method("get_policies", method!(Engine::get_policies, 0))?;
 
     // data operations
     engine_class.define_method("add_data", method!(Engine::add_data, 1))?;
@@ -325,5 +353,7 @@ fn init(ruby: &Ruby) -> Result<(), Error> {
     engine_class.define_method("set_gather_prints", method!(Engine::set_gather_prints, 1))?;
     engine_class.define_method("take_prints", method!(Engine::take_prints, 0))?;
 
+    // ast
+    engine_class.define_method("get_ast_as_json", method!(Engine::get_ast_as_json, 0))?;
     Ok(())
 }

bindings/ruby/lib/regorus/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Regorus
-  VERSION = "0.1.5"
+  VERSION = "0.2.1"
 end

bindings/wasm/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "regorusjs"
-version = "0.1.5"
+version = "0.2.2"
 edition = "2021"
 repository = "https://github.com/microsoft/regorus/bindings/wasm"
 description = "WASM bindings for Regorus - a fast, lightweight Rego interpreter written in Rust"
@@ -11,7 +11,9 @@ keywords = ["interpreter", "opa", "policy-as-code", "rego"]
 crate-type = ["cdylib"]
 
 [features]
-default = ["regorus/std", "regorus/full-opa"]
+default = ["ast", "coverage", "regorus/std", "regorus/full-opa"]
+ast = ["regorus/ast"]
+coverage = ["regorus/coverage"]
 
 [dependencies]
 regorus = { path  = "../..", default-features = false, features = ["arc"] }

bindings/wasm/src/lib.rs

@@ -72,6 +72,13 @@ impl Engine {
         self.engine.get_packages().map_err(error_to_jsvalue)
     }
 
+    /// Get the list of policies.
+    ///
+    /// See https://docs.rs/regorus/latest/regorus/struct.Engine.html#method.get_policies
+    pub fn getPolicies(&self) -> Result<String, JsValue> {
+        self.engine.get_policies_as_json().map_err(error_to_jsvalue)
+    }
+
     /// Clear policy data.
     ///
     /// See https://docs.rs/regorus/0.1.0-alpha.2/regorus/struct.Engine.html#method.clear_data
@@ -131,6 +138,7 @@ impl Engine {
     ///
     /// See https://docs.rs/regorus/latest/regorus/struct.Engine.html#method.set_enable_coverage
     /// * `b`: Whether to enable gathering coverage or not.
+    #[cfg(feature = "coverage")]
     pub fn setEnableCoverage(&mut self, enable: bool) {
         self.engine.set_enable_coverage(enable)
     }
@@ -138,6 +146,7 @@ impl Engine {
     /// Get the coverage report as json.
     ///
     /// See https://docs.rs/regorus/latest/regorus/struct.Engine.html#method.get_coverage_report
+    #[cfg(feature = "coverage")]
     pub fn getCoverageReport(&self) -> Result<String, JsValue> {
         let report = self
             .engine
@@ -149,6 +158,7 @@ impl Engine {
     /// Clear gathered coverage data.
     ///
     /// See https://docs.rs/regorus/latest/regorus/struct.Engine.html#method.clear_coverage_data
+    #[cfg(feature = "coverage")]
     pub fn clearCoverageData(&mut self) {
         self.engine.clear_coverage_data()
     }
@@ -156,6 +166,7 @@ impl Engine {
     /// Get ANSI color coded coverage report.
     ///
     /// See https://docs.rs/regorus/latest/regorus/coverage/struct.Report.html#method.to_string_pretty
+    #[cfg(feature = "coverage")]
     pub fn getCoverageReportPretty(&self) -> Result<String, JsValue> {
         let report = self
             .engine
@@ -163,10 +174,19 @@ impl Engine {
             .map_err(error_to_jsvalue)?;
         report.to_string_pretty().map_err(error_to_jsvalue)
     }
+
+    /// Get AST of policies.
+    ///
+    /// See https://docs.rs/regorus/latest/regorus/struct.Engine.html#method.get_ast_as_json
+    #[cfg(feature = "ast")]
+    pub fn getAstAsJson(&self) -> Result<String, JsValue> {
+        self.engine.get_ast_as_json().map_err(error_to_jsvalue)
+    }
 }
 
 #[cfg(test)]
 mod tests {
+    use crate::error_to_jsvalue;
     use wasm_bindgen::prelude::*;
     use wasm_bindgen_test::wasm_bindgen_test;
 
@@ -204,7 +224,7 @@ mod tests {
         assert_eq!(pkg, "data.test");
 
         let results = engine.evalQuery("data".to_string())?;
-        let r = regorus::Value::from_json_str(&results).map_err(crate::error_to_jsvalue)?;
+        let r = regorus::Value::from_json_str(&results).map_err(error_to_jsvalue)?;
 
         let v = &r["result"][0]["expressions"][0]["value"];
 
@@ -216,7 +236,7 @@ mod tests {
 
         // Use eval_rule to perform same query.
         let v = engine.evalRule("data.test.message".to_owned())?;
-        let v = regorus::Value::from_json_str(&v).map_err(crate::error_to_jsvalue)?;
+        let v = regorus::Value::from_json_str(&v).map_err(error_to_jsvalue)?;
 
         // Ensure that input and policy were evaluated.
         assert_eq!(v, regorus::Value::from("Hello"));
@@ -234,7 +254,7 @@ mod tests {
 
         // Test code coverage.
         let report = engine1.getCoverageReport()?;
-        let r = regorus::Value::from_json_str(&report).map_err(crate::error_to_jsvalue)?;
+        let r = regorus::Value::from_json_str(&report).map_err(error_to_jsvalue)?;
 
         assert_eq!(
             r["files"][0]["covered"]
@@ -246,6 +266,13 @@ mod tests {
         println!("{}", engine1.getCoverageReportPretty()?);
 
         engine1.clearCoverageData();
+
+        let policies = engine1.getPolicies()?;
+        let v = regorus::Value::from_json_str(&policies).map_err(error_to_jsvalue)?;
+        assert_eq!(
+            v[0]["path"].as_string().map_err(error_to_jsvalue)?.as_ref(),
+            "hello.rego"
+        );
         Ok(())
     }
 }

build.rs

@@ -8,6 +8,9 @@ fn main() -> Result<()> {
     // Copy hooks to appropriate location so that git will run them.
     // In git worktrees, .git is a symlink and the following commands fail.
     if Path::new(".git").is_dir() {
+        if !Path::new("./.git/hooks").exists() {
+            std::fs::create_dir_all("./.git/hooks")?;
+        }
         std::fs::copy("./scripts/pre-commit", "./.git/hooks/pre-commit")?;
         std::fs::copy("./scripts/pre-push", "./.git/hooks/pre-push")?;
     }

examples/extension_list/agent-extension-data-allow-only.json

@@ -0,0 +1,19 @@
+{
+    "azureGuestAgentPolicy": {
+        "signingRules": {
+            "extensionSigned": true
+        },
+        "allowListOnly": true
+    },
+    "azureGuestExtensionsPolicy": {
+        "test3": {
+            "runtimeRules": {}
+        },
+        "test2": {
+            "signingRules": {
+                "extensionSigned": false
+            },
+            "runtimeRules": {}
+        }
+    }
+}

examples/extension_list/agent-extension-default-data.json

@@ -0,0 +1,9 @@
+{
+    "azureGuestAgentPolicy": {
+        "policyVersion": "0.1.0",
+        "signingRules": {
+            "extensionSigned": false
+        },
+        "allowListOnly": false
+    }
+}

examples/extension_list/agent-extension-input.json

@@ -0,0 +1,28 @@
+{
+    "extensions": {
+        "Microsoft.Azure.ActiveDirectory.AADSSHLoginForLinux": {
+            "signingInfo": {
+
+                "extensionSigned": false
+
+            }
+
+        },
+        "test2": {
+            "signingInfo": {
+                "extensionSigned": true
+            }
+        },
+        "test3": {
+            "signingInfo": {
+                "extensionSigned": false
+            }
+        },
+        "test1": {
+            "signingInfo": {
+                "extensionSigned": false
+            }
+        },
+        "test4": {}
+    }
+}

examples/extension_list/agent_extension_policy.rego

@@ -0,0 +1,125 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+package agent_extension_policy
+
+import rego.v1
+
+policy_version := "0.1.0"
+
+default default_global_rules := {
+	"allowListOnly": false,
+	"signingRules": {
+		"extensionSigned": false,
+		"signingDetails": {},
+	},
+	"updateAllowed": true,
+	"uninstallAllowed": true,
+}
+
+default global_rules := {
+	"allowListOnly": false,
+	"signingRules": {
+		"extensionSigned": false,
+		"signingDetails": {},
+	},
+	"updateAllowed": true,
+	"uninstallAllowed": true,
+}
+
+global_rules := object.union(default_global_rules, data.azureGuestAgentPolicy) if {
+	data.azureGuestAgentPolicy
+}
+
+default any_extension_allowed := true
+
+any_extension_allowed := false if {
+	global_rules.allowListOnly
+}
+
+default default_signing_info := {"signingInfo": {}}
+
+# Download rule 1: if the extension is in the list and download rule satisfied: download allowed
+extensions_to_download[name] := extension if {
+	some name, input_extension in input.extensions
+	data.azureGuestExtensionsPolicy[name]
+	download_rule_validated(input_extension, data.azureGuestExtensionsPolicy[name])
+	extension := object.union(input_extension, {"downloadAllowed": true})
+}
+
+# Download rule 2: if the extension is in the list and download rule not satisfied: download denied
+extensions_to_download[name] := extension if {
+	some name, input_extension in input.extensions
+	data.azureGuestExtensionsPolicy[name]
+	not download_rule_validated(input_extension, data.azureGuestExtensionsPolicy[name])
+	extension := object.union(input_extension, {"downloadAllowed": false})
+}
+
+# Download rule 3: if the extension is not in the list: depending on allowListOnly on or off
+extensions_to_download[name] := extension if {
+	some name, input_extension in input.extensions
+	not data.azureGuestExtensionsPolicy[name]
+	extension := object.union(input_extension, {"downloadAllowed": any_extension_allowed})
+}
+
+# Validate rule 1: if individual signing rule exists, signing rule validated according to the rules
+extensions_validated[name] := extension if {
+	some name, input_extension in input.extensions
+	data.azureGuestExtensionsPolicy[name]
+
+	extension_global_rules := object.union(global_rules, data.azureGuestExtensionsPolicy[name])
+	extension_signing_info := object.union(extension_global_rules, default_signing_info)
+	output := object.union(input_extension, extension_signing_info)
+	signing_validated(output.signingInfo, output.signingRules)
+	extension := object.union(output, {"signingValidated": true})
+}
+
+# Validate rule 2: if indivual signing rule exists, signing rule not validated according to the rules
+extensions_validated[name] := extension if {
+	some name, input_extension in input.extensions
+	data.azureGuestExtensionsPolicy[name]
+
+	extension_global_rules := object.union(global_rules, data.azureGuestExtensionsPolicy[name])
+	extension_signing_info := object.union(extension_global_rules, default_signing_info)
+	output := object.union(input_extension, extension_signing_info)
+	not signing_validated(output.signingInfo, output.signingRules)
+	extension := object.union(output, {"signingValidated": false})
+}
+
+# Validate rule 3: if individual signing rule doesn't exist, signing rule validated according to global signing rule
+extensions_validated[name] := extension if {
+	some name, input_extension in input.extensions
+	not data.azureGuestExtensionsPolicy[name]
+	extension_global_rules := object.union(input_extension, global_rules)
+	output := object.union(extension_global_rules, default_signing_info)
+	signing_validated(output.signingInfo, output.signingRules)
+	extension := object.union(output, {"signingValidated": true})
+}
+
+# Validate rule 4: if individual signing rule doesn't exist, signing rule not validated according to the global rules
+extensions_validated[name] := extension if {
+	some name, input_extension in input.extensions
+	not data.azureGuestExtensionsPolicy[name]
+	extension_global_rules := object.union(input_extension, global_rules)
+	output := object.union(extension_global_rules, default_signing_info)
+	not signing_validated(output.signingInfo, output.signingRules)
+	extension := object.union(output, {"signingValidated": false})
+}
+
+# Currently if download rules doesn't exist, allow the extension because its name is in the list.
+# In the future additional rules can be checked with downloadRules present.
+download_rule_validated(_, rules) if {
+	not rules.downloadRules
+}
+
+# Signing is validated if input comes with extension signed, or the input of signing information is matching the
+# rules in data.
+signing_validated(signingInfo, signingRules) if {
+	signingInfo
+	signingRules
+	signingInfo.extensionSigned
+} else if {
+	signingInfo
+	signingRules
+	signingInfo.extensionSigned == signingRules.extensionSigned
+}

examples/regorus.rs

@@ -170,8 +170,40 @@ fn rego_parse(file: String) -> Result<()> {
     Ok(())
 }
 
+#[allow(unused_variables)]
+fn rego_ast(file: String) -> Result<()> {
+    #[cfg(feature = "ast")]
+    {
+        // Create engine.
+        let mut engine = regorus::Engine::new();
+
+        // Create source.
+        #[cfg(feature = "std")]
+        engine.add_policy_from_file(file)?;
+
+        #[cfg(not(feature = "std"))]
+        engine.add_policy(file.clone(), read_file(&file)?)?;
+
+        let ast = engine.get_ast_as_json()?;
+
+        println!("{ast}");
+        Ok(())
+    }
+
+    #[cfg(not(feature = "ast"))]
+    {
+        bail!("`ast` feature must be enabled");
+    }
+}
+
 #[derive(clap::Subcommand)]
 enum RegorusCommand {
+    /// Parse a Rego policy and dump AST.
+    Ast {
+        /// Rego policy file.
+        file: String,
+    },
+
     /// Evaluate a Rego Query.
     Eval {
         /// Directories containing Rego files.
@@ -254,5 +286,6 @@ fn main() -> Result<()> {
         ),
         RegorusCommand::Lex { file, verbose } => rego_lex(file, verbose),
         RegorusCommand::Parse { file } => rego_parse(file),
+        RegorusCommand::Ast { file } => rego_ast(file),
     }
 }

src/ast.rs

@@ -8,12 +8,14 @@ use crate::*;
 use core::{cmp, fmt, ops::Deref};
 
 #[derive(Debug, PartialEq, Eq, Clone)]
+#[cfg_attr(feature = "ast", derive(serde::Serialize))]
 pub enum BinOp {
     And,
     Or,
 }
 
 #[derive(Debug, PartialEq, Eq, Clone)]
+#[cfg_attr(feature = "ast", derive(serde::Serialize))]
 pub enum ArithOp {
     Add,
     Sub,
@@ -23,6 +25,7 @@ pub enum ArithOp {
 }
 
 #[derive(Debug, PartialEq, Eq, Clone)]
+#[cfg_attr(feature = "ast", derive(serde::Serialize))]
 pub enum BoolOp {
     Lt,
     Le,
@@ -33,12 +36,15 @@ pub enum BoolOp {
 }
 
 #[derive(Debug, PartialEq, Eq, Clone)]
+#[cfg_attr(feature = "ast", derive(serde::Serialize))]
 pub enum AssignOp {
     Eq,
     ColEq,
 }
 
+#[cfg_attr(feature = "ast", derive(serde::Serialize))]
 pub struct NodeRef<T> {
+    #[cfg_attr(feature = "ast", serde(flatten))]
     r: Rc<T>,
 }
 
@@ -97,6 +103,7 @@ impl<T> NodeRef<T> {
 pub type Ref<T> = NodeRef<T>;
 
 #[derive(Debug)]
+#[cfg_attr(feature = "ast", derive(serde::Serialize))]
 pub enum Expr {
     // Simple items that only have a span as content.
     String((Span, Value)),
@@ -230,6 +237,7 @@ impl Expr {
 }
 
 #[derive(Debug)]
+#[cfg_attr(feature = "ast", derive(serde::Serialize))]
 pub enum Literal {
     SomeVars {
         span: Span,
@@ -259,6 +267,7 @@ pub enum Literal {
 }
 
 #[derive(Debug)]
+#[cfg_attr(feature = "ast", derive(serde::Serialize))]
 pub struct WithModifier {
     pub span: Span,
     pub refr: Ref<Expr>,
@@ -266,19 +275,23 @@ pub struct WithModifier {
 }
 
 #[derive(Debug)]
+#[cfg_attr(feature = "ast", derive(serde::Serialize))]
 pub struct LiteralStmt {
     pub span: Span,
     pub literal: Literal,
+    #[cfg_attr(feature = "ast", serde(skip_serializing_if = "Vec::is_empty"))]
     pub with_mods: Vec<WithModifier>,
 }
 
 #[derive(Debug)]
+#[cfg_attr(feature = "ast", derive(serde::Serialize))]
 pub struct Query {
     pub span: Span,
     pub stmts: Vec<LiteralStmt>,
 }
 
 #[derive(Debug)]
+#[cfg_attr(feature = "ast", derive(serde::Serialize))]
 pub struct RuleAssign {
     pub span: Span,
     pub op: AssignOp,
@@ -286,6 +299,7 @@ pub struct RuleAssign {
 }
 
 #[derive(Debug)]
+#[cfg_attr(feature = "ast", derive(serde::Serialize))]
 pub struct RuleBody {
     pub span: Span,
     pub assign: Option<RuleAssign>,
@@ -293,6 +307,7 @@ pub struct RuleBody {
 }
 
 #[derive(Debug)]
+#[cfg_attr(feature = "ast", derive(serde::Serialize))]
 pub enum RuleHead {
     Compr {
         span: Span,
@@ -313,6 +328,7 @@ pub enum RuleHead {
 }
 
 #[derive(Debug)]
+#[cfg_attr(feature = "ast", derive(serde::Serialize))]
 pub enum Rule {
     Spec {
         span: Span,
@@ -337,22 +353,27 @@ impl Rule {
 }
 
 #[derive(Debug)]
+#[cfg_attr(feature = "ast", derive(serde::Serialize))]
 pub struct Package {
     pub span: Span,
     pub refr: Ref<Expr>,
 }
 
 #[derive(Debug)]
+#[cfg_attr(feature = "ast", derive(serde::Serialize))]
 pub struct Import {
     pub span: Span,
     pub refr: Ref<Expr>,
+    #[cfg_attr(feature = "ast", serde(skip_serializing_if = "Option::is_none"))]
     pub r#as: Option<Span>,
 }
 
 #[derive(Debug)]
+#[cfg_attr(feature = "ast", derive(serde::Serialize))]
 pub struct Module {
     pub package: Package,
     pub imports: Vec<Import>,
+    #[cfg_attr(feature = "ast", serde(rename(serialize = "rules")))]
     pub policy: Vec<Ref<Rule>>,
     pub rego_v1: bool,
 }

src/builtins/strings.rs

@@ -28,6 +28,7 @@ pub fn register(m: &mut builtins::BuiltinsMap<&'static str, builtins::BuiltinFcn
     m.insert("startswith", (startswith, 2));
     m.insert("strings.any_prefix_match", (any_prefix_match, 2));
     m.insert("strings.any_suffix_match", (any_suffix_match, 2));
+    m.insert("strings.count", (strings_count, 2));
     m.insert("strings.replace_n", (replace_n, 2));
     m.insert("strings.reverse", (reverse, 1));
     m.insert("substring", (substring, 3));
@@ -145,11 +146,18 @@ fn split(span: &Span, params: &[Ref<Expr>], args: &[Value], _strict: bool) -> Re
     let s = ensure_string(name, &params[0], &args[0])?;
     let delimiter = ensure_string(name, &params[1], &args[1])?;
 
-    Ok(Value::from_array(
+    // Handle https://github.com/microsoft/regorus/issues/291
+    let parts: Vec<Value> = if delimiter.as_ref() == "" {
+        // If delimiter is "", str::split returns a leading and trailing "" whereas Golang's split doesn't.
+        // Therefore avoid str::split and instead return each char as a Value::String.
+        s.chars().map(|c| Value::from(c.to_string())).collect()
+    } else {
         s.split(delimiter.as_ref())
             .map(|s| Value::String(s.into()))
-            .collect(),
-    ))
+            .collect()
+    };
+
+    Ok(Value::from(parts))
 }
 
 fn to_string(v: &Value, unescape: bool) -> String {
@@ -512,6 +520,27 @@ fn any_suffix_match(
     ))
 }
 
+fn strings_count(
+    span: &Span,
+    params: &[Ref<Expr>],
+    args: &[Value],
+    _strict: bool,
+) -> Result<Value> {
+    let name = "strings.count";
+    ensure_args_count(span, name, params, args, 2)?;
+
+    let search = ensure_string(name, &params[0], &args[0])?;
+    let substring = ensure_string(name, &params[0], &args[1])?;
+
+    Ok(Value::from(
+        search
+            .as_bytes()
+            .windows(substring.len())
+            .filter(|&w| w == substring.as_bytes())
+            .count(),
+    ))
+}
+
 fn startswith(span: &Span, params: &[Ref<Expr>], args: &[Value], _strict: bool) -> Result<Value> {
     let name = "startswith";
     ensure_args_count(span, name, params, args, 2)?;

src/builtins/uuid.rs

@@ -121,6 +121,7 @@ fn timestamp(uuid: &Uuid) -> Option<Timestamp> {
     // https://github.com/uuid-rs/uuid/blob/94ecea893fadac93248f1bd6f47673c09cec5912/src/lib.rs#L900-L904
     if uuid.get_version_num() == 2 {
         let (ticks, counter) = decode_rfc4122_timestamp(uuid);
+        #[allow(deprecated)]
         return Some(Timestamp::from_rfc4122(ticks, counter));
     }
 

src/engine.rs

@@ -129,6 +129,66 @@ impl Engine {
             .collect()
     }
 
+    /// Get the list of policy files.
+    /// ```
+    /// # use regorus::*;
+    /// # fn main() -> anyhow::Result<()> {
+    /// # let mut engine = Engine::new();
+    ///
+    /// let pkg = engine.add_policy("hello.rego".to_string(), "package test".to_string())?;
+    /// assert_eq!(pkg, "data.test");
+    ///
+    /// let policies = engine.get_policies()?;
+    ///
+    /// assert_eq!(policies[0].get_path(), "hello.rego");
+    /// assert_eq!(policies[0].get_contents(), "package test");
+    /// # Ok(())
+    /// # }
+    /// ```
+    pub fn get_policies(&self) -> Result<Vec<Source>> {
+        Ok(self
+            .modules
+            .iter()
+            .map(|m| m.package.refr.span().source.clone())
+            .collect())
+    }
+
+    /// Get the list of policy files as a JSON object.
+    /// ```
+    /// # use regorus::*;
+    /// # fn main() -> anyhow::Result<()> {
+    /// # let mut engine = Engine::new();
+    ///
+    /// let pkg = engine.add_policy("hello.rego".to_string(), "package test".to_string())?;
+    /// assert_eq!(pkg, "data.test");
+    ///
+    /// let policies = engine.get_policies_as_json()?;
+    ///
+    /// let v = Value::from_json_str(&policies)?;
+    /// assert_eq!(v[0]["path"].as_string()?.as_ref(), "hello.rego");
+    /// assert_eq!(v[0]["contents"].as_string()?.as_ref(), "package test");
+    /// # Ok(())
+    /// # }
+    /// ```
+    pub fn get_policies_as_json(&self) -> Result<String> {
+        #[derive(Serialize)]
+        struct Source<'a> {
+            path: &'a String,
+            contents: &'a String,
+        }
+
+        let mut sources = vec![];
+        for m in self.modules.iter() {
+            let source = &m.package.refr.span().source;
+            sources.push(Source {
+                path: source.get_path(),
+                contents: source.get_contents(),
+            });
+        }
+
+        serde_json::to_string_pretty(&sources).map_err(anyhow::Error::msg)
+    }
+
     /// Set the input document.
     ///
     /// * `input`: Input documented. Typically this [Value] is constructed from JSON or YAML.
@@ -179,7 +239,7 @@ impl Engine {
     /// # }
     /// ```
     pub fn clear_data(&mut self) {
-        self.interpreter.set_data(Value::new_object());
+        self.interpreter.set_init_data(Value::new_object());
         self.prepared = false;
     }
 
@@ -216,7 +276,40 @@ impl Engine {
             bail!("data must be object");
         }
         self.prepared = false;
-        self.interpreter.get_data_mut().merge(data)
+        self.interpreter.get_init_data_mut().merge(data)
+    }
+
+    /// Get the data document.
+    ///
+    /// The returned value is the data document that has been constructed using
+    /// one or more calls to [`Engine::add_data`]. The values of policy rules are
+    /// not included in the returned document.
+    ///
+    ///
+    /// ```
+    /// # use regorus::*;
+    /// # fn main() -> anyhow::Result<()> {
+    /// let mut engine = Engine::new();
+    ///
+    /// // If not set, data document is empty.
+    /// assert_eq!(engine.get_data(), Value::new_object());
+    ///
+    /// // Merge { "x" : 1, "y" : {} }
+    /// assert!(engine.add_data(Value::from_json_str(r#"{ "x" : 1, "y" : {}}"#)?).is_ok());
+    ///
+    /// // Merge { "z" : 2 }
+    /// assert!(engine.add_data(Value::from_json_str(r#"{ "z" : 2 }"#)?).is_ok());
+    ///
+    /// let data = engine.get_data();
+    /// assert_eq!(data["x"], Value::from(1));
+    /// assert_eq!(data["y"], Value::new_object());
+    /// assert_eq!(data["z"], Value::from(2));
+    ///
+    /// # Ok(())
+    /// # }
+    /// ```
+    pub fn get_data(&self) -> Value {
+        self.interpreter.get_init_data().clone()
     }
 
     pub fn add_data_json(&mut self, data_json: &str) -> Result<()> {
@@ -477,11 +570,7 @@ impl Engine {
             self.interpreter.set_modules(&self.modules);
 
             self.interpreter.clear_builtins_cache();
-            // when the interpreter is prepared the initial data is saved
-            // the data will be reset to init_data each time clean_internal_evaluation_state is called
-            let init_data = self.interpreter.get_data_mut().clone();
-            self.interpreter.set_init_data(init_data);
-
+            // clean_internal_evaluation_state will set data to an efficient clont of use supplied init_data
             // Initialize the with-document with initial data values.
             // with-modifiers will be applied to this document.
             self.interpreter.init_with_document()?;
@@ -743,4 +832,42 @@ impl Engine {
     pub fn take_prints(&mut self) -> Result<Vec<String>> {
         self.interpreter.take_prints()
     }
+
+    /// Get the policies and corresponding AST.
+    ///
+    ///
+    /// ```rust
+    /// # use regorus::*;
+    /// # use anyhow::{bail, Result};
+    /// # fn main() -> Result<()> {
+    /// # let mut engine = Engine::new();
+    /// engine.add_policy("test.rego".to_string(), "package test\n x := 1".to_string())?;
+    ///
+    /// let ast = engine.get_ast_as_json()?;
+    /// let value = Value::from_json_str(&ast)?;
+    ///
+    /// assert_eq!(value[0]["ast"]["package"]["refr"]["Var"][1].as_string()?.as_ref(), "test");
+    /// # Ok(())
+    /// # }
+    /// ```
+    #[cfg(feature = "ast")]
+    #[cfg_attr(docsrs, doc(cfg(feature = "ast")))]
+    pub fn get_ast_as_json(&self) -> Result<String> {
+        #[derive(Serialize)]
+        struct Policy<'a> {
+            source: &'a Source,
+            version: u32,
+            ast: &'a Module,
+        }
+        let mut ast = vec![];
+        for m in &self.modules {
+            ast.push(Policy {
+                source: &m.package.span.source,
+                version: 1,
+                ast: m,
+            });
+        }
+
+        serde_json::to_string_pretty(&ast).map_err(anyhow::Error::msg)
+    }
 }

src/interpreter.rs

@@ -216,18 +216,22 @@ impl Interpreter {
         self.modules = modules.to_vec();
     }
 
-    pub fn set_init_data(&mut self, init_data: Value) {
-        self.init_data = init_data;
-    }
-
-    pub fn set_data(&mut self, data: Value) {
-        self.data = data;
-    }
-
     pub fn get_data_mut(&mut self) -> &mut Value {
         &mut self.data
     }
 
+    pub fn set_init_data(&mut self, data: Value) {
+        self.init_data = data;
+    }
+
+    pub fn get_init_data(&self) -> &Value {
+        &self.init_data
+    }
+
+    pub fn get_init_data_mut(&mut self) -> &mut Value {
+        &mut self.init_data
+    }
+
     pub fn set_traces(&mut self, enable_tracing: bool) {
         self.traces = match enable_tracing {
             true => Some(vec![]),

src/lexer.rs

@@ -12,17 +12,34 @@ use crate::Value;
 use anyhow::{anyhow, bail, Result};
 
 #[derive(Clone)]
+#[cfg_attr(feature = "ast", derive(serde::Serialize))]
 struct SourceInternal {
     pub file: String,
     pub contents: String,
+    #[cfg_attr(feature = "ast", serde(skip_serializing))]
     pub lines: Vec<(u32, u32)>,
 }
 
+/// A policy file.
 #[derive(Clone)]
+#[cfg_attr(feature = "ast", derive(serde::Serialize))]
 pub struct Source {
+    #[cfg_attr(feature = "ast", serde(flatten))]
     src: Rc<SourceInternal>,
 }
 
+impl Source {
+    /// The path associated with the policy file.
+    pub fn get_path(&self) -> &String {
+        &self.src.file
+    }
+
+    /// The contents of the policy file.
+    pub fn get_contents(&self) -> &String {
+        &self.src.contents
+    }
+}
+
 impl cmp::Ord for Source {
     fn cmp(&self, other: &Source) -> cmp::Ordering {
         Rc::as_ptr(&self.src).cmp(&Rc::as_ptr(&other.src))
@@ -212,7 +229,9 @@ impl Source {
 }
 
 #[derive(Clone)]
+#[cfg_attr(feature = "ast", derive(serde::Serialize))]
 pub struct Span {
+    #[cfg_attr(feature = "ast", serde(skip_serializing))]
     pub source: Source,
     pub line: u32,
     pub col: u32,
@@ -274,6 +293,10 @@ pub struct Lexer<'source> {
     iter: Peekable<CharIndices<'source>>,
     line: u32,
     col: u32,
+    unknown_char_is_symbol: bool,
+    allow_slash_star_escape: bool,
+    comment_starts_with_double_slash: bool,
+    double_colon_token: bool,
 }
 
 impl<'source> Lexer<'source> {
@@ -283,9 +306,29 @@ impl<'source> Lexer<'source> {
             iter: source.contents().char_indices().peekable(),
             line: 1,
             col: 1,
+            unknown_char_is_symbol: false,
+            allow_slash_star_escape: false,
+            comment_starts_with_double_slash: false,
+            double_colon_token: false,
         }
     }
 
+    pub fn set_unknown_char_is_symbol(&mut self, b: bool) {
+        self.unknown_char_is_symbol = b;
+    }
+
+    pub fn set_allow_slash_star_escape(&mut self, b: bool) {
+        self.allow_slash_star_escape = b;
+    }
+
+    pub fn set_comment_starts_with_double_slash(&mut self, b: bool) {
+        self.comment_starts_with_double_slash = b;
+    }
+
+    pub fn set_double_colon_token(&mut self, b: bool) {
+        self.double_colon_token = b;
+    }
+
     fn peek(&mut self) -> (usize, char) {
         match self.iter.peek() {
             Some((index, chr)) => (*index, *chr),
@@ -465,6 +508,7 @@ impl<'source> Lexer<'source> {
                     match ch {
                         // json escape sequence
                         '"' | '\\' | '/' | 'b' | 'f' | 'n' | 'r' | 't' => (),
+                        '*' if self.allow_slash_star_escape => (),
                         'u' => {
                             for _i in 0..4 {
                                 let (offset, ch) = self.peek();
@@ -528,12 +572,24 @@ impl<'source> Lexer<'source> {
         ))
     }
 
+    #[inline]
+    fn skip_past_newline(&mut self) -> Result<()> {
+        self.iter.next();
+        loop {
+            match self.peek().1 {
+                '\n' | '\x00' => break,
+                _ => self.iter.next(),
+            };
+        }
+        Ok(())
+    }
+
     fn skip_ws(&mut self) -> Result<()> {
         // Only the 4 json whitespace characters are recognized.
         // https://www.crockford.com/mckeeman.html.
         // Additionally, comments are also skipped.
         // A tab is considered 4 space characters.
-        'outer: loop {
+        loop {
             match self.peek().1 {
                 ' ' => self.col += 1,
                 '\t' => self.col += 4,
@@ -550,14 +606,13 @@ impl<'source> Lexer<'source> {
                     self.col = 1;
                     self.line += 1;
                 }
-                '#' => {
-                    self.iter.next();
-                    loop {
-                        match self.peek().1 {
-                            '\n' | '\x00' => continue 'outer,
-                            _ => self.iter.next(),
-                        };
-                    }
+                '#' if !self.comment_starts_with_double_slash => {
+                    self.skip_past_newline()?;
+                    continue;
+                }
+                '/' if self.comment_starts_with_double_slash && self.peekahead(1).1 == '/' => {
+                    self.skip_past_newline()?;
+                    continue;
                 }
                 _ => break,
             }
@@ -601,7 +656,7 @@ impl<'source> Lexer<'source> {
 		self.col += 1;
 		self.iter.next();
 		let mut end = start as u32 + 1;
-		if self.peek().1 == '=' {
+		if self.peek().1 == '=' || (self.peek().1 == ':' && self.double_colon_token) {
 		    self.col += 1;
 		    self.iter.next();
 		    end += 1;
@@ -675,6 +730,17 @@ impl<'source> Lexer<'source> {
 		}
 		Ok(ident)
 	    }
+	    _ if self.unknown_char_is_symbol => {
+		self.col += 1;
+		self.iter.next();
+		Ok(Token(TokenKind::Symbol, Span {
+		    source: self.source.clone(),
+		    line: self.line,
+		    col,
+		    start: start as u32,
+		    end: start as u32 + 1,
+		}))
+	    }
 	    _ => Err(self.source.error(self.line, self.col, "invalid character"))
 	}
     }

src/lib.rs

@@ -1,6 +1,8 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT License.
 
+#![allow(unknown_lints)]
+#![allow(clippy::doc_lazy_continuation)]
 // Use README.md as crate documentation.
 #![doc = include_str!(concat!(env!("CARGO_MANIFEST_DIR"), "/README.md"))]
 // We'll default to building for no_std - use core, alloc instead of std.
@@ -28,6 +30,7 @@ mod utils;
 mod value;
 
 pub use engine::Engine;
+pub use lexer::Source;
 pub use value::Value;
 
 #[cfg(feature = "arc")]

src/scheduler.rs

@@ -386,7 +386,9 @@ pub struct Analyzer {
 }
 
 #[derive(Debug, Clone)]
+#[allow(dead_code)]
 pub struct Schedule {
+    #[allow(unused)]
     pub scopes: BTreeMap<Ref<Query>, Scope>,
     pub order: BTreeMap<Ref<Query>, Vec<u16>>,
 }
@@ -410,7 +412,7 @@ impl Analyzer {
     }
 
     pub fn analyze(mut self, modules: &[Ref<Module>]) -> Result<Schedule> {
-        self.add_rules(modules)?;
+        self.add_rules_and_aliases(modules)?;
         self.functions = gather_functions(modules)?;
 
         for m in modules {
@@ -428,7 +430,7 @@ impl Analyzer {
         modules: &[Ref<Module>],
         query: &Ref<Query>,
     ) -> Result<Schedule> {
-        self.add_rules(modules)?;
+        self.add_rules_and_aliases(modules)?;
         self.analyze_query(None, None, query, Scope::default())?;
 
         Ok(Schedule {
@@ -437,7 +439,7 @@ impl Analyzer {
         })
     }
 
-    fn add_rules(&mut self, modules: &[Ref<Module>]) -> Result<()> {
+    fn add_rules_and_aliases(&mut self, modules: &[Ref<Module>]) -> Result<()> {
         for m in modules {
             let path = get_path_string(&m.package.refr, Some("data"))?;
             let scope: &mut Scope = self.packages.entry(path).or_default();
@@ -454,6 +456,12 @@ impl Analyzer {
                 };
                 scope.unscoped.insert(var);
             }
+
+            for import in &m.imports {
+                if let Some(var) = &import.r#as {
+                    scope.unscoped.insert(var.source_str());
+                }
+            }
         }
 
         Ok(())

src/tests/interpreter/mod.rs

@@ -422,3 +422,32 @@ fn one_yaml() -> Result<()> {
 fn run(path: &str) {
     yaml_test(path).unwrap()
 }
+
+#[test]
+fn test_get_data() -> Result<()> {
+    let mut engine = Engine::new();
+
+    // Merge { "x" : 1, "y" : {} }
+    engine.add_data(Value::from_json_str(r#"{ "x" : 1, "y" : {}}"#)?)?;
+
+    // Merge { "z" : 2 }
+    engine.add_data(Value::from_json_str(r#"{ "z" : 2 }"#)?)?;
+
+    // Add a policy
+    engine.add_policy("policy.rego".to_string(), "package a".to_string())?;
+
+    // Evaluate virtual data document. The virtual document includes all rules as well.
+    let v_data = engine.eval_query("data".to_string(), false)?.result[0].expressions[0]
+        .value
+        .clone();
+    // There must be an empty package.
+    assert_eq!(v_data["a"], Value::new_object());
+
+    // Get the data document.
+    let data = engine.get_data();
+
+    // There must NOT be any value of `a`.
+    assert_eq!(data["a"], Value::Undefined);
+
+    Ok(())
+}

tests/interpreter/cases/builtins/strings/split.yaml

@@ -0,0 +1,17 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+cases:
+  - note: empty separator
+    data: {}
+    modules: []
+    query: "x := split(\"test\", \"\")"
+    want_result:
+      x: ["t", "e", "s", "t"]
+      
+  - note: empty separator, empty string
+    data: {}
+    modules: []
+    query: "x := split(\"\", \"\")"
+    want_result:
+      x: []

tests/interpreter/cases/import/tests.yaml

@@ -34,6 +34,7 @@ cases:
         
       - |
         package b
+        import rego.v1
         # Both the following imports are overridden by rules
         #import data.a.b as a
         #import data.a.b
@@ -43,6 +44,10 @@ cases:
 
         a = 10
         c = C + b
+
+        r if {
+          some v in [C]
+        }
     query: data
     want_result:
       a:
@@ -50,6 +55,7 @@ cases:
       b:
         a: 10
         c: 22
+        r: true
         
   - note: import overridden by rule
     modules:

tests/kata/main.rs

@@ -172,6 +172,64 @@ struct Cli {
     generate: bool,
 }
 
+fn stateful_policy_test() -> Result<()> {
+    // Create an engine for evaluating Rego policies.
+    let mut engine = regorus::Engine::new();
+
+    let policy = String::from(
+        r#"
+       package example
+       import rego.v1
+
+       default allow := false
+
+       allow if {
+         print("data.allowed_actions = ", data.allowed_actions)
+         input.action in data.allowed_actions["user1"]
+         print("This rule should be allowed")
+       }
+	"#,
+    );
+
+    // Add policy to the engine.
+    engine.add_policy(String::from("policy.rego"), policy)?;
+
+    // Evaluate first input. Expect to evaluate to false, since state is not set
+    engine.set_input(regorus::Value::from_json_str(
+        r#"{
+      "action": "write"
+    }"#,
+    )?);
+
+    let r = engine.eval_bool_query(String::from("data.example.allow"), false)?;
+    println!("Received result: {:?}", r);
+    assert_eq!(r, false);
+
+    // Add data to engine. Set state
+    engine.add_data(regorus::Value::from_json_str(
+        r#"{
+     "allowed_actions": {
+        "user1" : ["read", "write"]
+     }}"#,
+    )?)?;
+
+    // Evaluate second input. Expect to evaluate to true, since state has been set now
+    engine.set_input(regorus::Value::from_json_str(
+        r#"{
+      "action": "write"
+    }"#,
+    )?);
+
+    let r = engine.eval_bool_query(String::from("data.example.allow"), false)?;
+    println!("Received result: {:?}", r);
+    assert_eq!(
+        r, true,
+        "expect result to be true since rule evaluates to true after state has been updated, per rego logs"
+    );
+
+    Ok(())
+}
+
 fn main() -> Result<()> {
     let cli = Cli::parse();
     run_kata_tests(
@@ -179,5 +237,6 @@ fn main() -> Result<()> {
         &cli.name,
         cli.coverage,
         cli.generate,
-    )
+    )?;
+    stateful_policy_test()
 }

tests/opa.rs

@@ -13,7 +13,7 @@ use serde::{Deserialize, Serialize};
 use walkdir::WalkDir;
 
 const OPA_REPO: &str = "https://github.com/open-policy-agent/opa";
-const OPA_BRANCH: &str = "v0.64.0";
+const OPA_BRANCH: &str = "v0.67.0";
 
 #[derive(Serialize, Deserialize, PartialEq, Debug)]
 #[serde(deny_unknown_fields)]
@@ -325,9 +325,9 @@ fn run_opa_tests(opa_tests_dir: String, folders: &[String]) -> Result<()> {
     if npass == 0 && nfail == 0 {
         bail!("no matching tests found.");
     } else if nfail == 0 {
-        println!("\x1b[32m    {:42}: {npass:4} {nfail:4}\x1b[0m", "TOTAL");
+        println!("\x1b[32m    {:40}: {npass:4} {nfail:4}\x1b[0m", "TOTAL");
     } else {
-        println!("\x1b[31m    {:42}: {npass:4} {nfail:4}\x1b[0m", "TOTAL");
+        println!("\x1b[31m    {:40}: {npass:4} {nfail:4}\x1b[0m", "TOTAL");
     }
 
     if !missing_functions.is_empty() {