Basic benchmarking setup with Criterion (#408)

* Basic benchmarking setup with Criterion * Fix Clippy warnings Signed-off-by: Burak Varlı <burakvar@amazon.co.uk>
2026-08-05 02:16:11 +00:00 · 2025-05-14 17:41:06 +02:00
parent 5caac47b38
commit 60ac4a7a7c
3 changed files with 329 additions and 0 deletions
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -40,6 +40,12 @@ dependencies = [
 "libc",
 ]

+[[package]]
+name = "anes"
+version = "0.1.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299"
+
 [[package]]
 name = "anstream"
 version = "0.6.18"
@@ -157,6 +163,12 @@ version = "0.6.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "5ce89b21cab1437276d2650d57e971f9d548a2d9037cc231abdc0562b97498ce"

+[[package]]
+name = "cast"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5"
+
 [[package]]
 name = "cc"
 version = "1.2.20"
@@ -207,6 +219,33 @@ dependencies = [
 "phf_codegen",
 ]

+[[package]]
+name = "ciborium"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e"
+dependencies = [
+ "ciborium-io",
+ "ciborium-ll",
+ "serde",
+]
+
+[[package]]
+name = "ciborium-io"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757"
+
+[[package]]
+name = "ciborium-ll"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9"
+dependencies = [
+ "ciborium-io",
+ "half",
+]
+
 [[package]]
 name = "clap"
 version = "4.5.38"
@@ -259,6 +298,73 @@ version = "0.8.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b"

+[[package]]
+name = "criterion"
+version = "0.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f2b12d017a929603d80db1831cd3a24082f8137ce19c69e6447f54f5fc8d692f"
+dependencies = [
+ "anes",
+ "cast",
+ "ciborium",
+ "clap",
+ "criterion-plot",
+ "is-terminal",
+ "itertools",
+ "num-traits",
+ "once_cell",
+ "oorandom",
+ "plotters",
+ "rayon",
+ "regex",
+ "serde",
+ "serde_derive",
+ "serde_json",
+ "tinytemplate",
+ "walkdir",
+]
+
+[[package]]
+name = "criterion-plot"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1"
+dependencies = [
+ "cast",
+ "itertools",
+]
+
+[[package]]
+name = "crossbeam-deque"
+version = "0.8.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51"
+dependencies = [
+ "crossbeam-epoch",
+ "crossbeam-utils",
+]
+
+[[package]]
+name = "crossbeam-epoch"
+version = "0.9.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e"
+dependencies = [
+ "crossbeam-utils",
+]
+
+[[package]]
+name = "crossbeam-utils"
+version = "0.8.21"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28"
+
+[[package]]
+name = "crunchy"
+version = "0.2.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "43da5946c66ffcc7745f48db692ffbb10a83bfe0afd96235c5c2a4fb23994929"
+
 [[package]]
 name = "data-encoding"
 version = "2.9.0"
@@ -276,6 +382,12 @@ dependencies = [
 "syn 2.0.101",
 ]

+[[package]]
+name = "either"
+version = "1.15.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719"
+
 [[package]]
 name = "email_address"
 version = "0.2.9"
@@ -381,6 +493,16 @@ dependencies = [
 "regex-syntax",
 ]

+[[package]]
+name = "half"
+version = "2.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "459196ed295495a68f7d7fe1d84f6c4b7ff0e21fe3017b2f283c6fac3ad803c9"
+dependencies = [
+ "cfg-if",
+ "crunchy",
+]
+
 [[package]]
 name = "hashbrown"
 version = "0.15.2"
@@ -393,6 +515,12 @@ version = "0.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"

+[[package]]
+name = "hermit-abi"
+version = "0.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f154ce46856750ed433c8649605bf7ed2de3bc35fd9d2a9f30cddd873c80cb08"
+
 [[package]]
 name = "iana-time-zone"
 version = "0.1.63"
@@ -566,12 +694,32 @@ dependencies = [
 "hashbrown",
 ]

+[[package]]
+name = "is-terminal"
+version = "0.4.16"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e04d7f318608d35d4b61ddd75cbdaee86b023ebe2bd5a66ee0915f0bf93095a9"
+dependencies = [
+ "hermit-abi",
+ "libc",
+ "windows-sys",
+]
+
 [[package]]
 name = "is_terminal_polyfill"
 version = "1.70.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf"

+[[package]]
+name = "itertools"
+version = "0.10.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473"
+dependencies = [
+ "either",
+]
+
 [[package]]
 name = "itoa"
 version = "1.0.15"
@@ -751,6 +899,12 @@ version = "1.21.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d"

+[[package]]
+name = "oorandom"
+version = "11.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e"
+
 [[package]]
 name = "outref"
 version = "0.5.2"
@@ -848,6 +1002,34 @@ dependencies = [
 "siphasher",
 ]

+[[package]]
+name = "plotters"
+version = "0.3.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747"
+dependencies = [
+ "num-traits",
+ "plotters-backend",
+ "plotters-svg",
+ "wasm-bindgen",
+ "web-sys",
+]
+
+[[package]]
+name = "plotters-backend"
+version = "0.3.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "df42e13c12958a16b3f7f4386b9ab1f3e7933914ecea48da7139435263a4172a"
+
+[[package]]
+name = "plotters-svg"
+version = "0.3.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "51bae2ac328883f7acdfea3d66a7c35751187f870bc81f94563733a154d7a670"
+dependencies = [
+ "plotters-backend",
+]
+
 [[package]]
 name = "ppv-lite86"
 version = "0.2.21"
@@ -953,6 +1135,26 @@ dependencies = [
 "getrandom 0.3.2",
 ]

+[[package]]
+name = "rayon"
+version = "1.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa"
+dependencies = [
+ "either",
+ "rayon-core",
+]
+
+[[package]]
+name = "rayon-core"
+version = "1.12.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2"
+dependencies = [
+ "crossbeam-deque",
+ "crossbeam-utils",
+]
+
 [[package]]
 name = "redox_syscall"
 version = "0.5.11"
@@ -1034,6 +1236,7 @@ dependencies = [
 "chrono",
 "chrono-tz",
 "clap",
+ "criterion",
 "data-encoding",
 "globset",
 "jsonschema",
@@ -1242,6 +1445,16 @@ dependencies = [
 "zerovec",
 ]

+[[package]]
+name = "tinytemplate"
+version = "1.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc"
+dependencies = [
+ "serde",
+ "serde_json",
+]
+
 [[package]]
 name = "unicode-ident"
 version = "1.0.18"
@@ -1411,6 +1624,16 @@ dependencies = [
 "unicode-ident",
 ]

+[[package]]
+name = "web-sys"
+version = "0.3.77"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "33b6dd2ef9186f1f2072e409e99cd22a975331a6b3591b12c764e0e55c60d5d2"
+dependencies = [
+ "js-sys",
+ "wasm-bindgen",
+]
+
 [[package]]
 name = "winapi-util"
 version = "0.1.9"
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -118,6 +118,7 @@ prettydiff = { version = "0.8.0", default-features = false }
 serde_yaml = "0.9.16"
 test-generator = "0.3.1"
 walkdir = "2.3.2"
+criterion = { version = "0.5" }

 [build-dependencies]
 anyhow = "1.0"
@@ -143,6 +144,10 @@ name="kata"
 harness=false
 test=false

+[[bench]]
+name = "regorus_benchmark"
+harness = false
+
 [[example]]
 name="regorus"
 harness=false
--- a/benches/regorus_benchmark.rs
+++ b/benches/regorus_benchmark.rs
@@ -0,0 +1,101 @@
+use std::hint::black_box;
+
+use regorus::Engine;
+
+use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion};
+use serde_json::json;
+
+fn engine_with_policy(policy: &str) -> Engine {
+    let mut engine = Engine::new();
+    engine
+        .add_policy("policy.rego".to_string(), policy.to_string())
+        .unwrap();
+    engine
+}
+
+fn eval_principal(engine: &mut Engine) {
+    engine.set_input(black_box(json!({"principal": "admin"}).into()));
+    let result = engine
+        .eval_rule(black_box("data.bench.allow".to_string()))
+        .unwrap();
+    assert_eq!(result, true.into());
+}
+
+fn allow_with_simple_equality(c: &mut Criterion) {
+    c.bench_function("simple equality check with constant", |b| {
+        let mut engine = engine_with_policy(
+            r#"
+            package bench
+            allow if input.principal == "admin"
+            "#,
+        );
+
+        b.iter(|| eval_principal(&mut engine))
+    });
+
+    c.bench_function("simple equality check with data", |b| {
+        let mut engine = engine_with_policy(
+            r#"
+            package bench
+            allow if input.principal == data.allowed_principal
+            "#,
+        );
+        engine
+            .add_data(json!({"allowed_principal": "admin"}).into())
+            .unwrap();
+
+        b.iter(|| eval_principal(&mut engine))
+    });
+}
+
+fn allow_with_simple_membership(c: &mut Criterion) {
+    let generate_principals = |n: usize| {
+        (0..n)
+            .map(|i| i.to_string())
+            .chain(std::iter::once("admin".to_string()))
+            .collect::<Vec<_>>()
+    };
+
+    let mut group = c.benchmark_group("allow with simple membership");
+    for size in [32, 64, 128, 512, 1024, 2048].iter() {
+        group.bench_with_input(BenchmarkId::new("with constant", size), size, |b, &size| {
+            let principals = generate_principals(size).join("\",\"");
+            let mut engine = engine_with_policy(&format!(
+                r#"
+                package bench
+
+                allowed_principals := {{
+                    "{principals}"
+                }}
+
+                allow if input.principal in allowed_principals
+                "#
+            ));
+
+            b.iter(|| eval_principal(&mut engine))
+        });
+
+        group.bench_with_input(BenchmarkId::new("with data", size), size, |b, &size| {
+            let principals = generate_principals(size);
+            let mut engine = engine_with_policy(
+                r#"
+                package bench
+                allow if input.principal in data.allowed_principals
+                "#,
+            );
+            engine
+                .add_data(json!({"allowed_principals": principals}).into())
+                .unwrap();
+
+            b.iter(|| eval_principal(&mut engine))
+        });
+    }
+    group.finish();
+}
+
+criterion_group!(
+    benches,
+    allow_with_simple_equality,
+    allow_with_simple_membership
+);
+criterion_main!(benches);