kata-containers/regorus
1
0
Fork 0
mirror of https://github.com/microsoft/regorus.git synced 2026-08-05 02:16:11 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: add multi-threaded evaluation benchmark suite with comprehensive C# implementation (#457)

Browse Source 
This commit introduces a complete multi-threaded evaluation benchmark suite for both Rust and C# implementations of Regorus.

- Implemented engine evaluation benchmark with input and engine cloning strategies
- Implemented compiled policy evaluation benchmark with input cloning and shared compiled policy strategies.

- Created EngineEvaluationBenchmark.cs and CompiledPolicyEvaluationBenchmark.cs with time-based execution (3s warmup + 3s evaluation)
- Implemented configuration options matching Rust implementation (useClonedEngines, useSharedPolicies parameters)

- Created markdown analysis documentation with cross-platform performance analysis
- C# seems to achieve 58-89% of Rust performance on test machine.

Signed-off-by: Anand Krishnamoorthi <anakrish@microsoft.com>
This commit is contained in:
Anand Krishnamoorthi
2025-08-22 11:40:39 -05:00
committed by GitHub
parent a53c7c8192
commit d561531613
56 changed files with 2763 additions and 197 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
bindings/csharp/Benchmarks/Benchmarks.csproj Normal file
View File

@@ -0,0 +1,25 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>net8.0</TargetFramework>
<Nullable>Enable</Nullable>
</PropertyGroup>
<PropertyGroup>
<!-- If the environment variable is set (such as in a Github Action run), append the suffix to the version number -->
<RegorusPackageVersionSuffix Condition="'$(VersionSuffix)' != ''">-$(VersionSuffix)</RegorusPackageVersionSuffix>
</PropertyGroup>
<PropertyGroup>
<!-- If the environment variable is set (such as in a Github Action run), append the suffix to the version number -->
<RegorusPackageVersionSuffix Condition="'$(VersionSuffix)' != ''">-$(VersionSuffix)</RegorusPackageVersionSuffix>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Regorus" Version="0.6.0$(RegorusPackageVersionSuffix)"/>
</ItemGroup>
<ItemGroup>
<None Include="../../ffi/target/release/libregorus_ffi.dylib" CopyToOutputDirectory="PreserveNewest" />
</ItemGroup>
</Project>

278
bindings/csharp/Benchmarks/CompiledPolicyEvaluationBenchmark.cs Normal file
View File

@@ -0,0 +1,278 @@
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Regorus;
namespace Benchmarks
{
public class CompiledPolicyEvaluationBenchmark
{
private static readonly string TestDataPath = Path.Combine(
Directory.GetCurrentDirectory(),
"..", "..", "..",
"benches", "evaluation", "test_data"
);
private static readonly (string PolicyFile, string[] InputFiles)[] PolicyInputFiles = new[]
{
("rbac_policy.rego", new[] { "rbac_input.json", "rbac_input2.json", "rbac_input3.json" }),
("api_access_policy.rego", new[] { "api_access_input.json", "api_access_input2.json", "api_access_input3.json" }),
("data_sensitivity_policy.rego", new[] { "data_sensitivity_input.json", "data_sensitivity_input2.json", "data_sensitivity_input3.json" }),
("time_based_policy.rego", new[] { "time_based_input.json", "time_based_input2.json", "time_based_input3.json" }),
("data_processing_policy.rego", new[] { "data_processing_input.json", "data_processing_input2.json", "data_processing_input3.json" }),
("azure_vm_policy.rego", new[] { "azure_vm_input.json", "azure_vm_input2.json", "azure_vm_input3.json" }),
("azure_storage_policy.rego", new[] { "azure_storage_input.json", "azure_storage_input2.json", "azure_storage_input3.json" }),
("azure_keyvault_policy.rego", new[] { "azure_keyvault_input.json", "azure_keyvault_input2.json", "azure_keyvault_input3.json" }),
("azure_nsg_policy.rego", new[] { "azure_nsg_input.json", "azure_nsg_input2.json", "azure_nsg_input3.json" })
};
private static readonly string[] PolicyNames = new[]
{
"rbac_policy",
"api_access_policy",
"data_sensitivity_policy",
"time_based_policy",
"data_processing_policy",
"azure_vm_policy",
"azure_storage_policy",
"azure_keyvault_policy",
"azure_nsg_policy"
};
private static List<(string Policy, string[] Inputs)> LoadPoliciesWithInputs()
{
var result = new List<(string Policy, string[] Inputs)>();
foreach (var (policyFile, inputFiles) in PolicyInputFiles)
{
var policyPath = Path.Combine(TestDataPath, "policies", policyFile);
var policy = File.ReadAllText(policyPath);
var inputs = inputFiles.Select(inputFile =>
{
var inputPath = Path.Combine(TestDataPath, "inputs", inputFile);
return File.ReadAllText(inputPath);
}).ToArray();
result.Add((policy, inputs));
}
return result;
}
private static List<CompiledPolicy> PrepareSharedCompiledPolicies()
{
var policiesWithInputs = LoadPoliciesWithInputs();
var compiledPolicies = new List<CompiledPolicy>();
foreach (var (policy, _) in policiesWithInputs)
{
var modules = new[] { new PolicyModule { Id = "policy.rego", Content = policy } };
var compiled = Compiler.CompilePolicyWithEntrypoint("{}", modules, "data.bench.allow");
compiledPolicies.Add(compiled);
}
return compiledPolicies;
}
public static void RunCompiledPolicyEvaluationBenchmark()
{
var cpuCount = Environment.ProcessorCount;
var maxThreads = cpuCount * 2;
var threadCounts = new List<int> { 1, 2 };
// Add even numbers from 4 to maxThreads
for (int i = 4; i <= maxThreads; i += 2)
{
threadCounts.Add(i);
}
Console.WriteLine($"Running compiled policy benchmark with max_threads: {maxThreads}");
Console.WriteLine($"Testing with thread counts: {string.Join(", ", threadCounts)}");
Console.WriteLine();
// Benchmark both shared policies and per-iteration compilation
var configurations = new[]
{
(true, "compiled_shared_policies"),
(false, "compiled_per_iteration")
};
foreach (var (useSharedPolicies, groupName) in configurations)
{
Console.WriteLine($"=== {groupName} ===");
foreach (var threads in threadCounts)
{
RunCompiledPolicyBenchmark(threads, useSharedPolicies, groupName);
}
Console.WriteLine();
}
}
public static void RunCompiledPolicyBenchmark(int threads, bool useSharedPolicies, string groupName)
{
const int warmupSeconds = 3;
const int durationSeconds = 3;
var policiesWithInputs = LoadPoliciesWithInputs();
List<CompiledPolicy>? compiledPolicies = null;
if (useSharedPolicies)
{
compiledPolicies = PrepareSharedCompiledPolicies();
}
Console.WriteLine($"Warming up with {threads} threads for {warmupSeconds} seconds...");
// Warmup phase
var (_, _, _) = RunBenchmarkPhase(threads, warmupSeconds, policiesWithInputs, compiledPolicies, useSharedPolicies, isWarmup: true);
Console.WriteLine($"Running benchmark with {threads} threads for {durationSeconds} seconds...");
// Actual benchmark phase
var (totalEvaluations, evaluationTime, policyCounters) = RunBenchmarkPhase(threads, durationSeconds, policiesWithInputs, compiledPolicies, useSharedPolicies, isWarmup: false);
// Calculate throughput based on pure evaluation time (consistent with Rust benchmark)
var evalsPerSecond = totalEvaluations / evaluationTime.TotalSeconds;
var kelemsPerSecond = evalsPerSecond / 1000.0;
Console.WriteLine($"{groupName}/eval/{threads} threads");
Console.WriteLine($" time: [{evaluationTime.TotalMilliseconds:F2} ms]");
Console.WriteLine($" thrpt: [{kelemsPerSecond:F2} Kelem/s]");
// Clean up compiled policies if we created them
if (compiledPolicies != null)
{
foreach (var policy in compiledPolicies)
{
policy.Dispose();
}
}
// Verify that all policies were evaluated
var allEvaluated = policyCounters.Values.All(count => count > 0);
if (allEvaluated)
{
Console.WriteLine("✓ All policies were evaluated successfully");
}
else
{
Console.WriteLine("ERROR: Some policies were never evaluated successfully!");
}
}
private static (int totalEvaluations, TimeSpan evaluationTime, Dictionary<string, int> policyCounters) RunBenchmarkPhase(
int threads,
int durationSeconds,
List<(string Policy, string[] Inputs)> policiesWithInputs,
List<CompiledPolicy>? compiledPolicies,
bool useSharedPolicies,
bool isWarmup)
{
var barrier = new Barrier(threads);
var tasks = new Task[threads];
var policyCounters = new Dictionary<string, int>();
var evaluationTimes = new Dictionary<int, TimeSpan>();
var lockObject = new object();
var stopExecution = false;
// Initialize counters
foreach (var policyName in PolicyNames)
{
policyCounters[policyName] = 0;
}
var stopwatch = Stopwatch.StartNew();
for (int threadId = 0; threadId < threads; threadId++)
{
int tid = threadId;
tasks[threadId] = Task.Run(() =>
{
barrier.SignalAndWait();
int evaluationCount = 0;
var localEvaluationTime = TimeSpan.Zero;
while (!stopExecution)
{
// Use different policy for each iteration
int policyIdx = (tid + evaluationCount) % policiesWithInputs.Count;
var (policy, inputs) = policiesWithInputs[policyIdx];
// Use different input for the same policy based on iteration
int inputIdx = evaluationCount % inputs.Length;
var input = inputs[inputIdx];
try
{
// Measure only the evaluation call
var evalStopwatch = Stopwatch.StartNew();
if (useSharedPolicies)
{
var result = compiledPolicies![policyIdx].EvalWithInput(input);
}
else
{
// Compile policy in each iteration
var modules = new[] { new PolicyModule { Id = "policy.rego", Content = policy } };
var compiled = Compiler.CompilePolicyWithEntrypoint("{}", modules, "data.bench.allow");
var result = compiled.EvalWithInput(input);
compiled.Dispose();
}
evalStopwatch.Stop();
localEvaluationTime += evalStopwatch.Elapsed;
// Track successful evaluations (only during actual benchmark, not warmup)
if (!isWarmup)
{
lock (lockObject)
{
policyCounters[PolicyNames[policyIdx]]++;
}
}
}
catch (Exception)
{
// Ignore evaluation errors for benchmarking purposes
}
evaluationCount++;
}
// Store the actual evaluation time for this thread
if (!isWarmup)
{
lock (lockObject)
{
if (!evaluationTimes.ContainsKey(tid))
evaluationTimes[tid] = TimeSpan.Zero;
evaluationTimes[tid] = localEvaluationTime;
}
}
});
}
// Stop execution after the specified duration
Task.Delay(TimeSpan.FromSeconds(durationSeconds)).ContinueWith(_ => stopExecution = true);
Task.WaitAll(tasks);
stopwatch.Stop();
var totalEvaluations = policyCounters.Values.Sum();
var totalEvaluationTime = evaluationTimes.Values.Aggregate(TimeSpan.Zero, (sum, time) => sum + time);
// Use pure evaluation time (consistent with Rust benchmark)
var evaluationTime = totalEvaluationTime == TimeSpan.Zero ? stopwatch.Elapsed : totalEvaluationTime;
return (totalEvaluations, evaluationTime, policyCounters);
}
}
}

293
bindings/csharp/Benchmarks/EngineEvaluationBenchmark.cs Normal file
View File

@@ -0,0 +1,293 @@
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Regorus;
namespace Benchmarks
{
public class EngineEvaluationBenchmark
{
private static readonly string TestDataPath = Path.Combine(
Directory.GetCurrentDirectory(),
"..", "..", "..",
"benches", "evaluation", "test_data"
);
private static readonly (string PolicyFile, string[] InputFiles)[] PolicyInputFiles = new[]
{
("rbac_policy.rego", new[] { "rbac_input.json", "rbac_input2.json", "rbac_input3.json" }),
("api_access_policy.rego", new[] { "api_access_input.json", "api_access_input2.json", "api_access_input3.json" }),
("data_sensitivity_policy.rego", new[] { "data_sensitivity_input.json", "data_sensitivity_input2.json", "data_sensitivity_input3.json" }),
("time_based_policy.rego", new[] { "time_based_input.json", "time_based_input2.json", "time_based_input3.json" }),
("data_processing_policy.rego", new[] { "data_processing_input.json", "data_processing_input2.json", "data_processing_input3.json" }),
("azure_vm_policy.rego", new[] { "azure_vm_input.json", "azure_vm_input2.json", "azure_vm_input3.json" }),
("azure_storage_policy.rego", new[] { "azure_storage_input.json", "azure_storage_input2.json", "azure_storage_input3.json" }),
("azure_keyvault_policy.rego", new[] { "azure_keyvault_input.json", "azure_keyvault_input2.json", "azure_keyvault_input3.json" }),
("azure_nsg_policy.rego", new[] { "azure_nsg_input.json", "azure_nsg_input2.json", "azure_nsg_input3.json" })
};
private static readonly string[] PolicyNames = new[]
{
"rbac_policy",
"api_access_policy",
"data_sensitivity_policy",
"time_based_policy",
"data_processing_policy",
"azure_vm_policy",
"azure_storage_policy",
"azure_keyvault_policy",
"azure_nsg_policy"
};
private static List<(string Policy, string[] Inputs)> LoadPoliciesWithInputs()
{
var result = new List<(string Policy, string[] Inputs)>();
foreach (var (policyFile, inputFiles) in PolicyInputFiles)
{
var policyPath = Path.Combine(TestDataPath, "policies", policyFile);
var policy = File.ReadAllText(policyPath);
var inputs = inputFiles.Select(inputFile =>
{
var inputPath = Path.Combine(TestDataPath, "inputs", inputFile);
return File.ReadAllText(inputPath);
}).ToArray();
result.Add((policy, inputs));
}
return result;
}
private static List<Engine> PrepareClonedEngines()
{
var policiesWithInputs = LoadPoliciesWithInputs();
var engines = new List<Engine>();
foreach (var (policy, _) in policiesWithInputs)
{
var engine = new Engine();
engine.AddPolicy("policy.rego", policy);
// Warm up the engine to ensure it's fully prepared for evaluation
// This prevents each cloned engine from repeating preparation work
engine.SetInputJson("{}");
try
{
engine.EvalRule("data.bench.allow");
}
catch
{
// Ignore warmup errors
}
engines.Add(engine);
}
return engines;
}
public static void RunEngineEvaluationBenchmark()
{
var cpuCount = Environment.ProcessorCount;
var maxThreads = cpuCount * 2;
var threadCounts = new List<int> { 1, 2 };
// Add even numbers from 4 to maxThreads
for (int i = 4; i <= maxThreads; i += 2)
{
threadCounts.Add(i);
}
Console.WriteLine($"Running engine benchmark with max_threads: {maxThreads}");
Console.WriteLine($"Testing with thread counts: {string.Join(", ", threadCounts)}");
Console.WriteLine();
// Benchmark both cloned engines and fresh engines
var configurations = new[]
{
(true, "cloned_engines"),
(false, "fresh_engines")
};
foreach (var (useClonedEngines, groupName) in configurations)
{
Console.WriteLine($"=== {groupName} ===");
foreach (var threads in threadCounts)
{
RunEngineEvaluationBenchmark(threads, useClonedEngines, groupName);
}
Console.WriteLine();
}
}
public static void RunEngineEvaluationBenchmark(int threads, bool useClonedEngines, string groupName)
{
const int warmupSeconds = 3;
const int durationSeconds = 3;
var policiesWithInputs = LoadPoliciesWithInputs();
Console.WriteLine($"Warming up with {threads} threads for {warmupSeconds} seconds...");
// Warmup phase
var (_, _, _) = RunBenchmarkPhase(threads, warmupSeconds, policiesWithInputs, useClonedEngines, isWarmup: true);
Console.WriteLine($"Running benchmark with {threads} threads for {durationSeconds} seconds...");
// Actual benchmark phase
var (totalEvaluations, evaluationTime, policyCounters) = RunBenchmarkPhase(threads, durationSeconds, policiesWithInputs, useClonedEngines, isWarmup: false);
// Calculate throughput based on pure evaluation time (consistent with Rust benchmark)
var evalsPerSecond = totalEvaluations / evaluationTime.TotalSeconds;
var kelemsPerSecond = evalsPerSecond / 1000.0;
Console.WriteLine($"{groupName}/eval/{threads} threads");
Console.WriteLine($" time: [{evaluationTime.TotalMilliseconds:F2} ms]");
Console.WriteLine($" thrpt: [{kelemsPerSecond:F2} Kelem/s]");
// Verify that all policies were evaluated
var allEvaluated = policyCounters.Values.All(count => count > 0);
if (allEvaluated)
{
Console.WriteLine("✓ All policies were evaluated successfully");
}
else
{
Console.WriteLine("ERROR: Some policies were never evaluated successfully!");
}
}
private static (int totalEvaluations, TimeSpan evaluationTime, Dictionary<string, int> policyCounters) RunBenchmarkPhase(
int threads,
int durationSeconds,
List<(string Policy, string[] Inputs)> policiesWithInputs,
bool useClonedEngines,
bool isWarmup)
{
var barrier = new Barrier(threads);
var tasks = new Task[threads];
var policyCounters = new Dictionary<string, int>();
var evaluationTimes = new Dictionary<int, TimeSpan>();
var lockObject = new object();
var stopExecution = false;
// Initialize counters
foreach (var policyName in PolicyNames)
{
policyCounters[policyName] = 0;
}
// Pre-create engines if using cloned engines
List<Engine>? clonedEngines = null;
if (useClonedEngines)
{
clonedEngines = PrepareClonedEngines();
}
var stopwatch = Stopwatch.StartNew();
for (int threadId = 0; threadId < threads; threadId++)
{
int tid = threadId;
tasks[threadId] = Task.Run(() =>
{
barrier.SignalAndWait();
int evaluationCount = 0;
var localEvaluationTime = TimeSpan.Zero;
while (!stopExecution)
{
// Use different policy for each iteration
int policyIdx = (tid + evaluationCount) % policiesWithInputs.Count;
var (policy, inputs) = policiesWithInputs[policyIdx];
// Use different input for the same policy based on iteration
int inputIdx = evaluationCount % inputs.Length;
var input = inputs[inputIdx];
try
{
// Measure only the engine operations
var evalStopwatch = Stopwatch.StartNew();
Engine engine;
if (useClonedEngines)
{
engine = clonedEngines![policyIdx].Clone();
}
else
{
engine = new Engine();
engine.AddPolicy("policy.rego", policy);
}
engine.SetInputJson(input);
var result = engine.EvalRule("data.bench.allow");
engine.Dispose();
evalStopwatch.Stop();
localEvaluationTime += evalStopwatch.Elapsed;
// Track successful evaluations (only during actual benchmark, not warmup)
if (!isWarmup)
{
lock (lockObject)
{
policyCounters[PolicyNames[policyIdx]]++;
}
}
}
catch (Exception)
{
// Ignore evaluation errors for benchmarking purposes
}
evaluationCount++;
}
// Store the actual evaluation time for this thread
if (!isWarmup)
{
lock (lockObject)
{
if (!evaluationTimes.ContainsKey(tid))
evaluationTimes[tid] = TimeSpan.Zero;
evaluationTimes[tid] = localEvaluationTime;
}
}
});
}
// Stop execution after the specified duration
Task.Delay(TimeSpan.FromSeconds(durationSeconds)).ContinueWith(_ => stopExecution = true);
Task.WaitAll(tasks);
stopwatch.Stop();
// Clean up cloned engines if we created them
if (clonedEngines != null)
{
foreach (var engine in clonedEngines)
{
engine.Dispose();
}
}
var totalEvaluations = policyCounters.Values.Sum();
var totalEvaluationTime = evaluationTimes.Values.Aggregate(TimeSpan.Zero, (sum, time) => sum + time);
// Use pure evaluation time (consistent with Rust benchmark)
var evaluationTime = totalEvaluationTime == TimeSpan.Zero ? stopwatch.Elapsed : totalEvaluationTime;
return (totalEvaluations, evaluationTime, policyCounters);
}
}
}

36
bindings/csharp/Benchmarks/Program.cs Normal file
View File

@@ -0,0 +1,36 @@
using System;
namespace Benchmarks
{
class Program
{
static void Main(string[] args)
{
Console.WriteLine("=== Regorus C# Benchmarks ===\n");
try
{
Console.WriteLine("Running Engine Evaluation Benchmark...");
EngineEvaluationBenchmark.RunEngineEvaluationBenchmark();
}
catch (Exception ex)
{
Console.WriteLine($"Engine benchmark failed: {ex.Message}");
}
Console.WriteLine("\n" + new string('=', 80) + "\n");
try
{
Console.WriteLine("Running Compiled Policy Evaluation Benchmark...");
CompiledPolicyEvaluationBenchmark.RunCompiledPolicyEvaluationBenchmark();
}
catch (Exception ex)
{
Console.WriteLine($"Compiled policy benchmark failed: {ex.Message}");
}
Console.WriteLine("\n=== Benchmarks Complete ===");
}
}
}

103
bindings/csharp/Benchmarks/compiled_policy_evaluation_benchmark.md Normal file
View File

@@ -0,0 +1,103 @@
# Compiled Policy Evaluation Benchmark Results (C#/.NET)
## Test Environment
- **Platform**: Apple Silicon (M-Series)
- **CPU**: 16 cores
- **Architecture**: ARM64 (aarch64-apple-darwin)
- **.NET Version**: 8.0
- **Benchmark Framework**: Custom time-based benchmarking
- **Test Data**: 20,000 inputs per evaluation (distributed across threads)
- **Policy**: Complex authorization policy with nested rules
- **Warmup Duration**: 3 seconds per configuration
- **Evaluation Duration**: 3 seconds per configuration
## Benchmark Overview
The C# compiled policy evaluation benchmark tests Regorus compiled policy performance across multiple thread configurations (1-32 threads). It measures throughput (thousands of evaluations per second) for different combinations of compiled policy compilation strategies.
## Configuration Combinations
1. **Compiled Shared Policies**: All threads share pre-compiled policy instances - optimal for performance
2. **Compiled Per Iteration**: Each thread compiles the policy for each evaluation iteration
*Note: The C# implementation uses a simpler configuration model compared to Rust, which also varies input data handling (cloned vs fresh inputs). The C# benchmarks focus on compilation strategies with consistent input handling.*
## Performance Results
### Compiled Shared Policies (Best Performance)
| Threads | Total Evaluation Time (ms) | Throughput (Kelem/s) |
|--------:|---------------------------:|---------------------:|
| 1 | 2928.81 | 211 |
| 2 | 5892.53 | 146 |
| 4 | 11750.71 | 155 |
| 6 | 17686.92 | 134 |
| 8 | 23543.53 | 90 |
| 10 | 29503.80 | 72 |
| 12 | 35494.81 | 58 |
| 14 | 41408.36 | 50 |
| 16 | 47333.65 | 44 |
| 18 | 53050.24 | 38 |
| 20 | 58807.20 | 34 |
| 22 | 406022.45 | 32 |
| 24 | 65480.69 | 32 |
| 26 | 70952.34 | 30 |
| 28 | 72064.03 | 30 |
| 30 | 492405.74 | 27 |
| 32 | 81210.83 | 27 |
### Compiled Per Iteration
| Threads | Total Evaluation Time (ms) | Throughput (Kelem/s) |
|--------:|---------------------------:|---------------------:|
| 1 | 2984.00 | 39 |
| 2 | 5969.45 | 38 |
| 4 | 11948.28 | 32 |
| 6 | 17927.24 | 30 |
| 8 | 23889.01 | 24 |
| 10 | 29882.38 | 20 |
| 12 | 35865.06 | 18 |
| 14 | 41838.70 | 15 |
| 16 | 47800.92 | 14 |
| 18 | 53257.22 | 10 |
| 20 | 59596.93 | 11 |
| 22 | 435853.41 | 10 |
| 24 | 70870.86 | 9 |
| 26 | 76120.59 | 9 |
| 28 | 80717.51 | 8 |
| 30 | 544207.96 | 8 |
| 32 | 91540.91 | 7 |
## Analysis
The C# compiled policy benchmark demonstrates important performance characteristics:
1. **Compilation Strategy Impact**: Shared compiled policies significantly outperform per-iteration compilation (~5.4x at 1 thread)
2. **Scaling Patterns**:
- Best throughput achieved at 1 thread for shared policies
- Performance generally degrades with increased thread count
3. **Performance Hierarchy**:
- Shared compiled policies: Best performance (optimal configuration)
- Per-iteration compilation: ~82% reduction from optimal
4. **Compilation Overhead**: Per-iteration compilation creates substantial overhead, similar to fresh engine creation
5. **Thread Contention**: Significant performance degradation beyond 8 threads for both configurations
## Comparison with Rust Compiled Policy Evaluation
| Configuration | C# Performance (1 thread) | Rust Performance (1 thread) | Relative Performance |
|:-----------------|:----------------------------|:-----------------------------|---------------------:|
| Shared Policies | Best performance | Higher throughput | 0.40x-0.70x |
| Per-iteration | ~82% reduction from optimal | ~85% reduction from optimal | 0.47x-0.89x |
*Note: Rust benchmarks include additional input data variations (cloned vs fresh inputs) that are not present in the C# implementation.*
## Comparison with C# Engine Evaluation
| Configuration | Compiled Policy (1 thread) | Engine Evaluation (1 thread) | Performance Ratio |
|:---------------|:----------------------------|:------------------------------|------------------:|
| Optimal Config | Best performance | Slightly higher throughput | 0.96x |
## Performance Insights
1. **Compilation Efficiency**: Pre-compiled policies provide massive performance benefits over per-iteration compilation
2. **C# Performance Gap**: C# compiled policies achieve 40%-70% of Rust performance for shared policies
3. **Engine vs Compiled**: In C#, engine evaluation slightly outperforms compiled policies (96%-104% range)

99
bindings/csharp/Benchmarks/engine_evaluation_benchmark.md Normal file
View File

@@ -0,0 +1,99 @@
# Engine Evaluation Benchmark Results (C#/.NET)
## Test Environment
- **Platform**: Apple Silicon (M-Series)
- **CPU**: 16 cores
- **Architecture**: ARM64 (aarch64-apple-darwin)
- **.NET Version**: 8.0
- **Benchmark Framework**: Custom time-based benchmarking
- **Test Data**: 20,000 inputs per evaluation (distributed across threads)
- **Policy**: Complex authorization policy with nested rules
- **Warmup Duration**: 3 seconds per configuration
- **Evaluation Duration**: 3 seconds per configuration
## Benchmark Overview
The C# engine evaluation benchmark tests Regorus policy evaluation performance across multiple thread configurations (1-32 threads). It measures throughput (thousands of evaluations per second) for different combinations of engine reuse strategies.
## Configuration Combinations
1. **Cloned Engines**: Each thread uses its own cloned engine instance - optimal for performance
2. **Fresh Engines**: Each thread creates a new engine for each evaluation iteration
*Note: The C# implementation uses a simpler configuration model compared to Rust, which also varies input data handling (cloned vs fresh inputs). The C# benchmarks focus on engine reuse strategies with consistent input handling.*
## Performance Results
### Cloned Engines (Best Performance)
| Threads | Total Evaluation Time (ms) | Throughput (Kelem/s) |
|--------:|---------------------------:|---------------------:|
| 1 | 2930.56 | 219 |
| 2 | 5868.46 | 177 |
| 4 | 11771.01 | 146 |
| 6 | 17682.52 | 129 |
| 8 | 23633.65 | 78 |
| 10 | 29489.12 | 67 |
| 12 | 35455.23 | 57 |
| 14 | 41353.65 | 47 |
| 16 | 47378.91 | 42 |
| 18 | 52750.68 | 36 |
| 20 | 58131.31 | 35 |
| 22 | 62964.88 | 31 |
| 24 | 64337.75 | 34 |
| 26 | 70044.96 | 29 |
| 28 | 72553.98 | 28 |
| 30 | 79323.25 | 26 |
| 32 | 78624.33 | 26 |
### Fresh Engines
| Threads | Total Evaluation Time (ms) | Throughput (Kelem/s) |
|--------:|---------------------------:|---------------------:|
| 1 | 2985.49 | 41 |
| 2 | 5968.13 | 38 |
| 4 | 11942.10 | 34 |
| 6 | 17918.75 | 32 |
| 8 | 23873.57 | 25 |
| 10 | 29863.85 | 20 |
| 12 | 35823.98 | 19 |
| 14 | 41811.53 | 16 |
| 16 | 47819.89 | 14 |
| 18 | 53478.32 | 13 |
| 20 | 59191.93 | 12 |
| 22 | 64630.71 | 11 |
| 24 | 70215.54 | 10 |
| 26 | 75732.06 | 9 |
| 28 | 80897.59 | 9 |
| 30 | 949904.84 | 8 |
| 32 | 92592.64 | 8 |
## Analysis
The C# benchmark results demonstrate important performance characteristics:
1. **Engine Reuse Impact**: Cloned engines significantly outperform fresh engines (~5.3x at 1 thread)
2. **Scaling Patterns**:
- Best throughput achieved at 1 thread for both configurations
- Performance degrades with increased thread count due to contention
- Cloned engines show better relative scaling characteristics
3. **Performance Hierarchy**:
- Cloned engines: Best performance (optimal configuration)
- Fresh engines: ~81% reduction from optimal
4. **Thread Contention**: Significant performance drop beyond 8 threads, especially for fresh engines
5. **C# vs Rust Performance**: C# shows ~67% of Rust performance for equivalent cloned engine configuration
## Comparison with Rust Engine Evaluation
| Configuration | C# Performance (1 thread) | Rust Performance (1 thread) | Relative Performance |
|:---------------|:---------------------------|:-----------------------------|---------------------:|
| Cloned Engines | Best performance | Higher throughput | 0.67x-0.92x |
| Fresh Engines | ~81% reduction from optimal| ~87% reduction from optimal | 0.75x-0.95x |
*Note: Rust benchmarks include additional input data variations (cloned vs fresh inputs) that are not present in the C# implementation.*
## Performance Insights
1. **Engine Creation Overhead**: Fresh engine creation has massive performance impact in C# (~5.3x slower)
2. **Thread Scaling**: C# shows more significant thread contention than Rust implementation
3. **Memory Management**: .NET garbage collection may contribute to performance variations
4. **Interop Overhead**: C# bindings add measurable overhead compared to native Rust