mirror of
https://github.com/microsoft/regorus.git
synced 2026-08-05 02:16:11 +00:00
feat(memory): Allocator-backed global memory limits (#544)
Policy evaluation at scale needs to be able to set memory limits so that a bad policy does not hog memory or to ensure that policy evaluation itself does not use too much memory which could cause other components to suffer. This PR introduces capability to set and enforce global memory limits. It also lays the groundwork for enabling per evaluation limits in future. Once a global memory limit is set, Regorus maintains per thread counters to track memory activity (allocation, deallocation) of a thread. These counters are periodically flushed to global memory counters. Per thread counters avoid the contention that updating global counters on each alloc/free would cause. Policy evaluation periodically checks these counters and raises errors if allocated memory has exceeded the configured limit. Currently memory limit capability is exposed only to FFI and C#. Also update mimalloc to v2.2.6 Signed-off-by: Anand Krishnamoorthi <anakrish@microsoft.com>
This commit is contained in:
committed by
GitHub
parent
80686d6ed1
commit
fd59bb5a91
@@ -30,3 +30,33 @@ Once the workflow run completes, the generated Nuget can be downloaded by follow
|
||||
## Local
|
||||
|
||||
TODO
|
||||
|
||||
## Memory Usage Safeguards
|
||||
|
||||
The C# bindings expose allocator-backed memory tracking utilities via the static `Regorus.MemoryLimits` helper. Typical usage:
|
||||
|
||||
```csharp
|
||||
// Restrict total allocations to 128 MiB for the process
|
||||
Regorus.MemoryLimits.SetGlobalMemoryLimit(128 * 1024 * 1024);
|
||||
|
||||
// Optional: tune how frequently each thread flushes its allocation counters
|
||||
Regorus.MemoryLimits.SetThreadFlushThresholdOverride(256 * 1024);
|
||||
|
||||
// Engine operations throw InvalidOperationException with the allocator message if the budget is exceeded
|
||||
using var engine = new Regorus.Engine();
|
||||
var veryLargeJson = new string('x', 128 * 1024);
|
||||
try
|
||||
{
|
||||
engine.SetInputJson(veryLargeJson);
|
||||
}
|
||||
catch (InvalidOperationException ex)
|
||||
{
|
||||
Console.WriteLine($"Allocator reported: {ex.Message}");
|
||||
}
|
||||
|
||||
// Restore defaults once done
|
||||
Regorus.MemoryLimits.SetGlobalMemoryLimit(null);
|
||||
Regorus.MemoryLimits.SetThreadFlushThresholdOverride(null);
|
||||
```
|
||||
|
||||
See `bindings/csharp/Regorus.Tests/RegorusTests.cs` for scenario coverage and `bindings/csharp/TargetExampleApp/Program.cs` for end-to-end usage.
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
// Licensed under the MIT License.
|
||||
|
||||
using System;
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Nodes;
|
||||
using Microsoft.VisualStudio.TestTools.UnitTesting;
|
||||
using Regorus;
|
||||
@@ -11,6 +12,8 @@ namespace Regorus.Tests;
|
||||
[TestClass]
|
||||
public class RegorusTests
|
||||
{
|
||||
private static readonly object LimitLock = new();
|
||||
|
||||
[TestMethod]
|
||||
public void Basic_evaluation_succeeds()
|
||||
{
|
||||
@@ -215,6 +218,121 @@ public class RegorusTests
|
||||
Assert.AreEqual("b", parameters![0]["modifiers"][0]["name"].ToString());
|
||||
}
|
||||
|
||||
[TestMethod]
|
||||
public void Global_memory_limit_can_be_set_and_cleared()
|
||||
{
|
||||
lock (LimitLock)
|
||||
{
|
||||
using var guard = new MemoryLimitScope();
|
||||
|
||||
MemoryLimits.SetGlobalMemoryLimit(null);
|
||||
Assert.IsNull(MemoryLimits.GetGlobalMemoryLimit());
|
||||
|
||||
const ulong limit = 32 * 1024;
|
||||
MemoryLimits.SetGlobalMemoryLimit(limit);
|
||||
Assert.AreEqual(limit, MemoryLimits.GetGlobalMemoryLimit());
|
||||
|
||||
MemoryLimits.SetGlobalMemoryLimit(null);
|
||||
Assert.IsNull(MemoryLimits.GetGlobalMemoryLimit());
|
||||
}
|
||||
}
|
||||
|
||||
[TestMethod]
|
||||
public void Memory_limit_violations_surface_from_engine_calls()
|
||||
{
|
||||
lock (LimitLock)
|
||||
{
|
||||
using var guard = new MemoryLimitScope();
|
||||
using var engine = new Engine();
|
||||
|
||||
const ulong limit = 1;
|
||||
var payload = new string('x', 128 * 1024);
|
||||
|
||||
MemoryLimits.FlushThreadMemoryCounters();
|
||||
MemoryLimits.SetGlobalMemoryLimit(limit);
|
||||
|
||||
try
|
||||
{
|
||||
var ex = Assert.ThrowsException<InvalidOperationException>(
|
||||
() => engine.SetInputJson($"{{\"payload\":\"{payload}\"}}"));
|
||||
StringAssert.Contains(ex.Message, "execution exceeded memory limit");
|
||||
}
|
||||
finally
|
||||
{
|
||||
MemoryLimits.SetGlobalMemoryLimit(null);
|
||||
MemoryLimits.FlushThreadMemoryCounters();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
[TestMethod]
|
||||
public void Evaluation_fails_when_input_pushes_policy_over_global_limit()
|
||||
{
|
||||
lock (LimitLock)
|
||||
{
|
||||
using var guard = new MemoryLimitScope();
|
||||
using var engine = new Engine();
|
||||
|
||||
const string policy = """
|
||||
package memorylimit
|
||||
|
||||
import rego.v1
|
||||
|
||||
stretched := concat("", [input.block | numbers.range(0, input.repeat - 1)[_]])
|
||||
""";
|
||||
|
||||
engine.AddPolicy("memorylimit.rego", policy);
|
||||
|
||||
MemoryLimits.FlushThreadMemoryCounters();
|
||||
const ulong limit = 4 * 1024 * 1024;
|
||||
MemoryLimits.SetGlobalMemoryLimit(limit);
|
||||
|
||||
var block = new string('x', 16 * 1024);
|
||||
|
||||
var smallInput = JsonSerializer.Serialize(new { block, repeat = 16 });
|
||||
engine.SetInputJson(smallInput);
|
||||
var smallResult = engine.EvalRule("data.memorylimit.stretched");
|
||||
Assert.IsNotNull(smallResult);
|
||||
var stretched = JsonSerializer.Deserialize<string>(smallResult);
|
||||
Assert.IsNotNull(stretched, "Policy should return a string result.");
|
||||
Assert.AreEqual(block.Length * 16, stretched!.Length, "Policy should expand the payload under the limit.");
|
||||
|
||||
var largeInput = JsonSerializer.Serialize(new { block, repeat = 4096 });
|
||||
engine.SetInputJson(largeInput);
|
||||
|
||||
var ex = Assert.ThrowsException<InvalidOperationException>(
|
||||
() => engine.EvalRule("data.memorylimit.stretched"));
|
||||
StringAssert.Contains(ex.Message, "execution exceeded memory limit");
|
||||
}
|
||||
}
|
||||
|
||||
[TestMethod]
|
||||
public void Thread_flush_threshold_roundtrips()
|
||||
{
|
||||
lock (LimitLock)
|
||||
{
|
||||
var original = MemoryLimits.GetThreadMemoryFlushThreshold();
|
||||
try
|
||||
{
|
||||
const ulong threshold = 256 * 1024;
|
||||
MemoryLimits.SetThreadFlushThresholdOverride(threshold);
|
||||
Assert.AreEqual(threshold, MemoryLimits.GetThreadMemoryFlushThreshold());
|
||||
|
||||
MemoryLimits.SetThreadFlushThresholdOverride(null);
|
||||
var restored = MemoryLimits.GetThreadMemoryFlushThreshold();
|
||||
Assert.IsTrue(restored.HasValue, "Clearing override should restore allocator default.");
|
||||
if (original.HasValue)
|
||||
{
|
||||
Assert.AreEqual(original, restored);
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
MemoryLimits.SetThreadFlushThresholdOverride(original);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
[TestMethod]
|
||||
public void SetInputJson_has_negligible_allocations_after_warmup()
|
||||
{
|
||||
@@ -253,4 +371,20 @@ public class RegorusTests
|
||||
$"Expected ≤512 B/op after warmup, but observed {bytesPerOp:F2} B/op (total {allocated} bytes)."
|
||||
);
|
||||
}
|
||||
|
||||
private sealed class MemoryLimitScope : IDisposable
|
||||
{
|
||||
private readonly ulong? _originalLimit;
|
||||
|
||||
public MemoryLimitScope()
|
||||
{
|
||||
_originalLimit = MemoryLimits.GetGlobalMemoryLimit();
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
MemoryLimits.SetGlobalMemoryLimit(_originalLimit);
|
||||
MemoryLimits.FlushThreadMemoryCounters();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -2,7 +2,6 @@
|
||||
// Licensed under the MIT License.
|
||||
|
||||
using System;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using System.Threading;
|
||||
using Regorus.Internal;
|
||||
@@ -146,36 +145,23 @@ namespace Regorus
|
||||
throw new ObjectDisposedException(nameof(CompiledPolicy));
|
||||
}
|
||||
|
||||
private string? StringFromUTF8(IntPtr ptr)
|
||||
{
|
||||
#if NETSTANDARD2_1
|
||||
return System.Runtime.InteropServices.Marshal.PtrToStringUTF8(ptr);
|
||||
#else
|
||||
int len = 0;
|
||||
while (System.Runtime.InteropServices.Marshal.ReadByte(ptr, len) != 0) { ++len; }
|
||||
byte[] buffer = new byte[len];
|
||||
System.Runtime.InteropServices.Marshal.Copy(ptr, buffer, 0, buffer.Length);
|
||||
return Encoding.UTF8.GetString(buffer);
|
||||
#endif
|
||||
}
|
||||
|
||||
private string? CheckAndDropResult(Internal.RegorusResult result)
|
||||
{
|
||||
try
|
||||
{
|
||||
if (result.status != Internal.RegorusStatus.Ok)
|
||||
{
|
||||
var message = StringFromUTF8((IntPtr)result.error_message);
|
||||
var message = Internal.Utf8Marshaller.FromUtf8(result.error_message);
|
||||
throw result.status.CreateException(message);
|
||||
}
|
||||
|
||||
return result.data_type switch
|
||||
{
|
||||
Internal.RegorusDataType.String => StringFromUTF8((IntPtr)result.output),
|
||||
Internal.RegorusDataType.String => Internal.Utf8Marshaller.FromUtf8(result.output),
|
||||
Internal.RegorusDataType.Boolean => result.bool_value.ToString().ToLowerInvariant(),
|
||||
Internal.RegorusDataType.Integer => result.int_value.ToString(),
|
||||
Internal.RegorusDataType.None => null,
|
||||
_ => StringFromUTF8((IntPtr)result.output)
|
||||
_ => Internal.Utf8Marshaller.FromUtf8(result.output)
|
||||
};
|
||||
}
|
||||
finally
|
||||
|
||||
@@ -4,8 +4,6 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Text;
|
||||
using Regorus.Internal;
|
||||
|
||||
#nullable enable
|
||||
@@ -157,26 +155,13 @@ namespace Regorus
|
||||
}
|
||||
}
|
||||
|
||||
private static string? StringFromUTF8(IntPtr ptr)
|
||||
{
|
||||
#if NETSTANDARD2_1
|
||||
return System.Runtime.InteropServices.Marshal.PtrToStringUTF8(ptr);
|
||||
#else
|
||||
int len = 0;
|
||||
while (System.Runtime.InteropServices.Marshal.ReadByte(ptr, len) != 0) { ++len; }
|
||||
byte[] buffer = new byte[len];
|
||||
System.Runtime.InteropServices.Marshal.Copy(ptr, buffer, 0, buffer.Length);
|
||||
return Encoding.UTF8.GetString(buffer);
|
||||
#endif
|
||||
}
|
||||
|
||||
private static CompiledPolicy GetCompiledPolicyResult(Internal.RegorusResult result)
|
||||
{
|
||||
try
|
||||
{
|
||||
if (result.status != Internal.RegorusStatus.Ok)
|
||||
{
|
||||
var message = StringFromUTF8((IntPtr)result.error_message);
|
||||
var message = Utf8Marshaller.FromUtf8(result.error_message);
|
||||
throw result.status.CreateException(message);
|
||||
}
|
||||
|
||||
|
||||
@@ -2,8 +2,6 @@
|
||||
// Licensed under the MIT License.
|
||||
|
||||
using System;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Text;
|
||||
using Regorus.Internal;
|
||||
|
||||
|
||||
@@ -357,32 +355,24 @@ namespace Regorus
|
||||
});
|
||||
}
|
||||
|
||||
string? StringFromUTF8(IntPtr ptr)
|
||||
{
|
||||
|
||||
#if NETSTANDARD2_1
|
||||
return System.Runtime.InteropServices.Marshal.PtrToStringUTF8(ptr);
|
||||
#else
|
||||
int len = 0;
|
||||
while (Marshal.ReadByte(ptr, len) != 0) { ++len; }
|
||||
byte[] buffer = new byte[len];
|
||||
Marshal.Copy(ptr, buffer, 0, buffer.Length);
|
||||
return Encoding.UTF8.GetString(buffer);
|
||||
#endif
|
||||
}
|
||||
|
||||
string? CheckAndDropResult(Regorus.Internal.RegorusResult result)
|
||||
{
|
||||
try
|
||||
{
|
||||
if (result.status != Regorus.Internal.RegorusStatus.Ok)
|
||||
{
|
||||
var message = StringFromUTF8((IntPtr)result.error_message);
|
||||
var message = Utf8Marshaller.FromUtf8(result.error_message);
|
||||
throw result.status.CreateException(message);
|
||||
}
|
||||
|
||||
var output = result.output is not null ? StringFromUTF8((IntPtr)result.output) : null;
|
||||
return output ?? string.Empty;
|
||||
return result.data_type switch
|
||||
{
|
||||
Regorus.Internal.RegorusDataType.String => Utf8Marshaller.FromUtf8(result.output),
|
||||
Regorus.Internal.RegorusDataType.Boolean => result.bool_value.ToString().ToLowerInvariant(),
|
||||
Regorus.Internal.RegorusDataType.Integer => result.int_value.ToString(),
|
||||
Regorus.Internal.RegorusDataType.None => null,
|
||||
_ => Utf8Marshaller.FromUtf8(result.output)
|
||||
};
|
||||
}
|
||||
finally
|
||||
{
|
||||
|
||||
@@ -0,0 +1,126 @@
|
||||
// Copyright (c) Microsoft Corporation.
|
||||
// Licensed under the MIT License.
|
||||
|
||||
using System;
|
||||
using Regorus.Internal;
|
||||
|
||||
#nullable enable
|
||||
namespace Regorus
|
||||
{
|
||||
/// <summary>
|
||||
/// Helpers for configuring and inspecting Regorus memory limits via the native allocator bridge.
|
||||
/// </summary>
|
||||
public static class MemoryLimits
|
||||
{
|
||||
/// <summary>
|
||||
/// Configure the process-wide global memory limit in bytes. Pass <c>null</c> to remove the limit.
|
||||
/// </summary>
|
||||
/// <param name="bytes">Maximum number of bytes the allocator may reserve before signalling an error.</param>
|
||||
public static void SetGlobalMemoryLimit(ulong? bytes)
|
||||
{
|
||||
var result = API.regorus_set_global_memory_limit(bytes ?? 0, bytes.HasValue);
|
||||
EnsureSuccess(result, nameof(SetGlobalMemoryLimit));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Returns the currently configured global memory limit, if any.
|
||||
/// </summary>
|
||||
public static ulong? GetGlobalMemoryLimit()
|
||||
{
|
||||
var result = API.regorus_get_global_memory_limit();
|
||||
return ExtractOptionalU64(result, "Failed to get global memory limit");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Forces the allocator to flush this thread's pending counters into the global aggregates.
|
||||
/// </summary>
|
||||
public static void FlushThreadMemoryCounters()
|
||||
{
|
||||
var result = API.regorus_flush_thread_memory_counters();
|
||||
EnsureSuccess(result, nameof(FlushThreadMemoryCounters));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Immediately checks the global memory limit and throws if the allocator reports exhaustion.
|
||||
/// </summary>
|
||||
public static void CheckGlobalMemoryLimit()
|
||||
{
|
||||
var result = API.regorus_check_global_memory_limit();
|
||||
EnsureSuccess(result, nameof(CheckGlobalMemoryLimit));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Override the per-thread automatic flush threshold in bytes. Pass <c>null</c> to restore the default.
|
||||
/// </summary>
|
||||
public static void SetThreadFlushThresholdOverride(ulong? bytes)
|
||||
{
|
||||
var result = API.regorus_set_thread_flush_threshold_override(bytes ?? 0, bytes.HasValue);
|
||||
EnsureSuccess(result, nameof(SetThreadFlushThresholdOverride));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Returns the per-thread flush threshold, if automatic flushing is enabled.
|
||||
/// </summary>
|
||||
public static ulong? GetThreadMemoryFlushThreshold()
|
||||
{
|
||||
var result = API.regorus_get_thread_memory_flush_threshold();
|
||||
return ExtractOptionalU64(result, "Failed to get thread memory flush threshold");
|
||||
}
|
||||
|
||||
private static unsafe ulong? ExtractOptionalU64(RegorusResult result, string errorContext)
|
||||
{
|
||||
try
|
||||
{
|
||||
if (result.status != RegorusStatus.Ok)
|
||||
{
|
||||
var message = Utf8Marshaller.FromUtf8(result.error_message) ?? $"{errorContext}: native call failed";
|
||||
throw result.status.CreateException(message);
|
||||
}
|
||||
|
||||
if (!result.bool_value)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
if (result.data_type != RegorusDataType.Integer)
|
||||
{
|
||||
throw new InvalidOperationException(
|
||||
$"{errorContext}: native call returned {result.data_type} ({(int)result.data_type}) with bool_value={result.bool_value}"
|
||||
);
|
||||
}
|
||||
|
||||
if (result.int_value < 0)
|
||||
{
|
||||
throw new OverflowException($"{errorContext}: native value was negative ({result.int_value})");
|
||||
}
|
||||
|
||||
return (ulong)result.int_value;
|
||||
}
|
||||
finally
|
||||
{
|
||||
API.regorus_result_drop(result);
|
||||
}
|
||||
}
|
||||
|
||||
private static void EnsureSuccess(RegorusResult result, string operation)
|
||||
{
|
||||
try
|
||||
{
|
||||
if (result.status != RegorusStatus.Ok)
|
||||
{
|
||||
string? message;
|
||||
unsafe
|
||||
{
|
||||
message = Utf8Marshaller.FromUtf8(result.error_message);
|
||||
}
|
||||
|
||||
throw result.status.CreateException(message);
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
API.regorus_result_drop(result);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -28,6 +28,46 @@ namespace Regorus.Internal
|
||||
|
||||
#endregion
|
||||
|
||||
#region Memory Limit Methods
|
||||
|
||||
/// <summary>
|
||||
/// Set the global memory limit. Pass hasLimit=false to clear the limit.
|
||||
/// </summary>
|
||||
[DllImport(LibraryName, EntryPoint = "regorus_set_global_memory_limit", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true)]
|
||||
internal static extern RegorusResult regorus_set_global_memory_limit(ulong limit, [MarshalAs(UnmanagedType.U1)] bool hasLimit);
|
||||
|
||||
/// <summary>
|
||||
/// Get the current global memory limit. bool_value indicates whether a limit is set.
|
||||
/// </summary>
|
||||
[DllImport(LibraryName, EntryPoint = "regorus_get_global_memory_limit", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true)]
|
||||
internal static extern RegorusResult regorus_get_global_memory_limit();
|
||||
|
||||
/// <summary>
|
||||
/// Check the global memory limit immediately.
|
||||
/// </summary>
|
||||
[DllImport(LibraryName, EntryPoint = "regorus_check_global_memory_limit", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true)]
|
||||
internal static extern RegorusResult regorus_check_global_memory_limit();
|
||||
|
||||
/// <summary>
|
||||
/// Flush the current thread's pending allocation counters into global aggregates.
|
||||
/// </summary>
|
||||
[DllImport(LibraryName, EntryPoint = "regorus_flush_thread_memory_counters", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true)]
|
||||
internal static extern RegorusResult regorus_flush_thread_memory_counters();
|
||||
|
||||
/// <summary>
|
||||
/// Set the per-thread flush threshold override. Pass hasThreshold=false to restore defaults.
|
||||
/// </summary>
|
||||
[DllImport(LibraryName, EntryPoint = "regorus_set_thread_flush_threshold_override", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true)]
|
||||
internal static extern RegorusResult regorus_set_thread_flush_threshold_override(ulong threshold, [MarshalAs(UnmanagedType.U1)] bool hasThreshold);
|
||||
|
||||
/// <summary>
|
||||
/// Get the per-thread flush threshold. bool_value indicates whether a threshold is configured.
|
||||
/// </summary>
|
||||
[DllImport(LibraryName, EntryPoint = "regorus_get_thread_memory_flush_threshold", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true)]
|
||||
internal static extern RegorusResult regorus_get_thread_memory_flush_threshold();
|
||||
|
||||
#endregion
|
||||
|
||||
#region Engine Methods
|
||||
|
||||
/// <summary>
|
||||
@@ -520,6 +560,7 @@ namespace Regorus.Internal
|
||||
/// Boolean value.
|
||||
/// Valid when data_type is Boolean.
|
||||
/// </summary>
|
||||
[MarshalAs(UnmanagedType.I1)]
|
||||
public bool bool_value;
|
||||
/// <summary>
|
||||
/// Integer value.
|
||||
|
||||
@@ -2,7 +2,6 @@
|
||||
// Licensed under the MIT License.
|
||||
|
||||
using System;
|
||||
using System.Text;
|
||||
using Regorus.Internal;
|
||||
|
||||
#nullable enable
|
||||
@@ -220,36 +219,23 @@ namespace Regorus
|
||||
CheckAndDropResult(Internal.API.regorus_effect_schema_clear());
|
||||
}
|
||||
|
||||
private static string? StringFromUTF8(IntPtr ptr)
|
||||
{
|
||||
#if NETSTANDARD2_1
|
||||
return System.Runtime.InteropServices.Marshal.PtrToStringUTF8(ptr);
|
||||
#else
|
||||
int len = 0;
|
||||
while (System.Runtime.InteropServices.Marshal.ReadByte(ptr, len) != 0) { ++len; }
|
||||
byte[] buffer = new byte[len];
|
||||
System.Runtime.InteropServices.Marshal.Copy(ptr, buffer, 0, buffer.Length);
|
||||
return Encoding.UTF8.GetString(buffer);
|
||||
#endif
|
||||
}
|
||||
|
||||
private static string? CheckAndDropResult(Internal.RegorusResult result)
|
||||
{
|
||||
try
|
||||
{
|
||||
if (result.status != Internal.RegorusStatus.Ok)
|
||||
{
|
||||
var message = StringFromUTF8((IntPtr)result.error_message);
|
||||
var message = Utf8Marshaller.FromUtf8(result.error_message);
|
||||
throw result.status.CreateException(message);
|
||||
}
|
||||
|
||||
return result.data_type switch
|
||||
{
|
||||
Internal.RegorusDataType.String => StringFromUTF8((IntPtr)result.output),
|
||||
Internal.RegorusDataType.String => Utf8Marshaller.FromUtf8(result.output),
|
||||
Internal.RegorusDataType.Boolean => result.bool_value.ToString().ToLowerInvariant(),
|
||||
Internal.RegorusDataType.Integer => result.int_value.ToString(),
|
||||
Internal.RegorusDataType.None => null,
|
||||
_ => StringFromUTF8((IntPtr)result.output)
|
||||
_ => Utf8Marshaller.FromUtf8(result.output)
|
||||
};
|
||||
}
|
||||
finally
|
||||
@@ -264,7 +250,7 @@ namespace Regorus
|
||||
{
|
||||
if (result.status != Internal.RegorusStatus.Ok)
|
||||
{
|
||||
var message = StringFromUTF8((IntPtr)result.error_message);
|
||||
var message = Utf8Marshaller.FromUtf8(result.error_message);
|
||||
throw result.status.CreateException(message);
|
||||
}
|
||||
|
||||
@@ -282,7 +268,7 @@ namespace Regorus
|
||||
{
|
||||
if (result.status != Internal.RegorusStatus.Ok)
|
||||
{
|
||||
var message = StringFromUTF8((IntPtr)result.error_message);
|
||||
var message = Utf8Marshaller.FromUtf8(result.error_message);
|
||||
throw result.status.CreateException(message);
|
||||
}
|
||||
|
||||
|
||||
@@ -17,7 +17,7 @@ namespace Regorus.Internal
|
||||
{
|
||||
RegorusStatus.Panic => new InvalidOperationException($"Regorus engine panicked: {details}"),
|
||||
RegorusStatus.Poisoned => new InvalidOperationException($"Regorus engine is poisoned: {details}"),
|
||||
_ => new Exception(details),
|
||||
_ => new InvalidOperationException(details),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,7 +2,6 @@
|
||||
// Licensed under the MIT License.
|
||||
|
||||
using System;
|
||||
using System.Text;
|
||||
using Regorus.Internal;
|
||||
|
||||
#nullable enable
|
||||
@@ -115,36 +114,23 @@ namespace Regorus
|
||||
}
|
||||
}
|
||||
|
||||
private static string? StringFromUTF8(IntPtr ptr)
|
||||
{
|
||||
#if NETSTANDARD2_1
|
||||
return System.Runtime.InteropServices.Marshal.PtrToStringUTF8(ptr);
|
||||
#else
|
||||
int len = 0;
|
||||
while (System.Runtime.InteropServices.Marshal.ReadByte(ptr, len) != 0) { ++len; }
|
||||
byte[] buffer = new byte[len];
|
||||
System.Runtime.InteropServices.Marshal.Copy(ptr, buffer, 0, buffer.Length);
|
||||
return Encoding.UTF8.GetString(buffer);
|
||||
#endif
|
||||
}
|
||||
|
||||
private static string? CheckAndDropResult(Internal.RegorusResult result)
|
||||
{
|
||||
try
|
||||
{
|
||||
if (result.status != Internal.RegorusStatus.Ok)
|
||||
{
|
||||
var message = StringFromUTF8((IntPtr)result.error_message);
|
||||
var message = Utf8Marshaller.FromUtf8(result.error_message);
|
||||
throw result.status.CreateException(message);
|
||||
}
|
||||
|
||||
return result.data_type switch
|
||||
{
|
||||
Internal.RegorusDataType.String => StringFromUTF8((IntPtr)result.output),
|
||||
Internal.RegorusDataType.String => Utf8Marshaller.FromUtf8(result.output),
|
||||
Internal.RegorusDataType.Boolean => result.bool_value.ToString().ToLowerInvariant(),
|
||||
Internal.RegorusDataType.Integer => result.int_value.ToString(),
|
||||
Internal.RegorusDataType.None => null,
|
||||
_ => StringFromUTF8((IntPtr)result.output)
|
||||
_ => Utf8Marshaller.FromUtf8(result.output)
|
||||
};
|
||||
}
|
||||
finally
|
||||
@@ -159,7 +145,7 @@ namespace Regorus
|
||||
{
|
||||
if (result.status != Internal.RegorusStatus.Ok)
|
||||
{
|
||||
var message = StringFromUTF8((IntPtr)result.error_message);
|
||||
var message = Utf8Marshaller.FromUtf8(result.error_message);
|
||||
throw result.status.CreateException(message);
|
||||
}
|
||||
|
||||
@@ -177,7 +163,7 @@ namespace Regorus
|
||||
{
|
||||
if (result.status != Internal.RegorusStatus.Ok)
|
||||
{
|
||||
var message = StringFromUTF8((IntPtr)result.error_message);
|
||||
var message = Utf8Marshaller.FromUtf8(result.error_message);
|
||||
throw result.status.CreateException(message);
|
||||
}
|
||||
|
||||
|
||||
@@ -150,5 +150,48 @@ namespace Regorus.Internal
|
||||
{
|
||||
return new PinnedUtf8(value);
|
||||
}
|
||||
|
||||
internal static unsafe string? FromUtf8(byte* pointer)
|
||||
{
|
||||
if (pointer is null)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
#if NETSTANDARD2_1
|
||||
return Marshal.PtrToStringUTF8((IntPtr)pointer);
|
||||
#else
|
||||
var intPtr = (IntPtr)pointer;
|
||||
var length = 0;
|
||||
while (Marshal.ReadByte(intPtr, length) != 0)
|
||||
{
|
||||
length++;
|
||||
}
|
||||
|
||||
if (length == 0)
|
||||
{
|
||||
return string.Empty;
|
||||
}
|
||||
|
||||
var buffer = ArrayPool<byte>.Shared.Rent(length);
|
||||
try
|
||||
{
|
||||
Marshal.Copy(intPtr, buffer, 0, length);
|
||||
return Encoding.UTF8.GetString(buffer, 0, length);
|
||||
}
|
||||
finally
|
||||
{
|
||||
ArrayPool<byte>.Shared.Return(buffer);
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
internal static string? FromUtf8(IntPtr pointer)
|
||||
{
|
||||
unsafe
|
||||
{
|
||||
return FromUtf8((byte*)pointer);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -32,14 +32,14 @@ allow if {
|
||||
|
||||
# Allow network security groups with proper inbound rules
|
||||
allow if {
|
||||
input.type == ""Microsoft.Network/networkSecurityGroups""
|
||||
count([rule |
|
||||
rule := input.properties.securityRules[_]
|
||||
rule.properties.direction == ""Inbound""
|
||||
rule.properties.access == ""Allow""
|
||||
rule.properties.sourceAddressPrefix == ""*""
|
||||
rule.properties.destinationPortRange in [parameters.allowedPorts]
|
||||
]) == 0
|
||||
input.type == ""Microsoft.Network/networkSecurityGroups""
|
||||
count([rule |
|
||||
rule := input.properties.securityRules[_]
|
||||
rule.properties.direction == ""Inbound""
|
||||
rule.properties.access == ""Allow""
|
||||
rule.properties.sourceAddressPrefix == ""*""
|
||||
rule.properties.destinationPortRange in [parameters.allowedPorts]
|
||||
]) == 0
|
||||
}";
|
||||
|
||||
private const string AZURE_STORAGE_POLICY_ASSIGNMENT = @"
|
||||
@@ -52,42 +52,42 @@ parameters.allowedPorts = [""22"", ""3389""]";
|
||||
|
||||
// Test data constants
|
||||
private const string COMPLIANT_STORAGE_ACCOUNT = @"{
|
||||
""type"": ""Microsoft.Storage/storageAccounts"",
|
||||
""name"": ""compliantstorageacct"",
|
||||
""location"": ""eastus"",
|
||||
""kind"": ""StorageV2"",
|
||||
""properties"": {
|
||||
""supportsHttpsTrafficOnly"": true,
|
||||
""minimumTlsVersion"": ""TLS1_2"",
|
||||
""allowBlobPublicAccess"": false,
|
||||
""encryption"": {
|
||||
""services"": {
|
||||
""blob"": { ""enabled"": true },
|
||||
""file"": { ""enabled"": true }
|
||||
}
|
||||
""type"": ""Microsoft.Storage/storageAccounts"",
|
||||
""name"": ""compliantstorageacct"",
|
||||
""location"": ""eastus"",
|
||||
""kind"": ""StorageV2"",
|
||||
""properties"": {
|
||||
""supportsHttpsTrafficOnly"": true,
|
||||
""minimumTlsVersion"": ""TLS1_2"",
|
||||
""allowBlobPublicAccess"": false,
|
||||
""encryption"": {
|
||||
""services"": {
|
||||
""blob"": { ""enabled"": true },
|
||||
""file"": { ""enabled"": true }
|
||||
}
|
||||
}
|
||||
},
|
||||
""tags"": {
|
||||
""environment"": ""production""
|
||||
}
|
||||
},
|
||||
""tags"": {
|
||||
""environment"": ""production""
|
||||
}
|
||||
}";
|
||||
|
||||
private const string NON_COMPLIANT_STORAGE_ACCOUNT = @"{
|
||||
""type"": ""Microsoft.Storage/storageAccounts"",
|
||||
""name"": ""insecurestorageacct"",
|
||||
""location"": ""westus"",
|
||||
""kind"": ""Storage"",
|
||||
""properties"": {
|
||||
""supportsHttpsTrafficOnly"": false,
|
||||
""minimumTlsVersion"": ""TLS1_0"",
|
||||
""allowBlobPublicAccess"": true,
|
||||
""encryption"": {
|
||||
""services"": {
|
||||
""blob"": { ""enabled"": false },
|
||||
""file"": { ""enabled"": false }
|
||||
}
|
||||
""type"": ""Microsoft.Storage/storageAccounts"",
|
||||
""name"": ""insecurestorageacct"",
|
||||
""location"": ""westus"",
|
||||
""kind"": ""Storage"",
|
||||
""properties"": {
|
||||
""supportsHttpsTrafficOnly"": false,
|
||||
""minimumTlsVersion"": ""TLS1_0"",
|
||||
""allowBlobPublicAccess"": true,
|
||||
""encryption"": {
|
||||
""services"": {
|
||||
""blob"": { ""enabled"": false },
|
||||
""file"": { ""enabled"": false }
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}";
|
||||
|
||||
static void Main(string[] args)
|
||||
@@ -96,6 +96,7 @@ parameters.allowedPorts = [""22"", ""3389""]";
|
||||
|
||||
try
|
||||
{
|
||||
DemonstrateMemoryLimitHelpers();
|
||||
DemonstrateTargetFunctionality();
|
||||
Console.WriteLine("\n=== Target demonstration completed successfully! ===");
|
||||
}
|
||||
@@ -213,6 +214,45 @@ parameters.allowedPorts = [""22"", ""3389""]";
|
||||
Console.WriteLine("✓ No locks required - CompiledPolicy is thread-safe!");
|
||||
}
|
||||
|
||||
static void DemonstrateMemoryLimitHelpers()
|
||||
{
|
||||
Console.WriteLine("REGORUS MEMORY LIMIT UTILITIES");
|
||||
Console.WriteLine("==============================");
|
||||
|
||||
var originalLimit = Regorus.MemoryLimits.GetGlobalMemoryLimit();
|
||||
var originalThreshold = Regorus.MemoryLimits.GetThreadMemoryFlushThreshold();
|
||||
|
||||
try
|
||||
{
|
||||
const ulong demoLimit = 64 * 1024 * 1024;
|
||||
Regorus.MemoryLimits.SetGlobalMemoryLimit(demoLimit);
|
||||
Console.WriteLine($"✓ Global memory limit set to {Regorus.MemoryLimits.GetGlobalMemoryLimit():N0} bytes");
|
||||
|
||||
const ulong flushThreshold = 256 * 1024;
|
||||
Regorus.MemoryLimits.SetThreadFlushThresholdOverride(flushThreshold);
|
||||
Console.WriteLine($"✓ Thread flush threshold configured for {Regorus.MemoryLimits.GetThreadMemoryFlushThreshold():N0} bytes");
|
||||
|
||||
Console.WriteLine("Attempting an allocation that exceeds a 1 byte budget...");
|
||||
Regorus.MemoryLimits.SetGlobalMemoryLimit(1);
|
||||
|
||||
using var engine = new Regorus.Engine();
|
||||
var payload = new string('x', 128 * 1024);
|
||||
engine.SetInputJson($"{{\"payload\":\"{payload}\"}}");
|
||||
}
|
||||
catch (InvalidOperationException error)
|
||||
{
|
||||
Console.WriteLine($"✗ Memory limit exceeded: {error.Message}");
|
||||
}
|
||||
finally
|
||||
{
|
||||
Regorus.MemoryLimits.SetThreadFlushThresholdOverride(null);
|
||||
Regorus.MemoryLimits.SetGlobalMemoryLimit(originalLimit);
|
||||
Regorus.MemoryLimits.FlushThreadMemoryCounters();
|
||||
}
|
||||
|
||||
Console.WriteLine();
|
||||
}
|
||||
|
||||
static void DemonstratePolicyInfo(Regorus.CompiledPolicy compiledPolicy)
|
||||
{
|
||||
Console.WriteLine("Getting policy metadata using GetPolicyInfo()...");
|
||||
|
||||
@@ -11,10 +11,15 @@
|
||||
<PropertyGroup>
|
||||
<!-- If the environment variable is set (such as in a Github Action run), append the suffix to the version number -->
|
||||
<RegorusPackageVersionSuffix Condition="'$(VersionSuffix)' != ''">-$(VersionSuffix)</RegorusPackageVersionSuffix>
|
||||
<UseLocalRegorus Condition="'$(UseLocalRegorus)' == ''">false</UseLocalRegorus>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Regorus" Version="0.8.0$(RegorusPackageVersionSuffix)"/>
|
||||
<ItemGroup Condition="'$(UseLocalRegorus)' == 'true'">
|
||||
<ProjectReference Include="../Regorus/Regorus.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup Condition="'$(UseLocalRegorus)' != 'true'">
|
||||
<PackageReference Include="Regorus" Version="0.8.0$(RegorusPackageVersionSuffix)" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
|
||||
@@ -10,7 +10,17 @@
|
||||
<LangVersion>10.0</LangVersion>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="regorus" Version="0.8.0"/>
|
||||
<PropertyGroup>
|
||||
<!-- Allow CI to append the version suffix for locally built packages -->
|
||||
<RegorusPackageVersionSuffix Condition="'$(VersionSuffix)' != ''">-$(VersionSuffix)</RegorusPackageVersionSuffix>
|
||||
<UseLocalRegorus Condition="'$(UseLocalRegorus)' == ''">false</UseLocalRegorus>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup Condition="'$(UseLocalRegorus)' == 'true'">
|
||||
<ProjectReference Include="../Regorus/Regorus.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup Condition="'$(UseLocalRegorus)' != 'true'">
|
||||
<PackageReference Include="regorus" Version="0.8.0$(RegorusPackageVersionSuffix)" />
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
|
||||
Reference in New Issue
Block a user