feat(rvm): extend program metadata and bump serialization to v6 (#654)

Add typed metadata support to RVM programs so that language frontends
can store language identity and arbitrary annotations alongside the
compiled bytecode.

Program metadata:
- Add `language` field to identify the source language (e.g. "rego",
  "azure_policy") so the VM can adjust semantics at runtime
- Add `annotations` map (BTreeMap<String, MetadataValue>) for
  frontend-specific key-value metadata
- Add MetadataValue enum with String, Bool, Integer, Float, Array,
  and Object variants, plus full serde support
- Add to_value() conversion for runtime access from VM instructions
- Add has_host_await flag with recompute_host_await_presence()

Serialization:
- Bump binary format version from 5 to 6
- Add JSON serialization for the new metadata fields

Assembly listing:
- Display language and annotations in the program header

Compiler:
- Track has_host_await during Rego compilation
This commit is contained in:
Anand Krishnamoorthi
2026-04-03 12:53:42 -05:00
committed by GitHub
parent db8a9abf13
commit 95bffcb5f9
9 changed files with 488 additions and 26 deletions

View File

@@ -259,6 +259,9 @@ impl<'a> Compiler<'a> {
}
pub fn emit_instruction(&mut self, instruction: Instruction, span: &Span) {
if matches!(instruction, Instruction::HostAwait { .. }) {
self.program.has_host_await = true;
}
self.program.instructions.push(instruction);
let source_path = span.source.get_path().to_string();

View File

@@ -23,8 +23,8 @@ pub use error::{CompilerError, Result, SpannedCompilerError};
use crate::ast::ExprRef;
use crate::lexer::Span;
use crate::rvm::program::{Program, RuleType, SpanInfo};
use crate::value::Value;
use crate::CompiledPolicy;
use crate::Value;
use alloc::collections::{BTreeMap, BTreeSet};
use alloc::string::String;
use alloc::vec;

View File

@@ -8,7 +8,8 @@ use anyhow::Result as AnyResult;
use indexmap::IndexMap;
use serde::{Deserialize, Serialize};
use super::types::{BuiltinInfo, ProgramMetadata, RuleInfo, SourceFile, SpanInfo};
use super::metadata::ProgramMetadata;
use super::types::{BuiltinInfo, RuleInfo, SourceFile, SpanInfo};
use crate::builtins::BuiltinFcn;
use crate::rvm::instructions::InstructionData;
use crate::rvm::Instruction;
@@ -70,6 +71,11 @@ pub struct Program {
/// Flag indicating that VirtualDataDocumentLookup instruction was used and runtime recursion checking is needed
pub needs_runtime_recursion_check: bool,
/// Flag indicating whether this program contains any HostAwait instruction.
/// Clients can use this to decide whether suspendable execution mode is required.
#[serde(default)]
pub has_host_await: bool,
/// Flag indicating that recompilation is needed due to partial deserialization failure
/// This is set to true when the artifact section was successfully read but the extensible
/// section failed to deserialize (e.g., due to version incompatibility)
@@ -85,7 +91,7 @@ pub struct Program {
impl Program {
/// Current serialization format version
pub const SERIALIZATION_VERSION: u32 = 5;
pub const SERIALIZATION_VERSION: u32 = 6;
/// Magic bytes to identify Regorus program files
pub const MAGIC: [u8; 4] = *b"REGO";
/// Maximum instructions supported (matches u16 jump targets)
@@ -122,10 +128,13 @@ impl Program {
compiled_at: "unknown".to_string(),
source_info: "unknown".to_string(),
optimization_level: 0,
language: String::new(),
annotations: alloc::collections::BTreeMap::new(),
},
rule_tree: Value::new_object(),
resolved_builtins: Vec::new(),
needs_runtime_recursion_check: false,
has_host_await: false,
needs_recompilation: false,
rego_v0: false, // Default to Rego v1
}
@@ -330,10 +339,21 @@ impl Program {
/// Add instruction with optional span
pub fn add_instruction(&mut self, instruction: Instruction, span: Option<SpanInfo>) {
if matches!(instruction, Instruction::HostAwait { .. }) {
self.has_host_await = true;
}
self.instructions.push(instruction);
self.instruction_spans.push(span);
}
/// Recompute whether HostAwait is present by scanning instructions.
pub fn recompute_host_await_presence(&mut self) {
self.has_host_await = self
.instructions
.iter()
.any(|instruction| matches!(instruction, Instruction::HostAwait { .. }));
}
/// Add literal value and return its index
pub fn add_literal(&mut self, value: Value) -> usize {
for (i, existing) in self.literals.iter().enumerate() {
@@ -408,6 +428,11 @@ impl Program {
pub const fn is_fully_functional(&self) -> bool {
!self.needs_recompilation
}
/// Check whether HostAwait instructions are present in this program.
pub const fn has_host_await(&self) -> bool {
self.has_host_await
}
}
impl Default for Program {

View File

@@ -133,6 +133,19 @@ pub fn generate_assembly_listing(program: &Program, config: &AssemblyListingConf
program.metadata.optimization_level
),
);
if !program.metadata.language.is_empty() {
push_line(
&mut output,
format_args!("; language: {}", program.metadata.language),
);
}
if !program.metadata.annotations.is_empty() {
push_line(&mut output, format_args!("; annotations:"));
for (key, value) in &program.metadata.annotations {
let json = serde_json::to_string(value).unwrap_or_else(|_| "<invalid>".to_string());
push_line(&mut output, format_args!("; {}: {}", key, json));
}
}
}
push_line(&mut output, format_args!(";"));

393
src/rvm/program/metadata.rs Normal file
View File

@@ -0,0 +1,393 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.
//! Program metadata types and serialization bridge.
//!
//! [`ProgramMetadata`] stores compiler provenance, source-language identity,
//! and arbitrary key-value annotations alongside the compiled bytecode.
//!
//! Annotations are kept as regorus [`Value`](crate::value::Value) at runtime
//! for zero-cost access via `LoadMetadata`. For binary serialization the
//! values are converted through [`MetadataValue`] — a postcard/bincode-safe
//! enum that avoids `deserialize_any`.
use crate::Rc;
use alloc::collections::BTreeMap;
use alloc::collections::BTreeSet;
use alloc::string::String;
use alloc::vec::Vec;
use serde::{Deserialize, Serialize};
/// Program compilation metadata.
///
/// Annotations are stored as [`Value`](crate::value::Value) at runtime for
/// zero-cost access via `LoadMetadata`. Serialization converts through
/// [`MetadataValue`] — a postcard/bincode-safe enum that avoids
/// `deserialize_any`.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ProgramMetadata {
/// Compiler version that generated this program
pub compiler_version: String,
/// Compilation timestamp
pub compiled_at: String,
/// Source policy information
pub source_info: String,
/// Optimization level used
pub optimization_level: u8,
/// Source language that was compiled (e.g. "rego", "azure_policy", "cedar")
#[serde(default)]
pub language: String,
/// Language-specific and user-defined annotations for indexing and introspection.
/// Stored as `Value` for direct runtime use; serialized via `MetadataValue`.
#[serde(
default,
serialize_with = "metadata_serde::serialize_annotations",
deserialize_with = "metadata_serde::deserialize_annotations"
)]
pub annotations: BTreeMap<String, crate::value::Value>,
}
impl ProgramMetadata {
/// Convert the full metadata struct into a regorus `Value` for runtime access.
pub fn to_value(&self) -> crate::value::Value {
use crate::value::Value;
let mut obj = BTreeMap::new();
obj.insert(
Value::String("compiler_version".into()),
Value::String(self.compiler_version.as_str().into()),
);
obj.insert(
Value::String("compiled_at".into()),
Value::String(self.compiled_at.as_str().into()),
);
obj.insert(
Value::String("source_info".into()),
Value::String(self.source_info.as_str().into()),
);
obj.insert(
Value::String("optimization_level".into()),
Value::from(i64::from(self.optimization_level)),
);
obj.insert(
Value::String("language".into()),
Value::String(self.language.as_str().into()),
);
if !self.annotations.is_empty() {
let mut annotations_obj = BTreeMap::new();
for (k, v) in &self.annotations {
annotations_obj.insert(Value::String(k.as_str().into()), v.clone());
}
obj.insert(
Value::String("annotations".into()),
Value::Object(Rc::new(annotations_obj)),
);
}
Value::Object(Rc::new(obj))
}
}
// ── MetadataValue: postcard-safe serialization bridge ────────────────────────
/// A postcard-compatible, recursive value type used exclusively for serializing
/// program metadata annotations.
///
/// Unlike `serde_json::Value` or regorus `Value`, this enum uses explicit variant
/// tags and does not rely on `deserialize_any`, making it safe for use with
/// non-self-describing formats such as postcard and bincode.
///
/// At runtime, annotations are stored as `Value` for zero-cost access.
/// This type is only used during `Serialize` / `Deserialize` of `ProgramMetadata`.
///
/// # Lossy mappings
///
/// [`Null`](crate::value::Value::Null) and [`Undefined`](crate::value::Value::Undefined)
/// are mapped to [`MetadataValue::String("")`](MetadataValue::String) because metadata
/// annotations have no need for null semantics. A round-trip through
/// `from_value` → `to_value` will therefore turn `Null`/`Undefined` into an
/// empty string.
///
/// Floating-point numbers are truncated to `i64` because metadata annotations
/// are expected to contain only integer counts, flags, and identifiers —
/// fractional values are not anticipated.
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
pub enum MetadataValue {
/// A string value
String(String),
/// A set of unique strings (sorted)
StringSet(BTreeSet<String>),
/// A boolean value
Bool(bool),
/// A 64-bit signed integer (floats are truncated to i64)
Integer(i64),
/// An ordered list of metadata values (recursive)
List(Vec<MetadataValue>),
/// A string-keyed map of metadata values (recursive)
Map(BTreeMap<String, MetadataValue>),
}
impl MetadataValue {
/// Convert a regorus `Value` into a `MetadataValue` for serialization.
///
/// Values that cannot be represented exactly are mapped on a best-effort
/// basis:
/// - **Null / Undefined** → empty string (metadata has no null concept)
/// - **Float** → truncated to `i64` (metadata is integer-only)
/// - **Non-string object keys** → `Display`-formatted strings
pub fn from_value(value: &crate::value::Value) -> Self {
use crate::value::Value;
match *value {
Value::String(ref s) => MetadataValue::String(String::from(s.as_ref())),
Value::Bool(b) => MetadataValue::Bool(b),
Value::Number(ref n) => {
// Try integer first, fall back to f64 truncation
n.as_i64().map_or_else(
|| {
n.as_f64().map_or(MetadataValue::Integer(0), |f| {
// Deliberate truncation of f64 to i64 for metadata storage.
// Metadata annotations are expected to be integer-valued;
// fractional values are not anticipated.
#[expect(clippy::as_conversions)]
let i = f as i64;
MetadataValue::Integer(i)
})
},
MetadataValue::Integer,
)
}
Value::Array(ref arr) => {
MetadataValue::List(arr.iter().map(MetadataValue::from_value).collect())
}
Value::Set(ref set) => {
// If all elements are strings, use StringSet; otherwise List
let all_strings = set.iter().all(|v| matches!(*v, Value::String(_)));
if all_strings {
MetadataValue::StringSet(
set.iter()
.filter_map(|v| match *v {
Value::String(ref s) => Some(String::from(s.as_ref())),
_ => None,
})
.collect(),
)
} else {
MetadataValue::List(set.iter().map(MetadataValue::from_value).collect())
}
}
Value::Object(ref obj) => {
let mut map = BTreeMap::new();
for (k, v) in obj.iter() {
let key = match *k {
Value::String(ref s) => String::from(s.as_ref()),
ref other => alloc::format!("{}", other),
};
map.insert(key, MetadataValue::from_value(v));
}
MetadataValue::Map(map)
}
// Null and Undefined have no metadata representation; map to empty string.
Value::Null | Value::Undefined => MetadataValue::String(String::new()),
}
}
/// Convert this `MetadataValue` into a regorus `Value`.
pub fn to_value(&self) -> crate::value::Value {
use crate::value::Value;
match *self {
MetadataValue::String(ref s) => Value::String(s.as_str().into()),
MetadataValue::StringSet(ref set) => {
let mut bset = alloc::collections::BTreeSet::new();
for s in set {
bset.insert(Value::String(s.as_str().into()));
}
Value::Set(Rc::new(bset))
}
MetadataValue::Bool(b) => Value::Bool(b),
MetadataValue::Integer(n) => Value::from(n),
MetadataValue::List(ref list) => {
let values: Vec<Value> = list.iter().map(MetadataValue::to_value).collect();
Value::Array(Rc::new(values))
}
MetadataValue::Map(ref map) => {
let mut obj = BTreeMap::new();
for (k, v) in map {
obj.insert(Value::String(k.as_str().into()), v.to_value());
}
Value::Object(Rc::new(obj))
}
}
}
}
/// Serde helpers for `annotations: BTreeMap<String, Value>`.
/// Serializes via `BTreeMap<String, MetadataValue>` to stay postcard-compatible.
mod metadata_serde {
use super::*;
pub fn serialize_annotations<S>(
annotations: &BTreeMap<String, crate::value::Value>,
serializer: S,
) -> core::result::Result<S::Ok, S::Error>
where
S: serde::Serializer,
{
use serde::Serialize as _;
let bridge: BTreeMap<String, MetadataValue> = annotations
.iter()
.map(|(k, v)| (k.clone(), MetadataValue::from_value(v)))
.collect();
bridge.serialize(serializer)
}
pub fn deserialize_annotations<'de, D>(
deserializer: D,
) -> core::result::Result<BTreeMap<String, crate::value::Value>, D::Error>
where
D: serde::Deserializer<'de>,
{
let bridge: BTreeMap<String, MetadataValue> = BTreeMap::deserialize(deserializer)?;
Ok(bridge.into_iter().map(|(k, v)| (k, v.to_value())).collect())
}
}
#[cfg(test)]
#[allow(clippy::unwrap_used)]
mod tests {
use super::*;
use crate::value::Value;
use alloc::collections::BTreeSet;
/// Round-trip: Value → MetadataValue → Value must be equivalent for
/// all lossless variants (strings, bools, integers, arrays, objects).
///
/// Note: Null and Undefined are intentionally lossy — they round-trip
/// to empty string. See [`MetadataValue`] doc for rationale.
fn assert_round_trip(original: &Value, expected: &Value) {
let mv = MetadataValue::from_value(original);
let recovered = mv.to_value();
assert_eq!(
&recovered, expected,
"round-trip failed for {original:?} → {mv:?} → {recovered:?}"
);
}
#[test]
fn round_trip_string() {
let v = Value::String("hello".into());
assert_round_trip(&v, &v);
}
#[test]
fn round_trip_bool() {
assert_round_trip(&Value::Bool(true), &Value::Bool(true));
assert_round_trip(&Value::Bool(false), &Value::Bool(false));
}
#[test]
fn round_trip_integer() {
let v = Value::from(42_i64);
assert_round_trip(&v, &v);
}
#[test]
fn round_trip_array() {
let v = Value::from_json_str(r#"[1, "two", true]"#).unwrap();
assert_round_trip(&v, &v);
}
#[test]
fn round_trip_string_set() {
let mut set = BTreeSet::new();
set.insert(Value::String("a".into()));
set.insert(Value::String("b".into()));
let v = Value::Set(Rc::new(set));
assert_round_trip(&v, &v);
}
#[test]
fn round_trip_object() {
let v = Value::from_json_str(r#"{"key": "value", "n": 7}"#).unwrap();
assert_round_trip(&v, &v);
}
#[test]
fn null_maps_to_empty_string() {
let mv = MetadataValue::from_value(&Value::Null);
assert_eq!(mv, MetadataValue::String(String::new()));
}
#[test]
fn undefined_maps_to_empty_string() {
let mv = MetadataValue::from_value(&Value::Undefined);
assert_eq!(mv, MetadataValue::String(String::new()));
}
#[test]
fn mixed_set_uses_list() {
let mut set = BTreeSet::new();
set.insert(Value::String("a".into()));
set.insert(Value::from(1_i64));
let v = Value::Set(Rc::new(set));
let mv = MetadataValue::from_value(&v);
assert!(
matches!(mv, MetadataValue::List(_)),
"mixed-type set should produce List, got {mv:?}"
);
}
#[test]
fn float_truncated_to_integer() {
let v = Value::from(1.23_f64);
let mv = MetadataValue::from_value(&v);
assert_eq!(mv, MetadataValue::Integer(1));
}
#[test]
fn to_value_optimization_level_is_integer() {
let meta = ProgramMetadata {
compiler_version: String::new(),
compiled_at: String::new(),
source_info: String::new(),
optimization_level: 2,
language: String::new(),
annotations: BTreeMap::new(),
};
let val = meta.to_value();
// optimization_level must be emitted as an integer Value, not float.
let key = Value::String("optimization_level".into());
let opt = val.as_object().unwrap().get(&key).unwrap().clone();
assert_eq!(opt, Value::from(2_i64));
}
#[test]
fn serde_annotations_round_trip() {
let mut annotations = BTreeMap::new();
annotations.insert(String::from("flag"), Value::Bool(true));
annotations.insert(String::from("count"), Value::from(42_i64));
annotations.insert(String::from("name"), Value::String("test".into()));
let meta = ProgramMetadata {
compiler_version: String::from("1.0"),
compiled_at: String::from("now"),
source_info: String::from("test"),
optimization_level: 1,
language: String::from("rego"),
annotations,
};
// Round-trip through postcard (the format we care about).
let bytes = postcard::to_allocvec(&meta).unwrap();
let recovered: ProgramMetadata = postcard::from_bytes(&bytes).unwrap();
assert_eq!(meta.annotations.len(), recovered.annotations.len());
for (k, v) in &meta.annotations {
assert_eq!(
recovered.annotations.get(k).unwrap(),
v,
"annotation {k:?} mismatch after round-trip"
);
}
}
}

View File

@@ -3,6 +3,7 @@
mod core;
mod listing;
mod metadata;
mod recompile;
mod rule_tree;
mod serialization;
@@ -14,6 +15,6 @@ pub use listing::{
};
pub(crate) use serialization::value::{binaries_to_values, BinaryValue};
pub use serialization::{DeserializationResult, VersionedProgram};
pub use types::{
BuiltinInfo, FunctionInfo, ProgramMetadata, RuleInfo, RuleType, SourceFile, SpanInfo,
};
pub use types::{BuiltinInfo, FunctionInfo, RuleInfo, RuleType, SourceFile, SpanInfo};
pub use metadata::ProgramMetadata;

View File

@@ -151,13 +151,13 @@ impl Program {
}
match version {
1..=3 => {
1..=5 => {
let mut program = Program::new();
program.needs_recompilation = true;
program.rego_v0 = Self::legacy_rego_v0(data, version).unwrap_or(false);
Ok(DeserializationResult::Partial(program))
}
4 | 5 => {
6 => {
if data.len() < 29 {
return Err("Data too short for header".to_string());
}
@@ -281,7 +281,7 @@ impl Program {
let version = Self::read_u32(data, 4).ok();
match version {
Some(1..=5) => Ok(true),
Some(1..=6) => Ok(true),
_ => Ok(false),
}
}
@@ -289,7 +289,7 @@ impl Program {
fn legacy_rego_v0(data: &[u8], version: u32) -> Option<bool> {
match version {
1 => data.get(16).map(|value| *value != 0),
2 | 3 => data.get(24).map(|value| *value != 0),
2..=5 => data.get(24).map(|value| *value != 0),
_ => None,
}
}

View File

@@ -5,8 +5,8 @@ use alloc::format;
use alloc::string::{String, ToString as _};
use alloc::vec::Vec;
use super::super::types::SourceFile;
use super::super::types::{BuiltinInfo, ProgramMetadata, RuleInfo, SpanInfo};
use super::super::metadata::ProgramMetadata;
use super::super::types::{BuiltinInfo, RuleInfo, SourceFile, SpanInfo};
use super::Program;
use crate::rvm::instructions::InstructionData;
use crate::rvm::Instruction;
@@ -24,7 +24,10 @@ impl Program {
"optimization_level": self.metadata.optimization_level,
"rego_v0": self.rego_v0,
"needs_runtime_recursion_check": self.needs_runtime_recursion_check,
"needs_recompilation": self.needs_recompilation
"has_host_await": self.has_host_await,
"needs_recompilation": self.needs_recompilation,
"language": self.metadata.language,
"annotations": self.metadata.annotations
},
"program_structure": {
"main_entry_point": self.main_entry_point,
@@ -84,6 +87,26 @@ impl Program {
.and_then(|v| v.as_u64())
.and_then(|v| u8::try_from(v).ok())
.unwrap_or(0);
let language = metadata
.get("language")
.and_then(|v| v.as_str())
.unwrap_or("")
.to_string();
#[allow(clippy::needless_borrowed_reference)]
let annotations: alloc::collections::BTreeMap<String, Value> = match metadata
.get("annotations")
{
Some(&serde_json::Value::Object(ref map)) => {
let mut result = alloc::collections::BTreeMap::new();
for (k, json_val) in map {
let val = serde_json::from_value::<Value>(json_val.clone())
.map_err(|e| format!("Annotation '{}' deserialization failed: {}", k, e))?;
result.insert(k.clone(), val);
}
result
}
_ => alloc::collections::BTreeMap::new(),
};
let rego_v0 = metadata
.get("rego_v0")
.and_then(|v| v.as_bool())
@@ -92,6 +115,10 @@ impl Program {
.get("needs_runtime_recursion_check")
.and_then(|v| v.as_bool())
.unwrap_or(false);
let has_host_await = metadata
.get("has_host_await")
.and_then(|v| v.as_bool())
.unwrap_or(false);
let needs_recompilation = metadata
.get("needs_recompilation")
.and_then(|v| v.as_bool())
@@ -183,14 +210,27 @@ impl Program {
compiled_at,
source_info,
optimization_level,
language,
annotations,
},
rule_tree,
resolved_builtins: Vec::new(),
needs_runtime_recursion_check,
has_host_await,
needs_recompilation,
rego_v0,
};
// Recompute has_host_await when it was not provided in the JSON input
// or when the provided value is not a valid boolean.
if json_data
.get("metadata")
.and_then(|m| m.get("has_host_await").and_then(|v| v.as_bool()))
.is_none()
{
program.recompute_host_await_presence();
}
if !program.builtin_info_table.is_empty() {
let _ = program.initialize_resolved_builtins();
}

View File

@@ -173,16 +173,3 @@ impl SourceFile {
Self { name, content }
}
}
/// Program compilation metadata
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ProgramMetadata {
/// Compiler version that generated this program
pub compiler_version: String,
/// Compilation timestamp
pub compiled_at: String,
/// Source policy information
pub source_info: String,
/// Optimization level used
pub optimization_level: u8,
}