From bf85af907e000b02aa6327a59f5fdc6b3ac22b33 Mon Sep 17 00:00:00 2001 From: Shayon Mukherjee Date: Fri, 13 Mar 2026 05:47:16 -0700 Subject: [PATCH] vmm: config: add memory_restore_mode to RestoreConfig Add a MemoryRestoreMode enum (Copy | OnDemand) to RestoreConfig so the restore path can be selected at restore time. Copy preserves the existing eager read-copy behavior. OnDemand enables userfaultfd-based demand paging and fails restore if the kernel does not support it. Validate that prefault=on is not combined with OnDemand mode. Update the OpenAPI spec with the new enum field. Signed-off-by: Shayon Mukherjee --- docs/snapshot_restore.md | 22 ++++++ vmm/src/api/openapi/cloud-hypervisor.yaml | 7 ++ vmm/src/config.rs | 95 ++++++++++++++++++++++- 3 files changed, 121 insertions(+), 3 deletions(-) diff --git a/docs/snapshot_restore.md b/docs/snapshot_restore.md index df7248805..2cf8eda5a 100644 --- a/docs/snapshot_restore.md +++ b/docs/snapshot_restore.md @@ -93,6 +93,28 @@ start using it. At this point, the VM is fully restored and is identical to the VM which was snapshot earlier. +Restore also supports selecting how guest memory is populated: + +```bash +./cloud-hypervisor \ + --api-socket /tmp/cloud-hypervisor.sock \ + --restore source_url=file:///home/foo/snapshot,memory_restore_mode=ondemand +``` + +If `memory_restore_mode` is omitted, Cloud Hypervisor uses the eager-copy +restore path (`copy`). + +With `memory_restore_mode=ondemand`, restore uses `userfaultfd` to fault snapshot +pages in on first access instead of copying the full `memory-ranges` file into +guest RAM before restore completes. This mode is strict: if Cloud Hypervisor +cannot enable the `userfaultfd` restore path, restore fails instead of falling +back to `copy`. + +Current constraints for `memory_restore_mode=ondemand`: + +- `prefault=on` is not supported +- the snapshot memory ranges must be page-aligned + ## Restore a VM with new Net FDs For a VM created with FDs explicitly passed to NetConfig, a set of valid FDs need to be provided along with the VM restore command in the following syntax: diff --git a/vmm/src/api/openapi/cloud-hypervisor.yaml b/vmm/src/api/openapi/cloud-hypervisor.yaml index efdcf7a67..8bdf14e50 100644 --- a/vmm/src/api/openapi/cloud-hypervisor.yaml +++ b/vmm/src/api/openapi/cloud-hypervisor.yaml @@ -1342,6 +1342,11 @@ components: destination_url: type: string + MemoryRestoreMode: + type: string + enum: [Copy, OnDemand] + default: Copy + RestoreConfig: required: - source_url @@ -1351,6 +1356,8 @@ components: type: string prefault: type: boolean + memory_restore_mode: + $ref: "#/components/schemas/MemoryRestoreMode" ReceiveMigrationData: required: diff --git a/vmm/src/config.rs b/vmm/src/config.rs index 42cbcfdbc..8b284660a 100644 --- a/vmm/src/config.rs +++ b/vmm/src/config.rs @@ -356,6 +356,9 @@ pub enum ValidationError { /// Number of FDs passed during Restore are incorrect to the NetConfig #[error("Number of Net FDs passed for '{0}' during Restore: {1}. Expected: {2}")] RestoreNetFdCountMismatch(String, usize, usize), + /// Prefault cannot be combined with on-demand restore + #[error("'prefault' cannot be combined with 'memory_restore_mode=ondemand'")] + InvalidRestorePrefaultWithOnDemand, /// Path provided in landlock-rules doesn't exist #[error("Path {0:?} provided in landlock-rules does not exist")] LandlockPathDoesNotExist(PathBuf), @@ -2564,27 +2567,61 @@ where } } +#[derive(Clone, Copy, Debug, PartialEq, Eq, Deserialize, Serialize, Default)] +pub enum MemoryRestoreMode { + /// Restore by eagerly copying the snapshot into guest RAM before resume. + #[default] + Copy, + /// Restore lazily by faulting snapshot pages into guest RAM on demand. + OnDemand, +} + +#[derive(Debug, Error)] +pub enum MemoryRestoreModeParseError { + #[error("Invalid value: {0}")] + InvalidValue(String), +} + +impl FromStr for MemoryRestoreMode { + type Err = MemoryRestoreModeParseError; + + fn from_str(s: &str) -> result::Result { + match s.to_lowercase().as_str() { + "copy" => Ok(Self::Copy), + "ondemand" => Ok(Self::OnDemand), + _ => Err(MemoryRestoreModeParseError::InvalidValue(s.to_owned())), + } + } +} + #[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize, Default)] pub struct RestoreConfig { pub source_url: PathBuf, #[serde(default)] pub prefault: bool, #[serde(default)] + pub memory_restore_mode: MemoryRestoreMode, + #[serde(default)] pub net_fds: Option>, } impl RestoreConfig { pub const SYNTAX: &'static str = "Restore from a VM snapshot. \ - \nRestore parameters \"source_url=,prefault=on|off,\ + \nRestore parameters \"source_url=,prefault=on|off,memory_restore_mode=copy|ondemand,\ net_fds=\" \ \n`source_url` should be a valid URL (e.g file:///foo/bar or tcp://192.168.1.10/foo) \ - \n`prefault` brings memory pages in when enabled (disabled by default) \ + \n`prefault` controls eager prefaulting for the copy-based restore path (disabled by default) \ + \n`memory_restore_mode=copy` preserves the existing eager read-copy restore behavior, while `memory_restore_mode=ondemand` enables lazy demand paging and fails restore if userfaultfd support is unavailable \ \n`net_fds` is a list of net ids with new file descriptors. \ Only net devices backed by FDs directly are needed as input."; pub fn parse(restore: &str) -> Result { let mut parser = OptionParser::new(); - parser.add("source_url").add("prefault").add("net_fds"); + parser + .add("source_url") + .add("prefault") + .add("memory_restore_mode") + .add("net_fds"); parser.parse(restore).map_err(Error::ParseRestore)?; let source_url = parser @@ -2596,6 +2633,10 @@ impl RestoreConfig { .map_err(Error::ParseRestore)? .unwrap_or(Toggle(false)) .0; + let memory_restore_mode = parser + .convert::("memory_restore_mode") + .map_err(Error::ParseRestore)? + .unwrap_or_default(); let net_fds = parser .convert::>>("net_fds") .map_err(Error::ParseRestore)? @@ -2612,6 +2653,7 @@ impl RestoreConfig { Ok(RestoreConfig { source_url, prefault, + memory_restore_mode, net_fds, }) } @@ -2620,6 +2662,10 @@ impl RestoreConfig { // corresponding 'RestoreNetConfig' with a matched 'id' and expected // number of FDs. pub fn validate(&self, vm_config: &VmConfig) -> ValidationResult<()> { + if self.memory_restore_mode == MemoryRestoreMode::OnDemand && self.prefault { + return Err(ValidationError::InvalidRestorePrefaultWithOnDemand); + } + let mut restored_net_with_fds = HashMap::new(); for n in self.net_fds.iter().flatten() { assert_eq!( @@ -4498,6 +4544,7 @@ id=\"{id}\",pci_segment={pci_segment},queue_sizes={queue_sizes}" RestoreConfig { source_url: PathBuf::from("/path/to/snapshot"), prefault: false, + memory_restore_mode: MemoryRestoreMode::Copy, net_fds: None, } ); @@ -4508,6 +4555,7 @@ id=\"{id}\",pci_segment={pci_segment},queue_sizes={queue_sizes}" RestoreConfig { source_url: PathBuf::from("/path/to/snapshot"), prefault: false, + memory_restore_mode: MemoryRestoreMode::Copy, net_fds: Some(vec![ RestoredNetConfig { id: "net0".to_string(), @@ -4522,11 +4570,39 @@ id=\"{id}\",pci_segment={pci_segment},queue_sizes={queue_sizes}" ]), } ); + assert_eq!( + RestoreConfig::parse("source_url=/path/to/snapshot,memory_restore_mode=ondemand")?, + RestoreConfig { + source_url: PathBuf::from("/path/to/snapshot"), + prefault: false, + memory_restore_mode: MemoryRestoreMode::OnDemand, + net_fds: None, + } + ); // Parsing should fail as source_url is a required field RestoreConfig::parse("prefault=off").unwrap_err(); + RestoreConfig::parse("source_url=/path/to/snapshot,memory_restore_mode=bogus").unwrap_err(); Ok(()) } + #[test] + fn test_restore_config_serde() { + assert_eq!( + serde_json::from_str::(r#"{"source_url":"/path/to/snapshot"}"#) + .unwrap() + .memory_restore_mode, + MemoryRestoreMode::Copy + ); + assert_eq!( + serde_json::from_str::( + r#"{"source_url":"/path/to/snapshot","memory_restore_mode":"OnDemand"}"# + ) + .unwrap() + .memory_restore_mode, + MemoryRestoreMode::OnDemand + ); + } + #[test] fn test_restore_config_validation() { // interested in only VmConfig.net, so set rest to default values @@ -4589,6 +4665,7 @@ id=\"{id}\",pci_segment={pci_segment},queue_sizes={queue_sizes}" let valid_config = RestoreConfig { source_url: PathBuf::from("/path/to/snapshot"), prefault: false, + memory_restore_mode: MemoryRestoreMode::Copy, net_fds: Some(vec![ RestoredNetConfig { id: "net0".to_string(), @@ -4663,6 +4740,7 @@ id=\"{id}\",pci_segment={pci_segment},queue_sizes={queue_sizes}" let another_valid_config = RestoreConfig { source_url: PathBuf::from("/path/to/snapshot"), prefault: false, + memory_restore_mode: MemoryRestoreMode::Copy, net_fds: None, }; snapshot_vm_config.net = Some(vec![NetConfig { @@ -4671,6 +4749,17 @@ id=\"{id}\",pci_segment={pci_segment},queue_sizes={queue_sizes}" ..net_fixture() }]); another_valid_config.validate(&snapshot_vm_config).unwrap(); + + let invalid_restore_mode = RestoreConfig { + source_url: PathBuf::from("/path/to/snapshot"), + prefault: true, + memory_restore_mode: MemoryRestoreMode::OnDemand, + net_fds: None, + }; + assert_eq!( + invalid_restore_mode.validate(&snapshot_vm_config), + Err(ValidationError::InvalidRestorePrefaultWithOnDemand) + ); } fn platform_fixture() -> PlatformConfig {