vmm: config: add memory_restore_mode to RestoreConfig

Add a MemoryRestoreMode enum (Copy | OnDemand) to RestoreConfig so
the restore path can be selected at restore time. Copy preserves the
existing eager read-copy behavior. OnDemand enables userfaultfd-based
demand paging and fails restore if the kernel does not support it.

Validate that prefault=on is not combined with OnDemand mode.

Update the OpenAPI spec with the new enum field.

Signed-off-by: Shayon Mukherjee <shayonj@gmail.com>

docs/snapshot_restore.md

@@ -93,6 +93,28 @@ start using it.
 At this point, the VM is fully restored and is identical to the VM which was
 snapshot earlier.
 
+Restore also supports selecting how guest memory is populated:
+
+```bash
+./cloud-hypervisor \
+    --api-socket /tmp/cloud-hypervisor.sock \
+    --restore source_url=file:///home/foo/snapshot,memory_restore_mode=ondemand
+```
+
+If `memory_restore_mode` is omitted, Cloud Hypervisor uses the eager-copy
+restore path (`copy`).
+
+With `memory_restore_mode=ondemand`, restore uses `userfaultfd` to fault snapshot
+pages in on first access instead of copying the full `memory-ranges` file into
+guest RAM before restore completes. This mode is strict: if Cloud Hypervisor
+cannot enable the `userfaultfd` restore path, restore fails instead of falling
+back to `copy`.
+
+Current constraints for `memory_restore_mode=ondemand`:
+
+- `prefault=on` is not supported
+- the snapshot memory ranges must be page-aligned
+
 ## Restore a VM with new Net FDs
 For a VM created with FDs explicitly passed to NetConfig, a set of valid FDs
 need to be provided along with the VM restore command in the following syntax:

vmm/src/api/openapi/cloud-hypervisor.yaml

@@ -1342,6 +1342,11 @@ components:
         destination_url:
           type: string
 
+    MemoryRestoreMode:
+      type: string
+      enum: [Copy, OnDemand]
+      default: Copy
+
     RestoreConfig:
       required:
         - source_url
@@ -1351,6 +1356,8 @@ components:
           type: string
         prefault:
           type: boolean
+        memory_restore_mode:
+          $ref: "#/components/schemas/MemoryRestoreMode"
 
     ReceiveMigrationData:
       required:

vmm/src/config.rs

@@ -356,6 +356,9 @@ pub enum ValidationError {
     /// Number of FDs passed during Restore are incorrect to the NetConfig
     #[error("Number of Net FDs passed for '{0}' during Restore: {1}. Expected: {2}")]
     RestoreNetFdCountMismatch(String, usize, usize),
+    /// Prefault cannot be combined with on-demand restore
+    #[error("'prefault' cannot be combined with 'memory_restore_mode=ondemand'")]
+    InvalidRestorePrefaultWithOnDemand,
     /// Path provided in landlock-rules doesn't exist
     #[error("Path {0:?} provided in landlock-rules does not exist")]
     LandlockPathDoesNotExist(PathBuf),
@@ -2564,27 +2567,61 @@ where
     }
 }
 
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Deserialize, Serialize, Default)]
+pub enum MemoryRestoreMode {
+    /// Restore by eagerly copying the snapshot into guest RAM before resume.
+    #[default]
+    Copy,
+    /// Restore lazily by faulting snapshot pages into guest RAM on demand.
+    OnDemand,
+}
+
+#[derive(Debug, Error)]
+pub enum MemoryRestoreModeParseError {
+    #[error("Invalid value: {0}")]
+    InvalidValue(String),
+}
+
+impl FromStr for MemoryRestoreMode {
+    type Err = MemoryRestoreModeParseError;
+
+    fn from_str(s: &str) -> result::Result<Self, Self::Err> {
+        match s.to_lowercase().as_str() {
+            "copy" => Ok(Self::Copy),
+            "ondemand" => Ok(Self::OnDemand),
+            _ => Err(MemoryRestoreModeParseError::InvalidValue(s.to_owned())),
+        }
+    }
+}
+
 #[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize, Default)]
 pub struct RestoreConfig {
     pub source_url: PathBuf,
     #[serde(default)]
     pub prefault: bool,
     #[serde(default)]
+    pub memory_restore_mode: MemoryRestoreMode,
+    #[serde(default)]
     pub net_fds: Option<Vec<RestoredNetConfig>>,
 }
 
 impl RestoreConfig {
     pub const SYNTAX: &'static str = "Restore from a VM snapshot. \
-        \nRestore parameters \"source_url=<source_url>,prefault=on|off,\
+        \nRestore parameters \"source_url=<source_url>,prefault=on|off,memory_restore_mode=copy|ondemand,\
         net_fds=<list_of_net_ids_with_their_associated_fds>\" \
         \n`source_url` should be a valid URL (e.g file:///foo/bar or tcp://192.168.1.10/foo) \
-        \n`prefault` brings memory pages in when enabled (disabled by default) \
+        \n`prefault` controls eager prefaulting for the copy-based restore path (disabled by default) \
+        \n`memory_restore_mode=copy` preserves the existing eager read-copy restore behavior, while `memory_restore_mode=ondemand` enables lazy demand paging and fails restore if userfaultfd support is unavailable \
         \n`net_fds` is a list of net ids with new file descriptors. \
         Only net devices backed by FDs directly are needed as input.";
 
     pub fn parse(restore: &str) -> Result<Self> {
         let mut parser = OptionParser::new();
-        parser.add("source_url").add("prefault").add("net_fds");
+        parser
+            .add("source_url")
+            .add("prefault")
+            .add("memory_restore_mode")
+            .add("net_fds");
         parser.parse(restore).map_err(Error::ParseRestore)?;
 
         let source_url = parser
@@ -2596,6 +2633,10 @@ impl RestoreConfig {
             .map_err(Error::ParseRestore)?
             .unwrap_or(Toggle(false))
             .0;
+        let memory_restore_mode = parser
+            .convert::<MemoryRestoreMode>("memory_restore_mode")
+            .map_err(Error::ParseRestore)?
+            .unwrap_or_default();
         let net_fds = parser
             .convert::<Tuple<String, Vec<u64>>>("net_fds")
             .map_err(Error::ParseRestore)?
@@ -2612,6 +2653,7 @@ impl RestoreConfig {
         Ok(RestoreConfig {
             source_url,
             prefault,
+            memory_restore_mode,
             net_fds,
         })
     }
@@ -2620,6 +2662,10 @@ impl RestoreConfig {
     // corresponding 'RestoreNetConfig' with a matched 'id' and expected
     // number of FDs.
     pub fn validate(&self, vm_config: &VmConfig) -> ValidationResult<()> {
+        if self.memory_restore_mode == MemoryRestoreMode::OnDemand && self.prefault {
+            return Err(ValidationError::InvalidRestorePrefaultWithOnDemand);
+        }
+
         let mut restored_net_with_fds = HashMap::new();
         for n in self.net_fds.iter().flatten() {
             assert_eq!(
@@ -4498,6 +4544,7 @@ id=\"{id}\",pci_segment={pci_segment},queue_sizes={queue_sizes}"
             RestoreConfig {
                 source_url: PathBuf::from("/path/to/snapshot"),
                 prefault: false,
+                memory_restore_mode: MemoryRestoreMode::Copy,
                 net_fds: None,
             }
         );
@@ -4508,6 +4555,7 @@ id=\"{id}\",pci_segment={pci_segment},queue_sizes={queue_sizes}"
             RestoreConfig {
                 source_url: PathBuf::from("/path/to/snapshot"),
                 prefault: false,
+                memory_restore_mode: MemoryRestoreMode::Copy,
                 net_fds: Some(vec![
                     RestoredNetConfig {
                         id: "net0".to_string(),
@@ -4522,11 +4570,39 @@ id=\"{id}\",pci_segment={pci_segment},queue_sizes={queue_sizes}"
                 ]),
             }
         );
+        assert_eq!(
+            RestoreConfig::parse("source_url=/path/to/snapshot,memory_restore_mode=ondemand")?,
+            RestoreConfig {
+                source_url: PathBuf::from("/path/to/snapshot"),
+                prefault: false,
+                memory_restore_mode: MemoryRestoreMode::OnDemand,
+                net_fds: None,
+            }
+        );
         // Parsing should fail as source_url is a required field
         RestoreConfig::parse("prefault=off").unwrap_err();
+        RestoreConfig::parse("source_url=/path/to/snapshot,memory_restore_mode=bogus").unwrap_err();
         Ok(())
     }
 
+    #[test]
+    fn test_restore_config_serde() {
+        assert_eq!(
+            serde_json::from_str::<RestoreConfig>(r#"{"source_url":"/path/to/snapshot"}"#)
+                .unwrap()
+                .memory_restore_mode,
+            MemoryRestoreMode::Copy
+        );
+        assert_eq!(
+            serde_json::from_str::<RestoreConfig>(
+                r#"{"source_url":"/path/to/snapshot","memory_restore_mode":"OnDemand"}"#
+            )
+            .unwrap()
+            .memory_restore_mode,
+            MemoryRestoreMode::OnDemand
+        );
+    }
+
     #[test]
     fn test_restore_config_validation() {
         // interested in only VmConfig.net, so set rest to default values
@@ -4589,6 +4665,7 @@ id=\"{id}\",pci_segment={pci_segment},queue_sizes={queue_sizes}"
         let valid_config = RestoreConfig {
             source_url: PathBuf::from("/path/to/snapshot"),
             prefault: false,
+            memory_restore_mode: MemoryRestoreMode::Copy,
             net_fds: Some(vec![
                 RestoredNetConfig {
                     id: "net0".to_string(),
@@ -4663,6 +4740,7 @@ id=\"{id}\",pci_segment={pci_segment},queue_sizes={queue_sizes}"
         let another_valid_config = RestoreConfig {
             source_url: PathBuf::from("/path/to/snapshot"),
             prefault: false,
+            memory_restore_mode: MemoryRestoreMode::Copy,
             net_fds: None,
         };
         snapshot_vm_config.net = Some(vec![NetConfig {
@@ -4671,6 +4749,17 @@ id=\"{id}\",pci_segment={pci_segment},queue_sizes={queue_sizes}"
             ..net_fixture()
         }]);
         another_valid_config.validate(&snapshot_vm_config).unwrap();
+
+        let invalid_restore_mode = RestoreConfig {
+            source_url: PathBuf::from("/path/to/snapshot"),
+            prefault: true,
+            memory_restore_mode: MemoryRestoreMode::OnDemand,
+            net_fds: None,
+        };
+        assert_eq!(
+            invalid_restore_mode.validate(&snapshot_vm_config),
+            Err(ValidationError::InvalidRestorePrefaultWithOnDemand)
+        );
     }
 
     fn platform_fixture() -> PlatformConfig {