From 17b5deeaede5e1def80029bd64bec6866d56070a Mon Sep 17 00:00:00 2001 From: Sebastien Boeuf Date: Thu, 23 Jul 2026 01:43:25 -0700 Subject: [PATCH] vmm: Allow preserving the source VM after snapshot Extend the migration protocol with a `preserve_source` option so that a source VM can be preserved. This benefits the snapshot case where the offload daemon can now snapshot a VM without tearing it down. Signed-off-by: Sebastien Boeuf Assisted-by: Claude:claude-opus-4-8 --- docs/snapshot_restore.md | 40 +++++++++++++++++++++++++++++++------ vmm/src/api/mod.rs | 39 +++++++++++++++++++++++++++++++++++- vmm/src/lib.rs | 26 +++++++++++++++++++----- vmm/src/migration/worker.rs | 9 ++++++--- 4 files changed, 99 insertions(+), 15 deletions(-) diff --git a/docs/snapshot_restore.md b/docs/snapshot_restore.md index 86e7ca68f..c960a93a9 100644 --- a/docs/snapshot_restore.md +++ b/docs/snapshot_restore.md @@ -162,9 +162,11 @@ over the existing local live-migration protocol, playing the migration peer role: - On snapshot, CH acts as the migration sender and the daemon acts as the - receiver. The source VM shuts down on success, exactly as it would for a - local live migration. Memory is transferred via `SCM_RIGHTS`, CH handing - off the daemon one memfd per guest-memory slot. + receiver. By default the source VM shuts down on success, exactly as it + would for a local live migration. Passing `preserve_source=on` instead + leaves the source VM paused and owned by the VMM once the snapshot + completes, so it can be resumed afterwards. Memory is transferred via + `SCM_RIGHTS`, CH handing off the daemon one memfd per guest-memory slot. - On restore, CH acts as the migration receiver and the daemon acts as the sender. The daemon provides one memfd per slot, populated from its storage, and CH uses those memfds directly as guest RAM backing. @@ -201,6 +203,30 @@ migration today. send-migration destination_url=unix:/tmp/offload.sock,local=on ``` +### Preserve the source VM + +By default an offload snapshot destroys the source VM on success. If you want +to preserve the source VM, add `preserve_source=on` (only valid together +with `local=on`) to the `send-migration` command: + +```bash +./ch-remote --api-socket /tmp/cloud-hypervisor.sock pause +./ch-remote --api-socket /tmp/cloud-hypervisor.sock \ + send-migration destination_url=unix:/tmp/offload.sock,local=on,preserve_source=on +# The source VMM keeps running, the VM is left paused. Resume it when ready: +./ch-remote --api-socket /tmp/cloud-hypervisor.sock resume +``` + +With `preserve_source=on` the VM is left `paused` and owned by the VMM, with +its devices intact and its disk locks still held. + +Because the source keeps running and holding its disk locks, its live disk +content diverges from the memory captured in the snapshot, and a restore +cannot re-acquire the locks on those same images. To restore the snapshot +consistently, you should copy the disk images while the VM is still +paused, and restore the daemon's snapshot against those copies. Otherwise, +you might end up in an undefined state leading to possible bugs. + ### Restore offload usage ```bash @@ -258,9 +284,11 @@ The daemon implements the local live-migration wire protocol defined in On the snapshot path, the daemon must finish reading from every memory fd before it ACKs `CompletePaused`. Cloud Hypervisor blocks at the `CompletePaused` handshake until the daemon ACKs. Once it ACKs, the source -VM shuts down and the daemon's fds are the only remaining record of guest -RAM. The reference daemon dumps each slot to disk and `fsync`s before -ACKing. +VM shuts down (unless `preserve_source=on` was requested) and the daemon's +fds are the only remaining record of guest RAM. The reference daemon dumps +each slot to disk and `fsync`s before ACKing. This ordering matters even with +`preserve_source=on` because the source is only resumed after the handshake +returns, so the daemon always captures a consistent image. ### Reference daemon diff --git a/vmm/src/api/mod.rs b/vmm/src/api/mod.rs index 5087b4a52..ec4ed69fe 100644 --- a/vmm/src/api/mod.rs +++ b/vmm/src/api/mod.rs @@ -590,6 +590,10 @@ pub struct VmSendMigrationData { /// Send memory across socket without copying #[serde(default)] pub local: bool, + /// Keep the source VM alive in a paused state once the migration is + /// complete. + #[serde(default)] + pub preserve_source: bool, /// The maximum downtime the migration aims for. /// /// Usually, on the order of a few hundred milliseconds. @@ -621,7 +625,7 @@ pub struct VmSendMigrationData { impl VmSendMigrationData { pub const SYNTAX: &'static str = "VM send migration parameters \ - \"destination_url=[,local=on|off,\ + \"destination_url=[,local=on|off,preserve_source=on|off,\ downtime_ms=,timeout_s=,\ timeout_strategy=cancel|ignore,connections=,\ tls_dir=,memory_mode=precopy|postcopy]\""; @@ -649,6 +653,7 @@ impl VmSendMigrationData { parser .add("destination_url") .add("local") + .add("preserve_source") .add("downtime_ms") .add("timeout_s") .add("timeout_strategy") @@ -669,6 +674,11 @@ impl VmSendMigrationData { .map_err(VmSendMigrationConfigError::ParseError)? .unwrap_or(Toggle(false)) .0; + let preserve_source = parser + .convert::("preserve_source") + .map_err(VmSendMigrationConfigError::ParseError)? + .unwrap_or(Toggle(false)) + .0; let downtime_ms = match parser .convert::("downtime_ms") .map_err(VmSendMigrationConfigError::ParseError)? @@ -718,6 +728,7 @@ impl VmSendMigrationData { let data = Self { destination_url, local, + preserve_source, downtime_ms, timeout_s, timeout_strategy, @@ -786,6 +797,12 @@ impl VmSendMigrationData { } } + if self.preserve_source && !self.local { + return Err(VmSendMigrationConfigError::ValidationError( + "preserve_source option is only supported with the local option.".to_string(), + )); + } + if let Some(tls_dir) = &self.tls_dir { validate_tls_dir(tls_dir, TlsEndpoint::Client).map_err(|e| { VmSendMigrationConfigError::ValidationError(format!( @@ -2408,6 +2425,7 @@ mod unit_tests { VmSendMigrationData { destination_url: "tcp:192.168.1.1:8080".to_string(), local: false, + preserve_source: false, downtime_ms: NonZeroU64::new(150).unwrap(), timeout_s: VmSendMigrationData::default_timeout_s(), timeout_strategy: Default::default(), @@ -2429,6 +2447,7 @@ mod unit_tests { VmSendMigrationData { destination_url: "tcp:192.168.1.1:8080".to_string(), local: false, + preserve_source: false, downtime_ms: NonZeroU64::new(150).unwrap(), timeout_s: NonZeroU64::new(900).unwrap(), timeout_strategy: TimeoutStrategy::Ignore, @@ -2453,5 +2472,23 @@ mod unit_tests { "destination_url=tcp:192.168.1.1:8080,memory_mode=postcopy,connections=4", ) .unwrap_err(); + + // preserve_source is accepted together with local (offload snapshot). + let data = VmSendMigrationData::parse( + "destination_url=unix:/tmp/sock,local=on,preserve_source=on", + ) + .unwrap(); + assert!(data.preserve_source); + assert!(data.local); + + // preserve_source defaults to false when unspecified. + let data = VmSendMigrationData::parse("destination_url=unix:/tmp/sock,local=on").unwrap(); + assert!(!data.preserve_source); + + // preserve_source without local must be rejected. + VmSendMigrationData::parse("destination_url=tcp:192.168.1.1:8080,preserve_source=on") + .unwrap_err(); + VmSendMigrationData::parse("destination_url=unix:/tmp/sock,preserve_source=on") + .unwrap_err(); } } diff --git a/vmm/src/lib.rs b/vmm/src/lib.rs index d6ff2ae24..efdf9826f 100644 --- a/vmm/src/lib.rs +++ b/vmm/src/lib.rs @@ -1670,8 +1670,11 @@ impl Vmm { }; transport::send_config(&mut socket, &vm_migration_config)?; - // Let every Migratable object know about the migration being started. - vm.start_migration()?; + // Let every Migratable object know about the migration being started + // unless the source VM must be preserved. + if !send_data_migration.preserve_source { + vm.start_migration()?; + } if send_data_migration.local || matches!(send_data_migration.memory_mode, MigrationMode::Postcopy) @@ -1716,8 +1719,11 @@ impl Vmm { // We release the locks early to enable locking them on the destination host. // The VM is already stopped. - vm.release_disk_locks() - .map_err(|e| MigratableError::UnlockError(anyhow!("{e}")))?; + // Keep the locks held if the source VM must be preserved. + if !send_data_migration.preserve_source { + vm.release_disk_locks() + .map_err(|e| MigratableError::UnlockError(anyhow!("{e}")))?; + } // For postcopy, serve faults before sending State so the destination // can fault pages in during restore. @@ -1809,7 +1815,12 @@ impl Vmm { } // Let every Migratable object know about the migration being complete - vm.complete_migration() + // unless the source VM must be preserved. + if send_data_migration.preserve_source { + Ok(()) + } else { + vm.complete_migration() + } } /// Serve `Command::PageFault` requests from local guest memory on the fault @@ -2031,6 +2042,7 @@ impl Vmm { vm, migration_result: migration_res, initial_vm_state, + preserve_source, } = migration_worker_handle.join(); let mut try_resume_vm_after_failed_migration = |mut vm: Vm| { @@ -2056,6 +2068,10 @@ impl Vmm { }; match migration_res { + Ok(()) if preserve_source => { + // Give the source VM back to the VMM. + self.vm = VmOwnership::Owned(vm); + } Ok(()) => { self.vm = VmOwnership::None; let mut vm = vm; diff --git a/vmm/src/migration/worker.rs b/vmm/src/migration/worker.rs index d2b4a78cf..aaf7ca972 100644 --- a/vmm/src/migration/worker.rs +++ b/vmm/src/migration/worker.rs @@ -127,6 +127,7 @@ impl MigrationWorker { vm, migration_result, initial_vm_state: self.initial_vm_state, + preserve_source: self.config.preserve_source, } } @@ -180,11 +181,13 @@ impl MigrationWorker { pub struct MigrationWorkerResult { /// The VM that was migrated. /// - /// If `migration_result` is `Ok`, the VM is paused and can be deleted. - /// If `migration_result` is `Err`, the VM can be resumed and given back to - /// the VMM. + /// If `migration_result` is `Ok`, the VM is paused and can be deleted + /// unless `preserve_source` is true, which means the VM is given back + /// to the VMM in a paused state. + /// If `Err`, the VM can be resumed and given back to the VMM. pub vm: Vm, /// The result of [`Vmm::send_migration`]. pub migration_result: Result<(), MigratableError>, pub initial_vm_state: VmState, + pub preserve_source: bool, }