vmm: support migration of paused VMs

This extends migration to also support paused VMs, preserving the
paused state on the destination.

Changes:
- Add CompletePaused protocol command that finalizes migration without
 resuming the VM on the destination
- Skip the pause step during migration if the VM is already paused
- On migration failure, only restore the running state if
  the VM was originally running (not paused)

Signed-off-by: Nguyen Dinh Phi <phind.uet@gmail.com>

vm-migration/src/protocol.rs

@@ -103,6 +103,7 @@ use crate::bitpos_iterator::BitposIteratorExt;
 ///     Configured --> Configured: Memory
 ///     Configured --> StateReceived: State
 ///     StateReceived --> Completed: Complete
+///     StateReceived --> Completed: CompletePaused
 /// ```
 ///
 /// [live-migration protocol]: super::protocol
@@ -115,10 +116,14 @@ pub enum Command {
     Config,
     State,
     Memory,
-    /// Finalizes the migration and resumes the VM on the guest.
+    /// Finalizes the migration and resumes the VM on the destination.
+    /// Sent when the source VM was running at migration time.
     Complete,
     Abandon,
     MemoryFd,
+    /// Finalizes the migration without resuming the VM on the destination.
+    /// Sent when the source VM was paused at migration time.
+    CompletePaused,
 }
 
 #[repr(C)]
@@ -161,10 +166,16 @@ impl Request {
         Self::new(Command::MemoryFd, length)
     }
 
+    /// Finalizes the migration and resumes the VM on the destination.
     pub fn complete() -> Self {
         Self::new(Command::Complete, 0)
     }
 
+    /// Finalizes the migration without resuming the VM on the destination.
+    pub fn complete_paused() -> Self {
+        Self::new(Command::CompletePaused, 0)
+    }
+
     pub fn abandon() -> Self {
         Self::new(Command::Abandon, 0)
     }

vmm/src/lib.rs

@@ -986,6 +986,10 @@ impl Vmm {
             StateReceived {
                 state_receive_begin,
             } => match req.command() {
+                Command::CompletePaused => {
+                    debug!("Migration (incoming): Receiving final state of a paused VM");
+                    Ok(Completed)
+                }
                 Command::Complete => {
                     // The unwrap is safe, because the state machine makes sure we called
                     // vm_receive_state before, which creates the VM.
@@ -1361,7 +1365,9 @@ impl Vmm {
             mem_send,
         )?;
         let downtime_begin = Instant::now();
-        vm.pause()?;
+        if vm.get_state() != VmState::Paused {
+            vm.pause()?;
+        }
 
         // Send last batch of dirty pages: final iteration
         {
@@ -1391,6 +1397,7 @@ impl Vmm {
         #[cfg(all(feature = "kvm", target_arch = "x86_64"))]
         hypervisor: &dyn hypervisor::Hypervisor,
         send_data_migration: &VmSendMigrationData,
+        initial_vm_state: VmState,
     ) -> result::Result<(), MigratableError> {
         // State machine that is updated with more context as we progress.
         let mut ctx = OngoingMigrationContext::new();
@@ -1461,9 +1468,11 @@ impl Vmm {
         vm.start_migration()?;
 
         if send_data_migration.local {
-            // Now pause VM
+            // Now pause VM (skip if already paused, e.g. migrating a paused VM)
             let downtime_begin = Instant::now();
-            vm.pause()?;
+            if vm.get_state() != VmState::Paused {
+                vm.pause()?;
+            }
             ctx.set_vm_paused(
                 downtime_begin,
                 // No memory was transferred
@@ -1509,10 +1518,15 @@ impl Vmm {
         // When this returns, we know the VM was resumed (if it was running
         // before the migration) and that the receiving VMM acquired disk
         // locks again.
+        let complete_req = if initial_vm_state == VmState::Running {
+            Request::complete()
+        } else {
+            Request::complete_paused()
+        };
         let (_, complete_duration) = measure_ok(|| {
             migration_transport::send_request_expect_ok(
                 &mut socket,
-                Request::complete(),
+                complete_req,
                 MigratableError::MigrateSend(anyhow!("Error completing migration")),
             )
         })?;
@@ -2578,13 +2592,10 @@ impl RequestHandler for Vmm {
             .as_mut()
             .ok_or_else(|| MigratableError::MigrateSend(anyhow!("VM is not running")))?;
 
-        // Only running VMs can be migrated: Future work can fix this to allow
+        let initial_vm_state = vm.get_state();
-        // also the migration of paused VMs while preserving the state in success
+        if initial_vm_state != VmState::Running && initial_vm_state != VmState::Paused {
-        // and error case. See #7815.
-        if vm.get_state() != VmState::Running {
             return Err(MigratableError::MigrateSend(anyhow!(
-                "VM is not in running state: {:?}",
+                "VM is not running or paused: {initial_vm_state:?}"
-                vm.get_state()
             )));
         }
 
@@ -2594,6 +2605,7 @@ impl RequestHandler for Vmm {
             #[cfg(all(feature = "kvm", target_arch = "x86_64"))]
             self.hypervisor.as_ref(),
             &send_data_migration,
+            initial_vm_state,
         )
         .map_err(|migration_err| {
             error!("Migration failed: {migration_err:?}");
@@ -2606,7 +2618,10 @@ impl RequestHandler for Vmm {
                 return e;
             }
 
-            if vm.get_state() == VmState::Paused
+            // Only resume if the VM was originally running; a VM that was already
+            // paused before migration should remain paused after failure.
+            if initial_vm_state == VmState::Running
+                && vm.get_state() == VmState::Paused
                 && let Err(e) = vm.resume()
             {
                 return e;