From 4d727c489907c94da7102929803ff68d9c098960 Mon Sep 17 00:00:00 2001 From: Philipp Schuster Date: Wed, 24 Jun 2026 16:02:51 +0200 Subject: [PATCH] vmm: keep virtio activation alive in migration Live migration can deadlock if the guest triggers a virtio device activation while the migration worker owns the VM. The failure shows up when starting live migrations during boot and firmware startup, where the guest can reset and reinitialize virtio devices while precopy is running. In the failing case, the source log shows a pending virtio activation that never completes: 8.115833s _virtio-pci-net_0: Needs activation; returning barrier 8.115854s vmm/src/vm.rs:464 -- Waiting for barrier 24.875452s Entering downtime phase 24.875481s stopping vcpu throttling thread ... vCPU thread did not respond in 10ms to signal - retrying vCPU thread did not respond in 20ms to signal - retrying ... thread 'throttle-vcpu' (1029) panicked ... Pause(Error signalling vCPUs: Timeout when waiting for signal to be acknowledged) The vCPU blocks on the activation barrier and never reaches the normal pause checkpoint. Later, migration enters downtime and stops the vCPU throttle thread. In the failing case, that thread is still inside a CpuManager::pause() call, which waits for every vCPU to acknowledge the signal. The blocked vCPU never does, so the pause times out. Fix this by storing the DeviceManager inside VmOwnership::Migration. This keeps just enough state on the VMM thread to drain pending virtio activations while the migration worker owns the Vm. The barrier logic stays unchanged. The VMM now releases the same activation barrier during migration that it already released before migration started. This keeps the guest from getting stuck in the activation wait and lets the later pause succeed. Co-authored-by: Leander Kohler On-behalf-of: SAP leander.kohler@sap.com Signed-off-by: Leander Kohler On-behalf-of: SAP philipp.schuster@sap.com Signed-off-by: Philipp Schuster --- vmm/src/lib.rs | 38 +++++++++++++++++++++++++++++++------- vmm/src/vm.rs | 4 ++++ 2 files changed, 35 insertions(+), 7 deletions(-) diff --git a/vmm/src/lib.rs b/vmm/src/lib.rs index ef4ec5793..03b4dad25 100644 --- a/vmm/src/lib.rs +++ b/vmm/src/lib.rs @@ -14,7 +14,7 @@ use std::path::PathBuf; #[cfg(feature = "guest_debug")] use std::sync::mpsc; use std::sync::mpsc::{Receiver, RecvError, SendError, Sender, channel}; -use std::sync::{Arc, Mutex}; +use std::sync::{Arc, Mutex, Weak}; use std::time::{Duration, Instant}; use std::{any, io, iter, mem, panic, path, process, result, thread}; @@ -57,6 +57,7 @@ use crate::api::{ use crate::config::{MemoryRestoreMode, RestoreConfig, add_to_config}; #[cfg(all(target_arch = "x86_64", feature = "guest_debug"))] use crate::coredump::GuestDebuggable; +use crate::device_manager::DeviceManager; use crate::landlock::Landlock; use crate::memory_manager::MemoryManager; #[cfg(all(feature = "kvm", target_arch = "x86_64"))] @@ -629,13 +630,18 @@ pub struct VmmThreadHandle { /// Models the current ownership and associated state of the VM from the /// perspective of the VMM. -pub enum VmOwnership { +enum VmOwnership { Owned(Vm), /// The VM is temporarily owned by an ongoing migration worker. + /// + /// We deliberately do not use shared access to the VM to prevent a whole + /// class of race conditions. Migration { migration_worker_handle: MigrationWorkerHandle, /// Snapshot returned while the VMM cannot inspect the worker-owned VM. vm_info_response: VmInfoResponse, + /// Access to VM state needed during migration. + device_manager: Weak>, }, None, } @@ -2103,12 +2109,27 @@ impl Vmm { } } EpollDispatch::ActivateVirtioDevices => { - // TODO: Future follow-up must resolve virtio activation handling while migrating. let count = self.activate_evt.read().map_err(Error::EventFdRead)?; - if let VmOwnership::Owned(ref vm) = self.vm { - info!("Trying to activate pending virtio devices: count = {count}"); - vm.activate_virtio_devices() - .map_err(Error::ActivateVirtioDevices)?; + info!("Trying to activate pending virtio devices: count = {count}"); + match &self.vm { + VmOwnership::Owned(vm) => { + vm.activate_virtio_devices() + .map_err(Error::ActivateVirtioDevices)?; + } + VmOwnership::Migration { device_manager, .. } => { + // If the VM (and thus the device manager) were + // dropped at this point, we'd have a serious + // programming bug. + let device_manager = device_manager + .upgrade() + .expect("DeviceManager should remain alive during a migration"); + let device_manager = device_manager.lock().unwrap(); + device_manager + .activate_virtio_devices() + .map_err(VmError::ActivateVirtioDevices) + .map_err(Error::ActivateVirtioDevices)?; + } + VmOwnership::None => {} } } EpollDispatch::Api => { @@ -3130,6 +3151,8 @@ impl RequestHandler for Vmm { .take_owned_or(VmError::VmNotRunning) .expect("should have VM ownership as we just checked it"); + let device_manager = Arc::downgrade(vm.device_manager()); + match MigrationWorker::spawn( vm, check_migration_evt, @@ -3143,6 +3166,7 @@ impl RequestHandler for Vmm { self.vm = VmOwnership::Migration { migration_worker_handle: handle, vm_info_response: vm_info_snapshot, + device_manager, }; Ok(()) } diff --git a/vmm/src/vm.rs b/vmm/src/vm.rs index afbb135b9..71b9363cf 100644 --- a/vmm/src/vm.rs +++ b/vmm/src/vm.rs @@ -3237,6 +3237,10 @@ impl Vm { .restore_clock(&hv_vcpus, &saved.state, saved.mode) .map_err(|e| MigratableError::Resume(anyhow!("Could not restore guest clock: {e}"))) } + + pub fn device_manager(&self) -> &Arc> { + &self.device_manager + } } impl Pausable for Vm {