virtio-devices: vmm: Signal NEEDS_RESET on activation failure

When the guest writes DRIVER_OK and the device fails to activate, the
VMM previously bubbled the error up via VirtioActivate and never
released the activation barrier, leaving the vCPU that wrote DRIVER_OK
blocked on the barrier and effectively deadlocking the guest.

Per virtio 1.3 section 2.1.2, a device that has experienced an error
it cannot recover from should set DEVICE_NEEDS_RESET in its status and
notify the driver via a configuration change interrupt. Do that on
activation failure through the existing mark_device_needs_reset
helper, then release the activation barrier so the vCPU can resume.

DeviceManager::activate_virtio_devices now logs and continues instead
of aborting the whole pending list, so one failing device does not
take down the VMM or block pause and migration. The activator has
already reported the failure with the device id.

Signed-off-by: Anatol Belski <anbelski@linux.microsoft.com>

virtio-devices/src/transport/pci_device.rs

@@ -40,9 +40,9 @@ use vmm_sys_util::eventfd::EventFd;
 use super::pci_common_config::VirtioPciCommonConfigState;
 use crate::transport::{VIRTIO_PCI_COMMON_CONFIG_ID, VirtioPciCommonConfig, VirtioTransport};
 use crate::{
-    ActivateResult, DEVICE_ACKNOWLEDGE, DEVICE_DRIVER, DEVICE_DRIVER_OK, DEVICE_FAILED,
-    DEVICE_FEATURES_OK, DEVICE_INIT, GuestMemoryMmap, VirtioDevice, VirtioDeviceType,
-    VirtioInterrupt, VirtioInterruptType,
+    ActivateResult, ActivationContext, DEVICE_ACKNOWLEDGE, DEVICE_DRIVER, DEVICE_DRIVER_OK,
+    DEVICE_FAILED, DEVICE_FEATURES_OK, DEVICE_INIT, GuestMemoryMmap, VirtioDevice,
+    VirtioDeviceType, VirtioInterrupt, VirtioInterruptType, mark_device_needs_reset,
 };
 
 /// Vector value used to disable MSI for a queue.
@@ -331,22 +331,32 @@ pub struct VirtioPciDeviceActivator {
 
 impl VirtioPciDeviceActivator {
     pub fn activate(mut self) -> ActivateResult {
-        let mut locked_device = self.device.lock().unwrap();
-        locked_device.activate(crate::device::ActivationContext {
+        let result = self.device.lock().unwrap().activate(ActivationContext {
             mem: self.memory.take().unwrap(),
-            interrupt_cb: self.interrupt,
+            interrupt_cb: self.interrupt.clone(),
             queues: self.queues.take().unwrap(),
-            device_status: self.status,
-        })?;
-        self.device_activated.store(true, Ordering::SeqCst);
+            device_status: self.status.clone(),
+        });
 
+        if let Err(e) = &result {
+            mark_device_needs_reset(
+                &self.status,
+                self.interrupt.as_ref(),
+                format_args!("{}: virtio device activation failed: {e:?}", self.id),
+            );
+        } else {
+            self.device_activated.store(true, Ordering::SeqCst);
+        }
+
+        // Release the barrier regardless of outcome. A failing activate()
+        // would otherwise deadlock the vCPU that wrote DRIVER_OK.
         if let Some(barrier) = self.barrier.take() {
             info!("{}: Waiting for barrier", self.id);
             barrier.wait();
             info!("{}: Barrier released", self.id);
         }
 
-        Ok(())
+        result
     }
 }
 

vmm/src/device_manager.rs

@@ -88,8 +88,7 @@ use vfio_ioctls::{VfioContainer, VfioDevice, VfioDeviceFd, VfioOps};
 use virtio_devices::transport::{VirtioPciDevice, VirtioPciDeviceActivator, VirtioTransport};
 use virtio_devices::vhost_user::VhostUserConfig;
 use virtio_devices::{
-    AccessPlatformMapping, ActivateError, Block, Endpoint, IommuMapping, VdpaDmaMapping,
-    VirtioMemMappingSource,
+    AccessPlatformMapping, Block, Endpoint, IommuMapping, VdpaDmaMapping, VirtioMemMappingSource,
 };
 use vm_allocator::{AddressAllocator, InterruptAllocError, SystemAllocator};
 use vm_device::dma_mapping::ExternalDmaMapping;
@@ -629,10 +628,6 @@ pub enum DeviceManagerError {
     #[error("vfio-user socket path already in use: {0:?}")]
     UserDeviceSocketInUse(std::path::PathBuf),
 
-    /// Error activating virtio device
-    #[error("Error activating virtio device")]
-    VirtioActivate(#[source] ActivateError),
-
     /// Failed retrieving device state from snapshot
     #[error("Failed retrieving device state from snapshot")]
     RestoreGetState(#[source] MigratableError),
@@ -4699,9 +4694,9 @@ impl DeviceManager {
 
     pub fn activate_virtio_devices(&self) -> DeviceManagerResult<()> {
         for activator in self.pending_activations.lock().unwrap().drain(..) {
-            activator
-                .activate()
-                .map_err(DeviceManagerError::VirtioActivate)?;
+            // Failures are logged and signalled to the guest via
+            // NEEDS_RESET by the activator, hence keep going.
+            let _ = activator.activate();
         }
         Ok(())
     }