virtio-devices, vmm: Optimised async virtio device activation

In order to ensure that the virtio device thread is spawned from the vmm
thread we use an asynchronous activation mechanism for the virtio
devices. This change optimises that code so that we do not need to
iterate through all virtio devices on the platform in order to find the
one that requires activation. We solve this by creating a separate short
lived VirtioPciDeviceActivator that holds the required state for the
activation (e.g. the clones of the queues) this can then be stored onto
the device manager ready for asynchronous activation.

Signed-off-by: Rob Bradford <robert.bradford@intel.com>

vmm/src/device_manager.rs

@@ -79,10 +79,12 @@ use std::result;
 use std::sync::{Arc, Mutex};
 use std::time::Instant;
 use vfio_ioctls::{VfioContainer, VfioDevice};
-use virtio_devices::transport::VirtioPciDevice;
 use virtio_devices::transport::VirtioTransport;
+use virtio_devices::transport::{VirtioPciDevice, VirtioPciDeviceActivator};
 use virtio_devices::vhost_user::VhostUserConfig;
-use virtio_devices::{AccessPlatformMapping, VdpaDmaMapping, VirtioMemMappingSource};
+use virtio_devices::{
+    AccessPlatformMapping, ActivateError, VdpaDmaMapping, VirtioMemMappingSource,
+};
 use virtio_devices::{Endpoint, IommuMapping};
 use virtio_devices::{VirtioSharedMemory, VirtioSharedMemoryList};
 use vm_allocator::{AddressAllocator, SystemAllocator};
@@ -476,6 +478,9 @@ pub enum DeviceManagerError {
 
     /// Invalid identifier
     InvalidIdentifier(String),
+
+    /// Error activating virtio device
+    VirtioActivate(ActivateError),
 }
 pub type DeviceManagerResult<T> = result::Result<T, DeviceManagerError>;
 
@@ -939,6 +944,9 @@ pub struct DeviceManager {
 
     // Start time of the VM
     timestamp: Instant,
+
+    // Pending activations
+    pending_activations: Arc<Mutex<Vec<VirtioPciDeviceActivator>>>,
 }
 
 impl DeviceManager {
@@ -1080,6 +1088,7 @@ impl DeviceManager {
             io_uring_supported: None,
             boot_id_list,
             timestamp,
+            pending_activations: Arc::new(Mutex::new(Vec::default())),
         };
 
         let device_manager = Arc::new(Mutex::new(device_manager));
@@ -3520,6 +3529,7 @@ impl DeviceManager {
                 // The exception being if not on the default PCI segment.
                 pci_segment_id > 0 || device_type != VirtioDeviceType::Block as u32,
                 dma_handler,
+                self.pending_activations.clone(),
             )
             .map_err(DeviceManagerError::VirtioDevice)?,
         ));
@@ -3674,17 +3684,10 @@ impl DeviceManager {
     }
 
     pub fn activate_virtio_devices(&self) -> DeviceManagerResult<()> {
-        // Find virtio pci devices and activate any pending ones
-        let device_tree = self.device_tree.lock().unwrap();
-        for pci_device_node in device_tree.pci_devices() {
-            #[allow(irrefutable_let_patterns)]
-            if let PciDeviceHandle::Virtio(virtio_pci_device) = &pci_device_node
-                .pci_device_handle
-                .as_ref()
-                .ok_or(DeviceManagerError::MissingPciDevice)?
-            {
-                virtio_pci_device.lock().unwrap().maybe_activate();
-            }
+        for mut activator in self.pending_activations.lock().unwrap().drain(..) {
+            activator
+                .activate()
+                .map_err(DeviceManagerError::VirtioActivate)?;
         }
         Ok(())
     }