From f0b7412495864b255707745202316ee55076128f Mon Sep 17 00:00:00 2001 From: Samuel Ortiz Date: Tue, 19 Nov 2019 01:53:23 +0100 Subject: [PATCH] vmm: device_manager: Add all virtio devices to the migratable list We want to track all migratable devices through the DeviceManager. Fixes: #341 Signed-off-by: Samuel Ortiz --- vmm/src/device_manager.rs | 89 ++++++++++++++++++++++++++++++++++----- 1 file changed, 79 insertions(+), 10 deletions(-) diff --git a/vmm/src/device_manager.rs b/vmm/src/device_manager.rs index a2ca39324..d425b6465 100644 --- a/vmm/src/device_manager.rs +++ b/vmm/src/device_manager.rs @@ -456,6 +456,7 @@ impl DeviceManager { &address_manager, &mut mem_slots, &mut mmap_regions, + &mut migratable_devices, )?); DeviceManager::add_legacy_devices( @@ -481,6 +482,7 @@ impl DeviceManager { &mut virt_iommu, virtio_devices, &interrupt_info, + &mut migratable_devices, )?; } else if cfg!(feature = "mmio_support") { DeviceManager::add_mmio_devices( @@ -489,6 +491,7 @@ impl DeviceManager { virtio_devices, &interrupt_info, &mut cmdline_additions, + &mut migratable_devices, )?; } @@ -520,6 +523,7 @@ impl DeviceManager { virt_iommu: &mut Option<(u32, Vec)>, virtio_devices: Vec<(Arc>, bool)>, interrupt_info: &InterruptInfo, + migratable_devices: &mut Vec>>, ) -> DeviceManagerResult<()> { #[cfg(feature = "pci_support")] { @@ -553,6 +557,7 @@ impl DeviceManager { vm_info.vm_fd, &mut pci_bus, mapping, + migratable_devices, )?; if let Some(dev_id) = virtio_iommu_attach_dev { @@ -588,6 +593,7 @@ impl DeviceManager { vm_info.vm_fd, &mut pci_bus, &None, + migratable_devices, )?; *virt_iommu = Some((iommu_id, iommu_attached_devices)); @@ -620,6 +626,7 @@ impl DeviceManager { virtio_devices: Vec<(Arc>, bool)>, interrupt_info: &InterruptInfo, mut cmdline_additions: &mut Vec, + migratable_devices: &mut Vec>>, ) -> DeviceManagerResult<()> { #[cfg(feature = "mmio_support")] { @@ -638,6 +645,7 @@ impl DeviceManager { &interrupt_info, addr, &mut cmdline_additions, + migratable_devices, )?; } else { error!("Unable to allocate MMIO address!"); @@ -841,14 +849,24 @@ impl DeviceManager { address_manager: &Arc, mut mem_slots: &mut u32, mmap_regions: &mut Vec<(*mut libc::c_void, usize)>, + migratable_devices: &mut Vec>>, ) -> DeviceManagerResult> { let mut allocator = address_manager.allocator.lock().unwrap(); let mut devices: Vec<(Arc>, bool)> = Vec::new(); // Create "standard" virtio devices (net/block/rng) - devices.append(&mut DeviceManager::make_virtio_block_devices(vm_info)?); - devices.append(&mut DeviceManager::make_virtio_net_devices(vm_info)?); - devices.append(&mut DeviceManager::make_virtio_rng_devices(vm_info)?); + devices.append(&mut DeviceManager::make_virtio_block_devices( + vm_info, + migratable_devices, + )?); + devices.append(&mut DeviceManager::make_virtio_net_devices( + vm_info, + migratable_devices, + )?); + devices.append(&mut DeviceManager::make_virtio_rng_devices( + vm_info, + migratable_devices, + )?); // Add virtio-fs if required devices.append(&mut DeviceManager::make_virtio_fs_devices( @@ -856,6 +874,7 @@ impl DeviceManager { &mut allocator, &mut mem_slots, mmap_regions, + migratable_devices, )?); // Add virtio-pmem if required @@ -864,26 +883,33 @@ impl DeviceManager { &mut allocator, &mut mem_slots, mmap_regions, + migratable_devices, )?); // Add virtio-vhost-user-net if required devices.append(&mut DeviceManager::make_virtio_vhost_user_net_devices( vm_info, + migratable_devices, )?); // Add virtio-vhost-user-blk if required devices.append(&mut DeviceManager::make_virtio_vhost_user_blk_devices( vm_info, + migratable_devices, )?); // Add virtio-vsock if required - devices.append(&mut DeviceManager::make_virtio_vsock_devices(vm_info)?); + devices.append(&mut DeviceManager::make_virtio_vsock_devices( + vm_info, + migratable_devices, + )?); Ok(devices) } fn make_virtio_block_devices( vm_info: &VmInfo, + migratable_devices: &mut Vec>>, ) -> DeviceManagerResult> { let mut devices = Vec::new(); @@ -898,7 +924,7 @@ impl DeviceManager { let image_type = qcow::detect_image_type(&raw_img) .map_err(DeviceManagerError::DetectImageType)?; - let block = match image_type { + match image_type { ImageType::Raw => { let raw_img = vm_virtio::RawFile::new(raw_img); let dev = vm_virtio::Block::new( @@ -909,7 +935,13 @@ impl DeviceManager { ) .map_err(DeviceManagerError::CreateVirtioBlock)?; - Arc::new(Mutex::new(dev)) as Arc> + let block = Arc::new(Mutex::new(dev)); + + devices.push(( + Arc::clone(&block) as Arc>, + disk_cfg.iommu, + )); + migratable_devices.push(Arc::clone(&block) as Arc>); } ImageType::Qcow2 => { let qcow_img = QcowFile::from(raw_img) @@ -921,11 +953,16 @@ impl DeviceManager { disk_cfg.iommu, ) .map_err(DeviceManagerError::CreateVirtioBlock)?; - Arc::new(Mutex::new(dev)) as Arc> + + let block = Arc::new(Mutex::new(dev)); + + devices.push(( + Arc::clone(&block) as Arc>, + disk_cfg.iommu, + )); + migratable_devices.push(Arc::clone(&block) as Arc>); } }; - - devices.push((Arc::clone(&block), disk_cfg.iommu)); } } @@ -934,6 +971,7 @@ impl DeviceManager { fn make_virtio_net_devices( vm_info: &VmInfo, + migratable_devices: &mut Vec>>, ) -> DeviceManagerResult> { let mut devices = Vec::new(); @@ -962,6 +1000,8 @@ impl DeviceManager { Arc::clone(&virtio_net_device) as Arc>, net_cfg.iommu, )); + migratable_devices + .push(Arc::clone(&virtio_net_device) as Arc>); } } @@ -970,6 +1010,7 @@ impl DeviceManager { fn make_virtio_rng_devices( vm_info: &VmInfo, + migratable_devices: &mut Vec>>, ) -> DeviceManagerResult> { let mut devices = Vec::new(); @@ -984,6 +1025,8 @@ impl DeviceManager { Arc::clone(&virtio_rng_device) as Arc>, false, )); + + migratable_devices.push(Arc::clone(&virtio_rng_device) as Arc>); } Ok(devices) @@ -994,6 +1037,7 @@ impl DeviceManager { allocator: &mut SystemAllocator, mem_slots: &mut u32, mmap_regions: &mut Vec<(*mut libc::c_void, usize)>, + migratable_devices: &mut Vec>>, ) -> DeviceManagerResult> { let mut devices = Vec::new(); // Add virtio-fs if required @@ -1074,6 +1118,9 @@ impl DeviceManager { Arc::clone(&virtio_fs_device) as Arc>, false, )); + + migratable_devices + .push(Arc::clone(&virtio_fs_device) as Arc>); } } } @@ -1086,6 +1133,7 @@ impl DeviceManager { allocator: &mut SystemAllocator, mem_slots: &mut u32, mmap_regions: &mut Vec<(*mut libc::c_void, usize)>, + migratable_devices: &mut Vec>>, ) -> DeviceManagerResult> { let mut devices = Vec::new(); // Add virtio-pmem if required @@ -1177,6 +1225,9 @@ impl DeviceManager { Arc::clone(&virtio_pmem_device) as Arc>, false, )); + + migratable_devices + .push(Arc::clone(&virtio_pmem_device) as Arc>); } } @@ -1185,6 +1236,7 @@ impl DeviceManager { fn make_virtio_vhost_user_net_devices( vm_info: &VmInfo, + migratable_devices: &mut Vec>>, ) -> DeviceManagerResult> { let mut devices = Vec::new(); // Add vhost-user-net if required @@ -1204,6 +1256,9 @@ impl DeviceManager { Arc::clone(&vhost_user_net_device) as Arc>, false, )); + + migratable_devices + .push(Arc::clone(&vhost_user_net_device) as Arc>); } } @@ -1212,6 +1267,7 @@ impl DeviceManager { fn make_virtio_vhost_user_blk_devices( vm_info: &VmInfo, + migratable_devices: &mut Vec>>, ) -> DeviceManagerResult> { let mut devices = Vec::new(); // Add vhost-user-blk if required @@ -1231,6 +1287,9 @@ impl DeviceManager { Arc::clone(&vhost_user_blk_device) as Arc>, false, )); + + migratable_devices + .push(Arc::clone(&vhost_user_blk_device) as Arc>); } } @@ -1239,6 +1298,7 @@ impl DeviceManager { fn make_virtio_vsock_devices( vm_info: &VmInfo, + migratable_devices: &mut Vec>>, ) -> DeviceManagerResult> { let mut devices = Vec::new(); // Add vsock if required @@ -1261,6 +1321,8 @@ impl DeviceManager { Arc::clone(&vsock_device) as Arc>, false, )); + + migratable_devices.push(Arc::clone(&vsock_device) as Arc>); } } @@ -1373,6 +1435,7 @@ impl DeviceManager { vm_fd: &Arc, pci: &mut PciBus, iommu_mapping: &Option>, + migratable_devices: &mut Vec>>, ) -> DeviceManagerResult> { // Allows support for one MSI-X vector per queue. It also adds 1 // as we need to take into account the dedicated vector to notify @@ -1471,6 +1534,8 @@ impl DeviceManager { ) .map_err(DeviceManagerError::AddPciDevice)?; + migratable_devices.push(Arc::clone(&virtio_pci_device) as Arc>); + let ret = if iommu_mapping.is_some() { Some(dev_id) } else { @@ -1490,6 +1555,7 @@ impl DeviceManager { interrupt_info: &InterruptInfo, mmio_base: GuestAddress, cmdline_additions: &mut Vec, + migratable_devices: &mut Vec>>, ) -> DeviceManagerResult<()> { let mut mmio_device = vm_virtio::transport::MmioDevice::new(memory.clone(), virtio_device) .map_err(DeviceManagerError::VirtioDevice)?; @@ -1515,9 +1581,10 @@ impl DeviceManager { mmio_device.assign_interrupt(interrupt); + let mmio_device_arc = Arc::new(Mutex::new(mmio_device)); address_manager .mmio_bus - .insert(Arc::new(Mutex::new(mmio_device)), mmio_base.0, MMIO_LEN) + .insert(mmio_device_arc.clone(), mmio_base.0, MMIO_LEN) .map_err(DeviceManagerError::BusError)?; cmdline_additions.push(format!( @@ -1527,6 +1594,8 @@ impl DeviceManager { irq_num )); + migratable_devices.push(Arc::clone(&mmio_device_arc) as Arc>); + Ok(()) }