From a86f4369a7cd147b5b0983d0084a6923e911ecab Mon Sep 17 00:00:00 2001 From: Sebastien Boeuf Date: Fri, 28 Feb 2020 12:29:43 +0100 Subject: [PATCH] vmm: Add VFIO PCI device hotplug support This commit finalizes the VFIO PCI hotplug support, based on all the previous commits preparing for it. One thing to notice, this does not support vIOMMU yet. This means we can hotplug VFIO PCI devices, but we cannot attach them to an existing or a new virtio-iommu device. Signed-off-by: Sebastien Boeuf --- vmm/src/device_manager.rs | 92 ++++++++++++++++++++++++++++++++++++++- vmm/src/vm.rs | 34 ++++++++++----- 2 files changed, 112 insertions(+), 14 deletions(-) diff --git a/vmm/src/device_manager.rs b/vmm/src/device_manager.rs index d639da3de..f60af2019 100644 --- a/vmm/src/device_manager.rs +++ b/vmm/src/device_manager.rs @@ -211,6 +211,9 @@ pub enum DeviceManagerError { /// Failed to spawn the block backend SpawnBlockBackend(io::Error), + + /// Missing PCI bus. + NoPciBus, } pub type DeviceManagerResult = result::Result; @@ -447,6 +450,14 @@ pub struct DeviceManager { // Paravirtualized IOMMU #[cfg(feature = "pci_support")] iommu_device: Option>>, + + // Bitmap of PCI devices to hotplug. + #[cfg(feature = "pci_support")] + pci_devices_up: u32, + + // Bitmap of PCI devices to hotunplug. + #[cfg(feature = "pci_support")] + pci_devices_down: u32, } impl DeviceManager { @@ -541,6 +552,10 @@ impl DeviceManager { kvm_device_fd: None, #[cfg(feature = "pci_support")] iommu_device: None, + #[cfg(feature = "pci_support")] + pci_devices_up: 0, + #[cfg(feature = "pci_support")] + pci_devices_down: 0, }; device_manager @@ -1632,7 +1647,39 @@ impl DeviceManager { return Ok(()); } - pub fn add_device(&self, _path: String) -> DeviceManagerResult<()> { + #[cfg(feature = "pci_support")] + pub fn add_device(&mut self, path: String) -> DeviceManagerResult<()> { + let device_cfg = DeviceConfig { + path: PathBuf::from(path), + iommu: false, + }; + + let mut pci = if let Some(pci_bus) = &self.pci_bus { + pci_bus.lock().unwrap() + } else { + return Err(DeviceManagerError::NoPciBus); + }; + + let interrupt_manager = Arc::clone(&self.msi_interrupt_manager); + + let device_fd = if let Some(device_fd) = &self.kvm_device_fd { + Arc::clone(&device_fd) + } else { + // If the VFIO KVM device file descriptor has not been created yet, + // it is created here and stored in the DeviceManager structure for + // future needs. + let device_fd = DeviceManager::create_kvm_device(&self.address_manager.vm_fd)?; + let device_fd = Arc::new(device_fd); + self.kvm_device_fd = Some(Arc::clone(&device_fd)); + device_fd + }; + + let device_id = + self.add_vfio_device(&mut pci, &interrupt_manager, &device_fd, &device_cfg)?; + + // Update the PCIU bitmap + self.pci_devices_up |= 1 << (device_id >> 3); + Ok(()) } } @@ -1911,7 +1958,48 @@ impl Pausable for DeviceManager { impl Snapshotable for DeviceManager {} impl Migratable for DeviceManager {} -impl BusDevice for DeviceManager {} +#[cfg(feature = "pci_support")] +const PCIU_FIELD_OFFSET: u64 = 0; +#[cfg(feature = "pci_support")] +const PCID_FIELD_OFFSET: u64 = 4; + +#[cfg(feature = "pci_support")] +const PCIU_FIELD_SIZE: usize = 4; +#[cfg(feature = "pci_support")] +const PCID_FIELD_SIZE: usize = 4; + +impl BusDevice for DeviceManager { + fn read(&mut self, base: u64, offset: u64, data: &mut [u8]) { + #[cfg(feature = "pci_support")] + match offset { + PCIU_FIELD_OFFSET => { + assert!(data.len() == PCIU_FIELD_SIZE); + data.copy_from_slice(&self.pci_devices_up.to_le_bytes()); + // Clear the PCIU bitmap + self.pci_devices_up = 0; + } + PCID_FIELD_OFFSET => { + assert!(data.len() == PCID_FIELD_SIZE); + data.copy_from_slice(&self.pci_devices_down.to_le_bytes()); + // Clear the PCID bitmap + self.pci_devices_down = 0; + } + _ => {} + } + + debug!( + "PCI_HP_REG_R: base 0x{:x}, offset 0x{:x}, data {:?}", + base, offset, data + ) + } + + fn write(&mut self, base: u64, offset: u64, data: &[u8]) { + debug!( + "PCI_HP_REG_W: base 0x{:x}, offset 0x{:x}, data {:?}", + base, offset, data + ) + } +} impl Drop for DeviceManager { fn drop(&mut self) { diff --git a/vmm/src/vm.rs b/vmm/src/vm.rs index a813c091c..baf5de088 100755 --- a/vmm/src/vm.rs +++ b/vmm/src/vm.rs @@ -164,6 +164,9 @@ pub enum Error { /// Memory manager error MemoryManager(MemoryManagerError), + + /// No PCI support + NoPciSupport, } pub type Result = result::Result; @@ -540,20 +543,27 @@ impl Vm { Ok(()) } - pub fn add_device(&mut self, path: String) -> Result<()> { - self.devices - .lock() - .unwrap() - .add_device(path) - .map_err(Error::DeviceManager)?; + pub fn add_device(&mut self, _path: String) -> Result<()> { + if cfg!(feature = "pci_support") { + #[cfg(feature = "pci_support")] + { + self.devices + .lock() + .unwrap() + .add_device(_path) + .map_err(Error::DeviceManager)?; - self.devices - .lock() - .unwrap() - .notify_hotplug(HotPlugNotificationFlags::PCI_DEVICES_CHANGED) - .map_err(Error::DeviceManager)?; + self.devices + .lock() + .unwrap() + .notify_hotplug(HotPlugNotificationFlags::PCI_DEVICES_CHANGED) + .map_err(Error::DeviceManager)?; + } - Ok(()) + Ok(()) + } else { + Err(Error::NoPciSupport) + } } fn os_signal_handler(signals: Signals, console_input_clone: Arc, on_tty: bool) {