pci: Allow for registering IO and Memory BAR

This patch adds the support for both IO and Memory BARs by expecting
the function allocate_bars() to identify the type of each BAR.
Based on the type, register_mapping() insert the address range on the
appropriate bus (PIO or MMIO).

Signed-off-by: Sebastien Boeuf <sebastien.boeuf@intel.com>

pci/src/bus.rs

@@ -2,7 +2,9 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE-BSD-3-Clause file.
 
-use crate::configuration::{PciBridgeSubclass, PciClassCode, PciConfiguration, PciHeaderType};
+use crate::configuration::{
+    PciBarRegionType, PciBridgeSubclass, PciClassCode, PciConfiguration, PciHeaderType,
+};
 use crate::device::{Error as PciDeviceError, PciDevice};
 use byteorder::{ByteOrder, LittleEndian};
 use devices::BusDevice;
@@ -21,6 +23,8 @@ pub enum PciRootError {
     AllocateDeviceAddrs(PciDeviceError),
     /// Could not allocate an IRQ number.
     AllocateIrq,
+    /// Could not add a device to the port io bus.
+    PioInsert(devices::BusError),
     /// Could not add a device to the mmio bus.
     MmioInsert(devices::BusError),
 }
@@ -89,12 +93,23 @@ impl PciConfigIo {
     pub fn register_mapping(
         &self,
         dev: Arc<Mutex<dyn BusDevice>>,
-        bus: &mut devices::Bus,
+        io_bus: &mut devices::Bus,
-        bars: Vec<(GuestAddress, GuestUsize)>,
+        mmio_bus: &mut devices::Bus,
+        bars: Vec<(GuestAddress, GuestUsize, PciBarRegionType)>,
     ) -> Result<()> {
-        for (address, size) in bars {
+        for (address, size, type_) in bars {
-            bus.insert(dev.clone(), address.raw_value(), size)
+            match type_ {
-                .map_err(PciRootError::MmioInsert)?;
+                PciBarRegionType::IORegion => {
+                    io_bus
+                        .insert(dev.clone(), address.raw_value(), size)
+                        .map_err(PciRootError::PioInsert)?;
+                }
+                PciBarRegionType::Memory32BitRegion | PciBarRegionType::Memory64BitRegion => {
+                    mmio_bus
+                        .insert(dev.clone(), address.raw_value(), size)
+                        .map_err(PciRootError::MmioInsert)?;
+                }
+            }
         }
         Ok(())
     }

pci/src/device.rs

@@ -2,7 +2,7 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE-BSD-3-Clause file.
 
-use crate::configuration;
+use crate::configuration::{self, PciBarRegionType};
 use crate::msix::MsixTableEntry;
 use crate::PciInterruptPin;
 use devices::BusDevice;
@@ -66,7 +66,7 @@ pub trait PciDevice: BusDevice {
     fn allocate_bars(
         &mut self,
         _allocator: &mut SystemAllocator,
-    ) -> Result<Vec<(GuestAddress, GuestUsize)>> {
+    ) -> Result<Vec<(GuestAddress, GuestUsize, PciBarRegionType)>> {
         Ok(Vec::new())
     }
 

vm-virtio/src/transport/pci_device.rs

@@ -20,9 +20,9 @@ use std::sync::Mutex;
 use devices::BusDevice;
 use pci::{
     InterruptDelivery, InterruptParameters, MsixCap, MsixConfig, PciBarConfiguration,
-    PciCapability, PciCapabilityID, PciClassCode, PciConfiguration, PciDevice, PciDeviceError,
+    PciBarRegionType, PciCapability, PciCapabilityID, PciClassCode, PciConfiguration, PciDevice,
-    PciHeaderType, PciInterruptPin, PciMassStorageSubclass, PciNetworkControllerSubclass,
+    PciDeviceError, PciHeaderType, PciInterruptPin, PciMassStorageSubclass,
-    PciSubclass,
+    PciNetworkControllerSubclass, PciSubclass,
 };
 use vm_allocator::SystemAllocator;
 use vm_memory::{Address, ByteValued, GuestAddress, GuestMemoryMmap, GuestUsize, Le32};
@@ -444,7 +444,8 @@ impl PciDevice for VirtioPciDevice {
     fn allocate_bars(
         &mut self,
         allocator: &mut SystemAllocator,
-    ) -> std::result::Result<Vec<(GuestAddress, GuestUsize)>, PciDeviceError> {
+    ) -> std::result::Result<Vec<(GuestAddress, GuestUsize, PciBarRegionType)>, PciDeviceError>
+    {
         let mut ranges = Vec::new();
 
         // Allocate the virtio-pci capability BAR.
@@ -461,7 +462,11 @@ impl PciDevice for VirtioPciDevice {
                 PciDeviceError::IoRegistrationFailed(virtio_pci_bar_addr.raw_value(), e)
             })? as u8;
 
-        ranges.push((virtio_pci_bar_addr, CAPABILITY_BAR_SIZE));
+        ranges.push((
+            virtio_pci_bar_addr,
+            CAPABILITY_BAR_SIZE,
+            PciBarRegionType::Memory64BitRegion,
+        ));
 
         // Once the BARs are allocated, the capabilities can be added to the PCI configuration.
         self.add_pci_capabilities(virtio_pci_bar)?;
@@ -475,7 +480,11 @@ impl PciDevice for VirtioPciDevice {
             let _device_bar = self.configuration.add_pci_bar(&config).map_err(|e| {
                 PciDeviceError::IoRegistrationFailed(device_bar_addr.raw_value(), e)
             })?;
-            ranges.push((device_bar_addr, config.get_size()));
+            ranges.push((
+                device_bar_addr,
+                config.get_size(),
+                PciBarRegionType::Memory64BitRegion,
+            ));
         }
 
         Ok(ranges)

vmm/src/vm.rs

@@ -414,6 +414,11 @@ impl Vcpu {
     }
 }
 
+struct BusInfo<'a> {
+    io: &'a mut devices::Bus,
+    mmio: &'a mut devices::Bus,
+}
+
 struct InterruptInfo<'a> {
     msi_capable: bool,
     ioapic: &'a Option<Arc<Mutex<ioapic::Ioapic>>>,
@@ -488,9 +493,14 @@ impl DeviceManager {
         msi_capable: bool,
         userspace_ioapic: bool,
     ) -> DeviceManagerResult<Self> {
-        let io_bus = devices::Bus::new();
+        let mut io_bus = devices::Bus::new();
         let mut mmio_bus = devices::Bus::new();
 
+        let mut buses = BusInfo {
+            io: &mut io_bus,
+            mmio: &mut mmio_bus,
+        };
+
         let ioapic = if userspace_ioapic {
             // Create IOAPIC
             Some(Arc::new(Mutex::new(ioapic::Ioapic::new(vm_fd.clone()))))
@@ -578,7 +588,7 @@ impl DeviceManager {
                     allocator,
                     vm_fd,
                     &mut pci,
-                    &mut mmio_bus,
+                    &mut buses,
                     &interrupt_info,
                 )?;
             }
@@ -605,7 +615,7 @@ impl DeviceManager {
                     allocator,
                     vm_fd,
                     &mut pci,
-                    &mut mmio_bus,
+                    &mut buses,
                     &interrupt_info,
                 )?;
             }
@@ -622,7 +632,7 @@ impl DeviceManager {
                 allocator,
                 vm_fd,
                 &mut pci,
-                &mut mmio_bus,
+                &mut buses,
                 &interrupt_info,
             )?;
         }
@@ -645,7 +655,7 @@ impl DeviceManager {
                         allocator,
                         vm_fd,
                         &mut pci,
-                        &mut mmio_bus,
+                        &mut buses,
                         &interrupt_info,
                     )?;
                 }
@@ -710,7 +720,7 @@ impl DeviceManager {
                     allocator,
                     vm_fd,
                     &mut pci,
-                    &mut mmio_bus,
+                    &mut buses,
                     &interrupt_info,
                 )?;
             }
@@ -735,7 +745,7 @@ impl DeviceManager {
         allocator: &mut SystemAllocator,
         vm_fd: &Arc<VmFd>,
         pci: &mut PciConfigIo,
-        mmio_bus: &mut devices::Bus,
+        buses: &mut BusInfo,
         interrupt_info: &InterruptInfo,
     ) -> DeviceManagerResult<()> {
         let msix_num = if interrupt_info.msi_capable {
@@ -828,8 +838,13 @@ impl DeviceManager {
         pci.add_device(virtio_pci_device.clone())
             .map_err(DeviceManagerError::AddPciDevice)?;
 
-        pci.register_mapping(virtio_pci_device.clone(), mmio_bus, bars)
+        pci.register_mapping(
-            .map_err(DeviceManagerError::AddPciDevice)?;
+            virtio_pci_device.clone(),
+            &mut buses.io,
+            &mut buses.mmio,
+            bars,
+        )
+        .map_err(DeviceManagerError::AddPciDevice)?;
 
         Ok(())
     }