pci, vmm: Expose the PCI bus for configuration via MMIO

Refactor the PCI datastructures to move the device ownership to a PciBus
struct. This PciBus struct can then be used by both a PciConfigIo and
PciConfigMmio in order to expose the configuration space via both IO
port and also via MMIO for PCI MMCONFIG.

Signed-off-by: Rob Bradford <robert.bradford@intel.com>

pci/src/bus.rs

@@ -71,23 +71,19 @@ impl PciDevice for PciRoot {
     }
 }
 
-pub struct PciConfigIo {
+pub struct PciBus {
     /// Devices attached to this bus.
     /// Device 0 is host bridge.
     devices: Vec<Arc<Mutex<dyn PciDevice>>>,
-    /// Config space register.
-    config_address: u32,
 }
 
-impl PciConfigIo {
+impl PciBus {
     pub fn new(pci_root: PciRoot) -> Self {
         let mut devices: Vec<Arc<Mutex<dyn PciDevice>>> = Vec::new();
+
         devices.push(Arc::new(Mutex::new(pci_root)));
 
-        PciConfigIo {
+        PciBus { devices }
-            devices,
-            config_address: 0,
-        }
     }
 
     pub fn register_mapping(
@@ -118,6 +114,21 @@ impl PciConfigIo {
         self.devices.push(device);
         Ok(())
     }
+}
+
+pub struct PciConfigIo {
+    /// Config space register.
+    config_address: u32,
+    pci_bus: Arc<Mutex<PciBus>>,
+}
+
+impl PciConfigIo {
+    pub fn new(pci_bus: Arc<Mutex<PciBus>>) -> Self {
+        PciConfigIo {
+            pci_bus,
+            config_address: 0,
+        }
+    }
 
     pub fn config_space_read(&self) -> u32 {
         let enabled = (self.config_address & 0x8000_0000) != 0;
@@ -138,9 +149,14 @@ impl PciConfigIo {
             return 0xffff_ffff;
         }
 
-        self.devices.get(device).map_or(0xffff_ffff, |d| {
+        self.pci_bus
-            d.lock().unwrap().read_config_register(register)
+            .lock()
-        })
+            .unwrap()
+            .devices
+            .get(device)
+            .map_or(0xffff_ffff, |d| {
+                d.lock().unwrap().read_config_register(register)
+            })
     }
 
     pub fn config_space_write(&mut self, offset: u64, data: &[u8]) {
@@ -161,7 +177,7 @@ impl PciConfigIo {
             return;
         }
 
-        if let Some(d) = self.devices.get(device) {
+        if let Some(d) = self.pci_bus.lock().unwrap().devices.get(device) {
             d.lock()
                 .unwrap()
                 .write_config_register(register, offset, data);
@@ -223,17 +239,12 @@ impl BusDevice for PciConfigIo {
 
 /// Emulates PCI memory-mapped configuration access mechanism.
 pub struct PciConfigMmio {
-    /// Devices attached to this bus.
+    pci_bus: Arc<Mutex<PciBus>>,
-    /// Device 0 is host bridge.
-    devices: Vec<Arc<Mutex<dyn PciDevice>>>,
 }
 
 impl PciConfigMmio {
-    pub fn new(pci_root: PciRoot) -> Self {
+    pub fn new(pci_bus: Arc<Mutex<PciBus>>) -> Self {
-        let mut devices: Vec<Arc<Mutex<dyn PciDevice>>> = Vec::new();
+        PciConfigMmio { pci_bus }
-
-        devices.push(Arc::new(Mutex::new(pci_root)));
-        PciConfigMmio { devices }
     }
 
     fn config_space_read(&self, config_address: u32) -> u32 {
@@ -244,9 +255,14 @@ impl PciConfigMmio {
             return 0xffff_ffff;
         }
 
-        self.devices.get(device).map_or(0xffff_ffff, |d| {
+        self.pci_bus
-            d.lock().unwrap().read_config_register(register)
+            .lock()
-        })
+            .unwrap()
+            .devices
+            .get(device)
+            .map_or(0xffff_ffff, |d| {
+                d.lock().unwrap().read_config_register(register)
+            })
     }
 
     fn config_space_write(&mut self, config_address: u32, offset: u64, data: &[u8]) {
@@ -261,7 +277,7 @@ impl PciConfigMmio {
             return;
         }
 
-        if let Some(d) = self.devices.get(device) {
+        if let Some(d) = self.pci_bus.lock().unwrap().devices.get(device) {
             d.lock()
                 .unwrap()
                 .write_config_register(register, offset, data);

pci/src/lib.rs

@@ -16,7 +16,7 @@ mod device;
 mod msi;
 mod msix;
 
-pub use self::bus::{PciConfigIo, PciConfigMmio, PciRoot, PciRootError};
+pub use self::bus::{PciBus, PciConfigIo, PciConfigMmio, PciRoot, PciRootError};
 pub use self::configuration::{
     PciBarConfiguration, PciBarPrefetchable, PciBarRegionType, PciCapability, PciCapabilityID,
     PciClassCode, PciConfiguration, PciHeaderType, PciMassStorageSubclass,

vmm/src/device_manager.rs

@@ -21,7 +21,8 @@ use libc::{EFD_NONBLOCK, TIOCGWINSZ};
 use net_util::Tap;
 #[cfg(feature = "pci_support")]
 use pci::{
-    InterruptDelivery, InterruptParameters, PciConfigIo, PciDevice, PciInterruptPin, PciRoot,
+    InterruptDelivery, InterruptParameters, PciBus, PciConfigIo, PciConfigMmio, PciDevice,
+    PciInterruptPin, PciRoot,
 };
 use qcow::{self, ImageType, QcowFile};
 
@@ -428,7 +429,7 @@ impl DeviceManager {
             #[cfg(feature = "pci_support")]
             {
                 let pci_root = PciRoot::new(None);
-                let mut pci = PciConfigIo::new(pci_root);
+                let mut pci_bus = PciBus::new(pci_root);
 
                 for device in virtio_devices {
                     DeviceManager::add_virtio_pci_device(
@@ -436,18 +437,32 @@ impl DeviceManager {
                         vm_info.memory,
                         allocator,
                         vm_info.vm_fd,
-                        &mut pci,
+                        &mut pci_bus,
                         &mut buses,
                         &interrupt_info,
                     )?;
                 }
 
                 DeviceManager::add_vfio_devices(
-                    vm_info, allocator, &mut pci, &mut buses, mem_slots,
+                    vm_info,
+                    allocator,
+                    &mut pci_bus,
+                    &mut buses,
+                    mem_slots,
                 )?;
-                let pci = Arc::new(Mutex::new(pci));
+
+                let pci_bus = Arc::new(Mutex::new(pci_bus));
+                let pci_config_io = Arc::new(Mutex::new(PciConfigIo::new(pci_bus.clone())));
                 io_bus
-                    .insert(pci, 0xcf8, 0x8)
+                    .insert(pci_config_io, 0xcf8, 0x8)
+                    .map_err(DeviceManagerError::BusError)?;
+                let pci_config_mmio = Arc::new(Mutex::new(PciConfigMmio::new(pci_bus)));
+                mmio_bus
+                    .insert(
+                        pci_config_mmio,
+                        arch::layout::PCI_MMCONFIG_START.0,
+                        arch::layout::PCI_MMCONFIG_SIZE,
+                    )
                     .map_err(DeviceManagerError::BusError)?;
             }
         } else if cfg!(feature = "mmio_support") {
@@ -845,7 +860,7 @@ impl DeviceManager {
     fn add_vfio_devices(
         vm_info: &VmInfo,
         allocator: &mut SystemAllocator,
-        pci: &mut PciConfigIo,
+        pci: &mut PciBus,
         buses: &mut BusInfo,
         mem_slots: u32,
     ) -> DeviceManagerResult<()> {
@@ -889,7 +904,7 @@ impl DeviceManager {
         memory: &Arc<RwLock<GuestMemoryMmap>>,
         allocator: &mut SystemAllocator,
         vm_fd: &Arc<VmFd>,
-        pci: &mut PciConfigIo,
+        pci: &mut PciBus,
         buses: &mut BusInfo,
         interrupt_info: &InterruptInfo,
     ) -> DeviceManagerResult<()> {