vmm: Add fast path for PCI config IO port

Looking up devices on the port I/O bus is time consuming during the
boot at there is an O(lg n) tree lookup and the overhead from taking a
lock on the bus contents.

Avoid this by adding a fast path uses the hardcoded port address and
size and directs PCI config requests directly to the device.

Command line:
target/release/cloud-hypervisor --kernel ~/src/linux/vmlinux --cmdline "root=/dev/vda1 console=ttyS0" --serial tty --console off --disk path=~/workloads/focal-server-cloudimg-amd64-custom-20210609-0.raw --api-socket /tmp/api

PIO exit: 17913
PCI fast path: 17871
Percentage on fast path: 99.8%

perf before:

marvin:~/src/cloud-hypervisor (main *)$ perf report -g | grep resolve
     6.20%     6.20%  vcpu0            cloud-hypervisor    [.] vm_device::bus::Bus::resolve

perf after:

marvin:~/src/cloud-hypervisor (2021-09-17-ioapic-fast-path *)$ perf report -g | grep resolve
     0.08%     0.08%  vcpu0            cloud-hypervisor    [.] vm_device::bus::Bus::resolve

The compromise required to implement this fast path is bringing the
creation of the PciConfigIo device into the DeviceManager::new() so that
it can be used in the VmmOps struct which is created before
DeviceManager::create_devices() is called.

Signed-off-by: Rob Bradford <robert.bradford@intel.com>

pci/src/bus.rs

@@ -188,20 +188,25 @@ impl PciBus {
     }
 }
 
+#[derive(Default)]
 pub struct PciConfigIo {
     /// Config space register.
     config_address: u32,
-    pci_bus: Arc<Mutex<PciBus>>,
+    pci_bus: Option<Arc<Mutex<PciBus>>>,
 }
 
 impl PciConfigIo {
-    pub fn new(pci_bus: Arc<Mutex<PciBus>>) -> Self {
+    pub fn new() -> Self {
         PciConfigIo {
-            pci_bus,
             config_address: 0,
+            pci_bus: None,
         }
     }
 
+    pub fn set_bus(&mut self, pci_bus: Arc<Mutex<PciBus>>) {
+        self.pci_bus = Some(pci_bus)
+    }
+
     pub fn config_space_read(&self) -> u32 {
         let enabled = (self.config_address & 0x8000_0000) != 0;
         if !enabled {
@@ -222,6 +227,8 @@ impl PciConfigIo {
         }
 
         self.pci_bus
+            .as_ref()
+            .unwrap()
             .lock()
             .unwrap()
             .devices
@@ -249,7 +256,7 @@ impl PciConfigIo {
             return None;
         }
 
-        let pci_bus = self.pci_bus.lock().unwrap();
+        let pci_bus = self.pci_bus.as_ref().unwrap().lock().unwrap();
         if let Some(d) = pci_bus.devices.get(&(device as u32)) {
             let mut device = d.lock().unwrap();
 

pci/src/lib.rs

@@ -42,3 +42,8 @@ impl PciInterruptPin {
         self as u32
     }
 }
+
+#[cfg(target_arch = "x86_64")]
+pub const PCI_CONFIG_IO_PORT: u64 = 0xcf8;
+#[cfg(target_arch = "x86_64")]
+pub const PCI_CONFIG_IO_PORT_SIZE: u64 = 0x8;