pci: Enable GSI routing (MSI type) for AArch64

In this commit we saved the BDF of a PCI device and set it to "devid"
in GSI routing entry, because this field is mandatory for GICv3-ITS.

Signed-off-by: Michael Zhao <michael.zhao@arm.com>

devices/src/ioapic.rs

@@ -344,6 +344,7 @@ impl Ioapic {
             high_addr: 0x0,
             low_addr,
             data,
+            devid: 0,
         };
 
         self.interrupt_source_group

hypervisor/src/kvm/mod.rs

@@ -47,7 +47,7 @@ pub mod aarch64;
 pub use kvm_bindings;
 pub use kvm_bindings::{
     kvm_create_device, kvm_device_type_KVM_DEV_TYPE_VFIO, kvm_irq_routing, kvm_irq_routing_entry,
-    kvm_userspace_memory_region, KVM_IRQ_ROUTING_MSI, KVM_MEM_READONLY,
+    kvm_userspace_memory_region, KVM_IRQ_ROUTING_MSI, KVM_MEM_READONLY, KVM_MSI_VALID_DEVID,
 };
 pub use kvm_ioctls;
 pub use kvm_ioctls::{Cap, Kvm};

pci/src/msi.rs

@@ -201,6 +201,7 @@ impl MsiConfig {
                     high_addr: self.cap.msg_addr_hi,
                     low_addr: self.cap.msg_addr_lo,
                     data: self.cap.msg_data as u32,
+                    devid: 0,
                 };
 
                 if let Err(e) = self

pci/src/msix.rs

@@ -72,6 +72,7 @@ struct MsixConfigState {
 pub struct MsixConfig {
     pub table_entries: Vec<MsixTableEntry>,
     pub pba_entries: Vec<u64>,
+    pub devid: u32,
     interrupt_source_group: Arc<Box<dyn InterruptSourceGroup>>,
     masked: bool,
     enabled: bool,
@@ -81,6 +82,7 @@ impl MsixConfig {
     pub fn new(
         msix_vectors: u16,
         interrupt_source_group: Arc<Box<dyn InterruptSourceGroup>>,
+        devid: u32,
     ) -> Self {
         assert!(msix_vectors <= MAX_MSIX_VECTORS_PER_DEVICE);
 
@@ -93,6 +95,7 @@ impl MsixConfig {
         MsixConfig {
             table_entries,
             pba_entries,
+            devid,
             interrupt_source_group,
             masked: false,
             enabled: false,
@@ -124,6 +127,7 @@ impl MsixConfig {
                     high_addr: table_entry.msg_addr_hi,
                     low_addr: table_entry.msg_addr_lo,
                     data: table_entry.msg_data,
+                    devid: self.devid,
                 };
 
                 self.interrupt_source_group
@@ -162,6 +166,7 @@ impl MsixConfig {
                         high_addr: table_entry.msg_addr_hi,
                         low_addr: table_entry.msg_addr_lo,
                         data: table_entry.msg_data,
+                        devid: self.devid,
                     };
 
                     if let Err(e) = self
@@ -306,6 +311,7 @@ impl MsixConfig {
                 high_addr: table_entry.msg_addr_hi,
                 low_addr: table_entry.msg_addr_lo,
                 data: table_entry.msg_data,
+                devid: self.devid,
             };
 
             if let Err(e) = self.interrupt_source_group.update(

pci/src/vfio.rs

@@ -363,7 +363,7 @@ impl VfioPciDevice {
             })
             .unwrap();
 
-        let msix_config = MsixConfig::new(msix_cap.table_size(), interrupt_source_group.clone());
+        let msix_config = MsixConfig::new(msix_cap.table_size(), interrupt_source_group.clone(), 0);
 
         self.interrupt.msix = Some(VfioMsix {
             bar: msix_config,

virtio-devices/src/transport/pci_device.rs

@@ -336,6 +336,7 @@ impl VirtioPciDevice {
         msix_num: u16,
         iommu_mapping_cb: Option<Arc<VirtioIommuRemapping>>,
         interrupt_manager: &Arc<dyn InterruptManager<GroupConfig = MsiIrqGroupConfig>>,
+        pci_device_bdf: u32,
     ) -> Result<Self> {
         let device_clone = device.clone();
         let locked_device = device_clone.lock().unwrap();
@@ -364,6 +365,7 @@ impl VirtioPciDevice {
             let msix_config = Arc::new(Mutex::new(MsixConfig::new(
                 msix_num,
                 interrupt_source_group.clone(),
+                pci_device_bdf,
             )));
             let msix_config_clone = msix_config.clone();
             (Some(msix_config), Some(msix_config_clone))

vm-device/src/interrupt/mod.rs

@@ -83,6 +83,8 @@ pub struct MsiIrqSourceConfig {
     pub low_addr: u32,
     /// Data to write to delivery message signaled interrupt.
     pub data: u32,
+    /// Unique ID of the device to delivery message signaled interrupt.
+    pub devid: u32,
 }
 
 /// Configuration data for an interrupt source.

vmm/src/device_manager.rs

@@ -2615,6 +2615,7 @@ impl DeviceManager {
             msix_num,
             iommu_mapping_cb,
             interrupt_manager,
+            pci_device_bdf,
         )
         .map_err(DeviceManagerError::VirtioDevice)?;
 

vmm/src/interrupt.rs

@@ -4,7 +4,6 @@
 //
 
 use devices::interrupt_controller::InterruptController;
-
 use std::collections::HashMap;
 use std::io;
 use std::mem::size_of;
@@ -118,7 +117,11 @@ pub trait MsiInterruptGroupOps {
 }
 
 pub trait RoutingEntryExt {
-    fn make_entry(gsi: u32, config: &InterruptSourceConfig) -> Result<Box<Self>>;
+    fn make_entry(
+        vm: &Arc<dyn hypervisor::Vm>,
+        gsi: u32,
+        config: &InterruptSourceConfig,
+    ) -> Result<Box<Self>>;
 }
 
 impl<E> MsiInterruptGroup<E> {
@@ -178,7 +181,7 @@ where
 
     fn update(&self, index: InterruptIndex, config: InterruptSourceConfig) -> Result<()> {
         if let Some(route) = self.irq_routes.get(&index) {
-            let entry = RoutingEntry::<_>::make_entry(route.gsi, &config)?;
+            let entry = RoutingEntry::<_>::make_entry(&self.vm, route.gsi, &config)?;
             self.gsi_msi_routes
                 .lock()
                 .unwrap()
@@ -353,6 +356,7 @@ where
 
 pub mod kvm {
     use super::*;
+    use hypervisor::kvm::KVM_MSI_VALID_DEVID;
     use hypervisor::kvm::{kvm_irq_routing, kvm_irq_routing_entry, KVM_IRQ_ROUTING_MSI};
 
     type KvmMsiInterruptGroup = MsiInterruptGroup<kvm_irq_routing_entry>;
@@ -360,7 +364,11 @@ pub mod kvm {
     pub type KvmMsiInterruptManager = MsiInterruptManager<kvm_irq_routing_entry>;
 
     impl RoutingEntryExt for KvmRoutingEntry {
-        fn make_entry(gsi: u32, config: &InterruptSourceConfig) -> Result<Box<Self>> {
+        fn make_entry(
+            vm: &Arc<dyn hypervisor::Vm>,
+            gsi: u32,
+            config: &InterruptSourceConfig,
+        ) -> Result<Box<Self>> {
             if let InterruptSourceConfig::MsiIrq(cfg) = &config {
                 let mut kvm_route = kvm_irq_routing_entry {
                     gsi,
@@ -372,6 +380,11 @@ pub mod kvm {
                 kvm_route.u.msi.address_hi = cfg.high_addr;
                 kvm_route.u.msi.data = cfg.data;
 
+                if vm.check_extension(hypervisor::Cap::MsiDevid) {
+                    kvm_route.flags = KVM_MSI_VALID_DEVID;
+                    kvm_route.u.msi.__bindgen_anon_1.devid = cfg.devid;
+                }
+
                 let kvm_entry = KvmRoutingEntry {
                     route: kvm_route,
                     masked: false,