From cce6237536bbfd6c397f86baa56254c7573f732a Mon Sep 17 00:00:00 2001 From: Michael Zhao Date: Wed, 17 Jun 2020 21:58:15 +0800 Subject: [PATCH] pci: Enable GSI routing (MSI type) for AArch64 In this commit we saved the BDF of a PCI device and set it to "devid" in GSI routing entry, because this field is mandatory for GICv3-ITS. Signed-off-by: Michael Zhao --- devices/src/ioapic.rs | 1 + hypervisor/src/kvm/mod.rs | 2 +- pci/src/msi.rs | 1 + pci/src/msix.rs | 6 ++++++ pci/src/vfio.rs | 2 +- virtio-devices/src/transport/pci_device.rs | 2 ++ vm-device/src/interrupt/mod.rs | 2 ++ vmm/src/device_manager.rs | 1 + vmm/src/interrupt.rs | 21 +++++++++++++++++---- 9 files changed, 32 insertions(+), 6 deletions(-) diff --git a/devices/src/ioapic.rs b/devices/src/ioapic.rs index 565937f7a..d25f5903b 100644 --- a/devices/src/ioapic.rs +++ b/devices/src/ioapic.rs @@ -344,6 +344,7 @@ impl Ioapic { high_addr: 0x0, low_addr, data, + devid: 0, }; self.interrupt_source_group diff --git a/hypervisor/src/kvm/mod.rs b/hypervisor/src/kvm/mod.rs index 87b25a741..7ec67510b 100644 --- a/hypervisor/src/kvm/mod.rs +++ b/hypervisor/src/kvm/mod.rs @@ -47,7 +47,7 @@ pub mod aarch64; pub use kvm_bindings; pub use kvm_bindings::{ kvm_create_device, kvm_device_type_KVM_DEV_TYPE_VFIO, kvm_irq_routing, kvm_irq_routing_entry, - kvm_userspace_memory_region, KVM_IRQ_ROUTING_MSI, KVM_MEM_READONLY, + kvm_userspace_memory_region, KVM_IRQ_ROUTING_MSI, KVM_MEM_READONLY, KVM_MSI_VALID_DEVID, }; pub use kvm_ioctls; pub use kvm_ioctls::{Cap, Kvm}; diff --git a/pci/src/msi.rs b/pci/src/msi.rs index 05494cdac..61f7c09d7 100644 --- a/pci/src/msi.rs +++ b/pci/src/msi.rs @@ -201,6 +201,7 @@ impl MsiConfig { high_addr: self.cap.msg_addr_hi, low_addr: self.cap.msg_addr_lo, data: self.cap.msg_data as u32, + devid: 0, }; if let Err(e) = self diff --git a/pci/src/msix.rs b/pci/src/msix.rs index f37a7c138..1ff706412 100644 --- a/pci/src/msix.rs +++ b/pci/src/msix.rs @@ -72,6 +72,7 @@ struct MsixConfigState { pub struct MsixConfig { pub table_entries: Vec, pub pba_entries: Vec, + pub devid: u32, interrupt_source_group: Arc>, masked: bool, enabled: bool, @@ -81,6 +82,7 @@ impl MsixConfig { pub fn new( msix_vectors: u16, interrupt_source_group: Arc>, + devid: u32, ) -> Self { assert!(msix_vectors <= MAX_MSIX_VECTORS_PER_DEVICE); @@ -93,6 +95,7 @@ impl MsixConfig { MsixConfig { table_entries, pba_entries, + devid, interrupt_source_group, masked: false, enabled: false, @@ -124,6 +127,7 @@ impl MsixConfig { high_addr: table_entry.msg_addr_hi, low_addr: table_entry.msg_addr_lo, data: table_entry.msg_data, + devid: self.devid, }; self.interrupt_source_group @@ -162,6 +166,7 @@ impl MsixConfig { high_addr: table_entry.msg_addr_hi, low_addr: table_entry.msg_addr_lo, data: table_entry.msg_data, + devid: self.devid, }; if let Err(e) = self @@ -306,6 +311,7 @@ impl MsixConfig { high_addr: table_entry.msg_addr_hi, low_addr: table_entry.msg_addr_lo, data: table_entry.msg_data, + devid: self.devid, }; if let Err(e) = self.interrupt_source_group.update( diff --git a/pci/src/vfio.rs b/pci/src/vfio.rs index 23fa8eb25..ba83c1d8f 100644 --- a/pci/src/vfio.rs +++ b/pci/src/vfio.rs @@ -363,7 +363,7 @@ impl VfioPciDevice { }) .unwrap(); - let msix_config = MsixConfig::new(msix_cap.table_size(), interrupt_source_group.clone()); + let msix_config = MsixConfig::new(msix_cap.table_size(), interrupt_source_group.clone(), 0); self.interrupt.msix = Some(VfioMsix { bar: msix_config, diff --git a/virtio-devices/src/transport/pci_device.rs b/virtio-devices/src/transport/pci_device.rs index a0a942a72..e293ab66b 100644 --- a/virtio-devices/src/transport/pci_device.rs +++ b/virtio-devices/src/transport/pci_device.rs @@ -336,6 +336,7 @@ impl VirtioPciDevice { msix_num: u16, iommu_mapping_cb: Option>, interrupt_manager: &Arc>, + pci_device_bdf: u32, ) -> Result { let device_clone = device.clone(); let locked_device = device_clone.lock().unwrap(); @@ -364,6 +365,7 @@ impl VirtioPciDevice { let msix_config = Arc::new(Mutex::new(MsixConfig::new( msix_num, interrupt_source_group.clone(), + pci_device_bdf, ))); let msix_config_clone = msix_config.clone(); (Some(msix_config), Some(msix_config_clone)) diff --git a/vm-device/src/interrupt/mod.rs b/vm-device/src/interrupt/mod.rs index f6f77f693..360d52b48 100644 --- a/vm-device/src/interrupt/mod.rs +++ b/vm-device/src/interrupt/mod.rs @@ -83,6 +83,8 @@ pub struct MsiIrqSourceConfig { pub low_addr: u32, /// Data to write to delivery message signaled interrupt. pub data: u32, + /// Unique ID of the device to delivery message signaled interrupt. + pub devid: u32, } /// Configuration data for an interrupt source. diff --git a/vmm/src/device_manager.rs b/vmm/src/device_manager.rs index 3a33d7c61..2fa4f77a9 100644 --- a/vmm/src/device_manager.rs +++ b/vmm/src/device_manager.rs @@ -2615,6 +2615,7 @@ impl DeviceManager { msix_num, iommu_mapping_cb, interrupt_manager, + pci_device_bdf, ) .map_err(DeviceManagerError::VirtioDevice)?; diff --git a/vmm/src/interrupt.rs b/vmm/src/interrupt.rs index 9f02a1871..fc7734bb2 100644 --- a/vmm/src/interrupt.rs +++ b/vmm/src/interrupt.rs @@ -4,7 +4,6 @@ // use devices::interrupt_controller::InterruptController; - use std::collections::HashMap; use std::io; use std::mem::size_of; @@ -118,7 +117,11 @@ pub trait MsiInterruptGroupOps { } pub trait RoutingEntryExt { - fn make_entry(gsi: u32, config: &InterruptSourceConfig) -> Result>; + fn make_entry( + vm: &Arc, + gsi: u32, + config: &InterruptSourceConfig, + ) -> Result>; } impl MsiInterruptGroup { @@ -178,7 +181,7 @@ where fn update(&self, index: InterruptIndex, config: InterruptSourceConfig) -> Result<()> { if let Some(route) = self.irq_routes.get(&index) { - let entry = RoutingEntry::<_>::make_entry(route.gsi, &config)?; + let entry = RoutingEntry::<_>::make_entry(&self.vm, route.gsi, &config)?; self.gsi_msi_routes .lock() .unwrap() @@ -353,6 +356,7 @@ where pub mod kvm { use super::*; + use hypervisor::kvm::KVM_MSI_VALID_DEVID; use hypervisor::kvm::{kvm_irq_routing, kvm_irq_routing_entry, KVM_IRQ_ROUTING_MSI}; type KvmMsiInterruptGroup = MsiInterruptGroup; @@ -360,7 +364,11 @@ pub mod kvm { pub type KvmMsiInterruptManager = MsiInterruptManager; impl RoutingEntryExt for KvmRoutingEntry { - fn make_entry(gsi: u32, config: &InterruptSourceConfig) -> Result> { + fn make_entry( + vm: &Arc, + gsi: u32, + config: &InterruptSourceConfig, + ) -> Result> { if let InterruptSourceConfig::MsiIrq(cfg) = &config { let mut kvm_route = kvm_irq_routing_entry { gsi, @@ -372,6 +380,11 @@ pub mod kvm { kvm_route.u.msi.address_hi = cfg.high_addr; kvm_route.u.msi.data = cfg.data; + if vm.check_extension(hypervisor::Cap::MsiDevid) { + kvm_route.flags = KVM_MSI_VALID_DEVID; + kvm_route.u.msi.__bindgen_anon_1.devid = cfg.devid; + } + let kvm_entry = KvmRoutingEntry { route: kvm_route, masked: false,