From f2e484750a09b74c5f5fb31803c872626ca802dc Mon Sep 17 00:00:00 2001 From: Michael Zhao Date: Wed, 3 Jun 2020 16:30:33 +0800 Subject: [PATCH] arch: aarch64: Add PCIe node in FDT for AArch64 Signed-off-by: Michael Zhao --- arch/src/aarch64/fdt.rs | 55 +++++++++++++++++++++++++++++++++++++++-- arch/src/aarch64/gic.rs | 23 +++++++++++------ arch/src/aarch64/mod.rs | 12 ++++++--- vmm/src/vm.rs | 30 ++++++++++++++++++++-- 4 files changed, 104 insertions(+), 16 deletions(-) diff --git a/arch/src/aarch64/fdt.rs b/arch/src/aarch64/fdt.rs index 5b0f2d2db..7fde94a2f 100644 --- a/arch/src/aarch64/fdt.rs +++ b/arch/src/aarch64/fdt.rs @@ -17,7 +17,10 @@ use super::super::DeviceType; use super::super::InitramfsConfig; use super::get_fdt_addr; use super::gic::GICDevice; -use super::layout::FDT_MAX_SIZE; +use super::layout::{ + FDT_MAX_SIZE, MEM_32BIT_DEVICES_SIZE, MEM_32BIT_DEVICES_START, PCI_MMCONFIG_SIZE, + PCI_MMCONFIG_START, +}; use crate::aarch64::fdt::Error::CstringFDTTransform; use vm_memory::{Address, Bytes, GuestAddress, GuestMemory, GuestMemoryError, GuestMemoryMmap}; @@ -94,6 +97,7 @@ pub fn create_fdt( device_info: &HashMap<(DeviceType, String), T>, gic_device: &Box, initrd: &Option, + pci_space_address: &Option<(u64, u64)>, ) -> Result> { // Alocate stuff necessary for the holding the blob. let mut fdt = vec![0; FDT_MAX_SIZE]; @@ -122,6 +126,9 @@ pub fn create_fdt( create_clock_node(&mut fdt)?; create_psci_node(&mut fdt)?; create_devices_node(&mut fdt, device_info)?; + if let Some((pci_device_base, pci_device_size)) = pci_space_address { + create_pci_nodes(&mut fdt, *pci_device_base, *pci_device_size)?; + } // End Header node. append_end_node(&mut fdt)?; @@ -540,6 +547,49 @@ fn create_devices_node( Ok(()) } +fn create_pci_nodes(fdt: &mut Vec, pci_device_base: u64, pci_device_size: u64) -> Result<()> { + // Add node for PCIe controller. + // See Documentation/devicetree/bindings/pci/host-generic-pci.txt in the kernel + // and https://elinux.org/Device_Tree_Usage. + let ranges = generate_prop32(&[ + // mmio addresses + 0x2000000, // (ss = 10: 32-bit memory space) + (MEM_32BIT_DEVICES_START.0 >> 32) as u32, // PCI address + MEM_32BIT_DEVICES_START.0 as u32, + (MEM_32BIT_DEVICES_START.0 >> 32) as u32, // CPU address + MEM_32BIT_DEVICES_START.0 as u32, + (MEM_32BIT_DEVICES_SIZE >> 32) as u32, // size + MEM_32BIT_DEVICES_SIZE as u32, + // device addresses + 0x3000000, // (ss = 11: 64-bit memory space) + (pci_device_base >> 32) as u32, // PCI address + pci_device_base as u32, + (pci_device_base >> 32) as u32, // CPU address + pci_device_base as u32, + (pci_device_size >> 32) as u32, // size + pci_device_size as u32, + ]); + let bus_range = generate_prop32(&[0, 0]); // Only bus 0 + let reg = generate_prop64(&[PCI_MMCONFIG_START.0, PCI_MMCONFIG_SIZE]); + + append_begin_node(fdt, "pci")?; + append_property_string(fdt, "compatible", "pci-host-ecam-generic")?; + append_property_string(fdt, "device_type", "pci")?; + append_property(fdt, "ranges", &ranges)?; + append_property(fdt, "bus-range", &bus_range)?; + append_property_u32(fdt, "#address-cells", 3)?; + append_property_u32(fdt, "#size-cells", 2)?; + append_property(fdt, "reg", ®)?; + append_property_u32(fdt, "#interrupt-cells", 1)?; + append_property_null(fdt, "interrupt-map")?; + append_property_null(fdt, "interrupt-map-mask")?; + append_property_null(fdt, "dma-coherent")?; + append_property_u32(fdt, "msi-parent", MSI_PHANDLE)?; + append_end_node(fdt)?; + + Ok(()) +} + #[cfg(test)] mod tests { use super::*; @@ -603,7 +653,7 @@ mod tests { let kvm = hypervisor::kvm::KvmHypervisor::new().unwrap(); let hv: Arc = Arc::new(kvm); let vm = hv.create_vm().unwrap(); - let gic = create_gic(&vm, 1).unwrap(); + let gic = create_gic(&vm, 1, false).unwrap(); assert!(create_fdt( &mem, &CString::new("console=tty0").unwrap(), @@ -611,6 +661,7 @@ mod tests { &dev_info, &gic, &None, + &None, ) .is_ok()) } diff --git a/arch/src/aarch64/gic.rs b/arch/src/aarch64/gic.rs index 3e603b004..46825d3a2 100644 --- a/arch/src/aarch64/gic.rs +++ b/arch/src/aarch64/gic.rs @@ -1,13 +1,12 @@ // Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 -use kvm_ioctls::DeviceFd; -use std::sync::Arc; -use std::{boxed::Box, result}; - use super::gicv2::GICv2; use super::gicv3::GICv3; use super::gicv3_its::GICv3ITS; +use kvm_ioctls::DeviceFd; +use std::sync::Arc; +use std::{boxed::Box, result}; /// Errors thrown while setting up the GIC. #[derive(Debug)] @@ -159,9 +158,17 @@ pub trait GICDevice: Send + Sync { /// /// It will try to create by default a GICv3 device. If that fails it will try /// to fall-back to a GICv2 device. -pub fn create_gic(vm: &Arc, vcpu_count: u64) -> Result> { - GICv3ITS::new(vm, vcpu_count) - .or_else(|_| GICv3::new(vm, vcpu_count).or_else(|_| GICv2::new(vm, vcpu_count))) +pub fn create_gic( + vm: &Arc, + vcpu_count: u64, + its_required: bool, +) -> Result> { + if its_required { + GICv3ITS::new(vm, vcpu_count) + } else { + GICv3ITS::new(vm, vcpu_count) + .or_else(|_| GICv3::new(vm, vcpu_count).or_else(|_| GICv2::new(vm, vcpu_count))) + } } #[cfg(test)] @@ -174,6 +181,6 @@ mod tests { let hv: Arc = Arc::new(kvm); let vm = hv.create_vm().unwrap(); - assert!(create_gic(&vm, 1).is_ok()); + assert!(create_gic(&vm, 1, false).is_ok()); } } diff --git a/arch/src/aarch64/mod.rs b/arch/src/aarch64/mod.rs index 73f131c0e..04ae65e1e 100644 --- a/arch/src/aarch64/mod.rs +++ b/arch/src/aarch64/mod.rs @@ -135,8 +135,6 @@ pub fn arch_memory_regions(size: GuestUsize) -> Vec<(GuestAddress, usize, Region /// /// * `guest_mem` - The memory to be used by the guest. /// * `num_cpus` - Number of virtual CPUs the guest will have. -#[allow(clippy::too_many_arguments)] -#[allow(unused_variables)] pub fn configure_system( vm: &Arc, guest_mem: &GuestMemoryMmap, @@ -145,16 +143,22 @@ pub fn configure_system( vcpu_mpidr: Vec, device_info: &HashMap<(DeviceType, String), T>, initrd: &Option, + pci_space_address: &Option<(u64, u64)>, ) -> super::Result<()> { - let gic_device = gic::create_gic(vm, vcpu_count).map_err(Error::SetupGIC)?; + // If pci_space_address is present, it means PCI devices are used ("pci" feature enabled). + // Then GITv3-ITS is required for MSI messaging. + // Otherwise ("mmio" feature enabled), any version of GIC is OK. + let gic_device = + gic::create_gic(vm, vcpu_count, pci_space_address.is_some()).map_err(Error::SetupGIC)?; - let dtb = fdt::create_fdt( + fdt::create_fdt( guest_mem, cmdline_cstring, vcpu_mpidr, device_info, &gic_device, initrd, + pci_space_address, ) .map_err(Error::SetupFDT)?; diff --git a/vmm/src/vm.rs b/vmm/src/vm.rs index 701b9b1e5..8cecd78a7 100644 --- a/vmm/src/vm.rs +++ b/vmm/src/vm.rs @@ -60,9 +60,9 @@ use std::path::PathBuf; use std::sync::{Arc, Mutex, RwLock}; use std::{result, str, thread}; use url::Url; +use vm_memory::{Address, GuestAddress, GuestAddressSpace}; #[cfg(target_arch = "x86_64")] -use vm_memory::{Address, Bytes, GuestMemoryMmap}; -use vm_memory::{GuestAddress, GuestAddressSpace}; +use vm_memory::{Bytes, GuestMemoryMmap}; use vm_migration::{ Migratable, MigratableError, Pausable, Snapshot, SnapshotDataSection, Snapshottable, Transportable, @@ -618,6 +618,31 @@ impl Vm { .get_device_info() .clone(); + let pci_space: Option<(u64, u64)> = if cfg!(feature = "pci_support") { + let pci_space_start: GuestAddress = self + .memory_manager + .lock() + .as_ref() + .unwrap() + .start_of_device_area(); + + let pci_space_end: GuestAddress = self + .memory_manager + .lock() + .as_ref() + .unwrap() + .end_of_device_area(); + + let pci_space_size = pci_space_end + .checked_offset_from(pci_space_start) + .ok_or(Error::MemOverflow)? + + 1; + + Some((pci_space_start.0, pci_space_size)) + } else { + None + }; + arch::configure_system( &self.memory_manager.lock().as_ref().unwrap().vm, &mem, @@ -626,6 +651,7 @@ impl Vm { vcpu_mpidrs, device_info, &None, + &pci_space, ) .map_err(Error::ConfigureSystem)?;