diff --git a/arch/src/aarch64/gic/gicv3.rs b/arch/src/aarch64/gic/gicv3.rs index 428a7d28d..681da6600 100644 --- a/arch/src/aarch64/gic/gicv3.rs +++ b/arch/src/aarch64/gic/gicv3.rs @@ -5,11 +5,14 @@ pub mod kvm { use crate::aarch64::gic::dist_regs::{get_dist_regs, read_ctlr, set_dist_regs, write_ctlr}; use crate::aarch64::gic::icc_regs::{get_icc_regs, set_icc_regs}; use crate::aarch64::gic::kvm::{save_pending_tables, KvmGicDevice}; - use crate::aarch64::gic::redist_regs::{get_redist_regs, set_redist_regs}; + use crate::aarch64::gic::redist_regs::{ + construct_gicr_typers, get_redist_regs, set_redist_regs, + }; use crate::aarch64::gic::GicDevice; use crate::layout; use anyhow::anyhow; use hypervisor::kvm::kvm_bindings; + use hypervisor::CpuState; use std::any::Any; use std::convert::TryInto; use std::sync::Arc; @@ -165,7 +168,8 @@ pub mod kvm { self.vcpu_count } - fn set_gicr_typers(&mut self, gicr_typers: Vec) { + fn set_gicr_typers(&mut self, vcpu_states: &[CpuState]) { + let gicr_typers = construct_gicr_typers(vcpu_states); self.gicr_typers = gicr_typers; } diff --git a/arch/src/aarch64/gic/gicv3_its.rs b/arch/src/aarch64/gic/gicv3_its.rs index 4bec26f0a..c9ec07186 100644 --- a/arch/src/aarch64/gic/gicv3_its.rs +++ b/arch/src/aarch64/gic/gicv3_its.rs @@ -10,6 +10,7 @@ pub mod kvm { use crate::aarch64::gic::kvm::KvmGicDevice; use crate::aarch64::gic::{Error, GicDevice}; use hypervisor::kvm::kvm_bindings; + use hypervisor::CpuState; pub struct KvmGicV3Its { /// The hypervisor agnostic device @@ -70,7 +71,7 @@ pub mod kvm { self.vcpu_count } - fn set_gicr_typers(&mut self, _gicr_typers: Vec) {} + fn set_gicr_typers(&mut self, _vcpu_states: &[CpuState]) {} fn as_any_concrete_mut(&mut self) -> &mut dyn Any { self diff --git a/arch/src/aarch64/gic/mod.rs b/arch/src/aarch64/gic/mod.rs index d2737e735..62008a8b7 100644 --- a/arch/src/aarch64/gic/mod.rs +++ b/arch/src/aarch64/gic/mod.rs @@ -10,6 +10,7 @@ pub mod redist_regs; pub use self::dist_regs::{get_dist_regs, read_ctlr, set_dist_regs, write_ctlr}; pub use self::icc_regs::{get_icc_regs, set_icc_regs}; pub use self::redist_regs::{get_redist_regs, set_redist_regs}; +use hypervisor::CpuState; use std::any::Any; use std::result; use std::sync::Arc; @@ -58,7 +59,7 @@ pub trait GicDevice: Send { } /// Get the values of GICR_TYPER for each vCPU. - fn set_gicr_typers(&mut self, gicr_typers: Vec); + fn set_gicr_typers(&mut self, vcpu_states: &[CpuState]); /// Downcast the trait object to its concrete type. fn as_any_concrete_mut(&mut self) -> &mut dyn Any; diff --git a/arch/src/aarch64/gic/redist_regs.rs b/arch/src/aarch64/gic/redist_regs.rs index c7d7a27e8..8f7d66d69 100644 --- a/arch/src/aarch64/gic/redist_regs.rs +++ b/arch/src/aarch64/gic/redist_regs.rs @@ -3,6 +3,7 @@ use super::{Error, Result}; use hypervisor::kvm::kvm_bindings::{kvm_device_attr, KVM_DEV_ARM_VGIC_GRP_REDIST_REGS}; +use hypervisor::CpuState; use std::sync::Arc; // Relevant redistributor registers that we want to save/restore. @@ -177,3 +178,34 @@ pub fn set_redist_regs( true, ) } + +pub fn construct_gicr_typers(vcpu_states: &[CpuState]) -> Vec { + /* Pre-construct the GICR_TYPER: + * For our implementation: + * Top 32 bits are the affinity value of the associated CPU + * CommonLPIAff == 01 (redistributors with same Aff3 share LPI table) + * Processor_Number == CPU index starting from 0 + * DPGS == 0 (GICR_CTLR.DPG* not supported) + * Last == 1 if this is the last redistributor in a series of + * contiguous redistributor pages + * DirectLPI == 0 (direct injection of LPIs not supported) + * VLPIS == 0 (virtual LPIs not supported) + * PLPIS == 0 (physical LPIs not supported) + */ + let mut gicr_typers: Vec = Vec::new(); + for (index, state) in vcpu_states.iter().enumerate() { + let last = { + if index == vcpu_states.len() - 1 { + 1 + } else { + 0 + } + }; + //calculate affinity + let mut cpu_affid = state.mpidr & 1095233437695; + cpu_affid = ((cpu_affid & 0xFF00000000) >> 8) | (cpu_affid & 0xFFFFFF); + gicr_typers.push((cpu_affid << 32) | (1 << 24) | (index as u64) << 8 | (last << 4)); + } + + gicr_typers +} diff --git a/vmm/src/device_manager.rs b/vmm/src/device_manager.rs index c1658f79b..1830c382d 100644 --- a/vmm/src/device_manager.rs +++ b/vmm/src/device_manager.rs @@ -29,8 +29,6 @@ use crate::{device_node, DEVICE_MANAGER_SNAPSHOT_ID}; #[cfg(feature = "acpi")] use acpi_tables::{aml, aml::Aml}; use anyhow::anyhow; -#[cfg(target_arch = "aarch64")] -use arch::aarch64::gic::GicDevice; #[cfg(feature = "acpi")] use arch::layout; #[cfg(target_arch = "x86_64")] @@ -55,8 +53,6 @@ use devices::{ }; #[cfg(feature = "kvm")] use hypervisor::kvm_ioctls::*; -#[cfg(target_arch = "aarch64")] -use hypervisor::CpuState; #[cfg(feature = "mshv")] use hypervisor::IoEventAddress; use libc::{ @@ -828,9 +824,6 @@ pub struct DeviceManager { #[cfg(target_arch = "aarch64")] interrupt_controller: Option>>, - #[cfg(target_arch = "aarch64")] - gic_device_entity: Option>>>, - // Things to be added to the commandline (i.e. for virtio-mmio) cmdline_additions: Vec, @@ -966,8 +959,6 @@ impl DeviceManager { address_manager: Arc::clone(&address_manager), console: Arc::new(Console::default()), interrupt_controller: None, - #[cfg(target_arch = "aarch64")] - gic_device_entity: None, cmdline_additions: Vec::new(), #[cfg(feature = "acpi")] ged_notification_device: None, @@ -1266,51 +1257,6 @@ impl DeviceManager { self.interrupt_controller.as_ref() } - #[cfg(target_arch = "aarch64")] - pub fn set_gic_device_entity(&mut self, device_entity: Arc>>) { - self.gic_device_entity = Some(device_entity); - } - - #[cfg(target_arch = "aarch64")] - pub fn get_gic_device_entity(&self) -> Option<&Arc>>> { - self.gic_device_entity.as_ref() - } - #[cfg(target_arch = "aarch64")] - pub fn construct_gicr_typers(&self, vcpu_states: &[CpuState]) { - /* Pre-construct the GICR_TYPER: - * For our implementation: - * Top 32 bits are the affinity value of the associated CPU - * CommonLPIAff == 01 (redistributors with same Aff3 share LPI table) - * Processor_Number == CPU index starting from 0 - * DPGS == 0 (GICR_CTLR.DPG* not supported) - * Last == 1 if this is the last redistributor in a series of - * contiguous redistributor pages - * DirectLPI == 0 (direct injection of LPIs not supported) - * VLPIS == 0 (virtual LPIs not supported) - * PLPIS == 0 (physical LPIs not supported) - */ - let mut gicr_typers: Vec = Vec::new(); - for (index, state) in vcpu_states.iter().enumerate() { - let last = { - if index == vcpu_states.len() - 1 { - 1 - } else { - 0 - } - }; - //calculate affinity - let mut cpu_affid = state.mpidr & 1095233437695; - cpu_affid = ((cpu_affid & 0xFF00000000) >> 8) | (cpu_affid & 0xFFFFFF); - gicr_typers.push((cpu_affid << 32) | (1 << 24) | (index as u64) << 8 | (last << 4)); - } - - self.get_gic_device_entity() - .unwrap() - .lock() - .unwrap() - .set_gicr_typers(gicr_typers) - } - #[cfg(target_arch = "aarch64")] pub fn enable_interrupt_controller(&self) -> DeviceManagerResult<()> { if let Some(interrupt_controller) = &self.interrupt_controller { diff --git a/vmm/src/vm.rs b/vmm/src/vm.rs index ff8efd582..b3ee28460 100644 --- a/vmm/src/vm.rs +++ b/vmm/src/vm.rs @@ -1860,12 +1860,7 @@ impl Vm { vm_snapshot: &mut Snapshot, ) -> std::result::Result<(), MigratableError> { let saved_vcpu_states = self.cpu_manager.lock().unwrap().get_saved_states(); - self.device_manager - .lock() - .unwrap() - .construct_gicr_typers(&saved_vcpu_states); - - vm_snapshot.add_snapshot( + let gic_device = Arc::clone( self.device_manager .lock() .unwrap() @@ -1874,7 +1869,16 @@ impl Vm { .lock() .unwrap() .get_gic_device() - .unwrap() + .unwrap(), + ); + + gic_device + .lock() + .unwrap() + .set_gicr_typers(&saved_vcpu_states); + + vm_snapshot.add_snapshot( + gic_device .lock() .unwrap() .as_any_concrete_mut() @@ -1899,9 +1903,13 @@ impl Vm { // Creating a GIC device here, as the GIC will not be created when // restoring the device manager. Note that currently only the bare GICv3 // without ITS is supported. - let gic_device = create_gic(&self.vm, vcpu_numbers.try_into().unwrap()) + let mut gic_device = create_gic(&self.vm, vcpu_numbers.try_into().unwrap()) .map_err(|e| MigratableError::Restore(anyhow!("Could not create GIC: {:#?}", e)))?; + // Here we prepare the GICR_TYPER registers from the restored vCPU states. + gic_device.set_gicr_typers(&saved_vcpu_states); + + let gic_device = Arc::new(Mutex::new(gic_device)); // Update the GIC entity in device manager self.device_manager .lock() @@ -1910,25 +1918,11 @@ impl Vm { .unwrap() .lock() .unwrap() - .set_gic_device(Arc::new(Mutex::new(gic_device))); - - // Here we prepare the GICR_TYPER registers from the restored vCPU states. - self.device_manager - .lock() - .unwrap() - .construct_gicr_typers(&saved_vcpu_states); + .set_gic_device(Arc::clone(&gic_device)); // Restore GIC states. if let Some(gic_v3_snapshot) = vm_snapshot.snapshots.get(GIC_V3_SNAPSHOT_ID) { - self.device_manager - .lock() - .unwrap() - .get_interrupt_controller() - .unwrap() - .lock() - .unwrap() - .get_gic_device() - .unwrap() + gic_device .lock() .unwrap() .as_any_concrete_mut()