From 6d9c1eb63842257f9b592917a09101b7667118ae Mon Sep 17 00:00:00 2001 From: Bo Chen Date: Fri, 16 Jul 2021 11:33:03 -0700 Subject: [PATCH] arch, vmm: Add CPUID check to the 'Config' step of live migration We now send not only the 'VmConfig' at the 'Command::Config' step of live migration, but also send the 'common CPUID'. In this way, we can check the compatibility of CPUID features between the source and destination VMs, and abort live migration early if needed. Signed-off-by: Bo Chen --- arch/src/lib.rs | 2 +- arch/src/x86_64/mod.rs | 164 ++++++++++++++++++++++++++++++++++++++++- vmm/src/lib.rs | 87 +++++++++++++++++++++- 3 files changed, 247 insertions(+), 6 deletions(-) diff --git a/arch/src/lib.rs b/arch/src/lib.rs index 8d7589818..727b38dda 100644 --- a/arch/src/lib.rs +++ b/arch/src/lib.rs @@ -85,7 +85,7 @@ pub mod x86_64; pub use x86_64::{ arch_memory_regions, configure_system, configure_vcpu, generate_common_cpuid, get_host_cpu_phys_bits, initramfs_load_addr, layout, layout::CMDLINE_MAX_SIZE, - layout::CMDLINE_START, regs, EntryPoint, + layout::CMDLINE_START, regs, CpuidFeatureEntry, EntryPoint, }; /// Safe wrapper for `sysconf(_SC_PAGESIZE)`. diff --git a/arch/src/x86_64/mod.rs b/arch/src/x86_64/mod.rs index b2310cc3b..82c3a20df 100644 --- a/arch/src/x86_64/mod.rs +++ b/arch/src/x86_64/mod.rs @@ -185,6 +185,9 @@ pub enum Error { /// Error populating CPUID with CPU identification CpuidIdentification(vmm_sys_util::fam::Error), + + /// Error checking CPUID compatibility + CpuidCheckCompatibility, } impl From for super::Error { @@ -194,7 +197,7 @@ impl From for super::Error { } #[allow(dead_code, clippy::upper_case_acronyms)] -#[derive(Copy, Clone)] +#[derive(Copy, Clone, Debug)] pub enum CpuidReg { EAX, EBX, @@ -338,6 +341,165 @@ impl CpuidPatch { } } +pub struct CpuidFeatureEntry { + pub function: u32, + pub index: u32, + pub feature_reg: CpuidReg, +} + +impl CpuidFeatureEntry { + fn checked_feature_entry_list() -> Vec { + vec![ + // The following list includes all hardware features bits from + // the CPUID Wiki Page: https://en.wikipedia.org/wiki/CPUID + // Leaf 0x1, ECX/EDX, feature bits + CpuidFeatureEntry { + function: 1, + index: 0, + feature_reg: CpuidReg::ECX, + }, + CpuidFeatureEntry { + function: 1, + index: 0, + feature_reg: CpuidReg::EDX, + }, + // Leaf 0x7, EAX/EBX/ECX/EDX, extended features + CpuidFeatureEntry { + function: 7, + index: 0, + feature_reg: CpuidReg::EAX, + }, + CpuidFeatureEntry { + function: 7, + index: 0, + feature_reg: CpuidReg::EBX, + }, + CpuidFeatureEntry { + function: 7, + index: 0, + feature_reg: CpuidReg::ECX, + }, + CpuidFeatureEntry { + function: 7, + index: 0, + feature_reg: CpuidReg::EDX, + }, + // Leaf 0x7 subleaf 0x1, EAX, extended features + CpuidFeatureEntry { + function: 7, + index: 1, + feature_reg: CpuidReg::EAX, + }, + // Leaf 0x8000_0001, ECX/EDX, CPUID features bits + CpuidFeatureEntry { + function: 0x8000_0001, + index: 0, + feature_reg: CpuidReg::ECX, + }, + CpuidFeatureEntry { + function: 0x8000_0001, + index: 0, + feature_reg: CpuidReg::EDX, + }, + // KVM CPUID bits: https://www.kernel.org/doc/html/latest/virt/kvm/cpuid.html + // Leaf 0x4000_0001, EAX/EBX/ECX/EDX, KVM CPUID features + CpuidFeatureEntry { + function: 0x4000_0001, + index: 0, + feature_reg: CpuidReg::EAX, + }, + CpuidFeatureEntry { + function: 0x4000_0001, + index: 0, + feature_reg: CpuidReg::EBX, + }, + CpuidFeatureEntry { + function: 0x4000_0001, + index: 0, + feature_reg: CpuidReg::ECX, + }, + CpuidFeatureEntry { + function: 0x4000_0001, + index: 0, + feature_reg: CpuidReg::EDX, + }, + ] + } + + fn get_features_from_cpuid( + cpuid: &CpuId, + feature_entry_list: &[CpuidFeatureEntry], + ) -> Vec { + let mut features = vec![0; feature_entry_list.len()]; + for (i, feature_entry) in feature_entry_list.iter().enumerate() { + for cpuid_entry in cpuid.as_slice().iter() { + if cpuid_entry.function == feature_entry.function + && cpuid_entry.index == feature_entry.index + { + match feature_entry.feature_reg { + CpuidReg::EAX => { + features[i] = cpuid_entry.eax; + } + CpuidReg::EBX => { + features[i] = cpuid_entry.ebx; + } + CpuidReg::ECX => { + features[i] = cpuid_entry.ecx; + } + CpuidReg::EDX => { + features[i] = cpuid_entry.edx; + } + } + + break; + } + } + } + + features + } + + // The function returns `Error` (a.k.a. "incompatible"), when the CPUID features from `src_vm_cpuid` + // is not a subset of those of the `dest_vm_cpuid`. + pub fn check_cpuid_compatibility( + src_vm_cpuid: &CpuId, + dest_vm_cpuid: &CpuId, + ) -> Result<(), Error> { + let feature_entry_list = &Self::checked_feature_entry_list(); + let src_vm_features = Self::get_features_from_cpuid(src_vm_cpuid, feature_entry_list); + let dest_vm_features = Self::get_features_from_cpuid(dest_vm_cpuid, feature_entry_list); + + // Loop on feature bit and check if the 'source vm' feature is a subset + // of those of the 'destination vm' feature + let mut compatible = true; + for (i, (src_vm_feature, dest_vm_feature)) in src_vm_features + .iter() + .zip(dest_vm_features.iter()) + .enumerate() + { + let different_feature_bits = src_vm_feature ^ dest_vm_feature; + let src_vm_feature_bits_only = different_feature_bits & src_vm_feature; + if src_vm_feature_bits_only != 0 { + error!( + "Detected incompatible CPUID entry: leaf={:#02x} (subleaf={:#02x}), register='{:?}', \ + source VM feature='{:#04x}', destination VM feature'{:#04x}'.", + feature_entry_list[i].function, feature_entry_list[i].index, feature_entry_list[i].feature_reg, + src_vm_feature, dest_vm_feature + ); + + compatible = false; + } + } + + if compatible { + info!("No CPU incompatibility detected."); + Ok(()) + } else { + Err(Error::CpuidCheckCompatibility) + } + } +} + pub fn generate_common_cpuid( hypervisor: Arc, topology: Option<(u8, u8, u8)>, diff --git a/vmm/src/lib.rs b/vmm/src/lib.rs index e597bdd42..c5ba3c395 100644 --- a/vmm/src/lib.rs +++ b/vmm/src/lib.rs @@ -284,6 +284,13 @@ pub fn start_vmm_thread( Ok(thread) } +#[derive(Clone, Deserialize, Serialize)] +struct VmMigrationConfig { + vm_config: Arc>, + #[cfg(all(feature = "kvm", target_arch = "x86_64"))] + common_cpuid: hypervisor::CpuId, +} + pub struct Vmm { epoll: EpollContext, exit_evt: EventFd, @@ -743,8 +750,47 @@ impl Vmm { socket .read_exact(&mut data) .map_err(MigratableError::MigrateSocket)?; - let config: VmConfig = serde_json::from_slice(&data).map_err(|e| { - MigratableError::MigrateReceive(anyhow!("Error deserialising config: {}", e)) + + let vm_migration_config: VmMigrationConfig = + serde_json::from_slice(&data).map_err(|e| { + MigratableError::MigrateReceive(anyhow!("Error deserialising config: {}", e)) + })?; + + #[cfg(all(feature = "kvm", target_arch = "x86_64"))] + // We check the `CPUID` compatibility of between the source vm and destination, which is + // mostly about feature compatibility and "topology/sgx" leaves are not relevant. + let dest_cpuid = { + let vm_config = &vm_migration_config.vm_config.lock().unwrap(); + #[cfg(feature = "tdx")] + let tdx_enabled = vm_config.tdx.is_some(); + let phys_bits = vm::physical_bits( + vm_config.cpus.max_phys_bits, + #[cfg(feature = "tdx")] + tdx_enabled, + ); + arch::generate_common_cpuid( + self.hypervisor.clone(), + None, + None, + phys_bits, + vm_config.cpus.kvm_hyperv, + #[cfg(feature = "tdx")] + tdx_enabled, + ) + .map_err(|e| { + MigratableError::MigrateReceive(anyhow!("Error generating common cpuid': {:?}", e)) + })? + }; + #[cfg(all(feature = "kvm", target_arch = "x86_64"))] + arch::CpuidFeatureEntry::check_cpuid_compatibility( + &vm_migration_config.common_cpuid, + &dest_cpuid, + ) + .map_err(|e| { + MigratableError::MigrateReceive(anyhow!( + "Error checking cpu feature compatibility': {:?}", + e + )) })?; let exit_evt = self.exit_evt.try_clone().map_err(|e| { @@ -757,7 +803,7 @@ impl Vmm { MigratableError::MigrateReceive(anyhow!("Error cloning activate EventFd: {}", e)) })?; - self.vm_config = Some(Arc::new(Mutex::new(config))); + self.vm_config = Some(vm_migration_config.vm_config); let vm = Vm::new_from_migration( self.vm_config.clone().unwrap(), exit_evt, @@ -998,7 +1044,40 @@ impl Vmm { } // Send config - let config_data = serde_json::to_vec(&vm.get_config()).unwrap(); + let vm_config = vm.get_config(); + #[cfg(all(feature = "kvm", target_arch = "x86_64"))] + let common_cpuid = { + #[cfg(feature = "tdx")] + let tdx_enabled = vm_config.lock().unwrap().tdx.is_some(); + let phys_bits = vm::physical_bits( + vm_config.lock().unwrap().cpus.max_phys_bits, + #[cfg(feature = "tdx")] + tdx_enabled, + ); + arch::generate_common_cpuid( + self.hypervisor.clone(), + None, + None, + phys_bits, + vm_config.lock().unwrap().cpus.kvm_hyperv, + #[cfg(feature = "tdx")] + tdx_enabled, + ) + .map_err(|e| { + MigratableError::MigrateReceive(anyhow!( + "Error generating common cpuid': {:?}", + e + )) + })? + }; + + let vm_migration_config = VmMigrationConfig { + vm_config, + #[cfg(all(feature = "kvm", target_arch = "x86_64"))] + common_cpuid, + }; + + let config_data = serde_json::to_vec(&vm_migration_config).unwrap(); Request::config(config_data.len() as u64).write_to(&mut socket)?; socket .write_all(&config_data)