diff --git a/hypervisor/src/kvm/mod.rs b/hypervisor/src/kvm/mod.rs index 8b21002d1..89090294a 100644 --- a/hypervisor/src/kvm/mod.rs +++ b/hypervisor/src/kvm/mod.rs @@ -14,7 +14,10 @@ use std::any::Any; use std::collections::HashMap; #[cfg(any(target_arch = "aarch64", target_arch = "riscv64"))] use std::mem::offset_of; -#[cfg(feature = "tdx")] +#[cfg(feature = "sev_snp")] +use std::os::fd::FromRawFd; +use std::os::fd::OwnedFd; +#[cfg(any(feature = "sev_snp", feature = "tdx"))] use std::os::unix::io::AsRawFd; #[cfg(feature = "tdx")] use std::os::unix::io::RawFd; @@ -26,9 +29,12 @@ use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, RwLock}; use anyhow::anyhow; +#[cfg(feature = "sev_snp")] +use kvm_bindings::kvm_create_guest_memfd; use kvm_ioctls::{NoDatamatch, VcpuFd, VmFd}; #[cfg(target_arch = "x86_64")] use log::warn; +use vmm_sys_util::errno; use vmm_sys_util::eventfd::EventFd; #[cfg(target_arch = "aarch64")] @@ -50,8 +56,6 @@ pub use crate::riscv64::{ }; #[cfg(target_arch = "riscv64")] use crate::riscv64_reg_id; -use crate::vm::{self, InterruptSourceConfig, VmOps}; -use crate::{HypervisorType, HypervisorVmConfig, cpu, hypervisor}; // x86_64 dependencies #[cfg(target_arch = "x86_64")] pub mod x86_64; @@ -73,7 +77,12 @@ use crate::arch::x86::{ CpuIdEntry, FpuState, LapicState, MTRR_MSR_INDICES, MsrEntry, NUM_IOAPIC_PINS, SpecialRegisters, XsaveState, }; -use crate::{CpuState, IoEventAddress, IrqRoutingEntry, MpState, StandardRegisters}; +use crate::{ + CpuState, HypervisorType, HypervisorVmConfig, InterruptSourceConfig, IoEventAddress, + IrqRoutingEntry, MpState, StandardRegisters, USER_MEMORY_REGION_GUEST_MEMFD, + USER_MEMORY_REGION_LOG_DIRTY, USER_MEMORY_REGION_READ, USER_MEMORY_REGION_WRITE, + UserMemoryRegion, VmOps, cpu, hypervisor, vm, +}; // aarch64 dependencies #[cfg(target_arch = "aarch64")] pub mod aarch64; @@ -83,6 +92,8 @@ pub mod riscv64; #[cfg(target_arch = "aarch64")] use std::mem; +#[cfg(target_arch = "x86_64")] +use kvm_bindings::KVM_X86_DEFAULT_VM; /// /// Export generically-named wrappers of kvm-bindings for Unix-based platforms /// @@ -92,10 +103,11 @@ pub use kvm_bindings::kvm_vcpu_events as VcpuEvents; use kvm_bindings::nested::KvmNestedStateBuffer; pub use kvm_bindings::{ self, KVM_GUESTDBG_ENABLE, KVM_GUESTDBG_SINGLESTEP, KVM_IRQ_ROUTING_IRQCHIP, - KVM_IRQ_ROUTING_MSI, KVM_MEM_LOG_DIRTY_PAGES, KVM_MEM_READONLY, KVM_MSI_VALID_DEVID, - kvm_clock_data, kvm_create_device, kvm_create_device as CreateDevice, + KVM_IRQ_ROUTING_MSI, KVM_MEM_GUEST_MEMFD, KVM_MEM_LOG_DIRTY_PAGES, KVM_MEM_READONLY, + KVM_MSI_VALID_DEVID, kvm_clock_data, kvm_create_device, kvm_create_device as CreateDevice, kvm_device_attr as DeviceAttr, kvm_device_type_KVM_DEV_TYPE_VFIO, kvm_guest_debug, kvm_irq_routing, kvm_irq_routing_entry, kvm_mp_state, kvm_run, kvm_userspace_memory_region, + kvm_userspace_memory_region2, }; #[cfg(target_arch = "aarch64")] use kvm_bindings::{ @@ -107,7 +119,7 @@ use kvm_bindings::{ #[cfg(target_arch = "riscv64")] use kvm_bindings::{KVM_REG_RISCV_CORE, kvm_riscv_core}; #[cfg(feature = "tdx")] -use kvm_bindings::{KVM_X86_DEFAULT_VM, KVM_X86_SW_PROTECTED_VM, KVMIO, kvm_run__bindgen_ty_1}; +use kvm_bindings::{KVM_X86_SW_PROTECTED_VM, KVMIO, kvm_run__bindgen_ty_1}; #[cfg(target_arch = "x86_64")] use kvm_bindings::{Xsave as xsave2, kvm_xsave2}; pub use kvm_ioctls::{self, Cap, Kvm, VcpuExit}; @@ -238,6 +250,61 @@ pub struct KvmTdxExitVmcall { pub out_rdx: u64, } +impl From for UserMemoryRegion { + fn from(region: kvm_userspace_memory_region2) -> Self { + let mut flags = USER_MEMORY_REGION_READ; + if region.flags & KVM_MEM_READONLY == 0 { + flags |= USER_MEMORY_REGION_WRITE; + } + if region.flags & KVM_MEM_LOG_DIRTY_PAGES != 0 { + flags |= USER_MEMORY_REGION_LOG_DIRTY; + } + if region.flags & KVM_MEM_GUEST_MEMFD != 0 { + flags |= USER_MEMORY_REGION_GUEST_MEMFD; + } + + UserMemoryRegion { + slot: region.slot, + guest_phys_addr: region.guest_phys_addr, + memory_size: region.memory_size, + userspace_addr: region.userspace_addr, + flags, + guest_memfd: Some(region.guest_memfd), + guest_memfd_offset: Some(region.guest_memfd_offset), + } + } +} + +impl From for kvm_userspace_memory_region2 { + fn from(region: UserMemoryRegion) -> Self { + assert!( + region.flags & USER_MEMORY_REGION_READ != 0, + "KVM mapped memory is always readable" + ); + + let mut flags = 0; + if region.flags & USER_MEMORY_REGION_WRITE == 0 { + flags |= KVM_MEM_READONLY; + } + if region.flags & USER_MEMORY_REGION_LOG_DIRTY != 0 { + flags |= KVM_MEM_LOG_DIRTY_PAGES; + } + if region.flags & USER_MEMORY_REGION_GUEST_MEMFD != 0 { + flags |= KVM_MEM_GUEST_MEMFD; + } + + kvm_userspace_memory_region2 { + slot: region.slot, + guest_phys_addr: region.guest_phys_addr, + memory_size: region.memory_size, + userspace_addr: region.userspace_addr, + flags, + guest_memfd: region.guest_memfd.unwrap_or(0), + guest_memfd_offset: region.guest_memfd_offset.unwrap_or(0), + ..Default::default() + } + } +} impl From for MpState { fn from(s: kvm_mp_state) -> Self { MpState::Kvm(s) @@ -424,6 +491,9 @@ struct KvmDirtyLogSlot { guest_phys_addr: u64, memory_size: u64, userspace_addr: u64, + // Following fields are used by kvm_userspace_memory_region2. + guest_memfd_offset: u64, + guest_memfd: u32, } /// Wrapper over KVM VM ioctls. @@ -432,6 +502,7 @@ pub struct KvmVm { #[cfg(target_arch = "x86_64")] msrs: Vec, dirty_log_slots: RwLock>, + guest_memfds: Option>>, } impl KvmVm { @@ -494,6 +565,47 @@ impl KvmVm { fn translate_msi_ext_dest_id(address_lo: u32, address_hi: u32) -> (u32, u32) { (address_lo, address_hi) } + + /// Set user memory region to use guest_memfd when available. + /// guest_memfd is available on host linux kernel v6.8+ + /// + /// # Safety + /// + /// `region.userspace_addr` must point to `region.memory_size` bytes of + /// memory that will stay mapped until the slot is removed via + /// `remove_user_memory_region`. The memory region must + /// be uniquely owned by the caller, as mapping it into the guest + /// effectively creates a long-lived mutable reference. + unsafe fn set_user_memory_region( + &self, + region: kvm_userspace_memory_region2, + ) -> Result<(), errno::Error> { + if self.guest_memfds.is_some() { + // SAFETY: Safe as the caller guarantees that region is safe to map + // the guest and is non-overlapping. + unsafe { self.fd.set_user_memory_region2(region) } + } else { + // SAFETY: Safe because guest regions are guaranteed not to overlap. + unsafe { + self.fd.set_user_memory_region(kvm_userspace_memory_region { + slot: region.slot, + guest_phys_addr: region.guest_phys_addr, + userspace_addr: region.userspace_addr, + flags: region.flags, + memory_size: region.memory_size, + }) + } + } + } + + /// Get flag for kvm_userspace_memory_region based on memfd support. + fn get_kvm_userspace_memory_region_flag(&self, flag: u32) -> u32 { + flag | if self.guest_memfds.is_some() { + KVM_MEM_GUEST_MEMFD + } else { + 0 + } + } } /// Implementation of Vm trait for KVM @@ -759,14 +871,43 @@ impl vm::Vm for KvmVm { const _: () = assert!(core::mem::size_of::() <= core::mem::size_of::()); - let mut region = kvm_userspace_memory_region { + // Create a per-region guest_memfd when supported. + // Each region gets its own fd sized exactly to memory_size + #[cfg(feature = "sev_snp")] + let guest_memfd = if let Some(memfds) = &self.guest_memfds { + // SAFETY: Safe because guest regions are guaranteed not to overlap. + let fd = unsafe { + OwnedFd::from_raw_fd( + self.fd + .create_guest_memfd(kvm_create_guest_memfd { + size: memory_size as u64, + ..Default::default() + }) + .map_err(|e| vm::HypervisorVmError::CreateUserMemory(e.into()))?, + ) + }; + let raw_fd = fd.as_raw_fd() as u32; + memfds.write().unwrap().insert(slot, fd); + raw_fd + } else { + 0 + }; + #[cfg(not(feature = "sev_snp"))] + let guest_memfd = 0; + + let mut region = kvm_userspace_memory_region2 { slot, + flags: self.get_kvm_userspace_memory_region_flag(flags), guest_phys_addr, memory_size: memory_size as u64, userspace_addr: userspace_addr as usize as u64, - flags, + #[cfg(not(target_arch = "riscv64"))] + guest_memfd, + // Each guest_memfd is per-region and sized to memory_size, + // so the region's data always starts at offset 0. + guest_memfd_offset: 0, + ..Default::default() }; - if (region.flags & KVM_MEM_LOG_DIRTY_PAGES) != 0 { if (region.flags & KVM_MEM_READONLY) != 0 { return Err(vm::HypervisorVmError::CreateUserMemory(anyhow!( @@ -782,20 +923,22 @@ impl vm::Vm for KvmVm { guest_phys_addr: region.guest_phys_addr, memory_size: region.memory_size, userspace_addr: region.userspace_addr, + guest_memfd_offset: region.guest_memfd_offset, + guest_memfd: region.guest_memfd, }, ); // Always create guest physical memory region without `KVM_MEM_LOG_DIRTY_PAGES`. // For regions that need this flag, dirty pages log will be turned on in `start_dirty_log`. - region.flags = 0; + region.flags = self.get_kvm_userspace_memory_region_flag(0); } // SAFETY: Safe because caller promised this is safe. unsafe { - self.fd - .set_user_memory_region(region) - .map_err(|e| vm::HypervisorVmError::CreateUserMemory(e.into())) + self.set_user_memory_region(region) + .map_err(|e| vm::HypervisorVmError::CreateUserMemory(e.into()))?; } + Ok(()) } /// Removes a guest physical memory region. @@ -823,12 +966,13 @@ impl vm::Vm for KvmVm { const _: () = assert!(core::mem::size_of::() <= core::mem::size_of::()); - let mut region = kvm_userspace_memory_region { + let mut region = kvm_userspace_memory_region2 { slot, guest_phys_addr, memory_size: memory_size as u64, userspace_addr: userspace_addr as usize as u64, flags, + ..Default::default() }; // Remove the corresponding entry from "self.dirty_log_slots" if needed @@ -838,10 +982,16 @@ impl vm::Vm for KvmVm { region.memory_size = 0; // SAFETY: Safe because caller promised this is safe. unsafe { - self.fd - .set_user_memory_region(region) - .map_err(|e| vm::HypervisorVmError::RemoveUserMemory(e.into())) + self.set_user_memory_region(region) + .map_err(|e| vm::HypervisorVmError::RemoveUserMemory(e.into()))?; } + + // Close the per-region guest_memfd if one was created for this slot + if let Some(memfds) = &self.guest_memfds { + memfds.write().unwrap().remove(&slot); + } + + Ok(()) } /// @@ -932,17 +1082,19 @@ impl vm::Vm for KvmVm { fn start_dirty_log(&self) -> vm::Result<()> { let dirty_log_slots = self.dirty_log_slots.read().unwrap(); for (_, s) in dirty_log_slots.iter() { - let region = kvm_userspace_memory_region { + let region = kvm_userspace_memory_region2 { slot: s.slot, guest_phys_addr: s.guest_phys_addr, memory_size: s.memory_size, userspace_addr: s.userspace_addr, - flags: KVM_MEM_LOG_DIRTY_PAGES, + flags: self.get_kvm_userspace_memory_region_flag(KVM_MEM_LOG_DIRTY_PAGES), + guest_memfd: s.guest_memfd, + guest_memfd_offset: s.guest_memfd_offset, + ..Default::default() }; // SAFETY: Safe because guest regions are guaranteed not to overlap. unsafe { - self.fd - .set_user_memory_region(region) + self.set_user_memory_region(region) .map_err(|e| vm::HypervisorVmError::StartDirtyLog(e.into()))?; } } @@ -956,17 +1108,19 @@ impl vm::Vm for KvmVm { fn stop_dirty_log(&self) -> vm::Result<()> { let dirty_log_slots = self.dirty_log_slots.read().unwrap(); for (_, s) in dirty_log_slots.iter() { - let region = kvm_userspace_memory_region { + let region = kvm_userspace_memory_region2 { slot: s.slot, guest_phys_addr: s.guest_phys_addr, memory_size: s.memory_size, userspace_addr: s.userspace_addr, - flags: 0, + flags: self.get_kvm_userspace_memory_region_flag(0), + guest_memfd: s.guest_memfd, + guest_memfd_offset: s.guest_memfd_offset, + ..Default::default() }; // SAFETY: Safe because guest regions are guaranteed not to overlap. unsafe { - self.fd - .set_user_memory_region(region) + self.set_user_memory_region(region) .map_err(|e| vm::HypervisorVmError::StartDirtyLog(e.into()))?; } } @@ -1228,11 +1382,17 @@ impl hypervisor::Hypervisor for KvmHypervisor { vm_type = self.kvm.get_host_ipa_limit().try_into().unwrap(); } - #[cfg(feature = "tdx")] - if _config.tdx_enabled { - vm_type = KVM_X86_SW_PROTECTED_VM.into(); - } else { - vm_type = KVM_X86_DEFAULT_VM.into(); + #[cfg(target_arch = "x86_64")] + cfg_if::cfg_if! { + if #[cfg(feature = "tdx")] { + if _config.tdx_enabled { + vm_type = KVM_X86_SW_PROTECTED_VM.into(); + } else { + vm_type = KVM_X86_DEFAULT_VM.into(); + } + } else { + vm_type = KVM_X86_DEFAULT_VM.into(); + } } loop { @@ -1255,7 +1415,7 @@ impl hypervisor::Hypervisor for KvmHypervisor { { let msr_list = self.get_msr_list()?; let num_msrs = msr_list.as_fam_struct_ref().nmsrs as usize; - let mut msrs: Vec = vec![ + let mut msrs = vec![ MsrEntry { ..Default::default() }; @@ -1266,10 +1426,18 @@ impl hypervisor::Hypervisor for KvmHypervisor { msrs[pos].index = *index; } + #[allow(unused_mut)] + let mut guest_memfds = None; + #[cfg(feature = "sev_snp")] + if _config.sev_snp_enabled && fd.check_extension(Cap::GuestMemfd) { + guest_memfds = Some(RwLock::new(HashMap::new())); + } + Ok(Arc::new(KvmVm { fd, msrs, dirty_log_slots: RwLock::new(HashMap::new()), + guest_memfds, })) } @@ -1278,6 +1446,7 @@ impl hypervisor::Hypervisor for KvmHypervisor { Ok(Arc::new(KvmVm { fd, dirty_log_slots: RwLock::new(HashMap::new()), + guest_memfds: None, })) } } diff --git a/hypervisor/src/lib.rs b/hypervisor/src/lib.rs index 3d919e45c..787cfbf1f 100644 --- a/hypervisor/src/lib.rs +++ b/hypervisor/src/lib.rs @@ -118,6 +118,26 @@ pub fn vec_with_array_field(count: usize) -> Vec { vec_with_size_in_bytes(vec_size_bytes) } +/// User memory region structure +#[derive(Debug, Default, Eq, PartialEq)] +pub struct UserMemoryRegion { + pub slot: u32, + pub guest_phys_addr: u64, + pub memory_size: u64, + pub userspace_addr: u64, + pub flags: u32, + pub guest_memfd: Option, + pub guest_memfd_offset: Option, +} + +/// Flags for user memory region +pub const USER_MEMORY_REGION_READ: u32 = 1; +pub const USER_MEMORY_REGION_WRITE: u32 = 1 << 1; +pub const USER_MEMORY_REGION_EXECUTE: u32 = 1 << 2; +pub const USER_MEMORY_REGION_LOG_DIRTY: u32 = 1 << 3; +pub const USER_MEMORY_REGION_ADJUSTABLE: u32 = 1 << 4; +pub const USER_MEMORY_REGION_GUEST_MEMFD: u32 = 1 << 5; + #[derive(Debug)] pub enum MpState { #[cfg(feature = "kvm")] diff --git a/vmm/src/seccomp_filters.rs b/vmm/src/seccomp_filters.rs index 25da7f9c9..008fc60f2 100644 --- a/vmm/src/seccomp_filters.rs +++ b/vmm/src/seccomp_filters.rs @@ -90,6 +90,9 @@ mod kvm { pub const KVM_HAS_DEVICE_ATTR: u64 = 0x4018_aee3; pub const KVM_SET_ONE_REG: u64 = 0x4010_aeac; pub const KVM_SET_USER_MEMORY_REGION: u64 = 0x4020_ae46; + pub const KVM_SET_USER_MEMORY_REGION2: u64 = 0x40a0_ae49; + pub const KVM_SET_MEMORY_ATTRIBUTES: u64 = 0x4020_aed2; + pub const KVM_CREATE_GUEST_MEMFD: u64 = 0xc040_aed4; pub const KVM_IRQFD: u64 = 0x4020_ae76; pub const KVM_IOEVENTFD: u64 = 0x4040_ae79; pub const KVM_SET_VCPU_EVENTS: u64 = 0x4040_aea0; @@ -252,6 +255,14 @@ fn create_vmm_ioctl_seccomp_rule_common_kvm() -> Result, Backen and![Cond::new(1, ArgLen::Dword, Eq, KVM_SET_ONE_REG)?], and![Cond::new(1, ArgLen::Dword, Eq, KVM_SET_REGS)?], and![Cond::new(1, ArgLen::Dword, Eq, KVM_SET_USER_MEMORY_REGION,)?], + and![Cond::new( + 1, + ArgLen::Dword, + Eq, + KVM_SET_USER_MEMORY_REGION2, + )?], + and![Cond::new(1, ArgLen::Dword, Eq, KVM_SET_MEMORY_ATTRIBUTES,)?], + and![Cond::new(1, ArgLen::Dword, Eq, KVM_CREATE_GUEST_MEMFD,)?], and![Cond::new(1, ArgLen::Dword, Eq, KVM_SET_VCPU_EVENTS,)?], and![Cond::new(1, ArgLen::Dword, Eq, KVM_NMI)?], and![Cond::new(1, ArgLen::Dword, Eq, KVM_GET_NESTED_STATE)?], @@ -750,6 +761,14 @@ fn create_vcpu_ioctl_seccomp_rule_kvm() -> Result, BackendError and![Cond::new(1, ArgLen::Dword, Eq, KVM_SET_DEVICE_ATTR,)?], and![Cond::new(1, ArgLen::Dword, Eq, KVM_SET_GSI_ROUTING,)?], and![Cond::new(1, ArgLen::Dword, Eq, KVM_SET_USER_MEMORY_REGION,)?], + and![Cond::new( + 1, + ArgLen::Dword, + Eq, + KVM_SET_USER_MEMORY_REGION2, + )?], + and![Cond::new(1, ArgLen::Dword, Eq, KVM_CREATE_GUEST_MEMFD,)?], + and![Cond::new(1, ArgLen::Dword, Eq, KVM_SET_MEMORY_ATTRIBUTES,)?], and![Cond::new(1, ArgLen::Dword, Eq, KVM_RUN,)?], and![Cond::new(1, ArgLen::Dword, Eq, KVM_NMI)?], and![Cond::new(1, ArgLen::Dword, Eq, KVM_GET_NESTED_STATE)?],