diff --git a/hypervisor/src/cpu.rs b/hypervisor/src/cpu.rs index 777f571f0..ffecd1e46 100644 --- a/hypervisor/src/cpu.rs +++ b/hypervisor/src/cpu.rs @@ -311,6 +311,18 @@ pub enum HypervisorCpuError { /// #[error("Failed to set TSC frequency")] SetTscKhz(#[source] anyhow::Error), + #[cfg(target_arch = "x86_64")] + /// + /// Error getting TSC offset + /// + #[error("Failed to get TSC offset")] + GetTscOffset(#[source] anyhow::Error), + #[cfg(target_arch = "x86_64")] + /// + /// Error setting TSC offset + /// + #[error("Failed to set TSC offset")] + SetTscOffset(#[source] anyhow::Error), /// /// Error reading value at given GPA /// @@ -614,6 +626,21 @@ pub trait Vcpu: Send + Sync { } #[cfg(target_arch = "x86_64")] /// + /// Read the vCPU's TSC offset, or `None` when the hypervisor does not + /// expose it. + /// + fn tsc_offset(&self) -> Result> { + Ok(None) + } + #[cfg(target_arch = "x86_64")] + /// + /// Set the vCPU's TSC offset if available. + /// + fn set_tsc_offset(&self, _offset: u64) -> Result<()> { + Ok(()) + } + #[cfg(target_arch = "x86_64")] + /// /// X86 specific call to retrieve cpuid leaf /// fn get_cpuid_values( diff --git a/hypervisor/src/kvm/mod.rs b/hypervisor/src/kvm/mod.rs index b5c9b6186..788767ec1 100644 --- a/hypervisor/src/kvm/mod.rs +++ b/hypervisor/src/kvm/mod.rs @@ -140,7 +140,9 @@ use log::error; use thiserror::Error; use vfio_ioctls::VfioDeviceFd; #[cfg(target_arch = "x86_64")] -use vmm_sys_util::{fam::FamStruct, ioctl_io_nr}; +use vmm_sys_util::ioctl::ioctl_with_ref; +#[cfg(target_arch = "x86_64")] +use vmm_sys_util::{fam::FamStruct, ioctl_io_nr, ioctl_iow_nr}; #[cfg(feature = "tdx")] use vmm_sys_util::{ioctl::ioctl_with_val, ioctl_iowr_nr}; @@ -173,6 +175,28 @@ const NANOS_PER_SECOND: u128 = 1_000_000_000; #[cfg(target_arch = "x86_64")] ioctl_io_nr!(KVM_NMI, kvm_bindings::KVMIO, 0x9a); +// kvm-ioctls only exposes the vCPU device-attribute ioctls for aarch64. +#[cfg(target_arch = "x86_64")] +ioctl_iow_nr!( + KVM_SET_DEVICE_ATTR, + kvm_bindings::KVMIO, + 0xe1, + kvm_bindings::kvm_device_attr +); +#[cfg(target_arch = "x86_64")] +ioctl_iow_nr!( + KVM_GET_DEVICE_ATTR, + kvm_bindings::KVMIO, + 0xe2, + kvm_bindings::kvm_device_attr +); +#[cfg(target_arch = "x86_64")] +ioctl_iow_nr!( + KVM_HAS_DEVICE_ATTR, + kvm_bindings::KVMIO, + 0xe3, + kvm_bindings::kvm_device_attr +); #[cfg(feature = "sev_snp")] use igvm_defs::PAGE_SIZE_4K; @@ -1919,6 +1943,29 @@ impl KvmVcpu { } } +#[cfg(target_arch = "x86_64")] +impl KvmVcpu { + /// Device attribute selecting the vCPU's L1 TSC offset, `addr` pointing at + /// the caller's `u64` buffer. + fn tsc_offset_attr(addr: u64) -> DeviceAttr { + DeviceAttr { + group: kvm_bindings::KVM_VCPU_TSC_CTRL, + attr: u64::from(kvm_bindings::KVM_VCPU_TSC_OFFSET), + addr, + flags: 0, + } + } + + /// Whether the host kernel supports the vCPU TSC offset attribute (Linux + /// 5.16+). + fn has_tsc_offset_attr(&self) -> bool { + let attr = Self::tsc_offset_attr(0); + // SAFETY: FFI call with a valid kvm_device_attr; `addr` is unused here. + let ret = unsafe { ioctl_with_ref(&self.fd, KVM_HAS_DEVICE_ATTR(), &attr) }; + ret == 0 + } +} + /// Implementation of Vcpu trait for KVM /// /// # Examples @@ -3536,6 +3583,43 @@ impl cpu::Vcpu for KvmVcpu { } } + #[cfg(target_arch = "x86_64")] + /// + /// Read the vCPU's L1 TSC offset, or `None` if the host kernel lacks the + /// attribute (Linux < 5.16). + /// + fn tsc_offset(&self) -> cpu::Result> { + if !self.has_tsc_offset_attr() { + return Ok(None); + } + let mut offset = 0u64; + let attr = Self::tsc_offset_attr(&raw mut offset as u64); + // SAFETY: FFI call; `attr.addr` points to `offset`, filled in by the kernel. + let ret = unsafe { ioctl_with_ref(&self.fd, KVM_GET_DEVICE_ATTR(), &attr) }; + if ret < 0 { + return Err(cpu::HypervisorCpuError::GetTscOffset( + io::Error::last_os_error().into(), + )); + } + Ok(Some(offset)) + } + + #[cfg(target_arch = "x86_64")] + /// + /// Set the vCPU's L1 TSC offset. + /// + fn set_tsc_offset(&self, offset: u64) -> cpu::Result<()> { + let attr = Self::tsc_offset_attr(&raw const offset as u64); + // SAFETY: FFI call; `attr.addr` points to `offset`, read by the kernel. + let ret = unsafe { ioctl_with_ref(&self.fd, KVM_SET_DEVICE_ATTR(), &attr) }; + if ret < 0 { + return Err(cpu::HypervisorCpuError::SetTscOffset( + io::Error::last_os_error().into(), + )); + } + Ok(()) + } + #[cfg(target_arch = "x86_64")] /// /// Trigger NMI interrupt diff --git a/vmm/src/cpu.rs b/vmm/src/cpu.rs index 1c7cd2b4c..dd1eaf685 100644 --- a/vmm/src/cpu.rs +++ b/vmm/src/cpu.rs @@ -1120,9 +1120,36 @@ impl CpuManager { )?); } + #[cfg(target_arch = "x86_64")] + if snapshot.is_some() { + Self::synchronize_tsc_offsets(&vcpus); + } + Ok(vcpus) } + // Synchronize TSC offsets (mitigates each vCPU being restored at a slightly different time) + // and allows the TSC masterclock to engage. + #[cfg(target_arch = "x86_64")] + fn synchronize_tsc_offsets(vcpus: &[Arc>]) { + let Some(reference) = vcpus.first() else { + return; + }; + let offset = match reference.lock().unwrap().vcpu.tsc_offset() { + Ok(Some(offset)) => offset, + Ok(None) => return, + Err(e) => { + warn!("Could not read vCPU TSC offset for resync: {e}"); + return; + } + }; + for vcpu in vcpus { + if let Err(e) = vcpu.lock().unwrap().vcpu.set_tsc_offset(offset) { + warn!("Could not synchronize vCPU TSC offset: {e}"); + } + } + } + #[cfg(target_arch = "aarch64")] pub fn init_pmu(&self, irq: u32) -> Result { for cpu in self.vcpus.iter() {