diff --git a/hypervisor/src/cpu.rs b/hypervisor/src/cpu.rs index 81e98ca7e..777f571f0 100644 --- a/hypervisor/src/cpu.rs +++ b/hypervisor/src/cpu.rs @@ -516,6 +516,11 @@ pub trait Vcpu: Send + Sync { #[cfg(all(target_arch = "aarch64", feature = "kvm"))] fn get_cntvct(&self) -> Result; /// + /// Sets the guest virtual counter (`CNTVCT_EL0`). + /// + #[cfg(all(target_arch = "aarch64", feature = "kvm"))] + fn set_cntvct(&self, val: u64) -> Result<()>; + /// /// Gets the value of a non-core register on RISC-V 64-bit /// #[cfg(target_arch = "riscv64")] diff --git a/hypervisor/src/kvm/mod.rs b/hypervisor/src/kvm/mod.rs index 78800031c..5d70f0cf5 100644 --- a/hypervisor/src/kvm/mod.rs +++ b/hypervisor/src/kvm/mod.rs @@ -81,13 +81,13 @@ pub use x86_64::{CpuId, ExtendedControlRegisters, MsrEntries, VcpuKvmState}; #[cfg(target_arch = "x86_64")] use crate::ClockData; -#[cfg(any(target_arch = "x86_64", all(target_arch = "aarch64", feature = "kvm")))] -use crate::ClockState; #[cfg(target_arch = "x86_64")] use crate::arch::x86::{ CpuIdEntry, FpuState, LapicState, MTRR_MSR_INDICES, MsrEntry, NUM_IOAPIC_PINS, SpecialRegisters, XsaveState, }; +#[cfg(any(target_arch = "x86_64", target_arch = "aarch64"))] +use crate::{ClockRestoreMode, ClockState}; use crate::{ CpuState, HypervisorType, HypervisorVmConfig, InterruptSourceConfig, IoEventAddress, IrqRoutingEntry, MpState, StandardRegisters, USER_MEMORY_REGION_GUEST_MEMFD, @@ -168,6 +168,9 @@ const KVM_REG_ARM_TIMER_CNT: u64 = KVM_REG_ARM64 | ((3_u64 << KVM_REG_ARM64_SYSREG_CRM_SHIFT) & KVM_REG_ARM64_SYSREG_CRM_MASK as u64) | (2_u64 & KVM_REG_ARM64_SYSREG_OP2_MASK as u64); +#[cfg(target_arch = "aarch64")] +const NANOS_PER_SECOND: u128 = 1_000_000_000; + #[cfg(target_arch = "x86_64")] ioctl_io_nr!(KVM_NMI, kvm_bindings::KVMIO, 0x9a); @@ -1315,10 +1318,60 @@ impl vm::Vm for KvmVm { /// Restore kvmclock before the vCPUs resume. #[cfg(target_arch = "x86_64")] - fn restore_clock(&self, state: &ClockState) -> vm::Result<()> { + fn restore_clock( + &self, + _vcpus: &[&dyn cpu::Vcpu], + state: &ClockState, + _mode: ClockRestoreMode, + ) -> vm::Result<()> { self.set_clock(state) } + /// Advance the guest virtual counter to current wall time before the vCPUs + /// resume. A no-op for a `SameHostResume` (the counter free-ran across the + /// pause); on `SnapshotRestore`/migration-receive it must catch up. + #[cfg(target_arch = "aarch64")] + fn restore_clock( + &self, + vcpus: &[&dyn cpu::Vcpu], + saved: &ClockState, + mode: ClockRestoreMode, + ) -> vm::Result<()> { + if mode == ClockRestoreMode::SameHostResume { + return Ok(()); + } + // KVM does not rescale the counter frequency across hosts (unlike x86 + // TSC), so a differing destination frequency would scale the elapsed + // ticks incorrectly. Reject rather than corrupt the guest clock. + let host_cntfrq = get_cntfrq(); + if host_cntfrq != saved.cntfrq { + return Err(vm::HypervisorVmError::CntfrqMismatch { + saved: saved.cntfrq, + host: host_cntfrq, + }); + } + let now_ns = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_err(|e| vm::HypervisorVmError::RestoreTimerState(e.into()))? + .as_nanos() as u64; + let elapsed_ns = now_ns.saturating_sub(saved.host_realtime_ns); + let elapsed_ticks = (elapsed_ns as u128 * saved.cntfrq as u128 / NANOS_PER_SECOND) as u64; + let target = saved.cntvct.wrapping_add(elapsed_ticks); + // Linux >= 6.4 (KVM_CAP_COUNTER_OFFSET) tracks the vtimer offset VM-wide, so one + // write advances every vCPU; older kernels track it per-vCPU, so program each. + // Restore is the snapshot-boot hot path, so skip the redundant writes when we can. + let targets = if self.check_extension(Cap::CounterOffset) { + &vcpus[..1] + } else { + vcpus + }; + for vcpu in targets { + vcpu.set_cntvct(target) + .map_err(|e| vm::HypervisorVmError::RestoreTimerState(e.into()))?; + } + Ok(()) + } + /// Create a device that is used for passthrough fn create_passthrough_device(&self) -> vm::Result { let mut vfio_dev = kvm_create_device { @@ -2789,6 +2842,17 @@ impl cpu::Vcpu for KvmVcpu { Ok(u64::from_le_bytes(bytes)) } + /// + /// Sets the guest virtual counter via `KVM_REG_ARM_TIMER_CNT`. + /// + #[cfg(target_arch = "aarch64")] + fn set_cntvct(&self, val: u64) -> cpu::Result<()> { + self.fd + .set_one_reg(KVM_REG_ARM_TIMER_CNT, &val.to_le_bytes()) + .map_err(|e| cpu::HypervisorCpuError::SetSysRegister(e.into()))?; + Ok(()) + } + /// /// Gets the value of a non-core register /// diff --git a/hypervisor/src/lib.rs b/hypervisor/src/lib.rs index c688ddeb7..145183fd9 100644 --- a/hypervisor/src/lib.rs +++ b/hypervisor/src/lib.rs @@ -249,6 +249,17 @@ pub struct TimerState { pub cntfrq: u64, } +/// How the guest clock is re-established when the vCPUs resume: a same-host +/// pause/resume left it running, whereas a snapshot restore / migration-receive +/// means the guest was off-host and the clock must catch up to wall time. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ClockRestoreMode { + /// Same-host pause -> resume; the clock kept running, nothing to advance. + SameHostResume, + /// Restored from a snapshot or migrated in; advance to current wall time. + SnapshotRestore, +} + #[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] pub struct HypervisorVmConfig { #[cfg(feature = "tdx")] diff --git a/hypervisor/src/mshv/mod.rs b/hypervisor/src/mshv/mod.rs index 09a70a8f7..1f297ef7a 100644 --- a/hypervisor/src/mshv/mod.rs +++ b/hypervisor/src/mshv/mod.rs @@ -77,14 +77,14 @@ pub use { mshv_bindings::mshv_device_attr as DeviceAttr, mshv_ioctls, mshv_ioctls::DeviceFd, }; -#[cfg(target_arch = "x86_64")] -use crate::{ClockData, ClockState}; #[cfg(target_arch = "aarch64")] use crate::arch::aarch64::gic::{Vgic, VgicConfig}; #[cfg(target_arch = "aarch64")] use crate::arch::aarch64::regs; #[cfg(target_arch = "x86_64")] use crate::arch::x86::{CpuIdEntry, FpuState, MsrEntry}; +#[cfg(target_arch = "x86_64")] +use crate::{ClockData, ClockRestoreMode, ClockState}; use crate::{CpuState, IoEventAddress, IrqRoutingEntry, MpState}; pub const PAGE_SHIFT: usize = 12; @@ -1329,6 +1329,11 @@ impl cpu::Vcpu for MshvVcpu { unimplemented!() } + #[cfg(all(target_arch = "aarch64", feature = "kvm"))] + fn set_cntvct(&self, _val: u64) -> cpu::Result<()> { + unimplemented!() + } + #[cfg(target_arch = "aarch64")] fn get_reg_list(&self, _reg_list: &mut crate::RegList) -> cpu::Result<()> { unimplemented!() @@ -2213,7 +2218,12 @@ impl vm::Vm for MshvVm { /// Restore the partition reference time before the vCPUs resume. #[cfg(target_arch = "x86_64")] - fn restore_clock(&self, state: &ClockState) -> vm::Result<()> { + fn restore_clock( + &self, + _vcpus: &[&dyn cpu::Vcpu], + state: &ClockState, + _mode: ClockRestoreMode, + ) -> vm::Result<()> { self.set_clock(state) } diff --git a/hypervisor/src/vm.rs b/hypervisor/src/vm.rs index d1d30cf4d..ddafded64 100644 --- a/hypervisor/src/vm.rs +++ b/hypervisor/src/vm.rs @@ -24,7 +24,6 @@ use vmm_sys_util::eventfd::EventFd; #[cfg(target_arch = "x86_64")] use crate::ClockData; -use crate::ClockState; #[cfg(target_arch = "aarch64")] use crate::arch::aarch64::gic::{Vgic, VgicConfig}; #[cfg(target_arch = "riscv64")] @@ -32,7 +31,7 @@ use crate::arch::riscv64::aia::{Vaia, VaiaConfig}; #[cfg(feature = "tdx")] use crate::arch::x86::CpuIdEntry; use crate::cpu::Vcpu; -use crate::{IoEventAddress, IrqRoutingEntry}; +use crate::{ClockRestoreMode, ClockState, IoEventAddress, IrqRoutingEntry}; /// /// I/O events data matches (32 or 64 bits). @@ -147,6 +146,20 @@ pub enum HypervisorVmError { #[error("Failed to capture the guest timer state")] CaptureTimerState(#[source] anyhow::Error), /// + /// Restore guest timer state error (aarch64) + /// + #[cfg(all(target_arch = "aarch64", feature = "kvm"))] + #[error("Failed to restore the guest timer state")] + RestoreTimerState(#[source] anyhow::Error), + /// + /// Counter frequency mismatch on restore (aarch64) + /// + #[cfg(all(target_arch = "aarch64", feature = "kvm"))] + #[error( + "Saved counter frequency ({saved} Hz) != host ({host} Hz); refusing to advance guest counter" + )] + CntfrqMismatch { saved: u64, host: u64 }, + /// /// Create passthrough device /// #[error("Failed to create passthrough device")] @@ -398,9 +411,17 @@ pub trait Vm: Send + Sync + Any { fn snapshot_clock(&self, _boot_vcpu: &dyn Vcpu) -> Result> { Ok(None) } - /// Re-establish the guest clock before the vCPUs resume. - #[cfg(target_arch = "x86_64")] - fn restore_clock(&self, _state: &ClockState) -> Result<()> { + /// Re-establish the guest clock before the vCPUs resume. `mode` distinguishes a + /// same-host pause/resume (the clock kept running) from a restore/migration where + /// it must catch up to wall time; x86 ignores it (kvmclock is re-applied on every + /// resume). aarch64 writes the counter on all `vcpus` (older kernels track the + /// offset per-vCPU); x86 ignores `vcpus`. + fn restore_clock( + &self, + _vcpus: &[&dyn Vcpu], + _state: &ClockState, + _mode: ClockRestoreMode, + ) -> Result<()> { Ok(()) } /// Create a device that is used for passthrough diff --git a/vmm/src/lib.rs b/vmm/src/lib.rs index 89c692faa..06549c93e 100644 --- a/vmm/src/lib.rs +++ b/vmm/src/lib.rs @@ -625,7 +625,7 @@ pub struct VmmThreadHandle { /// Models the current ownership and associated state of the VM from the /// perspective of the VMM. -#[cfg_attr(feature = "tdx", expect(clippy::large_enum_variant))] +#[allow(clippy::large_enum_variant)] pub enum VmOwnership { Owned(Vm), None, diff --git a/vmm/src/vm.rs b/vmm/src/vm.rs index f8a44b456..67d3aec08 100644 --- a/vmm/src/vm.rs +++ b/vmm/src/vm.rs @@ -513,6 +513,16 @@ pub fn physical_bits(hypervisor: &dyn hypervisor::Hypervisor, max_phys_bits: u8) cmp::min(host_phys_bits, max_phys_bits) } +/// Guest clock baseline captured for snapshot/restore, plus how it must be +/// re-established on the next resume. `mode` is runtime-only (`SnapshotRestore` +/// after restore/migration-receive, `SameHostResume` for a live same-host capture) +/// and is not serialized: it is re-derived from "was this VM built from a snapshot". +#[derive(Clone, Copy)] +struct SavedClock { + mode: hypervisor::ClockRestoreMode, + state: hypervisor::ClockState, +} + pub struct Vm { #[cfg(feature = "tdx")] kernel: Option, @@ -526,7 +536,7 @@ pub struct Vm { #[cfg_attr(any(not(feature = "kvm"), target_arch = "aarch64"), allow(dead_code))] // The hypervisor abstracted virtual machine. vm: Arc, - saved_clock: Option, + saved_clock: Option, #[cfg(not(target_arch = "riscv64"))] numa_nodes: NumaNodes, #[cfg_attr(any(not(feature = "kvm"), target_arch = "aarch64"), allow(dead_code))] @@ -681,7 +691,12 @@ impl Vm { let saved_clock = if let Some(snapshot) = snapshot.as_ref() { let vm_snapshot = get_vm_snapshot(snapshot).map_err(Error::Restore)?; - vm_snapshot.clock + // Restored or migrated in: the guest clock must catch up to wall time + // on resume (the counter was reset to the saved value). + vm_snapshot.clock.map(|state| SavedClock { + mode: hypervisor::ClockRestoreMode::SnapshotRestore, + state, + }) } else { None }; @@ -3199,6 +3214,37 @@ impl Vm { .nmi() .map_err(Error::ErrorNmi); } + + /// Capture the guest clock for a same-host resume (mode `SameHostResume`). + /// `None` if the VM is not booted or the backend has no guest clock. + fn capture_guest_clock(&self) -> std::result::Result, MigratableError> { + let Some(boot_vcpu) = self.cpu_manager.lock().unwrap().boot_vcpu() else { + return Ok(None); + }; + let boot_vcpu = boot_vcpu.lock().unwrap(); + Ok(self + .vm + .snapshot_clock(boot_vcpu.hypervisor_vcpu()) + .map_err(|e| MigratableError::Pause(anyhow!("Could not capture guest clock: {e}")))? + .map(|state| SavedClock { + mode: hypervisor::ClockRestoreMode::SameHostResume, + state, + })) + } + + /// Re-establish the guest clock before the vCPUs resume. No-op if none captured. + fn restore_guest_clock(&self) -> std::result::Result<(), MigratableError> { + let Some(saved) = &self.saved_clock else { + return Ok(()); + }; + let vcpus = self.cpu_manager.lock().unwrap().vcpus(); + let guards: Vec<_> = vcpus.iter().map(|v| v.lock().unwrap()).collect(); + let hv_vcpus: Vec<&dyn hypervisor::Vcpu> = + guards.iter().map(|g| g.hypervisor_vcpu()).collect(); + self.vm + .restore_clock(&hv_vcpus, &saved.state, saved.mode) + .map_err(|e| MigratableError::Resume(anyhow!("Could not restore guest clock: {e}"))) + } } impl Pausable for Vm { @@ -3217,15 +3263,9 @@ impl Pausable for Vm { self.cpu_manager.lock().unwrap().pause()?; - // Capture the guest clock once the vCPUs are paused: aarch64 reads the - // boot vCPU's virtual counter, which requires the vCPU to be quiesced. - if let Some(boot_vcpu) = self.cpu_manager.lock().unwrap().boot_vcpu() { - let boot_vcpu = boot_vcpu.lock().unwrap(); - self.saved_clock = self - .vm - .snapshot_clock(boot_vcpu.hypervisor_vcpu()) - .map_err(|e| MigratableError::Pause(anyhow!("Could not capture guest clock: {e}")))?; - } + // Capture the guest clock now that the vCPUs are quiesced. + self.saved_clock = self.capture_guest_clock()?; + self.device_manager.lock().unwrap().pause()?; self.vm @@ -3247,16 +3287,8 @@ impl Pausable for Vm { .valid_transition(new_state) .map_err(|e| MigratableError::Resume(anyhow!("Invalid transition: {e:?}")))?; - // Restore the guest clock BEFORE the vCPUs start running, so they see the - // corrected time from the first instruction after resume. - #[cfg(target_arch = "x86_64")] - { - if let Some(state) = &self.saved_clock { - self.vm.restore_clock(state).map_err(|e| { - MigratableError::Resume(anyhow!("Could not restore guest clock: {e}")) - })?; - } - } + // Restore the guest clock before the vCPUs start running. + self.restore_guest_clock()?; if current_state == VmState::Paused { self.vm @@ -3276,7 +3308,7 @@ impl Pausable for Vm { #[derive(Serialize, Deserialize)] pub struct VmSnapshot { - #[serde(default)] + #[serde(default, skip_serializing_if = "Option::is_none")] pub clock: Option, #[cfg(all(feature = "kvm", target_arch = "x86_64"))] pub common_cpuid: Vec, @@ -3338,7 +3370,7 @@ impl Snapshottable for Vm { }; let vm_snapshot_state = VmSnapshot { - clock: self.saved_clock, + clock: self.saved_clock.map(|saved| saved.state), #[cfg(all(feature = "kvm", target_arch = "x86_64"))] common_cpuid, };