From e7c0d690d05c26eb6cde1eb8ee629a487c3eaf7f Mon Sep 17 00:00:00 2001 From: Saravanan D Date: Fri, 17 Apr 2026 08:29:56 +0000 Subject: [PATCH] pci: vfio: Implement save path state transitions Wire a migratable VFIO device's migration state to the VM lifecycle so the device's internal state survives snapshot and restore. A device such as a ConnectX VF bound to mlx5_vfio_pci would otherwise come back blank, because a plain snapshot saves only the PCI configuration Cloud Hypervisor owns, not the device's own state. On save, pause moves the device to STOP and snapshot() drives it through STOP_COPY to extract the opaque state blob, attached to the device snapshot as a base64 encoded child. resume() returns it to RUNNING. All new behavior is gated on migration_flags.is_some(), so devices without migration support (including vfio-user) retain their previous snapshot behavior. If the data read fails after STOP_COPY was entered, the device is returned to STOP before the error is bubbled, since the STOP_COPY to STOP arc stays valid. A failed transition into STOP_COPY returns immediately because a STOP from the resulting ERROR state cannot help. Full recovery including device reset is deferred. Since the non BAR write path goes directly to the VFIO device and not the shadow, the PciConfiguration shadow can get stale. Mirror every non BAR, non MSI config write into the shadow via write_byte / write_word / write_reg so snapshot() can capture PCI_COMMAND. Without this the shadow keeps the values set at device init and snapshot() encodes PCI_COMMAND as zero. Use the raw write_byte, write_word, and write_reg helpers rather than PciConfiguration::write_config_register, which would otherwise drain pending_bar_reprogram, consumed by the BAR block below, and rerun MSI-X set_msg_ctl, already done by update_msix_capabilities. Signed-off-by: Saravanan D --- Cargo.lock | 7 ++ Cargo.toml | 1 + fuzz/Cargo.lock | 7 ++ pci/Cargo.toml | 1 + pci/src/vfio.rs | 172 +++++++++++++++++++++++++++++++++++++++++++++++- 5 files changed, 185 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 257835a98..7d81eb162 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -287,6 +287,12 @@ dependencies = [ "windows-link", ] +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + [[package]] name = "bit-set" version = "0.8.0" @@ -1661,6 +1667,7 @@ name = "pci" version = "0.1.0" dependencies = [ "anyhow", + "base64", "byteorder", "hypervisor", "libc", diff --git a/Cargo.toml b/Cargo.toml index 832ad145e..4683f618d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -83,6 +83,7 @@ serde_with = { version = "3.19.0", default-features = false } # other crates anyhow = "1.0.102" +base64 = "0.22.1" bitflags = "2.11.1" byteorder = "1.5.0" cfg-if = "1.0.4" diff --git a/fuzz/Cargo.lock b/fuzz/Cargo.lock index 5ff43d821..3c1004443 100644 --- a/fuzz/Cargo.lock +++ b/fuzz/Cargo.lock @@ -113,6 +113,12 @@ version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + [[package]] name = "bitfield-struct" version = "0.13.0" @@ -980,6 +986,7 @@ name = "pci" version = "0.1.0" dependencies = [ "anyhow", + "base64", "byteorder", "hypervisor", "libc", diff --git a/pci/Cargo.toml b/pci/Cargo.toml index c1b69f985..1f73ad945 100644 --- a/pci/Cargo.toml +++ b/pci/Cargo.toml @@ -12,6 +12,7 @@ mshv = ["hypervisor/mshv", "vfio-ioctls/mshv"] [dependencies] anyhow = { workspace = true } +base64 = { workspace = true } byteorder = { workspace = true } hypervisor = { path = "../hypervisor" } libc = { workspace = true } diff --git a/pci/src/vfio.rs b/pci/src/vfio.rs index bc74bc84a..2c668b40b 100644 --- a/pci/src/vfio.rs +++ b/pci/src/vfio.rs @@ -12,13 +12,24 @@ use std::sync::{Arc, Barrier, Mutex}; use std::{cmp, io, result}; use anyhow::anyhow; +use base64::Engine; +use base64::engine::general_purpose::STANDARD as BASE64_STANDARD; use byteorder::{ByteOrder, LittleEndian}; use hypervisor::HypervisorVmError; use libc::{_SC_PAGESIZE, sysconf}; use log::{debug, error, info}; use serde::{Deserialize, Serialize}; use thiserror::Error; -use vfio_bindings::bindings::vfio::*; +use vfio_bindings::bindings::vfio::{ + vfio_device_mig_state_VFIO_DEVICE_STATE_ERROR as VFIO_DEV_STATE_ERROR, + vfio_device_mig_state_VFIO_DEVICE_STATE_PRE_COPY as VFIO_DEV_STATE_PRE_COPY, + vfio_device_mig_state_VFIO_DEVICE_STATE_PRE_COPY_P2P as VFIO_DEV_STATE_PRE_COPY_P2P, + vfio_device_mig_state_VFIO_DEVICE_STATE_RESUMING as VFIO_DEV_STATE_RESUMING, + vfio_device_mig_state_VFIO_DEVICE_STATE_RUNNING as VFIO_DEV_STATE_RUNNING, + vfio_device_mig_state_VFIO_DEVICE_STATE_RUNNING_P2P as VFIO_DEV_STATE_RUNNING_P2P, + vfio_device_mig_state_VFIO_DEVICE_STATE_STOP as VFIO_DEV_STATE_STOP, + vfio_device_mig_state_VFIO_DEVICE_STATE_STOP_COPY as VFIO_DEV_STATE_STOP_COPY, *, +}; use vfio_ioctls::{VfioDevice, VfioIrq, VfioOps, VfioRegionInfoCap, VfioRegionSparseMmapArea}; use vm_allocator::page_size::{ align_page_size_down, align_page_size_up, get_page_size, is_4k_aligned, is_4k_multiple, @@ -46,6 +57,7 @@ use crate::{ }; pub(crate) const VFIO_COMMON_ID: &str = "vfio_common"; +pub(crate) const VFIO_MIGRATION_ID: &str = "vfio_migration"; #[derive(Debug, Error)] pub enum VfioPciError { @@ -364,6 +376,55 @@ pub enum VfioError { KernelVfio(#[source] vfio_ioctls::VfioError), #[error("VFIO user error")] VfioUser(#[source] vfio_user::Error), + #[error("VFIO device does not support migration")] + NoMigrationSupport, + #[error("VFIO device reported unknown migration state {0}")] + InvalidMigrationState(u32), +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum VfioMigrationState { + Error, + Stop, + Running, + StopCopy, + Resuming, + RunningP2P, + PreCopy, + PreCopyP2P, +} + +impl From for u32 { + fn from(state: VfioMigrationState) -> u32 { + match state { + VfioMigrationState::Error => VFIO_DEV_STATE_ERROR, + VfioMigrationState::Stop => VFIO_DEV_STATE_STOP, + VfioMigrationState::Running => VFIO_DEV_STATE_RUNNING, + VfioMigrationState::StopCopy => VFIO_DEV_STATE_STOP_COPY, + VfioMigrationState::Resuming => VFIO_DEV_STATE_RESUMING, + VfioMigrationState::RunningP2P => VFIO_DEV_STATE_RUNNING_P2P, + VfioMigrationState::PreCopy => VFIO_DEV_STATE_PRE_COPY, + VfioMigrationState::PreCopyP2P => VFIO_DEV_STATE_PRE_COPY_P2P, + } + } +} + +impl TryFrom for VfioMigrationState { + type Error = VfioError; + + fn try_from(value: u32) -> Result { + match value { + VFIO_DEV_STATE_ERROR => Ok(Self::Error), + VFIO_DEV_STATE_STOP => Ok(Self::Stop), + VFIO_DEV_STATE_RUNNING => Ok(Self::Running), + VFIO_DEV_STATE_STOP_COPY => Ok(Self::StopCopy), + VFIO_DEV_STATE_RESUMING => Ok(Self::Resuming), + VFIO_DEV_STATE_RUNNING_P2P => Ok(Self::RunningP2P), + VFIO_DEV_STATE_PRE_COPY => Ok(Self::PreCopy), + VFIO_DEV_STATE_PRE_COPY_P2P => Ok(Self::PreCopyP2P), + other => Err(VfioError::InvalidMigrationState(other)), + } + } } pub(crate) trait Vfio: Send + Sync { @@ -441,6 +502,14 @@ pub(crate) trait Vfio: Send + Sync { fn migration_flags(&self) -> Result, VfioError> { Ok(None) } + + fn set_migration_state(&self, _state: VfioMigrationState) -> Result<(), VfioError> { + Err(VfioError::NoMigrationSupport) + } + + fn read_migration_data(&self) -> Result, VfioError> { + Err(VfioError::NoMigrationSupport) + } } struct VfioDeviceWrapper { @@ -489,6 +558,18 @@ impl Vfio for VfioDeviceWrapper { .query_migration_support() .map_err(VfioError::KernelVfio) } + + fn set_migration_state(&self, state: VfioMigrationState) -> Result<(), VfioError> { + self.device + .set_migration_state(state.into()) + .map_err(VfioError::KernelVfio) + } + + fn read_migration_data(&self) -> Result, VfioError> { + self.device + .read_migration_data_to_end() + .map_err(VfioError::KernelVfio) + } } #[derive(Serialize, Deserialize)] @@ -498,6 +579,11 @@ struct VfioCommonState { msix_state: Option, } +#[derive(Serialize, Deserialize)] +struct VfioMigrationData { + blob: String, +} + pub(crate) struct ConfigPatch { mask: u32, patch: u32, @@ -513,7 +599,6 @@ pub(crate) struct VfioCommon { pub(crate) patches: HashMap, x_nv_gpudirect_clique: Option, x_exclude_mmap_bars: Vec, - #[allow(dead_code)] pub(crate) migration_flags: Option, } @@ -1359,6 +1444,27 @@ impl VfioCommon { // to the device region to update the MSI Enable bit. self.vfio_wrapper.write_config((reg + offset) as u32, data); + // The non BAR write path goes directly to the VFIO device and not the shadow, + // so the PciConfiguration shadow can get stale. Mirror the write into the + // shadow here since snapshot() serializes it. Without this the shadow keeps its + // device init values and a snapshot encodes PCI_COMMAND as zero. + // + // Use the raw write_* helpers rather than the PciConfiguration method + // self.configuration.write_config_register(), which would otherwise drain + // pending_bar_reprogram, owned by the BAR block below, and rerun MSI-X + // set_msg_ctl, already done by update_msix_capabilities above. + let byte_offset = reg_idx * PCI_CONFIG_REGISTER_SIZE + offset as usize; + match data.len() { + 1 => self.configuration.write_byte(byte_offset, data[0]), + 2 => self + .configuration + .write_word(byte_offset, u16::from(data[0]) | (u16::from(data[1]) << 8)), + 4 => self + .configuration + .write_reg(reg_idx, LittleEndian::read_u32(data)), + _ => {} + } + // Return pending BAR repgrogramming if MSE bit is set let mut ret_param = self.configuration.pending_bar_reprogram(); if !ret_param.is_empty() { @@ -1470,6 +1576,37 @@ impl VfioCommon { Ok(()) } + + pub(crate) fn transition_migration_state( + &self, + target: VfioMigrationState, + ) -> anyhow::Result<()> { + debug!("VFIO migration transition -> {target:?}"); + self.vfio_wrapper + .set_migration_state(target) + .map_err(|e| anyhow!("VFIO set_migration_state({target:?}) failed: {e}")) + } + + pub(crate) fn save_migration_data(&self) -> Result, MigratableError> { + self.transition_migration_state(VfioMigrationState::StopCopy) + .map_err(MigratableError::Snapshot)?; + + let data = self.vfio_wrapper.read_migration_data(); + + // We entered STOP_COPY successfully, so the STOP_COPY to STOP arc is + // valid whether or not the read succeeded. Return the device to STOP + // either way, since a failed transition into STOP_COPY would have + // returned above and a STOP from ERROR cannot help. + let stop = self + .transition_migration_state(VfioMigrationState::Stop) + .map_err(MigratableError::Snapshot); + + let data = data.map_err(|e| { + MigratableError::Snapshot(anyhow!("VFIO migration data read failed: {e}")) + })?; + stop?; + Ok(data) + } } impl Pausable for VfioCommon {} @@ -1495,6 +1632,17 @@ impl Snapshottable for VfioCommon { vfio_common_snapshot.add_snapshot(msix.bar.id(), msix.bar.snapshot()?); } + if self.migration_flags.is_some() { + let data = self.save_migration_data()?; + let mig = VfioMigrationData { + blob: BASE64_STANDARD.encode(&data), + }; + vfio_common_snapshot.add_snapshot( + VFIO_MIGRATION_ID.to_string(), + Snapshot::new_from_state(&mig)?, + ); + } + Ok(vfio_common_snapshot) } } @@ -2101,7 +2249,25 @@ iova 0x{:x}, size 0x{:x}: {}, ", } } -impl Pausable for VfioPciDevice {} +impl Pausable for VfioPciDevice { + fn pause(&mut self) -> result::Result<(), MigratableError> { + if self.common.migration_flags.is_some() { + self.common + .transition_migration_state(VfioMigrationState::Stop) + .map_err(MigratableError::Pause)?; + } + Ok(()) + } + + fn resume(&mut self) -> result::Result<(), MigratableError> { + if self.common.migration_flags.is_some() { + self.common + .transition_migration_state(VfioMigrationState::Running) + .map_err(MigratableError::Resume)?; + } + Ok(()) + } +} impl Snapshottable for VfioPciDevice { fn id(&self) -> String {