diff --git a/Cargo.lock b/Cargo.lock index 257835a98..7d81eb162 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -287,6 +287,12 @@ dependencies = [ "windows-link", ] +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + [[package]] name = "bit-set" version = "0.8.0" @@ -1661,6 +1667,7 @@ name = "pci" version = "0.1.0" dependencies = [ "anyhow", + "base64", "byteorder", "hypervisor", "libc", diff --git a/Cargo.toml b/Cargo.toml index 832ad145e..4683f618d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -83,6 +83,7 @@ serde_with = { version = "3.19.0", default-features = false } # other crates anyhow = "1.0.102" +base64 = "0.22.1" bitflags = "2.11.1" byteorder = "1.5.0" cfg-if = "1.0.4" diff --git a/fuzz/Cargo.lock b/fuzz/Cargo.lock index 5ff43d821..3c1004443 100644 --- a/fuzz/Cargo.lock +++ b/fuzz/Cargo.lock @@ -113,6 +113,12 @@ version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + [[package]] name = "bitfield-struct" version = "0.13.0" @@ -980,6 +986,7 @@ name = "pci" version = "0.1.0" dependencies = [ "anyhow", + "base64", "byteorder", "hypervisor", "libc", diff --git a/pci/Cargo.toml b/pci/Cargo.toml index c1b69f985..1f73ad945 100644 --- a/pci/Cargo.toml +++ b/pci/Cargo.toml @@ -12,6 +12,7 @@ mshv = ["hypervisor/mshv", "vfio-ioctls/mshv"] [dependencies] anyhow = { workspace = true } +base64 = { workspace = true } byteorder = { workspace = true } hypervisor = { path = "../hypervisor" } libc = { workspace = true } diff --git a/pci/src/vfio.rs b/pci/src/vfio.rs index bc74bc84a..2c668b40b 100644 --- a/pci/src/vfio.rs +++ b/pci/src/vfio.rs @@ -12,13 +12,24 @@ use std::sync::{Arc, Barrier, Mutex}; use std::{cmp, io, result}; use anyhow::anyhow; +use base64::Engine; +use base64::engine::general_purpose::STANDARD as BASE64_STANDARD; use byteorder::{ByteOrder, LittleEndian}; use hypervisor::HypervisorVmError; use libc::{_SC_PAGESIZE, sysconf}; use log::{debug, error, info}; use serde::{Deserialize, Serialize}; use thiserror::Error; -use vfio_bindings::bindings::vfio::*; +use vfio_bindings::bindings::vfio::{ + vfio_device_mig_state_VFIO_DEVICE_STATE_ERROR as VFIO_DEV_STATE_ERROR, + vfio_device_mig_state_VFIO_DEVICE_STATE_PRE_COPY as VFIO_DEV_STATE_PRE_COPY, + vfio_device_mig_state_VFIO_DEVICE_STATE_PRE_COPY_P2P as VFIO_DEV_STATE_PRE_COPY_P2P, + vfio_device_mig_state_VFIO_DEVICE_STATE_RESUMING as VFIO_DEV_STATE_RESUMING, + vfio_device_mig_state_VFIO_DEVICE_STATE_RUNNING as VFIO_DEV_STATE_RUNNING, + vfio_device_mig_state_VFIO_DEVICE_STATE_RUNNING_P2P as VFIO_DEV_STATE_RUNNING_P2P, + vfio_device_mig_state_VFIO_DEVICE_STATE_STOP as VFIO_DEV_STATE_STOP, + vfio_device_mig_state_VFIO_DEVICE_STATE_STOP_COPY as VFIO_DEV_STATE_STOP_COPY, *, +}; use vfio_ioctls::{VfioDevice, VfioIrq, VfioOps, VfioRegionInfoCap, VfioRegionSparseMmapArea}; use vm_allocator::page_size::{ align_page_size_down, align_page_size_up, get_page_size, is_4k_aligned, is_4k_multiple, @@ -46,6 +57,7 @@ use crate::{ }; pub(crate) const VFIO_COMMON_ID: &str = "vfio_common"; +pub(crate) const VFIO_MIGRATION_ID: &str = "vfio_migration"; #[derive(Debug, Error)] pub enum VfioPciError { @@ -364,6 +376,55 @@ pub enum VfioError { KernelVfio(#[source] vfio_ioctls::VfioError), #[error("VFIO user error")] VfioUser(#[source] vfio_user::Error), + #[error("VFIO device does not support migration")] + NoMigrationSupport, + #[error("VFIO device reported unknown migration state {0}")] + InvalidMigrationState(u32), +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum VfioMigrationState { + Error, + Stop, + Running, + StopCopy, + Resuming, + RunningP2P, + PreCopy, + PreCopyP2P, +} + +impl From for u32 { + fn from(state: VfioMigrationState) -> u32 { + match state { + VfioMigrationState::Error => VFIO_DEV_STATE_ERROR, + VfioMigrationState::Stop => VFIO_DEV_STATE_STOP, + VfioMigrationState::Running => VFIO_DEV_STATE_RUNNING, + VfioMigrationState::StopCopy => VFIO_DEV_STATE_STOP_COPY, + VfioMigrationState::Resuming => VFIO_DEV_STATE_RESUMING, + VfioMigrationState::RunningP2P => VFIO_DEV_STATE_RUNNING_P2P, + VfioMigrationState::PreCopy => VFIO_DEV_STATE_PRE_COPY, + VfioMigrationState::PreCopyP2P => VFIO_DEV_STATE_PRE_COPY_P2P, + } + } +} + +impl TryFrom for VfioMigrationState { + type Error = VfioError; + + fn try_from(value: u32) -> Result { + match value { + VFIO_DEV_STATE_ERROR => Ok(Self::Error), + VFIO_DEV_STATE_STOP => Ok(Self::Stop), + VFIO_DEV_STATE_RUNNING => Ok(Self::Running), + VFIO_DEV_STATE_STOP_COPY => Ok(Self::StopCopy), + VFIO_DEV_STATE_RESUMING => Ok(Self::Resuming), + VFIO_DEV_STATE_RUNNING_P2P => Ok(Self::RunningP2P), + VFIO_DEV_STATE_PRE_COPY => Ok(Self::PreCopy), + VFIO_DEV_STATE_PRE_COPY_P2P => Ok(Self::PreCopyP2P), + other => Err(VfioError::InvalidMigrationState(other)), + } + } } pub(crate) trait Vfio: Send + Sync { @@ -441,6 +502,14 @@ pub(crate) trait Vfio: Send + Sync { fn migration_flags(&self) -> Result, VfioError> { Ok(None) } + + fn set_migration_state(&self, _state: VfioMigrationState) -> Result<(), VfioError> { + Err(VfioError::NoMigrationSupport) + } + + fn read_migration_data(&self) -> Result, VfioError> { + Err(VfioError::NoMigrationSupport) + } } struct VfioDeviceWrapper { @@ -489,6 +558,18 @@ impl Vfio for VfioDeviceWrapper { .query_migration_support() .map_err(VfioError::KernelVfio) } + + fn set_migration_state(&self, state: VfioMigrationState) -> Result<(), VfioError> { + self.device + .set_migration_state(state.into()) + .map_err(VfioError::KernelVfio) + } + + fn read_migration_data(&self) -> Result, VfioError> { + self.device + .read_migration_data_to_end() + .map_err(VfioError::KernelVfio) + } } #[derive(Serialize, Deserialize)] @@ -498,6 +579,11 @@ struct VfioCommonState { msix_state: Option, } +#[derive(Serialize, Deserialize)] +struct VfioMigrationData { + blob: String, +} + pub(crate) struct ConfigPatch { mask: u32, patch: u32, @@ -513,7 +599,6 @@ pub(crate) struct VfioCommon { pub(crate) patches: HashMap, x_nv_gpudirect_clique: Option, x_exclude_mmap_bars: Vec, - #[allow(dead_code)] pub(crate) migration_flags: Option, } @@ -1359,6 +1444,27 @@ impl VfioCommon { // to the device region to update the MSI Enable bit. self.vfio_wrapper.write_config((reg + offset) as u32, data); + // The non BAR write path goes directly to the VFIO device and not the shadow, + // so the PciConfiguration shadow can get stale. Mirror the write into the + // shadow here since snapshot() serializes it. Without this the shadow keeps its + // device init values and a snapshot encodes PCI_COMMAND as zero. + // + // Use the raw write_* helpers rather than the PciConfiguration method + // self.configuration.write_config_register(), which would otherwise drain + // pending_bar_reprogram, owned by the BAR block below, and rerun MSI-X + // set_msg_ctl, already done by update_msix_capabilities above. + let byte_offset = reg_idx * PCI_CONFIG_REGISTER_SIZE + offset as usize; + match data.len() { + 1 => self.configuration.write_byte(byte_offset, data[0]), + 2 => self + .configuration + .write_word(byte_offset, u16::from(data[0]) | (u16::from(data[1]) << 8)), + 4 => self + .configuration + .write_reg(reg_idx, LittleEndian::read_u32(data)), + _ => {} + } + // Return pending BAR repgrogramming if MSE bit is set let mut ret_param = self.configuration.pending_bar_reprogram(); if !ret_param.is_empty() { @@ -1470,6 +1576,37 @@ impl VfioCommon { Ok(()) } + + pub(crate) fn transition_migration_state( + &self, + target: VfioMigrationState, + ) -> anyhow::Result<()> { + debug!("VFIO migration transition -> {target:?}"); + self.vfio_wrapper + .set_migration_state(target) + .map_err(|e| anyhow!("VFIO set_migration_state({target:?}) failed: {e}")) + } + + pub(crate) fn save_migration_data(&self) -> Result, MigratableError> { + self.transition_migration_state(VfioMigrationState::StopCopy) + .map_err(MigratableError::Snapshot)?; + + let data = self.vfio_wrapper.read_migration_data(); + + // We entered STOP_COPY successfully, so the STOP_COPY to STOP arc is + // valid whether or not the read succeeded. Return the device to STOP + // either way, since a failed transition into STOP_COPY would have + // returned above and a STOP from ERROR cannot help. + let stop = self + .transition_migration_state(VfioMigrationState::Stop) + .map_err(MigratableError::Snapshot); + + let data = data.map_err(|e| { + MigratableError::Snapshot(anyhow!("VFIO migration data read failed: {e}")) + })?; + stop?; + Ok(data) + } } impl Pausable for VfioCommon {} @@ -1495,6 +1632,17 @@ impl Snapshottable for VfioCommon { vfio_common_snapshot.add_snapshot(msix.bar.id(), msix.bar.snapshot()?); } + if self.migration_flags.is_some() { + let data = self.save_migration_data()?; + let mig = VfioMigrationData { + blob: BASE64_STANDARD.encode(&data), + }; + vfio_common_snapshot.add_snapshot( + VFIO_MIGRATION_ID.to_string(), + Snapshot::new_from_state(&mig)?, + ); + } + Ok(vfio_common_snapshot) } } @@ -2101,7 +2249,25 @@ iova 0x{:x}, size 0x{:x}: {}, ", } } -impl Pausable for VfioPciDevice {} +impl Pausable for VfioPciDevice { + fn pause(&mut self) -> result::Result<(), MigratableError> { + if self.common.migration_flags.is_some() { + self.common + .transition_migration_state(VfioMigrationState::Stop) + .map_err(MigratableError::Pause)?; + } + Ok(()) + } + + fn resume(&mut self) -> result::Result<(), MigratableError> { + if self.common.migration_flags.is_some() { + self.common + .transition_migration_state(VfioMigrationState::Running) + .map_err(MigratableError::Resume)?; + } + Ok(()) + } +} impl Snapshottable for VfioPciDevice { fn id(&self) -> String {