diff --git a/Cargo.lock b/Cargo.lock index 8e89b3ed1..f11742711 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -667,13 +667,18 @@ dependencies = [ name = "pci" version = "0.1.0" dependencies = [ + "anyhow", "byteorder", "devices", "libc", "log 0.4.8", + "serde", + "serde_derive", + "serde_json", "vm-allocator", "vm-device", "vm-memory", + "vm-migration", ] [[package]] diff --git a/pci/Cargo.toml b/pci/Cargo.toml index ed4d6c2ac..dde9c14f1 100644 --- a/pci/Cargo.toml +++ b/pci/Cargo.toml @@ -5,10 +5,15 @@ authors = ["Samuel Ortiz "] edition = "2018" [dependencies] -vm-allocator = { path = "../vm-allocator" } +anyhow = "1.0" byteorder = "1.3.4" devices = { path = "../devices" } libc = "0.2.69" log = "0.4.8" +serde = {version = ">=1.0.27", features = ["rc"] } +serde_derive = ">=1.0.27" +serde_json = ">=1.0.9" +vm-allocator = { path = "../vm-allocator" } vm-device = { path = "../vm-device" } vm-memory = "0.2.0" +vm-migration = { path = "../vm-migration" } diff --git a/pci/src/lib.rs b/pci/src/lib.rs index 5e7ad586f..2658fea18 100644 --- a/pci/src/lib.rs +++ b/pci/src/lib.rs @@ -6,6 +6,10 @@ #[macro_use] extern crate log; extern crate devices; +extern crate serde; +#[macro_use] +extern crate serde_derive; +extern crate serde_json; extern crate vm_memory; mod bus; diff --git a/pci/src/msix.rs b/pci/src/msix.rs index a83218dca..f37a7c138 100644 --- a/pci/src/msix.rs +++ b/pci/src/msix.rs @@ -6,14 +6,17 @@ extern crate byteorder; extern crate vm_memory; -use std::sync::Arc; - use crate::{PciCapability, PciCapabilityID}; +use anyhow::anyhow; use byteorder::{ByteOrder, LittleEndian}; +use std::io; +use std::result; +use std::sync::Arc; use vm_device::interrupt::{ InterruptIndex, InterruptSourceConfig, InterruptSourceGroup, MsiIrqSourceConfig, }; use vm_memory::ByteValued; +use vm_migration::{MigratableError, Pausable, Snapshot, SnapshotDataSection, Snapshottable}; const MAX_MSIX_VECTORS_PER_DEVICE: u16 = 2048; const MSIX_TABLE_ENTRIES_MODULO: u64 = 16; @@ -25,7 +28,15 @@ const FUNCTION_MASK_MASK: u16 = (1 << FUNCTION_MASK_BIT) as u16; const MSIX_ENABLE_MASK: u16 = (1 << MSIX_ENABLE_BIT) as u16; pub const MSIX_TABLE_ENTRY_SIZE: usize = 16; -#[derive(Debug, Clone)] +#[derive(Debug)] +enum Error { + /// Failed enabling the interrupt route. + EnableInterruptRoute(io::Error), + /// Failed updating the interrupt route. + UpdateInterruptRoute(io::Error), +} + +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct MsixTableEntry { pub msg_addr_lo: u32, pub msg_addr_hi: u32, @@ -50,6 +61,14 @@ impl Default for MsixTableEntry { } } +#[derive(Serialize, Deserialize)] +struct MsixConfigState { + table_entries: Vec, + pba_entries: Vec, + masked: bool, + enabled: bool, +} + pub struct MsixConfig { pub table_entries: Vec, pub pba_entries: Vec, @@ -80,6 +99,46 @@ impl MsixConfig { } } + fn state(&self) -> MsixConfigState { + MsixConfigState { + table_entries: self.table_entries.clone(), + pba_entries: self.pba_entries.clone(), + masked: self.masked, + enabled: self.enabled, + } + } + + fn set_state(&mut self, state: &MsixConfigState) -> result::Result<(), Error> { + self.table_entries = state.table_entries.clone(); + self.pba_entries = state.pba_entries.clone(); + self.masked = state.masked; + self.enabled = state.enabled; + + if self.enabled && !self.masked { + for (idx, table_entry) in self.table_entries.iter().enumerate() { + if table_entry.masked() { + continue; + } + + let config = MsiIrqSourceConfig { + high_addr: table_entry.msg_addr_hi, + low_addr: table_entry.msg_addr_lo, + data: table_entry.msg_data, + }; + + self.interrupt_source_group + .update(idx as InterruptIndex, InterruptSourceConfig::MsiIrq(config)) + .map_err(Error::UpdateInterruptRoute)?; + + self.interrupt_source_group + .enable() + .map_err(Error::EnableInterruptRoute)?; + } + } + + Ok(()) + } + pub fn masked(&self) -> bool { self.masked } @@ -365,6 +424,52 @@ impl MsixConfig { } } +impl Pausable for MsixConfig {} + +impl Snapshottable for MsixConfig { + fn id(&self) -> String { + String::from("msix_config") + } + + fn snapshot(&self) -> std::result::Result { + let snapshot = + serde_json::to_vec(&self.state()).map_err(|e| MigratableError::Snapshot(e.into()))?; + + let mut msix_snapshot = Snapshot::new(self.id().as_str()); + msix_snapshot.add_data_section(SnapshotDataSection { + id: format!("{}-section", self.id()), + snapshot, + }); + + Ok(msix_snapshot) + } + + fn restore(&mut self, snapshot: Snapshot) -> std::result::Result<(), MigratableError> { + if let Some(msix_section) = snapshot + .snapshot_data + .get(&format!("{}-section", self.id())) + { + let msix_state = match serde_json::from_slice(&msix_section.snapshot) { + Ok(state) => state, + Err(error) => { + return Err(MigratableError::Restore(anyhow!( + "Could not deserialize MSI-X {}", + error + ))) + } + }; + + return self.set_state(&msix_state).map_err(|e| { + MigratableError::Restore(anyhow!("Could not restore MSI-X state {:?}", e)) + }); + } + + Err(MigratableError::Restore(anyhow!( + "Could not find MSI-X snapshot section" + ))) + } +} + #[allow(dead_code)] #[repr(packed)] #[derive(Clone, Copy, Default)]