diff --git a/vmm/src/lib.rs b/vmm/src/lib.rs index 047e204c9..c1004395e 100644 --- a/vmm/src/lib.rs +++ b/vmm/src/lib.rs @@ -56,7 +56,7 @@ use crate::config::{MemoryRestoreMode, RestoreConfig, add_to_config}; use crate::coredump::GuestDebuggable; use crate::device_manager::DeviceManager; use crate::landlock::Landlock; -use crate::memory_manager::MemoryManager; +use crate::memory_manager::{MemoryManager, MemoryRangePolicy}; #[cfg(all(feature = "kvm", target_arch = "x86_64"))] use crate::migration::get_vm_snapshot; use crate::migration::transport::{ @@ -1112,7 +1112,10 @@ impl Vmm { )) })?; let mm = config_data.memory_manager.clone(); - let saved_regions = mm.lock().unwrap().memory_range_table(false)?; + let saved_regions = mm + .lock() + .unwrap() + .memory_range_table(MemoryRangePolicy::Full)?; mm.lock() .unwrap() .start_postcopy_serving( @@ -1372,7 +1375,7 @@ impl Vmm { let iteration_begin = Instant::now(); let iteration_table = if ctx.iteration == 0 { - vm.memory_range_table()? + vm.memory_range_table(MemoryRangePolicy::Sparse)? } else { // TODO do this in a thread #7816 vm.dirty_log()? diff --git a/vmm/src/memory_manager.rs b/vmm/src/memory_manager.rs index d04fcbb35..b56e16e44 100644 --- a/vmm/src/memory_manager.rs +++ b/vmm/src/memory_manager.rs @@ -601,6 +601,18 @@ impl BusDevice for MemoryManager { } } +/// Memory range policy. +#[derive(Clone, Copy)] +pub enum MemoryRangePolicy { + /// Send every region in full. + Full, + /// Send every region in full, except those already persisted through a + /// shared, hardlinked backing file. + SkipPersisted, + /// Send only the populated ranges of each region, skipping holes. + Sparse, +} + impl MemoryManager { /// Creates all memory regions based on the available RAM ranges defined /// by `ram_regions`, and based on the description of the memory zones. @@ -2685,7 +2697,7 @@ impl MemoryManager { pub fn memory_range_table( &self, - snapshot: bool, + mode: MemoryRangePolicy, ) -> result::Result { let mut table = MemoryRangeTable::default(); @@ -2695,33 +2707,68 @@ impl MemoryManager { } for region in memory_zone.regions() { - if snapshot - && let Some(file_offset) = region.file_offset() - && (region.flags() & libc::MAP_SHARED == libc::MAP_SHARED) - && Self::is_hardlink(file_offset.file()) - { - // In this very specific case, we know the memory - // region is backed by a file on the host filesystem - // that can be accessed by the user, and additionally - // the mapping is shared, which means that modifications - // to the content are written to the actual file. - // When meeting these conditions, we can skip the - // copy of the memory content for this specific region, - // as we can assume the user will have it saved through - // the backing file already. - continue; + let gpa = region.start_addr().raw_value(); + let len = region.len(); + + let shared_file_offset = region + .file_offset() + .filter(|_| region.flags() & libc::MAP_SHARED == libc::MAP_SHARED); + + match mode { + MemoryRangePolicy::Full => {} + MemoryRangePolicy::SkipPersisted => { + // Content already persisted through the backing file. + if let Some(file_offset) = shared_file_offset + && Self::is_hardlink(file_offset.file()) + { + continue; + } + } + MemoryRangePolicy::Sparse => { + if let Some(file_offset) = shared_file_offset + && let Some(ranges) = Self::sparse_extents(file_offset, gpa, len) + { + for range in ranges { + table.push(range); + } + continue; + } + } } - table.push(MemoryRange { - gpa: region.start_addr().raw_value(), - length: region.len(), - }); + table.push(MemoryRange { gpa, length: len }); } } Ok(table) } + /// Return the populated extents of a memory range, or `None` if the + /// caller should fall back onto sending the entire region. + fn sparse_extents(file_offset: &FileOffset, gpa: u64, len: u64) -> Option> { + let offset = file_offset.start(); + let end = offset + len; + let mut cursor = offset; + let mut ranges = Vec::new(); + loop { + match next_data_extent(file_offset.file().as_fd(), cursor, end) { + Ok(Some((data_offset, ext_len))) => { + let range_gpa = data_offset + .checked_sub(offset) + .and_then(|delta| gpa.checked_add(delta))?; + ranges.push(MemoryRange { + gpa: range_gpa, + length: ext_len, + }); + cursor = data_offset + ext_len; + } + Ok(None) => break, + Err(_) => return None, + } + } + Some(ranges) + } + pub fn snapshot_data(&self) -> MemoryManagerSnapshotData { MemoryManagerSnapshotData { memory_ranges: self.snapshot_memory_ranges.clone(), @@ -2793,7 +2840,7 @@ impl MemoryManager { dump_state: &DumpState, ) -> result::Result<(), GuestDebuggableError> { let snapshot_memory_ranges = self - .memory_range_table(false) + .memory_range_table(MemoryRangePolicy::Full) .map_err(|e| GuestDebuggableError::Coredump(e.into()))?; if snapshot_memory_ranges.is_empty() { @@ -3200,7 +3247,7 @@ impl Snapshottable for MemoryManager { } fn snapshot(&mut self) -> result::Result { - let memory_ranges = self.memory_range_table(true)?; + let memory_ranges = self.memory_range_table(MemoryRangePolicy::SkipPersisted)?; // Store locally this list of ranges as it will be used through the // Transportable::send() implementation. The point is to avoid the diff --git a/vmm/src/vm.rs b/vmm/src/vm.rs index 3bcb5a34d..281066308 100644 --- a/vmm/src/vm.rs +++ b/vmm/src/vm.rs @@ -105,7 +105,7 @@ use crate::landlock::LandlockError; #[cfg(feature = "tdx")] use crate::memory_manager; use crate::memory_manager::{ - Error as MemoryManagerError, MemoryManager, MemoryManagerSnapshotData, + Error as MemoryManagerError, MemoryManager, MemoryManagerSnapshotData, MemoryRangePolicy, }; #[cfg(all(target_arch = "x86_64", feature = "guest_debug"))] use crate::migration::url_to_file; @@ -3029,11 +3029,11 @@ impl Vm { Ok(()) } - pub fn memory_range_table(&self) -> result::Result { - self.memory_manager - .lock() - .unwrap() - .memory_range_table(false) + pub fn memory_range_table( + &self, + mode: MemoryRangePolicy, + ) -> result::Result { + self.memory_manager.lock().unwrap().memory_range_table(mode) } pub fn guest_memory(&self) -> GuestMemoryAtomic {