From 39768704f3227afad4a77f4dd36ce526d5bf686d Mon Sep 17 00:00:00 2001 From: Philipp Schuster Date: Tue, 10 Mar 2026 09:20:03 +0100 Subject: [PATCH] vmm: migration: add iteration metrics and downtime estimation Add infrastructure to collect metrics during precopy memory migration iterations. For each iteration we now track transferred bytes, duration, bandwidth, and estimate the expected downtime based on the remaining memory of the current iteration and measured bandwidth. These metrics are logged and used to decide when to stop the precopy phase. This also introduces basic termination conditions such as: - maximum number of iterations - reaching a target downtime - maximum migration duration This is the fundament for an upcoming API call to publicly export statistics about an ongoing live migration. The changes are, however, self-contained and helpful by themselves. The new log now looks somewhat as in the following, providing lots of helpful insights (especially the bandwidth and estimated downtime are helpful). The metrics were measured with CHV build with `--release`, a VM under heavy load (lots of memory writes), same-host TCP migration and prefault=on: ``` cloud-hypervisor: 12.702682s: DEBUG:vmm/src/lib.rs:1313 -- Precopy: iter=0 total=6144MiB curr=6144MiB bw=1986.83MiB/s transfer=3.09s overhead=0ms est_downtime=0ms elapsed=3.11s avg_bw=1975.41MiB/s cloud-hypervisor: 15.728419s: DEBUG:vmm/src/lib.rs:1313 -- Precopy: iter=1 total=11562MiB curr=5418MiB bw=1824.44MiB/s transfer=2.97s overhead=56ms est_downtime=2726ms elapsed=6.14s avg_bw=1884.21MiB/s cloud-hypervisor: 18.710428s: DEBUG:vmm/src/lib.rs:1313 -- Precopy: iter=2 total=16980MiB curr=5418MiB bw=1854.25MiB/s transfer=2.92s overhead=59ms est_downtime=2969ms elapsed=9.12s avg_bw=1862.17MiB/s cloud-hypervisor: 21.783699s: DEBUG:vmm/src/lib.rs:1313 -- Precopy: iter=3 total=22407MiB curr=5428MiB bw=1799.43MiB/s transfer=3.02s overhead=56ms est_downtime=2926ms elapsed=12.19s avg_bw=1837.92MiB/s cloud-hypervisor: 25.785696s: DEBUG:vmm/src/lib.rs:1313 -- Precopy: iter=4 total=27825MiB curr=5418MiB bw=1375.53MiB/s transfer=3.94s overhead=62ms est_downtime=3010ms elapsed=16.19s avg_bw=1718.26MiB/s cloud-hypervisor: 29.000349s: DEBUG:vmm/src/lib.rs:1313 -- Precopy: iter=5 total=33243MiB curr=5418MiB bw=1727.60MiB/s transfer=3.14s overhead=78ms est_downtime=3938ms elapsed=19.41s avg_bw=1712.82MiB/s cloud-hypervisor: 32.215805s: DEBUG:vmm/src/lib.rs:1313 -- Precopy: iter=6 total=38671MiB curr=5429MiB bw=1724.03MiB/s transfer=3.15s overhead=66ms est_downtime=3142ms elapsed=22.62s avg_bw=1709.33MiB/s cloud-hypervisor: 32.275215s: DEBUG:vmm/src/lib.rs:1286 -- Precopy converged: iter=7 total=38671MiB curr=5418MiB bw=1720.46MiB/s transfer=3.15s overhead=66ms est_downtime=3142ms elapsed=22.68s avg_bw=1704.85MiB/s ... cloud-hypervisor: 33.411682s: INFO:vmm/src/lib.rs:1365 -- Precopy complete: iter=8 total=44339MiB curr=5668MiB bw=1799.98MiB/s transfer=3.15s overhead=66ms est_downtime=3142ms elapsed=23.82s avg_bw=1861.45MiB/s ``` # Outlook We can add user-configurable downtimes and migration downtimes next. These changes are inspired by [0] but differ significantly in details. [0] https://github.com/cloud-hypervisor/cloud-hypervisor/pull/7033 Signed-off-by: Philipp Schuster On-behalf-of: SAP philipp.schuster@sap.com --- vmm/src/lib.rs | 62 +++++++++++++++++++++++++++++++++++--------------- 1 file changed, 44 insertions(+), 18 deletions(-) diff --git a/vmm/src/lib.rs b/vmm/src/lib.rs index 418638678..007db270e 100644 --- a/vmm/src/lib.rs +++ b/vmm/src/lib.rs @@ -39,7 +39,10 @@ use tracer::trace_scoped; use vm_memory::bitmap::{AtomicBitmap, BitmapSlice}; use vm_memory::{ReadVolatile, VolatileMemoryError, VolatileSlice, WriteVolatile}; use vm_migration::protocol::*; -use vm_migration::{Migratable, MigratableError, Pausable, Snapshot, Snapshottable, Transportable}; +use vm_migration::{ + MemoryMigrationContext, Migratable, MigratableError, Pausable, Snapshot, Snapshottable, + Transportable, +}; use vmm_sys_util::eventfd::EventFd; use vmm_sys_util::signal::unblock_signal; use vmm_sys_util::sock_ctrl_msg::ScmSocket; @@ -1252,27 +1255,37 @@ impl Vmm { fn do_memory_iterations( vm: &mut Vm, socket: &mut SocketStream, - iteration_counter: &mut u64, - is_converged: impl Fn(u64) -> bool, + ctx: &mut MemoryMigrationContext, + is_converged: impl Fn(&MemoryMigrationContext) -> bool, ) -> result::Result { loop { - let iteration_table = if *iteration_counter == 0 { + let iteration_begin = Instant::now(); + + let iteration_table = if ctx.iteration == 0 { vm.memory_range_table()? } else { // TODO do this in a thread #7816 vm.dirty_log()? }; - if is_converged(*iteration_counter) { - debug!("Precopy converged: iter={iteration_counter}"); + ctx.update_metrics_before_transfer(iteration_begin, &iteration_table); + if is_converged(ctx) { + debug!("Precopy converged: {ctx}"); break Ok(iteration_table); } // Send the current dirty pages + let transfer_begin = Instant::now(); Self::vm_send_dirty_pages(vm, socket, &iteration_table)?; + let transfer_duration = transfer_begin.elapsed(); + ctx.update_metrics_after_transfer(transfer_begin, transfer_duration); - // Prepare next iteration. - *iteration_counter += 1; + // Log progress of the current iteration + debug!("Precopy: {ctx}"); + + // Increment iteration last: This way we ensure that the logging + // above matches the actual iteration. + ctx.iteration += 1; } } @@ -1286,22 +1299,35 @@ impl Vmm { vm: &mut Vm, socket: &mut SocketStream, ) -> result::Result<(), MigratableError> { - const MAX_ITERATIONS: u64 = 5; + const MAX_ITERATIONS: usize = 5; - let mut iteration_counter = 0; - let is_converged = |iteration_counter: u64| iteration_counter >= MAX_ITERATIONS; + let mut ctx = MemoryMigrationContext::new(); + let is_converged = |ctx: &MemoryMigrationContext| { + // TODO: Add check for configurable downtime and max migration time #7111 + ctx.iteration >= MAX_ITERATIONS || ctx.current_iteration_total_bytes == 0 + }; vm.start_dirty_log()?; - let remaining = - Self::do_memory_iterations(vm, socket, &mut iteration_counter, is_converged)?; + let remaining = Self::do_memory_iterations(vm, socket, &mut ctx, is_converged)?; vm.pause()?; - // Send last batch of dirty pages - let mut final_table = vm.dirty_log()?; - final_table.extend(remaining); - Vmm::vm_send_dirty_pages(vm, socket, &final_table)?; + // Send last batch of dirty pages: final iteration + { + let iteration_begin = Instant::now(); - info!("Memory migration complete"); + let mut final_table = vm.dirty_log()?; + final_table.extend(remaining); + + ctx.update_metrics_before_transfer(iteration_begin, &final_table); + let transfer_begin = Instant::now(); + Vmm::vm_send_dirty_pages(vm, socket, &final_table)?; + let transfer_duration = transfer_begin.elapsed(); + ctx.update_metrics_after_transfer(transfer_begin, transfer_duration); + ctx.iteration += 1; + } + ctx.finalize(); + + info!("Precopy complete: {ctx}"); Ok(()) }