diff --git a/vm-migration/src/protocol.rs b/vm-migration/src/protocol.rs index 550e3c76c..6c896fc02 100644 --- a/vm-migration/src/protocol.rs +++ b/vm-migration/src/protocol.rs @@ -10,14 +10,17 @@ use crate::MigratableError; // (The establishment is out of scope.) // 2: Source -> Dest : send "start command" // 3: Dest -> Source : sends "ok response" when read to accept state data -// 4: Source -> Dest : sends "state command" followed by state data, length -// in command is length of state data +// 4: Source -> Dest : sends "config command" followed by config data, length +// in command is length of config data // 5: Dest -> Source : sends "ok response" when ready to accept memory data // 6: Source -> Dest : send "memory command" followed by table of u64 pairs (GPA, size) // followed by the memory described in those pairs. // !! length is size of table i.e. 16 * number of ranges !! // 7: Dest -> Source : sends "ok response" when ready to accept more memory data -// 8..(n-2): Repeat steps 6 and 7 until source has no more memory to send +// 8..(n-4): Repeat steps 6 and 7 until source has no more memory to send +// (n-3): Source -> Dest : sends "state command" followed by state data, length +// in command is length of config data +// (n-2): Dest -> Source : sends "ok response" // (n-1): Source -> Dest : send "complete command" // n: Dest -> Source: sends "ok response" @@ -31,6 +34,7 @@ use std::io::{Read, Write}; pub enum Command { Invalid, Start, + Config, State, Memory, Complete, @@ -82,6 +86,10 @@ impl Request { Self::new(Command::State, length) } + pub fn config(length: u64) -> Self { + Self::new(Command::Config, length) + } + pub fn memory(length: u64) -> Self { Self::new(Command::Memory, length) } diff --git a/vmm/src/lib.rs b/vmm/src/lib.rs index 399c862ce..a70158c54 100644 --- a/vmm/src/lib.rs +++ b/vmm/src/lib.rs @@ -647,10 +647,53 @@ impl Vmm { } } + fn vm_receive_config( + &mut self, + req: &Request, + socket: &mut T, + ) -> std::result::Result + where + T: Read + Write, + { + // Read in config data + let mut data = Vec::with_capacity(req.length() as usize); + unsafe { + data.set_len(req.length() as usize); + } + socket + .read_exact(&mut data) + .map_err(MigratableError::MigrateSocket)?; + let config: VmConfig = serde_json::from_slice(&data).map_err(|e| { + MigratableError::MigrateReceive(anyhow!("Error deserialising config: {}", e)) + })?; + + let exit_evt = self.exit_evt.try_clone().map_err(|e| { + MigratableError::MigrateReceive(anyhow!("Error cloning exit EventFd: {}", e)) + })?; + let reset_evt = self.reset_evt.try_clone().map_err(|e| { + MigratableError::MigrateReceive(anyhow!("Error cloning reset EventFd: {}", e)) + })?; + let vm = Vm::new_from_migration( + Arc::new(Mutex::new(config)), + exit_evt, + reset_evt, + &self.seccomp_action, + self.hypervisor.clone(), + ) + .map_err(|e| { + MigratableError::MigrateReceive(anyhow!("Error creating VM from snapshot: {:?}", e)) + })?; + + Response::ok().write_to(socket)?; + + Ok(vm) + } + fn vm_receive_state( &mut self, req: &Request, socket: &mut T, + mut vm: Vm, ) -> std::result::Result<(), MigratableError> where T: Read + Write, @@ -668,26 +711,6 @@ impl Vmm { })?; // Create VM - let vm_snapshot = get_vm_snapshot(&snapshot)?; - self.vm_config = Some(Arc::clone(&vm_snapshot.config)); - let exit_evt = self.exit_evt.try_clone().map_err(|e| { - MigratableError::MigrateReceive(anyhow!("Error cloning exit EventFd: {}", e)) - })?; - let reset_evt = self.reset_evt.try_clone().map_err(|e| { - MigratableError::MigrateReceive(anyhow!("Error cloning reset EventFd: {}", e)) - })?; - let mut vm = Vm::new_from_snapshot( - &snapshot, - exit_evt, - reset_evt, - None, - false, - &self.seccomp_action, - self.hypervisor.clone(), - ) - .map_err(|e| { - MigratableError::MigrateReceive(anyhow!("Error creating VM from snapshot: {:?}", e)) - })?; vm.restore(snapshot).map_err(|e| { Response::error().write_to(socket).ok(); e @@ -703,6 +726,7 @@ impl Vmm { &mut self, req: &Request, socket: &mut T, + vm: &mut Vm, ) -> std::result::Result<(), MigratableError> where T: Read + Write, @@ -711,14 +735,10 @@ impl Vmm { let table = MemoryRangeTable::read_from(socket, req.length())?; // And then read the memory itself - self.vm - .as_mut() - .unwrap() - .receive_memory_regions(&table, socket) - .map_err(|e| { - Response::error().write_to(socket).ok(); - e - })?; + vm.receive_memory_regions(&table, socket).map_err(|e| { + Response::error().write_to(socket).ok(); + e + })?; Response::ok().write_to(socket)?; Ok(()) } @@ -759,6 +779,7 @@ impl Vmm { }; let mut started = false; + let mut vm: Option = None; loop { let req = Request::read_from(&mut socket)?; @@ -770,6 +791,16 @@ impl Vmm { Response::ok().write_to(&mut socket)?; } + Command::Config => { + info!("Config Command Received"); + + if !started { + warn!("Migration not started yet"); + Response::error().write_to(&mut socket)?; + continue; + } + vm = Some(self.vm_receive_config(&req, &mut socket)?); + } Command::State => { info!("State Command Received"); @@ -778,15 +809,12 @@ impl Vmm { Response::error().write_to(&mut socket)?; continue; } - - // TODO: Remove this when doing live migration - if self.vm.is_some() { - warn!("State already sent"); + if let Some(vm) = vm.take() { + self.vm_receive_state(&req, &mut socket, vm)?; + } else { + warn!("Configuration not sent yet"); Response::error().write_to(&mut socket)?; - continue; } - - self.vm_receive_state(&req, &mut socket)?; } Command::Memory => { info!("Memory Command Received"); @@ -796,8 +824,12 @@ impl Vmm { Response::error().write_to(&mut socket)?; continue; } - - self.vm_receive_memory(&req, &mut socket)?; + if let Some(ref mut vm) = vm.as_mut() { + self.vm_receive_memory(&req, &mut socket, vm)?; + } else { + warn!("Configuration not sent yet"); + Response::error().write_to(&mut socket)?; + } } Command::Complete => { info!("Complete Command Received"); @@ -853,6 +885,39 @@ impl Vmm { ))); } + // Send config + let config_data = serde_json::to_vec(&vm.get_config()).unwrap(); + Request::config(config_data.len() as u64).write_to(&mut socket)?; + socket + .write_all(&config_data) + .map_err(MigratableError::MigrateSocket)?; + let res = Response::read_from(&mut socket)?; + if res.status() != Status::Ok { + warn!("Error during config migration"); + Request::abandon().write_to(&mut socket)?; + Response::read_from(&mut socket).ok(); + return Err(MigratableError::MigrateSend(anyhow!( + "Error during config migration" + ))); + } + + // Send memory table + let table = vm.memory_range_table()?; + Request::memory(table.length()) + .write_to(&mut socket) + .unwrap(); + table.write_to(&mut socket)?; + // And then the memory itself + vm.send_memory_regions(&table, &mut socket)?; + if res.status() != Status::Ok { + warn!("Error during memory migration"); + Request::abandon().write_to(&mut socket)?; + Response::read_from(&mut socket).ok(); + return Err(MigratableError::MigrateSend(anyhow!( + "Error during memory migration" + ))); + } + // Capture snapshot and send it let vm_snapshot = vm.snapshot()?; let snapshot_data = serde_json::to_vec(&vm_snapshot).unwrap(); @@ -870,24 +935,6 @@ impl Vmm { ))); } - // Send memory table - let table = vm.memory_range_table()?; - Request::memory(table.length()) - .write_to(&mut socket) - .unwrap(); - table.write_to(&mut socket)?; - - // And then the memory itself - vm.send_memory_regions(&table, &mut socket)?; - if res.status() != Status::Ok { - warn!("Error during memory migration"); - Request::abandon().write_to(&mut socket)?; - Response::read_from(&mut socket).ok(); - return Err(MigratableError::MigrateSend(anyhow!( - "Error during memory migration" - ))); - } - // Complete the migration Request::complete().write_to(&mut socket)?; let res = Response::read_from(&mut socket)?; diff --git a/vmm/src/vm.rs b/vmm/src/vm.rs index 79d44b034..ee2bf7d21 100644 --- a/vmm/src/vm.rs +++ b/vmm/src/vm.rs @@ -732,6 +732,40 @@ impl Vm { ) } + pub fn new_from_migration( + config: Arc>, + exit_evt: EventFd, + reset_evt: EventFd, + seccomp_action: &SeccompAction, + hypervisor: Arc, + ) -> Result { + #[cfg(target_arch = "x86_64")] + hypervisor.check_required_extensions().unwrap(); + let vm = hypervisor.create_vm().unwrap(); + #[cfg(target_arch = "x86_64")] + vm.enable_split_irq().unwrap(); + let phys_bits = physical_bits(config.lock().unwrap().cpus.max_phys_bits); + + let memory_manager = MemoryManager::new( + vm.clone(), + &config.lock().unwrap().memory.clone(), + false, + phys_bits, + ) + .map_err(Error::MemoryManager)?; + + Vm::new_from_memory_manager( + config, + memory_manager, + vm, + exit_evt, + reset_evt, + seccomp_action, + hypervisor, + None, + ) + } + fn load_initramfs(&mut self, guest_mem: &GuestMemoryMmap) -> Result { let mut initramfs = self.initramfs.as_ref().unwrap(); let size: usize = initramfs