vm-migration, vmm: Send configuration in separate step

Prior to sending the memory the full state is not needed only the
configuration. This is sufficient to create the appropriate structures
in the guest and have the memory allocations ready for filling.

Update the protocol documentation to add a separate config step and move
the state to after the memory is transferred. As the VM is created in a
separate step to restoring it the requires a slightly different
constructor as well as saving the VM object for the subsequent commands.

Signed-off-by: Rob Bradford <robert.bradford@intel.com>

vm-migration/src/protocol.rs

@@ -10,14 +10,17 @@ use crate::MigratableError;
 // (The establishment is out of scope.)
 // 2: Source -> Dest : send "start command"
 // 3: Dest -> Source : sends "ok response" when read to accept state data
-// 4: Source -> Dest : sends "state command" followed by state data, length
+// 4: Source -> Dest : sends "config command" followed by config data, length
-//                     in command is length of state data
+//                     in command is length of config data
 // 5: Dest -> Source : sends "ok response" when ready to accept memory data
 // 6: Source -> Dest : send "memory command" followed by table of u64 pairs (GPA, size)
 //                     followed by the memory described in those pairs.
 //                     !! length is size of table i.e. 16 * number of ranges !!
 // 7: Dest -> Source : sends "ok response" when ready to accept more memory data
-// 8..(n-2): Repeat steps 6 and 7 until source has no more memory to send
+// 8..(n-4): Repeat steps 6 and 7 until source has no more memory to send
+// (n-3): Source -> Dest : sends "state command" followed by state data, length
+//                     in command is length of config data
+// (n-2): Dest -> Source : sends "ok response"
 // (n-1): Source -> Dest : send "complete command"
 // n: Dest -> Source: sends "ok response"
 
@@ -31,6 +34,7 @@ use std::io::{Read, Write};
 pub enum Command {
     Invalid,
     Start,
+    Config,
     State,
     Memory,
     Complete,
@@ -82,6 +86,10 @@ impl Request {
         Self::new(Command::State, length)
     }
 
+    pub fn config(length: u64) -> Self {
+        Self::new(Command::Config, length)
+    }
+
     pub fn memory(length: u64) -> Self {
         Self::new(Command::Memory, length)
     }

vmm/src/lib.rs

@@ -647,10 +647,53 @@ impl Vmm {
         }
     }
 
+    fn vm_receive_config<T>(
+        &mut self,
+        req: &Request,
+        socket: &mut T,
+    ) -> std::result::Result<Vm, MigratableError>
+    where
+        T: Read + Write,
+    {
+        // Read in config data
+        let mut data = Vec::with_capacity(req.length() as usize);
+        unsafe {
+            data.set_len(req.length() as usize);
+        }
+        socket
+            .read_exact(&mut data)
+            .map_err(MigratableError::MigrateSocket)?;
+        let config: VmConfig = serde_json::from_slice(&data).map_err(|e| {
+            MigratableError::MigrateReceive(anyhow!("Error deserialising config: {}", e))
+        })?;
+
+        let exit_evt = self.exit_evt.try_clone().map_err(|e| {
+            MigratableError::MigrateReceive(anyhow!("Error cloning exit EventFd: {}", e))
+        })?;
+        let reset_evt = self.reset_evt.try_clone().map_err(|e| {
+            MigratableError::MigrateReceive(anyhow!("Error cloning reset EventFd: {}", e))
+        })?;
+        let vm = Vm::new_from_migration(
+            Arc::new(Mutex::new(config)),
+            exit_evt,
+            reset_evt,
+            &self.seccomp_action,
+            self.hypervisor.clone(),
+        )
+        .map_err(|e| {
+            MigratableError::MigrateReceive(anyhow!("Error creating VM from snapshot: {:?}", e))
+        })?;
+
+        Response::ok().write_to(socket)?;
+
+        Ok(vm)
+    }
+
     fn vm_receive_state<T>(
         &mut self,
         req: &Request,
         socket: &mut T,
+        mut vm: Vm,
     ) -> std::result::Result<(), MigratableError>
     where
         T: Read + Write,
@@ -668,26 +711,6 @@ impl Vmm {
         })?;
 
         // Create VM
-        let vm_snapshot = get_vm_snapshot(&snapshot)?;
-        self.vm_config = Some(Arc::clone(&vm_snapshot.config));
-        let exit_evt = self.exit_evt.try_clone().map_err(|e| {
-            MigratableError::MigrateReceive(anyhow!("Error cloning exit EventFd: {}", e))
-        })?;
-        let reset_evt = self.reset_evt.try_clone().map_err(|e| {
-            MigratableError::MigrateReceive(anyhow!("Error cloning reset EventFd: {}", e))
-        })?;
-        let mut vm = Vm::new_from_snapshot(
-            &snapshot,
-            exit_evt,
-            reset_evt,
-            None,
-            false,
-            &self.seccomp_action,
-            self.hypervisor.clone(),
-        )
-        .map_err(|e| {
-            MigratableError::MigrateReceive(anyhow!("Error creating VM from snapshot: {:?}", e))
-        })?;
         vm.restore(snapshot).map_err(|e| {
             Response::error().write_to(socket).ok();
             e
@@ -703,6 +726,7 @@ impl Vmm {
         &mut self,
         req: &Request,
         socket: &mut T,
+        vm: &mut Vm,
     ) -> std::result::Result<(), MigratableError>
     where
         T: Read + Write,
@@ -711,14 +735,10 @@ impl Vmm {
         let table = MemoryRangeTable::read_from(socket, req.length())?;
 
         // And then read the memory itself
-        self.vm
+        vm.receive_memory_regions(&table, socket).map_err(|e| {
-            .as_mut()
+            Response::error().write_to(socket).ok();
-            .unwrap()
+            e
-            .receive_memory_regions(&table, socket)
+        })?;
-            .map_err(|e| {
-                Response::error().write_to(socket).ok();
-                e
-            })?;
         Response::ok().write_to(socket)?;
         Ok(())
     }
@@ -759,6 +779,7 @@ impl Vmm {
         };
 
         let mut started = false;
+        let mut vm: Option<Vm> = None;
 
         loop {
             let req = Request::read_from(&mut socket)?;
@@ -770,6 +791,16 @@ impl Vmm {
 
                     Response::ok().write_to(&mut socket)?;
                 }
+                Command::Config => {
+                    info!("Config Command Received");
+
+                    if !started {
+                        warn!("Migration not started yet");
+                        Response::error().write_to(&mut socket)?;
+                        continue;
+                    }
+                    vm = Some(self.vm_receive_config(&req, &mut socket)?);
+                }
                 Command::State => {
                     info!("State Command Received");
 
@@ -778,15 +809,12 @@ impl Vmm {
                         Response::error().write_to(&mut socket)?;
                         continue;
                     }
-
+                    if let Some(vm) = vm.take() {
-                    // TODO: Remove this when doing live migration
+                        self.vm_receive_state(&req, &mut socket, vm)?;
-                    if self.vm.is_some() {
+                    } else {
-                        warn!("State already sent");
+                        warn!("Configuration not sent yet");
                         Response::error().write_to(&mut socket)?;
-                        continue;
                     }
-
-                    self.vm_receive_state(&req, &mut socket)?;
                 }
                 Command::Memory => {
                     info!("Memory Command Received");
@@ -796,8 +824,12 @@ impl Vmm {
                         Response::error().write_to(&mut socket)?;
                         continue;
                     }
-
+                    if let Some(ref mut vm) = vm.as_mut() {
-                    self.vm_receive_memory(&req, &mut socket)?;
+                        self.vm_receive_memory(&req, &mut socket, vm)?;
+                    } else {
+                        warn!("Configuration not sent yet");
+                        Response::error().write_to(&mut socket)?;
+                    }
                 }
                 Command::Complete => {
                     info!("Complete Command Received");
@@ -853,6 +885,39 @@ impl Vmm {
                 )));
             }
 
+            // Send config
+            let config_data = serde_json::to_vec(&vm.get_config()).unwrap();
+            Request::config(config_data.len() as u64).write_to(&mut socket)?;
+            socket
+                .write_all(&config_data)
+                .map_err(MigratableError::MigrateSocket)?;
+            let res = Response::read_from(&mut socket)?;
+            if res.status() != Status::Ok {
+                warn!("Error during config migration");
+                Request::abandon().write_to(&mut socket)?;
+                Response::read_from(&mut socket).ok();
+                return Err(MigratableError::MigrateSend(anyhow!(
+                    "Error during config migration"
+                )));
+            }
+
+            // Send memory table
+            let table = vm.memory_range_table()?;
+            Request::memory(table.length())
+                .write_to(&mut socket)
+                .unwrap();
+            table.write_to(&mut socket)?;
+            // And then the memory itself
+            vm.send_memory_regions(&table, &mut socket)?;
+            if res.status() != Status::Ok {
+                warn!("Error during memory migration");
+                Request::abandon().write_to(&mut socket)?;
+                Response::read_from(&mut socket).ok();
+                return Err(MigratableError::MigrateSend(anyhow!(
+                    "Error during memory migration"
+                )));
+            }
+
             // Capture snapshot and send it
             let vm_snapshot = vm.snapshot()?;
             let snapshot_data = serde_json::to_vec(&vm_snapshot).unwrap();
@@ -870,24 +935,6 @@ impl Vmm {
                 )));
             }
 
-            // Send memory table
-            let table = vm.memory_range_table()?;
-            Request::memory(table.length())
-                .write_to(&mut socket)
-                .unwrap();
-            table.write_to(&mut socket)?;
-
-            // And then the memory itself
-            vm.send_memory_regions(&table, &mut socket)?;
-            if res.status() != Status::Ok {
-                warn!("Error during memory migration");
-                Request::abandon().write_to(&mut socket)?;
-                Response::read_from(&mut socket).ok();
-                return Err(MigratableError::MigrateSend(anyhow!(
-                    "Error during memory migration"
-                )));
-            }
-
             // Complete the migration
             Request::complete().write_to(&mut socket)?;
             let res = Response::read_from(&mut socket)?;

vmm/src/vm.rs

@@ -732,6 +732,40 @@ impl Vm {
         )
     }
 
+    pub fn new_from_migration(
+        config: Arc<Mutex<VmConfig>>,
+        exit_evt: EventFd,
+        reset_evt: EventFd,
+        seccomp_action: &SeccompAction,
+        hypervisor: Arc<dyn hypervisor::Hypervisor>,
+    ) -> Result<Self> {
+        #[cfg(target_arch = "x86_64")]
+        hypervisor.check_required_extensions().unwrap();
+        let vm = hypervisor.create_vm().unwrap();
+        #[cfg(target_arch = "x86_64")]
+        vm.enable_split_irq().unwrap();
+        let phys_bits = physical_bits(config.lock().unwrap().cpus.max_phys_bits);
+
+        let memory_manager = MemoryManager::new(
+            vm.clone(),
+            &config.lock().unwrap().memory.clone(),
+            false,
+            phys_bits,
+        )
+        .map_err(Error::MemoryManager)?;
+
+        Vm::new_from_memory_manager(
+            config,
+            memory_manager,
+            vm,
+            exit_evt,
+            reset_evt,
+            seccomp_action,
+            hypervisor,
+            None,
+        )
+    }
+
     fn load_initramfs(&mut self, guest_mem: &GuestMemoryMmap) -> Result<arch::InitramfsConfig> {
         let mut initramfs = self.initramfs.as_ref().unwrap();
         let size: usize = initramfs