vmm, ch-remote: Allow VFIO fd substitution at restore

A VFIO device restored onto a different host has a device path and
iommufd that are meaningless there, and an FD backed device cannot
serialize a live descriptor into the snapshot at all. Restoring one
therefore needs fresh descriptors supplied with the request.

RestoreConfig gains vfio_fds, pairing each device id with a cdev FD,
and iommufd_fd for the backing iommufd. Both arrive over SCM_RIGHTS on
the restore request. vm_restore swaps each named device's stale path
or FD for the received one and installs the iommufd before the VM is
built, so the device comes up FD backed.

The request is rejected when a substituted device lacks the iommufd
backend, when an id is unknown or repeated, or when an FD backed device
names no replacement.

ch-remote gains the vfio_fds and iommufd_fd options and forwards the
descriptors through the SCM_RIGHTS pool.

Signed-off-by: Saravanan D <saravanand@crusoe.ai>

vmm/src/api/http/http_endpoint.rs

@@ -35,9 +35,11 @@
 //! [special HTTP library]: https://github.com/firecracker-microvm/micro-http
 
 use std::fs::File;
+use std::os::fd::IntoRawFd;
 use std::result;
 use std::sync::mpsc::Sender;
 
+use log::error;
 use micro_http::{Body, Method, Request, Response, StatusCode, Version};
 use vmm_sys_util::eventfd::EventFd;
 
@@ -120,7 +122,7 @@ mod fds_helper {
         use std::slice::from_ref;
 
         use super::{ConfigWithFDs, ConfigWithVariableFDs};
-        use crate::config::RestoredNetConfig;
+        use crate::config::{RestoredNetConfig, RestoredVfioConfig};
         use crate::vm_config::{DeviceConfig, NetConfig};
 
         impl ConfigWithFDs for NetConfig {
@@ -174,6 +176,26 @@ mod fds_helper {
                 self.num_fds
             }
         }
+
+        impl ConfigWithFDs for RestoredVfioConfig {
+            fn id(&self) -> Option<&str> {
+                Some(self.id.as_str())
+            }
+
+            fn fds_from_http_body(&self) -> Option<&[RawFd]> {
+                self.fd.as_ref().map(from_ref)
+            }
+
+            fn set_fds(&mut self, fds: Option<Vec<RawFd>>) {
+                self.fd = fds.and_then(|mut v| v.pop());
+            }
+        }
+
+        impl ConfigWithVariableFDs for RestoredVfioConfig {
+            fn expected_num_fds(&self) -> usize {
+                1
+            }
+        }
     }
 
     fn attach_fds_to_cfg_inner<T: ConfigWithFDs>(
@@ -561,10 +583,37 @@ impl PutHandler for VmRestore {
         if let Some(body) = body {
             let mut restore_cfg: RestoreConfig = serde_json::from_slice(body.raw())?;
 
+            let net_total: usize = restore_cfg
+                .net_fds
+                .iter()
+                .flatten()
+                .map(|c| c.num_fds)
+                .sum();
+            let vfio_total = restore_cfg.vfio_fds.as_ref().map_or(0, |c| c.len());
+            let iommufd_total = usize::from(restore_cfg.vfio_fds.is_some());
+            let expected = net_total + vfio_total + iommufd_total;
+            if files.len() != expected {
+                error!(
+                    "Expected {expected} FDs in VmRestore request, received {}",
+                    files.len()
+                );
+                return Err(HttpError::BadRequest);
+            }
+
+            // Split in the order net_fds, then vfio_fds, then the iommufd FD.
+            let mut files = files;
             if let Some(cfgs) = restore_cfg.net_fds.as_mut() {
+                let net_files: Vec<File> = files.drain(..net_total).collect();
                 let mut cfgs = cfgs.iter_mut().collect::<Vec<&mut _>>();
-                let cfgs = cfgs.as_mut_slice();
-                attach_fds_to_cfgs(files, cfgs)?;
+                attach_fds_to_cfgs(net_files, cfgs.as_mut_slice())?;
+            }
+            if let Some(cfgs) = restore_cfg.vfio_fds.as_mut() {
+                let vfio_files: Vec<File> = files.drain(..vfio_total).collect();
+                let mut cfgs = cfgs.iter_mut().collect::<Vec<&mut _>>();
+                attach_fds_to_cfgs(vfio_files, cfgs.as_mut_slice())?;
+            }
+            if restore_cfg.vfio_fds.is_some() {
+                restore_cfg.iommufd_fd = Some(files.remove(0).into_raw_fd());
             }
 
             self.send(api_notifier, api_sender, restore_cfg)