vm-virtio: vhost: Implement the Pausable trait for all vhost-user devices

Due to the amount of code currently duplicated across vhost-user devices,
the stats for this commit is on the large side but it's mostly more
duplicated code, unfortunately.

Migratable and Snapshotable placeholder implementations are provided as
well, making all vhost-user devices Migratable.

Signed-off-by: Samuel Ortiz <sameo@linux.intel.com>

vm-virtio/src/vhost_user/blk.rs

@@ -6,12 +6,17 @@ use libc::EFD_NONBLOCK;
 use std::cmp;
 use std::io::Write;
 use std::ptr::null;
+use std::result;
+use std::sync::atomic::{AtomicBool, Ordering};
 use std::sync::{Arc, RwLock};
 use std::thread;
 use std::vec::Vec;
 
 use crate::VirtioInterrupt;
 
+use super::Error as DeviceError;
+
+use vm_device::{Migratable, MigratableError, Pausable, Snapshotable};
 use vm_memory::GuestMemoryMmap;
 use vmm_sys_util::eventfd::EventFd;
 
@@ -38,12 +43,15 @@ impl VhostUserMasterReqHandler for SlaveReqHandler {}
 pub struct Blk {
     vhost_user_blk: Master,
     kill_evt: Option<EventFd>,
+    pause_evt: Option<EventFd>,
     avail_features: u64,
     acked_features: u64,
     config_space: Vec<u8>,
     queue_sizes: Vec<u16>,
     queue_evts: Option<Vec<EventFd>>,
     interrupt_cb: Option<Arc<VirtioInterrupt>>,
+    epoll_thread: Option<thread::JoinHandle<result::Result<(), DeviceError>>>,
+    paused: Arc<AtomicBool>,
 }
 
 impl Blk {
@@ -118,12 +126,15 @@ impl Blk {
         Ok(Blk {
             vhost_user_blk,
             kill_evt: None,
+            pause_evt: None,
             avail_features,
             acked_features,
             config_space,
             queue_sizes: vec![vu_cfg.queue_size; vu_cfg.num_queues],
             queue_evts: None,
             interrupt_cb: None,
+            epoll_thread: None,
+            paused: Arc::new(AtomicBool::new(false)),
         })
     }
 }
@@ -216,16 +227,22 @@ impl VirtioDevice for Blk {
         queues: Vec<Queue>,
         queue_evts: Vec<EventFd>,
     ) -> ActivateResult {
-        let (self_kill_evt, kill_evt) =
-            match EventFd::new(EFD_NONBLOCK).and_then(|e| Ok((e.try_clone()?, e))) {
-                Ok(v) => v,
-                Err(e) => {
-                    error!("failed creating kill EventFd pair: {}", e);
-                    return Err(ActivateError::BadActivate);
-                }
-            };
+        let (self_kill_evt, kill_evt) = EventFd::new(EFD_NONBLOCK)
+            .and_then(|e| Ok((e.try_clone()?, e)))
+            .map_err(|e| {
+                error!("failed creating kill EventFd pair: {}", e);
+                ActivateError::BadActivate
+            })?;
         self.kill_evt = Some(self_kill_evt);
 
+        let (self_pause_evt, pause_evt) = EventFd::new(EFD_NONBLOCK)
+            .and_then(|e| Ok((e.try_clone()?, e)))
+            .map_err(|e| {
+                error!("failed creating pause EventFd pair: {}", e);
+                ActivateError::BadActivate
+            })?;
+        self.pause_evt = Some(self_pause_evt);
+
         // Save the interrupt EventFD as we need to return it on reset
         // but clone it to pass into the thread.
         self.interrupt_cb = Some(interrupt_cb.clone());
@@ -253,24 +270,30 @@ impl VirtioDevice for Blk {
         let mut handler = VhostUserEpollHandler::<SlaveReqHandler>::new(VhostUserEpollConfig {
             interrupt_cb,
             kill_evt,
+            pause_evt,
             vu_interrupt_list,
             slave_req_handler: None,
         });
 
-        let handler_result = thread::Builder::new()
+        let paused = self.paused.clone();
+        thread::Builder::new()
             .name("vhost_user_blk".to_string())
-            .spawn(move || {
-                if let Err(e) = handler.run() {
-                    error!("net worker thread exited with error {:?}!", e);
-                }
-            });
-        if let Err(e) = handler_result {
-            error!("vhost-user blk thread create failed with error {:?}", e);
-        }
+            .spawn(move || handler.run(paused))
+            .map(|thread| self.epoll_thread = Some(thread))
+            .map_err(|e| {
+                error!("failed to clone virtio epoll thread: {}", e);
+                ActivateError::BadActivate
+            })?;
+
         Ok(())
     }
 
     fn reset(&mut self) -> Option<(Arc<VirtioInterrupt>, Vec<EventFd>)> {
+        // We first must resume the virtio thread if it was paused.
+        if self.pause_evt.take().is_some() {
+            self.resume().ok()?;
+        }
+
         if let Err(e) = reset_vhost_user(&mut self.vhost_user_blk, self.queue_sizes.len()) {
             error!("Failed to reset vhost-user daemon: {:?}", e);
             return None;
@@ -288,3 +311,7 @@ impl VirtioDevice for Blk {
         ))
     }
 }
+
+virtio_pausable!(Blk);
+impl Snapshotable for Blk {}
+impl Migratable for Blk {}