vm-virtio, virtio-devices: Split device implementation from virt queues

Split the generic virtio code (queues and device type) from the
VirtioDevice trait, transport and device implementations.

This also simplifies the feature handling in vhost_user_backend as the
vm-virtio crate is no longer has any features.

Signed-off-by: Rob Bradford <robert.bradford@intel.com>

virtio-devices/src/vhost_user/blk.rs

@@ -0,0 +1,335 @@
+// Copyright 2019 Intel Corporation. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+use super::super::{ActivateError, ActivateResult, Queue, VirtioDevice, VirtioDeviceType};
+use super::handler::*;
+use super::vu_common_ctrl::*;
+use super::Error as DeviceError;
+use super::{Error, Result};
+use crate::block::VirtioBlockConfig;
+use crate::VirtioInterrupt;
+use libc::EFD_NONBLOCK;
+use std::cmp;
+use std::io::Write;
+use std::mem;
+use std::os::unix::io::AsRawFd;
+use std::result;
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::Arc;
+use std::thread;
+use std::vec::Vec;
+use vhost_rs::vhost_user::message::VhostUserConfigFlags;
+use vhost_rs::vhost_user::message::VHOST_USER_CONFIG_OFFSET;
+use vhost_rs::vhost_user::message::{VhostUserProtocolFeatures, VhostUserVirtioFeatures};
+use vhost_rs::vhost_user::{Master, VhostUserMaster, VhostUserMasterReqHandler};
+use vhost_rs::VhostBackend;
+use virtio_bindings::bindings::virtio_blk::*;
+use virtio_bindings::bindings::virtio_ring::VIRTIO_RING_F_EVENT_IDX;
+use vm_memory::{ByteValued, GuestAddressSpace, GuestMemoryAtomic, GuestMemoryMmap};
+use vm_migration::{Migratable, MigratableError, Pausable, Snapshottable, Transportable};
+use vmm_sys_util::eventfd::EventFd;
+
+struct SlaveReqHandler {}
+impl VhostUserMasterReqHandler for SlaveReqHandler {}
+
+pub struct Blk {
+    id: String,
+    vhost_user_blk: Master,
+    kill_evt: Option<EventFd>,
+    pause_evt: Option<EventFd>,
+    avail_features: u64,
+    acked_features: u64,
+    config: VirtioBlockConfig,
+    queue_sizes: Vec<u16>,
+    queue_evts: Option<Vec<EventFd>>,
+    interrupt_cb: Option<Arc<dyn VirtioInterrupt>>,
+    epoll_threads: Option<Vec<thread::JoinHandle<result::Result<(), DeviceError>>>>,
+    paused: Arc<AtomicBool>,
+}
+
+impl Blk {
+    /// Create a new vhost-user-blk device
+    pub fn new(id: String, vu_cfg: VhostUserConfig) -> Result<Blk> {
+        let mut vhost_user_blk = Master::connect(&vu_cfg.socket, vu_cfg.num_queues as u64)
+            .map_err(Error::VhostUserCreateMaster)?;
+
+        // Filling device and vring features VMM supports.
+        let mut avail_features = 1 << VIRTIO_BLK_F_SEG_MAX
+            | 1 << VIRTIO_BLK_F_RO
+            | 1 << VIRTIO_BLK_F_BLK_SIZE
+            | 1 << VIRTIO_BLK_F_FLUSH
+            | 1 << VIRTIO_BLK_F_TOPOLOGY
+            | 1 << VIRTIO_RING_F_EVENT_IDX
+            | 1 << VIRTIO_BLK_F_CONFIG_WCE
+            | 1 << VIRTIO_F_VERSION_1
+            | VhostUserVirtioFeatures::PROTOCOL_FEATURES.bits();
+
+        if vu_cfg.num_queues > 1 {
+            avail_features |= 1 << VIRTIO_BLK_F_MQ;
+        }
+
+        // Set vhost-user owner.
+        vhost_user_blk
+            .set_owner()
+            .map_err(Error::VhostUserSetOwner)?;
+
+        // Get features from backend, do negotiation to get a feature collection which
+        // both VMM and backend support.
+        let backend_features = vhost_user_blk
+            .get_features()
+            .map_err(Error::VhostUserGetFeatures)?;
+        avail_features &= backend_features;
+        // Set features back is required by the vhost crate mechanism, since the
+        // later vhost call will check if features is filled in master before execution.
+        vhost_user_blk
+            .set_features(avail_features)
+            .map_err(Error::VhostUserSetFeatures)?;
+
+        // Identify if protocol features are supported by the slave.
+        let mut acked_features = 0;
+        if avail_features & VhostUserVirtioFeatures::PROTOCOL_FEATURES.bits() != 0 {
+            acked_features |= VhostUserVirtioFeatures::PROTOCOL_FEATURES.bits();
+
+            let mut protocol_features = vhost_user_blk
+                .get_protocol_features()
+                .map_err(Error::VhostUserGetProtocolFeatures)?;
+            protocol_features |= VhostUserProtocolFeatures::MQ;
+            protocol_features &= !VhostUserProtocolFeatures::INFLIGHT_SHMFD;
+            vhost_user_blk
+                .set_protocol_features(protocol_features)
+                .map_err(Error::VhostUserSetProtocolFeatures)?;
+        }
+        // Get the max queues number from backend, and the queue number set
+        // should be less than this max queue number.
+        let max_queues_num = vhost_user_blk
+            .get_queue_num()
+            .map_err(Error::VhostUserGetQueueMaxNum)?;
+
+        if vu_cfg.num_queues > max_queues_num as usize {
+            error!("vhost-user-blk has queue number: {} larger than the max queue number: {} backend allowed\n",
+                vu_cfg.num_queues, max_queues_num);
+            return Err(Error::BadQueueNum);
+        }
+        let config_len = mem::size_of::<VirtioBlockConfig>();
+        let config_space: Vec<u8> = vec![0u8; config_len as usize];
+        let (_, config_space) = vhost_user_blk
+            .get_config(
+                VHOST_USER_CONFIG_OFFSET,
+                config_len as u32,
+                VhostUserConfigFlags::WRITABLE,
+                config_space.as_slice(),
+            )
+            .unwrap();
+        let mut config = VirtioBlockConfig::default();
+        if let Some(backend_config) = VirtioBlockConfig::from_slice(config_space.as_slice()) {
+            config = *backend_config;
+            // Only set num_queues value(u16).
+            config.num_queues = vu_cfg.num_queues as u16;
+        }
+
+        // Send set_vring_base here, since it could tell backends, like SPDK,
+        // how many virt queues to be handled, which backend required to know
+        // at early stage.
+        for i in 0..vu_cfg.num_queues {
+            vhost_user_blk
+                .set_vring_base(i, 0)
+                .map_err(Error::VhostUserSetVringBase)?;
+        }
+
+        Ok(Blk {
+            id,
+            vhost_user_blk,
+            kill_evt: None,
+            pause_evt: None,
+            avail_features,
+            acked_features,
+            config,
+            queue_sizes: vec![vu_cfg.queue_size; vu_cfg.num_queues],
+            queue_evts: None,
+            interrupt_cb: None,
+            epoll_threads: None,
+            paused: Arc::new(AtomicBool::new(false)),
+        })
+    }
+}
+
+impl Drop for Blk {
+    fn drop(&mut self) {
+        if let Some(kill_evt) = self.kill_evt.take() {
+            if let Err(e) = kill_evt.write(1) {
+                error!("failed to kill vhost-user-blk: {:?}", e);
+            }
+        }
+    }
+}
+
+impl VirtioDevice for Blk {
+    fn device_type(&self) -> u32 {
+        VirtioDeviceType::TYPE_BLOCK as u32
+    }
+
+    fn queue_max_sizes(&self) -> &[u16] {
+        &self.queue_sizes
+    }
+
+    fn features(&self) -> u64 {
+        self.avail_features
+    }
+
+    fn ack_features(&mut self, value: u64) {
+        let mut v = value;
+        // Check if the guest is ACK'ing a feature that we didn't claim to have.
+        let unrequested_features = v & !self.avail_features;
+        if unrequested_features != 0 {
+            warn!("Received acknowledge request for unknown feature: {:x}", v);
+            // Don't count these features as acked.
+            v &= !unrequested_features;
+        }
+        self.acked_features |= v;
+    }
+
+    fn read_config(&self, offset: u64, mut data: &mut [u8]) {
+        let config_slice = self.config.as_slice();
+        let config_len = config_slice.len() as u64;
+        if offset >= config_len {
+            error!("Failed to read config space");
+            return;
+        }
+        if let Some(end) = offset.checked_add(data.len() as u64) {
+            // This write can't fail, offset and end are checked against config_len.
+            data.write_all(&config_slice[offset as usize..cmp::min(end, config_len) as usize])
+                .unwrap();
+        }
+    }
+
+    fn write_config(&mut self, offset: u64, data: &[u8]) {
+        let config_slice = self.config.as_mut_slice();
+        let data_len = data.len() as u64;
+        let config_len = config_slice.len() as u64;
+        if offset + data_len > config_len {
+            error!("Failed to write config space");
+            return;
+        }
+        self.vhost_user_blk
+            .set_config(offset as u32, VhostUserConfigFlags::WRITABLE, data)
+            .expect("Failed to set config");
+        let (_, right) = config_slice.split_at_mut(offset as usize);
+        right.copy_from_slice(&data[..]);
+    }
+
+    fn activate(
+        &mut self,
+        mem: GuestMemoryAtomic<GuestMemoryMmap>,
+        interrupt_cb: Arc<dyn VirtioInterrupt>,
+        queues: Vec<Queue>,
+        queue_evts: Vec<EventFd>,
+    ) -> ActivateResult {
+        let (self_kill_evt, kill_evt) = EventFd::new(EFD_NONBLOCK)
+            .and_then(|e| Ok((e.try_clone()?, e)))
+            .map_err(|e| {
+                error!("failed creating kill EventFd pair: {}", e);
+                ActivateError::BadActivate
+            })?;
+        self.kill_evt = Some(self_kill_evt);
+
+        let (self_pause_evt, pause_evt) = EventFd::new(EFD_NONBLOCK)
+            .and_then(|e| Ok((e.try_clone()?, e)))
+            .map_err(|e| {
+                error!("failed creating pause EventFd pair: {}", e);
+                ActivateError::BadActivate
+            })?;
+        self.pause_evt = Some(self_pause_evt);
+
+        // Save the interrupt EventFD as we need to return it on reset
+        // but clone it to pass into the thread.
+        self.interrupt_cb = Some(interrupt_cb.clone());
+
+        let mut tmp_queue_evts: Vec<EventFd> = Vec::new();
+        for queue_evt in queue_evts.iter() {
+            // Save the queue EventFD as we need to return it on reset
+            // but clone it to pass into the thread.
+            tmp_queue_evts.push(queue_evt.try_clone().map_err(|e| {
+                error!("failed to clone queue EventFd: {}", e);
+                ActivateError::BadActivate
+            })?);
+        }
+        self.queue_evts = Some(tmp_queue_evts);
+
+        let mut vu_interrupt_list = setup_vhost_user(
+            &mut self.vhost_user_blk,
+            &mem.memory(),
+            queues,
+            queue_evts,
+            &interrupt_cb,
+            self.acked_features,
+        )
+        .map_err(ActivateError::VhostUserBlkSetup)?;
+
+        let mut epoll_threads = Vec::new();
+        for _ in 0..vu_interrupt_list.len() {
+            let mut interrupt_list_sub: Vec<(Option<EventFd>, Queue)> = Vec::with_capacity(1);
+            interrupt_list_sub.push(vu_interrupt_list.remove(0));
+
+            let mut handler = VhostUserEpollHandler::<SlaveReqHandler>::new(VhostUserEpollConfig {
+                interrupt_cb: interrupt_cb.clone(),
+                kill_evt: kill_evt.try_clone().unwrap(),
+                pause_evt: pause_evt.try_clone().unwrap(),
+                vu_interrupt_list: interrupt_list_sub,
+                slave_req_handler: None,
+            });
+
+            let paused = self.paused.clone();
+            thread::Builder::new()
+                .name("vhost_user_blk".to_string())
+                .spawn(move || handler.run(paused))
+                .map(|thread| epoll_threads.push(thread))
+                .map_err(|e| {
+                    error!("failed to clone virtio epoll thread: {}", e);
+                    ActivateError::BadActivate
+                })?;
+        }
+        self.epoll_threads = Some(epoll_threads);
+
+        Ok(())
+    }
+
+    fn reset(&mut self) -> Option<(Arc<dyn VirtioInterrupt>, Vec<EventFd>)> {
+        // We first must resume the virtio thread if it was paused.
+        if self.pause_evt.take().is_some() {
+            self.resume().ok()?;
+        }
+
+        if let Err(e) = reset_vhost_user(&mut self.vhost_user_blk, self.queue_sizes.len()) {
+            error!("Failed to reset vhost-user daemon: {:?}", e);
+            return None;
+        }
+
+        if let Some(kill_evt) = self.kill_evt.take() {
+            // Ignore the result because there is nothing we can do about it.
+            let _ = kill_evt.write(1);
+        }
+
+        // Return the interrupt and queue EventFDs
+        Some((
+            self.interrupt_cb.take().unwrap(),
+            self.queue_evts.take().unwrap(),
+        ))
+    }
+
+    fn shutdown(&mut self) {
+        let _ = unsafe { libc::close(self.vhost_user_blk.as_raw_fd()) };
+    }
+
+    fn update_memory(&mut self, mem: &GuestMemoryMmap) -> std::result::Result<(), crate::Error> {
+        update_mem_table(&mut self.vhost_user_blk, mem).map_err(crate::Error::VhostUserUpdateMemory)
+    }
+}
+
+virtio_pausable!(Blk);
+impl Snapshottable for Blk {
+    fn id(&self) -> String {
+        self.id.clone()
+    }
+}
+impl Transportable for Blk {}
+impl Migratable for Blk {}

virtio-devices/src/vhost_user/fs.rs

@@ -0,0 +1,617 @@
+// Copyright 2019 Intel Corporation. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+use super::vu_common_ctrl::{reset_vhost_user, setup_vhost_user, update_mem_table};
+use super::Error as DeviceError;
+use super::{Error, Result};
+use crate::vhost_user::handler::{VhostUserEpollConfig, VhostUserEpollHandler};
+use crate::{
+    ActivateError, ActivateResult, Queue, UserspaceMapping, VirtioDevice, VirtioDeviceType,
+    VirtioInterrupt, VirtioSharedMemoryList, VIRTIO_F_VERSION_1,
+};
+use libc::{self, c_void, off64_t, pread64, pwrite64, EFD_NONBLOCK};
+use std::cmp;
+use std::io;
+use std::io::Write;
+use std::os::unix::io::{AsRawFd, RawFd};
+use std::result;
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::{Arc, Mutex};
+use std::thread;
+use vhost_rs::vhost_user::message::{
+    VhostUserFSSlaveMsg, VhostUserFSSlaveMsgFlags, VhostUserProtocolFeatures,
+    VhostUserVirtioFeatures, VHOST_USER_FS_SLAVE_ENTRIES,
+};
+use vhost_rs::vhost_user::{
+    HandlerResult, Master, MasterReqHandler, VhostUserMaster, VhostUserMasterReqHandler,
+};
+use vhost_rs::VhostBackend;
+use vm_memory::{
+    Address, ByteValued, GuestAddress, GuestAddressSpace, GuestMemory, GuestMemoryAtomic,
+    GuestMemoryMmap, MmapRegion,
+};
+use vm_migration::{Migratable, MigratableError, Pausable, Snapshottable, Transportable};
+use vmm_sys_util::eventfd::EventFd;
+
+const NUM_QUEUE_OFFSET: usize = 1;
+
+struct SlaveReqHandler {
+    cache_offset: GuestAddress,
+    cache_size: u64,
+    mmap_cache_addr: u64,
+    mem: GuestMemoryAtomic<GuestMemoryMmap>,
+}
+
+impl SlaveReqHandler {
+    // Make sure request is within cache range
+    fn is_req_valid(&self, offset: u64, len: u64) -> bool {
+        let end = match offset.checked_add(len) {
+            Some(n) => n,
+            None => return false,
+        };
+
+        !(offset >= self.cache_size || end > self.cache_size)
+    }
+}
+
+impl VhostUserMasterReqHandler for SlaveReqHandler {
+    fn handle_config_change(&mut self) -> HandlerResult<u64> {
+        debug!("handle_config_change");
+        Ok(0)
+    }
+
+    fn fs_slave_map(&mut self, fs: &VhostUserFSSlaveMsg, fd: RawFd) -> HandlerResult<u64> {
+        debug!("fs_slave_map");
+
+        for i in 0..VHOST_USER_FS_SLAVE_ENTRIES {
+            let offset = fs.cache_offset[i];
+            let len = fs.len[i];
+
+            // Ignore if the length is 0.
+            if len == 0 {
+                continue;
+            }
+
+            if !self.is_req_valid(offset, len) {
+                return Err(io::Error::from_raw_os_error(libc::EINVAL));
+            }
+
+            let addr = self.mmap_cache_addr + offset;
+            let ret = unsafe {
+                libc::mmap(
+                    addr as *mut libc::c_void,
+                    len as usize,
+                    fs.flags[i].bits() as i32,
+                    libc::MAP_SHARED | libc::MAP_FIXED,
+                    fd,
+                    fs.fd_offset[i] as libc::off_t,
+                )
+            };
+            if ret == libc::MAP_FAILED {
+                return Err(io::Error::last_os_error());
+            }
+
+            let ret = unsafe { libc::close(fd) };
+            if ret == -1 {
+                return Err(io::Error::last_os_error());
+            }
+        }
+
+        Ok(0)
+    }
+
+    fn fs_slave_unmap(&mut self, fs: &VhostUserFSSlaveMsg) -> HandlerResult<u64> {
+        debug!("fs_slave_unmap");
+
+        for i in 0..VHOST_USER_FS_SLAVE_ENTRIES {
+            let offset = fs.cache_offset[i];
+            let mut len = fs.len[i];
+
+            // Ignore if the length is 0.
+            if len == 0 {
+                continue;
+            }
+
+            // Need to handle a special case where the slave ask for the unmapping
+            // of the entire mapping.
+            if len == 0xffff_ffff_ffff_ffff {
+                len = self.cache_size;
+            }
+
+            if !self.is_req_valid(offset, len) {
+                return Err(io::Error::from_raw_os_error(libc::EINVAL));
+            }
+
+            let addr = self.mmap_cache_addr + offset;
+            let ret = unsafe {
+                libc::mmap(
+                    addr as *mut libc::c_void,
+                    len as usize,
+                    libc::PROT_NONE,
+                    libc::MAP_ANONYMOUS | libc::MAP_PRIVATE | libc::MAP_FIXED,
+                    -1,
+                    0 as libc::off_t,
+                )
+            };
+            if ret == libc::MAP_FAILED {
+                return Err(io::Error::last_os_error());
+            }
+        }
+
+        Ok(0)
+    }
+
+    fn fs_slave_sync(&mut self, fs: &VhostUserFSSlaveMsg) -> HandlerResult<u64> {
+        debug!("fs_slave_sync");
+
+        for i in 0..VHOST_USER_FS_SLAVE_ENTRIES {
+            let offset = fs.cache_offset[i];
+            let len = fs.len[i];
+
+            // Ignore if the length is 0.
+            if len == 0 {
+                continue;
+            }
+
+            if !self.is_req_valid(offset, len) {
+                return Err(io::Error::from_raw_os_error(libc::EINVAL));
+            }
+
+            let addr = self.mmap_cache_addr + offset;
+            let ret =
+                unsafe { libc::msync(addr as *mut libc::c_void, len as usize, libc::MS_SYNC) };
+            if ret == -1 {
+                return Err(io::Error::last_os_error());
+            }
+        }
+
+        Ok(0)
+    }
+
+    fn fs_slave_io(&mut self, fs: &VhostUserFSSlaveMsg, fd: RawFd) -> HandlerResult<u64> {
+        debug!("fs_slave_io");
+
+        let mut done: u64 = 0;
+        for i in 0..VHOST_USER_FS_SLAVE_ENTRIES {
+            // Ignore if the length is 0.
+            if fs.len[i] == 0 {
+                continue;
+            }
+
+            let mut foffset = fs.fd_offset[i];
+            let mut len = fs.len[i] as usize;
+            let gpa = fs.cache_offset[i];
+            let cache_end = self.cache_offset.raw_value() + self.cache_size;
+            let efault = libc::EFAULT;
+
+            let mut ptr = if gpa >= self.cache_offset.raw_value() && gpa < cache_end {
+                let offset = gpa
+                    .checked_sub(self.cache_offset.raw_value())
+                    .ok_or_else(|| io::Error::from_raw_os_error(efault))?;
+                let end = gpa
+                    .checked_add(fs.len[i])
+                    .ok_or_else(|| io::Error::from_raw_os_error(efault))?;
+
+                if end >= cache_end {
+                    return Err(io::Error::from_raw_os_error(efault));
+                }
+
+                self.mmap_cache_addr + offset
+            } else {
+                self.mem
+                    .memory()
+                    .get_host_address(GuestAddress(gpa))
+                    .map_err(|e| {
+                        error!(
+                            "Failed to find RAM region associated with guest physical address 0x{:x}: {:?}",
+                            gpa, e
+                        );
+                        io::Error::from_raw_os_error(efault)
+                    })? as u64
+            };
+
+            while len > 0 {
+                let ret = if (fs.flags[i] & VhostUserFSSlaveMsgFlags::MAP_W)
+                    == VhostUserFSSlaveMsgFlags::MAP_W
+                {
+                    debug!("write: foffset={}, len={}", foffset, len);
+                    unsafe { pwrite64(fd, ptr as *const c_void, len as usize, foffset as off64_t) }
+                } else {
+                    debug!("read: foffset={}, len={}", foffset, len);
+                    unsafe { pread64(fd, ptr as *mut c_void, len as usize, foffset as off64_t) }
+                };
+
+                if ret < 0 {
+                    return Err(io::Error::last_os_error());
+                }
+
+                if ret == 0 {
+                    // EOF
+                    return Err(io::Error::new(
+                        io::ErrorKind::UnexpectedEof,
+                        "failed to access whole buffer",
+                    ));
+                }
+                len -= ret as usize;
+                foffset += ret as u64;
+                ptr += ret as u64;
+                done += ret as u64;
+            }
+        }
+
+        let ret = unsafe { libc::close(fd) };
+        if ret == -1 {
+            return Err(io::Error::last_os_error());
+        }
+
+        Ok(done)
+    }
+}
+
+#[derive(Copy, Clone)]
+#[repr(C, packed)]
+struct VirtioFsConfig {
+    tag: [u8; 36],
+    num_request_queues: u32,
+}
+
+impl Default for VirtioFsConfig {
+    fn default() -> Self {
+        VirtioFsConfig {
+            tag: [0; 36],
+            num_request_queues: 0,
+        }
+    }
+}
+
+unsafe impl ByteValued for VirtioFsConfig {}
+
+pub struct Fs {
+    id: String,
+    vu: Master,
+    queue_sizes: Vec<u16>,
+    avail_features: u64,
+    acked_features: u64,
+    config: VirtioFsConfig,
+    kill_evt: Option<EventFd>,
+    pause_evt: Option<EventFd>,
+    // Hold ownership of the memory that is allocated for the device
+    // which will be automatically dropped when the device is dropped
+    cache: Option<(VirtioSharedMemoryList, MmapRegion)>,
+    slave_req_support: bool,
+    queue_evts: Option<Vec<EventFd>>,
+    interrupt_cb: Option<Arc<dyn VirtioInterrupt>>,
+    epoll_threads: Option<Vec<thread::JoinHandle<result::Result<(), DeviceError>>>>,
+    paused: Arc<AtomicBool>,
+}
+
+impl Fs {
+    /// Create a new virtio-fs device.
+    pub fn new(
+        id: String,
+        path: &str,
+        tag: &str,
+        req_num_queues: usize,
+        queue_size: u16,
+        cache: Option<(VirtioSharedMemoryList, MmapRegion)>,
+    ) -> Result<Fs> {
+        let mut slave_req_support = false;
+
+        // Calculate the actual number of queues needed.
+        let num_queues = NUM_QUEUE_OFFSET + req_num_queues;
+
+        // Connect to the vhost-user socket.
+        let mut master =
+            Master::connect(path, num_queues as u64).map_err(Error::VhostUserCreateMaster)?;
+
+        // Filling device and vring features VMM supports.
+        let mut avail_features =
+            1 << VIRTIO_F_VERSION_1 | VhostUserVirtioFeatures::PROTOCOL_FEATURES.bits();
+
+        // Set vhost-user owner.
+        master.set_owner().map_err(Error::VhostUserSetOwner)?;
+
+        // Get features from backend, do negotiation to get a feature collection which
+        // both VMM and backend support.
+        let backend_features = master.get_features().map_err(Error::VhostUserGetFeatures)?;
+        avail_features &= backend_features;
+        // Set features back is required by the vhost crate mechanism, since the
+        // later vhost call will check if features is filled in master before execution.
+        master
+            .set_features(avail_features)
+            .map_err(Error::VhostUserSetFeatures)?;
+
+        // Identify if protocol features are supported by the slave.
+        let mut acked_features = 0;
+        if avail_features & VhostUserVirtioFeatures::PROTOCOL_FEATURES.bits() != 0 {
+            acked_features |= VhostUserVirtioFeatures::PROTOCOL_FEATURES.bits();
+
+            let mut protocol_features = master
+                .get_protocol_features()
+                .map_err(Error::VhostUserGetProtocolFeatures)?;
+
+            if cache.is_some() {
+                protocol_features &= VhostUserProtocolFeatures::MQ
+                    | VhostUserProtocolFeatures::REPLY_ACK
+                    | VhostUserProtocolFeatures::SLAVE_REQ
+                    | VhostUserProtocolFeatures::SLAVE_SEND_FD;
+            } else {
+                protocol_features &=
+                    VhostUserProtocolFeatures::MQ | VhostUserProtocolFeatures::REPLY_ACK;
+            }
+
+            master
+                .set_protocol_features(protocol_features)
+                .map_err(Error::VhostUserSetProtocolFeatures)?;
+
+            slave_req_support = true;
+        }
+
+        // Create virtio-fs device configuration.
+        let mut config = VirtioFsConfig::default();
+        let tag_bytes_vec = tag.to_string().into_bytes();
+        config.tag[..tag_bytes_vec.len()].copy_from_slice(tag_bytes_vec.as_slice());
+        config.num_request_queues = req_num_queues as u32;
+
+        Ok(Fs {
+            id,
+            vu: master,
+            queue_sizes: vec![queue_size; num_queues],
+            avail_features,
+            acked_features,
+            config,
+            kill_evt: None,
+            pause_evt: None,
+            cache,
+            slave_req_support,
+            queue_evts: None,
+            interrupt_cb: None,
+            epoll_threads: None,
+            paused: Arc::new(AtomicBool::new(false)),
+        })
+    }
+}
+
+impl Drop for Fs {
+    fn drop(&mut self) {
+        if let Some(kill_evt) = self.kill_evt.take() {
+            // Ignore the result because there is nothing we can do about it.
+            let _ = kill_evt.write(1);
+        }
+    }
+}
+
+impl VirtioDevice for Fs {
+    fn device_type(&self) -> u32 {
+        VirtioDeviceType::TYPE_FS as u32
+    }
+
+    fn queue_max_sizes(&self) -> &[u16] {
+        &self.queue_sizes.as_slice()
+    }
+
+    fn features(&self) -> u64 {
+        self.avail_features
+    }
+
+    fn ack_features(&mut self, value: u64) {
+        let mut v = value;
+        // Check if the guest is ACK'ing a feature that we didn't claim to have.
+        let unrequested_features = v & !self.avail_features;
+        if unrequested_features != 0 {
+            warn!("fs: virtio-fs got unknown feature ack: {:x}", v);
+
+            // Don't count these features as acked.
+            v &= !unrequested_features;
+        }
+        self.acked_features |= v;
+    }
+
+    fn read_config(&self, offset: u64, mut data: &mut [u8]) {
+        let config_slice = self.config.as_slice();
+        let config_len = config_slice.len() as u64;
+        if offset >= config_len {
+            error!("Failed to read config space");
+            return;
+        }
+        if let Some(end) = offset.checked_add(data.len() as u64) {
+            // This write can't fail, offset and end are checked against config_len.
+            data.write_all(&config_slice[offset as usize..cmp::min(end, config_len) as usize])
+                .unwrap();
+        }
+    }
+
+    fn write_config(&mut self, offset: u64, data: &[u8]) {
+        let config_slice = self.config.as_mut_slice();
+        let data_len = data.len() as u64;
+        let config_len = config_slice.len() as u64;
+        if offset + data_len > config_len {
+            error!("Failed to write config space");
+            return;
+        }
+        let (_, right) = config_slice.split_at_mut(offset as usize);
+        right.copy_from_slice(&data[..]);
+    }
+
+    fn activate(
+        &mut self,
+        mem: GuestMemoryAtomic<GuestMemoryMmap>,
+        interrupt_cb: Arc<dyn VirtioInterrupt>,
+        queues: Vec<Queue>,
+        queue_evts: Vec<EventFd>,
+    ) -> ActivateResult {
+        if queues.len() != self.queue_sizes.len() || queue_evts.len() != self.queue_sizes.len() {
+            error!(
+                "Cannot perform activate. Expected {} queue(s), got {}",
+                self.queue_sizes.len(),
+                queues.len()
+            );
+            return Err(ActivateError::BadActivate);
+        }
+
+        let (self_kill_evt, kill_evt) = EventFd::new(EFD_NONBLOCK)
+            .and_then(|e| Ok((e.try_clone()?, e)))
+            .map_err(|e| {
+                error!("failed creating kill EventFd pair: {}", e);
+                ActivateError::BadActivate
+            })?;
+        self.kill_evt = Some(self_kill_evt);
+
+        let (self_pause_evt, pause_evt) = EventFd::new(EFD_NONBLOCK)
+            .and_then(|e| Ok((e.try_clone()?, e)))
+            .map_err(|e| {
+                error!("failed creating pause EventFd pair: {}", e);
+                ActivateError::BadActivate
+            })?;
+        self.pause_evt = Some(self_pause_evt);
+
+        // Save the interrupt EventFD as we need to return it on reset
+        // but clone it to pass into the thread.
+        self.interrupt_cb = Some(interrupt_cb.clone());
+
+        let mut tmp_queue_evts: Vec<EventFd> = Vec::new();
+        for queue_evt in queue_evts.iter() {
+            // Save the queue EventFD as we need to return it on reset
+            // but clone it to pass into the thread.
+            tmp_queue_evts.push(queue_evt.try_clone().map_err(|e| {
+                error!("failed to clone queue EventFd: {}", e);
+                ActivateError::BadActivate
+            })?);
+        }
+        self.queue_evts = Some(tmp_queue_evts);
+
+        let vu_call_evt_queue_list = setup_vhost_user(
+            &mut self.vu,
+            &mem.memory(),
+            queues,
+            queue_evts,
+            &interrupt_cb,
+            self.acked_features,
+        )
+        .map_err(ActivateError::VhostUserSetup)?;
+
+        // Initialize slave communication.
+        let slave_req_handler = if self.slave_req_support {
+            if let Some(cache) = self.cache.as_ref() {
+                let vu_master_req_handler = Arc::new(Mutex::new(SlaveReqHandler {
+                    cache_offset: cache.0.addr,
+                    cache_size: cache.0.len,
+                    mmap_cache_addr: cache.0.host_addr,
+                    mem,
+                }));
+
+                let req_handler = MasterReqHandler::new(vu_master_req_handler).map_err(|e| {
+                    ActivateError::VhostUserSetup(Error::MasterReqHandlerCreation(e))
+                })?;
+                self.vu
+                    .set_slave_request_fd(req_handler.get_tx_raw_fd())
+                    .map_err(|e| {
+                        ActivateError::VhostUserSetup(Error::VhostUserSetSlaveRequestFd(e))
+                    })?;
+                Some(req_handler)
+            } else {
+                None
+            }
+        } else {
+            None
+        };
+
+        let mut handler = VhostUserEpollHandler::new(VhostUserEpollConfig {
+            vu_interrupt_list: vu_call_evt_queue_list,
+            interrupt_cb,
+            kill_evt,
+            pause_evt,
+            slave_req_handler,
+        });
+
+        let paused = self.paused.clone();
+        let mut epoll_threads = Vec::new();
+        thread::Builder::new()
+            .name("virtio_fs".to_string())
+            .spawn(move || handler.run(paused))
+            .map(|thread| epoll_threads.push(thread))
+            .map_err(|e| {
+                error!("failed to clone queue EventFd: {}", e);
+                ActivateError::BadActivate
+            })?;
+
+        self.epoll_threads = Some(epoll_threads);
+
+        Ok(())
+    }
+
+    fn reset(&mut self) -> Option<(Arc<dyn VirtioInterrupt>, Vec<EventFd>)> {
+        // We first must resume the virtio thread if it was paused.
+        if self.pause_evt.take().is_some() {
+            self.resume().ok()?;
+        }
+
+        if let Err(e) = reset_vhost_user(&mut self.vu, self.queue_sizes.len()) {
+            error!("Failed to reset vhost-user daemon: {:?}", e);
+            return None;
+        }
+
+        if let Some(kill_evt) = self.kill_evt.take() {
+            // Ignore the result because there is nothing we can do about it.
+            let _ = kill_evt.write(1);
+        }
+
+        // Return the interrupt and queue EventFDs
+        Some((
+            self.interrupt_cb.take().unwrap(),
+            self.queue_evts.take().unwrap(),
+        ))
+    }
+
+    fn shutdown(&mut self) {
+        let _ = unsafe { libc::close(self.vu.as_raw_fd()) };
+    }
+
+    fn get_shm_regions(&self) -> Option<VirtioSharedMemoryList> {
+        if let Some(cache) = self.cache.as_ref() {
+            Some(cache.0.clone())
+        } else {
+            None
+        }
+    }
+
+    fn set_shm_regions(
+        &mut self,
+        shm_regions: VirtioSharedMemoryList,
+    ) -> std::result::Result<(), crate::Error> {
+        if let Some(mut cache) = self.cache.as_mut() {
+            cache.0 = shm_regions;
+            Ok(())
+        } else {
+            Err(crate::Error::SetShmRegionsNotSupported)
+        }
+    }
+
+    fn update_memory(&mut self, mem: &GuestMemoryMmap) -> std::result::Result<(), crate::Error> {
+        update_mem_table(&mut self.vu, mem).map_err(crate::Error::VhostUserUpdateMemory)
+    }
+
+    fn userspace_mappings(&self) -> Vec<UserspaceMapping> {
+        let mut mappings = Vec::new();
+        if let Some(cache) = self.cache.as_ref() {
+            mappings.push(UserspaceMapping {
+                host_addr: cache.0.host_addr,
+                mem_slot: cache.0.mem_slot,
+                addr: cache.0.addr,
+                len: cache.0.len,
+                mergeable: false,
+            })
+        }
+
+        mappings
+    }
+}
+
+virtio_pausable!(Fs);
+impl Snapshottable for Fs {
+    fn id(&self) -> String {
+        self.id.clone()
+    }
+}
+impl Transportable for Fs {}
+impl Migratable for Fs {}

virtio-devices/src/vhost_user/handler.rs

@@ -0,0 +1,196 @@
+// Copyright (c) 2019 Intel Corporation. All rights reserved.
+// Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+//
+// Copyright 2017 The Chromium OS Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE-BSD-3-Clause file.
+//
+// SPDX-License-Identifier: Apache-2.0 AND BSD-3-Clause
+
+use super::super::{Queue, VirtioInterruptType};
+use super::{Error, Result};
+use vmm_sys_util::eventfd::EventFd;
+
+use crate::VirtioInterrupt;
+use std::fs::File;
+use std::io;
+use std::os::unix::io::{AsRawFd, FromRawFd};
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::Arc;
+use std::thread;
+use vhost_rs::vhost_user::{MasterReqHandler, VhostUserMasterReqHandler};
+
+/// Collection of common parameters required by vhost-user devices while
+/// call Epoll handler.
+///
+/// # Arguments
+/// * `interrupt_cb` interrupt for virtqueue change.
+/// * `kill_evt` - EventFd used to kill the vhost-user device.
+/// * `vu_interrupt_list` - virtqueue and EventFd to signal when buffer used.
+pub struct VhostUserEpollConfig<S: VhostUserMasterReqHandler> {
+    pub interrupt_cb: Arc<dyn VirtioInterrupt>,
+    pub kill_evt: EventFd,
+    pub pause_evt: EventFd,
+    pub vu_interrupt_list: Vec<(Option<EventFd>, Queue)>,
+    pub slave_req_handler: Option<MasterReqHandler<S>>,
+}
+
+pub struct VhostUserEpollHandler<S: VhostUserMasterReqHandler> {
+    vu_epoll_cfg: VhostUserEpollConfig<S>,
+}
+
+impl<S: VhostUserMasterReqHandler> VhostUserEpollHandler<S> {
+    /// Construct a new event handler for vhost-user based devices.
+    ///
+    /// # Arguments
+    /// * `vu_epoll_cfg` - collection of common parameters for vhost-user devices
+    ///
+    /// # Return
+    /// * `VhostUserEpollHandler` - epoll handler for vhost-user based devices
+    pub fn new(vu_epoll_cfg: VhostUserEpollConfig<S>) -> VhostUserEpollHandler<S> {
+        VhostUserEpollHandler { vu_epoll_cfg }
+    }
+
+    fn signal_used_queue(&self, queue: &Queue) -> Result<()> {
+        self.vu_epoll_cfg
+            .interrupt_cb
+            .trigger(&VirtioInterruptType::Queue, Some(queue))
+            .map_err(Error::FailedSignalingUsedQueue)
+    }
+
+    pub fn run(&mut self, paused: Arc<AtomicBool>) -> Result<()> {
+        // Create the epoll file descriptor
+        let epoll_fd = epoll::create(true).map_err(Error::EpollCreateFd)?;
+        // Use 'File' to enforce closing on 'epoll_fd'
+        let epoll_file = unsafe { File::from_raw_fd(epoll_fd) };
+
+        for (index, vhost_user_interrupt) in self.vu_epoll_cfg.vu_interrupt_list.iter().enumerate()
+        {
+            if let Some(eventfd) = &vhost_user_interrupt.0 {
+                // Add events
+                epoll::ctl(
+                    epoll_file.as_raw_fd(),
+                    epoll::ControlOptions::EPOLL_CTL_ADD,
+                    eventfd.as_raw_fd(),
+                    epoll::Event::new(epoll::Events::EPOLLIN, index as u64),
+                )
+                .map_err(Error::EpollCtl)?;
+            }
+        }
+
+        let kill_evt_index = self.vu_epoll_cfg.vu_interrupt_list.len();
+
+        epoll::ctl(
+            epoll_file.as_raw_fd(),
+            epoll::ControlOptions::EPOLL_CTL_ADD,
+            self.vu_epoll_cfg.kill_evt.as_raw_fd(),
+            epoll::Event::new(epoll::Events::EPOLLIN, kill_evt_index as u64),
+        )
+        .map_err(Error::EpollCtl)?;
+
+        let pause_evt_index = kill_evt_index + 1;
+
+        epoll::ctl(
+            epoll_file.as_raw_fd(),
+            epoll::ControlOptions::EPOLL_CTL_ADD,
+            self.vu_epoll_cfg.pause_evt.as_raw_fd(),
+            epoll::Event::new(epoll::Events::EPOLLIN, pause_evt_index as u64),
+        )
+        .map_err(Error::EpollCtl)?;
+
+        let mut index = pause_evt_index;
+
+        let slave_evt_index = if let Some(self_req_handler) = &self.vu_epoll_cfg.slave_req_handler {
+            index = pause_evt_index + 1;
+            epoll::ctl(
+                epoll_file.as_raw_fd(),
+                epoll::ControlOptions::EPOLL_CTL_ADD,
+                self_req_handler.as_raw_fd(),
+                epoll::Event::new(epoll::Events::EPOLLIN, index as u64),
+            )
+            .map_err(Error::EpollCtl)?;
+
+            Some(index)
+        } else {
+            None
+        };
+
+        let mut events = vec![epoll::Event::new(epoll::Events::empty(), 0); index + 1];
+
+        // Before jumping into the epoll loop, check if the device is expected
+        // to be in a paused state. This is helpful for the restore code path
+        // as the device thread should not start processing anything before the
+        // device has been resumed.
+        while paused.load(Ordering::SeqCst) {
+            thread::park();
+        }
+
+        'poll: loop {
+            let num_events = match epoll::wait(epoll_file.as_raw_fd(), -1, &mut events[..]) {
+                Ok(res) => res,
+                Err(e) => {
+                    if e.kind() == io::ErrorKind::Interrupted {
+                        // It's well defined from the epoll_wait() syscall
+                        // documentation that the epoll loop can be interrupted
+                        // before any of the requested events occurred or the
+                        // timeout expired. In both those cases, epoll_wait()
+                        // returns an error of type EINTR, but this should not
+                        // be considered as a regular error. Instead it is more
+                        // appropriate to retry, by calling into epoll_wait().
+                        continue;
+                    }
+                    return Err(Error::EpollWait(e));
+                }
+            };
+
+            for event in events.iter().take(num_events) {
+                let ev_type = event.data as usize;
+
+                match ev_type {
+                    x if x < kill_evt_index => {
+                        if let Some(eventfd) = &self.vu_epoll_cfg.vu_interrupt_list[x].0 {
+                            eventfd.read().map_err(Error::FailedReadingQueue)?;
+                            if let Err(e) =
+                                self.signal_used_queue(&self.vu_epoll_cfg.vu_interrupt_list[x].1)
+                            {
+                                error!("Failed to signal used queue: {:?}", e);
+                                break 'poll;
+                            }
+                        }
+                    }
+                    x if kill_evt_index == x => {
+                        debug!("KILL_EVENT received, stopping epoll loop");
+                        break 'poll;
+                    }
+                    x if pause_evt_index == x => {
+                        debug!("PAUSE_EVENT received, pausing vhost-user epoll loop");
+                        // We loop here to handle spurious park() returns.
+                        // Until we have not resumed, the paused boolean will
+                        // be true.
+                        while paused.load(Ordering::SeqCst) {
+                            thread::park();
+                        }
+
+                        // Drain pause event after the device has been resumed.
+                        // This ensures the pause event has been seen by each
+                        // and every thread related to this virtio device.
+                        let _ = self.vu_epoll_cfg.pause_evt.read();
+                    }
+                    x if (slave_evt_index.is_some() && slave_evt_index.unwrap() == x) => {
+                        if let Some(slave_req_handler) =
+                            self.vu_epoll_cfg.slave_req_handler.as_mut()
+                        {
+                            slave_req_handler
+                                .handle_request()
+                                .map_err(Error::VhostUserSlaveRequest)?;
+                        }
+                    }
+                    _ => {
+                        error!("Unknown event for vhost-user");
+                    }
+                }
+            }
+        }
+        Ok(())
+    }
+}

virtio-devices/src/vhost_user/mod.rs

@@ -0,0 +1,102 @@
+// Copyright 2019 Intel Corporation. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+extern crate epoll;
+extern crate net_util;
+extern crate vhost_rs;
+extern crate virtio_bindings;
+extern crate vm_memory;
+
+use std::io;
+use vhost_rs::Error as VhostError;
+use vm_memory::Error as MmapError;
+
+pub mod blk;
+pub mod fs;
+mod handler;
+pub mod net;
+pub mod vu_common_ctrl;
+
+pub use self::blk::Blk;
+pub use self::fs::*;
+pub use self::net::Net;
+pub use self::vu_common_ctrl::VhostUserConfig;
+
+#[derive(Debug)]
+pub enum Error {
+    /// Invalid available address.
+    AvailAddress,
+    /// Queue number  is not correct
+    BadQueueNum,
+    /// Creating kill eventfd failed.
+    CreateKillEventFd(io::Error),
+    /// Cloning kill eventfd failed.
+    CloneKillEventFd(io::Error),
+    /// Invalid descriptor table address.
+    DescriptorTableAddress,
+    /// Create Epoll eventfd failed
+    EpollCreateFd(io::Error),
+    /// Epoll ctl error
+    EpollCtl(io::Error),
+    /// Epoll wait error
+    EpollWait(io::Error),
+    /// Read queue failed.
+    FailedReadingQueue(io::Error),
+    /// Signal used queue failed.
+    FailedSignalingUsedQueue(io::Error),
+    /// Failed to read vhost eventfd.
+    MemoryRegions(MmapError),
+    /// Failed to create master.
+    VhostUserCreateMaster(VhostError),
+    /// Failed to open vhost device.
+    VhostUserOpen(VhostError),
+    /// Connection to socket failed.
+    VhostUserConnect(vhost_rs::Error),
+    /// Get features failed.
+    VhostUserGetFeatures(VhostError),
+    /// Get queue max number failed.
+    VhostUserGetQueueMaxNum(VhostError),
+    /// Get protocol features failed.
+    VhostUserGetProtocolFeatures(VhostError),
+    /// Vhost-user Backend not support vhost-user protocol.
+    VhostUserProtocolNotSupport,
+    /// Set owner failed.
+    VhostUserSetOwner(VhostError),
+    /// Reset owner failed.
+    VhostUserResetOwner(VhostError),
+    /// Set features failed.
+    VhostUserSetFeatures(VhostError),
+    /// Set protocol features failed.
+    VhostUserSetProtocolFeatures(VhostError),
+    /// Set mem table failed.
+    VhostUserSetMemTable(VhostError),
+    /// Set vring num failed.
+    VhostUserSetVringNum(VhostError),
+    /// Set vring addr failed.
+    VhostUserSetVringAddr(VhostError),
+    /// Set vring base failed.
+    VhostUserSetVringBase(VhostError),
+    /// Set vring call failed.
+    VhostUserSetVringCall(VhostError),
+    /// Set vring kick failed.
+    VhostUserSetVringKick(VhostError),
+    /// Set vring enable failed.
+    VhostUserSetVringEnable(VhostError),
+    /// Failed to create vhost eventfd.
+    VhostIrqCreate(io::Error),
+    /// Failed to read vhost eventfd.
+    VhostIrqRead(io::Error),
+    /// Failed to read vhost eventfd.
+    VhostUserMemoryRegion(MmapError),
+    /// Failed to handle vhost-user slave request.
+    VhostUserSlaveRequest(vhost_rs::vhost_user::Error),
+    /// Failed to create the master request handler from slave.
+    MasterReqHandlerCreation(vhost_rs::vhost_user::Error),
+    /// Set slave request fd failed.
+    VhostUserSetSlaveRequestFd(vhost_rs::Error),
+    /// Invalid used address.
+    UsedAddress,
+    /// Invalid features provided from vhost-user backend
+    InvalidFeatures,
+}
+type Result<T> = std::result::Result<T, Error>;

virtio-devices/src/vhost_user/net.rs

@@ -0,0 +1,375 @@
+// Copyright 2019 Intel Corporation. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+use super::super::net_util::{
+    build_net_config_space, CtrlVirtio, NetCtrlEpollHandler, VirtioNetConfig,
+};
+use super::super::Error as CtrlError;
+use super::super::{ActivateError, ActivateResult, Queue, VirtioDevice, VirtioDeviceType};
+use super::handler::*;
+use super::vu_common_ctrl::*;
+use super::Error as DeviceError;
+use super::{Error, Result};
+use crate::VirtioInterrupt;
+use libc::EFD_NONBLOCK;
+use net_util::MacAddr;
+use std::cmp;
+use std::io::Write;
+use std::os::unix::io::AsRawFd;
+use std::result;
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::Arc;
+use std::thread;
+use std::vec::Vec;
+use vhost_rs::vhost_user::message::{VhostUserProtocolFeatures, VhostUserVirtioFeatures};
+use vhost_rs::vhost_user::{Master, VhostUserMaster, VhostUserMasterReqHandler};
+use vhost_rs::VhostBackend;
+use virtio_bindings::bindings::virtio_net;
+use virtio_bindings::bindings::virtio_ring;
+use vm_memory::{ByteValued, GuestAddressSpace, GuestMemoryAtomic, GuestMemoryMmap};
+use vm_migration::{Migratable, MigratableError, Pausable, Snapshottable, Transportable};
+use vmm_sys_util::eventfd::EventFd;
+
+const DEFAULT_QUEUE_NUMBER: usize = 2;
+
+struct SlaveReqHandler {}
+impl VhostUserMasterReqHandler for SlaveReqHandler {}
+
+pub struct Net {
+    id: String,
+    vhost_user_net: Master,
+    kill_evt: Option<EventFd>,
+    pause_evt: Option<EventFd>,
+    avail_features: u64,
+    acked_features: u64,
+    backend_features: u64,
+    config: VirtioNetConfig,
+    queue_sizes: Vec<u16>,
+    queue_evts: Option<Vec<EventFd>>,
+    interrupt_cb: Option<Arc<dyn VirtioInterrupt>>,
+    epoll_threads: Option<Vec<thread::JoinHandle<result::Result<(), DeviceError>>>>,
+    ctrl_queue_epoll_thread: Option<thread::JoinHandle<result::Result<(), CtrlError>>>,
+    paused: Arc<AtomicBool>,
+}
+
+impl Net {
+    /// Create a new vhost-user-net device
+    /// Create a new vhost-user-net device
+    pub fn new(id: String, mac_addr: MacAddr, vu_cfg: VhostUserConfig) -> Result<Net> {
+        let mut vhost_user_net = Master::connect(&vu_cfg.socket, vu_cfg.num_queues as u64)
+            .map_err(Error::VhostUserCreateMaster)?;
+
+        // Filling device and vring features VMM supports.
+        let mut avail_features = 1 << virtio_net::VIRTIO_NET_F_GUEST_CSUM
+            | 1 << virtio_net::VIRTIO_NET_F_CSUM
+            | 1 << virtio_net::VIRTIO_NET_F_GUEST_TSO4
+            | 1 << virtio_net::VIRTIO_NET_F_GUEST_TSO6
+            | 1 << virtio_net::VIRTIO_NET_F_GUEST_ECN
+            | 1 << virtio_net::VIRTIO_NET_F_GUEST_UFO
+            | 1 << virtio_net::VIRTIO_NET_F_HOST_TSO4
+            | 1 << virtio_net::VIRTIO_NET_F_HOST_TSO6
+            | 1 << virtio_net::VIRTIO_NET_F_HOST_ECN
+            | 1 << virtio_net::VIRTIO_NET_F_HOST_UFO
+            | 1 << virtio_net::VIRTIO_NET_F_MRG_RXBUF
+            | 1 << virtio_net::VIRTIO_F_NOTIFY_ON_EMPTY
+            | 1 << virtio_net::VIRTIO_F_VERSION_1
+            | 1 << virtio_ring::VIRTIO_RING_F_EVENT_IDX
+            | VhostUserVirtioFeatures::PROTOCOL_FEATURES.bits();
+
+        vhost_user_net
+            .set_owner()
+            .map_err(Error::VhostUserSetOwner)?;
+
+        // Get features from backend, do negotiation to get a feature collection which
+        // both VMM and backend support.
+        let backend_features = vhost_user_net
+            .get_features()
+            .map_err(Error::VhostUserGetFeatures)?;
+        avail_features &= backend_features;
+        // Set features back is required by the vhost crate mechanism, since the
+        // later vhost call will check if features is filled in master before execution.
+        vhost_user_net
+            .set_features(avail_features)
+            .map_err(Error::VhostUserSetFeatures)?;
+
+        let protocol_features;
+        let mut acked_features = 0;
+        if avail_features & VhostUserVirtioFeatures::PROTOCOL_FEATURES.bits() != 0 {
+            acked_features |= VhostUserVirtioFeatures::PROTOCOL_FEATURES.bits();
+            protocol_features = vhost_user_net
+                .get_protocol_features()
+                .map_err(Error::VhostUserGetProtocolFeatures)?;
+        } else {
+            return Err(Error::VhostUserProtocolNotSupport);
+        }
+
+        let max_queue_number =
+            if protocol_features.bits() & VhostUserProtocolFeatures::MQ.bits() != 0 {
+                vhost_user_net
+                    .set_protocol_features(protocol_features & VhostUserProtocolFeatures::MQ)
+                    .map_err(Error::VhostUserSetProtocolFeatures)?;
+                match vhost_user_net.get_queue_num() {
+                    Ok(qn) => qn,
+                    Err(_) => DEFAULT_QUEUE_NUMBER as u64,
+                }
+            } else {
+                DEFAULT_QUEUE_NUMBER as u64
+            };
+        if vu_cfg.num_queues > max_queue_number as usize {
+            error!("vhost-user-net has queue number: {} larger than the max queue number: {} backend allowed\n",
+                vu_cfg.num_queues, max_queue_number);
+            return Err(Error::BadQueueNum);
+        }
+
+        avail_features |= 1 << virtio_net::VIRTIO_NET_F_CTRL_VQ;
+        let queue_num = vu_cfg.num_queues + 1;
+
+        let mut config = VirtioNetConfig::default();
+        build_net_config_space(
+            &mut config,
+            mac_addr,
+            vu_cfg.num_queues,
+            &mut avail_features,
+        );
+
+        // Send set_vring_base here, since it could tell backends, like OVS + DPDK,
+        // how many virt queues to be handled, which backend required to know at early stage.
+        for i in 0..vu_cfg.num_queues {
+            vhost_user_net
+                .set_vring_base(i, 0)
+                .map_err(Error::VhostUserSetVringBase)?;
+        }
+
+        Ok(Net {
+            id,
+            vhost_user_net,
+            kill_evt: None,
+            pause_evt: None,
+            avail_features,
+            acked_features,
+            backend_features,
+            config,
+            queue_sizes: vec![vu_cfg.queue_size; queue_num],
+            queue_evts: None,
+            interrupt_cb: None,
+            epoll_threads: None,
+            ctrl_queue_epoll_thread: None,
+            paused: Arc::new(AtomicBool::new(false)),
+        })
+    }
+}
+
+impl Drop for Net {
+    fn drop(&mut self) {
+        if let Some(kill_evt) = self.kill_evt.take() {
+            if let Err(e) = kill_evt.write(1) {
+                error!("failed to kill vhost-user-net: {:?}", e);
+            }
+        }
+    }
+}
+
+impl VirtioDevice for Net {
+    fn device_type(&self) -> u32 {
+        VirtioDeviceType::TYPE_NET as u32
+    }
+
+    fn queue_max_sizes(&self) -> &[u16] {
+        &self.queue_sizes
+    }
+
+    fn features(&self) -> u64 {
+        self.avail_features
+    }
+
+    fn ack_features(&mut self, value: u64) {
+        let mut v = value;
+        // Check if the guest is ACK'ing a feature that we didn't claim to have.
+        let unrequested_features = v & !self.avail_features;
+        if unrequested_features != 0 {
+            warn!("Received acknowledge request for unknown feature: {:x}", v);
+            // Don't count these features as acked.
+            v &= !unrequested_features;
+        }
+        self.acked_features |= v;
+    }
+
+    fn read_config(&self, offset: u64, mut data: &mut [u8]) {
+        let config_slice = self.config.as_slice();
+        let config_len = config_slice.len() as u64;
+        if offset >= config_len {
+            error!("Failed to read config space");
+            return;
+        }
+        if let Some(end) = offset.checked_add(data.len() as u64) {
+            // This write can't fail, offset and end are checked against config_len.
+            data.write_all(&config_slice[offset as usize..cmp::min(end, config_len) as usize])
+                .unwrap();
+        }
+    }
+
+    fn write_config(&mut self, offset: u64, data: &[u8]) {
+        let config_slice = self.config.as_mut_slice();
+        let data_len = data.len() as u64;
+        let config_len = config_slice.len() as u64;
+        if offset + data_len > config_len {
+            error!("Failed to write config space");
+            return;
+        }
+        let (_, right) = config_slice.split_at_mut(offset as usize);
+        right.copy_from_slice(&data[..]);
+    }
+
+    fn activate(
+        &mut self,
+        mem: GuestMemoryAtomic<GuestMemoryMmap>,
+        interrupt_cb: Arc<dyn VirtioInterrupt>,
+        mut queues: Vec<Queue>,
+        mut queue_evts: Vec<EventFd>,
+    ) -> ActivateResult {
+        if queues.len() != self.queue_sizes.len() || queue_evts.len() != self.queue_sizes.len() {
+            error!(
+                "Cannot perform activate. Expected {} queue(s), got {}",
+                self.queue_sizes.len(),
+                queues.len()
+            );
+            return Err(ActivateError::BadActivate);
+        }
+
+        let (self_kill_evt, kill_evt) = EventFd::new(EFD_NONBLOCK)
+            .and_then(|e| Ok((e.try_clone()?, e)))
+            .map_err(|e| {
+                error!("failed creating kill EventFd pair: {}", e);
+                ActivateError::BadActivate
+            })?;
+        self.kill_evt = Some(self_kill_evt);
+
+        let (self_pause_evt, pause_evt) = EventFd::new(EFD_NONBLOCK)
+            .and_then(|e| Ok((e.try_clone()?, e)))
+            .map_err(|e| {
+                error!("failed creating pause EventFd pair: {}", e);
+                ActivateError::BadActivate
+            })?;
+        self.pause_evt = Some(self_pause_evt);
+
+        // Save the interrupt EventFD as we need to return it on reset
+        // but clone it to pass into the thread.
+        self.interrupt_cb = Some(interrupt_cb.clone());
+
+        let mut tmp_queue_evts: Vec<EventFd> = Vec::new();
+        for queue_evt in queue_evts.iter() {
+            // Save the queue EventFD as we need to return it on reset
+            // but clone it to pass into the thread.
+            tmp_queue_evts.push(queue_evt.try_clone().map_err(|e| {
+                error!("failed to clone queue EventFd: {}", e);
+                ActivateError::BadActivate
+            })?);
+        }
+        self.queue_evts = Some(tmp_queue_evts);
+
+        let queue_num = queue_evts.len();
+
+        if (self.acked_features & 1 << virtio_net::VIRTIO_NET_F_CTRL_VQ) != 0 && queue_num % 2 != 0
+        {
+            let cvq_queue = queues.remove(queue_num - 1);
+            let cvq_queue_evt = queue_evts.remove(queue_num - 1);
+
+            let mut ctrl_handler = NetCtrlEpollHandler {
+                mem: mem.clone(),
+                kill_evt: kill_evt.try_clone().unwrap(),
+                pause_evt: pause_evt.try_clone().unwrap(),
+                ctrl_q: CtrlVirtio::new(cvq_queue, cvq_queue_evt),
+                epoll_fd: 0,
+            };
+
+            let paused = self.paused.clone();
+            thread::Builder::new()
+                .name("virtio_net".to_string())
+                .spawn(move || ctrl_handler.run_ctrl(paused))
+                .map(|thread| self.ctrl_queue_epoll_thread = Some(thread))
+                .map_err(|e| {
+                    error!("failed to clone queue EventFd: {}", e);
+                    ActivateError::BadActivate
+                })?;
+        }
+
+        let mut vu_interrupt_list = setup_vhost_user(
+            &mut self.vhost_user_net,
+            &mem.memory(),
+            queues,
+            queue_evts,
+            &interrupt_cb,
+            self.acked_features & self.backend_features,
+        )
+        .map_err(ActivateError::VhostUserNetSetup)?;
+
+        let mut epoll_threads = Vec::new();
+        for _ in 0..vu_interrupt_list.len() / 2 {
+            let mut interrupt_list_sub: Vec<(Option<EventFd>, Queue)> = Vec::with_capacity(2);
+            interrupt_list_sub.push(vu_interrupt_list.remove(0));
+            interrupt_list_sub.push(vu_interrupt_list.remove(0));
+
+            let mut handler = VhostUserEpollHandler::<SlaveReqHandler>::new(VhostUserEpollConfig {
+                interrupt_cb: interrupt_cb.clone(),
+                kill_evt: kill_evt.try_clone().unwrap(),
+                pause_evt: pause_evt.try_clone().unwrap(),
+                vu_interrupt_list: interrupt_list_sub,
+                slave_req_handler: None,
+            });
+
+            let paused = self.paused.clone();
+            thread::Builder::new()
+                .name("vhost_user_net".to_string())
+                .spawn(move || handler.run(paused))
+                .map(|thread| epoll_threads.push(thread))
+                .map_err(|e| {
+                    error!("failed to clone queue EventFd: {}", e);
+                    ActivateError::BadActivate
+                })?;
+        }
+
+        self.epoll_threads = Some(epoll_threads);
+
+        Ok(())
+    }
+
+    fn reset(&mut self) -> Option<(Arc<dyn VirtioInterrupt>, Vec<EventFd>)> {
+        // We first must resume the virtio thread if it was paused.
+        if self.pause_evt.take().is_some() {
+            self.resume().ok()?;
+        }
+
+        if let Err(e) = reset_vhost_user(&mut self.vhost_user_net, self.queue_sizes.len()) {
+            error!("Failed to reset vhost-user daemon: {:?}", e);
+            return None;
+        }
+
+        if let Some(kill_evt) = self.kill_evt.take() {
+            // Ignore the result because there is nothing we can do about it.
+            let _ = kill_evt.write(1);
+        }
+
+        // Return the interrupt and queue EventFDs
+        Some((
+            self.interrupt_cb.take().unwrap(),
+            self.queue_evts.take().unwrap(),
+        ))
+    }
+
+    fn shutdown(&mut self) {
+        let _ = unsafe { libc::close(self.vhost_user_net.as_raw_fd()) };
+    }
+
+    fn update_memory(&mut self, mem: &GuestMemoryMmap) -> std::result::Result<(), crate::Error> {
+        update_mem_table(&mut self.vhost_user_net, mem).map_err(crate::Error::VhostUserUpdateMemory)
+    }
+}
+
+virtio_ctrl_q_pausable!(Net);
+impl Snapshottable for Net {
+    fn id(&self) -> String {
+        self.id.clone()
+    }
+}
+impl Transportable for Net {}
+impl Migratable for Net {}

virtio-devices/src/vhost_user/vu_common_ctrl.rs

@@ -0,0 +1,147 @@
+// Copyright 2019 Intel Corporation. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+use super::super::{Descriptor, Queue};
+use super::{Error, Result};
+use crate::{VirtioInterrupt, VirtioInterruptType};
+use libc::EFD_NONBLOCK;
+use std::convert::TryInto;
+use std::os::unix::io::AsRawFd;
+use std::sync::Arc;
+use std::vec::Vec;
+use vfio_ioctls::get_host_address_range;
+use vhost_rs::vhost_user::{Master, VhostUserMaster};
+use vhost_rs::{VhostBackend, VhostUserMemoryRegionInfo, VringConfigData};
+use vm_memory::{Address, Error as MmapError, GuestMemory, GuestMemoryMmap, GuestMemoryRegion};
+use vmm_sys_util::eventfd::EventFd;
+
+#[derive(Debug, Clone)]
+pub struct VhostUserConfig {
+    pub socket: String,
+    pub num_queues: usize,
+    pub queue_size: u16,
+}
+
+pub fn update_mem_table(vu: &mut Master, mem: &GuestMemoryMmap) -> Result<()> {
+    let mut regions: Vec<VhostUserMemoryRegionInfo> = Vec::new();
+    mem.with_regions_mut(|_, region| {
+        let (mmap_handle, mmap_offset) = match region.file_offset() {
+            Some(_file_offset) => (_file_offset.file().as_raw_fd(), _file_offset.start()),
+            None => return Err(MmapError::NoMemoryRegion),
+        };
+
+        let vhost_user_net_reg = VhostUserMemoryRegionInfo {
+            guest_phys_addr: region.start_addr().raw_value(),
+            memory_size: region.len() as u64,
+            userspace_addr: region.as_ptr() as u64,
+            mmap_offset,
+            mmap_handle,
+        };
+
+        regions.push(vhost_user_net_reg);
+
+        Ok(())
+    })
+    .map_err(Error::VhostUserMemoryRegion)?;
+
+    vu.set_mem_table(regions.as_slice())
+        .map_err(Error::VhostUserSetMemTable)?;
+
+    Ok(())
+}
+
+pub fn setup_vhost_user_vring(
+    vu: &mut Master,
+    mem: &GuestMemoryMmap,
+    queues: Vec<Queue>,
+    queue_evts: Vec<EventFd>,
+    virtio_interrupt: &Arc<dyn VirtioInterrupt>,
+) -> Result<Vec<(Option<EventFd>, Queue)>> {
+    // Let's first provide the memory table to the backend.
+    update_mem_table(vu, mem)?;
+
+    let mut vu_interrupt_list = Vec::new();
+
+    for (queue_index, queue) in queues.into_iter().enumerate() {
+        let actual_size: usize = queue.actual_size().try_into().unwrap();
+
+        vu.set_vring_num(queue_index, queue.actual_size())
+            .map_err(Error::VhostUserSetVringNum)?;
+
+        let config_data = VringConfigData {
+            queue_max_size: queue.get_max_size(),
+            queue_size: queue.actual_size(),
+            flags: 0u32,
+            desc_table_addr: get_host_address_range(
+                mem,
+                queue.desc_table,
+                actual_size * std::mem::size_of::<Descriptor>(),
+            )
+            .ok_or_else(|| Error::DescriptorTableAddress)? as u64,
+            // The used ring is {flags: u16; idx: u16; virtq_used_elem [{id: u16, len: u16}; actual_size]},
+            // i.e. 4 + (4 + 4) * actual_size.
+            used_ring_addr: get_host_address_range(mem, queue.used_ring, 4 + actual_size * 8)
+                .ok_or_else(|| Error::UsedAddress)? as u64,
+            // The used ring is {flags: u16; idx: u16; elem [u16; actual_size]},
+            // i.e. 4 + (2) * actual_size.
+            avail_ring_addr: get_host_address_range(mem, queue.avail_ring, 4 + actual_size * 2)
+                .ok_or_else(|| Error::AvailAddress)? as u64,
+            log_addr: None,
+        };
+
+        vu.set_vring_addr(queue_index, &config_data)
+            .map_err(Error::VhostUserSetVringAddr)?;
+        vu.set_vring_base(queue_index, 0u16)
+            .map_err(Error::VhostUserSetVringBase)?;
+
+        if let Some(eventfd) = virtio_interrupt.notifier(&VirtioInterruptType::Queue, Some(&queue))
+        {
+            vu.set_vring_call(queue_index, &eventfd)
+                .map_err(Error::VhostUserSetVringCall)?;
+            vu_interrupt_list.push((None, queue));
+        } else {
+            let eventfd = EventFd::new(EFD_NONBLOCK).map_err(Error::VhostIrqCreate)?;
+            vu.set_vring_call(queue_index, &eventfd)
+                .map_err(Error::VhostUserSetVringCall)?;
+            vu_interrupt_list.push((Some(eventfd), queue));
+        }
+
+        vu.set_vring_kick(queue_index, &queue_evts[queue_index])
+            .map_err(Error::VhostUserSetVringKick)?;
+
+        vu.set_vring_enable(queue_index, true)
+            .map_err(Error::VhostUserSetVringEnable)?;
+    }
+
+    Ok(vu_interrupt_list)
+}
+
+pub fn setup_vhost_user(
+    vu: &mut Master,
+    mem: &GuestMemoryMmap,
+    queues: Vec<Queue>,
+    queue_evts: Vec<EventFd>,
+    virtio_interrupt: &Arc<dyn VirtioInterrupt>,
+    acked_features: u64,
+) -> Result<Vec<(Option<EventFd>, Queue)>> {
+    // Set features based on the acked features from the guest driver.
+    vu.set_features(acked_features)
+        .map_err(Error::VhostUserSetFeatures)?;
+
+    setup_vhost_user_vring(vu, mem, queues, queue_evts, virtio_interrupt)
+}
+
+pub fn reset_vhost_user(vu: &mut Master, num_queues: usize) -> Result<()> {
+    for queue_index in 0..num_queues {
+        // Disable the vrings.
+        vu.set_vring_enable(queue_index, false)
+            .map_err(Error::VhostUserSetVringEnable)?;
+
+        // Stop the vrings.
+        vu.get_vring_base(queue_index)
+            .map_err(Error::VhostUserSetFeatures)?;
+    }
+
+    // Reset the owner.
+    vu.reset_owner().map_err(Error::VhostUserResetOwner)
+}