build: Bump version to 0.5.1

Signed-off-by: Rob Bradford <robert.bradford@intel.com>

Cargo.toml

@@ -1,39 +1,40 @@
 [package]
 name = "cloud-hypervisor"
-version = "0.4.0"
+version = "0.5.1"
 authors = ["The Cloud Hypervisor Authors"]
 edition = "2018"
 default-run = "cloud-hypervisor"
 
 [dependencies]
+arc-swap = "0.4.4"
 clap = "2.33.0"
-epoll = ">=4.0.1"
+epoll = "4.1.0"
 lazy_static = "1.4.0"
 libc = "0.2.66"
 log = { version = "0.4.8", features = ["std"] }
-net_gen = { path = "net_gen" }
-net_util = { path = "net_util" }
-qcow = { path = "qcow" }
 vhost_user_backend = { path = "vhost_user_backend"}
+vhost_user_block = { path = "vhost_user_block"}
 vhost_user_fs = { path = "vhost_user_fs"}
+vhost_user_net = { path = "vhost_user_net"}
 virtio-bindings = "0.1.0"
 vmm = { path = "vmm" }
 vm-device = { path = "vm-device" }
 vm-memory = { git = "https://github.com/rust-vmm/vm-memory" }
-vmm-sys-util = ">=0.3.1"
+vmm-sys-util = "0.4.0"
 vm-virtio = { path = "vm-virtio" }
 
-[dev-dependencies]
-ssh2 = "0.6.0"
-dirs = "2.0.2"
-credibility = "0.1.3"
-tempdir= "0.3.7"
-lazy_static= "1.4.0"
-
 [dependencies.vhost_rs]
 path = "vhost_rs"
 features = ["vhost-user-slave"]
 
+[dev-dependencies]
+ssh2 = "0.7.1"
+dirs = "2.0.2"
+credibility = "0.1.3"
+tempdir= "0.3.7"
+lazy_static= "1.4.0"
+serde_json = "1.0.48"
+
 [features]
 default = ["acpi", "pci", "cmos"]
 acpi = ["vmm/acpi"]
@@ -46,5 +47,22 @@ integration_tests = []
 
 [workspace]
 members = [
+    "arch",
+    "devices",
+    "vhost_rs",
+    "qcow",
+    "pci",
+    "vmm",
+    "vm-virtio",
+    "vm-device",
+    "vhost_user_block",
     "vhost_user_backend",
+    "vhost_user_fs",
+    "vhost_user_net",
+    "vfio",
+    "net_util",
+    "acpi_tables",
+    "arch_gen",
+    "net_gen",
+    "vm-allocator",
 ]

README.md

@@ -86,6 +86,34 @@ $ popd
 
 This will build a `cloud-hypervisor` binary under `$CLOUDH/cloud-hypervisor/target/release/cloud-hypervisor`.
 
+### Containerized builds and tests
+
+If you want to build and test Cloud Hypervisor without having to install all the
+required dependencies (The rust toolchain, cargo tools, etc), you can also use
+Cloud Hypervisor's development script: `dev_cli.sh`. Please note that upon its
+first invocation, this script will pull a fairly large container image.
+
+For example, to build the Cloud Hypervisor release binary:
+
+```shell
+$ pushd $CLOUDH
+$ cd cloud-hypervisor
+$ ./scripts/dev_cli.sh build --release
+```
+
+With `dev_cli.sh`, one can also run the Cloud Hypervisor CI locally. This can be
+very convenient for debugging CI errors without having to fully rely on the
+Cloud Hypervisor CI infrastructure.
+
+For example, to run the Cloud Hypervisor unit tests:
+
+```shell
+$ ./scripts/dev_cli.sh tests --unit
+```
+
+Run the `./scripts/dev_cli.sh --help` command to view all the supported
+development script commands and their related options.
+
 ## Run
 
 You can run a guest VM by either using an existing cloud image or booting into your own kernel and disk image.
@@ -129,18 +157,19 @@ Multiple arguments can be given to the `--disk` parameter.
 #### Building your kernel
 
 `cloud-hypervisor` also supports direct kernel boot into a `vmlinux` ELF kernel
-image. You want to build such an image first:
+image. In order to support virtio-fs and virtio-iommu we have our own development branch. You are of course able to use your own kernel but these instructions will continue with the version that we develop and test against.
+
+To build the kernel:
 
 ```shell
 
-# Clone a 5.0 Linux kernel
+# Clone the Cloud Hypervisor Linux branch
 $ pushd $CLOUDH
-$ git clone git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git linux-cloud-hypervisor
-$ cd linux-cloud-hypervisor
-$ git reset --hard v5.0
+$ git clone --depth 1 https://github.com/cloud-hypervisor/linux.git -b virtio-fs-virtio-iommu-5.5-rc1 linux-cloud-hypervisor
+$ pushd linux-cloud-hypervisor
 
 # Use the cloud-hypervisor kernel config to build your kernel
-$ cp $CLOUDH/cloud-hypervisor/resources/linux-5.0-config .config
+$ cp $CLOUDH/cloud-hypervisor/resources/linux-virtio-fs-virtio-iommu-config .config
 $ make bzImage -j `nproc`
 $ popd
 ```

acpi_tables/src/aml.rs

@@ -3,6 +3,8 @@
 // SPDX-License-Identifier: Apache-2.0
 //
 
+use std::marker::PhantomData;
+
 pub trait Aml {
     fn to_aml_bytes(&self) -> Vec<u8>;
 }
@@ -1158,6 +1160,46 @@ impl Aml for Buffer {
     }
 }
 
+pub struct CreateField<'a, T> {
+    buffer: &'a dyn Aml,
+    offset: &'a dyn Aml,
+    field: Path,
+    phantom: PhantomData<&'a T>,
+}
+
+impl<'a, T> CreateField<'a, T> {
+    pub fn new(buffer: &'a dyn Aml, offset: &'a dyn Aml, field: Path) -> Self {
+        CreateField::<T> {
+            buffer,
+            offset,
+            field,
+            phantom: PhantomData::default(),
+        }
+    }
+}
+
+impl<'a> Aml for CreateField<'a, u64> {
+    fn to_aml_bytes(&self) -> Vec<u8> {
+        let mut bytes = Vec::new();
+        bytes.push(0x8f); /* CreateQWordFieldOp */
+        bytes.extend_from_slice(&self.buffer.to_aml_bytes());
+        bytes.extend_from_slice(&self.offset.to_aml_bytes());
+        bytes.extend_from_slice(&self.field.to_aml_bytes());
+        bytes
+    }
+}
+
+impl<'a> Aml for CreateField<'a, u32> {
+    fn to_aml_bytes(&self) -> Vec<u8> {
+        let mut bytes = Vec::new();
+        bytes.push(0x8a); /* CreateDWordFieldOp */
+        bytes.extend_from_slice(&self.buffer.to_aml_bytes());
+        bytes.extend_from_slice(&self.offset.to_aml_bytes());
+        bytes.extend_from_slice(&self.field.to_aml_bytes());
+        bytes
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -1888,4 +1930,59 @@ mod tests {
             &buffer_data[..]
         )
     }
+
+    #[test]
+    fn test_create_field() {
+        /*
+        Method (MCRS, 0, Serialized)
+        {
+            Name (MR64, ResourceTemplate ()
+            {
+                QWordMemory (ResourceProducer, PosDecode, MinFixed, MaxFixed, Cacheable, ReadWrite,
+                    0x0000000000000000, // Granularity
+                    0x0000000000000000, // Range Minimum
+                    0xFFFFFFFFFFFFFFFE, // Range Maximum
+                    0x0000000000000000, // Translation Offset
+                    0xFFFFFFFFFFFFFFFF, // Length
+                    ,, _Y00, AddressRangeMemory, TypeStatic)
+            })
+            CreateQWordField (MR64, \_SB.MHPC.MCRS._Y00._MIN, MIN)  // _MIN: Minimum Base Address
+            CreateQWordField (MR64, \_SB.MHPC.MCRS._Y00._MAX, MAX)  // _MAX: Maximum Base Address
+            CreateQWordField (MR64, \_SB.MHPC.MCRS._Y00._LEN, LEN)  // _LEN: Length
+        }
+        */
+        let data = [
+            0x14, 0x41, 0x06, 0x4D, 0x43, 0x52, 0x53, 0x08, 0x08, 0x4D, 0x52, 0x36, 0x34, 0x11,
+            0x33, 0x0A, 0x30, 0x8A, 0x2B, 0x00, 0x00, 0x0C, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00,
+            0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xFE, 0xFF, 0xFF,
+            0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xFF,
+            0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x79, 0x00, 0x8F, 0x4D, 0x52, 0x36, 0x34,
+            0x0A, 0x0E, 0x4D, 0x49, 0x4E, 0x5F, 0x8F, 0x4D, 0x52, 0x36, 0x34, 0x0A, 0x16, 0x4D,
+            0x41, 0x58, 0x5F, 0x8F, 0x4D, 0x52, 0x36, 0x34, 0x0A, 0x26, 0x4C, 0x45, 0x4E, 0x5F,
+        ];
+
+        assert_eq!(
+            Method::new(
+                "MCRS".into(),
+                0,
+                true,
+                vec![
+                    &Name::new(
+                        "MR64".into(),
+                        &ResourceTemplate::new(vec![&AddressSpace::new_memory(
+                            AddressSpaceCachable::Cacheable,
+                            true,
+                            0x0000_0000_0000_0000u64,
+                            0xFFFF_FFFF_FFFF_FFFEu64
+                        )])
+                    ),
+                    &CreateField::<u64>::new(&Path::new("MR64"), &14usize, "MIN_".into()),
+                    &CreateField::<u64>::new(&Path::new("MR64"), &22usize, "MAX_".into()),
+                    &CreateField::<u64>::new(&Path::new("MR64"), &38usize, "LEN_".into()),
+                ]
+            )
+            .to_aml_bytes(),
+            &data[..]
+        );
+    }
 }

arch/Cargo.toml

@@ -7,10 +7,10 @@ authors = ["The Chromium OS Authors"]
 default = []
 
 [dependencies]
-byteorder = "1.3.2"
+byteorder = "1.3.4"
 kvm-bindings = "0.2.0"
-kvm-ioctls = "0.4.0"
-libc = "0.2.60"
+kvm-ioctls = "0.5.0"
+libc = "0.2.66"
 
 acpi_tables = { path = "../acpi_tables", optional = true }
 arch_gen = { path = "../arch_gen" }
@@ -24,4 +24,4 @@ git = "https://github.com/rust-vmm/linux-loader"
 features = ["elf", "bzimage"]
 
 [dev-dependencies]
-rand = "0.7.0"
+rand = "0.7.3"

arch/src/lib.rs

@@ -22,7 +22,7 @@ extern crate vm_memory;
 
 use std::result;
 
-#[derive(Debug, PartialEq)]
+#[derive(Debug)]
 pub enum Error {
     #[cfg(target_arch = "x86_64")]
     /// X86_64 specific error triggered during system configuration.
@@ -30,7 +30,7 @@ pub enum Error {
     /// The zero page extends past the end of guest_mem.
     ZeroPagePastRamEnd,
     /// Error writing the zero page of guest memory.
-    ZeroPageSetup,
+    ZeroPageSetup(vm_memory::GuestMemoryError),
 }
 pub type Result<T> = result::Result<T, Error>;
 

arch/src/x86_64/mod.rs

@@ -32,7 +32,7 @@ struct BootParamsWrapper(boot_params);
 // It is safe to initialize BootParamsWrap which is a wrapper over `boot_params` (a series of ints).
 unsafe impl ByteValued for BootParamsWrapper {}
 
-#[derive(Debug, PartialEq)]
+#[derive(Debug)]
 pub enum Error {
     /// Invalid e820 setup params.
     E820Configuration,
@@ -135,7 +135,7 @@ pub fn configure_system(
 
     add_e820_entry(&mut params.0, 0, layout::EBDA_START.raw_value(), E820_RAM)?;
 
-    let mem_end = guest_mem.end_addr();
+    let mem_end = guest_mem.last_addr();
     if mem_end < layout::MEM_32BIT_RESERVED_START {
         add_e820_entry(
             &mut params.0,
@@ -177,7 +177,7 @@ pub fn configure_system(
         .ok_or(super::Error::ZeroPagePastRamEnd)?;
     guest_mem
         .write_obj(params, zero_page_addr)
-        .map_err(|_| super::Error::ZeroPageSetup)?;
+        .map_err(super::Error::ZeroPageSetup)?;
 
     Ok(())
 }
@@ -226,15 +226,9 @@ mod tests {
     #[test]
     fn test_system_configuration() {
         let no_vcpus = 4;
-        let gm = GuestMemoryMmap::new(&vec![(GuestAddress(0), 0x10000)]).unwrap();
+        let gm = GuestMemoryMmap::from_ranges(&vec![(GuestAddress(0), 0x10000)]).unwrap();
         let config_err = configure_system(&gm, GuestAddress(0), 0, 1, None, None);
         assert!(config_err.is_err());
-        assert_eq!(
-            config_err.unwrap_err(),
-            super::super::Error::X86_64Setup(super::Error::MpTableSetup(
-                mptable::Error::NotEnoughMemory
-            ))
-        );
 
         // Now assigning some memory that falls before the 32bit memory hole.
         let mem_size = 128 << 20;
@@ -244,7 +238,7 @@ mod tests {
             .filter(|r| r.2 == RegionType::Ram)
             .map(|r| (r.0, r.1))
             .collect();
-        let gm = GuestMemoryMmap::new(&ram_regions).unwrap();
+        let gm = GuestMemoryMmap::from_ranges(&ram_regions).unwrap();
         configure_system(&gm, GuestAddress(0), 0, no_vcpus, None, None).unwrap();
 
         // Now assigning some memory that is equal to the start of the 32bit memory hole.
@@ -255,7 +249,7 @@ mod tests {
             .filter(|r| r.2 == RegionType::Ram)
             .map(|r| (r.0, r.1))
             .collect();
-        let gm = GuestMemoryMmap::new(&ram_regions).unwrap();
+        let gm = GuestMemoryMmap::from_ranges(&ram_regions).unwrap();
         configure_system(&gm, GuestAddress(0), 0, no_vcpus, None, None).unwrap();
 
         // Now assigning some memory that falls after the 32bit memory hole.
@@ -266,7 +260,7 @@ mod tests {
             .filter(|r| r.2 == RegionType::Ram)
             .map(|r| (r.0, r.1))
             .collect();
-        let gm = GuestMemoryMmap::new(&ram_regions).unwrap();
+        let gm = GuestMemoryMmap::from_ranges(&ram_regions).unwrap();
         configure_system(&gm, GuestAddress(0), 0, no_vcpus, None, None).unwrap();
     }
 

arch/src/x86_64/mptable.rs

@@ -14,7 +14,7 @@ use libc::c_char;
 
 use arch_gen::x86::mpspec;
 use layout::{APIC_START, IOAPIC_START, MPTABLE_START};
-use vm_memory::{Address, ByteValued, Bytes, GuestMemory, GuestMemoryMmap};
+use vm_memory::{Address, ByteValued, Bytes, GuestMemory, GuestMemoryError, GuestMemoryMmap};
 
 // This is a workaround to the Rust enforcement specifying that any implementation of a foreign
 // trait (in this case `ByteValued`) where:
@@ -45,30 +45,30 @@ unsafe impl ByteValued for MpcTableWrapper {}
 unsafe impl ByteValued for MpcLintsrcWrapper {}
 unsafe impl ByteValued for MpfIntelWrapper {}
 
-#[derive(Debug, PartialEq)]
+#[derive(Debug)]
 pub enum Error {
     /// There was too little guest memory to store the entire MP table.
     NotEnoughMemory,
     /// The MP table has too little address space to be stored.
     AddressOverflow,
     /// Failure while zeroing out the memory for the MP table.
-    Clear,
+    Clear(GuestMemoryError),
     /// Number of CPUs exceeds the maximum supported CPUs
     TooManyCpus,
     /// Failure to write the MP floating pointer.
-    WriteMpfIntel,
+    WriteMpfIntel(GuestMemoryError),
     /// Failure to write MP CPU entry.
-    WriteMpcCpu,
+    WriteMpcCpu(GuestMemoryError),
     /// Failure to write MP ioapic entry.
-    WriteMpcIoapic,
+    WriteMpcIoapic(GuestMemoryError),
     /// Failure to write MP bus entry.
-    WriteMpcBus,
+    WriteMpcBus(GuestMemoryError),
     /// Failure to write MP interrupt source entry.
-    WriteMpcIntsrc,
+    WriteMpcIntsrc(GuestMemoryError),
     /// Failure to write MP local interrupt source entry.
-    WriteMpcLintsrc,
+    WriteMpcLintsrc(GuestMemoryError),
     /// Failure to write MP table header.
-    WriteMpcTable,
+    WriteMpcTable(GuestMemoryError),
 }
 
 pub type Result<T> = result::Result<T, Error>;
@@ -145,7 +145,7 @@ pub fn setup_mptable(mem: &GuestMemoryMmap, num_cpus: u8) -> Result<()> {
     }
 
     mem.read_exact_from(base_mp, &mut io::repeat(0), mp_size)
-        .map_err(|_| Error::Clear)?;
+        .map_err(Error::Clear)?;
 
     {
         let mut mpf_intel = MpfIntelWrapper(mpspec::mpf_intel::default());
@@ -156,7 +156,7 @@ pub fn setup_mptable(mem: &GuestMemoryMmap, num_cpus: u8) -> Result<()> {
         mpf_intel.0.physptr = (base_mp.raw_value() + size) as u32;
         mpf_intel.0.checksum = mpf_intel_compute_checksum(&mpf_intel.0);
         mem.write_obj(mpf_intel, base_mp)
-            .map_err(|_| Error::WriteMpfIntel)?;
+            .map_err(Error::WriteMpfIntel)?;
         base_mp = base_mp.unchecked_add(size);
     }
 
@@ -181,7 +181,7 @@ pub fn setup_mptable(mem: &GuestMemoryMmap, num_cpus: u8) -> Result<()> {
             mpc_cpu.0.cpufeature = CPU_STEPPING;
             mpc_cpu.0.featureflag = CPU_FEATURE_APIC | CPU_FEATURE_FPU;
             mem.write_obj(mpc_cpu, base_mp)
-                .map_err(|_| Error::WriteMpcCpu)?;
+                .map_err(Error::WriteMpcCpu)?;
             base_mp = base_mp.unchecked_add(size as u64);
             checksum = checksum.wrapping_add(compute_checksum(&mpc_cpu.0));
         }
@@ -193,7 +193,7 @@ pub fn setup_mptable(mem: &GuestMemoryMmap, num_cpus: u8) -> Result<()> {
         mpc_bus.0.busid = 0;
         mpc_bus.0.bustype = BUS_TYPE_ISA;
         mem.write_obj(mpc_bus, base_mp)
-            .map_err(|_| Error::WriteMpcBus)?;
+            .map_err(Error::WriteMpcBus)?;
         base_mp = base_mp.unchecked_add(size as u64);
         checksum = checksum.wrapping_add(compute_checksum(&mpc_bus.0));
     }
@@ -206,7 +206,7 @@ pub fn setup_mptable(mem: &GuestMemoryMmap, num_cpus: u8) -> Result<()> {
         mpc_ioapic.0.flags = mpspec::MPC_APIC_USABLE as u8;
         mpc_ioapic.0.apicaddr = IOAPIC_START.0 as u32;
         mem.write_obj(mpc_ioapic, base_mp)
-            .map_err(|_| Error::WriteMpcIoapic)?;
+            .map_err(Error::WriteMpcIoapic)?;
         base_mp = base_mp.unchecked_add(size as u64);
         checksum = checksum.wrapping_add(compute_checksum(&mpc_ioapic.0));
     }
@@ -222,7 +222,7 @@ pub fn setup_mptable(mem: &GuestMemoryMmap, num_cpus: u8) -> Result<()> {
         mpc_intsrc.0.dstapic = ioapicid;
         mpc_intsrc.0.dstirq = i;
         mem.write_obj(mpc_intsrc, base_mp)
-            .map_err(|_| Error::WriteMpcIntsrc)?;
+            .map_err(Error::WriteMpcIntsrc)?;
         base_mp = base_mp.unchecked_add(size as u64);
         checksum = checksum.wrapping_add(compute_checksum(&mpc_intsrc.0));
     }
@@ -237,7 +237,7 @@ pub fn setup_mptable(mem: &GuestMemoryMmap, num_cpus: u8) -> Result<()> {
         mpc_lintsrc.0.destapic = 0;
         mpc_lintsrc.0.destapiclint = 0;
         mem.write_obj(mpc_lintsrc, base_mp)
-            .map_err(|_| Error::WriteMpcLintsrc)?;
+            .map_err(Error::WriteMpcLintsrc)?;
         base_mp = base_mp.unchecked_add(size as u64);
         checksum = checksum.wrapping_add(compute_checksum(&mpc_lintsrc.0));
     }
@@ -252,7 +252,7 @@ pub fn setup_mptable(mem: &GuestMemoryMmap, num_cpus: u8) -> Result<()> {
         mpc_lintsrc.0.destapic = 0xFF; /* to all local APICs */
         mpc_lintsrc.0.destapiclint = 1;
         mem.write_obj(mpc_lintsrc, base_mp)
-            .map_err(|_| Error::WriteMpcLintsrc)?;
+            .map_err(Error::WriteMpcLintsrc)?;
         base_mp = base_mp.unchecked_add(size as u64);
         checksum = checksum.wrapping_add(compute_checksum(&mpc_lintsrc.0));
     }
@@ -271,7 +271,7 @@ pub fn setup_mptable(mem: &GuestMemoryMmap, num_cpus: u8) -> Result<()> {
         checksum = checksum.wrapping_add(compute_checksum(&mpc_table.0));
         mpc_table.0.checksum = (!checksum).wrapping_add(1) as i8;
         mem.write_obj(mpc_table, table_base)
-            .map_err(|_| Error::WriteMpcTable)?;
+            .map_err(Error::WriteMpcTable)?;
     }
 
     Ok(())
@@ -296,7 +296,8 @@ mod tests {
     #[test]
     fn bounds_check() {
         let num_cpus = 4;
-        let mem = GuestMemoryMmap::new(&[(MPTABLE_START, compute_mp_size(num_cpus))]).unwrap();
+        let mem =
+            GuestMemoryMmap::from_ranges(&[(MPTABLE_START, compute_mp_size(num_cpus))]).unwrap();
 
         setup_mptable(&mem, num_cpus).unwrap();
     }
@@ -304,7 +305,8 @@ mod tests {
     #[test]
     fn bounds_check_fails() {
         let num_cpus = 4;
-        let mem = GuestMemoryMmap::new(&[(MPTABLE_START, compute_mp_size(num_cpus) - 1)]).unwrap();
+        let mem = GuestMemoryMmap::from_ranges(&[(MPTABLE_START, compute_mp_size(num_cpus) - 1)])
+            .unwrap();
 
         assert!(setup_mptable(&mem, num_cpus).is_err());
     }
@@ -312,7 +314,8 @@ mod tests {
     #[test]
     fn mpf_intel_checksum() {
         let num_cpus = 1;
-        let mem = GuestMemoryMmap::new(&[(MPTABLE_START, compute_mp_size(num_cpus))]).unwrap();
+        let mem =
+            GuestMemoryMmap::from_ranges(&[(MPTABLE_START, compute_mp_size(num_cpus))]).unwrap();
 
         setup_mptable(&mem, num_cpus).unwrap();
 
@@ -327,7 +330,8 @@ mod tests {
     #[test]
     fn mpc_table_checksum() {
         let num_cpus = 4;
-        let mem = GuestMemoryMmap::new(&[(MPTABLE_START, compute_mp_size(num_cpus))]).unwrap();
+        let mem =
+            GuestMemoryMmap::from_ranges(&[(MPTABLE_START, compute_mp_size(num_cpus))]).unwrap();
 
         setup_mptable(&mem, num_cpus).unwrap();
 
@@ -356,9 +360,11 @@ mod tests {
 
     #[test]
     fn cpu_entry_count() {
-        let mem =
-            GuestMemoryMmap::new(&[(MPTABLE_START, compute_mp_size(MAX_SUPPORTED_CPUS as u8))])
-                .unwrap();
+        let mem = GuestMemoryMmap::from_ranges(&[(
+            MPTABLE_START,
+            compute_mp_size(MAX_SUPPORTED_CPUS as u8),
+        )])
+        .unwrap();
 
         for i in 0..MAX_SUPPORTED_CPUS as u8 {
             setup_mptable(&mem, i).unwrap();
@@ -391,9 +397,10 @@ mod tests {
     #[test]
     fn cpu_entry_count_max() {
         let cpus = MAX_SUPPORTED_CPUS + 1;
-        let mem = GuestMemoryMmap::new(&[(MPTABLE_START, compute_mp_size(cpus as u8))]).unwrap();
+        let mem =
+            GuestMemoryMmap::from_ranges(&[(MPTABLE_START, compute_mp_size(cpus as u8))]).unwrap();
 
-        let result = setup_mptable(&mem, cpus as u8).unwrap_err();
-        assert_eq!(result, Error::TooManyCpus);
+        let result = setup_mptable(&mem, cpus as u8);
+        assert!(result.is_err());
     }
 }

arch/src/x86_64/regs.rs

@@ -12,7 +12,7 @@ use arch_gen::x86::msr_index;
 use kvm_bindings::{kvm_fpu, kvm_msr_entry, kvm_regs, kvm_sregs, Msrs};
 use kvm_ioctls::VcpuFd;
 use layout::{BOOT_GDT_START, BOOT_IDT_START, PDE_START, PDPTE_START, PML4_START};
-use vm_memory::{Address, Bytes, GuestMemory, GuestMemoryMmap};
+use vm_memory::{Address, Bytes, GuestMemory, GuestMemoryError, GuestMemoryMmap};
 
 // MTRR constants
 const MTRR_ENABLE: u64 = 0x800; // IA32_MTRR_DEF_TYPE MSR: E (MTRRs enabled) flag, bit 11
@@ -30,16 +30,18 @@ pub enum Error {
     SetModelSpecificRegisters(kvm_ioctls::Error),
     /// Failed to set SREGs for this CPU.
     SetStatusRegisters(kvm_ioctls::Error),
+    /// Checking the GDT address failed.
+    CheckGDTAddr,
     /// Writing the GDT to RAM failed.
-    WriteGDT,
+    WriteGDT(GuestMemoryError),
     /// Writing the IDT to RAM failed.
-    WriteIDT,
+    WriteIDT(GuestMemoryError),
     /// Writing PDPTE to RAM failed.
-    WritePDPTEAddress,
+    WritePDPTEAddress(GuestMemoryError),
     /// Writing PDE to RAM failed.
-    WritePDEAddress,
+    WritePDEAddress(GuestMemoryError),
     /// Writing PML4 to RAM failed.
-    WritePML4Address,
+    WritePML4Address(GuestMemoryError),
 }
 
 pub type Result<T> = result::Result<T, Error>;
@@ -121,10 +123,8 @@ fn write_gdt_table(table: &[u64], guest_mem: &GuestMemoryMmap) -> Result<()> {
     for (index, entry) in table.iter().enumerate() {
         let addr = guest_mem
             .checked_offset(boot_gdt_addr, index * mem::size_of::<u64>())
-            .ok_or(Error::WriteGDT)?;
-        guest_mem
-            .write_obj(*entry, addr)
-            .map_err(|_| Error::WriteGDT)?;
+            .ok_or(Error::CheckGDTAddr)?;
+        guest_mem.write_obj(*entry, addr).map_err(Error::WriteGDT)?;
     }
     Ok(())
 }
@@ -133,7 +133,7 @@ fn write_idt_value(val: u64, guest_mem: &GuestMemoryMmap) -> Result<()> {
     let boot_idt_addr = BOOT_IDT_START;
     guest_mem
         .write_obj(val, boot_idt_addr)
-        .map_err(|_| Error::WriteIDT)
+        .map_err(Error::WriteIDT)
 }
 
 fn configure_segments_and_sregs(mem: &GuestMemoryMmap, sregs: &mut kvm_sregs) -> Result<()> {
@@ -177,16 +177,16 @@ fn setup_page_tables(mem: &GuestMemoryMmap, sregs: &mut kvm_sregs) -> Result<()>
 
     // Entry covering VA [0..512GB)
     mem.write_obj(PDPTE_START.raw_value() | 0x03, PML4_START)
-        .map_err(|_| Error::WritePML4Address)?;
+        .map_err(Error::WritePML4Address)?;
 
     // Entry covering VA [0..1GB)
     mem.write_obj(PDE_START.raw_value() | 0x03, PDPTE_START)
-        .map_err(|_| Error::WritePDPTEAddress)?;
+        .map_err(Error::WritePDPTEAddress)?;
     // 512 2MB entries together covering VA [0..1GB). Note we are assuming
     // CPU supports 2MB pages (/proc/cpuinfo has 'pse'). All modern CPUs do.
     for i in 0..512 {
         mem.write_obj((i << 21) + 0x83u64, PDE_START.unchecked_add(i * 8))
-            .map_err(|_| Error::WritePDEAddress)?;
+            .map_err(Error::WritePDEAddress)?;
     }
 
     sregs.cr3 = PML4_START.raw_value();
@@ -269,7 +269,7 @@ mod tests {
     use vm_memory::{GuestAddress, GuestMemoryMmap};
 
     fn create_guest_mem() -> GuestMemoryMmap {
-        GuestMemoryMmap::new(&vec![(GuestAddress(0), 0x10000)]).unwrap()
+        GuestMemoryMmap::from_ranges(&vec![(GuestAddress(0), 0x10000)]).unwrap()
     }
 
     fn read_u64(gm: &GuestMemoryMmap, offset: GuestAddress) -> u64 {

arch_gen/src/x86/mod.rs

@@ -16,7 +16,6 @@
     clippy::should_implement_trait,
     clippy::transmute_ptr_to_ptr
 )]
-pub mod bootparam;
 #[allow(non_camel_case_types)]
 #[allow(non_upper_case_globals)]
 #[allow(clippy::unreadable_literal, clippy::redundant_static_lifetimes)]

devices/Cargo.toml

@@ -4,14 +4,14 @@ version = "0.1.0"
 authors = ["The Chromium OS Authors"]
 
 [dependencies]
-byteorder = "1.3.2"
-epoll = ">=4.0.1"
-kvm-bindings = "0.2.0"
-kvm-ioctls = "0.4.0"
-libc = "0.2.60"
+bitflags = "1.2.1"
+byteorder = "1.3.4"
+epoll = "4.1.0"
+libc = "0.2.66"
 log = "0.4.8"
+vm-device = { path = "../vm-device" }
 vm-memory = { git = "https://github.com/rust-vmm/vm-memory" }
-vmm-sys-util = ">=0.3.1"
+vmm-sys-util = "0.4.0"
 
 [dev-dependencies]
 tempfile = "3.1.0"

devices/src/acpi.rs

@@ -3,10 +3,11 @@
 // SPDX-License-Identifier: Apache-2.0
 //
 
+use std::sync::Arc;
+use vm_device::interrupt::InterruptSourceGroup;
 use vmm_sys_util::eventfd::EventFd;
 use BusDevice;
-use HotPlugNotificationType;
-use Interrupt;
+use HotPlugNotificationFlags;
 
 /// A device for handling ACPI shutdown and reboot
 pub struct AcpiShutdownDevice {
@@ -46,6 +47,7 @@ impl BusDevice for AcpiShutdownDevice {
         const SLEEP_VALUE_BIT: u8 = 2;
         if data[0] == (S5_SLEEP_VALUE << SLEEP_VALUE_BIT) | (1 << SLEEP_STATUS_EN_BIT) {
             debug!("ACPI Shutdown signalled");
+            extern crate bitflags;
             if let Err(e) = self.exit_evt.write(1) {
                 error!("Error triggering ACPI shutdown event: {}", e);
             }
@@ -55,26 +57,26 @@ impl BusDevice for AcpiShutdownDevice {
 
 /// A device for handling ACPI GED event generation
 pub struct AcpiGEDDevice {
-    interrupt: Box<dyn Interrupt>,
-    notification_type: HotPlugNotificationType,
+    interrupt: Arc<Box<dyn InterruptSourceGroup>>,
+    notification_type: HotPlugNotificationFlags,
     ged_irq: u32,
 }
 
 impl AcpiGEDDevice {
-    pub fn new(interrupt: Box<dyn Interrupt>, ged_irq: u32) -> AcpiGEDDevice {
+    pub fn new(interrupt: Arc<Box<dyn InterruptSourceGroup>>, ged_irq: u32) -> AcpiGEDDevice {
         AcpiGEDDevice {
             interrupt,
-            notification_type: HotPlugNotificationType::NoDevicesChanged,
+            notification_type: HotPlugNotificationFlags::NO_DEVICES_CHANGED,
             ged_irq,
         }
     }
 
     pub fn notify(
         &mut self,
-        notification_type: HotPlugNotificationType,
+        notification_type: HotPlugNotificationFlags,
     ) -> Result<(), std::io::Error> {
-        self.notification_type = notification_type;
-        self.interrupt.deliver()
+        self.notification_type |= notification_type;
+        self.interrupt.trigger(0)
     }
 
     pub fn irq(&self) -> u32 {
@@ -86,8 +88,8 @@ impl AcpiGEDDevice {
 impl BusDevice for AcpiGEDDevice {
     // Spec has all fields as zero
     fn read(&mut self, _base: u64, _offset: u64, data: &mut [u8]) {
-        data[0] = self.notification_type as u8;
-        self.notification_type = HotPlugNotificationType::NoDevicesChanged;
+        data[0] = self.notification_type.bits();
+        self.notification_type = HotPlugNotificationFlags::NO_DEVICES_CHANGED;
     }
 
     fn write(&mut self, _base: u64, _offset: u64, _data: &[u8]) {}

devices/src/ioapic.rs

@@ -11,22 +11,35 @@
 
 use crate::BusDevice;
 use byteorder::{ByteOrder, LittleEndian};
-use kvm_bindings::kvm_msi;
-use kvm_ioctls::VmFd;
+use std::io;
 use std::result;
 use std::sync::Arc;
+use vm_device::interrupt::{
+    InterruptIndex, InterruptManager, InterruptSourceConfig, InterruptSourceGroup,
+    MsiIrqGroupConfig, MsiIrqSourceConfig,
+};
 use vm_memory::GuestAddress;
 
 #[derive(Debug)]
 pub enum Error {
-    /// Failed to send an interrupt.
-    InterruptFailed(kvm_ioctls::Error),
     /// Invalid destination mode.
     InvalidDestinationMode,
     /// Invalid trigger mode.
     InvalidTriggerMode,
     /// Invalid delivery mode.
     InvalidDeliveryMode,
+    /// Failed creating the interrupt source group.
+    CreateInterruptSourceGroup(io::Error),
+    /// Failed triggering the interrupt.
+    TriggerInterrupt(io::Error),
+    /// Failed masking the interrupt.
+    MaskInterrupt(io::Error),
+    /// Failed unmasking the interrupt.
+    UnmaskInterrupt(io::Error),
+    /// Failed updating the interrupt.
+    UpdateInterrupt(io::Error),
+    /// Failed enabling the interrupt.
+    EnableInterrupt(io::Error),
 }
 
 type Result<T> = result::Result<T, Error>;
@@ -156,8 +169,8 @@ pub struct Ioapic {
     id: u32,
     reg_sel: u32,
     reg_entries: [RedirectionTableEntry; NUM_IOAPIC_PINS],
-    vm_fd: Arc<VmFd>,
     apic_address: GuestAddress,
+    interrupt_source_group: Arc<Box<dyn InterruptSourceGroup>>,
 }
 
 impl BusDevice for Ioapic {
@@ -196,14 +209,28 @@ impl BusDevice for Ioapic {
 }
 
 impl Ioapic {
-    pub fn new(vm_fd: Arc<VmFd>, apic_address: GuestAddress) -> Ioapic {
-        Ioapic {
+    pub fn new(
+        apic_address: GuestAddress,
+        interrupt_manager: Arc<dyn InterruptManager<GroupConfig = MsiIrqGroupConfig>>,
+    ) -> Result<Ioapic> {
+        let interrupt_source_group = interrupt_manager
+            .create_group(MsiIrqGroupConfig {
+                base: 0 as InterruptIndex,
+                count: NUM_IOAPIC_PINS as InterruptIndex,
+            })
+            .map_err(Error::CreateInterruptSourceGroup)?;
+
+        interrupt_source_group
+            .enable()
+            .map_err(Error::EnableInterrupt)?;
+
+        Ok(Ioapic {
             id: 0,
             reg_sel: 0,
             reg_entries: [0; NUM_IOAPIC_PINS],
-            vm_fd,
             apic_address,
-        }
+            interrupt_source_group,
+        })
     }
 
     // The ioapic must be informed about EOIs in order to deassert interrupts
@@ -223,16 +250,30 @@ impl Ioapic {
     pub fn service_irq(&mut self, irq: usize) -> Result<()> {
         let entry = &mut self.reg_entries[irq];
 
-        // Don't inject the interrupt if the IRQ is masked
-        if interrupt_mask(*entry) == 1 {
-            return Ok(());
+        self.interrupt_source_group
+            .trigger(irq as InterruptIndex)
+            .map_err(Error::TriggerInterrupt)?;
+        debug!("Interrupt successfully delivered");
+
+        // If trigger mode is level sensitive, set the Remote IRR bit.
+        // It will be cleared when the EOI is received.
+        if trigger_mode(*entry) == 1 {
+            set_remote_irr(entry, 1);
         }
+        // Clear the Delivery Status bit
+        set_delivery_status(entry, 0);
+
+        Ok(())
+    }
+
+    fn update_entry(&self, irq: usize) -> Result<()> {
+        let entry = self.reg_entries[irq];
 
         // Validate Destination Mode value, and retrieve Destination ID
-        let destination_mode = destination_mode(*entry);
+        let destination_mode = destination_mode(entry);
         let destination_id: u8 = match destination_mode {
-            x if x == DestinationMode::Physical as u8 => destination_field_physical(*entry),
-            x if x == DestinationMode::Logical as u8 => destination_field_logical(*entry),
+            x if x == DestinationMode::Physical as u8 => destination_field_physical(entry),
+            x if x == DestinationMode::Logical as u8 => destination_field_logical(entry),
             _ => return Err(Error::InvalidDestinationMode),
         };
 
@@ -242,20 +283,20 @@ impl Ioapic {
         let redirection_hint: u8 = 1;
 
         // Generate MSI message address
-        let address_lo: u32 = self.apic_address.0 as u32
+        let low_addr: u32 = self.apic_address.0 as u32
             | u32::from(destination_id) << 12
             | u32::from(redirection_hint) << 3
             | u32::from(destination_mode) << 2;
 
         // Validate Trigger Mode value
-        let trigger_mode = trigger_mode(*entry);
+        let trigger_mode = trigger_mode(entry);
         match trigger_mode {
             x if (x == TriggerMode::Edge as u8) || (x == TriggerMode::Level as u8) => {}
             _ => return Err(Error::InvalidTriggerMode),
         }
 
         // Validate Delivery Mode value
-        let delivery_mode = delivery_mode(*entry);
+        let delivery_mode = delivery_mode(entry);
         match delivery_mode {
             x if (x == DeliveryMode::Fixed as u8)
                 || (x == DeliveryMode::Lowest as u8)
@@ -270,37 +311,31 @@ impl Ioapic {
 
         // Generate MSI message data
         let data: u32 = u32::from(trigger_mode) << 15
-            | u32::from(remote_irr(*entry)) << 14
+            | u32::from(remote_irr(entry)) << 14
             | u32::from(delivery_mode) << 8
-            | u32::from(vector(*entry));
+            | u32::from(vector(entry));
 
-        let msi = kvm_msi {
-            address_lo,
-            address_hi: 0x0,
+        let config = MsiIrqSourceConfig {
+            high_addr: 0x0,
+            low_addr,
             data,
-            flags: 0u32,
-            devid: 0u32,
-            pad: [0u8; 12],
         };
 
-        match self.vm_fd.signal_msi(msi) {
-            Ok(ret) => {
-                if ret > 0 {
-                    debug!("MSI message successfully delivered");
-                    // If trigger mode is level sensitive, set the Remote IRR bit.
-                    // It will be cleared when the EOI is received.
-                    if trigger_mode == 1 {
-                        set_remote_irr(entry, 1);
-                    }
-                    // Clear the Delivery Status bit
-                    set_delivery_status(entry, 0);
-                } else {
-                    warn!("failed to deliver MSI message, blocked by guest");
-                }
-                Ok(())
-            }
-            Err(e) => Err(Error::InterruptFailed(e)),
+        self.interrupt_source_group
+            .update(irq as InterruptIndex, InterruptSourceConfig::MsiIrq(config))
+            .map_err(Error::UpdateInterrupt)?;
+
+        if interrupt_mask(entry) == 1 {
+            self.interrupt_source_group
+                .mask(irq as InterruptIndex)
+                .map_err(Error::MaskInterrupt)?;
+        } else {
+            self.interrupt_source_group
+                .unmask(irq as InterruptIndex)
+                .map_err(Error::UnmaskInterrupt)?;
         }
+
+        Ok(())
     }
 
     fn ioapic_write(&mut self, val: u32) {
@@ -320,6 +355,11 @@ impl Ioapic {
                     self.reg_entries[index] &= 0xffff_ffff_0000_5000;
                     self.reg_entries[index] |= u64::from(val) & 0xffff_afff;
                 }
+                // The entry must be updated through the interrupt source
+                // group.
+                if let Err(e) = self.update_entry(index) {
+                    error!("Failed updating IOAPIC entry: {:?}", e);
+                }
             }
             _ => error!("IOAPIC: invalid write to register offset"),
         }

devices/src/legacy/serial.rs

@@ -5,9 +5,11 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE-BSD-3-Clause file.
 
-use crate::{BusDevice, Interrupt};
+use crate::BusDevice;
 use std::collections::VecDeque;
+use std::sync::Arc;
 use std::{io, result};
+use vm_device::interrupt::InterruptSourceGroup;
 use vmm_sys_util::errno::Result;
 
 const LOOP_SIZE: usize = 0x40;
@@ -55,7 +57,7 @@ const DEFAULT_BAUD_DIVISOR: u16 = 12; // 9600 bps
 pub struct Serial {
     interrupt_enable: u8,
     interrupt_identification: u8,
-    interrupt: Box<dyn Interrupt>,
+    interrupt: Arc<Box<dyn InterruptSourceGroup>>,
     line_control: u8,
     line_status: u8,
     modem_control: u8,
@@ -67,7 +69,10 @@ pub struct Serial {
 }
 
 impl Serial {
-    pub fn new(interrupt: Box<dyn Interrupt>, out: Option<Box<dyn io::Write + Send>>) -> Serial {
+    pub fn new(
+        interrupt: Arc<Box<dyn InterruptSourceGroup>>,
+        out: Option<Box<dyn io::Write + Send>>,
+    ) -> Serial {
         Serial {
             interrupt_enable: 0,
             interrupt_identification: DEFAULT_INTERRUPT_IDENTIFICATION,
@@ -84,12 +89,15 @@ impl Serial {
     }
 
     /// Constructs a Serial port ready for output.
-    pub fn new_out(interrupt: Box<dyn Interrupt>, out: Box<dyn io::Write + Send>) -> Serial {
+    pub fn new_out(
+        interrupt: Arc<Box<dyn InterruptSourceGroup>>,
+        out: Box<dyn io::Write + Send>,
+    ) -> Serial {
         Self::new(interrupt, Some(out))
     }
 
     /// Constructs a Serial port with no connected output.
-    pub fn new_sink(interrupt: Box<dyn Interrupt>) -> Serial {
+    pub fn new_sink(interrupt: Arc<Box<dyn InterruptSourceGroup>>) -> Serial {
         Self::new(interrupt, None)
     }
 
@@ -149,7 +157,7 @@ impl Serial {
     }
 
     fn trigger_interrupt(&mut self) -> result::Result<(), io::Error> {
-        self.interrupt.deliver()
+        self.interrupt.trigger(0)
     }
 
     fn iir_reset(&mut self) {
@@ -233,16 +241,24 @@ mod tests {
     use super::*;
     use std::io;
     use std::sync::{Arc, Mutex};
+    use vm_device::interrupt::{InterruptIndex, InterruptSourceConfig};
     use vmm_sys_util::eventfd::EventFd;
 
     struct TestInterrupt {
         event_fd: EventFd,
     }
 
-    impl Interrupt for TestInterrupt {
-        fn deliver(&self) -> result::Result<(), std::io::Error> {
+    impl InterruptSourceGroup for TestInterrupt {
+        fn trigger(&self, _index: InterruptIndex) -> result::Result<(), std::io::Error> {
             self.event_fd.write(1)
         }
+        fn update(
+            &self,
+            _index: InterruptIndex,
+            _config: InterruptSourceConfig,
+        ) -> result::Result<(), std::io::Error> {
+            Ok(())
+        }
     }
 
     impl TestInterrupt {
@@ -278,7 +294,7 @@ mod tests {
         let intr_evt = EventFd::new(0).unwrap();
         let serial_out = SharedBuffer::new();
         let mut serial = Serial::new_out(
-            Box::new(TestInterrupt::new(intr_evt.try_clone().unwrap())),
+            Arc::new(Box::new(TestInterrupt::new(intr_evt.try_clone().unwrap()))),
             Box::new(serial_out.clone()),
         );
 
@@ -297,7 +313,7 @@ mod tests {
         let intr_evt = EventFd::new(0).unwrap();
         let serial_out = SharedBuffer::new();
         let mut serial = Serial::new_out(
-            Box::new(TestInterrupt::new(intr_evt.try_clone().unwrap())),
+            Arc::new(Box::new(TestInterrupt::new(intr_evt.try_clone().unwrap()))),
             Box::new(serial_out.clone()),
         );
 
@@ -334,8 +350,9 @@ mod tests {
     #[test]
     fn serial_thr() {
         let intr_evt = EventFd::new(0).unwrap();
-        let mut serial =
-            Serial::new_sink(Box::new(TestInterrupt::new(intr_evt.try_clone().unwrap())));
+        let mut serial = Serial::new_sink(Arc::new(Box::new(TestInterrupt::new(
+            intr_evt.try_clone().unwrap(),
+        ))));
 
         // write 1 to the interrupt event fd, so that read doesn't block in case the event fd
         // counter doesn't change (for 0 it blocks)
@@ -354,8 +371,9 @@ mod tests {
     #[test]
     fn serial_dlab() {
         let intr_evt = EventFd::new(0).unwrap();
-        let mut serial =
-            Serial::new_sink(Box::new(TestInterrupt::new(intr_evt.try_clone().unwrap())));
+        let mut serial = Serial::new_sink(Arc::new(Box::new(TestInterrupt::new(
+            intr_evt.try_clone().unwrap(),
+        ))));
 
         serial.write(0, LCR as u64, &[LCR_DLAB_BIT as u8]);
         serial.write(0, DLAB_LOW as u64, &[0x12 as u8]);
@@ -373,8 +391,9 @@ mod tests {
     #[test]
     fn serial_modem() {
         let intr_evt = EventFd::new(0).unwrap();
-        let mut serial =
-            Serial::new_sink(Box::new(TestInterrupt::new(intr_evt.try_clone().unwrap())));
+        let mut serial = Serial::new_sink(Arc::new(Box::new(TestInterrupt::new(
+            intr_evt.try_clone().unwrap(),
+        ))));
 
         serial.write(0, MCR as u64, &[MCR_LOOP_BIT as u8]);
         serial.write(0, DATA as u64, &['a' as u8]);
@@ -397,8 +416,9 @@ mod tests {
     #[test]
     fn serial_scratch() {
         let intr_evt = EventFd::new(0).unwrap();
-        let mut serial =
-            Serial::new_sink(Box::new(TestInterrupt::new(intr_evt.try_clone().unwrap())));
+        let mut serial = Serial::new_sink(Arc::new(Box::new(TestInterrupt::new(
+            intr_evt.try_clone().unwrap(),
+        ))));
 
         serial.write(0, SCR as u64, &[0x12 as u8]);
 

devices/src/lib.rs

@@ -6,18 +6,19 @@
 // found in the LICENSE-BSD-3-Clause file.
 
 //! Emulates virtual and hardware devices.
+#[macro_use]
+extern crate bitflags;
 extern crate byteorder;
 extern crate epoll;
-extern crate kvm_bindings;
-extern crate kvm_ioctls;
 extern crate libc;
 #[macro_use]
 extern crate log;
+extern crate vm_device;
 extern crate vm_memory;
 extern crate vmm_sys_util;
 
 use std::fs::File;
-use std::{io, result};
+use std::io;
 
 #[cfg(feature = "acpi")]
 mod acpi;
@@ -67,12 +68,10 @@ pub enum Error {
     IoError(io::Error),
 }
 
-pub trait Interrupt: Send + Sync {
-    fn deliver(&self) -> result::Result<(), std::io::Error>;
-}
-
-#[derive(Clone, Copy)]
-pub enum HotPlugNotificationType {
-    NoDevicesChanged,
-    CPUDevicesChanged,
+bitflags! {
+    pub struct HotPlugNotificationFlags: u8 {
+        const NO_DEVICES_CHANGED = 0;
+        const CPU_DEVICES_CHANGED = 0b1;
+        const MEMORY_DEVICES_CHANGED = 0b10;
+    }
 }

docs/api.md

@@ -0,0 +1,371 @@
+- [Cloud Hypervisor API](#cloud-hypervisor-api)
+  * [External API](#external-api)
+    + [REST API](#rest-api)
+      - [Location and availability](#location-and-availability)
+      - [Endpoints](#endpoints)
+		* [Virtual Machine Manager (VMM) Actions](#virtual-machine-manager-vmm-actions)
+		* [Virtual Machine (VM) Actions](#virtual-machine-vm-actions)
+      - [REST API Examples](#rest-api-examples)
+        * [Create a Virtual Machine](#create-a-virtual-machine)
+	    * [Boot a Virtual Machine](#boot-a-virtual-machine)
+        * [Dump a Virtual Machine Information](#dump-a-virtual-machine-information)
+        * [Reboot a Virtual Machine](#reboot-a-virtual-machine)
+        * [Shut a Virtual Machine Down](#shut-a-virtual-machine-down)
+    + [Command Line Interface](#command-line-interface)
+    + [REST API and CLI Architecural Relationship](#rest-api-and-cli-architectural-relationship)
+  * [Internal API](#internal-api)
+    + [Goals and Design](#goals-and-design)
+  * [End to End Example](#end-to-end-example)
+
+# Cloud Hypervisor API
+
+The Cloud Hypervisor API is made of 2 distinct interfaces:
+
+1. **The external API**. This is the user facing API. Users and operators can
+   control and manage Cloud Hypervisor through either a REST API or a Command
+   Line Interface (CLI).
+1. **The internal API**, based on [rust's Multi-Producer, Single-Consumer (MPSC)](https://doc.rust-lang.org/std/sync/mpsc/)
+   module. This API is used internally by the Cloud Hypervisor threads to
+   communicate between each others.
+
+The goal of this document is to describe the Cloud Hypervisor API as a whole,
+and to outline how the internal and external APIs are architecturally related.
+
+## External API
+
+### REST API
+
+The Cloud Hypervisor [REST](https://en.wikipedia.org/wiki/Representational_state_transfer)
+API triggers VM and VMM specific actions, and as such it is designed as a
+collection of RPC-style, static methods.
+
+The API is [OpenAPI 3.0](https://github.com/OAI/OpenAPI-Specification/blob/master/versions/3.0.0.md)
+compliant. Please consult the [Cloud Hypervisor API](https://raw.githubusercontent.com/cloud-hypervisor/cloud-hypervisor/master/vmm/src/api/openapi/cloud-hypervisor.yaml)
+document for more details about the API payloads and responses.
+
+### Location and availability
+
+The REST API is available as soon as the Cloud Hypervisor binary is started,
+through a local UNIX socket.
+By default, it is located at `/run/user/{user ID}/cloud-hypervisor.{Cloud Hypervisor PID}`.
+For example, if you launched Cloud Hypervisor as user ID 1000 and its PID is
+123456, the Cloud Hypervisor REST API will be available at `/run/user/1000/cloud-hypervisor.123456`.
+
+The REST API default URL can be overridden through the Cloud Hypervisor
+option `--api-socket`:
+
+```
+$ ./target/debug/cloud-hypervisor --api-socket /tmp/cloud-hypervisor.sock
+Cloud Hypervisor Guest
+    API server: /tmp/cloud-hypervisor.sock
+    vCPUs: 1
+    Memory: 512 MB
+    Kernel: None
+    Kernel cmdline:
+    Disk(s): None
+```
+
+### Endpoints
+
+The Cloud Hypervisor API exposes the following actions through its endpoints:
+
+#### Virtual Machine Manager (VMM) Actions
+
+Action                              | Endpoint        | Request Body | Response Body              | Prerequisites
+------------------------------------|-----------------|--------------|----------------------------|---------------------------
+Check for the REST API availability | `/vmm.ping`     | N/A          | `/schemas/VmmPingResponse` | N/A
+Shut the VMM down                   | `/vmm.shutdown` | N/A          | N/A                        | The VMM is running
+
+#### Virtual Machine (VM) Actions
+
+Action                           | Endpoint       | Request Body        | Response Body     | Prerequisites
+---------------------------------|----------------|---------------------|-------------------|---------------------------
+Create the VM                    | `/vm.create`   | `/schemas/VmConfig` | N/A               | The VM is not created yet
+Delete the VM                    | `/vm.delete`   | N/A                 | N/A               | The VM is created but not booted
+Boot the VM                      | `/vm.boot`     | N/A                 | N/A               | The VM is created
+Shut the VM down                 | `/vm.shutdown` | N/A                 | N/A               | The VM is booted
+Reboot the VM                    | `/vm.reboot`   | N/A                 | N/A               | The VM is booted
+Pause the VM                     | `/vm.pause`    | N/A                 | N/A               | The VM is booted
+Resume the VM                    | `/vm.resume`   | N/A                 | N/A               | The VM is paused
+Add/remove CPUs to/from the VM   | `/vm.resize`   | `/schemas/VmResize` | N/A               | The VM is booted
+Remove memory from the VM        | `/vm.resize`   | `/schemas/VmResize` | N/A               | The VM is booted
+Dump the VM information          | `/vm.info`     | N/A                 | `/schemas/VmInfo` | The VM is created
+
+### REST API Examples
+
+For the following set of examples, we assume Cloud Hypervisor is started with
+the REST API available at `/tmp/cloud-hypervisor.sock`:
+
+```
+$ ./target/debug/cloud-hypervisor --api-socket /tmp/cloud-hypervisor.sock
+Cloud Hypervisor Guest
+    API server: /tmp/cloud-hypervisor.sock
+    vCPUs: 1
+    Memory: 512 MB
+    Kernel: None
+    Kernel cmdline:
+    Disk(s): None
+```
+
+#### Create a Virtual Machine
+
+We want to create a virtual machine with the following characteristics:
+
+* 4 vCPUs
+* 1 GB of RAM
+* 1 virtio based networking interface
+* Direct kernel boot from a custom 5.5.0 Linux kernel located at
+  `/opt/clh/kernel/vmlinux-virtio-fs-virtio-iommu`
+* Using a Clear Linux image as its root filesystem, located at
+  `/opt/clh/images/clear-30080-kvm.img`
+
+```shell
+#!/bin/bash
+
+curl --unix-socket /tmp/cloud-hypervisor.sock -i \
+     -X PUT 'http://localhost/api/v1/vm.create'  \
+     -H 'Accept: application/json'               \
+     -H 'Content-Type: application/json'         \
+     -d '{
+         "cpus":{"boot_vcpus": 4, "max_vcpus": 4},
+         "kernel":{"path":"/opt/clh/kernel/vmlinux-virtio-fs-virtio-iommu"},
+         "cmdline":{"args":"console=hvc0 reboot=k panic=1 nomodules i8042.noaux i8042.nomux i8042.nopnp i8042.dumbkbd root=/dev/vda3"},
+         "disks":[{"path":"/opt/clh/images/clear-30080-kvm.img"}],
+         "rng":{"src":"/dev/urandom"},
+         "net":[{"ip":"192.168.10.10", "mask":"255.255.255.0", "mac":"12:34:56:78:90:01"}]
+         }'
+```
+
+#### Boot a Virtual Machine
+
+Once the VM is created, we can boot it:
+
+```shell
+#!/bin/bash
+
+curl --unix-socket /tmp/cloud-hypervisor.sock -i -X PUT 'http://localhost/api/v1/vm.boot'
+```
+
+#### Dump a Virtual Machine Information
+
+We can fetch information about any VM, as soon as it's created:
+
+```shell
+#!/bin/bash
+
+curl --unix-socket /tmp/cloud-hypervisor.sock -i \
+     -X GET 'http://localhost/api/v1/vm.info' \
+     -H 'Accept: application/json'
+```
+
+#### Reboot a Virtual Machine
+
+We can reboot a VM that's already booted:
+
+```shell
+#!/bin/bash
+
+curl --unix-socket /tmp/cloud-hypervisor.sock -i -X PUT 'http://localhost/api/v1/vm.reboot'
+```
+
+#### Shut a Virtual Machine Down
+
+Once booted, we can shut a VM down from the REST API:
+
+```shell
+#!/bin/bash
+
+curl --unix-socket /tmp/cloud-hypervisor.sock -i -X PUT 'http://localhost/api/v1/vm.shutdown'
+```
+
+### Command Line Interface
+
+The Cloud Hypervisor Command Line Interface (CLI) can only be used for launching
+the Cloud Hypervisor binary, i.e. it can not be used for controlling the VMM or
+the launched VM once they're up and running.
+
+If you want to inspect the VMM, or control the VM after launching Cloud
+Hypervisor from the CLI, you must use the [REST API](#rest-api).
+
+From the CLI, one can either:
+
+1. Create and boot a complete virtual machine by using the CLI options to build
+   the VM config. Run `cloud-hypervisor --help` for a complete list of CLI
+   options. As soon as the `cloud-hypervisor` binary is launched, the
+   [REST API](#rest-api) is available for controlling and managing the VM.
+1. Start the [REST API](#rest-api) server only, by not passing any VM
+   configuration options. The VM can then be asynchronously created and booted
+   by sending HTTP commands to the [REST API](#rest-api). Check the
+   [REST API examples](#rest-api-examples) section for more details.
+
+### REST API and CLI Architectural Relationship
+
+The REST API and the CLI both rely on a common, [internal API](#internal-api).
+
+The CLI options are parsed by the
+[clap crate](https://docs.rs/clap/2.33.0/clap/) and then translated into
+[internal API](#internal-api) commands.
+
+The REST API is processed by an HTTP thread using the
+[Firecracker's `micro_http`](https://github.com/firecracker-microvm/firecracker/tree/master/src/micro_http)
+crate. As with the CLI, the HTTP requests eventually get translated into
+[internal API](#internal-api) commands.
+
+As a summary, the REST API and the CLI are essentially frontends for the
+[internal API](#internal-api):
+
+```
+                                  +------------------+
+                        REST API  |                  |
+                       +--------->+    micro_http    +--------+
+                       |          |                  |        |
+                       |          +------------------+        |
+                       |                                      |      +------------------------+
+                       |                                      |      |                        |
++------------+         |                                      |      |                        |
+|            |         |                                      |      | +--------------+       |
+|    User    +---------+                                      +------> | Internal API |       |
+|            |         |                                      |      | +--------------+       |
++------------+         |                                      |      |                        |
+                       |                                      |      |                        |
+                       |                                      |      +------------------------+
+                       |            +----------+              |                 VMM
+                       |     CLI    |          |              |
+                       +----------->+   clap   +--------------+
+                                    |          |
+                                    +----------+
+
+
+```
+
+## Internal API
+
+The Cloud Hypervisor internal API, as its name suggests, is used internally
+by the different Cloud Hypervisor threads (VMM, HTTP, control loop, etc) to
+send commands and responses to each others.
+
+It is based on [rust's Multi-Producer, Single-Consumer (MPSC)](https://doc.rust-lang.org/std/sync/mpsc/),
+and the single consumer (a.k.a. the API receiver) is the Cloud Hypervisor
+control loop.
+
+API producers are the HTTP thread handling the [REST API](#rest-api) and the
+main thread that initially parses the [CLI](#command-line-interface).
+
+### Goals and Design
+
+The internal API is designed for controlling, managing and inspecting a Cloud
+Hypervisor VMM and its guest. It is a backend for handling external, user
+visible requests through either the [REST API](#rest-api) or the
+[CLI](#command-line-interface) interfaces.
+
+The API follows a command-response scheme that closely maps the [REST API](#rest-api).
+Any command must be replied to with a response.
+
+Commands are [MPSC](https://doc.rust-lang.org/std/sync/mpsc/) based messages and
+are received and processed by the VMM control loop.
+
+In order for the VMM control loop to respond to any internal API command, it
+must be able to send a response back to the MPSC sender. For that purpose, all
+internal API command payload carry the [Sender](https://doc.rust-lang.org/std/sync/mpsc/struct.Sender.html)
+end of an [MPSC](https://doc.rust-lang.org/std/sync/mpsc/) channel. 
+
+The sender of any internal API command is therefore responsible for:
+
+1. Creating an [MPSC](https://doc.rust-lang.org/std/sync/mpsc/) response
+   channel.
+1. Passing the [Sender](https://doc.rust-lang.org/std/sync/mpsc/struct.Sender.html)
+   end of the response channel as part of the internal API command payload.
+1. Waiting for the internal API command's response on the [Receiver](https://doc.rust-lang.org/std/sync/mpsc/struct.Receiver.html)
+   end of the response channel.
+
+## End to End Example
+
+In order to further understand how the external and internal Cloud Hypervisor
+APIs work together, let's look at a complete VM creation flow, from the
+[REST API](#rest-api) call, to the reply the external user will receive:
+
+1. A user or operator sends an HTTP request to the Cloud Hypervisor
+   [REST API](#rest-api) in order to creates a virtual machine:
+   ```
+   shell
+   #!/bin/bash
+
+	curl --unix-socket /tmp/cloud-hypervisor.sock -i \
+		-X PUT 'http://localhost/api/v1/vm.create'  \
+		-H 'Accept: application/json'               \
+		-H 'Content-Type: application/json'         \
+		-d '{
+			"cpus":{"boot_vcpus": 4, "max_vcpus": 4},
+			"kernel":{"path":"/opt/clh/kernel/vmlinux-virtio-fs-virtio-iommu"},
+			"cmdline":{"args":"console=hvc0 reboot=k panic=1 nomodules i8042.noaux i8042.nomux i8042.nopnp i8042.dumbkbd root=/dev/vda3"},
+			"disks":[{"path":"/opt/clh/images/clear-30080-kvm.img"}],
+			"rng":{"src":"/dev/urandom"},
+			"net":[{"ip":"192.168.10.10", "mask":"255.255.255.0", "mac":"12:34:56:78:90:01"}]
+			}'
+   ```
+1. The Cloud Hypervisor HTTP thread processes the request and de-serializes the
+   HTTP request JSON body into an internal `VmConfig` structure.
+1. The Cloud Hypervisor HTTP thread creates an
+   [MPSC](https://doc.rust-lang.org/std/sync/mpsc/) channel for the internal API
+   server to send its response back.
+1. The Cloud Hypervisor HTTP thread prepares an internal API command for creating a
+   virtual machine. The command's payload is made of the de-serialized
+   `VmConfig` structure and the response channel:
+   ```Rust
+   VmCreate(Arc<Mutex<VmConfig>>, Sender<ApiResponse>)
+   ```
+1. The Cloud Hypervisor HTTP thread sends the internal API command, and waits
+   for the response:
+   ```Rust
+   // Send the VM creation request.
+    api_sender
+        .send(ApiRequest::VmCreate(config, response_sender))
+        .map_err(ApiError::RequestSend)?;
+    api_evt.write(1).map_err(ApiError::EventFdWrite)?;
+
+    response_receiver.recv().map_err(ApiError::ResponseRecv)??;
+   ```
+1. The Cloud Hypervisor control loop receives the command, as it listens on the
+   internal API [MPSC](https://doc.rust-lang.org/std/sync/mpsc/) channel:
+   ```Rust
+   // Read from the API receiver channel
+   let api_request = api_receiver.recv().map_err(Error::ApiRequestRecv)?;
+   ```
+1. The Cloud Hypervisor control loop matches the received internal API against
+   the `VmCreate` payload, and extracts both the `VmConfig` structure and the
+   [Sender](https://doc.rust-lang.org/std/sync/mpsc/struct.Sender.html) from the
+   command payload. It stores the `VmConfig` structure and replies back to the
+   sender ((The HTTP thread):
+   ```Rust
+   match api_request {
+	   ApiRequest::VmCreate(config, sender) => {
+		   // We only store the passed VM config.
+		   // The VM will be created when being asked to boot it.
+		   let response = if self.vm_config.is_none() {
+			   self.vm_config = Some(config);
+			   Ok(ApiResponsePayload::Empty)
+		   } else {
+			   Err(ApiError::VmAlreadyCreated)
+		   };
+		   
+	       sender.send(response).map_err(Error::ApiResponseSend)?;
+	   }
+   ```
+1. The Cloud Hypervisor HTTP thread receives the internal API command response
+   as the return value from its `VmCreate` HTTP handler. Depending on the
+   control loop internal API response, it generates the appropriate HTTP
+   response:
+   ```Rust
+   // Call vm_create()
+   match vm_create(api_notifier, api_sender, Arc::new(Mutex::new(vm_config)))
+	   .map_err(HttpError::VmCreate)
+   {
+	   Ok(_) => Response::new(Version::Http11, StatusCode::NoContent),
+	   Err(e) => error_response(e, StatusCode::InternalServerError),
+   }
+   ```
+1. The Cloud Hypervisor HTTP thread sends the formed HTTP response back to the
+   user. This is abstracted by the
+   [micro_http](https://github.com/firecracker-microvm/firecracker/tree/master/src/micro_http)
+   crate.
+   

docs/device_model.md

@@ -166,8 +166,8 @@ processes, we added support for vhost-user-blk backends. This enables
 `cloud-hypervisor` users to plug a `vhost-user` based block device (e.g. SPDK)
 into the VMM as their virtio block backend.
 
-This device is always built-in, and it is enabled based on the presence of the
-flag `--vhost-user-blk`.
+This device is always built-in, and it is enabled when `vhost_user=true` and
+`socket` are provided to the `--disk` parameter.
 
 ### vhost-user-fs
 
@@ -188,8 +188,8 @@ processes, we added support for [vhost-user-net](https://access.redhat.com/solut
 backends. This enables `cloud-hypervisor` users to plug a `vhost-user` based
 networking device (e.g. DPDK) into the VMM as their virtio network backend.
 
-This device is always built-in, and it is enabled based on the presence of the
-flag `--vhost-user-net`.
+This device is always built-in, and it is enabled when `vhost_user=true` and
+`socket` are provided to the `--net` parameter.
 
 ## VFIO
 

docs/fs.md

@@ -26,14 +26,14 @@ _Run virtiofsd_
 ```bash
 ./virtiofsd \
     -d \
-    -o vhost_user_socket=/tmp/virtiofs \
+    --socket-path=/tmp/virtiofs \
     -o source=/tmp/shared_dir \
-    -o cache=always
+    -o cache=none
 ```
 
-The `cache=always` option should be the default when using `virtiofsd` with the __cloud-hypervisor__ VMM. This allows the daemon to memory map the shared files, which gives better I/O performance.
+The `cache=none` option should be the default when using `virtiofsd` with the __cloud-hypervisor__ VMM. This prevents from using the guest page cache, which reduces the memory footprint of the guest. When running multiple virtual machines on the same host, this will let the host deal with page cache, which will increase the density of virtual machines which can be launched.
 
-The `cache=none` option is another way to run the daemon but because the _virtqueues_ are used to convey the files content in this case, the I/O performance is impacted.
+The `cache=always` option will allow for the guest page cache to be used, which will increase the memory footprint of the guest. This option should be used only for specific use cases where a single VM is going to be running on a host.
 
 ### The kernel
 
@@ -60,7 +60,7 @@ Assuming you have `clear-kvm.img` and `custom-vmlinux.bin` on your system, here
     --disk path=clear-kvm.img \
     --kernel custom-vmlinux.bin \
     --cmdline "console=ttyS0 reboot=k panic=1 nomodules root=/dev/vda3" \ 
-    --fs tag=virtiofs,sock=/tmp/virtiofs,num_queues=1,queue_size=512
+    --fs tag=myfs,sock=/tmp/virtiofs,num_queues=1,queue_size=512
 ```
 
 By default, DAX is enabled with a cache window of 8GiB. You can specify a custom size (let's say 4GiB for this example) for the cache by explicitly setting DAX and the cache size:
@@ -78,13 +78,11 @@ In case you don't want to use a shared window of cache to pass the shared files
 ```
 
 ### Mount the shared directory
-The last step is to mount the shared directory inside the guest, using the `virtio_fs` filesystem type.
+The last step is to mount the shared directory inside the guest, using the `virtiofs` filesystem type.
 ```bash
 mkdir mount_dir
-mount \
-    -t virtio_fs virtiofs mount_dir/ \
-    -o rootmode=040000,user_id=0,group_id=0,dax
+mount -t virtiofs -o dax myfs mount_dir/
 ```
-The `tag` needs to be consistent with what has been provided through the __cloud-hypervisor__ command line, which happens to be `virtiofs` in this example.
+The `tag` needs to be consistent with what has been provided through the __cloud-hypervisor__ command line, which happens to be `myfs` in this example.
 
-The `dax` option must be removed in case the shared cache region is not enabled from the VMM.
+The `-o dax` option must be removed in case the shared cache region is not enabled from the VMM.

docs/hotplug.md

@@ -11,10 +11,10 @@ This patch is integrated into the Clear Linux KVM and cloudguest images.
 
 ## CPU Hot Plug
 
-Extra vCPUs can be added (but not removed [1]) from a running Cloud Hypervisor instance. This is controlled by two mechanisms:
+Extra vCPUs can be added and removed from a running Cloud Hypervisor instance. This is controlled by two mechanisms:
 
 1. Specifying a number of maximum potential vCPUs that is greater than the number of default (boot) vCPUs.
-2. Making a HTTP API request to the VM to ask for the additional vCPUs to be added.
+2. Making a HTTP API request to the VMM to ask for the additional vCPUs to be added.
 
 To use CPU hotplug start the VM with the number of max vCPUs greater than the number of boot vCPUs, e.g.
 
@@ -40,7 +40,7 @@ To ask the VMM to add additional vCPUs then use the resize API:
 curl -H "Accept: application/json" -H "Content-Type: application/json" -i -XPUT --unix-socket /tmp/ch-socket -d "{ \"desired_vcpus\":8}" http://localhost/api/v1/vm.resize
 ```
 
-The extra vCPU threads will be created and advertised to the running kernel. The kernel does not bring up the CPUs immediately and instead the user must "on-line" them from inside the VM:
+The extra vCPU threads will be created and advertised to the running kernel. The kernel does not bring up the CPUs immediately and instead the user must "online" them from inside the VM:
 
 ```shell
 root@ch-guest ~ # lscpu | grep list:
@@ -54,4 +54,60 @@ On-line CPU(s) list:             0-7
 
 After a reboot the added CPUs will remain.
 
-[1]: It is not currently possible to remove CPUs after they are added however CPU hot unplug is included in our roadmap for a future version.
+Removing CPUs works similarly by reducing the number in the "desired_vcpus" field of the reisze API. The CPUs will be automatically offlined inside the guest so there is no need to run any commands inside the guest:
+
+```shell
+curl -H "Accept: application/json" -H "Content-Type: application/json" -i -XPUT --unix-socket /tmp/ch-socket -d "{ \"desired_vcpus\":2}" http://localhost/api/v1/vm.resize
+```
+
+As per adding CPUs to the guest, after a reboot the VM will be running with the reduced number of vCPUs.
+
+## Memory Hot Plug
+
+Extra memory can be added from a runing Cloud Hypervisor instance. This is controlled by two mechanisms:
+
+1. Allocating some of the guest physical address space for hotplug memory.
+2. Making a HTTP API request to the VMM to ask for a new amount of RAM to be assigned to the VM. In the case of expanding the memory for the VM the new memory will be hotplugged into the running VM, if reducing the size of the memory then change will take effect after the next reboot.
+
+To use memory hotplug start the VM specifying some size RAM in the "hotplug_size" parameter to the memory configuration. Not all the memory specified in this parameter will be available to hotplug as there are spacing and alignment requirements so it is recommended to make it larger than the hotplug RAM needed.
+
+```shell
+$ pushd $CLOUDH
+$ sudo setcap cap_net_admin+ep ./cloud-hypervisor/target/release/cloud-hypervisor
+$ ./cloud-hypervisor/target/release/cloud-hypervisor \
+	--kernel ./hypervisor-fw \
+	--disk path=clear-31890-kvm.img \
+	--cpus boot=4,max=8 \
+	--memory size=1024M,hotplug_size=8192M \
+	--net "tap=,mac=,ip=,mask=" \
+	--rng \
+    --api-socket=/tmp/ch-socket
+$ popd
+```
+
+Before issuing the API request it is necessary to run the following command inside the VM to make it automatically online the added memory:
+
+```shell
+root@ch-guest ~ # echo online | sudo tee /sys/devices/system/memory/auto_online_blocks
+```
+
+To ask the VMM to add expand the RAM for the VM:
+
+```shell
+curl -H "Accept: application/json" -H "Content-Type: application/json" -i -XPUT --unix-socket /tmp/ch-socket -d "{ \"desired_vcpus\": 4, \"desired_ram\" : 3221225472}" http://localhost/api/v1/vm.resize
+```
+
+The new memory is now available to use inside the VM:
+
+```shell
+free -h
+              total        used        free      shared  buff/cache   available
+Mem:          3.0Gi        71Mi       2.8Gi       0.0Ki        47Mi       2.8Gi
+Swap:          32Mi          0B        32Mi
+```
+
+Due to guest OS limitations is is necessary to ensure that amount of memory added (between currently assigned RAM and that which is desired) is a multiple of 128MiB.
+
+The same API can also be used to reduce the desired RAM for a VM but the change will not be applied until the VM is rebooted.
+
+Memory and CPU resizing can be combined together into the same HTTP API request.

docs/networking.md

@@ -2,6 +2,21 @@
 
 cloud-hypervisor can emulate one or more virtual network interfaces, represented at the hypervisor host by [tap devices](https://www.kernel.org/doc/Documentation/networking/tuntap.txt). This guide briefly describes, in a manual and distribution neutral way, how to setup and use networking with cloud-hypevisor.
 
+## Multiple queue support for net devices ##
+
+While multiple vcpus defined for guest, to gain the benefit of vcpu scalable to improve performance, it suggests to define multiple queue pairs for net devices, one Tx/Rx queue pair per one vcpu, that means the number of queue pairs at least is equal to the vcpu count. In that case, after virtnet driver set cpu affinity for virtqueues in guest kernel, vcpus could handle interrupt from different virtqueue pairs in parallel.
+
+It will gain better performance for guest that has multiple queues defined for net devices while it has multiple net sessions running in userspace.
+
+To enable multiple queue support in cloud-hypervisor, multiple queue pairs will be defined, while multiple tap fds will be opened for the same tap device, it will also have multiple threads started, each thread will monitor and handle the events from each virtqueue pairs and the associated tap fd.
+
+Note:
+
+- Currently, it does not support to use ethtool to change the combined queue numbers in guest.
+- Multiple queue is enabled for vhost-user-net backend in cloud-hypervisor, however, multiple thread is not added to handle mq, thus, the performance for vhost-user-net backend is not supposed to be improved. The multiple thread will be added for backend later.
+- Performance test for vhost-user-net will be covered once vhost-user-net backend has mulitple thread supported.
+- Performance test for virtio-net is done by comparing 2 queue pairs with 1 queue pairs, that to run 2 iperf3 sessions in the same test environments, throughput is improved about 37%.
+
 ## Start cloud-hypervisor with net devices
 
 Use one `--net` command-line argument from cloud-hypervisor to specify the emulation of one or more virtual NIC's. The example below instructs cloud-hypervisor to emulate for instance 2 virtual NIC's:
@@ -13,18 +28,29 @@ Use one `--net` command-line argument from cloud-hypervisor to specify the emula
     --disk path=my-root-disk.img \
     --kernel my-vmlinux.bin \
     --cmdline "console=ttyS0 reboot=k panic=1 nomodules root=/dev/vda3" \
-    --net tap=ich0,mac=a4:a1:c2:00:00:01,ip=192.168.4.2,mask=255.255.255.0 \
-          tap=ich1,mac=a4:a1:c2:00:00:02,ip=10.0.1.2,mask=255.255.255.0
+    --net tap=ich0,mac=a4:a1:c2:00:00:01,ip=192.168.4.2,mask=255.255.255.0,num_queues=2,queue_size=256 \
+          tap=ich1,mac=a4:a1:c2:00:00:02,ip=10.0.1.2,mask=255.255.255.0,num_queues=2,queue_size=256
 ```
- 
+
 The `--net` argument takes 1 or more space-separated strings of key value pairs containing the following 4 keys or fields:
 
-| Name     | Purpose                    | Optional  |
-| -------- |----------------------------| ----------|
-| tap      | tap device name            | Yes       |
-| mac      | vNIC mac address           | Yes       |
-| ip       | tap IP IP address          | yes       |
-| mask     | tap IP netmask             | Yes       |
+| Name       | Purpose                    | Optional  |
+| -----------|----------------------------| ----------|
+| tap        | tap device name            | Yes       |
+| mac        | vNIC mac address           | Yes       |
+| ip         | tap IP IP address          | yes       |
+| mask       | tap IP netmask             | Yes       |
+| num_queues | the number of queues       | yes       |
+| queue_size | the size of each queue     | Yes       |
+
+num_queues is the total number of tx and rx queues, the default value is 2, and it could be increased by multiples of 2. Additionally, num_queues is suggested to be as 2 times of vcpu count. The default value for queue_size is 256.
+
+If the tap device is pre-created on host before guest boot up. To use multiple queue support for net device in guest, the tap device should be opened like this from host.
+
+```bash
+[root@localhost ~]# ip tuntap add name ich0 mode tap multi_queue
+```
+
 
 ## Configure the tap devices
 

net_gen/Cargo.toml

@@ -4,4 +4,4 @@ version = "0.1.0"
 authors = ["The Chromium OS Authors"]
 
 [dependencies]
-vmm-sys-util = ">=0.3.1"
+vmm-sys-util = "0.4.0"

net_util/Cargo.toml

@@ -4,14 +4,14 @@ version = "0.1.0"
 authors = ["The Chromium OS Authors"]
 
 [dependencies]
-libc = "0.2.60"
-rand = "0.7.0"
-serde = "1.0.98"
-vmm-sys-util = ">=0.3.1"
+libc = "0.2.66"
+rand = "0.7.3"
+serde = "1.0.104"
+vmm-sys-util = "0.4.0"
 
 net_gen = { path = "../net_gen" }
 
 [dev-dependencies]
-lazy_static = "1.3.0"
-pnet = "0.22.0"
-serde_json = "1.0.40"
+lazy_static = "1.4.0"
+pnet = "0.25.0"
+serde_json = "1.0.48"

net_util/src/mac.rs

@@ -7,6 +7,7 @@
 
 use rand::Rng;
 use std::fmt;
+use std::io;
 use std::result::Result;
 
 use serde::de::{Deserialize, Deserializer, Error};
@@ -20,23 +21,31 @@ pub struct MacAddr {
 }
 
 impl MacAddr {
-    // The error contains the str that failed to be parsed, for nicer error message generation.
-    pub fn parse_str<S>(s: &S) -> Result<MacAddr, &str>
+    pub fn parse_str<S>(s: &S) -> Result<MacAddr, io::Error>
     where
         S: AsRef<str> + ?Sized,
     {
         let v: Vec<&str> = s.as_ref().split(':').collect();
         let mut bytes = [0u8; MAC_ADDR_LEN];
+        let common_err = Err(io::Error::new(
+            io::ErrorKind::Other,
+            format!("parsing of {} into a MAC address failed", s.as_ref()),
+        ));
 
         if v.len() != MAC_ADDR_LEN {
-            return Err(s.as_ref());
+            return common_err;
         }
 
         for i in 0..MAC_ADDR_LEN {
             if v[i].len() != 2 {
-                return Err(s.as_ref());
+                return common_err;
             }
-            bytes[i] = u8::from_str_radix(v[i], 16).map_err(|_| s.as_ref())?;
+            bytes[i] = u8::from_str_radix(v[i], 16).map_err(|e| {
+                io::Error::new(
+                    io::ErrorKind::Other,
+                    format!("parsing of {} into a MAC address failed: {}", s.as_ref(), e),
+                )
+            })?;
         }
 
         Ok(MacAddr { bytes })
@@ -106,7 +115,8 @@ impl<'de> Deserialize<'de> for MacAddr {
         D: Deserializer<'de>,
     {
         let s = String::deserialize(deserializer)?;
-        MacAddr::parse_str(&s).map_err(|_| D::Error::custom("The provided MAC address is invalid."))
+        MacAddr::parse_str(&s)
+            .map_err(|e| D::Error::custom(format!("The provided MAC address is invalid: {}", e)))
     }
 }
 

net_util/src/tap.rs

@@ -40,7 +40,7 @@ pub type Result<T> = ::std::result::Result<T, Error>;
 #[derive(Debug)]
 pub struct Tap {
     tap_file: File,
-    if_name: [u8; 16usize],
+    if_name: Vec<u8>,
 }
 
 impl PartialEq for Tap {
@@ -53,7 +53,7 @@ impl std::clone::Clone for Tap {
     fn clone(&self) -> Self {
         Tap {
             tap_file: self.tap_file.try_clone().unwrap(),
-            if_name: self.if_name,
+            if_name: self.if_name.clone(),
         }
     }
 }
@@ -78,7 +78,7 @@ fn build_terminated_if_name(if_name: &str) -> Result<Vec<u8>> {
 }
 
 impl Tap {
-    pub fn open_named(if_name: &str) -> Result<Tap> {
+    pub fn open_named(if_name: &str, num_queue_pairs: usize) -> Result<Tap> {
         let terminated_if_name = build_terminated_if_name(if_name)?;
 
         let fd = unsafe {
@@ -107,6 +107,9 @@ impl Tap {
             name_slice.copy_from_slice(terminated_if_name.as_slice());
             *ifru_flags =
                 (net_gen::IFF_TAP | net_gen::IFF_NO_PI | net_gen::IFF_VNET_HDR) as c_short;
+            if num_queue_pairs > 1 {
+                *ifru_flags |= net_gen::IFF_MULTI_QUEUE as c_short;
+            }
         }
 
         // ioctl is safe since we call it with a valid tap fd and check the return
@@ -117,16 +120,19 @@ impl Tap {
             return Err(Error::CreateTap(IoError::last_os_error()));
         }
 
+        let if_name_temp = unsafe { *ifreq.ifr_ifrn.ifrn_name.as_ref() };
+        let mut if_name = if_name_temp.to_vec();
+        if_name.truncate(terminated_if_name.len() - 1);
         // Safe since only the name is accessed, and it's cloned out.
         Ok(Tap {
             tap_file: tuntap,
-            if_name: unsafe { *ifreq.ifr_ifrn.ifrn_name.as_ref() },
+            if_name,
         })
     }
 
     /// Create a new tap interface.
-    pub fn new() -> Result<Tap> {
-        Self::open_named("vmtap%d")
+    pub fn new(num_queue_pairs: usize) -> Result<Tap> {
+        Self::open_named("vmtap%d", num_queue_pairs)
     }
 
     /// Set the host-side IP address for the tap interface.
@@ -232,11 +238,16 @@ impl Tap {
         // in a single-field union.
         unsafe {
             let ifrn_name = ifreq.ifr_ifrn.ifrn_name.as_mut();
-            ifrn_name.clone_from_slice(&self.if_name);
+            let name_slice = &mut ifrn_name[..self.if_name.len()];
+            name_slice.copy_from_slice(&self.if_name);
         }
 
         ifreq
     }
+
+    pub fn get_if_name(&self) -> Vec<u8> {
+        self.if_name.clone()
+    }
 }
 
 impl Read for Tap {
@@ -423,7 +434,7 @@ mod tests {
 
     #[test]
     fn test_tap_create() {
-        let t = Tap::new().unwrap();
+        let t = Tap::new(1).unwrap();
         println!("created tap: {:?}", t);
     }
 
@@ -435,7 +446,7 @@ mod tests {
         // the end of the function.
         let tap_ip_guard = TAP_IP_LOCK.lock().unwrap();
 
-        let tap = Tap::new().unwrap();
+        let tap = Tap::new(1).unwrap();
         let ip_addr: net::Ipv4Addr = (*tap_ip_guard).parse().unwrap();
         let netmask: net::Ipv4Addr = SUBNET_MASK.parse().unwrap();
 
@@ -448,21 +459,21 @@ mod tests {
     #[test]
     fn test_set_options() {
         // This line will fail to provide an initialized FD if the test is not run as root.
-        let tap = Tap::new().unwrap();
+        let tap = Tap::new(1).unwrap();
         tap.set_vnet_hdr_size(16).unwrap();
         tap.set_offload(0).unwrap();
     }
 
     #[test]
     fn test_tap_enable() {
-        let tap = Tap::new().unwrap();
+        let tap = Tap::new(1).unwrap();
         let ret = tap.enable();
         assert!(ret.is_ok());
     }
 
     #[test]
     fn test_tap_get_ifreq() {
-        let tap = Tap::new().unwrap();
+        let tap = Tap::new(1).unwrap();
         let ret = tap.get_ifreq();
         assert_eq!(
             "__BindgenUnionField",
@@ -472,7 +483,7 @@ mod tests {
 
     #[test]
     fn test_raw_fd() {
-        let tap = Tap::new().unwrap();
+        let tap = Tap::new(1).unwrap();
         assert_eq!(tap.as_raw_fd(), tap.tap_file.as_raw_fd());
     }
 
@@ -480,7 +491,7 @@ mod tests {
     fn test_read() {
         let tap_ip_guard = TAP_IP_LOCK.lock().unwrap();
 
-        let mut tap = Tap::new().unwrap();
+        let mut tap = Tap::new(1).unwrap();
         tap.set_ip_addr((*tap_ip_guard).parse().unwrap()).unwrap();
         tap.set_netmask(SUBNET_MASK.parse().unwrap()).unwrap();
         tap.enable().unwrap();
@@ -541,7 +552,7 @@ mod tests {
     fn test_write() {
         let tap_ip_guard = TAP_IP_LOCK.lock().unwrap();
 
-        let mut tap = Tap::new().unwrap();
+        let mut tap = Tap::new(1).unwrap();
         tap.set_ip_addr((*tap_ip_guard).parse().unwrap()).unwrap();
         tap.set_netmask(SUBNET_MASK.parse().unwrap()).unwrap();
         tap.enable().unwrap();

pci/Cargo.toml

@@ -6,9 +6,9 @@ edition = "2018"
 
 [dependencies]
 vm-allocator = { path = "../vm-allocator" }
-byteorder = "1.3.2"
+byteorder = "1.3.4"
 devices = { path = "../devices" }
-libc = "0.2.60"
+libc = "0.2.66"
 log = "0.4.8"
+vm-device = { path = "../vm-device" }
 vm-memory = { git = "https://github.com/rust-vmm/vm-memory" }
-vmm-sys-util = ">=0.3.1"

pci/src/bus.rs

@@ -67,7 +67,7 @@ impl PciDevice for PciRoot {
         self.config.write_config_register(reg_idx, offset, data);
     }
 
-    fn read_config_register(&self, reg_idx: usize) -> u32 {
+    fn read_config_register(&mut self, reg_idx: usize) -> u32 {
         self.config.read_reg(reg_idx)
     }
 

pci/src/configuration.rs

@@ -399,12 +399,17 @@ impl PciConfiguration {
     pub fn write_reg(&mut self, reg_idx: usize, value: u32) {
         let mut mask = self.writable_bits[reg_idx];
 
-        if value == 0xffff_ffff {
-            if reg_idx >= BAR0_REG && reg_idx < BAR0_REG + NUM_BAR_REGS {
-                // Handle very specific case where the BAR is being written with
-                // all 1's to retrieve the BAR size on next BAR reading.
+        if reg_idx >= BAR0_REG && reg_idx < BAR0_REG + NUM_BAR_REGS {
+            // Handle very specific case where the BAR is being written with
+            // all 1's to retrieve the BAR size during next BAR reading.
+            if value == 0xffff_ffff {
                 mask = self.bar_size[reg_idx - 4];
-            } else if reg_idx == ROM_BAR_REG {
+            }
+        } else if reg_idx == ROM_BAR_REG {
+            // Handle very specific case where the BAR is being written with
+            // all 1's on bits 31-11 to retrieve the BAR size during next BAR
+            // reading.
+            if value & ROM_BAR_ADDR_MASK == ROM_BAR_ADDR_MASK {
                 mask = self.rom_bar_size;
             }
         }
@@ -669,10 +674,6 @@ impl PciConfiguration {
 
         let value = LittleEndian::read_u32(data);
 
-        if value == 0xffff_ffff {
-            return None;
-        }
-
         let mask = self.writable_bits[reg_idx];
         if reg_idx >= BAR0_REG && reg_idx < BAR0_REG + NUM_BAR_REGS {
             let bar_idx = reg_idx - 4;
@@ -684,6 +685,12 @@ impl PciConfiguration {
                     match bar_type {
                         PciBarRegionType::Memory64BitRegion => {}
                         _ => {
+                            // Ignore the case where the BAR size is being
+                            // asked for.
+                            if value == 0xffff_ffff {
+                                return None;
+                            }
+
                             debug!(
                                 "DETECT BAR REPROG: current 0x{:x}, new 0x{:x}",
                                 self.registers[reg_idx], value
@@ -707,6 +714,17 @@ impl PciConfiguration {
                     && (self.registers[reg_idx - 1] & self.writable_bits[reg_idx - 1])
                         != (self.bar_addr[bar_idx - 1] & self.writable_bits[reg_idx - 1])
                 {
+                    // Ignore the case where the BAR size is being asked for.
+                    // Because we are in the 64bits case here, we have to check
+                    // if the lower 32bits of the current BAR have already been
+                    // asked for the BAR size too.
+                    if value == 0xffff_ffff
+                        && self.registers[reg_idx - 1] & self.writable_bits[reg_idx - 1]
+                            == self.bar_size[bar_idx - 1] & self.writable_bits[reg_idx - 1]
+                    {
+                        return None;
+                    }
+
                     debug!(
                         "DETECT BAR REPROG: current 0x{:x}, new 0x{:x}",
                         self.registers[reg_idx], value
@@ -731,6 +749,11 @@ impl PciConfiguration {
                 }
             }
         } else if reg_idx == ROM_BAR_REG && (value & mask) != (self.rom_bar_addr & mask) {
+            // Ignore the case where the BAR size is being asked for.
+            if value & ROM_BAR_ADDR_MASK == ROM_BAR_ADDR_MASK {
+                return None;
+            }
+
             debug!(
                 "DETECT ROM BAR REPROG: current 0x{:x}, new 0x{:x}",
                 self.registers[reg_idx], value

pci/src/device.rs

@@ -3,23 +3,13 @@
 // found in the LICENSE-BSD-3-Clause file.
 
 use crate::configuration::{self, PciBarRegionType};
-use crate::msix::MsixTableEntry;
-use crate::PciInterruptPin;
 use devices::BusDevice;
 use std::any::Any;
 use std::fmt::{self, Display};
-use std::sync::Arc;
 use std::{self, io, result};
 use vm_allocator::SystemAllocator;
 use vm_memory::{GuestAddress, GuestUsize};
 
-pub struct InterruptParameters<'a> {
-    pub msix: Option<&'a MsixTableEntry>,
-}
-
-pub type InterruptDelivery =
-    Box<dyn Fn(InterruptParameters) -> result::Result<(), io::Error> + Send + Sync>;
-
 #[derive(Debug)]
 pub enum Error {
     /// Setup of the device capabilities failed.
@@ -56,19 +46,6 @@ pub struct BarReprogrammingParams {
 }
 
 pub trait PciDevice: BusDevice {
-    /// Assign a legacy PCI IRQ to this device.
-    /// The device may write to `irq_evt` to trigger an interrupt.
-    fn assign_pin_irq(
-        &mut self,
-        _irq_cb: Arc<InterruptDelivery>,
-        _irq_num: u32,
-        _irq_pin: PciInterruptPin,
-    ) {
-    }
-
-    /// Assign MSI-X to this device.
-    fn assign_msix(&mut self, _msi_cb: Arc<InterruptDelivery>) {}
-
     /// Allocates the needed PCI BARs space using the `allocate` function which takes a size and
     /// returns an address. Returns a Vec of (GuestAddress, GuestUsize) tuples.
     fn allocate_bars(
@@ -84,7 +61,7 @@ pub trait PciDevice: BusDevice {
     fn write_config_register(&mut self, reg_idx: usize, offset: u64, data: &[u8]);
     /// Gets a register from the configuration space.
     /// * `reg_idx` - The index of the config register to read.
-    fn read_config_register(&self, reg_idx: usize) -> u32;
+    fn read_config_register(&mut self, reg_idx: usize) -> u32;
     /// Detects if a BAR is being reprogrammed.
     fn detect_bar_reprogramming(
         &mut self,

pci/src/lib.rs

@@ -7,7 +7,6 @@
 extern crate log;
 extern crate devices;
 extern crate vm_memory;
-extern crate vmm_sys_util;
 
 mod bus;
 mod configuration;
@@ -22,10 +21,9 @@ pub use self::configuration::{
     PciNetworkControllerSubclass, PciProgrammingInterface, PciSerialBusSubClass, PciSubclass,
 };
 pub use self::device::{
-    BarReprogrammingParams, DeviceRelocation, Error as PciDeviceError, InterruptDelivery,
-    InterruptParameters, PciDevice,
+    BarReprogrammingParams, DeviceRelocation, Error as PciDeviceError, PciDevice,
 };
-pub use self::msi::MsiCap;
+pub use self::msi::{msi_num_enabled_vectors, MsiCap, MsiConfig};
 pub use self::msix::{MsixCap, MsixConfig, MsixTableEntry, MSIX_TABLE_ENTRY_SIZE};
 
 /// PCI has four interrupt pins A->D.

pci/src/msi.rs

@@ -7,6 +7,10 @@ extern crate byteorder;
 extern crate vm_memory;
 
 use byteorder::{ByteOrder, LittleEndian};
+use std::sync::Arc;
+use vm_device::interrupt::{
+    InterruptIndex, InterruptSourceConfig, InterruptSourceGroup, MsiIrqSourceConfig,
+};
 
 // MSI control masks
 const MSI_CTL_ENABLE: u16 = 0x1;
@@ -21,6 +25,16 @@ const MSI_MSG_ADDR_LO_OFFSET: u64 = 0x4;
 // MSI message masks
 const MSI_MSG_ADDR_LO_MASK: u32 = 0xffff_fffc;
 
+pub fn msi_num_enabled_vectors(msg_ctl: u16) -> usize {
+    let field = (msg_ctl >> 4) & 0x7;
+
+    if field > 5 {
+        return 0;
+    }
+
+    1 << field
+}
+
 #[derive(Clone, Copy, Default)]
 pub struct MsiCap {
     // Message Control Register
@@ -58,21 +72,15 @@ impl MsiCap {
         self.msg_ctl & MSI_CTL_PER_VECTOR == MSI_CTL_PER_VECTOR
     }
 
-    pub fn enabled(&self) -> bool {
+    fn enabled(&self) -> bool {
         self.msg_ctl & MSI_CTL_ENABLE == MSI_CTL_ENABLE
     }
 
-    pub fn num_enabled_vectors(&self) -> usize {
-        let field = (self.msg_ctl >> 4) & 0x7;
-
-        if field > 5 {
-            return 0;
-        }
-
-        1 << field
+    fn num_enabled_vectors(&self) -> usize {
+        msi_num_enabled_vectors(self.msg_ctl)
     }
 
-    pub fn vector_masked(&self, vector: usize) -> bool {
+    fn vector_masked(&self, vector: usize) -> bool {
         if !self.per_vector_mask() {
             return false;
         }
@@ -80,7 +88,7 @@ impl MsiCap {
         (self.mask_bits >> vector) & 0x1 == 0x1
     }
 
-    pub fn size(&self) -> u64 {
+    fn size(&self) -> u64 {
         let mut size: u64 = 0xa;
 
         if self.addr_64_bits() {
@@ -93,7 +101,7 @@ impl MsiCap {
         size
     }
 
-    pub fn update(&mut self, offset: u64, data: &[u8]) {
+    fn update(&mut self, offset: u64, data: &[u8]) {
         // Calculate message data offset depending on the address being 32 or
         // 64 bits.
         // Calculate upper address offset if the address is 64 bits.
@@ -151,3 +159,73 @@ impl MsiCap {
         }
     }
 }
+
+pub struct MsiConfig {
+    cap: MsiCap,
+    interrupt_source_group: Arc<Box<dyn InterruptSourceGroup>>,
+}
+
+impl MsiConfig {
+    pub fn new(msg_ctl: u16, interrupt_source_group: Arc<Box<dyn InterruptSourceGroup>>) -> Self {
+        let cap = MsiCap {
+            msg_ctl,
+            ..Default::default()
+        };
+
+        MsiConfig {
+            cap,
+            interrupt_source_group,
+        }
+    }
+
+    pub fn enabled(&self) -> bool {
+        self.cap.enabled()
+    }
+
+    pub fn size(&self) -> u64 {
+        self.cap.size()
+    }
+
+    pub fn num_enabled_vectors(&self) -> usize {
+        self.cap.num_enabled_vectors()
+    }
+
+    pub fn update(&mut self, offset: u64, data: &[u8]) {
+        let old_enabled = self.cap.enabled();
+
+        self.cap.update(offset, data);
+
+        if self.cap.enabled() {
+            for idx in 0..self.num_enabled_vectors() {
+                let config = MsiIrqSourceConfig {
+                    high_addr: self.cap.msg_addr_hi,
+                    low_addr: self.cap.msg_addr_lo,
+                    data: self.cap.msg_data as u32,
+                };
+
+                if let Err(e) = self
+                    .interrupt_source_group
+                    .update(idx as InterruptIndex, InterruptSourceConfig::MsiIrq(config))
+                {
+                    error!("Failed updating vector: {:?}", e);
+                }
+
+                if self.cap.vector_masked(idx) {
+                    if let Err(e) = self.interrupt_source_group.mask(idx as InterruptIndex) {
+                        error!("Failed masking vector: {:?}", e);
+                    }
+                }
+            }
+
+            if !old_enabled {
+                if let Err(e) = self.interrupt_source_group.enable() {
+                    error!("Failed enabling irq_fd: {:?}", e);
+                }
+            }
+        } else if old_enabled {
+            if let Err(e) = self.interrupt_source_group.disable() {
+                error!("Failed disabling irq_fd: {:?}", e);
+            }
+        }
+    }
+}

pci/src/msix.rs

@@ -8,9 +8,11 @@ extern crate vm_memory;
 
 use std::sync::Arc;
 
-use crate::device::InterruptParameters;
-use crate::{InterruptDelivery, PciCapability, PciCapabilityID};
+use crate::{PciCapability, PciCapabilityID};
 use byteorder::{ByteOrder, LittleEndian};
+use vm_device::interrupt::{
+    InterruptIndex, InterruptSourceConfig, InterruptSourceGroup, MsiIrqSourceConfig,
+};
 use vm_memory::ByteValued;
 
 const MAX_MSIX_VECTORS_PER_DEVICE: u16 = 2048;
@@ -51,13 +53,16 @@ impl Default for MsixTableEntry {
 pub struct MsixConfig {
     pub table_entries: Vec<MsixTableEntry>,
     pub pba_entries: Vec<u64>,
-    interrupt_cb: Option<Arc<InterruptDelivery>>,
+    interrupt_source_group: Arc<Box<dyn InterruptSourceGroup>>,
     masked: bool,
     enabled: bool,
 }
 
 impl MsixConfig {
-    pub fn new(msix_vectors: u16) -> Self {
+    pub fn new(
+        msix_vectors: u16,
+        interrupt_source_group: Arc<Box<dyn InterruptSourceGroup>>,
+    ) -> Self {
         assert!(msix_vectors <= MAX_MSIX_VECTORS_PER_DEVICE);
 
         let mut table_entries: Vec<MsixTableEntry> = Vec::new();
@@ -69,16 +74,12 @@ impl MsixConfig {
         MsixConfig {
             table_entries,
             pba_entries,
-            interrupt_cb: None,
+            interrupt_source_group,
             masked: false,
             enabled: false,
         }
     }
 
-    pub fn register_interrupt_cb(&mut self, cb: Arc<InterruptDelivery>) {
-        self.interrupt_cb = Some(cb);
-    }
-
     pub fn masked(&self) -> bool {
         self.masked
     }
@@ -89,10 +90,47 @@ impl MsixConfig {
 
     pub fn set_msg_ctl(&mut self, reg: u16) {
         let old_masked = self.masked;
+        let old_enabled = self.enabled;
 
         self.masked = ((reg >> FUNCTION_MASK_BIT) & 1u16) == 1u16;
         self.enabled = ((reg >> MSIX_ENABLE_BIT) & 1u16) == 1u16;
 
+        // Update KVM routes
+        if old_masked != self.masked || old_enabled != self.enabled {
+            if self.enabled && !self.masked {
+                for (idx, table_entry) in self.table_entries.iter().enumerate() {
+                    let config = MsiIrqSourceConfig {
+                        high_addr: table_entry.msg_addr_hi,
+                        low_addr: table_entry.msg_addr_lo,
+                        data: table_entry.msg_data,
+                    };
+
+                    if let Err(e) = self
+                        .interrupt_source_group
+                        .update(idx as InterruptIndex, InterruptSourceConfig::MsiIrq(config))
+                    {
+                        error!("Failed updating vector: {:?}", e);
+                    }
+
+                    if table_entry.masked() {
+                        if let Err(e) = self.interrupt_source_group.mask(idx as InterruptIndex) {
+                            error!("Failed masking vector: {:?}", e);
+                        }
+                    }
+                }
+
+                if !old_enabled || old_masked {
+                    if let Err(e) = self.interrupt_source_group.enable() {
+                        error!("Failed enabling irq_fd: {:?}", e);
+                    }
+                }
+            } else if old_enabled || !old_masked {
+                if let Err(e) = self.interrupt_source_group.disable() {
+                    error!("Failed disabling irq_fd: {:?}", e);
+                }
+            }
+        }
+
         // If the Function Mask bit was set, and has just been cleared, it's
         // important to go through the entire PBA to check if there was any
         // pending MSI-X message to inject, given that the vector is not
@@ -198,6 +236,32 @@ impl MsixConfig {
             _ => error!("invalid data length"),
         };
 
+        // Update interrupt routes
+        if self.enabled && !self.masked {
+            let table_entry = &self.table_entries[index];
+
+            let config = MsiIrqSourceConfig {
+                high_addr: table_entry.msg_addr_hi,
+                low_addr: table_entry.msg_addr_lo,
+                data: table_entry.msg_data,
+            };
+
+            if let Err(e) = self.interrupt_source_group.update(
+                index as InterruptIndex,
+                InterruptSourceConfig::MsiIrq(config),
+            ) {
+                error!("Failed updating vector: {:?}", e);
+            }
+
+            if table_entry.masked() {
+                if let Err(e) = self.interrupt_source_group.mask(index as InterruptIndex) {
+                    error!("Failed masking vector: {:?}", e);
+                }
+            } else if let Err(e) = self.interrupt_source_group.unmask(index as InterruptIndex) {
+                error!("Failed unmasking vector: {:?}", e);
+            }
+        }
+
         // After the MSI-X table entry has been updated, it is necessary to
         // check if the vector control masking bit has changed. In case the
         // bit has been flipped from 1 to 0, we need to inject a MSI message
@@ -285,13 +349,12 @@ impl MsixConfig {
 
     fn inject_msix_and_clear_pba(&mut self, vector: usize) {
         // Inject the MSI message
-        if let Some(cb) = &self.interrupt_cb {
-            match (cb)(InterruptParameters {
-                msix: Some(&self.table_entries[vector]),
-            }) {
-                Ok(_) => debug!("MSI-X injected on vector control flip"),
-                Err(e) => error!("failed to inject MSI-X: {}", e),
-            };
+        match self
+            .interrupt_source_group
+            .trigger(vector as InterruptIndex)
+        {
+            Ok(_) => debug!("MSI-X injected on vector control flip"),
+            Err(e) => error!("failed to inject MSI-X: {}", e),
         }
 
         // Clear the bit from PBA

qcow/Cargo.toml

@@ -9,11 +9,12 @@ license = "BSD-3-Clause"
 path = "src/qcow.rs"
 
 [dependencies]
-byteorder = "1.3.2"
-libc = "0.2.60"
+byteorder = "1.3.4"
+libc = "0.2.66"
 log = "0.4.8"
-remain = "0.1.3"
-vmm-sys-util = ">=0.3.1"
+remain = "0.2.1"
+vmm-sys-util = "0.4.0"
+vm-virtio = { path = "../vm-virtio" }
 
 [dev-dependencies]
 tempfile = "3.1.0"

qcow/src/qcow.rs

@@ -12,6 +12,7 @@ mod vec_cache;
 use byteorder::{BigEndian, ReadBytesExt, WriteBytesExt};
 use libc::{EINVAL, ENOSPC, ENOTSUP};
 use remain::sorted;
+use vm_virtio::RawFile;
 use vmm_sys_util::{
     file_traits::FileSetLen, file_traits::FileSync, seek_hole::SeekHole, write_zeroes::PunchHole,
     write_zeroes::WriteZeroes,
@@ -19,10 +20,8 @@ use vmm_sys_util::{
 
 use std::cmp::{max, min};
 use std::fmt::{self, Display};
-use std::fs::File;
 use std::io::{self, Read, Seek, SeekFrom, Write};
 use std::mem::size_of;
-use std::os::unix::io::{AsRawFd, RawFd};
 
 use crate::qcow_raw_file::QcowRawFile;
 use crate::refcount::RefCount;
@@ -190,7 +189,7 @@ pub struct QcowHeader {
 
 impl QcowHeader {
     /// Creates a QcowHeader from a reference to a file.
-    pub fn new(f: &mut File) -> Result<QcowHeader> {
+    pub fn new(f: &mut RawFile) -> Result<QcowHeader> {
         f.seek(SeekFrom::Start(0)).map_err(Error::ReadingHeader)?;
         let magic = f.read_u32::<BigEndian>().map_err(Error::ReadingHeader)?;
         if magic != QCOW_MAGIC {
@@ -198,12 +197,12 @@ impl QcowHeader {
         }
 
         // Reads the next u32 from the file.
-        fn read_u32_from_file(f: &mut File) -> Result<u32> {
+        fn read_u32_from_file(f: &mut RawFile) -> Result<u32> {
             f.read_u32::<BigEndian>().map_err(Error::ReadingHeader)
         }
 
         // Reads the next u64 from the file.
-        fn read_u64_from_file(f: &mut File) -> Result<u64> {
+        fn read_u64_from_file(f: &mut RawFile) -> Result<u64> {
             f.read_u64::<BigEndian>().map_err(Error::ReadingHeader)
         }
 
@@ -368,9 +367,11 @@ fn max_refcount_clusters(refcount_order: u32, cluster_size: u32, num_clusters: u
 ///
 /// ```
 /// # use std::io::{Read, Seek, SeekFrom};
+/// # use vm_virtio::RawFile;
 /// # use qcow::{self, QcowFile};
 /// # fn test(file: std::fs::File) -> std::io::Result<()> {
-///     let mut q = QcowFile::from(file).expect("Can't open qcow file");
+///     let mut raw_img = RawFile::new(file, false);
+///     let mut q = QcowFile::from(raw_img).expect("Can't open qcow file");
 ///     let mut buf = [0u8; 12];
 ///     q.seek(SeekFrom::Start(10 as u64))?;
 ///     q.read(&mut buf[..])?;
@@ -395,7 +396,7 @@ pub struct QcowFile {
 
 impl QcowFile {
     /// Creates a QcowFile from `file`. File must be a valid qcow2 image.
-    pub fn from(mut file: File) -> Result<QcowFile> {
+    pub fn from(mut file: RawFile) -> Result<QcowFile> {
         let header = QcowHeader::new(&mut file)?;
 
         // Only v2 and v3 files are supported.
@@ -544,7 +545,7 @@ impl QcowFile {
     }
 
     /// Creates a new QcowFile at the given path.
-    pub fn new(mut file: File, version: u32, virtual_size: u64) -> Result<QcowFile> {
+    pub fn new(mut file: RawFile, version: u32, virtual_size: u64) -> Result<QcowFile> {
         let header = QcowHeader::create_for_size(version, virtual_size);
         file.seek(SeekFrom::Start(0)).map_err(Error::SeekingFile)?;
         header.write_to(&mut file)?;
@@ -1220,7 +1221,12 @@ impl QcowFile {
         let refcount = self
             .refcounts
             .get_cluster_refcount(&mut self.raw_file, cluster_addr)
-            .map_err(|_| std::io::Error::from_raw_os_error(EINVAL))?;
+            .map_err(|e| {
+                io::Error::new(
+                    io::ErrorKind::InvalidData,
+                    format!("failed to get cluster refcount: {}", e),
+                )
+            })?;
         if refcount == 0 {
             return Err(std::io::Error::from_raw_os_error(EINVAL));
         }
@@ -1394,12 +1400,6 @@ impl Drop for QcowFile {
     }
 }
 
-impl AsRawFd for QcowFile {
-    fn as_raw_fd(&self) -> RawFd {
-        self.raw_file.file().as_raw_fd()
-    }
-}
-
 impl Read for QcowFile {
     fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
         let address: u64 = self.current_offset as u64;
@@ -1634,7 +1634,7 @@ where
     Ok(())
 }
 
-fn convert_reader<R>(reader: &mut R, dst_file: File, dst_type: ImageType) -> Result<()>
+fn convert_reader<R>(reader: &mut R, dst_file: RawFile, dst_type: ImageType) -> Result<()>
 where
     R: Read + Seek + SeekHole,
 {
@@ -1666,8 +1666,8 @@ where
 /// Copy the contents of a disk image in `src_file` into `dst_file`.
 /// The type of `src_file` is automatically detected, and the output file type is
 /// determined by `dst_type`.
-pub fn convert(src_file: File, dst_file: File, dst_type: ImageType) -> Result<()> {
-    let src_type = detect_image_type(&src_file)?;
+pub fn convert(mut src_file: RawFile, dst_file: RawFile, dst_type: ImageType) -> Result<()> {
+    let src_type = detect_image_type(&mut src_file)?;
     match src_type {
         ImageType::Qcow2 => {
             let mut src_reader = QcowFile::from(src_file)?;
@@ -1682,17 +1682,18 @@ pub fn convert(src_file: File, dst_file: File, dst_type: ImageType) -> Result<()
 }
 
 /// Detect the type of an image file by checking for a valid qcow2 header.
-pub fn detect_image_type(file: &File) -> Result<ImageType> {
-    let mut f = file;
-    let orig_seek = f.seek(SeekFrom::Current(0)).map_err(Error::SeekingFile)?;
-    f.seek(SeekFrom::Start(0)).map_err(Error::SeekingFile)?;
-    let magic = f.read_u32::<BigEndian>().map_err(Error::ReadingHeader)?;
+pub fn detect_image_type(file: &mut RawFile) -> Result<ImageType> {
+    let orig_seek = file
+        .seek(SeekFrom::Current(0))
+        .map_err(Error::SeekingFile)?;
+    file.seek(SeekFrom::Start(0)).map_err(Error::SeekingFile)?;
+    let magic = file.read_u32::<BigEndian>().map_err(Error::ReadingHeader)?;
     let image_type = if magic == QCOW_MAGIC {
         ImageType::Qcow2
     } else {
         ImageType::Raw
     };
-    f.seek(SeekFrom::Start(orig_seek))
+    file.seek(SeekFrom::Start(orig_seek))
         .map_err(Error::SeekingFile)?;
     Ok(image_type)
 }
@@ -1700,9 +1701,9 @@ pub fn detect_image_type(file: &File) -> Result<ImageType> {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use std::fs::File;
     use std::io::{Read, Seek, SeekFrom, Write};
     use tempfile::tempfile;
+    use vm_virtio::RawFile;
 
     fn valid_header_v3() -> Vec<u8> {
         vec![
@@ -1771,9 +1772,9 @@ mod tests {
 
     fn with_basic_file<F>(header: &[u8], mut testfn: F)
     where
-        F: FnMut(File),
+        F: FnMut(RawFile),
     {
-        let mut disk_file: File = tempfile().unwrap();
+        let mut disk_file: RawFile = RawFile::new(tempfile().unwrap(), false);
         disk_file.write_all(&header).unwrap();
         disk_file.set_len(0x1_0000_0000).unwrap();
         disk_file.seek(SeekFrom::Start(0)).unwrap();
@@ -1781,11 +1782,11 @@ mod tests {
         testfn(disk_file); // File closed when the function exits.
     }
 
-    fn with_default_file<F>(file_size: u64, mut testfn: F)
+    fn with_default_file<F>(file_size: u64, direct: bool, mut testfn: F)
     where
         F: FnMut(QcowFile),
     {
-        let tmp = tempfile().unwrap();
+        let tmp: RawFile = RawFile::new(tempfile().unwrap(), direct);
         let qcow_file = QcowFile::new(tmp, 3, file_size).unwrap();
 
         testfn(qcow_file); // File closed when the function exits.
@@ -1794,7 +1795,7 @@ mod tests {
     #[test]
     fn default_header_v2() {
         let header = QcowHeader::create_for_size(2, 0x10_0000);
-        let mut disk_file: File = tempfile().unwrap();
+        let mut disk_file: RawFile = RawFile::new(tempfile().unwrap(), false);
         header
             .write_to(&mut disk_file)
             .expect("Failed to write header to temporary file.");
@@ -1805,7 +1806,7 @@ mod tests {
     #[test]
     fn default_header_v3() {
         let header = QcowHeader::create_for_size(3, 0x10_0000);
-        let mut disk_file: File = tempfile().unwrap();
+        let mut disk_file: RawFile = RawFile::new(tempfile().unwrap(), false);
         header
             .write_to(&mut disk_file)
             .expect("Failed to write header to temporary file.");
@@ -1815,13 +1816,13 @@ mod tests {
 
     #[test]
     fn header_read() {
-        with_basic_file(&valid_header_v2(), |mut disk_file: File| {
+        with_basic_file(&valid_header_v2(), |mut disk_file: RawFile| {
             let header = QcowHeader::new(&mut disk_file).expect("Failed to create Header.");
             assert_eq!(header.version, 2);
             assert_eq!(header.refcount_order, DEFAULT_REFCOUNT_ORDER);
             assert_eq!(header.header_size, V2_BARE_HEADER_SIZE);
         });
-        with_basic_file(&valid_header_v3(), |mut disk_file: File| {
+        with_basic_file(&valid_header_v3(), |mut disk_file: RawFile| {
             let header = QcowHeader::new(&mut disk_file).expect("Failed to create Header.");
             assert_eq!(header.version, 3);
             assert_eq!(header.refcount_order, DEFAULT_REFCOUNT_ORDER);
@@ -1832,7 +1833,7 @@ mod tests {
     #[test]
     fn invalid_magic() {
         let invalid_header = vec![0x51u8, 0x46, 0x4a, 0xfb];
-        with_basic_file(&invalid_header, |mut disk_file: File| {
+        with_basic_file(&invalid_header, |mut disk_file: RawFile| {
             QcowHeader::new(&mut disk_file).expect_err("Invalid header worked.");
         });
     }
@@ -1841,7 +1842,7 @@ mod tests {
     fn invalid_refcount_order() {
         let mut header = valid_header_v3();
         header[99] = 2;
-        with_basic_file(&header, |disk_file: File| {
+        with_basic_file(&header, |disk_file: RawFile| {
             QcowFile::from(disk_file).expect_err("Invalid refcount order worked.");
         });
     }
@@ -1850,7 +1851,7 @@ mod tests {
     fn invalid_cluster_bits() {
         let mut header = valid_header_v3();
         header[23] = 3;
-        with_basic_file(&header, |disk_file: File| {
+        with_basic_file(&header, |disk_file: RawFile| {
             QcowFile::from(disk_file).expect_err("Failed to create file.");
         });
     }
@@ -1858,7 +1859,7 @@ mod tests {
     #[test]
     fn test_header_huge_file() {
         let header = test_huge_header();
-        with_basic_file(&header, |disk_file: File| {
+        with_basic_file(&header, |disk_file: RawFile| {
             QcowFile::from(disk_file).expect_err("Failed to create file.");
         });
     }
@@ -1867,7 +1868,7 @@ mod tests {
     fn test_header_crazy_file_size_rejected() {
         let mut header = valid_header_v3();
         &mut header[24..32].copy_from_slice(&[0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x1e]);
-        with_basic_file(&header, |disk_file: File| {
+        with_basic_file(&header, |disk_file: RawFile| {
             QcowFile::from(disk_file).expect_err("Failed to create file.");
         });
     }
@@ -1876,7 +1877,7 @@ mod tests {
     fn test_huge_l1_table() {
         let mut header = valid_header_v3();
         header[36] = 0x12;
-        with_basic_file(&header, |disk_file: File| {
+        with_basic_file(&header, |disk_file: RawFile| {
             QcowFile::from(disk_file).expect_err("Failed to create file.");
         });
     }
@@ -1888,7 +1889,7 @@ mod tests {
         header[26] = 1;
         header[31] = 0;
         // 1 TB with the min cluster size makes the arrays too big, it should fail.
-        with_basic_file(&header, |disk_file: File| {
+        with_basic_file(&header, |disk_file: RawFile| {
             QcowFile::from(disk_file).expect_err("Failed to create file.");
         });
     }
@@ -1902,7 +1903,7 @@ mod tests {
         header[31] = 0;
         // set cluster_bits
         header[23] = 16;
-        with_basic_file(&header, |disk_file: File| {
+        with_basic_file(&header, |disk_file: RawFile| {
             let mut qcow = QcowFile::from(disk_file).expect("Failed to create file.");
             qcow.seek(SeekFrom::Start(0x100_0000_0000 - 8))
                 .expect("Failed to seek.");
@@ -1916,7 +1917,7 @@ mod tests {
     fn test_header_huge_num_refcounts() {
         let mut header = valid_header_v3();
         &mut header[56..60].copy_from_slice(&[0x02, 0x00, 0xe8, 0xff]);
-        with_basic_file(&header, |disk_file: File| {
+        with_basic_file(&header, |disk_file: RawFile| {
             QcowFile::from(disk_file).expect_err("Created disk with crazy refcount clusters");
         });
     }
@@ -1925,14 +1926,14 @@ mod tests {
     fn test_header_huge_refcount_offset() {
         let mut header = valid_header_v3();
         &mut header[48..56].copy_from_slice(&[0x00, 0x00, 0x09, 0x00, 0x00, 0x00, 0x02, 0x00]);
-        with_basic_file(&header, |disk_file: File| {
+        with_basic_file(&header, |disk_file: RawFile| {
             QcowFile::from(disk_file).expect_err("Created disk with crazy refcount offset");
         });
     }
 
     #[test]
     fn write_read_start() {
-        with_basic_file(&valid_header_v3(), |disk_file: File| {
+        with_basic_file(&valid_header_v3(), |disk_file: RawFile| {
             let mut q = QcowFile::from(disk_file).unwrap();
             q.write(b"test first bytes")
                 .expect("Failed to write test string.");
@@ -1945,7 +1946,7 @@ mod tests {
 
     #[test]
     fn offset_write_read() {
-        with_basic_file(&valid_header_v3(), |disk_file: File| {
+        with_basic_file(&valid_header_v3(), |disk_file: RawFile| {
             let mut q = QcowFile::from(disk_file).unwrap();
             let b = [0x55u8; 0x1000];
             q.seek(SeekFrom::Start(0xfff2000)).expect("Failed to seek.");
@@ -1959,7 +1960,7 @@ mod tests {
 
     #[test]
     fn write_zeroes_read() {
-        with_basic_file(&valid_header_v3(), |disk_file: File| {
+        with_basic_file(&valid_header_v3(), |disk_file: RawFile| {
             let mut q = QcowFile::from(disk_file).unwrap();
             // Write some test data.
             let b = [0x55u8; 0x1000];
@@ -1985,7 +1986,7 @@ mod tests {
         // Choose a size that is larger than a cluster.
         // valid_header uses cluster_bits = 12, which corresponds to a cluster size of 4096.
         const CHUNK_SIZE: usize = 4096 * 2 + 512;
-        with_basic_file(&valid_header_v3(), |disk_file: File| {
+        with_basic_file(&valid_header_v3(), |disk_file: RawFile| {
             let mut q = QcowFile::from(disk_file).unwrap();
             // Write some test data.
             let b = [0x55u8; CHUNK_SIZE];
@@ -2006,11 +2007,11 @@ mod tests {
 
     #[test]
     fn test_header() {
-        with_basic_file(&valid_header_v2(), |disk_file: File| {
+        with_basic_file(&valid_header_v2(), |disk_file: RawFile| {
             let q = QcowFile::from(disk_file).unwrap();
             assert_eq!(q.virtual_size(), 0x20_0000_0000);
         });
-        with_basic_file(&valid_header_v3(), |disk_file: File| {
+        with_basic_file(&valid_header_v3(), |disk_file: RawFile| {
             let q = QcowFile::from(disk_file).unwrap();
             assert_eq!(q.virtual_size(), 0x20_0000_0000);
         });
@@ -2018,7 +2019,7 @@ mod tests {
 
     #[test]
     fn read_small_buffer() {
-        with_basic_file(&valid_header_v3(), |disk_file: File| {
+        with_basic_file(&valid_header_v3(), |disk_file: RawFile| {
             let mut q = QcowFile::from(disk_file).unwrap();
             let mut b = [5u8; 16];
             q.seek(SeekFrom::Start(1000)).expect("Failed to seek.");
@@ -2030,7 +2031,7 @@ mod tests {
 
     #[test]
     fn replay_ext4() {
-        with_basic_file(&valid_header_v3(), |disk_file: File| {
+        with_basic_file(&valid_header_v3(), |disk_file: RawFile| {
             let mut q = QcowFile::from(disk_file).unwrap();
             const BUF_SIZE: usize = 0x1000;
             let mut b = [0u8; BUF_SIZE];
@@ -2406,7 +2407,16 @@ mod tests {
 
     #[test]
     fn combo_write_read() {
-        with_default_file(1024 * 1024 * 1024 * 256, |mut qcow_file| {
+        combo_write_read_common(false);
+    }
+
+    #[test]
+    fn combo_write_read_direct() {
+        combo_write_read_common(true);
+    }
+
+    fn combo_write_read_common(direct: bool) {
+        with_default_file(1024 * 1024 * 1024 * 256, direct, |mut qcow_file| {
             const NUM_BLOCKS: usize = 555;
             const BLOCK_SIZE: usize = 0x1_0000;
             const OFFSET: usize = 0x1_0000_0020;
@@ -2459,7 +2469,16 @@ mod tests {
 
     #[test]
     fn seek_data() {
-        with_default_file(0x30000, |mut file| {
+        seek_data_common(false);
+    }
+
+    #[test]
+    fn seek_data_direct() {
+        seek_data_common(true);
+    }
+
+    fn seek_data_common(direct: bool) {
+        with_default_file(0x30000, direct, |mut file| {
             // seek_data at or after the end of the file should return None
             assert_eq!(file.seek_data(0x10000).unwrap(), None);
             assert_eq!(seek_cur(&mut file), 0);
@@ -2492,7 +2511,16 @@ mod tests {
 
     #[test]
     fn seek_hole() {
-        with_default_file(0x30000, |mut file| {
+        seek_hole_common(false);
+    }
+
+    #[test]
+    fn seek_hole_direct() {
+        seek_hole_common(true);
+    }
+
+    fn seek_hole_common(direct: bool) {
+        with_default_file(0x30000, direct, |mut file| {
             // File consisting entirely of a hole
             assert_eq!(file.seek_hole(0).unwrap(), Some(0));
             assert_eq!(seek_cur(&mut file), 0);
@@ -2566,7 +2594,7 @@ mod tests {
 
     #[test]
     fn rebuild_refcounts() {
-        with_basic_file(&valid_header_v3(), |mut disk_file: File| {
+        with_basic_file(&valid_header_v3(), |mut disk_file: RawFile| {
             let header = QcowHeader::new(&mut disk_file).expect("Failed to create Header.");
             let cluster_size = 65536;
             let mut raw_file =

qcow/src/qcow_raw_file.rs

@@ -2,17 +2,17 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE-BSD-3-Clause file.
 
-use std::fs::File;
 use std::io::{self, BufWriter, Seek, SeekFrom};
 use std::mem::size_of;
 
 use byteorder::{BigEndian, ReadBytesExt, WriteBytesExt};
+use vm_virtio::RawFile;
 use vmm_sys_util::write_zeroes::WriteZeroes;
 
 /// A qcow file. Allows reading/writing clusters and appending clusters.
 #[derive(Debug)]
 pub struct QcowRawFile {
-    file: File,
+    file: RawFile,
     cluster_size: u64,
     cluster_mask: u64,
 }
@@ -20,7 +20,7 @@ pub struct QcowRawFile {
 impl QcowRawFile {
     /// Creates a `QcowRawFile` from the given `File`, `None` is returned if `cluster_size` is not
     /// a power of two.
-    pub fn from(file: File, cluster_size: u64) -> Option<Self> {
+    pub fn from(file: RawFile, cluster_size: u64) -> Option<Self> {
         if cluster_size.count_ones() != 1 {
             return None;
         }
@@ -67,7 +67,7 @@ impl QcowRawFile {
         non_zero_flags: u64,
     ) -> io::Result<()> {
         self.file.seek(SeekFrom::Start(offset))?;
-        let mut buffer = BufWriter::with_capacity(table.len() * size_of::<u64>(), &self.file);
+        let mut buffer = BufWriter::with_capacity(table.len() * size_of::<u64>(), &mut self.file);
         for addr in table {
             let val = if *addr == 0 {
                 0
@@ -92,7 +92,7 @@ impl QcowRawFile {
     /// Writes a refcount block to the file.
     pub fn write_refcount_block(&mut self, offset: u64, table: &[u16]) -> io::Result<()> {
         self.file.seek(SeekFrom::Start(offset))?;
-        let mut buffer = BufWriter::with_capacity(table.len() * size_of::<u16>(), &self.file);
+        let mut buffer = BufWriter::with_capacity(table.len() * size_of::<u16>(), &mut self.file);
         for count in table {
             buffer.write_u16::<BigEndian>(*count)?;
         }
@@ -115,13 +115,8 @@ impl QcowRawFile {
         Ok(Some(new_cluster_address))
     }
 
-    /// Returns a reference to the underlying file.
-    pub fn file(&self) -> &File {
-        &self.file
-    }
-
     /// Returns a mutable reference to the underlying file.
-    pub fn file_mut(&mut self) -> &mut File {
+    pub fn file_mut(&mut self) -> &mut RawFile {
         &mut self.file
     }
 

release-notes.md

@@ -1,37 +1,116 @@
+- [v0.5.1](#v051)
+- [v0.5.0](#v050)
+    - [Virtual Machine Dynamic Resizing](#virtual-machine-dynamic-resizing)
+    - [Multi-Queue, Multi-Threaded Paravirtualization](#multi-queue-multi-threaded-paravirtualization)
+    - [New Interrupt Management Framework](#new-interrupt-management-framework)
+    - [Development Tools](#development-tools)
+    - [Kata Containers Integration](#kata-containers-integration)
+    - [Contributors](#contributors)
 - [v0.4.0](#v040)
-    + [Dynamic virtual CPUs addition](#dynamic-virtual-cpus-addition)
-    + [Programmatic firmware tables generation](#programmatic-firmware-tables-generation)
-    + [Filesystem and block devices vhost-user backends](#filesystem-and-block-devices-vhost-user-backends)
-    + [Guest pause and resume](#guest-pause-and-resume)
-    + [Userspace IOAPIC by default](#userspace-ioapic-by-default)
-    + [PCI BAR reprogramming](#pci-bar-reprogramming)
-    + [New `cloud-hypervisor` organization](#new--cloud-hypervisor--organization)
-    + [Contributors](#contributors)
+    - [Dynamic virtual CPUs addition](#dynamic-virtual-cpus-addition)
+    - [Programmatic firmware tables generation](#programmatic-firmware-tables-generation)
+    - [Filesystem and block devices vhost-user backends](#filesystem-and-block-devices-vhost-user-backends)
+    - [Guest pause and resume](#guest-pause-and-resume)
+    - [Userspace IOAPIC by default](#userspace-ioapic-by-default)
+    - [PCI BAR reprogramming](#pci-bar-reprogramming)
+    - [New `cloud-hypervisor` organization](#new-cloud-hypervisor-organization)
+    - [Contributors](#contributors-1)
 - [v0.3.0](#v030)
-    + [Block device offloading](#block-device-offloading)
-    + [Network device backend](#network-device-backend)
-    + [Virtual sockets](#virtual-sockets)
-    + [HTTP based API](#http-based-api)
-    + [Memory mapped virtio transport](#memory-mapped-virtio-transport)
-    + [Paravirtualized IOMMU](#paravirtualized-iommu)
-    + [Ubuntu 19.10](#ubuntu-1910)
-    + [Guest large memory](#guest-large-memory)
+    - [Block device offloading](#block-device-offloading)
+    - [Network device backend](#network-device-backend)
+    - [Virtual sockets](#virtual-sockets)
+    - [HTTP based API](#http-based-api)
+    - [Memory mapped virtio transport](#memory-mapped-virtio-transport)
+    - [Paravirtualized IOMMU](#paravirtualized-iommu)
+    - [Ubuntu 19.10](#ubuntu-1910)
+    - [Large memory guests](#large-memory-guests)
 - [v0.2.0](#v020)
-    + [Network device offloading](#network-device-offloading)
-    + [Minimal hardware-reduced ACPI](#minimal-hardware-reduced-acpi)
-    + [Debug I/O port](#debug-i-o-port)
-    + [Improved direct device assignment](#improved-direct-device-assignment)
-    + [Improved shared filesystem](#improved-shared-filesystem)
-    + [Ubuntu bionic based CI](#ubuntu-bionic-based-ci)
+    - [Network device offloading](#network-device-offloading)
+    - [Minimal hardware-reduced ACPI](#minimal-hardware-reduced-acpi)
+    - [Debug I/O port](#debug-io-port)
+    - [Improved direct device assignment](#improved-direct-device-assignment)
+    - [Improved shared filesystem](#improved-shared-filesystem)
+    - [Ubuntu bionic based CI](#ubuntu-bionic-based-ci)
 - [v0.1.0](#v010)
-    + [Shared filesystem](#shared-filesystem)
-    + [Initial direct device assignment support](#initial-direct-device-assignment-support)
-    + [Userspace IOAPIC](#userspace-ioapic)
-    + [Virtual persistent memory](#virtual-persistent-memory)
-    + [Linux kernel bzImage](#linux-kernel-bzimage)
-    + [Console over virtio](#console-over-virtio)
-    + [Unit testing](#unit-testing)
-    + [Integration tests parallelization](#integration-tests-parallelization)
+    - [Shared filesystem](#shared-filesystem)
+    - [Initial direct device assignment support](#initial-direct-device-assignment-support)
+    - [Userspace IOAPIC](#userspace-ioapic)
+    - [Virtual persistent memory](#virtual-persistent-memory)
+    - [Linux kernel bzImage](#linux-kernel-bzimage)
+    - [Console over virtio](#console-over-virtio)
+    - [Unit testing](#unit-testing)
+    - [Integration tests parallelization](#integration-tests-parallelization)
+
+# v0.5.1
+
+This is a bugfix release branched off v0.5.0. It contains the following fixes:
+
+* Update DiskConfig to contain missing disk control features (#790) - Samuel Ortiz and Sergio Lopez
+* Prevent memory overcommit via virtio-fs (#763) - Sebastien Boeuf
+* Fixed error reporting for resize command - Samuel Ortiz
+* Double reboot workaround (#783) - Rob Bradford
+* Various CI and development tooling fixes - Sebastien Boeuf, Samuel Ortiz, Rob Bradford
+
+# v0.5.0
+
+This release has been tracked through the [0.5.0 project](https://github.com/cloud-hypervisor/cloud-hypervisor/projects/6).
+
+Highlights for `cloud-hypervisor` version 0.5.0 include:
+
+### Virtual Machine Dynamic Resizing
+
+With 0.4.0 we added support for CPU hot plug, and 0.5.0 adds CPU hot unplug and
+memory hot plug as well. This allows to dynamically resize Cloud Hypervisor
+guests which is needed for e.g. Kubernetes related use cases.
+The memory hot plug implementation is based on the same framework as the CPU hot
+plug/unplug one, i.e. hardware-reduced ACPI notifications to the guest.
+
+Next on our VM resizing roadmap is the PCI devices hotplug feature.
+
+### Multi-Queue, Multi-Threaded Paravirtualization
+
+We enhanced our virtio networking and block support by having both devices use
+multiple I/O queues handled by multiple threads. This improves our default
+paravirtualized networking and block devices throughput.
+
+### New Interrupt Management Framework
+
+We improved our interrupt management implementation by introducing an Interrupt
+Manager framework, based on the currently on-going [rust-vmm vm-device](https://github.com/rust-vmm/vm-device)
+crates discussions. This move made the code significantly cleaner, and allowed
+us to remove several KVM related dependencies from crates like the PCI and
+virtio ones.
+
+### Development Tools
+
+In order to provide a better developer experience, we worked on improving our
+build, development and testing tools.
+Somehow similar to the excellent
+[Firecracker's devtool](https://github.com/firecracker-microvm/firecracker/blob/master/tools/devtool),
+we now provide a [dev_cli script](https://github.com/cloud-hypervisor/cloud-hypervisor/blob/master/scripts/dev_cli.sh).
+
+With this new tool, our users and contributors will be able to build and test
+Cloud Hypervisor through a containerized environment.
+
+### Kata Containers Integration
+
+We spent some significant time and efforts debugging and fixing our integration
+with the [Kata Containers](https://github.com/kata-containers) project. Cloud
+Hypervisor is now a fully supported Kata Containers hypervisor, and is
+integrated into the project's CI.
+
+### Contributors
+
+Many thanks to everyone that contributed to the 0.5.0 release:
+
+* Bo Chen <chen.bo@intel.com>
+* Cathy Zhang <cathy.zhang@intel.com>
+* Qiu Wenbo <qiuwenbo@phytium.com.cn>
+* Rob Bradford <robert.bradford@intel.com>
+* Samuel Ortiz <sameo@linux.intel.com>
+* Sebastien Boeuf <sebastien.boeuf@intel.com>
+* Sergio Lopez <slp@redhat.com>
+* Yang Zhong <yang.zhong@intel.com>
 
 # v0.4.0
 

resources/Dockerfile

@@ -0,0 +1,53 @@
+FROM ubuntu:18.04 as dev
+
+ARG RUST_TOOLCHAIN="1.40.0"
+ARG CLH_SRC_DIR="/cloud-hypervisor"
+ARG CLH_BUILD_DIR="$CLH_SRC_DIR/build"
+ARG CARGO_REGISTRY_DIR="$CLH_BUILD_DIR/cargo_registry"
+ARG CARGO_GIT_REGISTRY_DIR="$CLH_BUILD_DIR/cargo_git_registry"
+
+ENV CARGO_HOME=/usr/local/rust
+ENV RUSTUP_HOME=$CARGO_HOME
+ENV PATH="$PATH:$CARGO_HOME/bin"
+
+# Install all CI dependencies
+RUN apt-get update
+RUN apt-get -yq upgrade
+RUN DEBIAN_FRONTEND=noninteractive apt-get install -yq \
+	build-essential \
+	bc \
+	docker.io \
+	curl \
+	wget \
+	sudo \
+	mtools \
+	libssl-dev \
+	pkg-config \
+	flex \
+	bison \
+	libelf-dev \
+	qemu-utils \
+	qemu-system \
+	libglib2.0-dev \
+	libpixman-1-dev \
+	libseccomp-dev \
+	libcap-ng-dev \
+	socat \
+	dosfstools \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
+
+# Install the rust toolchain
+RUN nohup curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain "$RUST_TOOLCHAIN" \
+    && rustup component add rustfmt \
+    && rustup component add clippy \
+    && cargo install cargo-audit \
+    && rm -rf "$CARGO_HOME/registry" \
+    && ln -s "$CARGO_REGISTRY_DIR" "$CARGO_HOME/registry" \
+    && rm -rf "$CARGO_HOME/git" \
+    && ln -s "$CARGO_GIT_REGISTRY_DIR" "$CARGO_HOME/git"
+
+# Set the rust environment
+RUN echo 'source $CARGO_HOME/env' >> $HOME/.bashrc \
+    && mkdir $HOME/.cargo \
+    && ln -s $CARGO_HOME/env $HOME/.cargo/env

resources/linux-virtio-fs-virtio-iommu-config

@@ -1,6 +1,6 @@
 #
 # Automatically generated file; DO NOT EDIT.
-# Linux/x86 5.3.0-rc3 Kernel Configuration
+# Linux/x86 5.5.0-rc1 Kernel Configuration
 #
 
 #
@@ -11,6 +11,7 @@ CONFIG_GCC_VERSION=80300
 CONFIG_CLANG_VERSION=0
 CONFIG_CC_CAN_LINK=y
 CONFIG_CC_HAS_ASM_GOTO=y
+CONFIG_CC_HAS_ASM_INLINE=y
 CONFIG_CC_HAS_WARN_MAYBE_UNINITIALIZED=y
 CONFIG_IRQ_WORK=y
 CONFIG_BUILDTIME_EXTABLE_SORT=y
@@ -21,7 +22,6 @@ CONFIG_THREAD_INFO_IN_TASK=y
 #
 CONFIG_INIT_ENV_ARG_LIMIT=32
 # CONFIG_COMPILE_TEST is not set
-# CONFIG_HEADER_TEST is not set
 CONFIG_LOCALVERSION=""
 # CONFIG_LOCALVERSION_AUTO is not set
 CONFIG_BUILD_SALT=""
@@ -139,6 +139,7 @@ CONFIG_HAVE_UNSTABLE_SCHED_CLOCK=y
 
 CONFIG_ARCH_SUPPORTS_NUMA_BALANCING=y
 CONFIG_ARCH_WANT_BATCHED_UNMAP_TLB_FLUSH=y
+CONFIG_CC_HAS_INT128=y
 CONFIG_ARCH_SUPPORTS_INT128=y
 CONFIG_NUMA_BALANCING=y
 # CONFIG_NUMA_BALANCING_DEFAULT_ENABLED is not set
@@ -187,7 +188,6 @@ CONFIG_EXPERT=y
 CONFIG_MULTIUSER=y
 CONFIG_SGETMASK_SYSCALL=y
 CONFIG_SYSFS_SYSCALL=y
-# CONFIG_SYSCTL_SYSCALL is not set
 CONFIG_FHANDLE=y
 CONFIG_POSIX_TIMERS=y
 CONFIG_PRINTK=y
@@ -301,6 +301,7 @@ CONFIG_PARAVIRT=y
 CONFIG_PARAVIRT_SPINLOCKS=y
 # CONFIG_XEN is not set
 CONFIG_KVM_GUEST=y
+CONFIG_ARCH_CPUIDLE_HALTPOLL=y
 CONFIG_PVH=y
 CONFIG_KVM_DEBUG_FS=y
 CONFIG_PARAVIRT_TIME_ACCOUNTING=y
@@ -328,7 +329,6 @@ CONFIG_CPU_SUP_ZHAOXIN=y
 CONFIG_HPET_TIMER=y
 CONFIG_DMI=y
 # CONFIG_GART_IOMMU is not set
-# CONFIG_CALGARY_IOMMU is not set
 # CONFIG_MAXSMP is not set
 CONFIG_NR_CPUS_RANGE_BEGIN=2
 CONFIG_NR_CPUS_RANGE_END=512
@@ -354,6 +354,7 @@ CONFIG_PERF_EVENTS_INTEL_CSTATE=y
 CONFIG_X86_16BIT=y
 CONFIG_X86_ESPFIX64=y
 CONFIG_X86_VSYSCALL_EMULATION=y
+# CONFIG_X86_IOPL_IOPERM is not set
 # CONFIG_I8K is not set
 # CONFIG_MICROCODE is not set
 CONFIG_X86_MSR=y
@@ -361,7 +362,6 @@ CONFIG_X86_CPUID=y
 # CONFIG_X86_5LEVEL is not set
 CONFIG_X86_DIRECT_GBPAGES=y
 # CONFIG_X86_CPA_STATISTICS is not set
-CONFIG_ARCH_HAS_MEM_ENCRYPT=y
 CONFIG_AMD_MEM_ENCRYPT=y
 # CONFIG_AMD_MEM_ENCRYPT_ACTIVE_BY_DEFAULT is not set
 CONFIG_NUMA=y
@@ -388,9 +388,12 @@ CONFIG_X86_PAT=y
 CONFIG_ARCH_USES_PG_UNCACHED=y
 CONFIG_ARCH_RANDOM=y
 CONFIG_X86_SMAP=y
-CONFIG_X86_INTEL_UMIP=y
+# CONFIG_X86_UMIP is not set
 # CONFIG_X86_INTEL_MPX is not set
 # CONFIG_X86_INTEL_MEMORY_PROTECTION_KEYS is not set
+CONFIG_X86_INTEL_TSX_MODE_OFF=y
+# CONFIG_X86_INTEL_TSX_MODE_ON is not set
+# CONFIG_X86_INTEL_TSX_MODE_AUTO is not set
 CONFIG_EFI=y
 CONFIG_EFI_STUB=y
 # CONFIG_EFI_MIXED is not set
@@ -404,7 +407,7 @@ CONFIG_SCHED_HRTICK=y
 # CONFIG_KEXEC is not set
 CONFIG_KEXEC_FILE=y
 CONFIG_ARCH_HAS_KEXEC_PURGATORY=y
-CONFIG_KEXEC_VERIFY_SIG=y
+# CONFIG_KEXEC_SIG is not set
 # CONFIG_CRASH_DUMP is not set
 CONFIG_PHYSICAL_START=0x1000000
 CONFIG_RELOCATABLE=y
@@ -473,12 +476,11 @@ CONFIG_ACPI_PROCESSOR=y
 CONFIG_ACPI_HOTPLUG_CPU=y
 # CONFIG_ACPI_PROCESSOR_AGGREGATOR is not set
 CONFIG_ACPI_THERMAL=y
-CONFIG_ACPI_NUMA=y
 CONFIG_ARCH_HAS_ACPI_TABLE_UPGRADE=y
 # CONFIG_ACPI_DEBUG is not set
 # CONFIG_ACPI_PCI_SLOT is not set
 CONFIG_ACPI_CONTAINER=y
-# CONFIG_ACPI_HOTPLUG_MEMORY is not set
+CONFIG_ACPI_HOTPLUG_MEMORY=y
 CONFIG_ACPI_HOTPLUG_IOAPIC=y
 # CONFIG_ACPI_SBS is not set
 # CONFIG_ACPI_HED is not set
@@ -486,6 +488,7 @@ CONFIG_ACPI_HOTPLUG_IOAPIC=y
 # CONFIG_ACPI_BGRT is not set
 CONFIG_ACPI_REDUCED_HARDWARE_ONLY=y
 # CONFIG_ACPI_NFIT is not set
+CONFIG_ACPI_NUMA=y
 # CONFIG_ACPI_HMAT is not set
 CONFIG_HAVE_ACPI_APEI=y
 CONFIG_HAVE_ACPI_APEI_NMI=y
@@ -493,7 +496,6 @@ CONFIG_HAVE_ACPI_APEI_NMI=y
 # CONFIG_DPTF_POWER is not set
 # CONFIG_PMIC_OPREGION is not set
 # CONFIG_ACPI_CONFIGFS is not set
-CONFIG_ACPI_IORT=y
 CONFIG_X86_PM_TIMER=y
 # CONFIG_SFI is not set
 
@@ -536,6 +538,8 @@ CONFIG_CPU_IDLE=y
 CONFIG_CPU_IDLE_GOV_LADDER=y
 CONFIG_CPU_IDLE_GOV_MENU=y
 # CONFIG_CPU_IDLE_GOV_TEO is not set
+# CONFIG_CPU_IDLE_GOV_HALTPOLL is not set
+# CONFIG_HALTPOLL_CPUIDLE is not set
 # end of CPU Idle
 
 CONFIG_INTEL_IDLE=y
@@ -584,6 +588,7 @@ CONFIG_EFI_RUNTIME_WRAPPERS=y
 # CONFIG_EFI_TEST is not set
 # CONFIG_APPLE_PROPERTIES is not set
 # CONFIG_RESET_ATTACK_MITIGATION is not set
+# CONFIG_EFI_RCI2_TABLE is not set
 # end of EFI (Extensible Firmware Interface) Support
 
 CONFIG_EFI_EARLYCON=y
@@ -643,6 +648,7 @@ CONFIG_ARCH_HAS_SET_MEMORY=y
 CONFIG_ARCH_HAS_SET_DIRECT_MAP=y
 CONFIG_HAVE_ARCH_THREAD_STRUCT_WHITELIST=y
 CONFIG_ARCH_WANTS_DYNAMIC_TASK_STRUCT=y
+CONFIG_HAVE_ASM_MODVERSIONS=y
 CONFIG_HAVE_REGS_AND_STACK_ACCESS_API=y
 CONFIG_HAVE_RSEQ=y
 CONFIG_HAVE_FUNCTION_ARG_ACCESS_API=y
@@ -688,17 +694,16 @@ CONFIG_ARCH_MMAP_RND_BITS=28
 CONFIG_HAVE_COPY_THREAD_TLS=y
 CONFIG_HAVE_STACK_VALIDATION=y
 CONFIG_HAVE_RELIABLE_STACKTRACE=y
-CONFIG_64BIT_TIME=y
+# CONFIG_COMPAT_32BIT_TIME is not set
 CONFIG_HAVE_ARCH_VMAP_STACK=y
 CONFIG_VMAP_STACK=y
 CONFIG_ARCH_HAS_STRICT_KERNEL_RWX=y
 CONFIG_STRICT_KERNEL_RWX=y
 CONFIG_ARCH_HAS_STRICT_MODULE_RWX=y
-CONFIG_ARCH_HAS_REFCOUNT=y
-# CONFIG_REFCOUNT_FULL is not set
 CONFIG_HAVE_ARCH_PREL32_RELOCATIONS=y
 CONFIG_ARCH_USE_MEMREMAP_PROT=y
 # CONFIG_LOCK_EVENT_COUNTS is not set
+CONFIG_ARCH_HAS_MEM_ENCRYPT=y
 
 #
 # GCOV-based kernel profiling
@@ -717,6 +722,7 @@ CONFIG_BASE_SMALL=0
 CONFIG_MODULES_TREE_LOOKUP=y
 CONFIG_BLOCK=y
 CONFIG_BLK_SCSI_REQUEST=y
+CONFIG_BLK_CGROUP_RWSTAT=y
 CONFIG_BLK_DEV_BSG=y
 CONFIG_BLK_DEV_BSGLIB=y
 CONFIG_BLK_DEV_INTEGRITY=y
@@ -726,6 +732,7 @@ CONFIG_BLK_DEV_THROTTLING=y
 CONFIG_BLK_CMDLINE_PARSER=y
 CONFIG_BLK_WBT=y
 # CONFIG_BLK_CGROUP_IOLATENCY is not set
+# CONFIG_BLK_CGROUP_IOCOST is not set
 CONFIG_BLK_WBT_MQ=y
 CONFIG_BLK_DEBUG_FS=y
 # CONFIG_BLK_SED_OPAL is not set
@@ -845,10 +852,10 @@ CONFIG_GENERIC_EARLY_IOREMAP=y
 CONFIG_ARCH_HAS_PTE_DEVMAP=y
 CONFIG_ZONE_DEVICE=y
 CONFIG_DEV_PAGEMAP_OPS=y
-# CONFIG_HMM_MIRROR is not set
 # CONFIG_DEVICE_PRIVATE is not set
 CONFIG_PERCPU_STATS=y
 # CONFIG_GUP_BENCHMARK is not set
+# CONFIG_READ_ONLY_THP_FOR_FS is not set
 CONFIG_ARCH_HAS_PTE_SPECIAL=y
 # end of Memory Management options
 
@@ -865,6 +872,7 @@ CONFIG_UNIX_SCM=y
 CONFIG_UNIX_DIAG=y
 CONFIG_TLS=y
 # CONFIG_TLS_DEVICE is not set
+# CONFIG_TLS_TOE is not set
 CONFIG_XFRM=y
 CONFIG_XFRM_ALGO=y
 CONFIG_XFRM_USER=y
@@ -988,7 +996,6 @@ CONFIG_PCIEAER=y
 # CONFIG_PCIEAER_INJECT is not set
 # CONFIG_PCIE_ECRC is not set
 CONFIG_PCIEASPM=y
-# CONFIG_PCIEASPM_DEBUG is not set
 CONFIG_PCIEASPM_DEFAULT=y
 # CONFIG_PCIEASPM_POWERSAVE is not set
 # CONFIG_PCIEASPM_POWER_SUPERSAVE is not set
@@ -1013,12 +1020,6 @@ CONFIG_PCI_LABEL=y
 #
 # PCI controller drivers
 #
-
-#
-# Cadence PCIe controllers support
-#
-# end of Cadence PCIe controllers support
-
 # CONFIG_VMD is not set
 
 #
@@ -1027,6 +1028,11 @@ CONFIG_PCI_LABEL=y
 # CONFIG_PCIE_DW_PLAT_HOST is not set
 # CONFIG_PCI_MESON is not set
 # end of DesignWare PCI Core Support
+
+#
+# Cadence PCIe controllers support
+#
+# end of Cadence PCIe controllers support
 # end of PCI controller drivers
 
 #
@@ -1063,6 +1069,7 @@ CONFIG_EXTRA_FIRMWARE=""
 CONFIG_FW_LOADER_USER_HELPER=y
 # CONFIG_FW_LOADER_USER_HELPER_FALLBACK is not set
 # CONFIG_FW_LOADER_COMPRESS is not set
+# CONFIG_FW_CACHE is not set
 # end of Firmware loader
 
 CONFIG_ALLOW_DEV_COREDUMP=y
@@ -1131,7 +1138,6 @@ CONFIG_VIRTIO_BLK=y
 # CONFIG_DUMMY_IRQ is not set
 # CONFIG_IBM_ASM is not set
 # CONFIG_PHANTOM is not set
-# CONFIG_SGI_IOC4 is not set
 # CONFIG_TIFM_CORE is not set
 # CONFIG_ENCLOSURE_SERVICES is not set
 # CONFIG_HP_ILO is not set
@@ -1165,41 +1171,9 @@ CONFIG_VIRTIO_BLK=y
 #
 # Intel MIC & related support
 #
-
-#
-# Intel MIC Bus Driver
-#
 # CONFIG_INTEL_MIC_BUS is not set
-
-#
-# SCIF Bus Driver
-#
 # CONFIG_SCIF_BUS is not set
-
-#
-# VOP Bus Driver
-#
 # CONFIG_VOP_BUS is not set
-
-#
-# Intel MIC Host Driver
-#
-
-#
-# Intel MIC Card Driver
-#
-
-#
-# SCIF Driver
-#
-
-#
-# Intel MIC Coprocessor State Management (COSM) Drivers
-#
-
-#
-# VOP Driver
-#
 # end of Intel MIC & related support
 
 # CONFIG_GENWQE is not set
@@ -1253,10 +1227,6 @@ CONFIG_VIRTIO_NET=y
 # CONFIG_NLMON is not set
 # CONFIG_ARCNET is not set
 
-#
-# CAIF transport drivers
-#
-
 #
 # Distributed Switch Architecture drivers
 #
@@ -1379,11 +1349,11 @@ CONFIG_SERIAL_8250_EXAR=y
 CONFIG_SERIAL_8250_NR_UARTS=1
 CONFIG_SERIAL_8250_RUNTIME_UARTS=1
 # CONFIG_SERIAL_8250_EXTENDED is not set
+CONFIG_SERIAL_8250_DWLIB=y
 # CONFIG_SERIAL_8250_DW is not set
 # CONFIG_SERIAL_8250_RT288X is not set
 CONFIG_SERIAL_8250_LPSS=y
 CONFIG_SERIAL_8250_MID=y
-# CONFIG_SERIAL_8250_MOXA is not set
 
 #
 # Non-8250 serial port support
@@ -1400,6 +1370,7 @@ CONFIG_SERIAL_ARC=y
 CONFIG_SERIAL_ARC_NR_PORTS=1
 # CONFIG_SERIAL_RP2 is not set
 # CONFIG_SERIAL_FSL_LPUART is not set
+# CONFIG_SERIAL_FSL_LINFLEXUART is not set
 # end of Serial drivers
 
 CONFIG_SERIAL_DEV_BUS=y
@@ -1428,6 +1399,7 @@ CONFIG_DEVPORT=y
 # end of Character devices
 
 # CONFIG_RANDOM_TRUST_CPU is not set
+# CONFIG_RANDOM_TRUST_BOOTLOADER is not set
 
 #
 # I2C support
@@ -1462,6 +1434,7 @@ CONFIG_PTP_1588_CLOCK=y
 # Enable PHYLIB and NETWORK_PHY_TIMESTAMPING to see the additional clocks.
 #
 CONFIG_PTP_1588_CLOCK_KVM=y
+# CONFIG_PTP_1588_CLOCK_IDTCM is not set
 # end of PTP clock support
 
 # CONFIG_PINCTRL is not set
@@ -1483,7 +1456,6 @@ CONFIG_THERMAL_GOV_FAIR_SHARE=y
 CONFIG_THERMAL_GOV_STEP_WISE=y
 # CONFIG_THERMAL_GOV_BANG_BANG is not set
 CONFIG_THERMAL_GOV_USER_SPACE=y
-# CONFIG_THERMAL_GOV_POWER_ALLOCATOR is not set
 # CONFIG_THERMAL_EMULATION is not set
 
 #
@@ -1510,7 +1482,6 @@ CONFIG_BCMA_POSSIBLE=y
 #
 # Multifunction device drivers
 #
-# CONFIG_MFD_CROS_EC is not set
 # CONFIG_MFD_MADERA is not set
 # CONFIG_HTC_PASIC3 is not set
 # CONFIG_MFD_INTEL_QUARK_I2C_GPIO is not set
@@ -1619,7 +1590,7 @@ CONFIG_LCD_CLASS_DEVICE=y
 CONFIG_BACKLIGHT_CLASS_DEVICE=y
 CONFIG_BACKLIGHT_GENERIC=y
 # CONFIG_BACKLIGHT_APPLE is not set
-# CONFIG_BACKLIGHT_PM8941_WLED is not set
+# CONFIG_BACKLIGHT_QCOM_WLED is not set
 # CONFIG_BACKLIGHT_SAHARA is not set
 # end of Backlight & LCD device support
 
@@ -1683,7 +1654,6 @@ CONFIG_HID_GENERIC=y
 # CONFIG_HID_KENSINGTON is not set
 # CONFIG_HID_LCPOWER is not set
 # CONFIG_HID_LENOVO is not set
-# CONFIG_HID_LOGITECH is not set
 # CONFIG_HID_MAGICMOUSE is not set
 # CONFIG_HID_MALTRON is not set
 # CONFIG_HID_MAYFLASH is not set
@@ -1727,7 +1697,6 @@ CONFIG_HID_REDRAGON=y
 
 CONFIG_USB_OHCI_LITTLE_ENDIAN=y
 # CONFIG_USB_SUPPORT is not set
-# CONFIG_UWB is not set
 # CONFIG_MMC is not set
 # CONFIG_MEMSTICK is not set
 # CONFIG_NEW_LEDS is not set
@@ -1759,6 +1728,7 @@ CONFIG_DW_DMAC_CORE=y
 # CONFIG_DW_EDMA is not set
 # CONFIG_DW_EDMA_PCIE is not set
 CONFIG_HSU_DMA=y
+# CONFIG_SF_PDMA is not set
 
 #
 # DMA Clients
@@ -1811,9 +1781,11 @@ CONFIG_VIRTIO_MMIO_CMDLINE_DEVICES=y
 # CONFIG_HYPERV is not set
 # end of Microsoft Hyper-V guest support
 
+# CONFIG_GREYBUS is not set
 # CONFIG_STAGING is not set
 # CONFIG_X86_PLATFORM_DEVICES is not set
 CONFIG_PMC_ATOM=y
+# CONFIG_MFD_CROS_EC is not set
 # CONFIG_CHROME_PLATFORMS is not set
 # CONFIG_MELLANOX_PLATFORM is not set
 CONFIG_CLKDEV_LOOKUP=y
@@ -1854,6 +1826,7 @@ CONFIG_IOMMU_DMA=y
 # CONFIG_INTEL_IOMMU is not set
 # CONFIG_IRQ_REMAP is not set
 CONFIG_VIRTIO_IOMMU=y
+CONFIG_VIRTIO_IOMMU_TOPOLOGY=y
 
 #
 # Remoteproc drivers
@@ -1899,13 +1872,6 @@ CONFIG_VIRTIO_IOMMU=y
 #
 # end of i.MX SoC drivers
 
-#
-# IXP4xx SoC drivers
-#
-# CONFIG_IXP4XX_QMGR is not set
-# CONFIG_IXP4XX_NPE is not set
-# end of IXP4xx SoC drivers
-
 #
 # Qualcomm SoC drivers
 #
@@ -2023,6 +1989,7 @@ CONFIG_EXPORTFS=y
 CONFIG_FILE_LOCKING=y
 CONFIG_MANDATORY_FILE_LOCKING=y
 CONFIG_FS_ENCRYPTION=y
+# CONFIG_FS_VERITY is not set
 CONFIG_FSNOTIFY=y
 CONFIG_DNOTIFY=y
 CONFIG_INOTIFY_USER=y
@@ -2149,6 +2116,7 @@ CONFIG_NLS_MAC_TURKISH=y
 CONFIG_NLS_UTF8=y
 # CONFIG_DLM is not set
 # CONFIG_UNICODE is not set
+CONFIG_IO_WQ=y
 # end of File systems
 
 #
@@ -2195,8 +2163,8 @@ CONFIG_CRYPTO_ALGAPI=y
 CONFIG_CRYPTO_ALGAPI2=y
 CONFIG_CRYPTO_AEAD=y
 CONFIG_CRYPTO_AEAD2=y
-CONFIG_CRYPTO_BLKCIPHER=y
-CONFIG_CRYPTO_BLKCIPHER2=y
+CONFIG_CRYPTO_SKCIPHER=y
+CONFIG_CRYPTO_SKCIPHER2=y
 CONFIG_CRYPTO_HASH=y
 CONFIG_CRYPTO_HASH2=y
 CONFIG_CRYPTO_RNG=y
@@ -2226,6 +2194,8 @@ CONFIG_CRYPTO_GLUE_HELPER_X86=y
 # CONFIG_CRYPTO_DH is not set
 # CONFIG_CRYPTO_ECDH is not set
 # CONFIG_CRYPTO_ECRDSA is not set
+# CONFIG_CRYPTO_CURVE25519 is not set
+# CONFIG_CRYPTO_CURVE25519_X86 is not set
 
 #
 # Authenticated Encryption with Associated Data
@@ -2234,16 +2204,7 @@ CONFIG_CRYPTO_GLUE_HELPER_X86=y
 CONFIG_CRYPTO_GCM=y
 # CONFIG_CRYPTO_CHACHA20POLY1305 is not set
 # CONFIG_CRYPTO_AEGIS128 is not set
-# CONFIG_CRYPTO_AEGIS128L is not set
-# CONFIG_CRYPTO_AEGIS256 is not set
 # CONFIG_CRYPTO_AEGIS128_AESNI_SSE2 is not set
-# CONFIG_CRYPTO_AEGIS128L_AESNI_SSE2 is not set
-# CONFIG_CRYPTO_AEGIS256_AESNI_SSE2 is not set
-# CONFIG_CRYPTO_MORUS640 is not set
-# CONFIG_CRYPTO_MORUS640_SSE2 is not set
-# CONFIG_CRYPTO_MORUS1280 is not set
-# CONFIG_CRYPTO_MORUS1280_SSE2 is not set
-# CONFIG_CRYPTO_MORUS1280_AVX2 is not set
 CONFIG_CRYPTO_SEQIV=y
 CONFIG_CRYPTO_ECHAINIV=y
 
@@ -2263,6 +2224,7 @@ CONFIG_CRYPTO_XTS=y
 # CONFIG_CRYPTO_NHPOLY1305_SSE2 is not set
 # CONFIG_CRYPTO_NHPOLY1305_AVX2 is not set
 # CONFIG_CRYPTO_ADIANTUM is not set
+# CONFIG_CRYPTO_ESSIV is not set
 
 #
 # Hash modes
@@ -2280,6 +2242,9 @@ CONFIG_CRYPTO_CRC32C_INTEL=y
 # CONFIG_CRYPTO_CRC32 is not set
 # CONFIG_CRYPTO_CRC32_PCLMUL is not set
 # CONFIG_CRYPTO_XXHASH is not set
+# CONFIG_CRYPTO_BLAKE2B is not set
+# CONFIG_CRYPTO_BLAKE2S is not set
+# CONFIG_CRYPTO_BLAKE2S_X86 is not set
 CONFIG_CRYPTO_CRCT10DIF=y
 # CONFIG_CRYPTO_CRCT10DIF_PCLMUL is not set
 CONFIG_CRYPTO_GHASH=y
@@ -2310,10 +2275,8 @@ CONFIG_CRYPTO_SHA512=y
 #
 CONFIG_CRYPTO_AES=y
 # CONFIG_CRYPTO_AES_TI is not set
-CONFIG_CRYPTO_AES_X86_64=y
 CONFIG_CRYPTO_AES_NI_INTEL=y
 # CONFIG_CRYPTO_ANUBIS is not set
-CONFIG_CRYPTO_LIB_ARC4=y
 CONFIG_CRYPTO_ARC4=y
 # CONFIG_CRYPTO_BLOWFISH is not set
 # CONFIG_CRYPTO_BLOWFISH_X86_64 is not set
@@ -2370,6 +2333,21 @@ CONFIG_CRYPTO_USER_API=y
 # CONFIG_CRYPTO_USER_API_SKCIPHER is not set
 CONFIG_CRYPTO_USER_API_RNG=y
 # CONFIG_CRYPTO_USER_API_AEAD is not set
+
+#
+# Crypto library routines
+#
+CONFIG_CRYPTO_LIB_AES=y
+CONFIG_CRYPTO_LIB_ARC4=y
+# CONFIG_CRYPTO_LIB_BLAKE2S is not set
+# CONFIG_CRYPTO_LIB_CHACHA is not set
+# CONFIG_CRYPTO_LIB_CURVE25519 is not set
+CONFIG_CRYPTO_LIB_DES=y
+CONFIG_CRYPTO_LIB_POLY1305_RSIZE=4
+CONFIG_CRYPTO_LIB_POLY1305_GENERIC=y
+# CONFIG_CRYPTO_LIB_POLY1305 is not set
+# CONFIG_CRYPTO_LIB_CHACHA20POLY1305 is not set
+CONFIG_CRYPTO_LIB_SHA256=y
 # CONFIG_CRYPTO_HW is not set
 # CONFIG_ASYMMETRIC_KEY_TYPE is not set
 
@@ -2434,7 +2412,6 @@ CONFIG_CPU_RMAP=y
 CONFIG_DQL=y
 CONFIG_NLATTR=y
 CONFIG_IRQ_POLL=y
-# CONFIG_DIMLIB is not set
 CONFIG_UCS2_STRING=y
 CONFIG_HAVE_GENERIC_VDSO=y
 CONFIG_GENERIC_GETTIMEOFDAY=y
@@ -2443,6 +2420,7 @@ CONFIG_FONT_SUPPORT=y
 CONFIG_FONT_8x8=y
 CONFIG_FONT_8x16=y
 CONFIG_ARCH_HAS_PMEM_API=y
+CONFIG_MEMREGION=y
 CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE=y
 CONFIG_ARCH_STACKWALK=y
 CONFIG_SBITMAP=y
@@ -2463,6 +2441,8 @@ CONFIG_CONSOLE_LOGLEVEL_QUIET=4
 CONFIG_MESSAGE_LOGLEVEL_DEFAULT=4
 # CONFIG_BOOT_PRINTK_DELAY is not set
 CONFIG_DYNAMIC_DEBUG=y
+CONFIG_SYMBOLIC_ERRNAME=y
+# CONFIG_DEBUG_BUGVERBOSE is not set
 # end of printk and dmesg options
 
 #
@@ -2473,8 +2453,6 @@ CONFIG_DYNAMIC_DEBUG=y
 CONFIG_FRAME_WARN=2048
 CONFIG_STRIP_ASM_SYMS=y
 # CONFIG_READABLE_ASM is not set
-CONFIG_UNUSED_SYMBOLS=y
-CONFIG_DEBUG_FS=y
 # CONFIG_HEADERS_INSTALL is not set
 CONFIG_OPTIMIZE_INLINING=y
 CONFIG_DEBUG_SECTION_MISMATCH=y
@@ -2484,9 +2462,20 @@ CONFIG_STACK_VALIDATION=y
 # CONFIG_DEBUG_FORCE_WEAK_PER_CPU is not set
 # end of Compile-time checks and compiler options
 
+#
+# Generic Kernel Debugging Instruments
+#
 CONFIG_MAGIC_SYSRQ=y
 CONFIG_MAGIC_SYSRQ_DEFAULT_ENABLE=0x1
 CONFIG_MAGIC_SYSRQ_SERIAL=y
+CONFIG_DEBUG_FS=y
+CONFIG_HAVE_ARCH_KGDB=y
+# CONFIG_KGDB is not set
+CONFIG_ARCH_HAS_UBSAN_SANITIZE_ALL=y
+# CONFIG_UBSAN is not set
+CONFIG_UBSAN_ALIGNMENT=y
+# end of Generic Kernel Debugging Instruments
+
 CONFIG_DEBUG_KERNEL=y
 CONFIG_DEBUG_MISC=y
 
@@ -2504,39 +2493,42 @@ CONFIG_DEBUG_MISC=y
 CONFIG_HAVE_DEBUG_KMEMLEAK=y
 # CONFIG_DEBUG_KMEMLEAK is not set
 # CONFIG_DEBUG_STACK_USAGE is not set
+# CONFIG_SCHED_STACK_END_CHECK is not set
 # CONFIG_DEBUG_VM is not set
 CONFIG_ARCH_HAS_DEBUG_VIRTUAL=y
 # CONFIG_DEBUG_VIRTUAL is not set
 CONFIG_DEBUG_MEMORY_INIT=y
 # CONFIG_DEBUG_PER_CPU_MAPS is not set
 CONFIG_HAVE_ARCH_KASAN=y
+CONFIG_HAVE_ARCH_KASAN_VMALLOC=y
 CONFIG_CC_HAS_KASAN_GENERIC=y
 # CONFIG_KASAN is not set
 CONFIG_KASAN_STACK=1
 # end of Memory Debugging
 
-CONFIG_ARCH_HAS_KCOV=y
-CONFIG_CC_HAS_SANCOV_TRACE_PC=y
-# CONFIG_KCOV is not set
 # CONFIG_DEBUG_SHIRQ is not set
 
 #
-# Debug Lockups and Hangs
+# Debug Oops, Lockups and Hangs
 #
+# CONFIG_PANIC_ON_OOPS is not set
+CONFIG_PANIC_ON_OOPS_VALUE=0
+CONFIG_PANIC_TIMEOUT=0
 # CONFIG_SOFTLOCKUP_DETECTOR is not set
 CONFIG_HARDLOCKUP_CHECK_TIMESTAMP=y
 # CONFIG_HARDLOCKUP_DETECTOR is not set
 # CONFIG_DETECT_HUNG_TASK is not set
 # CONFIG_WQ_WATCHDOG is not set
-# end of Debug Lockups and Hangs
+# end of Debug Oops, Lockups and Hangs
 
-# CONFIG_PANIC_ON_OOPS is not set
-CONFIG_PANIC_ON_OOPS_VALUE=0
-CONFIG_PANIC_TIMEOUT=0
+#
+# Scheduler Debugging
+#
 # CONFIG_SCHED_DEBUG is not set
 CONFIG_SCHED_INFO=y
 # CONFIG_SCHEDSTATS is not set
-# CONFIG_SCHED_STACK_END_CHECK is not set
+# end of Scheduler Debugging
+
 # CONFIG_DEBUG_TIMEKEEPING is not set
 
 #
@@ -2560,11 +2552,17 @@ CONFIG_LOCK_DEBUGGING_SUPPORT=y
 CONFIG_STACKTRACE=y
 # CONFIG_WARN_ALL_UNSEEDED_RANDOM is not set
 # CONFIG_DEBUG_KOBJECT is not set
-CONFIG_DEBUG_BUGVERBOSE=y
+
+#
+# Debug kernel data structures
+#
 CONFIG_DEBUG_LIST=y
 # CONFIG_DEBUG_PLIST is not set
 # CONFIG_DEBUG_SG is not set
 # CONFIG_DEBUG_NOTIFIERS is not set
+CONFIG_BUG_ON_DATA_CORRUPTION=y
+# end of Debug kernel data structures
+
 # CONFIG_DEBUG_CREDENTIALS is not set
 
 #
@@ -2580,14 +2578,13 @@ CONFIG_RCU_CPU_STALL_TIMEOUT=59
 # CONFIG_DEBUG_WQ_FORCE_RR_CPU is not set
 # CONFIG_DEBUG_BLOCK_EXT_DEVT is not set
 # CONFIG_CPU_HOTPLUG_STATE_CONTROL is not set
-# CONFIG_NOTIFIER_ERROR_INJECTION is not set
-# CONFIG_FAULT_INJECTION is not set
 # CONFIG_LATENCYTOP is not set
 CONFIG_USER_STACKTRACE_SUPPORT=y
 CONFIG_HAVE_FUNCTION_TRACER=y
 CONFIG_HAVE_FUNCTION_GRAPH_TRACER=y
 CONFIG_HAVE_DYNAMIC_FTRACE=y
 CONFIG_HAVE_DYNAMIC_FTRACE_WITH_REGS=y
+CONFIG_HAVE_DYNAMIC_FTRACE_WITH_DIRECT_CALLS=y
 CONFIG_HAVE_FTRACE_MCOUNT_RECORD=y
 CONFIG_HAVE_SYSCALL_TRACEPOINTS=y
 CONFIG_HAVE_FENTRY=y
@@ -2595,6 +2592,17 @@ CONFIG_HAVE_C_RECORDMCOUNT=y
 CONFIG_TRACING_SUPPORT=y
 # CONFIG_FTRACE is not set
 # CONFIG_PROVIDE_OHCI1394_DMA_INIT is not set
+# CONFIG_KUNIT is not set
+# CONFIG_NOTIFIER_ERROR_INJECTION is not set
+# CONFIG_FAULT_INJECTION is not set
+# end of Kernel hacking
+
+#
+# Kernel Testing and Coverage
+#
+CONFIG_ARCH_HAS_KCOV=y
+CONFIG_CC_HAS_SANCOV_TRACE_PC=y
+# CONFIG_KCOV is not set
 CONFIG_RUNTIME_TESTING_MENU=y
 # CONFIG_LKDTM is not set
 # CONFIG_TEST_LIST_SORT is not set
@@ -2625,16 +2633,14 @@ CONFIG_RUNTIME_TESTING_MENU=y
 # CONFIG_TEST_STACKINIT is not set
 # CONFIG_TEST_MEMINIT is not set
 # CONFIG_MEMTEST is not set
-CONFIG_BUG_ON_DATA_CORRUPTION=y
 # CONFIG_SAMPLES is not set
-CONFIG_HAVE_ARCH_KGDB=y
-# CONFIG_KGDB is not set
-CONFIG_ARCH_HAS_UBSAN_SANITIZE_ALL=y
-# CONFIG_UBSAN is not set
-CONFIG_UBSAN_ALIGNMENT=y
 CONFIG_ARCH_HAS_DEVMEM_IS_ALLOWED=y
 CONFIG_STRICT_DEVMEM=y
 # CONFIG_IO_STRICT_DEVMEM is not set
+
+#
+# x86 Debugging
+#
 CONFIG_TRACE_IRQFLAGS_SUPPORT=y
 CONFIG_X86_VERBOSE_BOOTUP=y
 CONFIG_EARLY_PRINTK=y
@@ -2646,6 +2652,7 @@ CONFIG_EARLY_PRINTK=y
 CONFIG_DOUBLEFAULT=y
 # CONFIG_DEBUG_TLBFLUSH is not set
 CONFIG_HAVE_MMIOTRACE_SUPPORT=y
+# CONFIG_X86_DECODER_SELFTEST is not set
 CONFIG_IO_DELAY_0X80=y
 # CONFIG_IO_DELAY_0XED is not set
 # CONFIG_IO_DELAY_UDELAY is not set
@@ -2659,4 +2666,5 @@ CONFIG_IO_DELAY_0X80=y
 # CONFIG_UNWINDER_ORC is not set
 CONFIG_UNWINDER_FRAME_POINTER=y
 # CONFIG_UNWINDER_GUESS is not set
-# end of Kernel hacking
+# end of x86 Debugging
+# end of Kernel Testing and Coverage

scripts/dev_cli.sh

@@ -0,0 +1,314 @@
+#!/bin/bash
+
+# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# Copyright © 2020 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+CLI_NAME="Cloud Hypervisor"
+
+CTR_IMAGE_TAG="cloudhypervisor/dev"
+CTR_IMAGE_VERSION="v1"
+CTR_IMAGE="${CTR_IMAGE_TAG}:${CTR_IMAGE_VERSION}"
+
+DOCKER_RUNTIME="docker"
+
+# Host paths
+CLH_SCRIPTS_DIR=$(cd "$(dirname "$0")" && pwd)
+CLH_ROOT_DIR=$(cd "${CLH_SCRIPTS_DIR}/.." && pwd)
+CLH_BUILD_DIR="${CLH_ROOT_DIR}/build"
+CLH_CARGO_TARGET="${CLH_BUILD_DIR}/cargo_target"
+CLH_DOCKERFILE="${CLH_SCRIPTS_DIR}/../resources/Dockerfile"
+CLH_CTR_BUILD_DIR="/tmp/cloud-hypervisor/ctr-build"
+CLH_INTEGRATION_WORKLOADS="${HOME}/workloads"
+
+# Container paths
+CTR_CLH_ROOT_DIR="/cloud-hypervisor"
+CTR_CLH_CARGO_TARGET="${CTR_CLH_ROOT_DIR}/build/cargo_target"
+CTR_CLH_INTEGRATION_WORKLOADS="/root/workloads"
+
+# Cargo paths
+# Full path to the cargo registry dir on the host. This appears on the host
+# because we want to persist the cargo registry across container invocations.
+# Otherwise, any rust crates from crates.io would be downloaded again each time
+# we build or test.
+CARGO_REGISTRY_DIR="${CLH_BUILD_DIR}/cargo_registry"
+
+# Full path to the cargo git registry on the host. This serves the same purpose
+# as CARGO_REGISTRY_DIR, for crates downloaded from GitHub repos instead of
+# crates.io.
+CARGO_GIT_REGISTRY_DIR="${CLH_BUILD_DIR}/cargo_git_registry"
+
+# Full path to the cargo target dir on the host.
+CARGO_TARGET_DIR="${CLH_BUILD_DIR}/cargo_target"
+
+# Send a decorated message to stdout, followed by a new line
+#
+say() {
+    [ -t 1 ] && [ -n "$TERM" ] \
+        && echo "$(tput setaf 2)[$CLI_NAME]$(tput sgr0) $*" \
+        || echo "[$CLI_NAME] $*"
+}
+
+# Send a decorated message to stdout, without a trailing new line
+#
+say_noln() {
+    [ -t 1 ] && [ -n "$TERM" ] \
+        && echo -n "$(tput setaf 2)[$CLI_NAME]$(tput sgr0) $*" \
+        || echo "[$CLI_NAME] $*"
+}
+
+# Send a text message to stderr
+#
+say_err() {
+    [ -t 2 ] && [ -n "$TERM" ] \
+        && echo "$(tput setaf 1)[$CLI_NAME] $*$(tput sgr0)" 1>&2 \
+        || echo "[$CLI_NAME] $*" 1>&2
+}
+
+# Send a warning-highlighted text to stdout
+say_warn() {
+    [ -t 1 ] && [ -n "$TERM" ] \
+        && echo "$(tput setaf 3)[$CLI_NAME] $*$(tput sgr0)" \
+        || echo "[$CLI_NAME] $*"
+}
+
+# Exit with an error message and (optional) code
+# Usage: die [-c <error code>] <error message>
+#
+die() {
+    code=1
+    [[ "$1" = "-c" ]] && {
+        code="$2"
+        shift 2
+    }
+    say_err "$@"
+    exit $code
+}
+
+# Exit with an error message if the last exit code is not 0
+#
+ok_or_die() {
+    code=$?
+    [[ $code -eq 0 ]] || die -c $code "$@"
+}
+
+# Make sure the build/ dirs are available. Exit if we can't create them.
+# Upon returning from this call, the caller can be certain the build/ dirs exist.
+#
+ensure_build_dir() {
+    for dir in "$CLH_BUILD_DIR" \
+		   "$CLH_INTEGRATION_WORKLOADS" \
+		   "$CLH_CTR_BUILD_DIR" \
+		   "$CARGO_TARGET_DIR" \
+		   "$CARGO_REGISTRY_DIR" \
+		   "$CARGO_GIT_REGISTRY_DIR"; do
+        mkdir -p "$dir" || die "Error: cannot create dir $dir"
+        [ -x "$dir" ] && [ -w "$dir" ] || \
+            {
+                say "Wrong permissions for $dir. Attempting to fix them ..."
+                chmod +x+w "$dir"
+            } || \
+            die "Error: wrong permissions for $dir. Should be +x+w"
+    done
+}
+
+cmd_help() {
+    echo ""
+    echo "Cloud Hypervisor $(basename $0)"
+    echo "Usage: $(basename $0) <command> [<command args>]"
+    echo ""
+    echo "Available commands:"
+    echo ""
+    echo "    build [--debug|--release] [-- [<cargo args>]]"
+    echo "        Build the Cloud Hypervisor binaries."
+    echo "        --debug               Build the debug binaries. This is the default."
+    echo "        --release             Build the release binaries."
+    echo ""
+    echo "    tests [--unit|--cargo|--all] [-- [<cargo test args>]]"
+    echo "        Run the Cloud Hypervisor tests."
+    echo "        --unit               Run the unit tests."
+    echo "        --cargo              Run the cargo tests."
+    echo "        --integration        Run the integration tests."
+    echo "        --all                Run all tests."
+    echo ""
+    echo "    build-container [--type]"
+    echo "        Build the Cloud Hypervisor container."
+    echo "        --dev                Build dev container. This is the default."
+    echo ""
+    echo "    clean [<cargo args>]]"
+    echo "        Remove the Cloud Hypervisor artifacts."
+    echo ""
+    echo "    help"
+    echo "        Display this help message."
+    echo ""
+}
+
+cmd_build() {
+    build="debug"
+
+    while [ $# -gt 0 ]; do
+	case "$1" in
+            "-h"|"--help")  { cmd_help; exit 1;     } ;;
+            "--debug")      { build="debug";      } ;;
+            "--release")    { build="release";    } ;;
+            "--")           { shift; break;         } ;;
+            *)
+		die "Unknown build argument: $1. Please use --help for help."
+		;;
+	esac
+	shift
+    done
+
+    cargo_args=("$@")
+    [ $build = "release" ] && cargo_args+=("--release")
+
+    $DOCKER_RUNTIME run \
+	   -ti \
+	   --workdir "$CTR_CLH_ROOT_DIR" \
+	   --rm \
+	   --volume /dev:/dev \
+	   --volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR" \
+	   "$CTR_IMAGE" \
+	   cargo build \
+	         --target-dir "$CTR_CLH_CARGO_TARGET" \
+	         "${cargo_args[@]}"
+
+    ret=$?
+
+    # If `cargo build` was successful, let's copy the binaries to a more
+    # accessible location.
+    [ $ret -eq 0 ] && {
+        cargo_bin_dir="$CLH_CARGO_TARGET/$build"
+        say "Binaries placed under $cargo_bin_dir"
+    }
+}
+
+cmd_clean() {
+    cargo_args=("$@")
+
+    $DOCKER_RUNTIME run \
+	   -ti \
+	   --workdir "$CTR_CLH_ROOT_DIR" \
+	   --rm \
+	   --volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR" \
+	   "$CTR_IMAGE" \
+	   cargo clean \
+	         --target-dir "$CTR_CLH_CARGO_TARGET" \
+	         "${cargo_args[@]}"
+    }
+
+cmd_tests() {
+    unit=false
+    cargo=false
+    integration=false
+
+    while [ $# -gt 0 ]; do
+	case "$1" in
+            "-h"|"--help")    { cmd_help; exit 1;     } ;;
+            "--unit")         { unit=true;      } ;;
+            "--cargo")        { cargo=true;    } ;;
+	    "--integration")  { integration=true;    } ;;
+	    "--all")          { cargo=true; unit=true; integration=true;  } ;;
+            "--")             { shift; break;         } ;;
+            *)
+		die "Unknown tests argument: $1. Please use --help for help."
+		;;
+	esac
+	shift
+    done
+
+    if [ "$unit" = true ] ;  then
+	say "Running unit tests..."
+	$DOCKER_RUNTIME run \
+	       -ti \
+	       --workdir "$CTR_CLH_ROOT_DIR" \
+	       --rm \
+	       --device /dev/kvm \
+	       --device /dev/net/tun \
+	       --cap-add net_admin \
+	       --volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR" \
+	       "$CTR_IMAGE" \
+	       ./scripts/run_unit_tests.sh "$@"
+    fi
+
+    if [ "$cargo" = true ] ;  then
+	say "Running cargo tests..."
+	$DOCKER_RUNTIME run \
+	       -ti \
+	       --workdir "$CTR_CLH_ROOT_DIR" \
+	       --rm \
+	       --volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR" \
+	       "$CTR_IMAGE" \
+	       ./scripts/run_cargo_tests.sh
+    fi
+
+    if [ "$integration" = true ] ;  then
+	say "Running integration tests..."
+	$DOCKER_RUNTIME run \
+	       -ti \
+	       --workdir "$CTR_CLH_ROOT_DIR" \
+	       --rm \
+	       --privileged \
+	       --volume /dev:/dev \
+	       --volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR" \
+	       --volume "$CLH_INTEGRATION_WORKLOADS:$CTR_CLH_INTEGRATION_WORKLOADS" \
+	       "$CTR_IMAGE" \
+	       ./scripts/run_integration_tests.sh "$@"
+    fi
+}
+
+cmd_build-container() {
+    container_type="dev"
+
+    while [ $# -gt 0 ]; do
+	case "$1" in
+            "-h"|"--help")  { cmd_help; exit 1;     } ;;
+            "--dev")        { container_type="dev"; } ;;
+            "--")           { shift; break;         } ;;
+            *)
+		die "Unknown build-container argument: $1. Please use --help for help."
+		;;
+	esac
+	shift
+    done
+
+    BUILD_DIR=/tmp/cloud-hypervisor/container/
+
+    mkdir -p $BUILD_DIR
+    cp $CLH_DOCKERFILE $BUILD_DIR
+
+    $DOCKER_RUNTIME build \
+	   --target $container_type \
+	   -t $CTR_IMAGE \
+	   -f $BUILD_DIR/Dockerfile \
+	   $BUILD_DIR
+}
+
+# Parse main command line args.
+#
+while [ $# -gt 0 ]; do
+    case "$1" in
+        -h|--help)              { cmd_help; exit 1;     } ;;
+        -y|--unattended)        { OPT_UNATTENDED=true;  } ;;
+        -*)
+            die "Unknown arg: $1. Please use \`$0 help\` for help."
+            ;;
+        *)
+            break
+            ;;
+    esac
+    shift
+done
+
+# $1 is now a command name. Check if it is a valid command and, if so,
+# run it.
+#
+declare -f "cmd_$1" > /dev/null
+ok_or_die "Unknown command: $1. Please use \`$0 help\` for help."
+
+cmd=cmd_$1
+shift
+
+ensure_build_dir
+
+$cmd "$@"

scripts/run_cargo_tests.sh

@@ -5,21 +5,24 @@ set -x
 source $HOME/.cargo/env
 
 # Install cargo components
-rustup component add clippy
-rustup component add rustfmt
-cargo install --force cargo-audit
+time rustup component add clippy
+time rustup component add rustfmt
+time which cargo-audit || cargo install cargo-audit
 
 # Run cargo builds and checks
-cargo rustc --bin cloud-hypervisor -- -D warnings
-cargo rustc --bin vhost_user_net -- -D warnings
-cargo test
-cargo audit
-cargo rustc --bin cloud-hypervisor --no-default-features --features "pci,acpi"  -- -D warnings
-cargo rustc --bin vhost_user_net --no-default-features --features "pci,acpi"  -- -D warnings
-cargo clippy --all-targets --all-features -- -D warnings
-cargo rustc --bin cloud-hypervisor --no-default-features --features "pci"  -- -D warnings
-cargo rustc --bin vhost_user_net --no-default-features --features "pci"  -- -D warnings
-cargo rustc --bin cloud-hypervisor --no-default-features --features "mmio"  -- -D warnings
-cargo rustc --bin vhost_user_net --no-default-features --features "mmio"  -- -D warnings
-find . \( -name "*.rs" ! -wholename "*/out/*.rs" \) | xargs rustfmt --check
-cargo build --release
+time cargo clippy --all-targets --all-features -- -D warnings
+time cargo rustc --bin cloud-hypervisor -- -D warnings
+time cargo rustc --bin vhost_user_net -- -D warnings
+time cargo test
+time cargo audit
+time cargo clippy --all-targets --no-default-features --features "pci,acpi" -- -D warnings
+time cargo rustc --bin cloud-hypervisor --no-default-features --features "pci,acpi"  -- -D warnings
+time cargo rustc --bin vhost_user_net --no-default-features --features "pci,acpi"  -- -D warnings
+time cargo clippy --all-targets --no-default-features --features "pci" -- -D warnings
+time cargo rustc --bin cloud-hypervisor --no-default-features --features "pci"  -- -D warnings
+time cargo rustc --bin vhost_user_net --no-default-features --features "pci"  -- -D warnings
+time cargo clippy --all-targets --no-default-features --features "mmio" -- -D warnings
+time cargo rustc --bin cloud-hypervisor --no-default-features --features "mmio"  -- -D warnings
+time cargo rustc --bin vhost_user_net --no-default-features --features "mmio"  -- -D warnings
+time cargo fmt -- --check
+time cargo build --release

scripts/run_integration_tests.sh

@@ -10,7 +10,7 @@ FW_URL=$(curl --silent https://api.github.com/repos/cloud-hypervisor/rust-hyperv
 FW="$WORKLOADS_DIR/hypervisor-fw"
 if [ ! -f "$FW" ]; then
     pushd $WORKLOADS_DIR
-    wget --quiet $FW_URL
+    time wget --quiet $FW_URL
     popd
 fi
 
@@ -19,7 +19,7 @@ CLEAR_OS_IMAGE_URL="https://cloudhypervisorstorage.blob.core.windows.net/images/
 CLEAR_OS_IMAGE="$WORKLOADS_DIR/$CLEAR_OS_IMAGE_NAME"
 if [ ! -f "$CLEAR_OS_IMAGE" ]; then
     pushd $WORKLOADS_DIR
-    wget --quiet $CLEAR_OS_IMAGE_URL
+    time wget --quiet $CLEAR_OS_IMAGE_URL
     popd
 fi
 
@@ -27,7 +27,7 @@ CLEAR_OS_RAW_IMAGE_NAME="clear-31310-cloudguest-raw.img"
 CLEAR_OS_RAW_IMAGE="$WORKLOADS_DIR/$CLEAR_OS_RAW_IMAGE_NAME"
 if [ ! -f "$CLEAR_OS_RAW_IMAGE" ]; then
     pushd $WORKLOADS_DIR
-    qemu-img convert -p -f qcow2 -O raw $CLEAR_OS_IMAGE_NAME $CLEAR_OS_RAW_IMAGE_NAME
+    time qemu-img convert -p -f qcow2 -O raw $CLEAR_OS_IMAGE_NAME $CLEAR_OS_RAW_IMAGE_NAME
     popd
 fi
 
@@ -36,7 +36,7 @@ BIONIC_OS_IMAGE_URL="https://cloudhypervisorstorage.blob.core.windows.net/images
 BIONIC_OS_IMAGE="$WORKLOADS_DIR/$BIONIC_OS_IMAGE_NAME"
 if [ ! -f "$BIONIC_OS_IMAGE" ]; then
     pushd $WORKLOADS_DIR
-    wget --quiet $BIONIC_OS_IMAGE_URL
+    time wget --quiet $BIONIC_OS_IMAGE_URL
     popd
 fi
 
@@ -44,7 +44,7 @@ BIONIC_OS_RAW_IMAGE_NAME="bionic-server-cloudimg-amd64-raw.img"
 BIONIC_OS_RAW_IMAGE="$WORKLOADS_DIR/$BIONIC_OS_RAW_IMAGE_NAME"
 if [ ! -f "$BIONIC_OS_RAW_IMAGE" ]; then
     pushd $WORKLOADS_DIR
-    qemu-img convert -p -f qcow2 -O raw $BIONIC_OS_IMAGE_NAME $BIONIC_OS_RAW_IMAGE_NAME
+    time qemu-img convert -p -f qcow2 -O raw $BIONIC_OS_IMAGE_NAME $BIONIC_OS_RAW_IMAGE_NAME
     popd
 fi
 
@@ -54,7 +54,7 @@ EOAN_OS_IMAGE_URL="https://cloudhypervisorstorage.blob.core.windows.net/images/$
 EOAN_OS_IMAGE="$WORKLOADS_DIR/$EOAN_OS_IMAGE_NAME"
 if [ ! -f "$EOAN_OS_IMAGE" ]; then
     pushd $WORKLOADS_DIR
-    wget --quiet $EOAN_OS_IMAGE_URL
+    time wget --quiet $EOAN_OS_IMAGE_URL
     popd
 fi
 
@@ -62,10 +62,17 @@ EOAN_OS_RAW_IMAGE_NAME="eoan-server-cloudimg-amd64-raw.img"
 EOAN_OS_RAW_IMAGE="$WORKLOADS_DIR/$EOAN_OS_RAW_IMAGE_NAME"
 if [ ! -f "$EOAN_OS_RAW_IMAGE" ]; then
     pushd $WORKLOADS_DIR
-    qemu-img convert -p -f qcow2 -O raw $EOAN_OS_IMAGE_NAME $EOAN_OS_RAW_IMAGE_NAME
+    time qemu-img convert -p -f qcow2 -O raw $EOAN_OS_IMAGE_NAME $EOAN_OS_RAW_IMAGE_NAME
     popd
 fi
 
+pushd $WORKLOADS_DIR
+curl --silent "https://cloudhypervisorstorage.blob.core.windows.net/images/sha1sums" | sha1sum --check
+if [ $? -ne 0 ]; then
+    echo "sha1sum validation of images failed, remove invalid images to fix the issue."
+    exit 1
+fi
+popd
 
 # Build custom kernel based on virtio-pmem and virtio-fs upstream patches
 VMLINUX_IMAGE="$WORKLOADS_DIR/vmlinux"
@@ -76,10 +83,10 @@ LINUX_CUSTOM_DIR="linux-custom"
 if [ ! -f "$VMLINUX_IMAGE" ]; then
     SRCDIR=$PWD
     pushd $WORKLOADS_DIR
-    git clone --depth 1 "https://github.com/cloud-hypervisor/linux.git" -b "virtio-fs-virtio-iommu" $LINUX_CUSTOM_DIR
+    time git clone --depth 1 "https://github.com/cloud-hypervisor/linux.git" -b "virtio-fs-virtio-iommu-topo-5.5-rc1" $LINUX_CUSTOM_DIR
     pushd $LINUX_CUSTOM_DIR
     cp $SRCDIR/resources/linux-virtio-fs-virtio-iommu-config .config
-    make bzImage -j `nproc`
+    time make bzImage -j `nproc`
     cp vmlinux $VMLINUX_IMAGE
     cp arch/x86/boot/bzImage $BZIMAGE_IMAGE
     popd
@@ -91,10 +98,10 @@ VIRTIOFSD="$WORKLOADS_DIR/virtiofsd"
 QEMU_DIR="qemu_build"
 if [ ! -f "$VIRTIOFSD" ]; then
     pushd $WORKLOADS_DIR
-    git clone --depth 1 "https://gitlab.com/virtio-fs/qemu.git" -b "virtio-fs-dev" $QEMU_DIR
+    git clone --depth 1 "https://github.com/sboeuf/qemu.git" -b "virtio-fs" $QEMU_DIR
     pushd $QEMU_DIR
-    ./configure --prefix=$PWD --target-list=x86_64-softmmu
-    make virtiofsd -j `nproc`
+    time ./configure --prefix=$PWD --target-list=x86_64-softmmu
+    time make virtiofsd -j `nproc`
     cp virtiofsd $VIRTIOFSD
     popd
     rm -rf $QEMU_DIR
@@ -124,12 +131,11 @@ if [ ! -d "$SHARED_DIR" ]; then
 fi
 
 VFIO_DIR="$WORKLOADS_DIR/vfio"
-if [ ! -d "$VFIO_DIR" ]; then
-    mkdir -p $VFIO_DIR
-    cp $CLEAR_OS_IMAGE $VFIO_DIR
-    cp $FW $VFIO_DIR
-    cp $VMLINUX_IMAGE $VFIO_DIR
-fi
+rm -rf $VFIO_DIR
+mkdir -p $VFIO_DIR
+cp $CLEAR_OS_IMAGE $VFIO_DIR
+cp $FW $VFIO_DIR
+cp $VMLINUX_IMAGE $VFIO_DIR
 
 # VFIO test network setup.
 # We reserve a different IP class for it: 172.17.0.0/24.
@@ -159,18 +165,22 @@ cp target/release/cloud-hypervisor $VFIO_DIR
 
 # Enable KSM with some reasonable parameters so that it won't take too long
 # for the memory to be merged between two processes.
-sudo bash -c "echo 10000 > /sys/kernel/mm/ksm/pages_to_scan"
+sudo bash -c "echo 1000000 > /sys/kernel/mm/ksm/pages_to_scan"
 sudo bash -c "echo 10 > /sys/kernel/mm/ksm/sleep_millisecs"
 sudo bash -c "echo 1 > /sys/kernel/mm/ksm/run"
 
 # Ensure test binary has the same caps as the cloud-hypervisor one
-cargo test --no-run --features "integration_tests" -- --nocapture
+time cargo test --no-run --features "integration_tests" -- --nocapture
 ls target/debug/deps/cloud_hypervisor-* | xargs -n 1 sudo setcap cap_net_admin+ep
 
+# test_vfio relies on hugepages
+echo 4096 | sudo tee /proc/sys/vm/nr_hugepages
+sudo chmod a+rwX /dev/hugepages
+
 sudo adduser $USER kvm
 newgrp kvm << EOF
 export RUST_BACKTRACE=1
-cargo test --features "integration_tests" -- --nocapture
+time cargo test --features "integration_tests" "$@" -- --nocapture
 EOF
 RES=$?
 
@@ -181,7 +191,7 @@ if [ $RES -eq 0 ]; then
 
     newgrp kvm << EOF
 export RUST_BACKTRACE=1
-cargo test --features "integration_tests,mmio" -- --nocapture
+time cargo test --features "integration_tests,mmio" "$@" -- --nocapture
 EOF
 
     RES=$?

scripts/run_unit_tests.sh

@@ -2,23 +2,13 @@
 
 source $HOME/.cargo/env
 
-# More effective than just cargo test --all as it captures crates within crates
-for f in $(find . -name Cargo.toml -printf '%h\n' | sort -u); do
-  pushd $f > /dev/null;
-  cargo test --no-run || exit 1;
-  popd > /dev/null;
-done
-
+cargo test --workspace --no-run
 pushd target/debug
 ls  | grep net_util | grep -v "\.d" | xargs -n 1 sudo setcap cap_net_admin,cap_net_raw+ep
 popd
 
 sudo adduser $USER kvm
 newgrp kvm << EOF || exit 1
-for f in \$(find . -name Cargo.toml -printf '%h\n' | sort -u); do
-  pushd \$f > /dev/null;
   export RUST_BACKTRACE=1
-  cargo test || exit 1;
-  popd > /dev/null;
-done
+  cargo test --workspace "$@" || exit 1;
 EOF

src/bin/vhost_user_blk.rs

@@ -10,237 +10,10 @@
 
 #[macro_use(crate_version, crate_authors)]
 extern crate clap;
-extern crate log;
-extern crate vhost_rs;
-extern crate vhost_user_backend;
-extern crate vm_virtio;
+extern crate vhost_user_block;
 
 use clap::{App, Arg};
-use epoll;
-use log::*;
-use std::fs::File;
-use std::fs::OpenOptions;
-use std::io::Read;
-use std::io::{Seek, SeekFrom, Write};
-use std::mem;
-use std::path::PathBuf;
-use std::process;
-use std::slice;
-use std::sync::{Arc, RwLock};
-use std::vec::Vec;
-
-use qcow::{self, ImageType, QcowFile};
-
-use vhost_rs::vhost_user::message::*;
-use vhost_user_backend::{VhostUserBackend, VhostUserDaemon, Vring, VringWorker};
-
-use virtio_bindings::bindings::virtio_blk::*;
-use vm_memory::{Bytes, GuestMemoryError, GuestMemoryMmap};
-use vm_virtio::block::{build_disk_image_id, Request};
-
-const QUEUE_SIZE: usize = 1024;
-const NUM_QUEUES: usize = 1;
-const SECTOR_SHIFT: u8 = 9;
-const SECTOR_SIZE: u64 = (0x01 as u64) << SECTOR_SHIFT;
-const BLK_SIZE: u32 = 512;
-
-trait DiskFile: Read + Seek + Write + Send + Sync {}
-impl<D: Read + Seek + Write + Send + Sync> DiskFile for D {}
-
-pub type Result<T> = std::result::Result<T, Error>;
-pub type VhostUserBackendResult<T> = std::result::Result<T, std::io::Error>;
-
-#[derive(Debug)]
-pub enum Error {
-    /// Failed to detect image type.
-    DetectImageType,
-    /// Bad memory address.
-    GuestMemory(GuestMemoryError),
-    /// Can't open image file.
-    OpenImage,
-    /// Failed to parse image parameter.
-    ParseImageParam,
-    /// Failed to parse sock parameter.
-    ParseSockParam,
-}
-
-struct VhostUserBlkBackend {
-    mem: Option<GuestMemoryMmap>,
-    vring_worker: Option<Arc<VringWorker>>,
-    disk_image: Box<dyn DiskFile>,
-    disk_image_id: Vec<u8>,
-    disk_nsectors: u64,
-    config: virtio_blk_config,
-}
-
-impl VhostUserBlkBackend {
-    pub fn new(image_path: String) -> Result<Self> {
-        let raw_img: File = OpenOptions::new()
-            .read(true)
-            .write(true)
-            .open(&image_path)
-            .unwrap();
-
-        let image_id = build_disk_image_id(&PathBuf::from(&image_path));
-        let image_type = qcow::detect_image_type(&raw_img).unwrap();
-        let mut image = match image_type {
-            ImageType::Raw => Box::new(vm_virtio::RawFile::new(raw_img)) as Box<dyn DiskFile>,
-            ImageType::Qcow2 => Box::new(QcowFile::from(raw_img).unwrap()) as Box<dyn DiskFile>,
-        };
-
-        let nsectors = (image.seek(SeekFrom::End(0)).unwrap() as u64) / SECTOR_SIZE;
-        let mut config = virtio_blk_config::default();
-
-        config.capacity = nsectors;
-        config.blk_size = BLK_SIZE;
-        config.size_max = 65535;
-        config.seg_max = 128 - 2;
-        config.min_io_size = 1;
-        config.opt_io_size = 1;
-        config.num_queues = 1;
-
-        Ok(VhostUserBlkBackend {
-            mem: None,
-            vring_worker: None,
-            disk_image: image,
-            disk_image_id: image_id,
-            disk_nsectors: nsectors,
-            config,
-        })
-    }
-
-    pub fn process_queue(&mut self, vring: &mut Vring) -> bool {
-        let mut used_any = false;
-        let mem = match self.mem.as_ref() {
-            Some(m) => m,
-            None => return false,
-        };
-
-        while let Some(head) = vring.mut_queue().iter(mem).next() {
-            debug!("got an element in the queue");
-            let len;
-            match Request::parse(&head, mem) {
-                Ok(request) => {
-                    debug!("element is a valid request");
-                    let status = match request.execute(
-                        &mut self.disk_image,
-                        self.disk_nsectors,
-                        mem,
-                        &self.disk_image_id,
-                    ) {
-                        Ok(l) => {
-                            len = l;
-                            VIRTIO_BLK_S_OK
-                        }
-                        Err(e) => {
-                            len = 1;
-                            e.status()
-                        }
-                    };
-                    mem.write_obj(status, request.status_addr).unwrap();
-                }
-                Err(err) => {
-                    error!("failed to parse available descriptor chain: {:?}", err);
-                    len = 0;
-                }
-            }
-            vring.mut_queue().add_used(mem, head.index, len);
-            used_any = true;
-        }
-
-        used_any
-    }
-}
-
-impl VhostUserBackend for VhostUserBlkBackend {
-    fn num_queues(&self) -> usize {
-        NUM_QUEUES
-    }
-
-    fn max_queue_size(&self) -> usize {
-        QUEUE_SIZE
-    }
-
-    fn features(&self) -> u64 {
-        1 << VIRTIO_BLK_F_MQ
-            | 1 << VIRTIO_F_VERSION_1
-            | VhostUserVirtioFeatures::PROTOCOL_FEATURES.bits()
-    }
-
-    fn protocol_features(&self) -> VhostUserProtocolFeatures {
-        VhostUserProtocolFeatures::CONFIG
-    }
-
-    fn update_memory(&mut self, mem: GuestMemoryMmap) -> VhostUserBackendResult<()> {
-        self.mem = Some(mem);
-        Ok(())
-    }
-
-    fn handle_event(
-        &mut self,
-        device_event: u16,
-        evset: epoll::Events,
-        vrings: &[Arc<RwLock<Vring>>],
-    ) -> VhostUserBackendResult<bool> {
-        if evset != epoll::Events::EPOLLIN {
-            warn!("invalid events operation");
-            return Ok(false);
-        }
-
-        debug!("event received: {:?}", device_event);
-
-        let mut vring = vrings[0].write().unwrap();
-        if self.process_queue(&mut vring) {
-            debug!("signalling queue");
-            vring.signal_used_queue().unwrap();
-        }
-
-        Ok(false)
-    }
-
-    fn get_config(&self, _offset: u32, _size: u32) -> Vec<u8> {
-        // self.config is a statically allocated virtio_blk_config
-        let buf = unsafe {
-            slice::from_raw_parts(
-                &self.config as *const virtio_blk_config as *const _,
-                mem::size_of::<virtio_blk_config>(),
-            )
-        };
-
-        buf.to_vec()
-    }
-}
-
-pub struct VhostUserBlkBackendConfig<'a> {
-    pub image: &'a str,
-    pub sock: &'a str,
-}
-
-impl<'a> VhostUserBlkBackendConfig<'a> {
-    pub fn parse(backend: &'a str) -> Result<Self> {
-        let params_list: Vec<&str> = backend.split(',').collect();
-
-        let mut image: &str = "";
-        let mut sock: &str = "";
-
-        for param in params_list.iter() {
-            if param.starts_with("image=") {
-                image = &param[6..];
-            } else if param.starts_with("sock=") {
-                sock = &param[5..];
-            }
-        }
-
-        if image.is_empty() {
-            return Err(Error::ParseImageParam);
-        }
-        if sock.is_empty() {
-            return Err(Error::ParseSockParam);
-        }
-
-        Ok(VhostUserBlkBackendConfig { image, sock })
-    }
-}
+use vhost_user_block::start_block_backend;
 
 fn main() {
     let cmd_arguments = App::new("vhost-user-blk backend")
@@ -248,53 +21,18 @@ fn main() {
         .author(crate_authors!())
         .about("Launch a vhost-user-blk backend.")
         .arg(
-            Arg::with_name("backend")
-                .long("backend")
+            Arg::with_name("block-backend")
+                .long("block-backend")
                 .help(
-                    "Backend parameters \"image=<image_path>,\
-                     sock=<socket_path>\"",
+                    "vhost-user-block backend parameters \"image=<image_path>,\
+                     sock=<socket_path>,num_queues=<number_of_queues>,\
+                     readonly=true|false,direct=true|false\"",
                 )
                 .takes_value(true)
                 .min_values(1),
         )
         .get_matches();
 
-    let vhost_user_blk_backend = cmd_arguments.value_of("backend").unwrap();
-
-    let backend_config = match VhostUserBlkBackendConfig::parse(vhost_user_blk_backend) {
-        Ok(config) => config,
-        Err(e) => {
-            println!("Failed parsing parameters {:?}", e);
-            process::exit(1);
-        }
-    };
-
-    let blk_backend = Arc::new(RwLock::new(
-        VhostUserBlkBackend::new(backend_config.image.to_string()).unwrap(),
-    ));
-
-    debug!("blk_backend is created!\n");
-
-    let name = "vhost-user-blk-backend";
-    let mut blk_daemon = VhostUserDaemon::new(
-        name.to_string(),
-        backend_config.sock.to_string(),
-        blk_backend.clone(),
-    )
-    .unwrap();
-    debug!("blk_daemon is created!\n");
-
-    let vring_worker = blk_daemon.get_vring_worker();
-
-    blk_backend.write().unwrap().vring_worker = Some(vring_worker);
-
-    if let Err(e) = blk_daemon.start() {
-        println!(
-            "failed to start daemon for vhost-user-blk with error: {:?}\n",
-            e
-        );
-        process::exit(1);
-    }
-
-    blk_daemon.wait().unwrap();
+    let backend_command = cmd_arguments.value_of("block-backend").unwrap();
+    start_block_backend(backend_command);
 }

src/bin/vhost_user_fs.rs

@@ -44,7 +44,7 @@ type VhostUserBackendResult<T> = std::result::Result<T, std::io::Error>;
 #[derive(Debug)]
 enum Error {
     /// Failed to create kill eventfd.
-    CreateKillEventFd,
+    CreateKillEventFd(io::Error),
     /// Failed to handle event other than input event.
     HandleEventNotEpollIn,
     /// Failed to handle unknown event.
@@ -89,7 +89,7 @@ impl<F: FileSystem + Send + Sync + 'static> VhostUserFsBackend<F> {
     fn new(fs: F) -> Result<Self> {
         Ok(VhostUserFsBackend {
             mem: None,
-            kill_evt: EventFd::new(EFD_NONBLOCK).map_err(|_| Error::CreateKillEventFd)?,
+            kill_evt: EventFd::new(EFD_NONBLOCK).map_err(Error::CreateKillEventFd)?,
             server: Arc::new(Server::new(fs)),
         })
     }

src/bin/vhost_user_net.rs

@@ -8,536 +8,10 @@
 
 #[macro_use(crate_version, crate_authors)]
 extern crate clap;
-extern crate log;
-extern crate net_util;
-extern crate vhost_rs;
-extern crate vhost_user_backend;
-extern crate vm_virtio;
+extern crate vhost_user_net;
 
 use clap::{App, Arg};
-use epoll;
-use libc::{self, EAGAIN, EFD_NONBLOCK};
-use log::*;
-use std::cmp;
-use std::fmt;
-use std::io::Read;
-use std::io::{self, Write};
-use std::mem;
-use std::net::Ipv4Addr;
-use std::os::unix::io::AsRawFd;
-use std::process;
-use std::sync::{Arc, RwLock};
-use std::vec::Vec;
-
-use vhost_rs::vhost_user::message::*;
-use vhost_rs::vhost_user::Error as VhostUserError;
-use vhost_user_backend::{VhostUserBackend, VhostUserDaemon, Vring, VringWorker};
-
-use net_gen;
-
-use net_util::{Tap, TapError};
-use virtio_bindings::bindings::virtio_net::*;
-use vm_memory::{Bytes, GuestAddress, GuestMemoryMmap};
-use vmm_sys_util::eventfd::EventFd;
-
-/// The maximum buffer size when segmentation offload is enabled. This
-/// includes the 12-byte virtio net header.
-/// http://docs.oasis-open.org/virtio/virtio/v1.0/virtio-v1.0.html#x1-1740003
-const MAX_BUFFER_SIZE: usize = 65562;
-const QUEUE_SIZE: usize = 1024;
-const NUM_QUEUES: usize = 2;
-
-// The guest has made a buffer available to receive a frame into.
-const RX_QUEUE_EVENT: u16 = 0;
-// The transmit queue has a frame that is ready to send from the guest.
-const TX_QUEUE_EVENT: u16 = 1;
-// A frame is available for reading from the tap device to receive in the guest.
-const RX_TAP_EVENT: u16 = 2;
-// The device has been dropped.
-const KILL_EVENT: u16 = 3;
-
-pub type VhostUserResult<T> = std::result::Result<T, VhostUserError>;
-pub type Result<T> = std::result::Result<T, Error>;
-pub type VhostUserBackendResult<T> = std::result::Result<T, std::io::Error>;
-
-#[derive(Debug)]
-pub enum Error {
-    /// Failed to activate device.
-    BadActivate,
-    /// Failed to create kill eventfd
-    CreateKillEventFd,
-    /// Failed to add event.
-    EpollCtl(io::Error),
-    /// Fail to wait event.
-    EpollWait(io::Error),
-    /// Failed to create EventFd.
-    EpollCreateFd,
-    /// Failed to read Tap.
-    FailedReadTap,
-    /// Failed to signal used queue.
-    FailedSignalingUsedQueue,
-    /// Failed to handle event other than input event.
-    HandleEventNotEpollIn,
-    /// Failed to handle unknown event.
-    HandleEventUnknownEvent,
-    /// Invalid vring address.
-    InvalidVringAddr,
-    /// No vring call fd to notify.
-    NoVringCallFdNotify,
-    /// No memory configured.
-    NoMemoryConfigured,
-    /// Failed to parse sock parameter.
-    ParseSockParam,
-    /// Failed to parse ip parameter.
-    ParseIpParam,
-    /// Failed to parse mask parameter.
-    ParseMaskParam,
-    /// Open tap device failed.
-    TapOpen(TapError),
-    /// Setting tap IP failed.
-    TapSetIp(TapError),
-    /// Setting tap netmask failed.
-    TapSetNetmask(TapError),
-    /// Setting tap interface offload flags failed.
-    TapSetOffload(TapError),
-    /// Setting vnet header size failed.
-    TapSetVnetHdrSize(TapError),
-    /// Enabling tap interface failed.
-    TapEnable(TapError),
-}
-
-impl fmt::Display for Error {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        write!(f, "vhost_user_net_error: {:?}", self)
-    }
-}
-
-impl std::error::Error for Error {}
-
-impl std::convert::From<Error> for std::io::Error {
-    fn from(e: Error) -> Self {
-        std::io::Error::new(io::ErrorKind::Other, e)
-    }
-}
-
-#[derive(Clone)]
-struct TxVirtio {
-    iovec: Vec<(GuestAddress, usize)>,
-    frame_buf: [u8; MAX_BUFFER_SIZE],
-}
-
-impl TxVirtio {
-    fn new() -> Self {
-        TxVirtio {
-            iovec: Vec::new(),
-            frame_buf: [0u8; MAX_BUFFER_SIZE],
-        }
-    }
-}
-
-#[derive(Clone)]
-struct RxVirtio {
-    deferred_frame: bool,
-    deferred_irqs: bool,
-    bytes_read: usize,
-    frame_buf: [u8; MAX_BUFFER_SIZE],
-}
-
-impl RxVirtio {
-    fn new() -> Self {
-        RxVirtio {
-            deferred_frame: false,
-            deferred_irqs: false,
-            bytes_read: 0,
-            frame_buf: [0u8; MAX_BUFFER_SIZE],
-        }
-    }
-}
-
-fn vnet_hdr_len() -> usize {
-    mem::size_of::<virtio_net_hdr_v1>()
-}
-
-struct VhostUserNetBackend {
-    mem: Option<GuestMemoryMmap>,
-    vring_worker: Option<Arc<VringWorker>>,
-    kill_evt: EventFd,
-    tap: Tap,
-    rx: RxVirtio,
-    tx: TxVirtio,
-    rx_tap_listening: bool,
-}
-
-impl std::clone::Clone for VhostUserNetBackend {
-    fn clone(&self) -> Self {
-        VhostUserNetBackend {
-            mem: self.mem.clone(),
-            vring_worker: self.vring_worker.clone(),
-            kill_evt: self.kill_evt.try_clone().unwrap(),
-            tap: self.tap.clone(),
-            rx: self.rx.clone(),
-            tx: self.tx.clone(),
-            rx_tap_listening: self.rx_tap_listening,
-        }
-    }
-}
-
-impl VhostUserNetBackend {
-    /// Create a new virtio network device with the given TAP interface.
-    pub fn new_with_tap(tap: Tap) -> Result<Self> {
-        // Set offload flags to match the virtio features below.
-        tap.set_offload(
-            net_gen::TUN_F_CSUM | net_gen::TUN_F_UFO | net_gen::TUN_F_TSO4 | net_gen::TUN_F_TSO6,
-        )
-        .map_err(Error::TapSetOffload)?;
-
-        let vnet_hdr_size = vnet_hdr_len() as i32;
-        tap.set_vnet_hdr_size(vnet_hdr_size)
-            .map_err(Error::TapSetVnetHdrSize)?;
-
-        let rx = RxVirtio::new();
-        let tx = TxVirtio::new();
-
-        Ok(VhostUserNetBackend {
-            mem: None,
-            vring_worker: None,
-            kill_evt: EventFd::new(EFD_NONBLOCK).map_err(|_| Error::CreateKillEventFd)?,
-            tap,
-            rx,
-            tx,
-            rx_tap_listening: false,
-        })
-    }
-
-    /// Create a new virtio network device with the given IP address and
-    /// netmask.
-    pub fn new(ip_addr: Ipv4Addr, netmask: Ipv4Addr) -> Result<Self> {
-        let tap = Tap::new().map_err(Error::TapOpen)?;
-        tap.set_ip_addr(ip_addr).map_err(Error::TapSetIp)?;
-        tap.set_netmask(netmask).map_err(Error::TapSetNetmask)?;
-        tap.enable().map_err(Error::TapEnable)?;
-
-        Self::new_with_tap(tap)
-    }
-
-    // Copies a single frame from `self.rx.frame_buf` into the guest. Returns true
-    // if a buffer was used, and false if the frame must be deferred until a buffer
-    // is made available by the driver.
-    fn rx_single_frame(&mut self, vring: &mut Vring) -> Result<bool> {
-        let mem = self.mem.as_ref().ok_or(Error::NoMemoryConfigured)?;
-
-        let mut next_desc = vring.mut_queue().iter(&mem).next();
-
-        if next_desc.is_none() {
-            // Queue has no available descriptors
-            if self.rx_tap_listening {
-                self.vring_worker
-                    .as_ref()
-                    .unwrap()
-                    .unregister_listener(
-                        self.tap.as_raw_fd(),
-                        epoll::Events::EPOLLIN,
-                        u64::from(RX_TAP_EVENT),
-                    )
-                    .unwrap();
-                self.rx_tap_listening = false;
-            }
-            return Ok(false);
-        }
-
-        // We just checked that the head descriptor exists.
-        let head_index = next_desc.as_ref().unwrap().index;
-        let mut write_count = 0;
-
-        // Copy from frame into buffer, which may span multiple descriptors.
-        loop {
-            match next_desc {
-                Some(desc) => {
-                    if !desc.is_write_only() {
-                        break;
-                    }
-                    let limit = cmp::min(write_count + desc.len as usize, self.rx.bytes_read);
-                    let source_slice = &self.rx.frame_buf[write_count..limit];
-                    let write_result = mem.write_slice(source_slice, desc.addr);
-
-                    match write_result {
-                        Ok(_) => {
-                            write_count = limit;
-                        }
-                        Err(e) => {
-                            error!("Failed to write slice: {:?}", e);
-                            break;
-                        }
-                    };
-
-                    if write_count >= self.rx.bytes_read {
-                        break;
-                    }
-                    next_desc = desc.next_descriptor();
-                }
-                None => {
-                    warn!("Receiving buffer is too small to hold frame of current size");
-                    break;
-                }
-            }
-        }
-
-        vring
-            .mut_queue()
-            .add_used(&mem, head_index, write_count as u32);
-
-        // Mark that we have at least one pending packet and we need to interrupt the guest.
-        self.rx.deferred_irqs = true;
-
-        Ok(write_count >= self.rx.bytes_read)
-    }
-
-    fn process_rx(&mut self, vring: &mut Vring) -> Result<()> {
-        // Read as many frames as possible.
-        loop {
-            match self.read_tap() {
-                Ok(count) => {
-                    self.rx.bytes_read = count;
-                    if !self.rx_single_frame(vring)? {
-                        self.rx.deferred_frame = true;
-                        break;
-                    }
-                }
-                Err(e) => {
-                    // The tap device is non-blocking, so any error aside from EAGAIN is
-                    // unexpected.
-                    match e.raw_os_error() {
-                        Some(err) if err == EAGAIN => (),
-                        _ => {
-                            error!("Failed to read tap: {:?}", e);
-                            return Err(Error::FailedReadTap);
-                        }
-                    };
-                    break;
-                }
-            }
-        }
-        if self.rx.deferred_irqs {
-            self.rx.deferred_irqs = false;
-            vring.signal_used_queue().unwrap();
-            Ok(())
-        } else {
-            Ok(())
-        }
-    }
-
-    fn resume_rx(&mut self, vring: &mut Vring) -> Result<()> {
-        if self.rx.deferred_frame {
-            if self.rx_single_frame(vring)? {
-                self.rx.deferred_frame = false;
-                // process_rx() was interrupted possibly before consuming all
-                // packets in the tap; try continuing now.
-                self.process_rx(vring)
-            } else if self.rx.deferred_irqs {
-                self.rx.deferred_irqs = false;
-                vring.signal_used_queue().unwrap();
-                Ok(())
-            } else {
-                Ok(())
-            }
-        } else {
-            Ok(())
-        }
-    }
-
-    fn process_tx(&mut self, vring: &mut Vring) -> Result<()> {
-        let mem = self.mem.as_ref().ok_or(Error::NoMemoryConfigured)?;
-
-        let mut used_desc_heads = [(0, 0); QUEUE_SIZE];
-        let mut used_count = 0;
-        while let Some(avail_desc) = vring.mut_queue().iter(&mem).next() {
-            let head_index = avail_desc.index;
-            let mut read_count = 0;
-            let mut next_desc = Some(avail_desc);
-
-            self.tx.iovec.clear();
-            while let Some(desc) = next_desc {
-                if desc.is_write_only() {
-                    break;
-                }
-                self.tx.iovec.push((desc.addr, desc.len as usize));
-                read_count += desc.len as usize;
-                next_desc = desc.next_descriptor();
-            }
-            used_desc_heads[used_count] = (head_index, read_count);
-            used_count += 1;
-            read_count = 0;
-            // Copy buffer from across multiple descriptors.
-            // TODO(performance - Issue #420): change this to use `writev()` instead of `write()`
-            // and get rid of the intermediate buffer.
-            for (desc_addr, desc_len) in self.tx.iovec.drain(..) {
-                let limit = cmp::min((read_count + desc_len) as usize, self.tx.frame_buf.len());
-
-                let read_result = mem.read_slice(
-                    &mut self.tx.frame_buf[read_count..limit as usize],
-                    desc_addr,
-                );
-                match read_result {
-                    Ok(_) => {
-                        // Increment by number of bytes actually read
-                        read_count += limit - read_count;
-                    }
-                    Err(e) => {
-                        error!("Failed to read slice: {:?}", e);
-                        break;
-                    }
-                }
-            }
-
-            let write_result = self.tap.write(&self.tx.frame_buf[..read_count as usize]);
-            match write_result {
-                Ok(_) => {}
-                Err(e) => {
-                    error!("net: tx: error failed to write to tap: {}", e);
-                }
-            };
-        }
-
-        if used_count > 0 {
-            for &(desc_index, _) in &used_desc_heads[..used_count] {
-                vring.mut_queue().add_used(&mem, desc_index, 0);
-            }
-            vring.signal_used_queue().unwrap();
-        }
-
-        Ok(())
-    }
-
-    fn read_tap(&mut self) -> io::Result<usize> {
-        self.tap.read(&mut self.rx.frame_buf)
-    }
-}
-
-impl VhostUserBackend for VhostUserNetBackend {
-    fn num_queues(&self) -> usize {
-        NUM_QUEUES
-    }
-
-    fn max_queue_size(&self) -> usize {
-        QUEUE_SIZE
-    }
-
-    fn features(&self) -> u64 {
-        1 << VIRTIO_NET_F_GUEST_CSUM
-            | 1 << VIRTIO_NET_F_CSUM
-            | 1 << VIRTIO_NET_F_GUEST_TSO4
-            | 1 << VIRTIO_NET_F_GUEST_UFO
-            | 1 << VIRTIO_NET_F_HOST_TSO4
-            | 1 << VIRTIO_NET_F_HOST_UFO
-            | 1 << VIRTIO_F_VERSION_1
-            | VhostUserVirtioFeatures::PROTOCOL_FEATURES.bits()
-    }
-
-    fn protocol_features(&self) -> VhostUserProtocolFeatures {
-        VhostUserProtocolFeatures::all()
-    }
-
-    fn update_memory(&mut self, mem: GuestMemoryMmap) -> VhostUserBackendResult<()> {
-        self.mem = Some(mem);
-        Ok(())
-    }
-
-    fn handle_event(
-        &mut self,
-        device_event: u16,
-        evset: epoll::Events,
-        vrings: &[Arc<RwLock<Vring>>],
-    ) -> VhostUserBackendResult<bool> {
-        if evset != epoll::Events::EPOLLIN {
-            return Err(Error::HandleEventNotEpollIn.into());
-        }
-
-        match device_event {
-            RX_QUEUE_EVENT => {
-                let mut vring = vrings[0].write().unwrap();
-                self.resume_rx(&mut vring)?;
-
-                if !self.rx_tap_listening {
-                    self.vring_worker.as_ref().unwrap().register_listener(
-                        self.tap.as_raw_fd(),
-                        epoll::Events::EPOLLIN,
-                        u64::from(RX_TAP_EVENT),
-                    )?;
-                    self.rx_tap_listening = true;
-                }
-            }
-            TX_QUEUE_EVENT => {
-                let mut vring = vrings[1].write().unwrap();
-                self.process_tx(&mut vring)?;
-            }
-            RX_TAP_EVENT => {
-                let mut vring = vrings[0].write().unwrap();
-                if self.rx.deferred_frame
-                // Process a deferred frame first if available. Don't read from tap again
-                // until we manage to receive this deferred frame.
-                {
-                    if self.rx_single_frame(&mut vring)? {
-                        self.rx.deferred_frame = false;
-                        self.process_rx(&mut vring)?;
-                    } else if self.rx.deferred_irqs {
-                        self.rx.deferred_irqs = false;
-                        vring.signal_used_queue()?;
-                    }
-                } else {
-                    self.process_rx(&mut vring)?;
-                }
-            }
-            KILL_EVENT => {
-                self.kill_evt.read().unwrap();
-                return Ok(true);
-            }
-            _ => return Err(Error::HandleEventUnknownEvent.into()),
-        }
-
-        Ok(false)
-    }
-}
-
-pub struct VhostUserNetBackendConfig<'a> {
-    pub ip: Ipv4Addr,
-    pub mask: Ipv4Addr,
-    pub sock: &'a str,
-}
-
-impl<'a> VhostUserNetBackendConfig<'a> {
-    pub fn parse(backend: &'a str) -> Result<Self> {
-        let params_list: Vec<&str> = backend.split(',').collect();
-
-        let mut ip_str: &str = "";
-        let mut mask_str: &str = "";
-        let mut sock: &str = "";
-
-        for param in params_list.iter() {
-            if param.starts_with("ip=") {
-                ip_str = &param[3..];
-            } else if param.starts_with("mask=") {
-                mask_str = &param[5..];
-            } else if param.starts_with("sock=") {
-                sock = &param[5..];
-            }
-        }
-
-        let mut ip: Ipv4Addr = Ipv4Addr::new(192, 168, 100, 1);
-        let mut mask: Ipv4Addr = Ipv4Addr::new(255, 255, 255, 0);
-
-        if sock.is_empty() {
-            return Err(Error::ParseSockParam);
-        }
-        if !ip_str.is_empty() {
-            ip = ip_str.parse().map_err(|_| Error::ParseIpParam)?;
-        }
-        if !mask_str.is_empty() {
-            mask = mask_str.parse().map_err(|_| Error::ParseMaskParam)?;
-        }
-
-        Ok(VhostUserNetBackendConfig { ip, mask, sock })
-    }
-}
+use vhost_user_net::start_net_backend;
 
 fn main() {
     let cmd_arguments = App::new("vhost-user-net backend")
@@ -545,57 +19,19 @@ fn main() {
         .author(crate_authors!())
         .about("Launch a vhost-user-net backend.")
         .arg(
-            Arg::with_name("backend")
-                .long("backend")
+            Arg::with_name("net-backend")
+                .long("net-backend")
                 .help(
-                    "Backend parameters \"ip=<ip_addr>,\
-                     mask=<net_mask>,sock=<socket_path>\"",
+                    "vhost-user-net backend parameters \"ip=<ip_addr>,\
+                     mask=<net_mask>,sock=<socket_path>,\
+                     num_queues=<number_of_queues>,\
+                     queue_size=<size_of_each_queue>\"",
                 )
                 .takes_value(true)
                 .min_values(1),
         )
         .get_matches();
 
-    let vhost_user_net_backend = cmd_arguments.value_of("backend").unwrap();
-
-    let backend_config = match VhostUserNetBackendConfig::parse(vhost_user_net_backend) {
-        Ok(config) => config,
-        Err(e) => {
-            println!("Failed parsing parameters {:?}", e);
-            process::exit(1);
-        }
-    };
-
-    let net_backend = Arc::new(RwLock::new(
-        VhostUserNetBackend::new(backend_config.ip, backend_config.mask).unwrap(),
-    ));
-    let name = "vhost-user-net-backend";
-    let mut net_daemon = VhostUserDaemon::new(
-        name.to_string(),
-        backend_config.sock.to_string(),
-        net_backend.clone(),
-    )
-    .unwrap();
-    let vring_worker = net_daemon.get_vring_worker();
-
-    if let Err(e) = vring_worker.register_listener(
-        net_backend.read().unwrap().kill_evt.as_raw_fd(),
-        epoll::Events::EPOLLIN,
-        u64::from(KILL_EVENT),
-    ) {
-        println!("failed to register listener for kill event: {:?}", e);
-        process::exit(1);
-    }
-
-    net_backend.write().unwrap().vring_worker = Some(vring_worker);
-
-    if let Err(e) = net_daemon.start() {
-        println!(
-            "failed to start daemon for vhost-user-net with error: {:?}",
-            e
-        );
-        process::exit(1);
-    }
-
-    net_daemon.wait().unwrap();
+    let backend_command = cmd_arguments.value_of("net-backend").unwrap();
+    start_net_backend(backend_command);
 }

test_data/cloud-init/clear/openstack/latest/user_data

@@ -58,10 +58,12 @@ write_files:
     content: |
         #!/bin/bash
 
-        mount -t virtio_fs virtiofs /mnt -o rootmode=040000,user_id=0,group_id=0,dax
+        mount -t virtiofs -o dax myfs /mnt
         bash -c "echo 0000:00:05.0 > /sys/bus/pci/devices/0000\:00\:05.0/driver/unbind"
         bash -c "echo 1af4 1041 > /sys/bus/pci/drivers/vfio-pci/new_id"
         bash -c "echo 0000:00:06.0 > /sys/bus/pci/devices/0000\:00\:06.0/driver/unbind"
         bash -c "echo 1af4 1041 > /sys/bus/pci/drivers/vfio-pci/new_id"
-
-        /mnt/cloud-hypervisor --kernel /mnt/vmlinux --cmdline "console=hvc0 reboot=k panic=1 nomodules i8042.noaux i8042.nomux i8042.nopnp i8042.dumbkbd root=/dev/vda2 VFIOTAG" --disk path=/mnt/clear-31310-cloudguest.img path=/mnt/cloudinit.img --cpus 1 --memory size=512M --rng --device path=/sys/bus/pci/devices/0000:00:05.0/ path=/sys/bus/pci/devices/0000:00:06.0/
+        # 512M ram requires 256 pages
+        echo 256 | sudo tee /proc/sys/vm/nr_hugepages
+        sudo chmod a+rwX /dev/hugepages
+        /mnt/cloud-hypervisor --kernel /mnt/vmlinux --cmdline "console=hvc0 reboot=k panic=1 nomodules i8042.noaux i8042.nomux i8042.nopnp i8042.dumbkbd root=/dev/vda2 VFIOTAG" --disk path=/mnt/clear-31310-cloudguest.img path=/mnt/cloudinit.img --cpus boot=1 --memory size=512M,file=/dev/hugepages --device path=/sys/bus/pci/devices/0000:00:05.0/ path=/sys/bus/pci/devices/0000:00:06.0/

vfio/Cargo.toml

@@ -4,18 +4,22 @@ version = "0.0.1"
 authors = ["The Cloud Hypervisor Authors"]
 
 [dependencies]
-byteorder = "1.3.2"
+arc-swap = "0.4.4"
+byteorder = "1.3.4"
 devices = { path = "../devices" }
 kvm-bindings = "0.2.0"
-kvm-ioctls = "0.4.0"
-libc = "0.2.60"
+kvm-ioctls = "0.5.0"
+libc = "0.2.66"
 log = "0.4.8"
 pci = { path = "../pci" }
-vfio-bindings = "0.1.0"
 vm-allocator = { path = "../vm-allocator" }
 vm-device = { path = "../vm-device" }
-vmm-sys-util = ">=0.3.1"
+vmm-sys-util = "0.4.0"
 
 [dependencies.vm-memory]
 git = "https://github.com/rust-vmm/vm-memory"
 features = ["backend-mmap"]
+
+[dependencies.vfio-bindings]
+git = "https://github.com/rust-vmm/vfio-bindings"
+features = ["fam-wrappers"]

vfio/src/lib.rs

@@ -5,6 +5,7 @@
 
 //#![deny(missing_docs)]
 //! Virtual Function I/O (VFIO) API
+extern crate arc_swap;
 extern crate byteorder;
 extern crate devices;
 extern crate kvm_bindings;

vfio/src/vfio_device.rs

@@ -3,6 +3,7 @@
 // SPDX-License-Identifier: Apache-2.0 OR BSD-3-Clause
 //
 use crate::vec_with_array_field;
+use arc_swap::ArcSwap;
 use byteorder::{ByteOrder, LittleEndian};
 use kvm_ioctls::*;
 use std::collections::HashMap;
@@ -12,17 +13,20 @@ use std::fmt;
 use std::fs::{File, OpenOptions};
 use std::io;
 use std::mem;
+use std::num;
 use std::os::unix::io::{AsRawFd, FromRawFd, RawFd};
 use std::os::unix::prelude::FileExt;
 use std::path::{Path, PathBuf};
 use std::result;
-use std::sync::{Arc, RwLock};
+use std::sync::Arc;
 use std::u32;
 use vfio_bindings::bindings::vfio::*;
+use vfio_bindings::bindings::IrqSet;
 use vfio_ioctls::*;
 use vm_device::{get_host_address_range, ExternalDmaMapping};
 use vm_memory::{Address, GuestAddress, GuestMemory, GuestMemoryMmap, GuestMemoryRegion};
 use vmm_sys_util::eventfd::EventFd;
+use vmm_sys_util::fam::FamStruct;
 use vmm_sys_util::ioctl::*;
 
 #[derive(Debug)]
@@ -47,6 +51,8 @@ pub enum VfioError {
     IommuDmaUnmap,
     VfioDeviceGetIrqInfo,
     VfioDeviceSetIrq,
+    ReadLink(io::Error),
+    ParseInt(num::ParseIntError),
 }
 pub type Result<T> = std::result::Result<T, VfioError>;
 
@@ -95,6 +101,8 @@ impl fmt::Display for VfioError {
             }
             VfioError::VfioDeviceGetIrqInfo => write!(f, "failed to get vfio device irq info"),
             VfioError::VfioDeviceSetIrq => write!(f, "failed to set vfio deviece irq"),
+            VfioError::ReadLink(e) => write!(f, "failed to read link from path: {}", e),
+            VfioError::ParseInt(e) => write!(f, "failed to parse integer: {}", e),
         }
     }
 }
@@ -436,7 +444,7 @@ impl VfioDeviceInfo {
                 ioctl_with_mut_ref(&self.device, VFIO_DEVICE_GET_REGION_INFO(), &mut reg_info)
             };
             if ret < 0 {
-                error!("Could not get region #{} info", i);
+                warn!("Could not get region #{} info", i);
                 continue;
             }
 
@@ -462,7 +470,7 @@ impl VfioDeviceInfo {
                     )
                 };
                 if ret < 0 {
-                    error!("Could not get region #{} info", i);
+                    warn!("Could not get region #{} info", i);
                     continue;
                 }
                 // region_with_cap[0].cap_info may contain vfio_region_info_cap_sparse_mmap
@@ -514,11 +522,11 @@ impl VfioDeviceInfo {
 /// associated with a specific VFIO container.
 pub struct VfioDmaMapping {
     container: Arc<VfioContainer>,
-    memory: Arc<RwLock<GuestMemoryMmap>>,
+    memory: Arc<ArcSwap<GuestMemoryMmap>>,
 }
 
 impl VfioDmaMapping {
-    pub fn new(container: Arc<VfioContainer>, memory: Arc<RwLock<GuestMemoryMmap>>) -> Self {
+    pub fn new(container: Arc<VfioContainer>, memory: Arc<ArcSwap<GuestMemoryMmap>>) -> Self {
         VfioDmaMapping { container, memory }
     }
 }
@@ -526,7 +534,7 @@ impl VfioDmaMapping {
 impl ExternalDmaMapping for VfioDmaMapping {
     fn map(&self, iova: u64, gpa: u64, size: u64) -> result::Result<(), io::Error> {
         let user_addr = if let Some(addr) = get_host_address_range(
-            &self.memory.read().unwrap(),
+            &self.memory.load(),
             GuestAddress(gpa),
             size.try_into().unwrap(),
         ) {
@@ -577,7 +585,7 @@ pub struct VfioDevice {
     group: VfioGroup,
     regions: Vec<VfioRegion>,
     irqs: HashMap<u32, VfioIrq>,
-    mem: Arc<RwLock<GuestMemoryMmap>>,
+    mem: Arc<ArcSwap<GuestMemoryMmap>>,
     iommu_attached: bool,
 }
 
@@ -588,16 +596,14 @@ impl VfioDevice {
     pub fn new(
         sysfspath: &Path,
         device_fd: Arc<DeviceFd>,
-        mem: Arc<RwLock<GuestMemoryMmap>>,
+        mem: Arc<ArcSwap<GuestMemoryMmap>>,
         iommu_attached: bool,
     ) -> Result<Self> {
         let uuid_path: PathBuf = [sysfspath, Path::new("iommu_group")].iter().collect();
-        let group_path = uuid_path.read_link().map_err(|_| VfioError::InvalidPath)?;
+        let group_path = uuid_path.read_link().map_err(VfioError::ReadLink)?;
         let group_osstr = group_path.file_name().ok_or(VfioError::InvalidPath)?;
         let group_str = group_osstr.to_str().ok_or(VfioError::InvalidPath)?;
-        let group_id = group_str
-            .parse::<u32>()
-            .map_err(|_| VfioError::InvalidPath)?;
+        let group_id = group_str.parse::<u32>().map_err(VfioError::ParseInt)?;
 
         let group = VfioGroup::new(group_id, device_fd)?;
         let device_info = group.get_device(sysfspath)?;
@@ -641,34 +647,25 @@ impl VfioDevice {
             return Err(VfioError::VfioDeviceSetIrq);
         }
 
-        let mut irq_set = vec_with_array_field::<vfio_irq_set, u32>(event_fds.len());
-        irq_set[0].argsz = mem::size_of::<vfio_irq_set>() as u32
-            + (event_fds.len() * mem::size_of::<u32>()) as u32;
-        irq_set[0].flags = VFIO_IRQ_SET_DATA_EVENTFD | VFIO_IRQ_SET_ACTION_TRIGGER;
-        irq_set[0].index = irq_index;
-        irq_set[0].start = 0;
-        irq_set[0].count = irq.count;
+        let mut irq_set_wrapper = IrqSet::new(event_fds.len() * mem::size_of::<u32>());
+        let mut irq_set = irq_set_wrapper.as_mut_fam_struct();
+        let fds = irq_set.as_mut_slice();
 
-        {
-            // irq_set.data could be none, bool or fd according to flags, so irq_set.data
-            // is u8 default, here irq_set.data is a vector of fds as u32, so 4 default u8
-            // are combined together as u32 for each fd.
-            // It is safe as enough space is reserved through
-            // vec_with_array_field(u32)<event_fds.len()>.
-            let fds = unsafe {
-                irq_set[0]
-                    .data
-                    .as_mut_slice(event_fds.len() * mem::size_of::<u32>())
-            };
-            for (index, event_fd) in event_fds.iter().enumerate() {
-                let fds_offset = index * mem::size_of::<u32>();
-                let fd = &mut fds[fds_offset..fds_offset + mem::size_of::<u32>()];
-                LittleEndian::write_u32(fd, event_fd.as_raw_fd() as u32);
-            }
+        for (index, event_fd) in event_fds.iter().enumerate() {
+            let fds_offset = index * mem::size_of::<u32>();
+            let fd = &mut fds[fds_offset..fds_offset + mem::size_of::<u32>()];
+            LittleEndian::write_u32(fd, event_fd.as_raw_fd() as u32);
         }
 
+        irq_set.argsz = mem::size_of::<vfio_irq_set>() as u32
+            + (event_fds.len() * mem::size_of::<u32>()) as u32;
+        irq_set.flags = VFIO_IRQ_SET_DATA_EVENTFD | VFIO_IRQ_SET_ACTION_TRIGGER;
+        irq_set.index = irq_index;
+        irq_set.start = 0;
+        irq_set.count = irq.count;
+
         // Safe as we are the owner of self and irq_set which are valid value
-        let ret = unsafe { ioctl_with_ref(self, VFIO_DEVICE_SET_IRQS(), &irq_set[0]) };
+        let ret = unsafe { ioctl_with_ref(self, VFIO_DEVICE_SET_IRQS(), irq_set) };
         if ret < 0 {
             return Err(VfioError::VfioDeviceSetIrq);
         }
@@ -845,7 +842,7 @@ impl VfioDevice {
     /// then vfio kernel driver could access guest memory from gfn
     pub fn setup_dma_map(&self) -> Result<()> {
         if !self.iommu_attached {
-            self.mem.read().unwrap().with_regions(|_index, region| {
+            self.mem.load().with_regions(|_index, region| {
                 self.vfio_dma_map(
                     region.start_addr().raw_value(),
                     region.len() as u64,
@@ -860,7 +857,7 @@ impl VfioDevice {
     /// then vfio kernel driver couldn't access this guest memory
     pub fn unset_dma_map(&self) -> Result<()> {
         if !self.iommu_attached {
-            self.mem.read().unwrap().with_regions(|_index, region| {
+            self.mem.load().with_regions(|_index, region| {
                 self.vfio_dma_unmap(region.start_addr().raw_value(), region.len() as u64)
             })?;
         }

vfio/src/vfio_pci.rs

@@ -7,27 +7,26 @@ extern crate devices;
 extern crate pci;
 extern crate vm_allocator;
 
-use crate::vec_with_array_field;
 use crate::vfio_device::VfioDevice;
 use byteorder::{ByteOrder, LittleEndian};
 use devices::BusDevice;
-use kvm_bindings::{
-    kvm_irq_routing, kvm_irq_routing_entry, kvm_userspace_memory_region, KVM_IRQ_ROUTING_MSI,
-};
+use kvm_bindings::kvm_userspace_memory_region;
 use kvm_ioctls::*;
 use pci::{
-    BarReprogrammingParams, MsiCap, MsixCap, MsixConfig, PciBarConfiguration, PciBarRegionType,
-    PciCapabilityID, PciClassCode, PciConfiguration, PciDevice, PciDeviceError, PciHeaderType,
-    PciSubclass, MSIX_TABLE_ENTRY_SIZE,
+    msi_num_enabled_vectors, BarReprogrammingParams, MsiConfig, MsixCap, MsixConfig,
+    PciBarConfiguration, PciBarRegionType, PciCapabilityID, PciClassCode, PciConfiguration,
+    PciDevice, PciDeviceError, PciHeaderType, PciSubclass, MSIX_TABLE_ENTRY_SIZE,
 };
 use std::any::Any;
-use std::collections::HashMap;
 use std::os::unix::io::AsRawFd;
 use std::ptr::null_mut;
-use std::sync::{Arc, Mutex};
+use std::sync::Arc;
 use std::{fmt, io, result};
 use vfio_bindings::bindings::vfio::*;
 use vm_allocator::SystemAllocator;
+use vm_device::interrupt::{
+    InterruptIndex, InterruptManager, InterruptSourceGroup, MsiIrqGroupConfig,
+};
 use vm_memory::{Address, GuestAddress, GuestUsize};
 use vmm_sys_util::eventfd::EventFd;
 
@@ -35,12 +34,15 @@ use vmm_sys_util::eventfd::EventFd;
 pub enum VfioPciError {
     AllocateGsi,
     EventFd(io::Error),
+    InterruptSourceGroupCreate(io::Error),
     IrqFd(kvm_ioctls::Error),
     NewVfioPciDevice,
     MapRegionGuest(kvm_ioctls::Error),
     SetGsiRouting(kvm_ioctls::Error),
     MsiNotConfigured,
     MsixNotConfigured,
+    UpdateMsiEventFd,
+    UpdateMsixEventFd,
 }
 pub type Result<T> = std::result::Result<T, VfioPciError>;
 
@@ -49,6 +51,9 @@ impl fmt::Display for VfioPciError {
         match self {
             VfioPciError::AllocateGsi => write!(f, "failed to allocate GSI"),
             VfioPciError::EventFd(e) => write!(f, "failed to create eventfd: {}", e),
+            VfioPciError::InterruptSourceGroupCreate(e) => {
+                write!(f, "failed to create interrupt source group: {}", e)
+            }
             VfioPciError::IrqFd(e) => write!(f, "failed to register irqfd: {}", e),
             VfioPciError::NewVfioPciDevice => write!(f, "failed to create VFIO PCI device"),
             VfioPciError::MapRegionGuest(e) => {
@@ -57,6 +62,8 @@ impl fmt::Display for VfioPciError {
             VfioPciError::SetGsiRouting(e) => write!(f, "failed to set GSI routes for KVM: {}", e),
             VfioPciError::MsiNotConfigured => write!(f, "MSI interrupt not yet configured"),
             VfioPciError::MsixNotConfigured => write!(f, "MSI-X interrupt not yet configured"),
+            VfioPciError::UpdateMsiEventFd => write!(f, "failed to update MSI eventfd"),
+            VfioPciError::UpdateMsixEventFd => write!(f, "failed to update MSI-X eventfd"),
         }
     }
 }
@@ -79,19 +86,19 @@ enum InterruptUpdateAction {
     DisableMsix,
 }
 
-#[derive(Copy, Clone)]
 struct VfioMsi {
-    cap: MsiCap,
+    cfg: MsiConfig,
     cap_offset: u32,
+    interrupt_source_group: Arc<Box<dyn InterruptSourceGroup>>,
 }
 
 impl VfioMsi {
     fn update(&mut self, offset: u64, data: &[u8]) -> Option<InterruptUpdateAction> {
-        let old_enabled = self.cap.enabled();
+        let old_enabled = self.cfg.enabled();
 
-        self.cap.update(offset, data);
+        self.cfg.update(offset, data);
 
-        let new_enabled = self.cap.enabled();
+        let new_enabled = self.cfg.enabled();
 
         if !old_enabled && new_enabled {
             return Some(InterruptUpdateAction::EnableMsi);
@@ -109,18 +116,19 @@ struct VfioMsix {
     bar: MsixConfig,
     cap: MsixCap,
     cap_offset: u32,
+    interrupt_source_group: Arc<Box<dyn InterruptSourceGroup>>,
 }
 
 impl VfioMsix {
     fn update(&mut self, offset: u64, data: &[u8]) -> Option<InterruptUpdateAction> {
-        let old_enabled = self.cap.enabled();
+        let old_enabled = self.bar.enabled();
 
         // Update "Message Control" word
         if offset == 2 && data.len() == 2 {
-            self.cap.set_msg_ctl(LittleEndian::read_u16(data));
+            self.bar.set_msg_ctl(LittleEndian::read_u16(data));
         }
 
-        let new_enabled = self.cap.enabled();
+        let new_enabled = self.bar.enabled();
 
         if !old_enabled && new_enabled {
             return Some(InterruptUpdateAction::EnableMsix);
@@ -169,7 +177,7 @@ impl Interrupt {
     fn accessed(&self, offset: u64) -> Option<(PciCapabilityID, u64)> {
         if let Some(msi) = &self.msi {
             if offset >= u64::from(msi.cap_offset)
-                && offset < u64::from(msi.cap_offset) + msi.cap.size()
+                && offset < u64::from(msi.cap_offset) + msi.cfg.size()
             {
                 return Some((
                     PciCapabilityID::MessageSignalledInterrupts,
@@ -210,30 +218,6 @@ impl Interrupt {
     }
 }
 
-struct InterruptRoute {
-    gsi: u32,
-    irq_fd: EventFd,
-}
-
-impl InterruptRoute {
-    fn new(allocator: &mut SystemAllocator) -> Result<Self> {
-        let irq_fd = EventFd::new(libc::EFD_NONBLOCK).map_err(VfioPciError::EventFd)?;
-        let gsi = allocator.allocate_gsi().ok_or(VfioPciError::AllocateGsi)?;
-
-        Ok(InterruptRoute { gsi, irq_fd })
-    }
-
-    fn enable(&self, vm: &Arc<VmFd>) -> Result<()> {
-        vm.register_irqfd(&self.irq_fd, self.gsi)
-            .map_err(VfioPciError::IrqFd)
-    }
-
-    fn disable(&self, vm: &Arc<VmFd>) -> Result<()> {
-        vm.unregister_irqfd(&self.irq_fd, self.gsi)
-            .map_err(VfioPciError::IrqFd)
-    }
-}
-
 #[derive(Copy, Clone)]
 struct MmioRegion {
     start: GuestAddress,
@@ -297,17 +281,14 @@ pub struct VfioPciDevice {
     configuration: PciConfiguration,
     mmio_regions: Vec<MmioRegion>,
     interrupt: Interrupt,
-    interrupt_routes: Vec<InterruptRoute>,
-    gsi_msi_routes: Arc<Mutex<HashMap<u32, kvm_irq_routing_entry>>>,
 }
 
 impl VfioPciDevice {
     /// Constructs a new Vfio Pci device for the given Vfio device
     pub fn new(
         vm_fd: &Arc<VmFd>,
-        allocator: &mut SystemAllocator,
         device: VfioDevice,
-        gsi_msi_routes: Arc<Mutex<HashMap<u32, kvm_irq_routing_entry>>>,
+        interrupt_manager: &Arc<dyn InterruptManager<GroupConfig = MsiIrqGroupConfig>>,
     ) -> Result<Self> {
         let device = Arc::new(device);
         device.reset();
@@ -336,65 +317,18 @@ impl VfioPciDevice {
                 msi: None,
                 msix: None,
             },
-            interrupt_routes: Vec::new(),
-            gsi_msi_routes,
         };
 
-        vfio_pci_device.parse_capabilities();
-
-        // Allocate temporary interrupt routes for now.
-        // The MSI vectors will be filled when the guest driver programs the device.
-        let max_interrupts = vfio_pci_device.device.max_interrupts();
-        for _ in 0..max_interrupts {
-            let route = InterruptRoute::new(allocator)?;
-            vfio_pci_device.interrupt_routes.push(route);
-        }
+        vfio_pci_device.parse_capabilities(interrupt_manager);
 
         Ok(vfio_pci_device)
     }
 
-    fn enable_irq_fds(&self) -> Result<Vec<&EventFd>> {
-        let mut irq_fds: Vec<&EventFd> = Vec::new();
-
-        for r in &self.interrupt_routes {
-            r.enable(&self.vm_fd)?;
-            irq_fds.push(&r.irq_fd);
-        }
-
-        Ok(irq_fds)
-    }
-
-    fn disable_irq_fds(&self) -> Result<()> {
-        for r in &self.interrupt_routes {
-            r.disable(&self.vm_fd)?;
-        }
-
-        Ok(())
-    }
-
-    fn set_kvm_routes(&self) -> Result<()> {
-        let mut entry_vec: Vec<kvm_irq_routing_entry> = Vec::new();
-        for (_, entry) in self.gsi_msi_routes.lock().unwrap().iter() {
-            entry_vec.push(*entry);
-        }
-
-        let mut irq_routing =
-            vec_with_array_field::<kvm_irq_routing, kvm_irq_routing_entry>(entry_vec.len());
-        irq_routing[0].nr = entry_vec.len() as u32;
-        irq_routing[0].flags = 0;
-
-        unsafe {
-            let entries: &mut [kvm_irq_routing_entry] =
-                irq_routing[0].entries.as_mut_slice(entry_vec.len());
-            entries.copy_from_slice(&entry_vec);
-        }
-
-        self.vm_fd
-            .set_gsi_routing(&irq_routing[0])
-            .map_err(VfioPciError::SetGsiRouting)
-    }
-
-    fn parse_msix_capabilities(&mut self, cap: u8) {
+    fn parse_msix_capabilities(
+        &mut self,
+        cap: u8,
+        interrupt_manager: &Arc<dyn InterruptManager<GroupConfig = MsiIrqGroupConfig>>,
+    ) {
         let msg_ctl = self
             .vfio_pci_configuration
             .read_config_word((cap + 2).into());
@@ -412,30 +346,53 @@ impl VfioPciDevice {
             table,
             pba,
         };
-        let msix_config = MsixConfig::new(msix_cap.table_size());
+
+        let interrupt_source_group = interrupt_manager
+            .create_group(MsiIrqGroupConfig {
+                base: 0,
+                count: msix_cap.table_size() as InterruptIndex,
+            })
+            .unwrap();
+
+        let msix_config = MsixConfig::new(msix_cap.table_size(), interrupt_source_group.clone());
 
         self.interrupt.msix = Some(VfioMsix {
             bar: msix_config,
             cap: msix_cap,
             cap_offset: cap.into(),
+            interrupt_source_group,
         });
     }
 
-    fn parse_msi_capabilities(&mut self, cap: u8) {
+    fn parse_msi_capabilities(
+        &mut self,
+        cap: u8,
+        interrupt_manager: &Arc<dyn InterruptManager<GroupConfig = MsiIrqGroupConfig>>,
+    ) {
         let msg_ctl = self
             .vfio_pci_configuration
             .read_config_word((cap + 2).into());
 
+        let interrupt_source_group = interrupt_manager
+            .create_group(MsiIrqGroupConfig {
+                base: 0,
+                count: msi_num_enabled_vectors(msg_ctl) as InterruptIndex,
+            })
+            .unwrap();
+
+        let msi_config = MsiConfig::new(msg_ctl, interrupt_source_group.clone());
+
         self.interrupt.msi = Some(VfioMsi {
-            cap: MsiCap {
-                msg_ctl,
-                ..Default::default()
-            },
+            cfg: msi_config,
             cap_offset: cap.into(),
+            interrupt_source_group,
         });
     }
 
-    fn parse_capabilities(&mut self) {
+    fn parse_capabilities(
+        &mut self,
+        interrupt_manager: &Arc<dyn InterruptManager<GroupConfig = MsiIrqGroupConfig>>,
+    ) {
         let mut cap_next = self
             .vfio_pci_configuration
             .read_config_byte(PCI_CONFIG_CAPABILITY_OFFSET);
@@ -447,10 +404,10 @@ impl VfioPciDevice {
 
             match PciCapabilityID::from(cap_id) {
                 PciCapabilityID::MessageSignalledInterrupts => {
-                    self.parse_msi_capabilities(cap_next);
+                    self.parse_msi_capabilities(cap_next, interrupt_manager);
                 }
                 PciCapabilityID::MSIX => {
-                    self.parse_msix_capabilities(cap_next);
+                    self.parse_msix_capabilities(cap_next, interrupt_manager);
                 }
                 _ => {}
             };
@@ -461,158 +418,66 @@ impl VfioPciDevice {
         }
     }
 
-    fn update_msi_interrupt_routes(&self, msi: &VfioMsi) -> Result<()> {
-        if msi.cap.enabled() {
-            let mut gsi_msi_routes = self.gsi_msi_routes.lock().unwrap();
-
-            for (idx, route) in self.interrupt_routes.iter().enumerate() {
-                // Ignore MSI vector if the amount of vectors supported by the
-                // guest OS does not match the expected amount. This is related
-                // to "Multiple Message Capable" and "Multiple Message Enable"
-                // fields from the "Message Control" register.
-                if idx >= msi.cap.num_enabled_vectors() {
-                    continue;
-                }
-
-                // Ignore MSI vector if masked.
-                if msi.cap.vector_masked(idx) {
-                    continue;
-                }
-
-                let mut entry = kvm_irq_routing_entry {
-                    gsi: route.gsi,
-                    type_: KVM_IRQ_ROUTING_MSI,
-                    ..Default::default()
-                };
-
-                entry.u.msi.address_lo = msi.cap.msg_addr_lo;
-                entry.u.msi.address_hi = msi.cap.msg_addr_hi;
-                entry.u.msi.data = u32::from(msi.cap.msg_data) | (idx as u32);
-
-                gsi_msi_routes.insert(route.gsi, entry);
-            }
-        } else {
-            let mut gsi_msi_routes = self.gsi_msi_routes.lock().unwrap();
-
-            for route in self.interrupt_routes.iter() {
-                gsi_msi_routes.remove(&route.gsi);
-            }
-        }
-
-        self.set_kvm_routes()
-    }
-
-    fn update_msix_interrupt_routes(&self, msix: &VfioMsix) -> Result<()> {
-        if msix.cap.enabled() && !msix.cap.masked() {
-            let mut gsi_msi_routes = self.gsi_msi_routes.lock().unwrap();
-
-            for (idx, table_entry) in msix.bar.table_entries.iter().enumerate() {
-                // Ignore MSI-X vector if masked.
-                if table_entry.masked() {
-                    continue;
-                }
-
-                let gsi = self.interrupt_routes[idx].gsi;
-
-                let mut entry = kvm_irq_routing_entry {
-                    gsi,
-                    type_: KVM_IRQ_ROUTING_MSI,
-                    ..Default::default()
-                };
-
-                entry.u.msi.address_lo = table_entry.msg_addr_lo;
-                entry.u.msi.address_hi = table_entry.msg_addr_hi;
-                entry.u.msi.data = table_entry.msg_data;
-
-                gsi_msi_routes.insert(gsi, entry);
-            }
-        } else {
-            let mut gsi_msi_routes = self.gsi_msi_routes.lock().unwrap();
-
-            for route in self.interrupt_routes.iter() {
-                gsi_msi_routes.remove(&route.gsi);
-            }
-        }
-
-        self.set_kvm_routes()
-    }
-
-    fn read_msix_table(&mut self, offset: u64, data: &mut [u8]) {
-        self.interrupt.msix_read_table(offset, data);
-    }
-
-    fn write_msix_table(&mut self, offset: u64, data: &[u8]) {
-        self.interrupt.msix_write_table(offset, data);
-
-        if let Some(msix) = &self.interrupt.msix {
-            if let Err(e) = self.update_msix_interrupt_routes(&msix) {
-                error!("Could not update MSI-X interrupt routes: {}", e);
-            }
-        }
-    }
-
     fn update_msi_capabilities(&mut self, offset: u64, data: &[u8]) -> Result<()> {
         match self.interrupt.update_msi(offset, data) {
-            Some(InterruptUpdateAction::EnableMsi) => match self.enable_irq_fds() {
-                Ok(fds) => {
-                    if let Err(e) = self.device.enable_msi(fds) {
+            Some(InterruptUpdateAction::EnableMsi) => {
+                if let Some(msi) = &self.interrupt.msi {
+                    let mut irq_fds: Vec<&EventFd> = Vec::new();
+                    for i in 0..msi.cfg.num_enabled_vectors() {
+                        if let Some(eventfd) =
+                            msi.interrupt_source_group.notifier(i as InterruptIndex)
+                        {
+                            irq_fds.push(eventfd);
+                        } else {
+                            return Err(VfioPciError::UpdateMsiEventFd);
+                        }
+                    }
+
+                    if let Err(e) = self.device.enable_msi(irq_fds) {
                         warn!("Could not enable MSI: {}", e);
                     }
                 }
-                Err(e) => warn!("Could not get IRQ fds: {}", e),
-            },
+            }
             Some(InterruptUpdateAction::DisableMsi) => {
                 if let Err(e) = self.device.disable_msi() {
                     warn!("Could not disable MSI: {}", e);
                 }
-                if let Err(e) = self.disable_irq_fds() {
-                    warn!("Could not disable MSI: {}", e);
-                }
             }
             _ => {}
         }
 
-        // Update the gsi_msi_routes table now that the MSI cache has been
-        // updated. The point is to always update the table based on latest
-        // changes to the cache, and based on the state of masking flags, the
-        // KVM GSI routes should be configured.
-        if let Some(msi) = &self.interrupt.msi {
-            return self.update_msi_interrupt_routes(&msi);
-        }
-
-        // If the code reach this point, something went wrong.
-        Err(VfioPciError::MsiNotConfigured)
+        Ok(())
     }
 
     fn update_msix_capabilities(&mut self, offset: u64, data: &[u8]) -> Result<()> {
         match self.interrupt.update_msix(offset, data) {
-            Some(InterruptUpdateAction::EnableMsix) => match self.enable_irq_fds() {
-                Ok(fds) => {
-                    if let Err(e) = self.device.enable_msix(fds) {
+            Some(InterruptUpdateAction::EnableMsix) => {
+                if let Some(msix) = &self.interrupt.msix {
+                    let mut irq_fds: Vec<&EventFd> = Vec::new();
+                    for i in 0..msix.bar.table_entries.len() {
+                        if let Some(eventfd) =
+                            msix.interrupt_source_group.notifier(i as InterruptIndex)
+                        {
+                            irq_fds.push(eventfd);
+                        } else {
+                            return Err(VfioPciError::UpdateMsiEventFd);
+                        }
+                    }
+
+                    if let Err(e) = self.device.enable_msix(irq_fds) {
                         warn!("Could not enable MSI-X: {}", e);
                     }
                 }
-                Err(e) => warn!("Could not get IRQ fds: {}", e),
-            },
+            }
             Some(InterruptUpdateAction::DisableMsix) => {
                 if let Err(e) = self.device.disable_msix() {
                     warn!("Could not disable MSI-X: {}", e);
                 }
-                if let Err(e) = self.disable_irq_fds() {
-                    warn!("Could not disable MSI: {}", e);
-                }
             }
             _ => {}
         }
 
-        // Update the gsi_msi_routes table because the state of the enable bit
-        // changed.
-        if let Some(msix) = &self.interrupt.msix {
-            return self.update_msix_interrupt_routes(&msix);
-        }
-
-        // If the code reach this point, something went wrong.
-        Err(VfioPciError::MsixNotConfigured)
+        Ok(())
     }
 
     fn find_region(&self, addr: u64) -> Option<MmioRegion> {
@@ -729,13 +594,13 @@ impl Drop for VfioPciDevice {
         self.unmap_mmio_regions();
 
         if let Some(msix) = &self.interrupt.msix {
-            if msix.cap.enabled() && self.device.disable_msix().is_err() {
+            if msix.bar.enabled() && self.device.disable_msix().is_err() {
                 error!("Could not disable MSI-X");
             }
         }
 
         if let Some(msi) = &self.interrupt.msi {
-            if msi.cap.enabled() && self.device.disable_msi().is_err() {
+            if msi.cfg.enabled() && self.device.disable_msi().is_err() {
                 error!("Could not disable MSI");
             }
         }
@@ -864,7 +729,7 @@ impl PciDevice for VfioPciDevice {
                     let msb_bar_offset: u32 = PCI_CONFIG_BAR_OFFSET + (bar_id + 1) * 4;
 
                     self.vfio_pci_configuration
-                        .write_config_dword(msb_bar_offset, msb_size);
+                        .write_config_dword(msb_size, msb_bar_offset);
 
                     msb_size = self
                         .vfio_pci_configuration
@@ -1002,7 +867,7 @@ impl PciDevice for VfioPciDevice {
             .region_write(VFIO_PCI_CONFIG_REGION_INDEX, data, reg + offset);
     }
 
-    fn read_config_register(&self, reg_idx: usize) -> u32 {
+    fn read_config_register(&mut self, reg_idx: usize) -> u32 {
         // When reading the BARs, we trap it and return what comes
         // from our local configuration space. We want the guest to
         // use that and not the VFIO device BARs as it does not map
@@ -1050,7 +915,7 @@ impl PciDevice for VfioPciDevice {
             let offset = addr - region.start.raw_value();
 
             if self.interrupt.msix_table_accessed(region.index, offset) {
-                self.read_msix_table(offset, data);
+                self.interrupt.msix_read_table(offset, data);
             } else {
                 self.device.region_read(region.index, data, offset);
             }
@@ -1064,7 +929,7 @@ impl PciDevice for VfioPciDevice {
 
             // If the MSI-X table is written to, we need to update our cache.
             if self.interrupt.msix_table_accessed(region.index, offset) {
-                self.write_msix_table(offset, data);
+                self.interrupt.msix_write_table(offset, data);
             } else {
                 self.device.region_write(region.index, data, offset);
             }

vhost_rs/Cargo.toml

@@ -13,9 +13,9 @@ vhost-user-master = []
 vhost-user-slave = []
 
 [dependencies]
-bitflags = "1.1.0"
-libc = "0.2.60"
-vmm-sys-util = ">=0.3.1"
+bitflags = "1.2.1"
+libc = "0.2.66"
+vmm-sys-util = "0.4.0"
 
 [dependencies.vm-memory]
 git = "https://github.com/rust-vmm/vm-memory"

vhost_rs/src/vhost_user/connection.rs

@@ -489,7 +489,6 @@ mod tests {
 
     use self::tempfile::tempfile;
     use super::*;
-    use libc;
     use std::fs::File;
     use std::io::{Read, Seek, SeekFrom, Write};
     use std::os::unix::io::FromRawFd;
@@ -513,18 +512,10 @@ mod tests {
         // accept on a fd without incoming connection
         let conn = listener.accept().unwrap();
         assert!(conn.is_none());
-
-        listener.set_nonblocking(true).unwrap();
-
-        // accept on a closed fd
-        unsafe {
-            libc::close(listener.as_raw_fd());
-        }
-        let conn2 = listener.accept();
-        assert!(conn2.is_err());
     }
 
     #[test]
+    #[ignore]
     fn send_data() {
         let listener = Listener::new(UNIX_SOCKET_DATA, true).unwrap();
         listener.set_nonblocking(true).unwrap();
@@ -550,6 +541,7 @@ mod tests {
     }
 
     #[test]
+    #[ignore]
     fn send_fd() {
         let listener = Listener::new(UNIX_SOCKET_FD, true).unwrap();
         listener.set_nonblocking(true).unwrap();
@@ -703,6 +695,7 @@ mod tests {
     }
 
     #[test]
+    #[ignore]
     fn send_recv() {
         let listener = Listener::new(UNIX_SOCKET_SEND, true).unwrap();
         listener.set_nonblocking(true).unwrap();

vhost_rs/src/vhost_user/master.rs

@@ -631,6 +631,7 @@ mod tests {
     }
 
     #[test]
+    #[ignore]
     fn create_master() {
         let listener = Listener::new(UNIX_SOCKET_MASTER, true).unwrap();
         listener.set_nonblocking(true).unwrap();
@@ -656,6 +657,7 @@ mod tests {
     }
 
     #[test]
+    #[ignore]
     fn test_create_failure() {
         let _ = Listener::new(UNIX_SOCKET_MASTER2, true).unwrap();
         let _ = Listener::new(UNIX_SOCKET_MASTER2, false).is_err();
@@ -670,6 +672,7 @@ mod tests {
     }
 
     #[test]
+    #[ignore]
     fn test_features() {
         let (mut master, mut peer) = create_pair(UNIX_SOCKET_MASTER3);
 
@@ -701,6 +704,7 @@ mod tests {
     }
 
     #[test]
+    #[ignore]
     fn test_protocol_features() {
         let (mut master, mut peer) = create_pair(UNIX_SOCKET_MASTER4);
 

vhost_rs/src/vhost_user/message.rs

@@ -631,7 +631,7 @@ bitflags! {
 }
 
 /// Max entries in one virtio-fs slave request.
-const VHOST_USER_FS_SLAVE_ENTRIES: usize = 8;
+pub const VHOST_USER_FS_SLAVE_ENTRIES: usize = 8;
 
 /// Slave request message to update the MMIO window.
 #[repr(packed)]
@@ -782,6 +782,7 @@ mod tests {
     }
 
     #[test]
+    #[ignore]
     fn check_user_config_msg() {
         let mut msg = VhostUserConfig::new(
             VHOST_USER_CONFIG_OFFSET,

vhost_rs/src/vhost_user/mod.rs

@@ -178,7 +178,7 @@ mod tests {
         backend: Arc<Mutex<S>>,
     ) -> (Master, SlaveReqHandler<S>) {
         let mut slave_listener = SlaveListener::new(path, true, backend).unwrap();
-        let master = Master::connect(path).unwrap();
+        let master = Master::connect(path, 1).unwrap();
         (master, slave_listener.accept().unwrap().unwrap())
     }
 

vhost_user_backend/Cargo.toml

@@ -10,11 +10,11 @@ pci_support = ["vm-virtio/pci_support"]
 mmio_support = ["vm-virtio/mmio_support"]
 
 [dependencies]
-epoll = ">=4.0.1"
+epoll = "4.1.0"
 libc = "0.2.66"
 vm-memory = { git = "https://github.com/rust-vmm/vm-memory" }
 vm-virtio = { path = "../vm-virtio" }
-vmm-sys-util = ">=0.3.1"
+vmm-sys-util = "0.4.0"
 
 [dependencies.vhost_rs]
 path = "../vhost_rs"

vhost_user_backend/src/lib.rs

@@ -536,7 +536,7 @@ impl<S: VhostUserBackend> VhostUserSlaveReqHandler for VhostUserHandler<S> {
             });
         }
 
-        let mem = GuestMemoryMmap::with_files(regions).map_err(|e| {
+        let mem = GuestMemoryMmap::from_ranges_with_files(regions).map_err(|e| {
             VhostUserError::ReqHandlerError(io::Error::new(io::ErrorKind::Other, e))
         })?;
         self.backend

vhost_user_block/Cargo.toml

@@ -0,0 +1,18 @@
+[package]
+name = "vhost_user_block"
+version = "0.1.0"
+authors = ["The Cloud Hypervisor Authors"]
+edition = "2018"
+
+[dependencies]
+bitflags = "1.2.1"
+epoll = "4.1.0"
+libc = "0.2.66"
+log = "0.4.8"
+qcow = { path = "../qcow" }
+vhost_user_backend = { path = "../vhost_user_backend" }
+vhost_rs = { path = "../vhost_rs" }
+virtio-bindings = "0.1.0"
+vm-memory = { git = "https://github.com/rust-vmm/vm-memory" }
+vm-virtio = { path = "../vm-virtio" }
+vmm-sys-util = "0.4.0"

vhost_user_block/src/lib.rs

@@ -0,0 +1,336 @@
+// Copyright 2019 Red Hat, Inc. All Rights Reserved.
+//
+// Portions Copyright 2019 Intel Corporation. All Rights Reserved.
+//
+// Portions Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+//
+// Portions Copyright 2017 The Chromium OS Authors. All rights reserved.
+//
+// SPDX-License-Identifier: (Apache-2.0 AND BSD-3-Clause)
+
+extern crate log;
+extern crate vhost_rs;
+extern crate vhost_user_backend;
+extern crate vm_virtio;
+
+use epoll;
+use log::*;
+use qcow::{self, ImageType, QcowFile};
+use std::fs::File;
+use std::fs::OpenOptions;
+use std::io::Read;
+use std::io::{Seek, SeekFrom, Write};
+use std::mem;
+use std::os::unix::fs::OpenOptionsExt;
+use std::path::PathBuf;
+use std::process;
+use std::slice;
+use std::sync::{Arc, RwLock};
+use std::vec::Vec;
+use vhost_rs::vhost_user::message::*;
+use vhost_user_backend::{VhostUserBackend, VhostUserDaemon, Vring, VringWorker};
+use virtio_bindings::bindings::virtio_blk::*;
+use vm_memory::{Bytes, GuestMemoryError, GuestMemoryMmap};
+use vm_virtio::block::{build_disk_image_id, Request};
+
+const QUEUE_SIZE: usize = 1024;
+const SECTOR_SHIFT: u8 = 9;
+const SECTOR_SIZE: u64 = (0x01 as u64) << SECTOR_SHIFT;
+const BLK_SIZE: u32 = 512;
+
+trait DiskFile: Read + Seek + Write + Send + Sync {}
+impl<D: Read + Seek + Write + Send + Sync> DiskFile for D {}
+
+pub type Result<T> = std::result::Result<T, Error>;
+pub type VhostUserBackendResult<T> = std::result::Result<T, std::io::Error>;
+
+#[derive(Debug)]
+pub enum Error {
+    /// Failed to detect image type.
+    DetectImageType,
+    /// Bad memory address.
+    GuestMemory(GuestMemoryError),
+    /// Can't open image file.
+    OpenImage,
+    /// Failed to parse direct parameter.
+    ParseDirectParam,
+    /// Failed to parse image parameter.
+    ParseImageParam,
+    /// Failed to parse sock parameter.
+    ParseSockParam,
+    /// Failed to parse readonly parameter.
+    ParseReadOnlyParam,
+    /// Failed parsing fs number of queues parameter.
+    ParseBlkNumQueuesParam(std::num::ParseIntError),
+}
+
+pub struct VhostUserBlkBackend {
+    mem: Option<GuestMemoryMmap>,
+    vring_worker: Option<Arc<VringWorker>>,
+    disk_image: Box<dyn DiskFile>,
+    disk_image_id: Vec<u8>,
+    disk_nsectors: u64,
+    config: virtio_blk_config,
+    rdonly: bool,
+}
+
+impl VhostUserBlkBackend {
+    pub fn new(image_path: String, num_queues: usize, rdonly: bool, direct: bool) -> Result<Self> {
+        let mut options = OpenOptions::new();
+        options.read(true);
+        options.write(!rdonly);
+        if direct {
+            options.custom_flags(libc::O_DIRECT);
+        }
+        let image: File = options.open(&image_path).unwrap();
+        let mut raw_img: vm_virtio::RawFile = vm_virtio::RawFile::new(image, direct);
+
+        let image_id = build_disk_image_id(&PathBuf::from(&image_path));
+        let image_type = qcow::detect_image_type(&mut raw_img).unwrap();
+        let mut image = match image_type {
+            ImageType::Raw => Box::new(raw_img) as Box<dyn DiskFile>,
+            ImageType::Qcow2 => Box::new(QcowFile::from(raw_img).unwrap()) as Box<dyn DiskFile>,
+        };
+
+        let nsectors = (image.seek(SeekFrom::End(0)).unwrap() as u64) / SECTOR_SIZE;
+        let mut config = virtio_blk_config::default();
+
+        config.capacity = nsectors;
+        config.blk_size = BLK_SIZE;
+        config.size_max = 65535;
+        config.seg_max = 128 - 2;
+        config.min_io_size = 1;
+        config.opt_io_size = 1;
+        config.num_queues = num_queues as u16;
+        config.wce = 1;
+
+        Ok(VhostUserBlkBackend {
+            mem: None,
+            vring_worker: None,
+            disk_image: image,
+            disk_image_id: image_id,
+            disk_nsectors: nsectors,
+            config,
+            rdonly,
+        })
+    }
+
+    pub fn process_queue(&mut self, vring: &mut Vring) -> bool {
+        let mut used_any = false;
+        let mem = match self.mem.as_ref() {
+            Some(m) => m,
+            None => return false,
+        };
+
+        while let Some(head) = vring.mut_queue().iter(mem).next() {
+            debug!("got an element in the queue");
+            let len;
+            match Request::parse(&head, mem) {
+                Ok(request) => {
+                    debug!("element is a valid request");
+                    let status = match request.execute(
+                        &mut self.disk_image,
+                        self.disk_nsectors,
+                        mem,
+                        &self.disk_image_id,
+                    ) {
+                        Ok(l) => {
+                            len = l;
+                            VIRTIO_BLK_S_OK
+                        }
+                        Err(e) => {
+                            len = 1;
+                            e.status()
+                        }
+                    };
+                    mem.write_obj(status, request.status_addr).unwrap();
+                }
+                Err(err) => {
+                    error!("failed to parse available descriptor chain: {:?}", err);
+                    len = 0;
+                }
+            }
+            vring.mut_queue().add_used(mem, head.index, len);
+            used_any = true;
+        }
+
+        used_any
+    }
+
+    pub fn set_vring_worker(&mut self, vring_worker: Option<Arc<VringWorker>>) {
+        self.vring_worker = vring_worker;
+    }
+}
+
+impl VhostUserBackend for VhostUserBlkBackend {
+    fn num_queues(&self) -> usize {
+        self.config.num_queues as usize
+    }
+
+    fn max_queue_size(&self) -> usize {
+        QUEUE_SIZE
+    }
+
+    fn features(&self) -> u64 {
+        let mut avail_features = 1 << VIRTIO_BLK_F_MQ
+            | 1 << VIRTIO_BLK_F_CONFIG_WCE
+            | 1 << VIRTIO_F_VERSION_1
+            | VhostUserVirtioFeatures::PROTOCOL_FEATURES.bits();
+
+        if self.rdonly {
+            avail_features |= 1 << VIRTIO_BLK_F_RO;
+        }
+        avail_features
+    }
+
+    fn protocol_features(&self) -> VhostUserProtocolFeatures {
+        VhostUserProtocolFeatures::CONFIG
+    }
+
+    fn update_memory(&mut self, mem: GuestMemoryMmap) -> VhostUserBackendResult<()> {
+        self.mem = Some(mem);
+        Ok(())
+    }
+
+    fn handle_event(
+        &mut self,
+        device_event: u16,
+        evset: epoll::Events,
+        vrings: &[Arc<RwLock<Vring>>],
+    ) -> VhostUserBackendResult<bool> {
+        if evset != epoll::Events::EPOLLIN {
+            warn!("invalid events operation");
+            return Ok(false);
+        }
+
+        debug!("event received: {:?}", device_event);
+
+        let mut vring = vrings[device_event as usize].write().unwrap();
+        if self.process_queue(&mut vring) {
+            debug!("signalling queue");
+            vring.signal_used_queue().unwrap();
+        }
+
+        Ok(false)
+    }
+
+    fn get_config(&self, _offset: u32, _size: u32) -> Vec<u8> {
+        // self.config is a statically allocated virtio_blk_config
+        let buf = unsafe {
+            slice::from_raw_parts(
+                &self.config as *const virtio_blk_config as *const _,
+                mem::size_of::<virtio_blk_config>(),
+            )
+        };
+
+        buf.to_vec()
+    }
+}
+
+pub struct VhostUserBlkBackendConfig<'a> {
+    pub image: &'a str,
+    pub sock: &'a str,
+    pub num_queues: usize,
+    pub readonly: bool,
+    pub direct: bool,
+}
+
+impl<'a> VhostUserBlkBackendConfig<'a> {
+    pub fn parse(backend: &'a str) -> Result<Self> {
+        let params_list: Vec<&str> = backend.split(',').collect();
+
+        let mut image: &str = "";
+        let mut sock: &str = "";
+        let mut num_queues_str: &str = "";
+        let mut readonly: bool = false;
+        let mut direct: bool = false;
+
+        for param in params_list.iter() {
+            if param.starts_with("image=") {
+                image = &param[6..];
+            } else if param.starts_with("sock=") {
+                sock = &param[5..];
+            } else if param.starts_with("num_queues=") {
+                num_queues_str = &param[11..];
+            } else if param.starts_with("readonly=") {
+                readonly = match param[9..].parse::<bool>() {
+                    Ok(b) => b,
+                    Err(_) => return Err(Error::ParseReadOnlyParam),
+                }
+            } else if param.starts_with("direct=") {
+                direct = match param[7..].parse::<bool>() {
+                    Ok(b) => b,
+                    Err(_) => return Err(Error::ParseDirectParam),
+                }
+            }
+        }
+
+        let mut num_queues: usize = 1;
+        if image.is_empty() {
+            return Err(Error::ParseImageParam);
+        }
+        if sock.is_empty() {
+            return Err(Error::ParseSockParam);
+        }
+        if !num_queues_str.is_empty() {
+            num_queues = num_queues_str
+                .parse()
+                .map_err(Error::ParseBlkNumQueuesParam)?;
+        }
+        Ok(VhostUserBlkBackendConfig {
+            image,
+            sock,
+            num_queues,
+            readonly,
+            direct,
+        })
+    }
+}
+
+pub fn start_block_backend(backend_command: &str) {
+    let backend_config = match VhostUserBlkBackendConfig::parse(backend_command) {
+        Ok(config) => config,
+        Err(e) => {
+            println!("Failed parsing parameters {:?}", e);
+            process::exit(1);
+        }
+    };
+
+    let blk_backend = Arc::new(RwLock::new(
+        VhostUserBlkBackend::new(
+            backend_config.image.to_string(),
+            backend_config.num_queues,
+            backend_config.readonly,
+            backend_config.direct,
+        )
+        .unwrap(),
+    ));
+
+    debug!("blk_backend is created!\n");
+
+    let name = "vhost-user-blk-backend";
+    let mut blk_daemon = VhostUserDaemon::new(
+        name.to_string(),
+        backend_config.sock.to_string(),
+        blk_backend.clone(),
+    )
+    .unwrap();
+    debug!("blk_daemon is created!\n");
+
+    let vring_worker = blk_daemon.get_vring_worker();
+
+    blk_backend
+        .write()
+        .unwrap()
+        .set_vring_worker(Some(vring_worker));
+
+    if let Err(e) = blk_daemon.start() {
+        println!(
+            "failed to start daemon for vhost-user-blk with error: {:?}\n",
+            e
+        );
+        process::exit(1);
+    }
+
+    blk_daemon.wait().unwrap();
+}

vhost_user_fs/Cargo.toml

@@ -5,8 +5,8 @@ authors = ["The Cloud Hypervisor Authors"]
 edition = "2018"
 
 [dependencies]
-bitflags = "1.1.0"
-libc = "0.2.65"
+bitflags = "1.2.1"
+libc = "0.2.66"
 log = "0.4.8"
 vm-memory = { git = "https://github.com/rust-vmm/vm-memory" }
 vm-virtio = { path = "../vm-virtio" }

vhost_user_fs/src/descriptor_utils.rs

@@ -546,7 +546,7 @@ mod tests {
         use DescriptorType::*;
 
         let memory_start_addr = GuestAddress(0x0);
-        let memory = GuestMemoryMmap::new(&vec![(memory_start_addr, 0x10000)]).unwrap();
+        let memory = GuestMemoryMmap::from_ranges(&vec![(memory_start_addr, 0x10000)]).unwrap();
 
         let chain = create_descriptor_chain(
             &memory,
@@ -587,7 +587,7 @@ mod tests {
         use DescriptorType::*;
 
         let memory_start_addr = GuestAddress(0x0);
-        let memory = GuestMemoryMmap::new(&vec![(memory_start_addr, 0x10000)]).unwrap();
+        let memory = GuestMemoryMmap::from_ranges(&vec![(memory_start_addr, 0x10000)]).unwrap();
 
         let chain = create_descriptor_chain(
             &memory,
@@ -628,7 +628,7 @@ mod tests {
         use DescriptorType::*;
 
         let memory_start_addr = GuestAddress(0x0);
-        let memory = GuestMemoryMmap::new(&vec![(memory_start_addr, 0x10000)]).unwrap();
+        let memory = GuestMemoryMmap::from_ranges(&vec![(memory_start_addr, 0x10000)]).unwrap();
 
         let chain = create_descriptor_chain(
             &memory,
@@ -653,7 +653,7 @@ mod tests {
         use DescriptorType::*;
 
         let memory_start_addr = GuestAddress(0x0);
-        let memory = GuestMemoryMmap::new(&vec![(memory_start_addr, 0x10000)]).unwrap();
+        let memory = GuestMemoryMmap::from_ranges(&vec![(memory_start_addr, 0x10000)]).unwrap();
 
         let chain = create_descriptor_chain(
             &memory,
@@ -678,7 +678,7 @@ mod tests {
         use DescriptorType::*;
 
         let memory_start_addr = GuestAddress(0x0);
-        let memory = GuestMemoryMmap::new(&vec![(memory_start_addr, 0x10000)]).unwrap();
+        let memory = GuestMemoryMmap::from_ranges(&vec![(memory_start_addr, 0x10000)]).unwrap();
 
         let chain = create_descriptor_chain(
             &memory,
@@ -726,7 +726,7 @@ mod tests {
         use DescriptorType::*;
 
         let memory_start_addr = GuestAddress(0x0);
-        let memory = GuestMemoryMmap::new(&vec![(memory_start_addr, 0x10000)]).unwrap();
+        let memory = GuestMemoryMmap::from_ranges(&vec![(memory_start_addr, 0x10000)]).unwrap();
 
         let secret: Le32 = 0x12345678.into();
 
@@ -765,7 +765,7 @@ mod tests {
         use DescriptorType::*;
 
         let memory_start_addr = GuestAddress(0x0);
-        let memory = GuestMemoryMmap::new(&vec![(memory_start_addr, 0x10000)]).unwrap();
+        let memory = GuestMemoryMmap::from_ranges(&vec![(memory_start_addr, 0x10000)]).unwrap();
 
         let chain = create_descriptor_chain(
             &memory,
@@ -795,7 +795,7 @@ mod tests {
         use DescriptorType::*;
 
         let memory_start_addr = GuestAddress(0x0);
-        let memory = GuestMemoryMmap::new(&vec![(memory_start_addr, 0x10000)]).unwrap();
+        let memory = GuestMemoryMmap::from_ranges(&vec![(memory_start_addr, 0x10000)]).unwrap();
 
         let chain = create_descriptor_chain(
             &memory,
@@ -824,7 +824,7 @@ mod tests {
         use DescriptorType::*;
 
         let memory_start_addr = GuestAddress(0x0);
-        let memory = GuestMemoryMmap::new(&vec![(memory_start_addr, 0x10000)]).unwrap();
+        let memory = GuestMemoryMmap::from_ranges(&vec![(memory_start_addr, 0x10000)]).unwrap();
 
         let chain = create_descriptor_chain(
             &memory,
@@ -853,7 +853,7 @@ mod tests {
         use DescriptorType::*;
 
         let memory_start_addr = GuestAddress(0x0);
-        let memory = GuestMemoryMmap::new(&vec![(memory_start_addr, 0x10000)]).unwrap();
+        let memory = GuestMemoryMmap::from_ranges(&vec![(memory_start_addr, 0x10000)]).unwrap();
 
         let chain = create_descriptor_chain(
             &memory,
@@ -882,7 +882,7 @@ mod tests {
         use DescriptorType::*;
 
         let memory_start_addr = GuestAddress(0x0);
-        let memory = GuestMemoryMmap::new(&vec![(memory_start_addr, 0x10000)]).unwrap();
+        let memory = GuestMemoryMmap::from_ranges(&vec![(memory_start_addr, 0x10000)]).unwrap();
 
         let chain = create_descriptor_chain(
             &memory,
@@ -911,7 +911,7 @@ mod tests {
         use DescriptorType::*;
 
         let memory_start_addr = GuestAddress(0x0);
-        let memory = GuestMemoryMmap::new(&vec![(memory_start_addr, 0x10000)]).unwrap();
+        let memory = GuestMemoryMmap::from_ranges(&vec![(memory_start_addr, 0x10000)]).unwrap();
 
         let chain = create_descriptor_chain(
             &memory,
@@ -940,7 +940,7 @@ mod tests {
         use DescriptorType::*;
 
         let memory_start_addr = GuestAddress(0x0);
-        let memory = GuestMemoryMmap::new(&vec![(memory_start_addr, 0x10000)]).unwrap();
+        let memory = GuestMemoryMmap::from_ranges(&vec![(memory_start_addr, 0x10000)]).unwrap();
 
         let chain = create_descriptor_chain(
             &memory,
@@ -964,7 +964,7 @@ mod tests {
         use DescriptorType::*;
 
         let memory_start_addr = GuestAddress(0x0);
-        let memory = GuestMemoryMmap::new(&vec![(memory_start_addr, 0x10000)]).unwrap();
+        let memory = GuestMemoryMmap::from_ranges(&vec![(memory_start_addr, 0x10000)]).unwrap();
 
         let chain = create_descriptor_chain(
             &memory,

vhost_user_net/Cargo.toml

@@ -0,0 +1,18 @@
+[package]
+name = "vhost_user_net"
+version = "0.1.0"
+authors = ["The Cloud Hypervisor Authors"]
+edition = "2018"
+
+[dependencies]
+bitflags = "1.2.1"
+epoll = "4.1.0"
+libc = "0.2.66"
+log = "0.4.8"
+net_util = { path = "../net_util" }
+vhost_user_backend = { path = "../vhost_user_backend" }
+vhost_rs = { path = "../vhost_rs" }
+virtio-bindings = "0.1.0"
+vm-memory = { git = "https://github.com/rust-vmm/vm-memory" }
+vm-virtio = { path = "../vm-virtio" }
+vmm-sys-util = "0.4.0"

vhost_user_net/src/lib.rs

@@ -0,0 +1,483 @@
+// Copyright 2019 Intel Corporation. All Rights Reserved.
+//
+// Portions Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+//
+// Portions Copyright 2017 The Chromium OS Authors. All rights reserved.
+//
+// SPDX-License-Identifier: (Apache-2.0 AND BSD-3-Clause)
+
+extern crate log;
+extern crate net_util;
+extern crate vhost_rs;
+extern crate vhost_user_backend;
+extern crate vm_virtio;
+
+use epoll;
+use libc::{self, EAGAIN, EFD_NONBLOCK};
+use log::*;
+use net_util::Tap;
+use std::convert::TryFrom;
+use std::fmt;
+use std::io::Read;
+use std::io::{self};
+use std::net::Ipv4Addr;
+use std::os::unix::io::AsRawFd;
+use std::os::unix::io::RawFd;
+use std::process;
+use std::sync::{Arc, RwLock};
+use std::vec::Vec;
+use vhost_rs::vhost_user::message::*;
+use vhost_rs::vhost_user::Error as VhostUserError;
+use vhost_user_backend::{VhostUserBackend, VhostUserDaemon, Vring, VringWorker};
+use virtio_bindings::bindings::virtio_net::*;
+use vm_memory::GuestMemoryMmap;
+use vm_virtio::net_util::{open_tap, RxVirtio, TxVirtio};
+use vm_virtio::Queue;
+use vmm_sys_util::eventfd::EventFd;
+
+pub type VhostUserResult<T> = std::result::Result<T, VhostUserError>;
+pub type Result<T> = std::result::Result<T, Error>;
+pub type VhostUserBackendResult<T> = std::result::Result<T, std::io::Error>;
+
+#[derive(Debug)]
+pub enum Error {
+    /// Failed to activate device.
+    BadActivate,
+    /// Failed to create kill eventfd
+    CreateKillEventFd(io::Error),
+    /// Failed to add event.
+    EpollCtl(io::Error),
+    /// Fail to wait event.
+    EpollWait(io::Error),
+    /// Failed to create EventFd.
+    EpollCreateFd,
+    /// Failed to read Tap.
+    FailedReadTap,
+    /// Failed to signal used queue.
+    FailedSignalingUsedQueue,
+    /// Failed to handle event other than input event.
+    HandleEventNotEpollIn,
+    /// Failed to handle unknown event.
+    HandleEventUnknownEvent,
+    /// Invalid vring address.
+    InvalidVringAddr,
+    /// No vring call fd to notify.
+    NoVringCallFdNotify,
+    /// No memory configured.
+    NoMemoryConfigured,
+    /// Failed to parse sock parameter.
+    ParseSockParam,
+    /// Failed to parse ip parameter.
+    ParseIpParam(std::net::AddrParseError),
+    /// Failed to parse mask parameter.
+    ParseMaskParam(std::net::AddrParseError),
+    /// Failed to parse queue number.
+    ParseQueueNumParam(std::num::ParseIntError),
+    /// Failed to parse queue size.
+    ParseQueueSizeParam(std::num::ParseIntError),
+    /// Open tap device failed.
+    OpenTap(vm_virtio::net_util::Error),
+}
+
+impl fmt::Display for Error {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "vhost_user_net_error: {:?}", self)
+    }
+}
+
+impl std::error::Error for Error {}
+
+impl std::convert::From<Error> for std::io::Error {
+    fn from(e: Error) -> Self {
+        std::io::Error::new(io::ErrorKind::Other, e)
+    }
+}
+
+pub struct VhostUserNetBackend {
+    mem: Option<GuestMemoryMmap>,
+    vring_worker: Option<Arc<VringWorker>>,
+    kill_evt: EventFd,
+    taps: Vec<(Tap, usize)>,
+    rxs: Vec<RxVirtio>,
+    txs: Vec<TxVirtio>,
+    rx_tap_listenings: Vec<bool>,
+    num_queues: usize,
+    queue_size: u16,
+}
+
+impl std::clone::Clone for VhostUserNetBackend {
+    fn clone(&self) -> Self {
+        VhostUserNetBackend {
+            mem: self.mem.clone(),
+            vring_worker: self.vring_worker.clone(),
+            kill_evt: self.kill_evt.try_clone().unwrap(),
+            taps: self.taps.clone(),
+            rxs: self.rxs.clone(),
+            txs: self.txs.clone(),
+            rx_tap_listenings: self.rx_tap_listenings.clone(),
+            num_queues: self.num_queues,
+            queue_size: self.queue_size,
+        }
+    }
+}
+
+impl VhostUserNetBackend {
+    /// Create a new virtio network device with the given TAP interface.
+    pub fn new_with_tap(taps: Vec<Tap>, num_queues: usize, queue_size: u16) -> Result<Self> {
+        let mut taps_v: Vec<(Tap, usize)> = Vec::new();
+        for (i, tap) in taps.iter().enumerate() {
+            taps_v.push((tap.clone(), num_queues + i));
+        }
+
+        let mut rxs: Vec<RxVirtio> = Vec::new();
+        let mut txs: Vec<TxVirtio> = Vec::new();
+        let mut rx_tap_listenings: Vec<bool> = Vec::new();
+
+        for _ in 0..taps.len() {
+            let rx = RxVirtio::new();
+            rxs.push(rx);
+            let tx = TxVirtio::new();
+            txs.push(tx);
+            rx_tap_listenings.push(false);
+        }
+
+        Ok(VhostUserNetBackend {
+            mem: None,
+            vring_worker: None,
+            kill_evt: EventFd::new(EFD_NONBLOCK).map_err(Error::CreateKillEventFd)?,
+            taps: taps_v,
+            rxs,
+            txs,
+            rx_tap_listenings,
+            num_queues,
+            queue_size,
+        })
+    }
+
+    /// Create a new virtio network device with the given IP address and
+    /// netmask.
+    pub fn new(
+        ip_addr: Ipv4Addr,
+        netmask: Ipv4Addr,
+        num_queues: usize,
+        queue_size: u16,
+    ) -> Result<Self> {
+        let taps =
+            open_tap(None, Some(ip_addr), Some(netmask), num_queues / 2).map_err(Error::OpenTap)?;
+
+        Self::new_with_tap(taps, num_queues, queue_size)
+    }
+
+    // Copies a single frame from `self.rx.frame_buf` into the guest. Returns true
+    // if a buffer was used, and false if the frame must be deferred until a buffer
+    // is made available by the driver.
+    fn rx_single_frame(&mut self, mut queue: &mut Queue, index: usize) -> Result<bool> {
+        let mem = self.mem.as_ref().ok_or(Error::NoMemoryConfigured)?;
+
+        let next_desc = queue.iter(&mem).next();
+
+        if next_desc.is_none() {
+            // Queue has no available descriptors
+            if self.rx_tap_listenings[index] {
+                self.vring_worker
+                    .as_ref()
+                    .unwrap()
+                    .unregister_listener(
+                        self.taps[index].0.as_raw_fd(),
+                        epoll::Events::EPOLLIN,
+                        u64::try_from(self.taps[index].1).unwrap(),
+                    )
+                    .unwrap();
+                self.rx_tap_listenings[index] = false;
+            }
+            return Ok(false);
+        }
+
+        let write_complete = self.rxs[index].process_desc_chain(&mem, next_desc, &mut queue);
+
+        Ok(write_complete)
+    }
+
+    fn process_rx(&mut self, vring: &mut Vring, index: usize) -> Result<()> {
+        // Read as many frames as possible.
+        loop {
+            match self.read_tap(index) {
+                Ok(count) => {
+                    self.rxs[index].bytes_read = count;
+                    if !self.rx_single_frame(&mut vring.mut_queue(), index)? {
+                        self.rxs[index].deferred_frame = true;
+                        break;
+                    }
+                }
+                Err(e) => {
+                    // The tap device is non-blocking, so any error aside from EAGAIN is
+                    // unexpected.
+                    match e.raw_os_error() {
+                        Some(err) if err == EAGAIN => (),
+                        _ => {
+                            error!("Failed to read tap: {:?}", e);
+                            return Err(Error::FailedReadTap);
+                        }
+                    };
+                    break;
+                }
+            }
+        }
+        if self.rxs[index].deferred_irqs {
+            self.rxs[index].deferred_irqs = false;
+            vring.signal_used_queue().unwrap();
+            Ok(())
+        } else {
+            Ok(())
+        }
+    }
+
+    fn resume_rx(&mut self, vring: &mut Vring, index: usize) -> Result<()> {
+        if self.rxs[index].deferred_frame {
+            if self.rx_single_frame(&mut vring.mut_queue(), index)? {
+                self.rxs[index].deferred_frame = false;
+                // process_rx() was interrupted possibly before consuming all
+                // packets in the tap; try continuing now.
+                self.process_rx(vring, index)
+            } else if self.rxs[index].deferred_irqs {
+                self.rxs[index].deferred_irqs = false;
+                vring.signal_used_queue().unwrap();
+                Ok(())
+            } else {
+                Ok(())
+            }
+        } else {
+            Ok(())
+        }
+    }
+
+    fn process_tx(&mut self, mut queue: &mut Queue, index: usize) -> Result<()> {
+        let mem = self.mem.as_ref().ok_or(Error::NoMemoryConfigured)?;
+
+        self.txs[index].process_desc_chain(&mem, &mut self.taps[index].0, &mut queue);
+
+        Ok(())
+    }
+
+    fn read_tap(&mut self, index: usize) -> io::Result<usize> {
+        self.taps[index].0.read(&mut self.rxs[index].frame_buf)
+    }
+
+    pub fn set_vring_worker(&mut self, vring_worker: Option<Arc<VringWorker>>) {
+        self.vring_worker = vring_worker;
+    }
+
+    pub fn get_kill_event(&self) -> (u16, RawFd) {
+        (
+            (self.num_queues + (self.num_queues / 2)) as u16,
+            self.kill_evt.as_raw_fd(),
+        )
+    }
+}
+
+impl VhostUserBackend for VhostUserNetBackend {
+    fn num_queues(&self) -> usize {
+        self.num_queues
+    }
+
+    fn max_queue_size(&self) -> usize {
+        self.queue_size as usize
+    }
+
+    fn features(&self) -> u64 {
+        1 << VIRTIO_NET_F_GUEST_CSUM
+            | 1 << VIRTIO_NET_F_CSUM
+            | 1 << VIRTIO_NET_F_GUEST_TSO4
+            | 1 << VIRTIO_NET_F_GUEST_UFO
+            | 1 << VIRTIO_NET_F_HOST_TSO4
+            | 1 << VIRTIO_NET_F_HOST_UFO
+            | 1 << VIRTIO_F_VERSION_1
+            | VhostUserVirtioFeatures::PROTOCOL_FEATURES.bits()
+    }
+
+    fn protocol_features(&self) -> VhostUserProtocolFeatures {
+        VhostUserProtocolFeatures::all()
+    }
+
+    fn update_memory(&mut self, mem: GuestMemoryMmap) -> VhostUserBackendResult<()> {
+        self.mem = Some(mem);
+        Ok(())
+    }
+
+    fn handle_event(
+        &mut self,
+        device_event: u16,
+        evset: epoll::Events,
+        vrings: &[Arc<RwLock<Vring>>],
+    ) -> VhostUserBackendResult<bool> {
+        if evset != epoll::Events::EPOLLIN {
+            return Err(Error::HandleEventNotEpollIn.into());
+        }
+
+        let tap_start_index = self.num_queues as u16;
+        let tap_end_index = (self.num_queues + self.num_queues / 2 - 1) as u16;
+        let kill_index = tap_end_index + 1;
+
+        match device_event {
+            x if ((x < self.num_queues as u16) && (x % 2 == 0)) => {
+                let index = (x / 2) as usize;
+                let mut vring = vrings[x as usize].write().unwrap();
+                self.resume_rx(&mut vring, index)?;
+
+                if !self.rx_tap_listenings[index] {
+                    self.vring_worker.as_ref().unwrap().register_listener(
+                        self.taps[index].0.as_raw_fd(),
+                        epoll::Events::EPOLLIN,
+                        u64::try_from(self.taps[index].1).unwrap(),
+                    )?;
+                    self.rx_tap_listenings[index] = true;
+                }
+            }
+            x if ((x < self.num_queues as u16) && (x % 2 != 0)) => {
+                let index = ((x - 1) / 2) as usize;
+                let mut vring = vrings[x as usize].write().unwrap();
+                self.process_tx(&mut vring.mut_queue(), index)?;
+            }
+            x if x >= tap_start_index && x <= tap_end_index => {
+                let index = x as usize - self.num_queues;
+                let mut vring = vrings[2 * index].write().unwrap();
+                if self.rxs[index].deferred_frame
+                // Process a deferred frame first if available. Don't read from tap again
+                // until we manage to receive this deferred frame.
+                {
+                    if self.rx_single_frame(&mut vring.mut_queue(), index)? {
+                        self.rxs[index].deferred_frame = false;
+                        self.process_rx(&mut vring, index)?;
+                    } else if self.rxs[index].deferred_irqs {
+                        self.rxs[index].deferred_irqs = false;
+                        vring.signal_used_queue()?;
+                    }
+                } else {
+                    self.process_rx(&mut vring, index)?;
+                }
+            }
+            x if x == kill_index => {
+                self.kill_evt.read().unwrap();
+                return Ok(true);
+            }
+            _ => return Err(Error::HandleEventUnknownEvent.into()),
+        }
+
+        Ok(false)
+    }
+}
+
+pub struct VhostUserNetBackendConfig<'a> {
+    pub ip: Ipv4Addr,
+    pub mask: Ipv4Addr,
+    pub sock: &'a str,
+    pub num_queues: usize,
+    pub queue_size: u16,
+}
+
+impl<'a> VhostUserNetBackendConfig<'a> {
+    pub fn parse(backend: &'a str) -> Result<Self> {
+        let params_list: Vec<&str> = backend.split(',').collect();
+
+        let mut ip_str: &str = "";
+        let mut mask_str: &str = "";
+        let mut sock: &str = "";
+        let mut num_queues_str: &str = "";
+        let mut queue_size_str: &str = "";
+
+        for param in params_list.iter() {
+            if param.starts_with("ip=") {
+                ip_str = &param[3..];
+            } else if param.starts_with("mask=") {
+                mask_str = &param[5..];
+            } else if param.starts_with("sock=") {
+                sock = &param[5..];
+            } else if param.starts_with("num_queues=") {
+                num_queues_str = &param[11..];
+            } else if param.starts_with("queue_size=") {
+                queue_size_str = &param[11..];
+            }
+        }
+
+        let mut ip: Ipv4Addr = Ipv4Addr::new(192, 168, 100, 1);
+        let mut mask: Ipv4Addr = Ipv4Addr::new(255, 255, 255, 0);
+        let mut num_queues: usize = 2;
+        let mut queue_size: u16 = 256;
+
+        if sock.is_empty() {
+            return Err(Error::ParseSockParam);
+        }
+        if !ip_str.is_empty() {
+            ip = ip_str.parse().map_err(Error::ParseIpParam)?;
+        }
+        if !mask_str.is_empty() {
+            mask = mask_str.parse().map_err(Error::ParseMaskParam)?;
+        }
+        if !num_queues_str.is_empty() {
+            num_queues = num_queues_str.parse().map_err(Error::ParseQueueNumParam)?;
+        }
+        if !queue_size_str.is_empty() {
+            queue_size = queue_size_str.parse().map_err(Error::ParseQueueSizeParam)?;
+        }
+
+        Ok(VhostUserNetBackendConfig {
+            ip,
+            mask,
+            sock,
+            num_queues,
+            queue_size,
+        })
+    }
+}
+
+pub fn start_net_backend(backend_command: &str) {
+    let backend_config = match VhostUserNetBackendConfig::parse(backend_command) {
+        Ok(config) => config,
+        Err(e) => {
+            println!("Failed parsing parameters {:?}", e);
+            process::exit(1);
+        }
+    };
+
+    let net_backend = Arc::new(RwLock::new(
+        VhostUserNetBackend::new(
+            backend_config.ip,
+            backend_config.mask,
+            backend_config.num_queues,
+            backend_config.queue_size,
+        )
+        .unwrap(),
+    ));
+
+    let mut net_daemon = VhostUserDaemon::new(
+        "vhost-user-net-backend".to_string(),
+        backend_config.sock.to_string(),
+        net_backend.clone(),
+    )
+    .unwrap();
+
+    let (kill_index, kill_evt_fd) = net_backend.read().unwrap().get_kill_event();
+    let vring_worker = net_daemon.get_vring_worker();
+
+    if let Err(e) =
+        vring_worker.register_listener(kill_evt_fd, epoll::Events::EPOLLIN, u64::from(kill_index))
+    {
+        println!("failed to register listener for kill event: {:?}", e);
+        process::exit(1);
+    }
+
+    net_backend
+        .write()
+        .unwrap()
+        .set_vring_worker(Some(vring_worker));
+
+    if let Err(e) = net_daemon.start() {
+        println!(
+            "failed to start daemon for vhost-user-net with error: {:?}",
+            e
+        );
+        process::exit(1);
+    }
+
+    net_daemon.wait().unwrap();
+}

vm-allocator/Cargo.toml

@@ -5,5 +5,5 @@ authors = ["The Chromium OS Authors"]
 edition = "2018"
 
 [dependencies]
-libc = "0.2.60"
+libc = "0.2.66"
 vm-memory = { git = "https://github.com/rust-vmm/vm-memory" }

vm-device/Cargo.toml

@@ -5,12 +5,13 @@ authors = ["The Cloud Hypervisor Authors"]
 edition = "2018"
 
 [dependencies]
-anyhow = "1.0"
-thiserror = "1.0"
-serde = {version = ">=1.0.27", features = ["rc"] }
-serde_derive = ">=1.0.27"
-serde_json = ">=1.0.9"
+anyhow = "1.0.26"
+thiserror = "1.0.11"
+serde = { version = "1.0.104", features = ["rc"] }
+serde_derive = "1.0.104"
+serde_json = "1.0.48"
+vmm-sys-util = "0.4.0"
 
 [dependencies.vm-memory]
 git = "https://github.com/rust-vmm/vm-memory"
-features = ["backend-mmap"]
+features = ["backend-mmap"]

vm-device/src/interrupt/mod.rs

@@ -0,0 +1,196 @@
+// Copyright (C) 2019 Alibaba Cloud. All rights reserved.
+// Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// Copyright © 2019 Intel Corporation
+//
+// SPDX-License-Identifier: Apache-2.0 OR BSD-3-Clause
+
+//! Traits and Structs to manage interrupt sources for devices.
+//!
+//! In system programming, an interrupt is a signal to the processor emitted by hardware or
+//! software indicating an event that needs immediate attention. An interrupt alerts the processor
+//! to a high-priority condition requiring the interruption of the current code the processor is
+//! executing. The processor responds by suspending its current activities, saving its state, and
+//! executing a function called an interrupt handler (or an interrupt service routine, ISR) to deal
+//! with the event. This interruption is temporary, and, after the interrupt handler finishes,
+//! unless handling the interrupt has emitted a fatal error, the processor resumes normal
+//! activities.
+//!
+//! Hardware interrupts are used by devices to communicate that they require attention from the
+//! operating system, or a bare-metal program running on the CPU if there are no OSes. The act of
+//! initiating a hardware interrupt is referred to as an interrupt request (IRQ). Different devices
+//! are usually associated with different interrupts using a unique value associated with each
+//! interrupt. This makes it possible to know which hardware device caused which interrupts.
+//! These interrupt values are often called IRQ lines, or just interrupt lines.
+//!
+//! Nowadays, IRQ lines is not the only mechanism to deliver device interrupts to processors.
+//! MSI [(Message Signaled Interrupt)](https://en.wikipedia.org/wiki/Message_Signaled_Interrupts)
+//! is another commonly used alternative in-band method of signaling an interrupt, using special
+//! in-band messages to replace traditional out-of-band assertion of dedicated interrupt lines.
+//! While more complex to implement in a device, message signaled interrupts have some significant
+//! advantages over pin-based out-of-band interrupt signaling. Message signaled interrupts are
+//! supported in PCI bus since its version 2.2, and in later available PCI Express bus. Some
+//! non-PCI architectures also use message signaled interrupts.
+//!
+//! While IRQ is a term commonly used by Operating Systems when dealing with hardware
+//! interrupts, the IRQ numbers managed by OSes are independent of the ones managed by VMM.
+//! For simplicity sake, the term `Interrupt Source` is used instead of IRQ to represent both
+//! pin-based interrupts and MSI interrupts.
+//!
+//! A device may support multiple types of interrupts, and each type of interrupt may support one
+//! or multiple interrupt sources. For example, a PCI device may support:
+//! * Legacy Irq: exactly one interrupt source.
+//! * PCI MSI Irq: 1,2,4,8,16,32 interrupt sources.
+//! * PCI MSIx Irq: 2^n(n=0-11) interrupt sources.
+//!
+//! A distinct Interrupt Source Identifier (ISID) will be assigned to each interrupt source.
+//! An ID allocator will be used to allocate and free Interrupt Source Identifiers for devices.
+//! To decouple the vm-device crate from the ID allocator, the vm-device crate doesn't take the
+//! responsibility to allocate/free Interrupt Source IDs but only makes use of assigned IDs.
+//!
+//! The overall flow to deal with interrupts is:
+//! * The VMM creates an interrupt manager
+//! * The VMM creates a device manager, passing on an reference to the interrupt manager
+//! * The device manager passes on an reference to the interrupt manager to all registered devices
+//! * The guest kernel loads drivers for virtual devices
+//! * The guest device driver determines the type and number of interrupts needed, and update the
+//!   device configuration
+//! * The virtual device backend requests the interrupt manager to create an interrupt group
+//!   according to guest configuration information
+
+use std::sync::Arc;
+use vmm_sys_util::eventfd::EventFd;
+
+/// Reuse std::io::Result to simplify interoperability among crates.
+pub type Result<T> = std::io::Result<T>;
+
+/// Data type to store an interrupt source identifier.
+pub type InterruptIndex = u32;
+
+/// Configuration data for legacy interrupts.
+///
+/// On x86 platforms, legacy interrupts means those interrupts routed through PICs or IOAPICs.
+#[derive(Copy, Clone, Debug)]
+pub struct LegacyIrqSourceConfig {}
+
+/// Configuration data for MSI/MSI-X interrupts.
+///
+/// On x86 platforms, these interrupts are vectors delivered directly to the LAPIC.
+#[derive(Copy, Clone, Debug, Default)]
+pub struct MsiIrqSourceConfig {
+    /// High address to delivery message signaled interrupt.
+    pub high_addr: u32,
+    /// Low address to delivery message signaled interrupt.
+    pub low_addr: u32,
+    /// Data to write to delivery message signaled interrupt.
+    pub data: u32,
+}
+
+/// Configuration data for an interrupt source.
+#[derive(Copy, Clone, Debug)]
+pub enum InterruptSourceConfig {
+    /// Configuration data for Legacy interrupts.
+    LegacyIrq(LegacyIrqSourceConfig),
+    /// Configuration data for PciMsi, PciMsix and generic MSI interrupts.
+    MsiIrq(MsiIrqSourceConfig),
+}
+
+/// Configuration data for legacy, pin based interrupt groups.
+///
+/// A legacy interrupt group only takes one irq number as its configuration.
+#[derive(Copy, Clone, Debug)]
+pub struct LegacyIrqGroupConfig {
+    /// Legacy irq number.
+    pub irq: InterruptIndex,
+}
+
+/// Configuration data for MSI/MSI-X interrupt groups
+///
+/// MSI/MSI-X interrupt groups are basically a set of vectors.
+#[derive(Copy, Clone, Debug)]
+pub struct MsiIrqGroupConfig {
+    /// First index of the MSI/MSI-X interrupt vectors
+    pub base: InterruptIndex,
+    /// Number of vectors in the MSI/MSI-X group.
+    pub count: InterruptIndex,
+}
+
+/// Trait to manage interrupt sources for virtual device backends.
+///
+/// The InterruptManager implementations should protect itself from concurrent accesses internally,
+/// so it could be invoked from multi-threaded context.
+pub trait InterruptManager {
+    type GroupConfig;
+
+    /// Create an [InterruptSourceGroup](trait.InterruptSourceGroup.html) object to manage
+    /// interrupt sources for a virtual device
+    ///
+    /// An [InterruptSourceGroup](trait.InterruptSourceGroup.html) object manages all interrupt
+    /// sources of the same type for a virtual device.
+    ///
+    /// # Arguments
+    /// * interrupt_type: type of interrupt source.
+    /// * base: base Interrupt Source ID to be managed by the group object.
+    /// * count: number of Interrupt Sources to be managed by the group object.
+    fn create_group(&self, config: Self::GroupConfig)
+        -> Result<Arc<Box<dyn InterruptSourceGroup>>>;
+
+    /// Destroy an [InterruptSourceGroup](trait.InterruptSourceGroup.html) object created by
+    /// [create_group()](trait.InterruptManager.html#tymethod.create_group).
+    ///
+    /// Assume the caller takes the responsibility to disable all interrupt sources of the group
+    /// before calling destroy_group(). This assumption helps to simplify InterruptSourceGroup
+    /// implementations.
+    fn destroy_group(&self, group: Arc<Box<dyn InterruptSourceGroup>>) -> Result<()>;
+}
+
+pub trait InterruptSourceGroup: Send + Sync {
+    /// Enable the interrupt sources in the group to generate interrupts.
+    fn enable(&self) -> Result<()> {
+        // Not all interrupt sources can be enabled.
+        // To accommodate this, we can have a no-op here.
+        Ok(())
+    }
+
+    /// Disable the interrupt sources in the group to generate interrupts.
+    fn disable(&self) -> Result<()> {
+        // Not all interrupt sources can be disabled.
+        // To accommodate this, we can have a no-op here.
+        Ok(())
+    }
+
+    /// Inject an interrupt from this interrupt source into the guest.
+    fn trigger(&self, index: InterruptIndex) -> Result<()>;
+
+    /// Returns an interrupt notifier from this interrupt.
+    ///
+    /// An interrupt notifier allows for external components and processes
+    /// to inject interrupts into a guest, by writing to the file returned
+    /// by this method.
+    #[allow(unused_variables)]
+    fn notifier(&self, index: InterruptIndex) -> Option<&EventFd> {
+        // One use case of the notifier is to implement vhost user backends.
+        // For all other implementations we can just return None here.
+        None
+    }
+
+    /// Update the interrupt source group configuration.
+    ///
+    /// # Arguments
+    /// * index: sub-index into the group.
+    /// * config: configuration data for the interrupt source.
+    fn update(&self, index: InterruptIndex, config: InterruptSourceConfig) -> Result<()>;
+
+    /// Mask an interrupt from this interrupt source.
+    fn mask(&self, _index: InterruptIndex) -> Result<()> {
+        // Not all interrupt sources can be disabled.
+        // To accommodate this, we can have a no-op here.
+        Ok(())
+    }
+
+    /// Unmask an interrupt from this interrupt source.
+    fn unmask(&self, _index: InterruptIndex) -> Result<()> {
+        // Not all interrupt sources can be disabled.
+        // To accommodate this, we can have a no-op here.
+        Ok(())
+    }
+}

vm-device/src/lib.rs

@@ -2,6 +2,8 @@ extern crate serde;
 extern crate thiserror;
 extern crate vm_memory;
 
+pub mod interrupt;
+
 use vm_memory::{
     Address, GuestAddress, GuestMemory, GuestMemoryMmap, GuestMemoryRegion, GuestRegionMmap,
     MemoryRegionAddress,
@@ -87,7 +89,7 @@ mod tests {
         let start_addr1 = GuestAddress(0x0);
         let start_addr2 = GuestAddress(0x1000);
         let guest_mem =
-            GuestMemoryMmap::new(&[(start_addr1, 0x400), (start_addr2, 0x400)]).unwrap();
+            GuestMemoryMmap::from_ranges(&[(start_addr1, 0x400), (start_addr2, 0x400)]).unwrap();
 
         assert!(get_host_address_range(&guest_mem, GuestAddress(0x600), 0x100).is_none());
 

vm-virtio/Cargo.toml

@@ -10,19 +10,20 @@ pci_support = ["pci"]
 mmio_support = []
 
 [dependencies]
-byteorder = "1.3.2"
+arc-swap = "0.4.4"
+byteorder = "1.3.4"
 devices = { path = "../devices" }
-epoll = ">=4.0.1"
-libc = "0.2.60"
+epoll = "4.1.0"
+libc = "0.2.66"
 log = "0.4.8"
 net_gen = { path = "../net_gen" }
 net_util = { path = "../net_util" }
 pci = { path = "../pci", optional = true }
 tempfile = "3.1.0"
-virtio-bindings = { git = "https://github.com/rust-vmm/virtio-bindings", version = "0.1", features = ["virtio-v5_0_0"]}
+virtio-bindings = { version = "0.1.0", features = ["virtio-v5_0_0"] }
 vm-allocator = { path = "../vm-allocator" }
 vm-device = { path = "../vm-device" }
-vmm-sys-util = ">=0.3.1"
+vmm-sys-util = "0.4.0"
 
 [dependencies.vhost_rs]
 path = "../vhost_rs"

vm-virtio/src/block.rs

@@ -8,36 +8,36 @@
 //
 // SPDX-License-Identifier: Apache-2.0 AND BSD-3-Clause
 
-use epoll;
-use libc::EFD_NONBLOCK;
-use std::cmp;
-use std::fs::File;
-use std::io::{self, Read, Seek, SeekFrom, Write};
-use std::os::linux::fs::MetadataExt;
-use std::os::unix::io::AsRawFd;
-use std::path::PathBuf;
-use std::result;
-use std::sync::atomic::{AtomicBool, Ordering};
-use std::sync::{Arc, RwLock};
-use std::thread;
-
 use super::Error as DeviceError;
 use super::{
     ActivateError, ActivateResult, DescriptorChain, DeviceEventT, Queue, VirtioDevice,
     VirtioDeviceType, VirtioInterruptType,
 };
 use crate::VirtioInterrupt;
+use arc_swap::ArcSwap;
+use epoll;
+use libc::{c_void, EFD_NONBLOCK};
+use std::alloc::{alloc_zeroed, dealloc, Layout};
+use std::cmp;
+use std::convert::TryInto;
+use std::fs::{File, Metadata};
+use std::io::{self, Read, Seek, SeekFrom, Write};
+use std::ops::DerefMut;
+use std::os::linux::fs::MetadataExt;
+use std::os::unix::io::{AsRawFd, RawFd};
+use std::path::PathBuf;
+use std::result;
+use std::slice;
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::{Arc, Mutex};
+use std::thread;
 use virtio_bindings::bindings::virtio_blk::*;
 use vm_device::{Migratable, MigratableError, Pausable, Snapshotable};
-use vm_memory::{Bytes, GuestAddress, GuestMemory, GuestMemoryError, GuestMemoryMmap};
-use vmm_sys_util::eventfd::EventFd;
+use vm_memory::{ByteValued, Bytes, GuestAddress, GuestMemory, GuestMemoryError, GuestMemoryMmap};
+use vmm_sys_util::{eventfd::EventFd, seek_hole::SeekHole, write_zeroes::PunchHole};
 
-const CONFIG_SPACE_SIZE: usize = 8;
 const SECTOR_SHIFT: u8 = 9;
 pub const SECTOR_SIZE: u64 = (0x01 as u64) << SECTOR_SHIFT;
-const QUEUE_SIZE: u16 = 256;
-const NUM_QUEUES: usize = 1;
-const QUEUE_SIZES: &[u16] = &[QUEUE_SIZE];
 
 // New descriptors are pending on the virtio queue.
 const QUEUE_AVAIL_EVENT: DeviceEventT = 0;
@@ -94,31 +94,251 @@ impl ExecuteError {
 pub trait DiskFile: Read + Seek + Write + Clone {}
 impl<D: Read + Seek + Write + Clone> DiskFile for D {}
 
+#[derive(Debug)]
 pub struct RawFile {
     file: File,
+    alignment: usize,
+    position: u64,
+}
+
+const BLK_ALIGNMENTS: [usize; 2] = [512, 4096];
+
+fn is_valid_alignment(fd: RawFd, alignment: usize) -> bool {
+    let layout = Layout::from_size_align(alignment, alignment).unwrap();
+    let ptr = unsafe { alloc_zeroed(layout) };
+
+    let ret = unsafe {
+        ::libc::pread(
+            fd,
+            ptr as *mut c_void,
+            alignment,
+            alignment.try_into().unwrap(),
+        )
+    };
+
+    unsafe { dealloc(ptr, layout) };
+
+    ret >= 0
 }
 
 impl RawFile {
-    pub fn new(file: File) -> Self {
-        RawFile { file }
+    pub fn new(file: File, direct_io: bool) -> Self {
+        // Assume no alignment restrictions if we aren't using O_DIRECT.
+        let mut alignment = 0;
+        if direct_io {
+            for align in &BLK_ALIGNMENTS {
+                if is_valid_alignment(file.as_raw_fd(), *align) {
+                    alignment = *align;
+                    break;
+                }
+            }
+        }
+        RawFile {
+            file,
+            alignment: alignment.try_into().unwrap(),
+            position: 0,
+        }
+    }
+
+    fn round_up(&self, offset: u64) -> u64 {
+        let align: u64 = self.alignment.try_into().unwrap();
+        ((offset / (align + 1)) + 1) * align
+    }
+
+    fn round_down(&self, offset: u64) -> u64 {
+        let align: u64 = self.alignment.try_into().unwrap();
+        (offset / align) * align
+    }
+
+    fn is_aligned(&self, buf: &[u8]) -> bool {
+        if self.alignment == 0 {
+            return true;
+        }
+
+        let align64: u64 = self.alignment.try_into().unwrap();
+
+        (self.position % align64 == 0)
+            && ((buf.as_ptr() as usize) % self.alignment == 0)
+            && (buf.len() % self.alignment == 0)
+    }
+
+    pub fn set_len(&self, size: u64) -> std::io::Result<()> {
+        self.file.set_len(size)
+    }
+
+    pub fn metadata(&self) -> std::io::Result<Metadata> {
+        self.file.metadata()
+    }
+
+    pub fn try_clone(&self) -> std::io::Result<RawFile> {
+        Ok(RawFile {
+            file: self.file.try_clone().expect("RawFile cloning failed"),
+            alignment: self.alignment,
+            position: self.position,
+        })
+    }
+
+    pub fn sync_all(&self) -> std::io::Result<()> {
+        self.file.sync_all()
+    }
+
+    pub fn sync_data(&self) -> std::io::Result<()> {
+        self.file.sync_data()
     }
 }
 
 impl Read for RawFile {
     fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
-        self.file.read(buf)
-    }
-}
+        if self.is_aligned(buf) {
+            match self.file.read(buf) {
+                Ok(r) => {
+                    self.position = self.position.checked_add(r.try_into().unwrap()).unwrap();
+                    Ok(r)
+                }
+                Err(e) => Err(e),
+            }
+        } else {
+            let rounded_pos: u64 = self.round_down(self.position);
+            let file_offset: usize = self
+                .position
+                .checked_sub(rounded_pos)
+                .unwrap()
+                .try_into()
+                .unwrap();
+            let buf_len: usize = buf.len();
+            let rounded_len: usize = self
+                .round_up(
+                    file_offset
+                        .checked_add(buf_len)
+                        .unwrap()
+                        .try_into()
+                        .unwrap(),
+                )
+                .try_into()
+                .unwrap();
 
-impl Seek for RawFile {
-    fn seek(&mut self, pos: SeekFrom) -> std::io::Result<u64> {
-        self.file.seek(pos)
+            let layout = Layout::from_size_align(rounded_len, self.alignment).unwrap();
+            let tmp_ptr = unsafe { alloc_zeroed(layout) };
+            let tmp_buf = unsafe { slice::from_raw_parts_mut(tmp_ptr, rounded_len) };
+
+            // This can eventually replaced with read_at once its interface
+            // has been stabilized.
+            let ret = unsafe {
+                ::libc::pread64(
+                    self.file.as_raw_fd(),
+                    tmp_buf.as_mut_ptr() as *mut c_void,
+                    tmp_buf.len(),
+                    rounded_pos.try_into().unwrap(),
+                )
+            };
+            if ret < 0 {
+                unsafe { dealloc(tmp_ptr, layout) };
+                return Err(io::Error::last_os_error());
+            }
+
+            let read: usize = ret.try_into().unwrap();
+            if read < file_offset {
+                unsafe { dealloc(tmp_ptr, layout) };
+                return Ok(0);
+            }
+
+            let mut to_copy = read - file_offset;
+            if to_copy > buf_len {
+                to_copy = buf_len;
+            }
+
+            buf.copy_from_slice(&tmp_buf[file_offset..(file_offset + buf_len)]);
+            unsafe { dealloc(tmp_ptr, layout) };
+
+            self.seek(SeekFrom::Current(to_copy.try_into().unwrap()))
+                .unwrap();
+            Ok(to_copy.try_into().unwrap())
+        }
     }
 }
 
 impl Write for RawFile {
     fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
-        self.file.write(buf)
+        if self.is_aligned(buf) {
+            match self.file.write(buf) {
+                Ok(r) => {
+                    self.position = self.position.checked_add(r.try_into().unwrap()).unwrap();
+                    Ok(r)
+                }
+                Err(e) => Err(e),
+            }
+        } else {
+            let rounded_pos: u64 = self.round_down(self.position);
+            let file_offset: usize = self
+                .position
+                .checked_sub(rounded_pos)
+                .unwrap()
+                .try_into()
+                .unwrap();
+            let buf_len: usize = buf.len();
+            let rounded_len: usize = self
+                .round_up(
+                    file_offset
+                        .checked_add(buf_len)
+                        .unwrap()
+                        .try_into()
+                        .unwrap(),
+                )
+                .try_into()
+                .unwrap();
+
+            let layout = Layout::from_size_align(rounded_len, self.alignment).unwrap();
+            let tmp_ptr = unsafe { alloc_zeroed(layout) };
+            let tmp_buf = unsafe { slice::from_raw_parts_mut(tmp_ptr, rounded_len) };
+
+            // This can eventually replaced with read_at once its interface
+            // has been stabilized.
+            let ret = unsafe {
+                ::libc::pread64(
+                    self.file.as_raw_fd(),
+                    tmp_buf.as_mut_ptr() as *mut c_void,
+                    tmp_buf.len(),
+                    rounded_pos.try_into().unwrap(),
+                )
+            };
+            if ret < 0 {
+                unsafe { dealloc(tmp_ptr, layout) };
+                return Err(io::Error::last_os_error());
+            };
+
+            tmp_buf[file_offset..(file_offset + buf_len)].copy_from_slice(buf);
+
+            // This can eventually replaced with write_at once its interface
+            // has been stabilized.
+            let ret = unsafe {
+                ::libc::pwrite64(
+                    self.file.as_raw_fd(),
+                    tmp_buf.as_ptr() as *const c_void,
+                    tmp_buf.len(),
+                    rounded_pos.try_into().unwrap(),
+                )
+            };
+
+            unsafe { dealloc(tmp_ptr, layout) };
+
+            if ret < 0 {
+                return Err(io::Error::last_os_error());
+            }
+
+            let written: usize = ret.try_into().unwrap();
+            if written < file_offset {
+                Ok(0)
+            } else {
+                let mut to_seek = written - file_offset;
+                if to_seek > buf_len {
+                    to_seek = buf_len;
+                }
+
+                self.seek(SeekFrom::Current(to_seek.try_into().unwrap()))
+                    .unwrap();
+                Ok(to_seek.try_into().unwrap())
+            }
+        }
     }
 
     fn flush(&mut self) -> std::io::Result<()> {
@@ -126,10 +346,56 @@ impl Write for RawFile {
     }
 }
 
+impl Seek for RawFile {
+    fn seek(&mut self, newpos: SeekFrom) -> std::io::Result<u64> {
+        match self.file.seek(newpos) {
+            Ok(pos) => {
+                self.position = pos;
+                Ok(pos)
+            }
+            Err(e) => Err(e),
+        }
+    }
+}
+
+impl PunchHole for RawFile {
+    fn punch_hole(&mut self, offset: u64, length: u64) -> std::io::Result<()> {
+        self.file.punch_hole(offset, length)
+    }
+}
+
+impl SeekHole for RawFile {
+    fn seek_hole(&mut self, offset: u64) -> std::io::Result<Option<u64>> {
+        match self.file.seek_hole(offset) {
+            Ok(pos) => {
+                if let Some(p) = pos {
+                    self.position = p;
+                }
+                Ok(pos)
+            }
+            Err(e) => Err(e),
+        }
+    }
+
+    fn seek_data(&mut self, offset: u64) -> std::io::Result<Option<u64>> {
+        match self.file.seek_data(offset) {
+            Ok(pos) => {
+                if let Some(p) = pos {
+                    self.position = p;
+                }
+                Ok(pos)
+            }
+            Err(e) => Err(e),
+        }
+    }
+}
+
 impl Clone for RawFile {
     fn clone(&self) -> Self {
         RawFile {
             file: self.file.try_clone().expect("RawFile cloning failed"),
+            alignment: self.alignment,
+            position: self.position,
         }
     }
 }
@@ -178,8 +444,7 @@ fn build_device_id(disk_path: &PathBuf) -> result::Result<String, Error> {
         blk_metadata.st_dev(),
         blk_metadata.st_rdev(),
         blk_metadata.st_ino()
-    )
-    .to_owned();
+    );
     Ok(device_id)
 }
 
@@ -324,29 +589,31 @@ impl Request {
 }
 
 struct BlockEpollHandler<T: DiskFile> {
-    queues: Vec<Queue>,
-    mem: Arc<RwLock<GuestMemoryMmap>>,
-    disk_image: T,
+    queue: Queue,
+    mem: Arc<ArcSwap<GuestMemoryMmap>>,
+    disk_image: Arc<Mutex<T>>,
     disk_nsectors: u64,
-    interrupt_cb: Arc<VirtioInterrupt>,
+    interrupt_cb: Arc<dyn VirtioInterrupt>,
     disk_image_id: Vec<u8>,
     kill_evt: EventFd,
     pause_evt: EventFd,
 }
 
 impl<T: DiskFile> BlockEpollHandler<T> {
-    fn process_queue(&mut self, queue_index: usize) -> bool {
-        let queue = &mut self.queues[queue_index];
+    fn process_queue(&mut self) -> bool {
+        let queue = &mut self.queue;
 
-        let mut used_desc_heads = [(0, 0); QUEUE_SIZE as usize];
+        let mut used_desc_heads = Vec::new();
         let mut used_count = 0;
-        let mem = self.mem.read().unwrap();
+        let mem = self.mem.load();
         for avail_desc in queue.iter(&mem) {
             let len;
             match Request::parse(&avail_desc, &mem) {
                 Ok(request) => {
+                    let mut disk_image_locked = self.disk_image.lock().unwrap();
+                    let mut disk_image = disk_image_locked.deref_mut();
                     let status = match request.execute(
-                        &mut self.disk_image,
+                        &mut disk_image,
                         self.disk_nsectors,
                         &mem,
                         &self.disk_image_id,
@@ -370,38 +637,37 @@ impl<T: DiskFile> BlockEpollHandler<T> {
                     len = 0;
                 }
             }
-            used_desc_heads[used_count] = (avail_desc.index, len);
+            used_desc_heads.push((avail_desc.index, len));
             used_count += 1;
         }
 
-        for &(desc_index, len) in &used_desc_heads[..used_count] {
+        for &(desc_index, len) in used_desc_heads.iter() {
             queue.add_used(&mem, desc_index, len);
         }
         used_count > 0
     }
 
-    fn signal_used_queue(&self, queue_index: usize) -> result::Result<(), DeviceError> {
-        (self.interrupt_cb)(&VirtioInterruptType::Queue, Some(&self.queues[queue_index])).map_err(
-            |e| {
+    fn signal_used_queue(&self) -> result::Result<(), DeviceError> {
+        self.interrupt_cb
+            .trigger(&VirtioInterruptType::Queue, Some(&self.queue))
+            .map_err(|e| {
                 error!("Failed to signal used queue: {:?}", e);
                 DeviceError::FailedSignalingUsedQueue(e)
-            },
-        )
+            })
     }
 
     #[allow(dead_code)]
     fn update_disk_image(
         &mut self,
-        disk_image: T,
+        mut disk_image: T,
         disk_path: &PathBuf,
     ) -> result::Result<(), DeviceError> {
-        self.disk_image = disk_image;
-        self.disk_nsectors = self
-            .disk_image
+        self.disk_nsectors = disk_image
             .seek(SeekFrom::End(0))
             .map_err(DeviceError::IoError)?
             / SECTOR_SIZE;
         self.disk_image_id = build_disk_image_id(disk_path);
+        self.disk_image = Arc::new(Mutex::new(disk_image));
         Ok(())
     }
 
@@ -465,8 +731,8 @@ impl<T: DiskFile> BlockEpollHandler<T> {
                         if let Err(e) = queue_evt.read() {
                             error!("Failed to get queue event: {:?}", e);
                             break 'epoll;
-                        } else if self.process_queue(0) {
-                            if let Err(e) = self.signal_used_queue(0) {
+                        } else if self.process_queue() {
+                            if let Err(e) = self.signal_used_queue() {
                                 error!("Failed to signal used queue: {:?}", e);
                                 break 'epoll;
                             }
@@ -496,32 +762,57 @@ impl<T: DiskFile> BlockEpollHandler<T> {
     }
 }
 
+#[derive(Copy, Clone, Debug, Default)]
+#[repr(C, packed)]
+pub struct VirtioBlockGeometry {
+    pub cylinders: u16,
+    pub heads: u8,
+    pub sectors: u8,
+}
+
+unsafe impl ByteValued for VirtioBlockGeometry {}
+
+#[derive(Copy, Clone, Debug, Default)]
+#[repr(C, packed)]
+pub struct VirtioBlockConfig {
+    pub capacity: u64,
+    pub size_max: u32,
+    pub seg_max: u32,
+    pub geometry: VirtioBlockGeometry,
+    pub blk_size: u32,
+    pub physical_block_exp: u8,
+    pub alignment_offset: u8,
+    pub min_io_size: u16,
+    pub opt_io_size: u32,
+    pub wce: u8,
+    unused: u8,
+    pub num_queues: u16,
+    pub max_discard_sectors: u32,
+    pub max_discard_seg: u32,
+    pub discard_sector_alignment: u32,
+    pub max_write_zeroes_sectors: u32,
+    pub max_write_zeroes_seg: u32,
+    pub write_zeroes_may_unmap: u8,
+    unused1: [u8; 3],
+}
+
+unsafe impl ByteValued for VirtioBlockConfig {}
+
 /// Virtio device for exposing block level read/write operations on a host file.
 pub struct Block<T: DiskFile> {
     kill_evt: Option<EventFd>,
-    disk_image: Option<T>,
+    disk_image: Arc<Mutex<T>>,
     disk_path: PathBuf,
     disk_nsectors: u64,
     avail_features: u64,
     acked_features: u64,
-    config_space: Vec<u8>,
-    queue_evt: Option<EventFd>,
-    interrupt_cb: Option<Arc<VirtioInterrupt>>,
-    epoll_thread: Option<thread::JoinHandle<result::Result<(), DeviceError>>>,
+    config: VirtioBlockConfig,
+    queue_evts: Option<Vec<EventFd>>,
+    interrupt_cb: Option<Arc<dyn VirtioInterrupt>>,
+    epoll_threads: Option<Vec<thread::JoinHandle<result::Result<(), DeviceError>>>>,
     pause_evt: Option<EventFd>,
     paused: Arc<AtomicBool>,
-}
-
-pub fn build_config_space(disk_size: u64) -> Vec<u8> {
-    // We only support disk size, which uses the first two words of the configuration space.
-    // If the image is not a multiple of the sector size, the tail bits are not exposed.
-    // The config space is little endian.
-    let mut config = Vec::with_capacity(CONFIG_SPACE_SIZE);
-    let num_sectors = disk_size >> SECTOR_SHIFT;
-    for i in 0..8 {
-        config.push((num_sectors >> (8 * i)) as u8);
-    }
-    config
+    queue_size: Vec<u16>,
 }
 
 impl<T: DiskFile> Block<T> {
@@ -533,6 +824,8 @@ impl<T: DiskFile> Block<T> {
         disk_path: PathBuf,
         is_disk_read_only: bool,
         iommu: bool,
+        num_queues: usize,
+        queue_size: u16,
     ) -> io::Result<Block<T>> {
         let disk_size = disk_image.seek(SeekFrom::End(0))? as u64;
         if disk_size % SECTOR_SIZE != 0 {
@@ -551,21 +844,33 @@ impl<T: DiskFile> Block<T> {
 
         if is_disk_read_only {
             avail_features |= 1u64 << VIRTIO_BLK_F_RO;
+        }
+
+        let disk_nsectors = disk_size / SECTOR_SIZE;
+        let mut config = VirtioBlockConfig {
+            capacity: disk_nsectors,
+            ..Default::default()
         };
 
+        if num_queues > 1 {
+            avail_features |= 1u64 << VIRTIO_BLK_F_MQ;
+            config.num_queues = num_queues as u16;
+        }
+
         Ok(Block {
             kill_evt: None,
-            disk_image: Some(disk_image),
+            disk_image: Arc::new(Mutex::new(disk_image)),
             disk_path,
-            disk_nsectors: disk_size / SECTOR_SIZE,
+            disk_nsectors,
             avail_features,
             acked_features: 0u64,
-            config_space: build_config_space(disk_size),
-            queue_evt: None,
+            config,
+            queue_evts: None,
             interrupt_cb: None,
-            epoll_thread: None,
+            epoll_threads: None,
             pause_evt: None,
             paused: Arc::new(AtomicBool::new(false)),
+            queue_size: vec![queue_size; num_queues],
         })
     }
 }
@@ -585,32 +890,15 @@ impl<T: 'static + DiskFile + Send> VirtioDevice for Block<T> {
     }
 
     fn queue_max_sizes(&self) -> &[u16] {
-        QUEUE_SIZES
+        self.queue_size.as_slice()
     }
 
-    fn features(&self, page: u32) -> u32 {
-        match page {
-            // Get the lower 32-bits of the features bitfield.
-            0 => self.avail_features as u32,
-            // Get the upper 32-bits of the features bitfield.
-            1 => (self.avail_features >> 32) as u32,
-            _ => {
-                warn!("Received request for unknown features page.");
-                0u32
-            }
-        }
+    fn features(&self) -> u64 {
+        self.avail_features
     }
 
-    fn ack_features(&mut self, page: u32, value: u32) {
-        let mut v = match page {
-            0 => u64::from(value),
-            1 => u64::from(value) << 32,
-            _ => {
-                warn!("Cannot acknowledge unknown features page.");
-                0u64
-            }
-        };
-
+    fn ack_features(&mut self, value: u64) {
+        let mut v = value;
         // Check if the guest is ACK'ing a feature that we didn't claim to have.
         let unrequested_features = v & !self.avail_features;
         if unrequested_features != 0 {
@@ -623,40 +911,42 @@ impl<T: 'static + DiskFile + Send> VirtioDevice for Block<T> {
     }
 
     fn read_config(&self, offset: u64, mut data: &mut [u8]) {
-        let config_len = self.config_space.len() as u64;
+        let config_slice = self.config.as_slice();
+        let config_len = config_slice.len() as u64;
         if offset >= config_len {
             error!("Failed to read config space");
             return;
         }
         if let Some(end) = offset.checked_add(data.len() as u64) {
             // This write can't fail, offset and end are checked against config_len.
-            data.write_all(&self.config_space[offset as usize..cmp::min(end, config_len) as usize])
+            data.write_all(&config_slice[offset as usize..cmp::min(end, config_len) as usize])
                 .unwrap();
         }
     }
 
     fn write_config(&mut self, offset: u64, data: &[u8]) {
+        let config_slice = self.config.as_mut_slice();
         let data_len = data.len() as u64;
-        let config_len = self.config_space.len() as u64;
+        let config_len = config_slice.len() as u64;
         if offset + data_len > config_len {
             error!("Failed to write config space");
             return;
         }
-        let (_, right) = self.config_space.split_at_mut(offset as usize);
+        let (_, right) = config_slice.split_at_mut(offset as usize);
         right.copy_from_slice(&data[..]);
     }
 
     fn activate(
         &mut self,
-        mem: Arc<RwLock<GuestMemoryMmap>>,
-        interrupt_cb: Arc<VirtioInterrupt>,
-        queues: Vec<Queue>,
+        mem: Arc<ArcSwap<GuestMemoryMmap>>,
+        interrupt_cb: Arc<dyn VirtioInterrupt>,
+        mut queues: Vec<Queue>,
         mut queue_evts: Vec<EventFd>,
     ) -> ActivateResult {
-        if queues.len() != NUM_QUEUES || queue_evts.len() != NUM_QUEUES {
+        if queues.len() != self.queue_size.len() || queue_evts.len() != self.queue_size.len() {
             error!(
                 "Cannot perform activate. Expected {} queue(s), got {}",
-                NUM_QUEUES,
+                self.queue_size.len(),
                 queues.len()
             );
             return Err(ActivateError::BadActivate);
@@ -679,49 +969,65 @@ impl<T: 'static + DiskFile + Send> VirtioDevice for Block<T> {
             })?;
         self.pause_evt = Some(self_pause_evt);
 
-        if let Some(disk_image) = self.disk_image.clone() {
-            let disk_image_id = build_disk_image_id(&self.disk_path);
-
-            // Save the interrupt EventFD as we need to return it on reset
-            // but clone it to pass into the thread.
-            self.interrupt_cb = Some(interrupt_cb);
-            let interrupt_cb = self.interrupt_cb.as_ref().unwrap().clone();
+        let disk_image_id = build_disk_image_id(&self.disk_path);
 
+        let mut tmp_queue_evts: Vec<EventFd> = Vec::new();
+        for queue_evt in queue_evts.iter() {
             // Save the queue EventFD as we need to return it on reset
             // but clone it to pass into the thread.
-            self.queue_evt = Some(queue_evts.remove(0));
-            let queue_evt = self.queue_evt.as_ref().unwrap().try_clone().map_err(|e| {
+            tmp_queue_evts.push(queue_evt.try_clone().map_err(|e| {
                 error!("failed to clone queue EventFd: {}", e);
                 ActivateError::BadActivate
-            })?;
+            })?);
+        }
+        self.queue_evts = Some(tmp_queue_evts);
 
+        let mut tmp_queue_evts: Vec<EventFd> = Vec::new();
+        for queue_evt in queue_evts.iter() {
+            // Save the queue EventFD as we need to return it on reset
+            // but clone it to pass into the thread.
+            tmp_queue_evts.push(queue_evt.try_clone().map_err(|e| {
+                error!("failed to clone queue EventFd: {}", e);
+                ActivateError::BadActivate
+            })?);
+        }
+        self.queue_evts = Some(tmp_queue_evts);
+
+        let mut epoll_threads = Vec::new();
+        for _ in 0..self.queue_size.len() {
             let mut handler = BlockEpollHandler {
-                queues,
-                mem,
-                disk_image,
+                queue: queues.remove(0),
+                mem: mem.clone(),
+                disk_image: self.disk_image.clone(),
                 disk_nsectors: self.disk_nsectors,
-                interrupt_cb,
-                disk_image_id,
-                kill_evt,
-                pause_evt,
+                interrupt_cb: interrupt_cb.clone(),
+                disk_image_id: disk_image_id.clone(),
+                kill_evt: kill_evt.try_clone().unwrap(),
+                pause_evt: pause_evt.try_clone().unwrap(),
             };
 
+            let queue_evt = queue_evts.remove(0);
             let paused = self.paused.clone();
             thread::Builder::new()
                 .name("virtio_blk".to_string())
                 .spawn(move || handler.run(queue_evt, paused))
-                .map(|thread| self.epoll_thread = Some(thread))
+                .map(|thread| epoll_threads.push(thread))
                 .map_err(|e| {
                     error!("failed to clone the virtio-blk epoll thread: {}", e);
                     ActivateError::BadActivate
                 })?;
-
-            return Ok(());
         }
-        Err(ActivateError::BadActivate)
+
+        // Save the interrupt EventFD as we need to return it on reset
+        // but clone it to pass into the thread.
+        self.interrupt_cb = Some(interrupt_cb);
+
+        self.epoll_threads = Some(epoll_threads);
+
+        Ok(())
     }
 
-    fn reset(&mut self) -> Option<(Arc<VirtioInterrupt>, Vec<EventFd>)> {
+    fn reset(&mut self) -> Option<(Arc<dyn VirtioInterrupt>, Vec<EventFd>)> {
         // We first must resume the virtio thread if it was paused.
         if self.pause_evt.take().is_some() {
             self.resume().ok()?;
@@ -735,14 +1041,11 @@ impl<T: 'static + DiskFile + Send> VirtioDevice for Block<T> {
         // Return the interrupt and queue EventFDs
         Some((
             self.interrupt_cb.take().unwrap(),
-            vec![self.queue_evt.take().unwrap()],
+            self.queue_evts.take().unwrap(),
         ))
     }
 }
 
-impl<T: 'static + DiskFile + Send> Pausable for Block<T> {
-    virtio_pausable_inner!();
-}
-
+virtio_pausable!(Block, T: 'static + DiskFile + Send);
 impl<T: 'static + DiskFile + Send> Snapshotable for Block<T> {}
 impl<T: 'static + DiskFile + Send> Migratable for Block<T> {}

vm-virtio/src/console.rs

@@ -1,6 +1,13 @@
 // Copyright 2019 Intel Corporation. All Rights Reserved.
 // SPDX-License-Identifier: Apache-2.0
 
+use super::Error as DeviceError;
+use super::{
+    ActivateError, ActivateResult, DeviceEventT, Queue, VirtioDevice, VirtioDeviceType,
+    VirtioInterruptType, VIRTIO_F_IOMMU_PLATFORM, VIRTIO_F_VERSION_1,
+};
+use crate::VirtioInterrupt;
+use arc_swap::ArcSwap;
 use epoll;
 use libc::EFD_NONBLOCK;
 use std;
@@ -11,16 +18,9 @@ use std::io::Write;
 use std::ops::DerefMut;
 use std::os::unix::io::AsRawFd;
 use std::result;
-use std::sync::{Arc, Mutex, RwLock};
-use std::thread;
-
-use super::Error as DeviceError;
-use super::{
-    ActivateError, ActivateResult, DeviceEventT, Queue, VirtioDevice, VirtioDeviceType,
-    VirtioInterruptType, VIRTIO_F_IOMMU_PLATFORM, VIRTIO_F_VERSION_1,
-};
-use crate::VirtioInterrupt;
 use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
+use std::sync::{Arc, Mutex};
+use std::thread;
 use vm_device::{Migratable, MigratableError, Pausable, Snapshotable};
 use vm_memory::{ByteValued, Bytes, GuestMemoryMmap};
 use vmm_sys_util::eventfd::EventFd;
@@ -58,8 +58,8 @@ unsafe impl ByteValued for VirtioConsoleConfig {}
 
 struct ConsoleEpollHandler {
     queues: Vec<Queue>,
-    mem: Arc<RwLock<GuestMemoryMmap>>,
-    interrupt_cb: Arc<VirtioInterrupt>,
+    mem: Arc<ArcSwap<GuestMemoryMmap>>,
+    interrupt_cb: Arc<dyn VirtioInterrupt>,
     in_buffer: Arc<Mutex<VecDeque<u8>>>,
     out: Arc<Mutex<Box<dyn io::Write + Send + Sync + 'static>>>,
     input_queue_evt: EventFd,
@@ -79,37 +79,28 @@ impl ConsoleEpollHandler {
      */
     fn process_input_queue(&mut self) -> bool {
         let mut in_buffer = self.in_buffer.lock().unwrap();
-        let count = in_buffer.len();
         let recv_queue = &mut self.queues[0]; //receiveq
         let mut used_desc_heads = [(0, 0); QUEUE_SIZE as usize];
         let mut used_count = 0;
-        let mut write_count = 0;
 
-        let mem = self.mem.read().unwrap();
+        if in_buffer.is_empty() {
+            return false;
+        }
+
+        let mem = self.mem.load();
         for avail_desc in recv_queue.iter(&mem) {
-            let len;
-
-            let limit = cmp::min(write_count + avail_desc.len as u32, count as u32);
-            let source_slice = in_buffer
-                .drain(write_count as usize..limit as usize)
-                .collect::<Vec<u8>>();
-            let write_result = mem.write_slice(&source_slice[..], avail_desc.addr);
-
-            match write_result {
-                Ok(_) => {
-                    len = limit - write_count; //avail_desc.len;
-                    write_count = limit;
-                }
-                Err(e) => {
-                    error!("Failed to write slice: {:?}", e);
-                    break;
-                }
+            let len = cmp::min(avail_desc.len as u32, in_buffer.len() as u32);
+            let source_slice = in_buffer.drain(..len as usize).collect::<Vec<u8>>();
+            if let Err(e) = mem.write_slice(&source_slice[..], avail_desc.addr) {
+                error!("Failed to write slice: {:?}", e);
+                recv_queue.go_to_previous_position();
+                break;
             }
 
             used_desc_heads[used_count] = (avail_desc.index, len);
             used_count += 1;
 
-            if write_count >= count as u32 {
+            if in_buffer.is_empty() {
                 break;
             }
         }
@@ -117,6 +108,7 @@ impl ConsoleEpollHandler {
         for &(desc_index, len) in &used_desc_heads[..used_count] {
             recv_queue.add_used(&mem, desc_index, len);
         }
+
         used_count > 0
     }
 
@@ -132,7 +124,7 @@ impl ConsoleEpollHandler {
         let mut used_desc_heads = [(0, 0); QUEUE_SIZE as usize];
         let mut used_count = 0;
 
-        let mem = self.mem.read().unwrap();
+        let mem = self.mem.load();
         for avail_desc in trans_queue.iter(&mem) {
             let len;
             let mut out = self.out.lock().unwrap();
@@ -155,10 +147,12 @@ impl ConsoleEpollHandler {
     }
 
     fn signal_used_queue(&self) -> result::Result<(), DeviceError> {
-        (self.interrupt_cb)(&VirtioInterruptType::Queue, Some(&self.queues[0])).map_err(|e| {
-            error!("Failed to signal used queue: {:?}", e);
-            DeviceError::FailedSignalingUsedQueue(e)
-        })
+        self.interrupt_cb
+            .trigger(&VirtioInterruptType::Queue, Some(&self.queues[0]))
+            .map_err(|e| {
+                error!("Failed to signal used queue: {:?}", e);
+                DeviceError::FailedSignalingUsedQueue(e)
+            })
     }
 
     fn run(&mut self, paused: Arc<AtomicBool>) -> result::Result<(), DeviceError> {
@@ -239,6 +233,11 @@ impl ConsoleEpollHandler {
                         if let Err(e) = self.input_queue_evt.read() {
                             error!("Failed to get queue event: {:?}", e);
                             break 'epoll;
+                        } else if self.process_input_queue() {
+                            if let Err(e) = self.signal_used_queue() {
+                                error!("Failed to signal used queue: {:?}", e);
+                                break 'epoll;
+                            }
                         }
                     }
                     OUTPUT_QUEUE_EVENT => {
@@ -264,8 +263,9 @@ impl ConsoleEpollHandler {
                         if let Err(e) = self.config_evt.read() {
                             error!("Failed to get config event: {:?}", e);
                             break 'epoll;
-                        } else if let Err(e) =
-                            (self.interrupt_cb)(&VirtioInterruptType::Config, None)
+                        } else if let Err(e) = self
+                            .interrupt_cb
+                            .trigger(&VirtioInterruptType::Config, None)
                         {
                             error!("Failed to signal console driver: {:?}", e);
                         }
@@ -350,8 +350,8 @@ pub struct Console {
     input: Arc<ConsoleInput>,
     out: Arc<Mutex<Box<dyn io::Write + Send + Sync + 'static>>>,
     queue_evts: Option<Vec<EventFd>>,
-    interrupt_cb: Option<Arc<VirtioInterrupt>>,
-    epoll_thread: Option<thread::JoinHandle<result::Result<(), DeviceError>>>,
+    interrupt_cb: Option<Arc<dyn VirtioInterrupt>>,
+    epoll_threads: Option<Vec<thread::JoinHandle<result::Result<(), DeviceError>>>>,
     paused: Arc<AtomicBool>,
 }
 
@@ -391,7 +391,7 @@ impl Console {
                 out: Arc::new(Mutex::new(out)),
                 queue_evts: None,
                 interrupt_cb: None,
-                epoll_thread: None,
+                epoll_threads: None,
                 paused: Arc::new(AtomicBool::new(false)),
             },
             console_input,
@@ -417,29 +417,12 @@ impl VirtioDevice for Console {
         QUEUE_SIZES
     }
 
-    fn features(&self, page: u32) -> u32 {
-        match page {
-            // Get the lower 32-bits of the features bitfield.
-            0 => self.avail_features as u32,
-            // Get the upper 32-bits of the features bitfield.
-            1 => (self.avail_features >> 32) as u32,
-            _ => {
-                warn!("Received request for unknown features page.");
-                0u32
-            }
-        }
+    fn features(&self) -> u64 {
+        self.avail_features
     }
 
-    fn ack_features(&mut self, page: u32, value: u32) {
-        let mut v = match page {
-            0 => u64::from(value),
-            1 => u64::from(value) << 32,
-            _ => {
-                warn!("Cannot acknowledge unknown features page.");
-                0u64
-            }
-        };
-
+    fn ack_features(&mut self, value: u64) {
+        let mut v = value;
         // Check if the guest is ACK'ing a feature that we didn't claim to have.
         let unrequested_features = v & !self.avail_features;
         if unrequested_features != 0 {
@@ -473,8 +456,8 @@ impl VirtioDevice for Console {
 
     fn activate(
         &mut self,
-        mem: Arc<RwLock<GuestMemoryMmap>>,
-        interrupt_cb: Arc<VirtioInterrupt>,
+        mem: Arc<ArcSwap<GuestMemoryMmap>>,
+        interrupt_cb: Arc<dyn VirtioInterrupt>,
         queues: Vec<Queue>,
         mut queue_evts: Vec<EventFd>,
     ) -> ActivateResult {
@@ -524,7 +507,7 @@ impl VirtioDevice for Console {
             .store(self.acked_features, Ordering::Relaxed);
 
         if (self.acked_features & (1u64 << VIRTIO_CONSOLE_F_SIZE)) != 0 {
-            if let Err(e) = (interrupt_cb)(&VirtioInterruptType::Config, None) {
+            if let Err(e) = interrupt_cb.trigger(&VirtioInterruptType::Config, None) {
                 error!("Failed to signal console driver: {:?}", e);
             }
         }
@@ -544,19 +527,22 @@ impl VirtioDevice for Console {
         };
 
         let paused = self.paused.clone();
+        let mut epoll_threads = Vec::new();
         thread::Builder::new()
             .name("virtio_console".to_string())
             .spawn(move || handler.run(paused))
-            .map(|thread| self.epoll_thread = Some(thread))
+            .map(|thread| epoll_threads.push(thread))
             .map_err(|e| {
                 error!("failed to clone the virtio-console epoll thread: {}", e);
                 ActivateError::BadActivate
             })?;
 
+        self.epoll_threads = Some(epoll_threads);
+
         Ok(())
     }
 
-    fn reset(&mut self) -> Option<(Arc<VirtioInterrupt>, Vec<EventFd>)> {
+    fn reset(&mut self) -> Option<(Arc<dyn VirtioInterrupt>, Vec<EventFd>)> {
         // We first must resume the virtio thread if it was paused.
         if self.pause_evt.take().is_some() {
             self.resume().ok()?;

vm-virtio/src/device.rs

@@ -7,7 +7,8 @@
 // SPDX-License-Identifier: Apache-2.0 AND BSD-3-Clause
 
 use super::*;
-use std::sync::{Arc, RwLock};
+use arc_swap::ArcSwap;
+use std::sync::Arc;
 use vm_memory::{GuestAddress, GuestMemoryMmap, GuestUsize};
 use vmm_sys_util::eventfd::EventFd;
 
@@ -16,11 +17,20 @@ pub enum VirtioInterruptType {
     Queue,
 }
 
-pub type VirtioInterrupt = Box<
-    dyn Fn(&VirtioInterruptType, Option<&Queue>) -> std::result::Result<(), std::io::Error>
-        + Send
-        + Sync,
->;
+pub trait VirtioInterrupt: Send + Sync {
+    fn trigger(
+        &self,
+        int_type: &VirtioInterruptType,
+        queue: Option<&Queue>,
+    ) -> std::result::Result<(), std::io::Error>;
+    fn notifier(
+        &self,
+        _int_type: &VirtioInterruptType,
+        _queue: Option<&Queue>,
+    ) -> Option<&EventFd> {
+        None
+    }
+}
 
 pub type VirtioIommuRemapping =
     Box<dyn Fn(u64) -> std::result::Result<u64, std::io::Error> + Send + Sync>;
@@ -52,14 +62,15 @@ pub trait VirtioDevice: Send {
     /// The maximum size of each queue that this device supports.
     fn queue_max_sizes(&self) -> &[u16];
 
-    /// The set of feature bits shifted by `page * 32`.
-    fn features(&self, page: u32) -> u32 {
-        let _ = page;
+    /// The set of feature bits that this device supports.
+    fn features(&self) -> u64 {
         0
     }
 
     /// Acknowledges that this set of features should be enabled.
-    fn ack_features(&mut self, page: u32, value: u32);
+    fn ack_features(&mut self, value: u64) {
+        let _ = value;
+    }
 
     /// Reads this device configuration space at `offset`.
     fn read_config(&self, offset: u64, data: &mut [u8]);
@@ -70,15 +81,15 @@ pub trait VirtioDevice: Send {
     /// Activates this device for real usage.
     fn activate(
         &mut self,
-        mem: Arc<RwLock<GuestMemoryMmap>>,
-        interrupt_evt: Arc<VirtioInterrupt>,
+        mem: Arc<ArcSwap<GuestMemoryMmap>>,
+        interrupt_evt: Arc<dyn VirtioInterrupt>,
         queues: Vec<Queue>,
         queue_evts: Vec<EventFd>,
     ) -> ActivateResult;
 
     /// Optionally deactivates this device and returns ownership of the guest memory map, interrupt
     /// event, and queue events.
-    fn reset(&mut self) -> Option<(Arc<VirtioInterrupt>, Vec<EventFd>)> {
+    fn reset(&mut self) -> Option<(Arc<dyn VirtioInterrupt>, Vec<EventFd>)> {
         None
     }
 
@@ -103,9 +114,19 @@ pub trait DmaRemapping: Send + Sync {
 }
 
 #[macro_export]
-macro_rules! virtio_pausable_inner {
+macro_rules! virtio_pausable_trait_definition {
     () => {
-        fn pause(&mut self) -> result::Result<(), MigratableError> {
+        trait VirtioPausable {
+            fn virtio_pause(&mut self) -> std::result::Result<(), MigratableError>;
+            fn virtio_resume(&mut self) -> std::result::Result<(), MigratableError>;
+        }
+    };
+}
+#[macro_export]
+macro_rules! virtio_pausable_trait_inner {
+    () => {
+        // This is the common Pausable trait implementation for virtio.
+        fn virtio_pause(&mut self) -> result::Result<(), MigratableError> {
             debug!(
                 "Pausing virtio-{}",
                 VirtioDeviceType::from(self.device_type())
@@ -120,26 +141,94 @@ macro_rules! virtio_pausable_inner {
             Ok(())
         }
 
-        fn resume(&mut self) -> result::Result<(), MigratableError> {
+        fn virtio_resume(&mut self) -> result::Result<(), MigratableError> {
             debug!(
                 "Resuming virtio-{}",
                 VirtioDeviceType::from(self.device_type())
             );
             self.paused.store(false, Ordering::SeqCst);
-            if let Some(epoll_thread) = &self.epoll_thread {
-                epoll_thread.thread().unpark();
+            if let Some(epoll_threads) = &self.epoll_threads {
+                for i in 0..epoll_threads.len() {
+                    epoll_threads[i].thread().unpark();
+                }
             }
 
             Ok(())
         }
-    }
+    };
+}
+
+#[macro_export]
+macro_rules! virtio_pausable_trait {
+    ($type:ident) => {
+        virtio_pausable_trait_definition!();
+
+        impl VirtioPausable for $type {
+            virtio_pausable_trait_inner!();
+        }
+    };
+
+    ($type:ident, T: $($bounds:tt)+) => {
+        virtio_pausable_trait_definition!();
+
+        impl<T: $($bounds)+ > VirtioPausable for $type<T> {
+            virtio_pausable_trait_inner!();
+        }
+    };
+}
+
+#[macro_export]
+macro_rules! virtio_pausable_inner {
+    ($type:ident) => {
+        fn pause(&mut self) -> result::Result<(), MigratableError> {
+            self.virtio_pause()
+        }
+
+        fn resume(&mut self) -> result::Result<(), MigratableError> {
+            self.virtio_resume()
+        }
+    };
 }
 
 #[macro_export]
 macro_rules! virtio_pausable {
-    ($name:ident) => {
-        impl Pausable for $name {
-            virtio_pausable_inner!();
+    ($type:ident) => {
+        virtio_pausable_trait!($type);
+
+        impl Pausable for $type {
+            virtio_pausable_inner!($type);
+        }
+    };
+
+    // For type bound virtio types
+    ($type:ident, T: $($bounds:tt)+) => {
+        virtio_pausable_trait!($type, T: $($bounds)+);
+
+        impl<T: $($bounds)+ > Pausable for $type<T> {
+            virtio_pausable_inner!($type);
+        }
+    };
+}
+
+#[macro_export]
+macro_rules! virtio_ctrl_q_pausable {
+    ($type:ident) => {
+        virtio_pausable_trait!($type);
+
+        impl Pausable for $type {
+            fn pause(&mut self) -> result::Result<(), MigratableError> {
+                self.virtio_pause()
+            }
+
+            fn resume(&mut self) -> result::Result<(), MigratableError> {
+                self.virtio_resume()?;
+
+                if let Some(ctrl_queue_epoll_thread) = &self.ctrl_queue_epoll_thread {
+                    ctrl_queue_epoll_thread.thread().unpark();
+                }
+
+                Ok(())
+            }
         }
     };
 }

vm-virtio/src/iommu.rs

@@ -2,6 +2,13 @@
 //
 // SPDX-License-Identifier: Apache-2.0 AND BSD-3-Clause
 
+use super::Error as DeviceError;
+use super::{
+    ActivateError, ActivateResult, DescriptorChain, DeviceEventT, Queue, VirtioDevice,
+    VirtioDeviceType, VIRTIO_F_VERSION_1,
+};
+use crate::{DmaRemapping, VirtioInterrupt, VirtioInterruptType};
+use arc_swap::ArcSwap;
 use epoll;
 use libc::EFD_NONBLOCK;
 use std::cmp;
@@ -15,13 +22,6 @@ use std::result;
 use std::sync::atomic::{AtomicBool, Ordering};
 use std::sync::{Arc, RwLock};
 use std::thread;
-
-use super::Error as DeviceError;
-use super::{
-    ActivateError, ActivateResult, DescriptorChain, DeviceEventT, Queue, VirtioDevice,
-    VirtioDeviceType, VIRTIO_F_VERSION_1,
-};
-use crate::{DmaRemapping, VirtioInterrupt, VirtioInterruptType};
 use vm_device::{ExternalDmaMapping, Migratable, MigratableError, Pausable, Snapshotable};
 use vm_memory::{Address, ByteValued, Bytes, GuestAddress, GuestMemoryError, GuestMemoryMmap};
 use vmm_sys_util::eventfd::EventFd;
@@ -44,6 +44,17 @@ const KILL_EVENT: DeviceEventT = 2;
 /// The device should be paused.
 const PAUSE_EVENT: DeviceEventT = 3;
 
+/// PROBE properties size.
+/// This is the minimal size to provide at least one RESV_MEM property.
+/// Because virtio-iommu expects one MSI reserved region, we must provide it,
+/// otherwise the driver in the guest will define a predefined one between
+/// 0x8000000 and 0x80FFFFF, which is only relevant for ARM architecture, but
+/// will conflict with x86.
+const PROBE_PROP_SIZE: u32 =
+    (size_of::<VirtioIommuProbeProperty>() + size_of::<VirtioIommuProbeResvMem>()) as u32;
+const MSI_IOVA_START: u64 = 0xfee0_0000;
+const MSI_IOVA_END: u64 = 0xfeef_ffff;
+
 /// Virtio IOMMU features
 #[allow(unused)]
 const VIRTIO_IOMMU_F_INPUT_RANGE: u32 = 0;
@@ -53,35 +64,83 @@ const VIRTIO_IOMMU_F_DOMAIN_BITS: u32 = 1;
 const VIRTIO_IOMMU_F_MAP_UNMAP: u32 = 2;
 #[allow(unused)]
 const VIRTIO_IOMMU_F_BYPASS: u32 = 3;
-#[allow(unused)]
 const VIRTIO_IOMMU_F_PROBE: u32 = 4;
+#[allow(unused)]
+const VIRTIO_IOMMU_F_MMIO: u32 = 5;
+#[allow(unused)]
+const VIRTIO_IOMMU_F_TOPOLOGY: u32 = 6;
 
 // Support 2MiB and 4KiB page sizes.
-#[allow(unused)]
 const VIRTIO_IOMMU_PAGE_SIZE_MASK: u64 = (2 << 20) | (4 << 10);
 
-#[allow(unused)]
 #[derive(Copy, Clone, Debug, Default)]
 #[repr(packed)]
-struct VirtioIommuRange {
+struct VirtioIommuRange32 {
+    start: u32,
+    end: u32,
+}
+
+unsafe impl ByteValued for VirtioIommuRange32 {}
+
+#[derive(Copy, Clone, Debug, Default)]
+#[repr(packed)]
+struct VirtioIommuRange64 {
     start: u64,
     end: u64,
 }
 
-unsafe impl ByteValued for VirtioIommuRange {}
+unsafe impl ByteValued for VirtioIommuRange64 {}
+
+#[derive(Copy, Clone, Debug, Default)]
+#[repr(packed)]
+struct VirtioIommuTopoConfig {
+    offset: u32,
+    num_items: u32,
+    item_length: u32,
+}
+
+unsafe impl ByteValued for VirtioIommuTopoConfig {}
 
 #[derive(Copy, Clone, Debug, Default)]
 #[repr(packed)]
 struct VirtioIommuConfig {
     page_size_mask: u64,
-    input_range: VirtioIommuRange,
-    domain_bits: u8,
-    padding: [u8; 3],
+    input_range: VirtioIommuRange64,
+    domain_range: VirtioIommuRange32,
     probe_size: u32,
+    topo_config: VirtioIommuTopoConfig,
 }
 
 unsafe impl ByteValued for VirtioIommuConfig {}
 
+#[allow(unused)]
+const VIRTIO_IOMMU_TOPO_PCI_RANGE: u16 = 1;
+#[allow(unused)]
+const VIRTIO_IOMMU_TOPO_ENDPOINT: u16 = 2;
+
+#[derive(Copy, Clone, Debug, Default)]
+#[repr(packed)]
+struct VirtioIommuTopoPciRange {
+    type_: u16,
+    hierarchy: u16,
+    requester_start: u16,
+    requester_end: u16,
+    endpoint_start: u32,
+}
+
+unsafe impl ByteValued for VirtioIommuTopoPciRange {}
+
+#[derive(Copy, Clone, Debug, Default)]
+#[repr(packed)]
+struct VirtioIommuTopoEndpoint {
+    type_: u16,
+    reserved: u16,
+    endpoint: u32,
+    address: u64,
+}
+
+unsafe impl ByteValued for VirtioIommuTopoEndpoint {}
+
 /// Virtio IOMMU request type
 const VIRTIO_IOMMU_T_ATTACH: u8 = 1;
 const VIRTIO_IOMMU_T_DETACH: u8 = 2;
@@ -89,7 +148,6 @@ const VIRTIO_IOMMU_T_MAP: u8 = 3;
 const VIRTIO_IOMMU_T_UNMAP: u8 = 4;
 const VIRTIO_IOMMU_T_PROBE: u8 = 5;
 
-#[allow(unused)]
 #[derive(Copy, Clone, Debug, Default)]
 #[repr(packed)]
 struct VirtioIommuReqHead {
@@ -116,7 +174,6 @@ const VIRTIO_IOMMU_S_NOENT: u8 = 6;
 #[allow(unused)]
 const VIRTIO_IOMMU_S_FAULT: u8 = 7;
 
-#[allow(unused)]
 #[derive(Copy, Clone, Debug, Default)]
 #[repr(packed)]
 struct VirtioIommuReqTail {
@@ -127,7 +184,6 @@ struct VirtioIommuReqTail {
 unsafe impl ByteValued for VirtioIommuReqTail {}
 
 /// ATTACH request
-#[allow(unused)]
 #[derive(Copy, Clone, Debug, Default)]
 #[repr(packed)]
 struct VirtioIommuReqAttach {
@@ -139,7 +195,6 @@ struct VirtioIommuReqAttach {
 unsafe impl ByteValued for VirtioIommuReqAttach {}
 
 /// DETACH request
-#[allow(unused)]
 #[derive(Copy, Clone, Debug, Default)]
 #[repr(packed)]
 struct VirtioIommuReqDetach {
@@ -161,7 +216,6 @@ const VIRTIO_IOMMU_MAP_F_EXEC: u32 = 1 << 2;
 const VIRTIO_IOMMU_MAP_F_MMIO: u32 = 1 << 3;
 
 /// MAP request
-#[allow(unused)]
 #[derive(Copy, Clone, Debug, Default)]
 #[repr(packed)]
 struct VirtioIommuReqMap {
@@ -175,7 +229,6 @@ struct VirtioIommuReqMap {
 unsafe impl ByteValued for VirtioIommuReqMap {}
 
 /// UNMAP request
-#[allow(unused)]
 #[derive(Copy, Clone, Debug, Default)]
 #[repr(packed)]
 struct VirtioIommuReqUnmap {
@@ -189,14 +242,12 @@ unsafe impl ByteValued for VirtioIommuReqUnmap {}
 
 /// Virtio IOMMU request PROBE types
 #[allow(unused)]
-const VIRTIO_IOMMU_PROBE_T_MASK: u32 = 0xfff;
+const VIRTIO_IOMMU_PROBE_T_MASK: u16 = 0xfff;
 #[allow(unused)]
-const VIRTIO_IOMMU_PROBE_T_NONE: u32 = 0;
-#[allow(unused)]
-const VIRTIO_IOMMU_PROBE_T_RESV_MEM: u32 = 1;
+const VIRTIO_IOMMU_PROBE_T_NONE: u16 = 0;
+const VIRTIO_IOMMU_PROBE_T_RESV_MEM: u16 = 1;
 
 /// PROBE request
-#[allow(unused)]
 #[derive(Copy, Clone, Debug, Default)]
 #[repr(packed)]
 struct VirtioIommuReqProbe {
@@ -206,7 +257,6 @@ struct VirtioIommuReqProbe {
 
 unsafe impl ByteValued for VirtioIommuReqProbe {}
 
-#[allow(unused)]
 #[derive(Copy, Clone, Debug, Default)]
 #[repr(packed)]
 struct VirtioIommuProbeProperty {
@@ -218,15 +268,12 @@ unsafe impl ByteValued for VirtioIommuProbeProperty {}
 
 /// Virtio IOMMU request PROBE property RESV_MEM subtypes
 #[allow(unused)]
-const VIRTIO_IOMMU_RESV_MEM_T_RESERVED: u32 = 0;
-#[allow(unused)]
-const VIRTIO_IOMMU_RESV_MEM_T_MSI: u32 = 1;
+const VIRTIO_IOMMU_RESV_MEM_T_RESERVED: u8 = 0;
+const VIRTIO_IOMMU_RESV_MEM_T_MSI: u8 = 1;
 
-#[allow(unused)]
 #[derive(Copy, Clone, Debug, Default)]
 #[repr(packed)]
 struct VirtioIommuProbeResvMem {
-    head: VirtioIommuProbeProperty,
     subtype: u8,
     reserved: [u8; 3],
     start: u64,
@@ -320,20 +367,7 @@ impl Display for Error {
     }
 }
 
-#[derive(Debug, PartialEq)]
-enum RequestType {
-    Attach,
-    Detach,
-    Map,
-    Unmap,
-    Probe,
-}
-
-struct Request {
-    #[allow(unused)]
-    type_: RequestType,
-    status_addr: GuestAddress,
-}
+struct Request {}
 
 impl Request {
     // Parse the available vring buffer. Based on the hashmap table of external
@@ -350,7 +384,7 @@ impl Request {
         mapping: &Arc<IommuMapping>,
         ext_mapping: &BTreeMap<u32, Arc<dyn ExternalDmaMapping>>,
         ext_domain_mapping: &mut BTreeMap<u32, Arc<dyn ExternalDmaMapping>>,
-    ) -> result::Result<Request, Error> {
+    ) -> result::Result<usize, Error> {
         // The head contains the request type which MUST be readable.
         if avail_desc.is_write_only() {
             return Err(Error::UnexpectedWriteOnlyDescriptor);
@@ -370,7 +404,10 @@ impl Request {
             return Err(Error::InvalidRequest);
         };
 
-        let request_type = match req_head.type_ {
+        // Create the reply
+        let mut reply: Vec<u8> = Vec::new();
+
+        let hdr_len = match req_head.type_ {
             VIRTIO_IOMMU_T_ATTACH => {
                 if desc_size_left != size_of::<VirtioIommuReqAttach>() {
                     return Err(Error::InvalidAttachRequest);
@@ -401,7 +438,7 @@ impl Request {
                     mappings.insert(domain, BTreeMap::new());
                 }
 
-                RequestType::Attach
+                0
             }
             VIRTIO_IOMMU_T_DETACH => {
                 if desc_size_left != size_of::<VirtioIommuReqDetach>() {
@@ -427,7 +464,7 @@ impl Request {
                 // Remove endpoint associated with specific domain
                 mapping.endpoints.write().unwrap().remove(&endpoint);
 
-                RequestType::Detach
+                0
             }
             VIRTIO_IOMMU_T_MAP => {
                 if desc_size_left != size_of::<VirtioIommuReqMap>() {
@@ -463,7 +500,7 @@ impl Request {
                     return Err(Error::InvalidMapRequest);
                 }
 
-                RequestType::Map
+                0
             }
             VIRTIO_IOMMU_T_UNMAP => {
                 if desc_size_left != size_of::<VirtioIommuReqUnmap>() {
@@ -492,7 +529,7 @@ impl Request {
                     entry.remove(&virt_start);
                 }
 
-                RequestType::Unmap
+                0
             }
             VIRTIO_IOMMU_T_PROBE => {
                 if desc_size_left != size_of::<VirtioIommuReqProbe>() {
@@ -504,7 +541,21 @@ impl Request {
                     .map_err(Error::GuestMemory)?;
                 debug!("Probe request {:?}", req);
 
-                RequestType::Probe
+                let probe_prop = VirtioIommuProbeProperty {
+                    type_: VIRTIO_IOMMU_PROBE_T_RESV_MEM,
+                    length: size_of::<VirtioIommuProbeResvMem>() as u16,
+                };
+                reply.extend_from_slice(probe_prop.as_slice());
+
+                let resv_mem = VirtioIommuProbeResvMem {
+                    subtype: VIRTIO_IOMMU_RESV_MEM_T_MSI,
+                    start: MSI_IOVA_START,
+                    end: MSI_IOVA_END,
+                    ..Default::default()
+                };
+                reply.extend_from_slice(resv_mem.as_slice());
+
+                PROBE_PROP_SIZE
             }
             _ => return Err(Error::InvalidRequest),
         };
@@ -518,21 +569,27 @@ impl Request {
             return Err(Error::UnexpectedReadOnlyDescriptor);
         }
 
-        if (status_desc.len as usize) < size_of::<VirtioIommuReqTail>() {
+        if status_desc.len < hdr_len + size_of::<VirtioIommuReqTail>() as u32 {
             return Err(Error::BufferLengthTooSmall);
         }
 
-        Ok(Request {
-            type_: request_type,
-            status_addr: status_desc.addr,
-        })
+        let tail = VirtioIommuReqTail {
+            status: VIRTIO_IOMMU_S_OK,
+            ..Default::default()
+        };
+        reply.extend_from_slice(tail.as_slice());
+
+        mem.write_slice(reply.as_slice(), status_desc.addr)
+            .map_err(Error::GuestMemory)?;
+
+        Ok((hdr_len as usize) + size_of::<VirtioIommuReqTail>())
     }
 }
 
 struct IommuEpollHandler {
     queues: Vec<Queue>,
-    mem: Arc<RwLock<GuestMemoryMmap>>,
-    interrupt_cb: Arc<VirtioInterrupt>,
+    mem: Arc<ArcSwap<GuestMemoryMmap>>,
+    interrupt_cb: Arc<dyn VirtioInterrupt>,
     queue_evts: Vec<EventFd>,
     kill_evt: EventFd,
     pause_evt: EventFd,
@@ -545,7 +602,7 @@ impl IommuEpollHandler {
     fn request_queue(&mut self) -> bool {
         let mut used_desc_heads = [(0, 0); QUEUE_SIZE as usize];
         let mut used_count = 0;
-        let mem = self.mem.read().unwrap();
+        let mem = self.mem.load();
         for avail_desc in self.queues[0].iter(&mem) {
             let len = match Request::parse(
                 &avail_desc,
@@ -554,22 +611,9 @@ impl IommuEpollHandler {
                 &self.ext_mapping,
                 &mut self.ext_domain_mapping,
             ) {
-                Ok(ref req) => {
-                    let reply = VirtioIommuReqTail {
-                        status: VIRTIO_IOMMU_S_OK,
-                        ..Default::default()
-                    };
-
-                    match mem.write_obj(reply, req.status_addr) {
-                        Ok(_) => size_of::<VirtioIommuReqTail>() as u32,
-                        Err(e) => {
-                            error!("bad guest memory address: {}", e);
-                            0
-                        }
-                    }
-                }
+                Ok(len) => len as u32,
                 Err(e) => {
-                    error!("Failed to parse available descriptor chain: {:?}", e);
+                    error!("failed parsing descriptor: {}", e);
                     0
                 }
             };
@@ -589,10 +633,12 @@ impl IommuEpollHandler {
     }
 
     fn signal_used_queue(&self, queue: &Queue) -> result::Result<(), DeviceError> {
-        (self.interrupt_cb)(&VirtioInterruptType::Queue, Some(queue)).map_err(|e| {
-            error!("Failed to signal used queue: {:?}", e);
-            DeviceError::FailedSignalingUsedQueue(e)
-        })
+        self.interrupt_cb
+            .trigger(&VirtioInterruptType::Queue, Some(queue))
+            .map_err(|e| {
+                error!("Failed to signal used queue: {:?}", e);
+                DeviceError::FailedSignalingUsedQueue(e)
+            })
     }
 
     fn run(&mut self, paused: Arc<AtomicBool>) -> result::Result<(), DeviceError> {
@@ -747,11 +793,12 @@ pub struct Iommu {
     avail_features: u64,
     acked_features: u64,
     config: VirtioIommuConfig,
+    config_topo_pci_ranges: Vec<VirtioIommuTopoPciRange>,
     mapping: Arc<IommuMapping>,
     ext_mapping: BTreeMap<u32, Arc<dyn ExternalDmaMapping>>,
     queue_evts: Option<Vec<EventFd>>,
-    interrupt_cb: Option<Arc<VirtioInterrupt>>,
-    epoll_thread: Option<thread::JoinHandle<result::Result<(), DeviceError>>>,
+    interrupt_cb: Option<Arc<dyn VirtioInterrupt>>,
+    epoll_threads: Option<Vec<thread::JoinHandle<result::Result<(), DeviceError>>>>,
     paused: Arc<AtomicBool>,
 }
 
@@ -759,6 +806,7 @@ impl Iommu {
     pub fn new() -> io::Result<(Self, Arc<IommuMapping>)> {
         let config = VirtioIommuConfig {
             page_size_mask: VIRTIO_IOMMU_PAGE_SIZE_MASK,
+            probe_size: PROBE_PROP_SIZE,
             ..Default::default()
         };
 
@@ -771,20 +819,61 @@ impl Iommu {
             Iommu {
                 kill_evt: None,
                 pause_evt: None,
-                avail_features: 1u64 << VIRTIO_F_VERSION_1 | 1u64 << VIRTIO_IOMMU_F_MAP_UNMAP,
+                avail_features: 1u64 << VIRTIO_F_VERSION_1
+                    | 1u64 << VIRTIO_IOMMU_F_MAP_UNMAP
+                    | 1u64 << VIRTIO_IOMMU_F_PROBE,
                 acked_features: 0u64,
                 config,
+                config_topo_pci_ranges: Vec::new(),
                 mapping: mapping.clone(),
                 ext_mapping: BTreeMap::new(),
                 queue_evts: None,
                 interrupt_cb: None,
-                epoll_thread: None,
+                epoll_threads: None,
                 paused: Arc::new(AtomicBool::new(false)),
             },
             mapping,
         ))
     }
 
+    // This function lets the caller specify a list of devices attached to the
+    // virtual IOMMU. This list is translated into a virtio-iommu configuration
+    // topology, so that it can be understood by the guest driver.
+    //
+    // The topology is overriden everytime this function is being invoked.
+    //
+    // This function is dedicated to PCI, which means it will exclusively
+    // create VIRTIO_IOMMU_TOPO_PCI_RANGE entries.
+    pub fn attach_pci_devices(&mut self, domain: u16, device_ids: Vec<u32>) {
+        if device_ids.is_empty() {
+            warn!("No device to attach to virtual IOMMU");
+            return;
+        }
+
+        // If there is at least one device attached to the virtual IOMMU, we
+        // need the topology feature to be enabled.
+        self.avail_features |= 1u64 << VIRTIO_IOMMU_F_TOPOLOGY;
+
+        // Update the topology.
+        let mut topo_pci_ranges = Vec::new();
+        for device_id in device_ids.iter() {
+            let dev_id = *device_id;
+            topo_pci_ranges.push(VirtioIommuTopoPciRange {
+                type_: VIRTIO_IOMMU_TOPO_PCI_RANGE,
+                hierarchy: domain,
+                requester_start: dev_id as u16,
+                requester_end: dev_id as u16,
+                endpoint_start: dev_id,
+            });
+        }
+        self.config_topo_pci_ranges = topo_pci_ranges;
+
+        // Update the configuration to include the topology.
+        self.config.topo_config.offset = size_of::<VirtioIommuConfig>() as u32;
+        self.config.topo_config.num_items = self.config_topo_pci_ranges.len() as u32;
+        self.config.topo_config.item_length = size_of::<VirtioIommuTopoPciRange>() as u32;
+    }
+
     pub fn add_external_mapping(&mut self, device_id: u32, mapping: Arc<dyn ExternalDmaMapping>) {
         self.ext_mapping.insert(device_id, mapping);
     }
@@ -808,29 +897,12 @@ impl VirtioDevice for Iommu {
         QUEUE_SIZES
     }
 
-    fn features(&self, page: u32) -> u32 {
-        match page {
-            // Get the lower 32-bits of the features bitfield.
-            0 => self.avail_features as u32,
-            // Get the upper 32-bits of the features bitfield.
-            1 => (self.avail_features >> 32) as u32,
-            _ => {
-                warn!("Received request for unknown features page.");
-                0u32
-            }
-        }
+    fn features(&self) -> u64 {
+        self.avail_features
     }
 
-    fn ack_features(&mut self, page: u32, value: u32) {
-        let mut v = match page {
-            0 => u64::from(value),
-            1 => u64::from(value) << 32,
-            _ => {
-                warn!("Cannot acknowledge unknown features page.");
-                0u64
-            }
-        };
-
+    fn ack_features(&mut self, value: u64) {
+        let mut v = value;
         // Check if the guest is ACK'ing a feature that we didn't claim to have.
         let unrequested_features = v & !self.avail_features;
         if unrequested_features != 0 {
@@ -843,7 +915,13 @@ impl VirtioDevice for Iommu {
     }
 
     fn read_config(&self, offset: u64, mut data: &mut [u8]) {
-        let config_slice = self.config.as_slice();
+        let mut config: Vec<u8> = Vec::new();
+        config.extend_from_slice(self.config.as_slice());
+        for config_topo_pci_range in self.config_topo_pci_ranges.iter() {
+            config.extend_from_slice(config_topo_pci_range.as_slice());
+        }
+
+        let config_slice = config.as_slice();
         let config_len = config_slice.len() as u64;
         if offset >= config_len {
             error!("Failed to read config space");
@@ -863,8 +941,8 @@ impl VirtioDevice for Iommu {
 
     fn activate(
         &mut self,
-        mem: Arc<RwLock<GuestMemoryMmap>>,
-        interrupt_cb: Arc<VirtioInterrupt>,
+        mem: Arc<ArcSwap<GuestMemoryMmap>>,
+        interrupt_cb: Arc<dyn VirtioInterrupt>,
         queues: Vec<Queue>,
         queue_evts: Vec<EventFd>,
     ) -> ActivateResult {
@@ -921,19 +999,22 @@ impl VirtioDevice for Iommu {
         };
 
         let paused = self.paused.clone();
+        let mut epoll_threads = Vec::new();
         thread::Builder::new()
             .name("virtio_iommu".to_string())
             .spawn(move || handler.run(paused))
-            .map(|thread| self.epoll_thread = Some(thread))
+            .map(|thread| epoll_threads.push(thread))
             .map_err(|e| {
                 error!("failed to clone the virtio-iommu epoll thread: {}", e);
                 ActivateError::BadActivate
             })?;
 
+        self.epoll_threads = Some(epoll_threads);
+
         Ok(())
     }
 
-    fn reset(&mut self) -> Option<(Arc<VirtioInterrupt>, Vec<EventFd>)> {
+    fn reset(&mut self) -> Option<(Arc<dyn VirtioInterrupt>, Vec<EventFd>)> {
         // We first must resume the virtio thread if it was paused.
         if self.pause_evt.take().is_some() {
             self.resume().ok()?;

vm-virtio/src/lib.rs

@@ -9,6 +9,8 @@
 // SPDX-License-Identifier: Apache-2.0 AND BSD-3-Clause
 
 //! Implements virtio devices, queues, and transport mechanisms.
+
+extern crate arc_swap;
 extern crate epoll;
 #[macro_use]
 extern crate log;
@@ -28,6 +30,7 @@ pub mod block;
 mod console;
 mod iommu;
 pub mod net;
+pub mod net_util;
 mod pmem;
 mod queue;
 mod rng;
@@ -41,6 +44,7 @@ pub use self::console::*;
 pub use self::device::*;
 pub use self::iommu::*;
 pub use self::net::*;
+pub use self::net_util::*;
 pub use self::pmem::*;
 pub use self::queue::*;
 pub use self::rng::*;
@@ -83,6 +87,7 @@ impl From<u32> for VirtioDeviceType {
         match t {
             1 => VirtioDeviceType::TYPE_NET,
             2 => VirtioDeviceType::TYPE_BLOCK,
+            3 => VirtioDeviceType::TYPE_CONSOLE,
             4 => VirtioDeviceType::TYPE_RNG,
             5 => VirtioDeviceType::TYPE_BALLOON,
             9 => VirtioDeviceType::TYPE_9P,
@@ -105,14 +110,17 @@ impl fmt::Display for VirtioDeviceType {
         let output = match *self {
             VirtioDeviceType::TYPE_NET => "net",
             VirtioDeviceType::TYPE_BLOCK => "block",
+            VirtioDeviceType::TYPE_CONSOLE => "console",
             VirtioDeviceType::TYPE_RNG => "rng",
             VirtioDeviceType::TYPE_BALLOON => "balloon",
             VirtioDeviceType::TYPE_GPU => "gpu",
             VirtioDeviceType::TYPE_9P => "9p",
+            VirtioDeviceType::TYPE_INPUT => "input",
             VirtioDeviceType::TYPE_VSOCK => "vsock",
+            VirtioDeviceType::TYPE_IOMMU => "iommu",
             VirtioDeviceType::TYPE_FS => "fs",
             VirtioDeviceType::TYPE_PMEM => "pmem",
-            _ => return Err(std::fmt::Error),
+            VirtioDeviceType::TYPE_UNKNOWN => "UNKNOWN",
         };
         write!(f, "{}", output)
     }
@@ -122,6 +130,8 @@ impl fmt::Display for VirtioDeviceType {
 const INTERRUPT_STATUS_USED_RING: u32 = 0x1;
 #[allow(dead_code)]
 const INTERRUPT_STATUS_CONFIG_CHANGED: u32 = 0x2;
+#[cfg(feature = "pci_support")]
+const VIRTIO_MSI_NO_VECTOR: u16 = 0xffff;
 
 #[derive(Debug)]
 pub enum ActivateError {

vm-virtio/src/net.rs

@@ -5,125 +5,50 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the THIRD-PARTY file.
 
+use super::net_util::{
+    build_net_config_space, build_net_config_space_with_mq, open_tap, register_listener,
+    unregister_listener, CtrlVirtio, NetCtrlEpollHandler, RxVirtio, TxVirtio, VirtioNetConfig,
+    KILL_EVENT, NET_EVENTS_COUNT, PAUSE_EVENT, RX_QUEUE_EVENT, RX_TAP_EVENT, TX_QUEUE_EVENT,
+};
+use super::Error as DeviceError;
+use super::{
+    ActivateError, ActivateResult, Queue, VirtioDevice, VirtioDeviceType, VirtioInterruptType,
+};
+use crate::VirtioInterrupt;
+use arc_swap::ArcSwap;
 use epoll;
 use libc::EAGAIN;
 use libc::EFD_NONBLOCK;
+use net_util::{MacAddr, Tap};
 use std::cmp;
 use std::io::Read;
 use std::io::{self, Write};
-use std::mem;
 use std::net::Ipv4Addr;
 use std::os::unix::io::{AsRawFd, RawFd};
 use std::result;
 use std::sync::atomic::{AtomicBool, Ordering};
-use std::sync::{Arc, RwLock};
+use std::sync::Arc;
 use std::thread;
 use std::vec::Vec;
-
-use net_gen;
-
-use super::Error as DeviceError;
-use super::{
-    ActivateError, ActivateResult, DeviceEventT, Queue, VirtioDevice, VirtioDeviceType,
-    VirtioInterruptType,
-};
-use crate::VirtioInterrupt;
-use net_util::{MacAddr, Tap, TapError, MAC_ADDR_LEN};
 use virtio_bindings::bindings::virtio_net::*;
 use vm_device::{Migratable, MigratableError, Pausable, Snapshotable};
-use vm_memory::{Bytes, GuestAddress, GuestMemoryMmap};
+use vm_memory::{ByteValued, GuestMemoryMmap};
 use vmm_sys_util::eventfd::EventFd;
 
-/// The maximum buffer size when segmentation offload is enabled. This
-/// includes the 12-byte virtio net header.
-/// http://docs.oasis-open.org/virtio/virtio/v1.0/virtio-v1.0.html#x1-1740003
-const MAX_BUFFER_SIZE: usize = 65562;
-const QUEUE_SIZE: u16 = 256;
-const NUM_QUEUES: usize = 2;
-const QUEUE_SIZES: &[u16] = &[QUEUE_SIZE; NUM_QUEUES];
-
-// A frame is available for reading from the tap device to receive in the guest.
-const RX_TAP_EVENT: DeviceEventT = 0;
-// The guest has made a buffer available to receive a frame into.
-const RX_QUEUE_EVENT: DeviceEventT = 1;
-// The transmit queue has a frame that is ready to send from the guest.
-const TX_QUEUE_EVENT: DeviceEventT = 2;
-// The device has been dropped.
-pub const KILL_EVENT: DeviceEventT = 3;
-// Number of DeviceEventT events supported by this implementation.
-pub const NET_EVENTS_COUNT: usize = 4;
-// The device should be paused.
-const PAUSE_EVENT: DeviceEventT = 5;
-
 #[derive(Debug)]
 pub enum Error {
-    /// Open tap device failed.
-    TapOpen(TapError),
-    /// Setting tap IP failed.
-    TapSetIp(TapError),
-    /// Setting tap netmask failed.
-    TapSetNetmask(TapError),
-    /// Setting tap interface offload flags failed.
-    TapSetOffload(TapError),
-    /// Setting vnet header size failed.
-    TapSetVnetHdrSize(TapError),
-    /// Enabling tap interface failed.
-    TapEnable(TapError),
+    /// Failed to open taps.
+    OpenTap(super::net_util::Error),
 }
 
 pub type Result<T> = result::Result<T, Error>;
 
-struct TxVirtio {
-    queue_evt: EventFd,
-    queue: Queue,
-    iovec: Vec<(GuestAddress, usize)>,
-    frame_buf: [u8; MAX_BUFFER_SIZE],
-}
-
-impl TxVirtio {
-    fn new(queue: Queue, queue_evt: EventFd) -> Self {
-        let tx_queue_max_size = queue.get_max_size() as usize;
-        TxVirtio {
-            queue_evt,
-            queue,
-            iovec: Vec::with_capacity(tx_queue_max_size),
-            frame_buf: [0u8; MAX_BUFFER_SIZE],
-        }
-    }
-}
-
-struct RxVirtio {
-    queue_evt: EventFd,
-    deferred_frame: bool,
-    deferred_irqs: bool,
-    queue: Queue,
-    bytes_read: usize,
-    frame_buf: [u8; MAX_BUFFER_SIZE],
-}
-
-impl RxVirtio {
-    fn new(queue: Queue, queue_evt: EventFd) -> Self {
-        RxVirtio {
-            queue_evt,
-            deferred_frame: false,
-            deferred_irqs: false,
-            queue,
-            bytes_read: 0,
-            frame_buf: [0u8; MAX_BUFFER_SIZE],
-        }
-    }
-}
-
-fn vnet_hdr_len() -> usize {
-    mem::size_of::<virtio_net_hdr_v1>()
-}
-
 struct NetEpollHandler {
-    mem: Arc<RwLock<GuestMemoryMmap>>,
+    mem: Arc<ArcSwap<GuestMemoryMmap>>,
     tap: Tap,
     rx: RxVirtio,
     tx: TxVirtio,
-    interrupt_cb: Arc<VirtioInterrupt>,
+    interrupt_cb: Arc<dyn VirtioInterrupt>,
     kill_evt: EventFd,
     pause_evt: EventFd,
     epoll_fd: RawFd,
@@ -132,80 +57,46 @@ struct NetEpollHandler {
 
 impl NetEpollHandler {
     fn signal_used_queue(&self, queue: &Queue) -> result::Result<(), DeviceError> {
-        (self.interrupt_cb)(&VirtioInterruptType::Queue, Some(queue)).map_err(|e| {
-            error!("Failed to signal used queue: {:?}", e);
-            DeviceError::FailedSignalingUsedQueue(e)
-        })
+        self.interrupt_cb
+            .trigger(&VirtioInterruptType::Queue, Some(queue))
+            .map_err(|e| {
+                error!("Failed to signal used queue: {:?}", e);
+                DeviceError::FailedSignalingUsedQueue(e)
+            })
     }
 
     // Copies a single frame from `self.rx.frame_buf` into the guest. Returns true
     // if a buffer was used, and false if the frame must be deferred until a buffer
     // is made available by the driver.
-    fn rx_single_frame(&mut self) -> bool {
-        let mem = self.mem.read().unwrap();
-        let mut next_desc = self.rx.queue.iter(&mem).next();
+    fn rx_single_frame(&mut self, mut queue: &mut Queue) -> bool {
+        let mem = self.mem.load();
+        let next_desc = queue.iter(&mem).next();
 
         if next_desc.is_none() {
             // Queue has no available descriptors
             if self.rx_tap_listening {
-                self.unregister_tap_rx_listener().unwrap();
+                unregister_listener(
+                    self.epoll_fd,
+                    self.tap.as_raw_fd(),
+                    epoll::Events::EPOLLIN,
+                    u64::from(RX_TAP_EVENT),
+                )
+                .unwrap();
                 self.rx_tap_listening = false;
             }
             return false;
         }
 
-        // We just checked that the head descriptor exists.
-        let head_index = next_desc.as_ref().unwrap().index;
-        let mut write_count = 0;
-
-        // Copy from frame into buffer, which may span multiple descriptors.
-        loop {
-            match next_desc {
-                Some(desc) => {
-                    if !desc.is_write_only() {
-                        break;
-                    }
-                    let limit = cmp::min(write_count + desc.len as usize, self.rx.bytes_read);
-                    let source_slice = &self.rx.frame_buf[write_count..limit];
-                    let write_result = mem.write_slice(source_slice, desc.addr);
-
-                    match write_result {
-                        Ok(_) => {
-                            write_count = limit;
-                        }
-                        Err(e) => {
-                            error!("Failed to write slice: {:?}", e);
-                            break;
-                        }
-                    };
-
-                    if write_count >= self.rx.bytes_read {
-                        break;
-                    }
-                    next_desc = desc.next_descriptor();
-                }
-                None => {
-                    warn!("Receiving buffer is too small to hold frame of current size");
-                    break;
-                }
-            }
-        }
-
-        self.rx.queue.add_used(&mem, head_index, write_count as u32);
-
-        // Mark that we have at least one pending packet and we need to interrupt the guest.
-        self.rx.deferred_irqs = true;
-
-        write_count >= self.rx.bytes_read
+        self.rx.process_desc_chain(&mem, next_desc, &mut queue)
     }
 
-    fn process_rx(&mut self) -> result::Result<(), DeviceError> {
+    fn process_rx(&mut self, queue: &mut Queue) -> result::Result<(), DeviceError> {
         // Read as many frames as possible.
         loop {
             match self.read_tap() {
                 Ok(count) => {
                     self.rx.bytes_read = count;
-                    if !self.rx_single_frame() {
+                    if !self.rx_single_frame(queue) {
                         self.rx.deferred_frame = true;
                         break;
                     }
@@ -226,22 +117,22 @@ impl NetEpollHandler {
         }
         if self.rx.deferred_irqs {
             self.rx.deferred_irqs = false;
-            self.signal_used_queue(&self.rx.queue)
+            self.signal_used_queue(queue)
         } else {
             Ok(())
         }
     }
 
-    fn resume_rx(&mut self) -> result::Result<(), DeviceError> {
+    fn resume_rx(&mut self, queue: &mut Queue) -> result::Result<(), DeviceError> {
         if self.rx.deferred_frame {
-            if self.rx_single_frame() {
+            if self.rx_single_frame(queue) {
                 self.rx.deferred_frame = false;
                 // process_rx() was interrupted possibly before consuming all
                 // packets in the tap; try continuing now.
-                self.process_rx()
+                self.process_rx(queue)
             } else if self.rx.deferred_irqs {
                 self.rx.deferred_irqs = false;
-                self.signal_used_queue(&self.rx.queue)
+                self.signal_used_queue(queue)
             } else {
                 Ok(())
             }
@@ -250,56 +141,10 @@ impl NetEpollHandler {
         }
     }
 
-    fn process_tx(&mut self) -> result::Result<(), DeviceError> {
-        let mem = self.mem.read().unwrap();
-        while let Some(avail_desc) = self.tx.queue.iter(&mem).next() {
-            let head_index = avail_desc.index;
-            let mut read_count = 0;
-            let mut next_desc = Some(avail_desc);
+    fn process_tx(&mut self, mut queue: &mut Queue) -> result::Result<(), DeviceError> {
+        let mem = self.mem.load();
 
-            self.tx.iovec.clear();
-            while let Some(desc) = next_desc {
-                if desc.is_write_only() {
-                    break;
-                }
-                self.tx.iovec.push((desc.addr, desc.len as usize));
-                read_count += desc.len as usize;
-                next_desc = desc.next_descriptor();
-            }
-
-            read_count = 0;
-            // Copy buffer from across multiple descriptors.
-            // TODO(performance - Issue #420): change this to use `writev()` instead of `write()`
-            // and get rid of the intermediate buffer.
-            for (desc_addr, desc_len) in self.tx.iovec.drain(..) {
-                let limit = cmp::min((read_count + desc_len) as usize, self.tx.frame_buf.len());
-
-                let read_result = mem.read_slice(
-                    &mut self.tx.frame_buf[read_count..limit as usize],
-                    desc_addr,
-                );
-                match read_result {
-                    Ok(_) => {
-                        // Increment by number of bytes actually read
-                        read_count += limit - read_count;
-                    }
-                    Err(e) => {
-                        error!("Failed to read slice: {:?}", e);
-                        break;
-                    }
-                }
-            }
-
-            let write_result = self.tap.write(&self.tx.frame_buf[..read_count as usize]);
-            match write_result {
-                Ok(_) => {}
-                Err(e) => {
-                    warn!("net: tx: error failed to write to tap: {}", e);
-                }
-            };
-
-            self.tx.queue.add_used(&mem, head_index, 0);
-        }
+        self.tx.process_desc_chain(&mem, &mut self.tap, &mut queue);
 
         Ok(())
     }
@@ -308,47 +153,73 @@ impl NetEpollHandler {
         self.tap.read(&mut self.rx.frame_buf)
     }
 
-    fn register_tap_rx_listener(&self) -> std::result::Result<(), std::io::Error> {
-        epoll::ctl(
-            self.epoll_fd,
-            epoll::ControlOptions::EPOLL_CTL_ADD,
-            self.tap.as_raw_fd(),
-            epoll::Event::new(epoll::Events::EPOLLIN, u64::from(RX_TAP_EVENT)),
-        )?;
-        Ok(())
+    fn handle_rx_event(&mut self, mut queue: &mut Queue, queue_evt: &EventFd) {
+        if let Err(e) = queue_evt.read() {
+            error!("Failed to get rx queue event: {:?}", e);
+        }
+
+        self.resume_rx(&mut queue).unwrap();
+        if !self.rx_tap_listening {
+            register_listener(
+                self.epoll_fd,
+                self.tap.as_raw_fd(),
+                epoll::Events::EPOLLIN,
+                u64::from(RX_TAP_EVENT),
+            )
+            .unwrap();
+            self.rx_tap_listening = true;
+        }
     }
 
-    fn unregister_tap_rx_listener(&self) -> std::result::Result<(), std::io::Error> {
-        epoll::ctl(
-            self.epoll_fd,
-            epoll::ControlOptions::EPOLL_CTL_DEL,
-            self.tap.as_raw_fd(),
-            epoll::Event::new(epoll::Events::EPOLLIN, u64::from(RX_TAP_EVENT)),
-        )?;
-        Ok(())
+    fn handle_tx_event(&mut self, mut queue: &mut Queue, queue_evt: &EventFd) {
+        if let Err(e) = queue_evt.read() {
+            error!("Failed to get tx queue event: {:?}", e);
+        }
+
+        self.process_tx(&mut queue).unwrap();
     }
 
-    fn run(&mut self, paused: Arc<AtomicBool>) -> result::Result<(), DeviceError> {
+    fn handle_rx_tap_event(&mut self, mut queue: &mut Queue) {
+        if self.rx.deferred_frame
+        // Process a deferred frame first if available. Don't read from tap again
+        // until we manage to receive this deferred frame.
+        {
+            if self.rx_single_frame(&mut queue) {
+                self.rx.deferred_frame = false;
+                self.process_rx(&mut queue).unwrap();
+            } else if self.rx.deferred_irqs {
+                self.rx.deferred_irqs = false;
+                self.signal_used_queue(&queue).unwrap();
+            }
+        } else {
+            self.process_rx(&mut queue).unwrap();
+        }
+    }
+
+    fn run(
+        &mut self,
+        paused: Arc<AtomicBool>,
+        mut queues: Vec<Queue>,
+        queue_evts: Vec<EventFd>,
+    ) -> result::Result<(), DeviceError> {
         // Create the epoll file descriptor
         self.epoll_fd = epoll::create(true).map_err(DeviceError::EpollCreateFd)?;
         // Add events
+        // Add events
         epoll::ctl(
             self.epoll_fd,
             epoll::ControlOptions::EPOLL_CTL_ADD,
-            self.rx.queue_evt.as_raw_fd(),
+            queue_evts[0].as_raw_fd(),
             epoll::Event::new(epoll::Events::EPOLLIN, u64::from(RX_QUEUE_EVENT)),
         )
         .map_err(DeviceError::EpollCtl)?;
         epoll::ctl(
             self.epoll_fd,
             epoll::ControlOptions::EPOLL_CTL_ADD,
-            self.tx.queue_evt.as_raw_fd(),
+            queue_evts[1].as_raw_fd(),
             epoll::Event::new(epoll::Events::EPOLLIN, u64::from(TX_QUEUE_EVENT)),
         )
         .map_err(DeviceError::EpollCtl)?;
-        self.register_tap_rx_listener()
-            .map_err(DeviceError::EpollCtl)?;
-        self.rx_tap_listening = true;
         epoll::ctl(
             self.epoll_fd,
             epoll::ControlOptions::EPOLL_CTL_ADD,
@@ -364,8 +235,7 @@ impl NetEpollHandler {
         )
         .map_err(DeviceError::EpollCtl)?;
 
-        const EPOLL_EVENTS_LEN: usize = 100;
-        let mut events = vec![epoll::Event::new(epoll::Events::empty(), 0); EPOLL_EVENTS_LEN];
+        let mut events = vec![epoll::Event::new(epoll::Events::empty(), 0); NET_EVENTS_COUNT];
 
         'epoll: loop {
             let num_events = match epoll::wait(self.epoll_fd, -1, &mut events[..]) {
@@ -390,43 +260,13 @@ impl NetEpollHandler {
 
                 match ev_type {
                     RX_QUEUE_EVENT => {
-                        debug!("RX_QUEUE_EVENT received");
-                        if let Err(e) = self.rx.queue_evt.read() {
-                            error!("Failed to get rx queue event: {:?}", e);
-                            break 'epoll;
-                        }
-
-                        self.resume_rx().unwrap();
-                        if !self.rx_tap_listening {
-                            self.register_tap_rx_listener().unwrap();
-                            self.rx_tap_listening = true;
-                        }
+                        self.handle_rx_event(&mut queues[0], &queue_evts[0]);
                     }
                     TX_QUEUE_EVENT => {
-                        debug!("TX_QUEUE_EVENT received");
-                        if let Err(e) = self.tx.queue_evt.read() {
-                            error!("Failed to get tx queue event: {:?}", e);
-                            break 'epoll;
-                        }
-
-                        self.process_tx().unwrap();
+                        self.handle_tx_event(&mut queues[1], &queue_evts[1]);
                     }
                     RX_TAP_EVENT => {
-                        debug!("RX_TAP_EVENT received");
-                        if self.rx.deferred_frame
-                        // Process a deferred frame first if available. Don't read from tap again
-                        // until we manage to receive this deferred frame.
-                        {
-                            if self.rx_single_frame() {
-                                self.rx.deferred_frame = false;
-                                self.process_rx().unwrap();
-                            } else if self.rx.deferred_irqs {
-                                self.rx.deferred_irqs = false;
-                                self.signal_used_queue(&self.rx.queue).unwrap();
-                            }
-                        } else {
-                            self.process_rx().unwrap();
-                        }
+                        self.handle_rx_tap_event(&mut queues[0]);
                     }
                     KILL_EVENT => {
                         debug!("KILL_EVENT received, stopping epoll loop");
@@ -447,7 +287,6 @@ impl NetEpollHandler {
                 }
             }
         }
-
         Ok(())
     }
 }
@@ -455,31 +294,27 @@ impl NetEpollHandler {
 pub struct Net {
     kill_evt: Option<EventFd>,
     pause_evt: Option<EventFd>,
-    tap: Option<Tap>,
+    taps: Option<Vec<Tap>>,
     avail_features: u64,
     acked_features: u64,
-    // The config space will only consist of the MAC address specified by the user,
-    // or nothing, if no such address if provided.
-    config_space: Vec<u8>,
+    config: VirtioNetConfig,
     queue_evts: Option<Vec<EventFd>>,
-    interrupt_cb: Option<Arc<VirtioInterrupt>>,
-    epoll_thread: Option<thread::JoinHandle<result::Result<(), DeviceError>>>,
+    interrupt_cb: Option<Arc<dyn VirtioInterrupt>>,
+    epoll_threads: Option<Vec<thread::JoinHandle<result::Result<(), DeviceError>>>>,
+    ctrl_queue_epoll_thread: Option<thread::JoinHandle<result::Result<(), DeviceError>>>,
     paused: Arc<AtomicBool>,
+    queue_size: Vec<u16>,
 }
 
 impl Net {
     /// Create a new virtio network device with the given TAP interface.
-    pub fn new_with_tap(tap: Tap, guest_mac: Option<&MacAddr>, iommu: bool) -> Result<Self> {
-        // Set offload flags to match the virtio features below.
-        tap.set_offload(
-            net_gen::TUN_F_CSUM | net_gen::TUN_F_UFO | net_gen::TUN_F_TSO4 | net_gen::TUN_F_TSO6,
-        )
-        .map_err(Error::TapSetOffload)?;
-
-        let vnet_hdr_size = vnet_hdr_len() as i32;
-        tap.set_vnet_hdr_size(vnet_hdr_size)
-            .map_err(Error::TapSetVnetHdrSize)?;
-
+    pub fn new_with_tap(
+        taps: Vec<Tap>,
+        guest_mac: Option<MacAddr>,
+        iommu: bool,
+        num_queues: usize,
+        queue_size: u16,
+    ) -> Result<Self> {
         let mut avail_features = 1 << VIRTIO_NET_F_GUEST_CSUM
             | 1 << VIRTIO_NET_F_CSUM
             | 1 << VIRTIO_NET_F_GUEST_TSO4
@@ -492,47 +327,46 @@ impl Net {
             avail_features |= 1u64 << VIRTIO_F_IOMMU_PLATFORM;
         }
 
-        let mut config_space;
+        avail_features |= 1 << VIRTIO_NET_F_CTRL_VQ;
+        let queue_num = num_queues + 1;
+
+        let mut config = VirtioNetConfig::default();
         if let Some(mac) = guest_mac {
-            config_space = Vec::with_capacity(MAC_ADDR_LEN);
-            // This is safe, because we know the capacity is large enough.
-            unsafe { config_space.set_len(MAC_ADDR_LEN) }
-            config_space[..].copy_from_slice(mac.get_bytes());
-            // When this feature isn't available, the driver generates a random MAC address.
-            // Otherwise, it should attempt to read the device MAC address from the config space.
-            avail_features |= 1 << VIRTIO_NET_F_MAC;
+            build_net_config_space(&mut config, mac, num_queues, &mut avail_features);
         } else {
-            config_space = Vec::new();
+            build_net_config_space_with_mq(&mut config, num_queues, &mut avail_features);
         }
 
         Ok(Net {
             kill_evt: None,
             pause_evt: None,
-            tap: Some(tap),
+            taps: Some(taps),
             avail_features,
             acked_features: 0u64,
-            config_space,
+            config,
             queue_evts: None,
             interrupt_cb: None,
-            epoll_thread: None,
+            epoll_threads: None,
+            ctrl_queue_epoll_thread: None,
             paused: Arc::new(AtomicBool::new(false)),
+            queue_size: vec![queue_size; queue_num],
         })
     }
 
     /// Create a new virtio network device with the given IP address and
     /// netmask.
     pub fn new(
-        ip_addr: Ipv4Addr,
-        netmask: Ipv4Addr,
-        guest_mac: Option<&MacAddr>,
+        if_name: Option<&str>,
+        ip_addr: Option<Ipv4Addr>,
+        netmask: Option<Ipv4Addr>,
+        guest_mac: Option<MacAddr>,
         iommu: bool,
+        num_queues: usize,
+        queue_size: u16,
     ) -> Result<Self> {
-        let tap = Tap::new().map_err(Error::TapOpen)?;
-        tap.set_ip_addr(ip_addr).map_err(Error::TapSetIp)?;
-        tap.set_netmask(netmask).map_err(Error::TapSetNetmask)?;
-        tap.enable().map_err(Error::TapEnable)?;
+        let taps = open_tap(if_name, ip_addr, netmask, num_queues / 2).map_err(Error::OpenTap)?;
 
-        Self::new_with_tap(tap, guest_mac, iommu)
+        Self::new_with_tap(taps, guest_mac, iommu, num_queues, queue_size)
     }
 }
 
@@ -551,32 +385,15 @@ impl VirtioDevice for Net {
     }
 
     fn queue_max_sizes(&self) -> &[u16] {
-        QUEUE_SIZES
+        &self.queue_size.as_slice()
     }
 
-    fn features(&self, page: u32) -> u32 {
-        match page {
-            // Get the lower 32-bits of the features bitfield.
-            0 => self.avail_features as u32,
-            // Get the upper 32-bits of the features bitfield.
-            1 => (self.avail_features >> 32) as u32,
-            _ => {
-                warn!("Received request for unknown features page: {}", page);
-                0u32
-            }
-        }
+    fn features(&self) -> u64 {
+        self.avail_features
     }
 
-    fn ack_features(&mut self, page: u32, value: u32) {
-        let mut v = match page {
-            0 => u64::from(value),
-            1 => u64::from(value) << 32,
-            _ => {
-                warn!("Cannot acknowledge unknown features page: {}", page);
-                0u64
-            }
-        };
-
+    fn ack_features(&mut self, value: u64) {
+        let mut v = value;
         // Check if the guest is ACK'ing a feature that we didn't claim to have.
         let unrequested_features = v & !self.avail_features;
         if unrequested_features != 0 {
@@ -588,40 +405,42 @@ impl VirtioDevice for Net {
     }
 
     fn read_config(&self, offset: u64, mut data: &mut [u8]) {
-        let config_len = self.config_space.len() as u64;
+        let config_slice = self.config.as_slice();
+        let config_len = config_slice.len() as u64;
         if offset >= config_len {
             error!("Failed to read config space");
             return;
         }
         if let Some(end) = offset.checked_add(data.len() as u64) {
             // This write can't fail, offset and end are checked against config_len.
-            data.write_all(&self.config_space[offset as usize..cmp::min(end, config_len) as usize])
+            data.write_all(&config_slice[offset as usize..cmp::min(end, config_len) as usize])
                 .unwrap();
         }
     }
 
     fn write_config(&mut self, offset: u64, data: &[u8]) {
+        let config_slice = self.config.as_mut_slice();
         let data_len = data.len() as u64;
-        let config_len = self.config_space.len() as u64;
+        let config_len = config_slice.len() as u64;
         if offset + data_len > config_len {
             error!("Failed to write config space");
             return;
         }
-        let (_, right) = self.config_space.split_at_mut(offset as usize);
+        let (_, right) = config_slice.split_at_mut(offset as usize);
         right.copy_from_slice(&data[..]);
     }
 
     fn activate(
         &mut self,
-        mem: Arc<RwLock<GuestMemoryMmap>>,
-        interrupt_cb: Arc<VirtioInterrupt>,
+        mem: Arc<ArcSwap<GuestMemoryMmap>>,
+        interrupt_cb: Arc<dyn VirtioInterrupt>,
         mut queues: Vec<Queue>,
         mut queue_evts: Vec<EventFd>,
     ) -> ActivateResult {
-        if queues.len() != NUM_QUEUES || queue_evts.len() != NUM_QUEUES {
+        if queues.len() != self.queue_size.len() || queue_evts.len() != self.queue_size.len() {
             error!(
                 "Cannot perform activate. Expected {} queue(s), got {}",
-                NUM_QUEUES,
+                self.queue_size.len(),
                 queues.len()
             );
             return Err(ActivateError::BadActivate);
@@ -643,7 +462,7 @@ impl VirtioDevice for Net {
             })?;
         self.pause_evt = Some(self_pause_evt);
 
-        if let Some(tap) = self.tap.clone() {
+        if let Some(mut taps) = self.taps.clone() {
             // Save the interrupt EventFD as we need to return it on reset
             // but clone it to pass into the thread.
             self.interrupt_cb = Some(interrupt_cb.clone());
@@ -659,38 +478,75 @@ impl VirtioDevice for Net {
             }
             self.queue_evts = Some(tmp_queue_evts);
 
-            let rx_queue = queues.remove(0);
-            let tx_queue = queues.remove(0);
-            let rx_queue_evt = queue_evts.remove(0);
-            let tx_queue_evt = queue_evts.remove(0);
-            let mut handler = NetEpollHandler {
-                mem,
-                tap,
-                rx: RxVirtio::new(rx_queue, rx_queue_evt),
-                tx: TxVirtio::new(tx_queue, tx_queue_evt),
-                interrupt_cb,
-                kill_evt,
-                pause_evt,
-                epoll_fd: 0,
-                rx_tap_listening: false,
-            };
+            let queue_num = queues.len();
+            if (self.acked_features & 1 << VIRTIO_NET_F_CTRL_VQ) != 0 && queue_num % 2 != 0 {
+                let cvq_queue = queues.remove(queue_num - 1);
+                let cvq_queue_evt = queue_evts.remove(queue_num - 1);
 
-            let paused = self.paused.clone();
-            thread::Builder::new()
-                .name("virtio_net".to_string())
-                .spawn(move || handler.run(paused))
-                .map(|thread| self.epoll_thread = Some(thread))
-                .map_err(|e| {
-                    error!("failed to clone the virtio-net epoll thread: {}", e);
-                    ActivateError::BadActivate
-                })?;
+                let mut ctrl_handler = NetCtrlEpollHandler {
+                    mem: mem.clone(),
+                    kill_evt: kill_evt.try_clone().unwrap(),
+                    pause_evt: pause_evt.try_clone().unwrap(),
+                    ctrl_q: CtrlVirtio::new(cvq_queue, cvq_queue_evt),
+                    epoll_fd: 0,
+                };
+
+                let paused = self.paused.clone();
+                thread::Builder::new()
+                    .name("virtio_net".to_string())
+                    .spawn(move || ctrl_handler.run_ctrl(paused))
+                    .map(|thread| self.ctrl_queue_epoll_thread = Some(thread))
+                    .map_err(|e| {
+                        error!("failed to clone queue EventFd: {}", e);
+                        ActivateError::BadActivate
+                    })?;
+            }
+
+            let mut epoll_threads = Vec::new();
+            for _ in 0..taps.len() {
+                let rx = RxVirtio::new();
+                let tx = TxVirtio::new();
+                let rx_tap_listening = false;
+
+                let mut queue_pair = Vec::new();
+                queue_pair.push(queues.remove(0));
+                queue_pair.push(queues.remove(0));
+
+                let mut queue_evt_pair = Vec::new();
+                queue_evt_pair.push(queue_evts.remove(0));
+                queue_evt_pair.push(queue_evts.remove(0));
+
+                let mut handler = NetEpollHandler {
+                    mem: mem.clone(),
+                    tap: taps.remove(0),
+                    rx,
+                    tx,
+                    interrupt_cb: interrupt_cb.clone(),
+                    kill_evt: kill_evt.try_clone().unwrap(),
+                    pause_evt: pause_evt.try_clone().unwrap(),
+                    epoll_fd: 0,
+                    rx_tap_listening,
+                };
+
+                let paused = self.paused.clone();
+                thread::Builder::new()
+                    .name("virtio_net".to_string())
+                    .spawn(move || handler.run(paused, queue_pair, queue_evt_pair))
+                    .map(|thread| epoll_threads.push(thread))
+                    .map_err(|e| {
+                        error!("failed to clone queue EventFd: {}", e);
+                        ActivateError::BadActivate
+                    })?;
+            }
+
+            self.epoll_threads = Some(epoll_threads);
 
             return Ok(());
         }
         Err(ActivateError::BadActivate)
     }
 
-    fn reset(&mut self) -> Option<(Arc<VirtioInterrupt>, Vec<EventFd>)> {
+    fn reset(&mut self) -> Option<(Arc<dyn VirtioInterrupt>, Vec<EventFd>)> {
         // We first must resume the virtio thread if it was paused.
         if self.pause_evt.take().is_some() {
             self.resume().ok()?;
@@ -709,6 +565,6 @@ impl VirtioDevice for Net {
     }
 }
 
-virtio_pausable!(Net);
+virtio_ctrl_q_pausable!(Net);
 impl Snapshotable for Net {}
 impl Migratable for Net {}

vm-virtio/src/net_util.rs

@@ -0,0 +1,498 @@
+// Copyright (c) 2019 Intel Corporation. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0 AND BSD-3-Clause
+
+use super::Error as DeviceError;
+use super::{DescriptorChain, DeviceEventT, Queue};
+use arc_swap::ArcSwap;
+use net_util::{MacAddr, Tap, TapError};
+use std::cmp;
+use std::io::{self, Write};
+use std::mem;
+use std::net::Ipv4Addr;
+use std::os::unix::io::{AsRawFd, RawFd};
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::Arc;
+use virtio_bindings::bindings::virtio_net::*;
+use vm_memory::{ByteValued, Bytes, GuestAddress, GuestMemoryError, GuestMemoryMmap};
+use vmm_sys_util::eventfd::EventFd;
+
+type Result<T> = std::result::Result<T, Error>;
+
+/// The maximum buffer size when segmentation offload is enabled. This
+/// includes the 12-byte virtio net header.
+/// http://docs.oasis-open.org/virtio/virtio/v1.0/virtio-v1.0.html#x1-1740003
+const MAX_BUFFER_SIZE: usize = 65562;
+const QUEUE_SIZE: usize = 256;
+
+// The guest has made a buffer available to receive a frame into.
+pub const RX_QUEUE_EVENT: DeviceEventT = 0;
+// The transmit queue has a frame that is ready to send from the guest.
+pub const TX_QUEUE_EVENT: DeviceEventT = 1;
+// A frame is available for reading from the tap device to receive in the guest.
+pub const RX_TAP_EVENT: DeviceEventT = 2;
+// The device has been dropped.
+pub const KILL_EVENT: DeviceEventT = 3;
+// The device should be paused.
+pub const PAUSE_EVENT: DeviceEventT = 4;
+// Number of DeviceEventT events supported by this implementation.
+pub const NET_EVENTS_COUNT: usize = 5;
+// The device has been dropped.
+const CTRL_QUEUE_EVENT: DeviceEventT = 0;
+// Number of DeviceEventT events supported by this implementation.
+const CTRL_EVENT_COUNT: usize = 3;
+
+#[repr(C, packed)]
+#[derive(Copy, Clone, Debug, Default)]
+pub struct VirtioNetConfig {
+    pub mac: [u8; 6],
+    pub status: u16,
+    pub max_virtqueue_pairs: u16,
+    pub mtu: u16,
+    pub speed: u32,
+    pub duplex: u8,
+}
+
+// Safe because it only has data and has no implicit padding.
+unsafe impl ByteValued for VirtioNetConfig {}
+
+#[derive(Debug)]
+pub enum Error {
+    /// Read process MQ.
+    FailedProcessMQ,
+    /// Read queue failed.
+    GuestMemory(GuestMemoryError),
+    /// Invalid ctrl class
+    InvalidCtlClass,
+    /// Invalid ctrl command
+    InvalidCtlCmd,
+    /// Invalid descriptor
+    InvalidDesc,
+    /// Invalid queue pairs number
+    InvalidQueuePairsNum,
+    /// No memory passed in.
+    NoMemory,
+    /// No ueue pairs nummber.
+    NoQueuePairsNum,
+    /// Open tap device failed.
+    TapOpen(TapError),
+    /// Setting tap IP failed.
+    TapSetIp(TapError),
+    /// Setting tap netmask failed.
+    TapSetNetmask(TapError),
+    /// Setting tap interface offload flags failed.
+    TapSetOffload(TapError),
+    /// Setting vnet header size failed.
+    TapSetVnetHdrSize(TapError),
+    /// Enabling tap interface failed.
+    TapEnable(TapError),
+}
+
+pub struct CtrlVirtio {
+    pub queue_evt: EventFd,
+    pub queue: Queue,
+}
+
+impl std::clone::Clone for CtrlVirtio {
+    fn clone(&self) -> Self {
+        CtrlVirtio {
+            queue_evt: self.queue_evt.try_clone().unwrap(),
+            queue: self.queue.clone(),
+        }
+    }
+}
+
+impl CtrlVirtio {
+    pub fn new(queue: Queue, queue_evt: EventFd) -> Self {
+        CtrlVirtio { queue_evt, queue }
+    }
+
+    fn process_mq(&self, mem: &GuestMemoryMmap, avail_desc: DescriptorChain) -> Result<()> {
+        let mq_desc = if avail_desc.has_next() {
+            avail_desc.next_descriptor().unwrap()
+        } else {
+            return Err(Error::NoQueuePairsNum);
+        };
+        let queue_pairs = mem
+            .read_obj::<u16>(mq_desc.addr)
+            .map_err(Error::GuestMemory)?;
+        if (queue_pairs < VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MIN as u16)
+            || (queue_pairs > VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX as u16)
+        {
+            return Err(Error::InvalidQueuePairsNum);
+        }
+        let status_desc = if mq_desc.has_next() {
+            mq_desc.next_descriptor().unwrap()
+        } else {
+            return Err(Error::NoQueuePairsNum);
+        };
+        mem.write_obj::<u8>(0, status_desc.addr)
+            .map_err(Error::GuestMemory)?;
+
+        Ok(())
+    }
+
+    pub fn process_cvq(&mut self, mem: &GuestMemoryMmap) -> Result<()> {
+        let mut used_desc_heads = [(0, 0); QUEUE_SIZE];
+        let mut used_count = 0;
+        if let Some(avail_desc) = self.queue.iter(&mem).next() {
+            used_desc_heads[used_count] = (avail_desc.index, avail_desc.len);
+            used_count += 1;
+            let ctrl_hdr = mem
+                .read_obj::<u16>(avail_desc.addr)
+                .map_err(Error::GuestMemory)?;
+            let ctrl_hdr_v = ctrl_hdr.as_slice();
+            let class = ctrl_hdr_v[0];
+            let cmd = ctrl_hdr_v[1];
+            match u32::from(class) {
+                VIRTIO_NET_CTRL_MQ => {
+                    if u32::from(cmd) != VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET {
+                        return Err(Error::InvalidCtlCmd);
+                    }
+                    if let Err(_e) = self.process_mq(&mem, avail_desc) {
+                        return Err(Error::FailedProcessMQ);
+                    }
+                }
+                _ => return Err(Error::InvalidCtlClass),
+            }
+        } else {
+            return Err(Error::InvalidDesc);
+        }
+        for &(desc_index, len) in &used_desc_heads[..used_count] {
+            self.queue.add_used(&mem, desc_index, len);
+        }
+
+        Ok(())
+    }
+}
+
+pub fn register_listener(
+    epoll_fd: RawFd,
+    fd: RawFd,
+    ev_type: epoll::Events,
+    data: u64,
+) -> std::result::Result<(), io::Error> {
+    epoll::ctl(
+        epoll_fd,
+        epoll::ControlOptions::EPOLL_CTL_ADD,
+        fd,
+        epoll::Event::new(ev_type, data),
+    )
+}
+
+pub fn unregister_listener(
+    epoll_fd: RawFd,
+    fd: RawFd,
+    ev_type: epoll::Events,
+    data: u64,
+) -> std::result::Result<(), io::Error> {
+    epoll::ctl(
+        epoll_fd,
+        epoll::ControlOptions::EPOLL_CTL_DEL,
+        fd,
+        epoll::Event::new(ev_type, data),
+    )
+}
+
+pub struct NetCtrlEpollHandler {
+    pub mem: Arc<ArcSwap<GuestMemoryMmap>>,
+    pub kill_evt: EventFd,
+    pub pause_evt: EventFd,
+    pub ctrl_q: CtrlVirtio,
+    pub epoll_fd: RawFd,
+}
+
+impl NetCtrlEpollHandler {
+    pub fn run_ctrl(&mut self, paused: Arc<AtomicBool>) -> std::result::Result<(), DeviceError> {
+        // Create the epoll file descriptor
+        self.epoll_fd = epoll::create(true).map_err(DeviceError::EpollCreateFd)?;
+
+        register_listener(
+            self.epoll_fd,
+            self.ctrl_q.queue_evt.as_raw_fd(),
+            epoll::Events::EPOLLIN,
+            u64::from(CTRL_QUEUE_EVENT),
+        )
+        .unwrap();
+        register_listener(
+            self.epoll_fd,
+            self.kill_evt.as_raw_fd(),
+            epoll::Events::EPOLLIN,
+            u64::from(KILL_EVENT),
+        )
+        .unwrap();
+        register_listener(
+            self.epoll_fd,
+            self.pause_evt.as_raw_fd(),
+            epoll::Events::EPOLLIN,
+            u64::from(PAUSE_EVENT),
+        )
+        .unwrap();
+
+        let mut events = vec![epoll::Event::new(epoll::Events::empty(), 0); CTRL_EVENT_COUNT];
+
+        'epoll: loop {
+            let num_events = match epoll::wait(self.epoll_fd, -1, &mut events[..]) {
+                Ok(res) => res,
+                Err(e) => {
+                    if e.kind() == std::io::ErrorKind::Interrupted {
+                        continue;
+                    }
+                    return Err(DeviceError::EpollWait(e));
+                }
+            };
+
+            for event in events.iter().take(num_events) {
+                let ev_type = event.data as u16;
+
+                match ev_type {
+                    CTRL_QUEUE_EVENT => {
+                        let mem = self.mem.load();
+                        if let Err(e) = self.ctrl_q.queue_evt.read() {
+                            error!("failed to get ctl queue event: {:?}", e);
+                        }
+                        if let Err(e) = self.ctrl_q.process_cvq(&mem) {
+                            error!("failed to process ctrl queue: {:?}", e);
+                        }
+                    }
+                    KILL_EVENT => {
+                        break 'epoll;
+                    }
+                    PAUSE_EVENT => {
+                        debug!("PAUSE_EVENT received, pausing vhost-user epoll loop");
+                        // We loop here to handle spurious park() returns.
+                        // Until we have not resumed, the paused boolean will
+                        // be true.
+                        while paused.load(Ordering::SeqCst) {
+                            std::thread::park();
+                        }
+                    }
+                    _ => {
+                        error!("Unknown event for virtio-net");
+                    }
+                }
+            }
+        }
+
+        Ok(())
+    }
+}
+
+#[derive(Clone)]
+pub struct TxVirtio {
+    pub iovec: Vec<(GuestAddress, usize)>,
+    pub frame_buf: [u8; MAX_BUFFER_SIZE],
+}
+
+impl Default for TxVirtio {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl TxVirtio {
+    pub fn new() -> Self {
+        TxVirtio {
+            iovec: Vec::new(),
+            frame_buf: [0u8; MAX_BUFFER_SIZE],
+        }
+    }
+
+    pub fn process_desc_chain(&mut self, mem: &GuestMemoryMmap, tap: &mut Tap, queue: &mut Queue) {
+        while let Some(avail_desc) = queue.iter(&mem).next() {
+            let head_index = avail_desc.index;
+            let mut read_count = 0;
+            let mut next_desc = Some(avail_desc);
+
+            self.iovec.clear();
+            while let Some(desc) = next_desc {
+                if desc.is_write_only() {
+                    break;
+                }
+                self.iovec.push((desc.addr, desc.len as usize));
+                read_count += desc.len as usize;
+                next_desc = desc.next_descriptor();
+            }
+
+            read_count = 0;
+            // Copy buffer from across multiple descriptors.
+            // TODO(performance - Issue #420): change this to use `writev()` instead of `write()`
+            // and get rid of the intermediate buffer.
+            for (desc_addr, desc_len) in self.iovec.drain(..) {
+                let limit = cmp::min((read_count + desc_len) as usize, self.frame_buf.len());
+
+                let read_result =
+                    mem.read_slice(&mut self.frame_buf[read_count..limit as usize], desc_addr);
+                match read_result {
+                    Ok(_) => {
+                        // Increment by number of bytes actually read
+                        read_count += limit - read_count;
+                    }
+                    Err(e) => {
+                        println!("Failed to read slice: {:?}", e);
+                        break;
+                    }
+                }
+            }
+
+            let write_result = tap.write(&self.frame_buf[..read_count]);
+            match write_result {
+                Ok(_) => {}
+                Err(e) => {
+                    println!("net: tx: error failed to write to tap: {}", e);
+                }
+            };
+            queue.add_used(&mem, head_index, 0);
+        }
+    }
+}
+
+#[derive(Clone)]
+pub struct RxVirtio {
+    pub deferred_frame: bool,
+    pub deferred_irqs: bool,
+    pub bytes_read: usize,
+    pub frame_buf: [u8; MAX_BUFFER_SIZE],
+}
+
+impl Default for RxVirtio {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl RxVirtio {
+    pub fn new() -> Self {
+        RxVirtio {
+            deferred_frame: false,
+            deferred_irqs: false,
+            bytes_read: 0,
+            frame_buf: [0u8; MAX_BUFFER_SIZE],
+        }
+    }
+
+    pub fn process_desc_chain(
+        &mut self,
+        mem: &GuestMemoryMmap,
+        mut next_desc: Option<DescriptorChain>,
+        queue: &mut Queue,
+    ) -> bool {
+        let head_index = next_desc.as_ref().unwrap().index;
+        let mut write_count = 0;
+
+        // Copy from frame into buffer, which may span multiple descriptors.
+        loop {
+            match next_desc {
+                Some(desc) => {
+                    if !desc.is_write_only() {
+                        break;
+                    }
+                    let limit = cmp::min(write_count + desc.len as usize, self.bytes_read);
+                    let source_slice = &self.frame_buf[write_count..limit];
+                    let write_result = mem.write_slice(source_slice, desc.addr);
+
+                    match write_result {
+                        Ok(_) => {
+                            write_count = limit;
+                        }
+                        Err(e) => {
+                            error!("Failed to write slice: {:?}", e);
+                            break;
+                        }
+                    };
+
+                    if write_count >= self.bytes_read {
+                        break;
+                    }
+                    next_desc = desc.next_descriptor();
+                }
+                None => {
+                    warn!("Receiving buffer is too small to hold frame of current size");
+                    break;
+                }
+            }
+        }
+
+        queue.add_used(&mem, head_index, write_count as u32);
+
+        // Mark that we have at least one pending packet and we need to interrupt the guest.
+        self.deferred_irqs = true;
+
+        write_count >= self.bytes_read
+    }
+}
+
+pub fn build_net_config_space(
+    mut config: &mut VirtioNetConfig,
+    mac: MacAddr,
+    num_queues: usize,
+    mut avail_features: &mut u64,
+) {
+    config.mac.copy_from_slice(mac.get_bytes());
+    *avail_features |= 1 << VIRTIO_NET_F_MAC;
+
+    build_net_config_space_with_mq(&mut config, num_queues, &mut avail_features);
+}
+
+pub fn build_net_config_space_with_mq(
+    config: &mut VirtioNetConfig,
+    num_queues: usize,
+    avail_features: &mut u64,
+) {
+    let num_queue_pairs = (num_queues / 2) as u16;
+    if (num_queue_pairs >= VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MIN as u16)
+        && (num_queue_pairs <= VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX as u16)
+    {
+        config.max_virtqueue_pairs = num_queue_pairs;
+        *avail_features |= 1 << VIRTIO_NET_F_MQ;
+    }
+}
+
+fn vnet_hdr_len() -> usize {
+    mem::size_of::<virtio_net_hdr_v1>()
+}
+
+/// Create a new virtio network device with the given IP address and
+/// netmask.
+pub fn open_tap(
+    if_name: Option<&str>,
+    ip_addr: Option<Ipv4Addr>,
+    netmask: Option<Ipv4Addr>,
+    num_rx_q: usize,
+) -> Result<Vec<Tap>> {
+    let mut taps: Vec<Tap> = Vec::new();
+    let mut ifname: String = String::new();
+    let vnet_hdr_size = vnet_hdr_len() as i32;
+    let flag = net_gen::TUN_F_CSUM | net_gen::TUN_F_UFO | net_gen::TUN_F_TSO4 | net_gen::TUN_F_TSO6;
+
+    for i in 0..num_rx_q {
+        let tap: Tap;
+        if i == 0 {
+            tap = match if_name {
+                Some(name) => Tap::open_named(name, num_rx_q).map_err(Error::TapOpen)?,
+                None => Tap::new(num_rx_q).map_err(Error::TapOpen)?,
+            };
+            if let Some(ip) = ip_addr {
+                tap.set_ip_addr(ip).map_err(Error::TapSetIp)?;
+            }
+            if let Some(mask) = netmask {
+                tap.set_netmask(mask).map_err(Error::TapSetNetmask)?;
+            }
+            tap.enable().map_err(Error::TapEnable)?;
+            tap.set_offload(flag).map_err(Error::TapSetOffload)?;
+
+            tap.set_vnet_hdr_size(vnet_hdr_size)
+                .map_err(Error::TapSetVnetHdrSize)?;
+
+            ifname = String::from_utf8(tap.get_if_name()).unwrap();
+        } else {
+            tap = Tap::open_named(ifname.as_str(), num_rx_q).map_err(Error::TapOpen)?;
+            tap.set_offload(flag).map_err(Error::TapSetOffload)?;
+
+            tap.set_vnet_hdr_size(vnet_hdr_size)
+                .map_err(Error::TapSetVnetHdrSize)?;
+        }
+        taps.push(tap);
+    }
+    Ok(taps)
+}

vm-virtio/src/pmem.rs

@@ -6,6 +6,13 @@
 //
 // SPDX-License-Identifier: Apache-2.0 AND BSD-3-Clause
 
+use super::Error as DeviceError;
+use super::{
+    ActivateError, ActivateResult, DescriptorChain, DeviceEventT, Queue, VirtioDevice,
+    VirtioDeviceType, VIRTIO_F_IOMMU_PLATFORM, VIRTIO_F_VERSION_1,
+};
+use crate::{VirtioInterrupt, VirtioInterruptType};
+use arc_swap::ArcSwap;
 use epoll;
 use libc::EFD_NONBLOCK;
 use std::cmp;
@@ -16,15 +23,8 @@ use std::mem::size_of;
 use std::os::unix::io::AsRawFd;
 use std::result;
 use std::sync::atomic::{AtomicBool, Ordering};
-use std::sync::{Arc, RwLock};
+use std::sync::Arc;
 use std::thread;
-
-use super::Error as DeviceError;
-use super::{
-    ActivateError, ActivateResult, DescriptorChain, DeviceEventT, Queue, VirtioDevice,
-    VirtioDeviceType, VIRTIO_F_IOMMU_PLATFORM, VIRTIO_F_VERSION_1,
-};
-use crate::{VirtioInterrupt, VirtioInterruptType};
 use vm_device::{Migratable, MigratableError, Pausable, Snapshotable};
 use vm_memory::{
     Address, ByteValued, Bytes, GuestAddress, GuestMemoryError, GuestMemoryMmap, GuestUsize,
@@ -158,9 +158,9 @@ impl Request {
 
 struct PmemEpollHandler {
     queue: Queue,
-    mem: Arc<RwLock<GuestMemoryMmap>>,
+    mem: Arc<ArcSwap<GuestMemoryMmap>>,
     disk: File,
-    interrupt_cb: Arc<VirtioInterrupt>,
+    interrupt_cb: Arc<dyn VirtioInterrupt>,
     queue_evt: EventFd,
     kill_evt: EventFd,
     pause_evt: EventFd,
@@ -170,7 +170,7 @@ impl PmemEpollHandler {
     fn process_queue(&mut self) -> bool {
         let mut used_desc_heads = [(0, 0); QUEUE_SIZE as usize];
         let mut used_count = 0;
-        let mem = self.mem.read().unwrap();
+        let mem = self.mem.load();
         for avail_desc in self.queue.iter(&mem) {
             let len = match Request::parse(&avail_desc, &mem) {
                 Ok(ref req) if (req.type_ == RequestType::Flush) => {
@@ -213,10 +213,12 @@ impl PmemEpollHandler {
     }
 
     fn signal_used_queue(&self) -> result::Result<(), DeviceError> {
-        (self.interrupt_cb)(&VirtioInterruptType::Queue, Some(&self.queue)).map_err(|e| {
-            error!("Failed to signal used queue: {:?}", e);
-            DeviceError::FailedSignalingUsedQueue(e)
-        })
+        self.interrupt_cb
+            .trigger(&VirtioInterruptType::Queue, Some(&self.queue))
+            .map_err(|e| {
+                error!("Failed to signal used queue: {:?}", e);
+                DeviceError::FailedSignalingUsedQueue(e)
+            })
     }
 
     fn run(&mut self, paused: Arc<AtomicBool>) -> result::Result<(), DeviceError> {
@@ -315,8 +317,8 @@ pub struct Pmem {
     acked_features: u64,
     config: VirtioPmemConfig,
     queue_evts: Option<Vec<EventFd>>,
-    interrupt_cb: Option<Arc<VirtioInterrupt>>,
-    epoll_thread: Option<thread::JoinHandle<result::Result<(), DeviceError>>>,
+    interrupt_cb: Option<Arc<dyn VirtioInterrupt>>,
+    epoll_threads: Option<Vec<thread::JoinHandle<result::Result<(), DeviceError>>>>,
     paused: Arc<AtomicBool>,
 }
 
@@ -342,7 +344,7 @@ impl Pmem {
             config,
             queue_evts: None,
             interrupt_cb: None,
-            epoll_thread: None,
+            epoll_threads: None,
             paused: Arc::new(AtomicBool::new(false)),
         })
     }
@@ -366,29 +368,12 @@ impl VirtioDevice for Pmem {
         QUEUE_SIZES
     }
 
-    fn features(&self, page: u32) -> u32 {
-        match page {
-            // Get the lower 32-bits of the features bitfield.
-            0 => self.avail_features as u32,
-            // Get the upper 32-bits of the features bitfield.
-            1 => (self.avail_features >> 32) as u32,
-            _ => {
-                warn!("Received request for unknown features page.");
-                0u32
-            }
-        }
+    fn features(&self) -> u64 {
+        self.avail_features
     }
 
-    fn ack_features(&mut self, page: u32, value: u32) {
-        let mut v = match page {
-            0 => u64::from(value),
-            1 => u64::from(value) << 32,
-            _ => {
-                warn!("Cannot acknowledge unknown features page.");
-                0u64
-            }
-        };
-
+    fn ack_features(&mut self, value: u64) {
+        let mut v = value;
         // Check if the guest is ACK'ing a feature that we didn't claim to have.
         let unrequested_features = v & !self.avail_features;
         if unrequested_features != 0 {
@@ -421,8 +406,8 @@ impl VirtioDevice for Pmem {
 
     fn activate(
         &mut self,
-        mem: Arc<RwLock<GuestMemoryMmap>>,
-        interrupt_cb: Arc<VirtioInterrupt>,
+        mem: Arc<ArcSwap<GuestMemoryMmap>>,
+        interrupt_cb: Arc<dyn VirtioInterrupt>,
         mut queues: Vec<Queue>,
         mut queue_evts: Vec<EventFd>,
     ) -> ActivateResult {
@@ -482,21 +467,24 @@ impl VirtioDevice for Pmem {
             };
 
             let paused = self.paused.clone();
+            let mut epoll_threads = Vec::new();
             thread::Builder::new()
                 .name("virtio_pmem".to_string())
                 .spawn(move || handler.run(paused))
-                .map(|thread| self.epoll_thread = Some(thread))
+                .map(|thread| epoll_threads.push(thread))
                 .map_err(|e| {
                     error!("failed to clone virtio-pmem epoll thread: {}", e);
                     ActivateError::BadActivate
                 })?;
 
+            self.epoll_threads = Some(epoll_threads);
+
             return Ok(());
         }
         Err(ActivateError::BadActivate)
     }
 
-    fn reset(&mut self) -> Option<(Arc<VirtioInterrupt>, Vec<EventFd>)> {
+    fn reset(&mut self) -> Option<(Arc<dyn VirtioInterrupt>, Vec<EventFd>)> {
         // We first must resume the virtio thread if it was paused.
         if self.pause_evt.take().is_some() {
             self.resume().ok()?;

vm-virtio/src/queue.rs

@@ -731,7 +731,7 @@ pub(crate) mod tests {
 
     #[test]
     fn test_checked_new_descriptor_chain() {
-        let m = &GuestMemoryMmap::new(&[(GuestAddress(0), 0x10000)]).unwrap();
+        let m = &GuestMemoryMmap::from_ranges(&[(GuestAddress(0), 0x10000)]).unwrap();
         let vq = VirtQueue::new(GuestAddress(0), m, 16);
 
         assert!(vq.end().0 < 0x1000);
@@ -792,7 +792,7 @@ pub(crate) mod tests {
 
     #[test]
     fn test_queue_and_iterator() {
-        let m = &GuestMemoryMmap::new(&[(GuestAddress(0), 0x10000)]).unwrap();
+        let m = &GuestMemoryMmap::from_ranges(&[(GuestAddress(0), 0x10000)]).unwrap();
         let vq = VirtQueue::new(GuestAddress(0), m, 16);
 
         let mut q = vq.create_queue();
@@ -897,7 +897,7 @@ pub(crate) mod tests {
 
     #[test]
     fn test_add_used() {
-        let m = &GuestMemoryMmap::new(&[(GuestAddress(0), 0x10000)]).unwrap();
+        let m = &GuestMemoryMmap::from_ranges(&[(GuestAddress(0), 0x10000)]).unwrap();
         let vq = VirtQueue::new(GuestAddress(0), m, 16);
 
         let mut q = vq.create_queue();

vm-virtio/src/rng.rs

@@ -2,6 +2,13 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
+use super::Error as DeviceError;
+use super::{
+    ActivateError, ActivateResult, DeviceEventT, Queue, VirtioDevice, VirtioDeviceType,
+    VIRTIO_F_IOMMU_PLATFORM, VIRTIO_F_VERSION_1,
+};
+use crate::{VirtioInterrupt, VirtioInterruptType};
+use arc_swap::ArcSwap;
 use epoll;
 use libc::EFD_NONBLOCK;
 use std;
@@ -10,15 +17,8 @@ use std::io;
 use std::os::unix::io::AsRawFd;
 use std::result;
 use std::sync::atomic::{AtomicBool, Ordering};
-use std::sync::{Arc, RwLock};
+use std::sync::Arc;
 use std::thread;
-
-use super::Error as DeviceError;
-use super::{
-    ActivateError, ActivateResult, DeviceEventT, Queue, VirtioDevice, VirtioDeviceType,
-    VIRTIO_F_IOMMU_PLATFORM, VIRTIO_F_VERSION_1,
-};
-use crate::{VirtioInterrupt, VirtioInterruptType};
 use vm_device::{Migratable, MigratableError, Pausable, Snapshotable};
 use vm_memory::{Bytes, GuestMemoryMmap};
 use vmm_sys_util::eventfd::EventFd;
@@ -36,9 +36,9 @@ const PAUSE_EVENT: DeviceEventT = 2;
 
 struct RngEpollHandler {
     queues: Vec<Queue>,
-    mem: Arc<RwLock<GuestMemoryMmap>>,
+    mem: Arc<ArcSwap<GuestMemoryMmap>>,
     random_file: File,
-    interrupt_cb: Arc<VirtioInterrupt>,
+    interrupt_cb: Arc<dyn VirtioInterrupt>,
     queue_evt: EventFd,
     kill_evt: EventFd,
     pause_evt: EventFd,
@@ -50,7 +50,7 @@ impl RngEpollHandler {
 
         let mut used_desc_heads = [(0, 0); QUEUE_SIZE as usize];
         let mut used_count = 0;
-        let mem = self.mem.read().unwrap();
+        let mem = self.mem.load();
         for avail_desc in queue.iter(&mem) {
             let mut len = 0;
 
@@ -80,10 +80,12 @@ impl RngEpollHandler {
     }
 
     fn signal_used_queue(&self) -> result::Result<(), DeviceError> {
-        (self.interrupt_cb)(&VirtioInterruptType::Queue, Some(&self.queues[0])).map_err(|e| {
-            error!("Failed to signal used queue: {:?}", e);
-            DeviceError::FailedSignalingUsedQueue(e)
-        })
+        self.interrupt_cb
+            .trigger(&VirtioInterruptType::Queue, Some(&self.queues[0]))
+            .map_err(|e| {
+                error!("Failed to signal used queue: {:?}", e);
+                DeviceError::FailedSignalingUsedQueue(e)
+            })
     }
 
     fn run(&mut self, paused: Arc<AtomicBool>) -> result::Result<(), DeviceError> {
@@ -181,8 +183,8 @@ pub struct Rng {
     avail_features: u64,
     acked_features: u64,
     queue_evts: Option<Vec<EventFd>>,
-    interrupt_cb: Option<Arc<VirtioInterrupt>>,
-    epoll_thread: Option<thread::JoinHandle<result::Result<(), DeviceError>>>,
+    interrupt_cb: Option<Arc<dyn VirtioInterrupt>>,
+    epoll_threads: Option<Vec<thread::JoinHandle<result::Result<(), DeviceError>>>>,
     paused: Arc<AtomicBool>,
 }
 
@@ -204,7 +206,7 @@ impl Rng {
             acked_features: 0u64,
             queue_evts: None,
             interrupt_cb: None,
-            epoll_thread: None,
+            epoll_threads: None,
             paused: Arc::new(AtomicBool::new(false)),
         })
     }
@@ -228,29 +230,12 @@ impl VirtioDevice for Rng {
         QUEUE_SIZES
     }
 
-    fn features(&self, page: u32) -> u32 {
-        match page {
-            // Get the lower 32-bits of the features bitfield.
-            0 => self.avail_features as u32,
-            // Get the upper 32-bits of the features bitfield.
-            1 => (self.avail_features >> 32) as u32,
-            _ => {
-                warn!("Received request for unknown features page.");
-                0u32
-            }
-        }
+    fn features(&self) -> u64 {
+        self.avail_features
     }
 
-    fn ack_features(&mut self, page: u32, value: u32) {
-        let mut v = match page {
-            0 => u64::from(value),
-            1 => u64::from(value) << 32,
-            _ => {
-                warn!("Cannot acknowledge unknown features page.");
-                0u64
-            }
-        };
-
+    fn ack_features(&mut self, value: u64) {
+        let mut v = value;
         // Check if the guest is ACK'ing a feature that we didn't claim to have.
         let unrequested_features = v & !self.avail_features;
         if unrequested_features != 0 {
@@ -272,8 +257,8 @@ impl VirtioDevice for Rng {
 
     fn activate(
         &mut self,
-        mem: Arc<RwLock<GuestMemoryMmap>>,
-        interrupt_cb: Arc<VirtioInterrupt>,
+        mem: Arc<ArcSwap<GuestMemoryMmap>>,
+        interrupt_cb: Arc<dyn VirtioInterrupt>,
         queues: Vec<Queue>,
         mut queue_evts: Vec<EventFd>,
     ) -> ActivateResult {
@@ -333,21 +318,24 @@ impl VirtioDevice for Rng {
             };
 
             let paused = self.paused.clone();
+            let mut epoll_threads = Vec::new();
             thread::Builder::new()
                 .name("virtio_rng".to_string())
                 .spawn(move || handler.run(paused))
-                .map(|thread| self.epoll_thread = Some(thread))
+                .map(|thread| epoll_threads.push(thread))
                 .map_err(|e| {
                     error!("failed to clone the virtio-rng epoll thread: {}", e);
                     ActivateError::BadActivate
                 })?;
 
+            self.epoll_threads = Some(epoll_threads);
+
             return Ok(());
         }
         Err(ActivateError::BadActivate)
     }
 
-    fn reset(&mut self) -> Option<(Arc<VirtioInterrupt>, Vec<EventFd>)> {
+    fn reset(&mut self) -> Option<(Arc<dyn VirtioInterrupt>, Vec<EventFd>)> {
         // We first must resume the virtio thread if it was paused.
         if self.pause_evt.take().is_some() {
             self.resume().ok()?;

vm-virtio/src/transport/mmio.rs

@@ -2,20 +2,20 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
-use std::result;
-use std::sync::atomic::{AtomicUsize, Ordering};
-use std::sync::{Arc, Mutex, RwLock};
-
-use byteorder::{ByteOrder, LittleEndian};
-use libc::EFD_NONBLOCK;
-
 use crate::transport::{VirtioTransport, NOTIFY_REG_OFFSET};
 use crate::{
     Queue, VirtioDevice, VirtioInterrupt, VirtioInterruptType, DEVICE_ACKNOWLEDGE, DEVICE_DRIVER,
     DEVICE_DRIVER_OK, DEVICE_FAILED, DEVICE_FEATURES_OK, DEVICE_INIT,
     INTERRUPT_STATUS_CONFIG_CHANGED, INTERRUPT_STATUS_USED_RING,
 };
-use devices::{BusDevice, Interrupt};
+use arc_swap::ArcSwap;
+use byteorder::{ByteOrder, LittleEndian};
+use devices::BusDevice;
+use libc::EFD_NONBLOCK;
+use std::result;
+use std::sync::atomic::{AtomicUsize, Ordering};
+use std::sync::{Arc, Mutex};
+use vm_device::interrupt::InterruptSourceGroup;
 use vm_device::{Migratable, MigratableError, Pausable, Snapshotable};
 use vm_memory::{GuestAddress, GuestMemoryMmap};
 use vmm_sys_util::{errno::Result, eventfd::EventFd};
@@ -25,6 +25,40 @@ const VENDOR_ID: u32 = 0;
 const MMIO_MAGIC_VALUE: u32 = 0x7472_6976;
 const MMIO_VERSION: u32 = 2;
 
+pub struct VirtioInterruptIntx {
+    interrupt_status: Arc<AtomicUsize>,
+    interrupt: Arc<Box<dyn InterruptSourceGroup>>,
+}
+
+impl VirtioInterruptIntx {
+    pub fn new(
+        interrupt_status: Arc<AtomicUsize>,
+        interrupt: Arc<Box<dyn InterruptSourceGroup>>,
+    ) -> Self {
+        VirtioInterruptIntx {
+            interrupt_status,
+            interrupt,
+        }
+    }
+}
+
+impl VirtioInterrupt for VirtioInterruptIntx {
+    fn trigger(
+        &self,
+        int_type: &VirtioInterruptType,
+        _queue: Option<&Queue>,
+    ) -> std::result::Result<(), std::io::Error> {
+        let status = match int_type {
+            VirtioInterruptType::Config => INTERRUPT_STATUS_CONFIG_CHANGED,
+            VirtioInterruptType::Queue => INTERRUPT_STATUS_USED_RING,
+        };
+        self.interrupt_status
+            .fetch_or(status as usize, Ordering::SeqCst);
+
+        self.interrupt.trigger(0)
+    }
+}
+
 /// Implements the
 /// [MMIO](http://docs.oasis-open.org/virtio/virtio/v1.0/cs04/virtio-v1.0-cs04.html#x1-1090002)
 /// transport for virtio devices.
@@ -47,18 +81,18 @@ pub struct MmioDevice {
     acked_features_select: u32,
     queue_select: u32,
     interrupt_status: Arc<AtomicUsize>,
-    interrupt_cb: Option<Arc<VirtioInterrupt>>,
+    interrupt_cb: Option<Arc<dyn VirtioInterrupt>>,
     driver_status: u32,
     config_generation: u32,
     queues: Vec<Queue>,
     queue_evts: Vec<EventFd>,
-    mem: Option<Arc<RwLock<GuestMemoryMmap>>>,
+    mem: Option<Arc<ArcSwap<GuestMemoryMmap>>>,
 }
 
 impl MmioDevice {
     /// Constructs a new MMIO transport for the given virtio device.
     pub fn new(
-        mem: Arc<RwLock<GuestMemoryMmap>>,
+        mem: Arc<ArcSwap<GuestMemoryMmap>>,
         device: Arc<Mutex<dyn VirtioDevice>>,
     ) -> Result<MmioDevice> {
         let device_clone = device.clone();
@@ -102,7 +136,7 @@ impl MmioDevice {
 
     fn are_queues_valid(&self) -> bool {
         if let Some(mem) = self.mem.as_ref() {
-            self.queues.iter().all(|q| q.is_valid(&mem.read().unwrap()))
+            self.queues.iter().all(|q| q.is_valid(mem.load().as_ref()))
         } else {
             false
         }
@@ -127,21 +161,11 @@ impl MmioDevice {
         }
     }
 
-    pub fn assign_interrupt(&mut self, interrupt: Box<dyn Interrupt>) {
-        let interrupt_status = self.interrupt_status.clone();
-        let cb = Arc::new(Box::new(
-            move |int_type: &VirtioInterruptType, _queue: Option<&Queue>| {
-                let status = match int_type {
-                    VirtioInterruptType::Config => INTERRUPT_STATUS_CONFIG_CHANGED,
-                    VirtioInterruptType::Queue => INTERRUPT_STATUS_USED_RING,
-                };
-                interrupt_status.fetch_or(status as usize, Ordering::SeqCst);
-
-                interrupt.deliver()
-            },
-        ) as VirtioInterrupt);
-
-        self.interrupt_cb = Some(cb);
+    pub fn assign_interrupt(&mut self, interrupt: Arc<Box<dyn InterruptSourceGroup>>) {
+        self.interrupt_cb = Some(Arc::new(VirtioInterruptIntx::new(
+            self.interrupt_status.clone(),
+            interrupt,
+        )));
     }
 }
 
@@ -165,8 +189,12 @@ impl BusDevice for MmioDevice {
                     0x08 => self.device.lock().unwrap().device_type(),
                     0x0c => VENDOR_ID, // vendor id
                     0x10 => {
-                        self.device.lock().unwrap().features(self.features_select)
-                            | if self.features_select == 1 { 0x1 } else { 0x0 }
+                        if self.features_select < 2 {
+                            (self.device.lock().unwrap().features() >> (self.features_select * 32))
+                                as u32
+                        } else {
+                            0
+                        }
                     }
                     0x34 => self.with_queue(0, |q| u32::from(q.get_max_size())),
                     0x44 => self.with_queue(0, |q| q.ready as u32),
@@ -210,11 +238,19 @@ impl BusDevice for MmioDevice {
                 let v = LittleEndian::read_u32(data);
                 match offset {
                     0x14 => self.features_select = v,
-                    0x20 => self
-                        .device
-                        .lock()
-                        .unwrap()
-                        .ack_features(self.acked_features_select, v),
+                    0x20 => {
+                        if self.acked_features_select < 2 {
+                            self.device
+                                .lock()
+                                .unwrap()
+                                .ack_features(u64::from(v) << (self.acked_features_select * 32));
+                        } else {
+                            warn!(
+                                "invalid ack_features (page {}, value 0x{:x})",
+                                self.acked_features_select, v
+                            );
+                        }
+                    }
                     0x24 => self.acked_features_select = v,
                     0x30 => self.queue_select = v,
                     0x38 => mut_q = self.with_queue_mut(|q| q.size = v as u16),

vm-virtio/src/transport/pci_common_config.rs

@@ -7,13 +7,12 @@
 // SPDX-License-Identifier: Apache-2.0 AND BSD-3-Clause
 extern crate byteorder;
 
+use crate::{Queue, VirtioDevice};
 use byteorder::{ByteOrder, LittleEndian};
 use std::sync::atomic::{AtomicU16, Ordering};
 use std::sync::{Arc, Mutex};
 use vm_memory::GuestAddress;
 
-use crate::{Queue, VirtioDevice};
-
 /// Contains the data for reading and writing the common configuration structure of a virtio PCI
 /// device.
 ///
@@ -165,7 +164,7 @@ impl VirtioPciCommonConfig {
                 // Only 64 bits of features (2 pages) are defined for now, so limit
                 // device_feature_select to avoid shifting by 64 or more bits.
                 if self.device_feature_select < 2 {
-                    locked_device.features(self.device_feature_select)
+                    (locked_device.features() >> (self.device_feature_select * 32)) as u32
                 } else {
                     0
                 }
@@ -200,7 +199,8 @@ impl VirtioPciCommonConfig {
             0x0c => {
                 if self.driver_feature_select < 2 {
                     let mut locked_device = device.lock().unwrap();
-                    locked_device.ack_features(self.driver_feature_select, value);
+                    locked_device
+                        .ack_features(u64::from(value) << (self.driver_feature_select * 32));
                 } else {
                     warn!(
                         "invalid ack_features (page {}, value 0x{:x})",
@@ -255,8 +255,8 @@ impl VirtioPciCommonConfig {
 mod tests {
     use super::*;
     use crate::{ActivateResult, VirtioInterrupt};
-
-    use std::sync::{Arc, RwLock};
+    use arc_swap::ArcSwap;
+    use std::sync::Arc;
     use vm_memory::GuestMemoryMmap;
     use vmm_sys_util::eventfd::EventFd;
 
@@ -273,19 +273,19 @@ mod tests {
         }
         fn activate(
             &mut self,
-            _mem: Arc<RwLock<GuestMemoryMmap>>,
-            _interrupt_evt: Arc<VirtioInterrupt>,
+            _mem: Arc<ArcSwap<GuestMemoryMmap>>,
+            _interrupt_evt: Arc<dyn VirtioInterrupt>,
             _queues: Vec<Queue>,
             _queue_evts: Vec<EventFd>,
         ) -> ActivateResult {
             Ok(())
         }
 
-        fn features(&self, _page: u32) -> u32 {
-            DUMMY_FEATURES as u32
+        fn features(&self) -> u64 {
+            DUMMY_FEATURES
         }
 
-        fn ack_features(&mut self, _page: u32, _value: u32) {}
+        fn ack_features(&mut self, _value: u64) {}
 
         fn read_config(&self, _offset: u64, _data: &mut [u8]) {}
 
@@ -303,41 +303,41 @@ mod tests {
             msix_config: Arc::new(AtomicU16::new(0)),
         };
 
-        let dev = &mut DummyDevice(0) as &mut dyn VirtioDevice;
+        let dev = Arc::new(Mutex::new(DummyDevice(0)));
         let mut queues = Vec::new();
 
         // Can set all bits of driver_status.
-        regs.write(0x14, &[0x55], &mut queues, dev);
+        regs.write(0x14, &[0x55], &mut queues, dev.clone());
         let mut read_back = vec![0x00];
-        regs.read(0x14, &mut read_back, &mut queues, dev);
+        regs.read(0x14, &mut read_back, &mut queues, dev.clone());
         assert_eq!(read_back[0], 0x55);
 
         // The config generation register is read only.
-        regs.write(0x15, &[0xaa], &mut queues, dev);
+        regs.write(0x15, &[0xaa], &mut queues, dev.clone());
         let mut read_back = vec![0x00];
-        regs.read(0x15, &mut read_back, &mut queues, dev);
+        regs.read(0x15, &mut read_back, &mut queues, dev.clone());
         assert_eq!(read_back[0], 0x55);
 
         // Device features is read-only and passed through from the device.
-        regs.write(0x04, &[0, 0, 0, 0], &mut queues, dev);
+        regs.write(0x04, &[0, 0, 0, 0], &mut queues, dev.clone());
         let mut read_back = vec![0, 0, 0, 0];
-        regs.read(0x04, &mut read_back, &mut queues, dev);
+        regs.read(0x04, &mut read_back, &mut queues, dev.clone());
         assert_eq!(LittleEndian::read_u32(&read_back), DUMMY_FEATURES as u32);
 
         // Feature select registers are read/write.
-        regs.write(0x00, &[1, 2, 3, 4], &mut queues, dev);
+        regs.write(0x00, &[1, 2, 3, 4], &mut queues, dev.clone());
         let mut read_back = vec![0, 0, 0, 0];
-        regs.read(0x00, &mut read_back, &mut queues, dev);
+        regs.read(0x00, &mut read_back, &mut queues, dev.clone());
         assert_eq!(LittleEndian::read_u32(&read_back), 0x0403_0201);
-        regs.write(0x08, &[1, 2, 3, 4], &mut queues, dev);
+        regs.write(0x08, &[1, 2, 3, 4], &mut queues, dev.clone());
         let mut read_back = vec![0, 0, 0, 0];
-        regs.read(0x08, &mut read_back, &mut queues, dev);
+        regs.read(0x08, &mut read_back, &mut queues, dev.clone());
         assert_eq!(LittleEndian::read_u32(&read_back), 0x0403_0201);
 
         // 'queue_select' can be read and written.
-        regs.write(0x16, &[0xaa, 0x55], &mut queues, dev);
+        regs.write(0x16, &[0xaa, 0x55], &mut queues, dev.clone());
         let mut read_back = vec![0x00, 0x00];
-        regs.read(0x16, &mut read_back, &mut queues, dev);
+        regs.read(0x16, &mut read_back, &mut queues, dev.clone());
         assert_eq!(read_back[0], 0xaa);
         assert_eq!(read_back[1], 0x55);
     }

vm-virtio/src/transport/pci_device.rs

@@ -13,31 +13,34 @@ extern crate vm_allocator;
 extern crate vm_memory;
 extern crate vmm_sys_util;
 
-use libc::EFD_NONBLOCK;
-use std::any::Any;
-use std::result;
-use std::sync::atomic::{AtomicU16, AtomicUsize, Ordering};
-use std::sync::{Arc, Mutex, RwLock};
-
-use devices::BusDevice;
-use pci::{
-    BarReprogrammingParams, InterruptDelivery, InterruptParameters, MsixCap, MsixConfig,
-    PciBarConfiguration, PciBarRegionType, PciCapability, PciCapabilityID, PciClassCode,
-    PciConfiguration, PciDevice, PciDeviceError, PciHeaderType, PciInterruptPin,
-    PciMassStorageSubclass, PciNetworkControllerSubclass, PciSubclass,
-};
-use vm_allocator::SystemAllocator;
-use vm_device::{Migratable, MigratableError, Pausable, Snapshotable};
-use vm_memory::{Address, ByteValued, GuestAddress, GuestMemoryMmap, GuestUsize, Le32};
-use vmm_sys_util::{errno::Result, eventfd::EventFd};
-
 use super::VirtioPciCommonConfig;
 use crate::transport::VirtioTransport;
 use crate::{
     Queue, VirtioDevice, VirtioDeviceType, VirtioInterrupt, VirtioInterruptType,
     VirtioIommuRemapping, DEVICE_ACKNOWLEDGE, DEVICE_DRIVER, DEVICE_DRIVER_OK, DEVICE_FAILED,
-    DEVICE_FEATURES_OK, DEVICE_INIT, INTERRUPT_STATUS_CONFIG_CHANGED, INTERRUPT_STATUS_USED_RING,
+    DEVICE_FEATURES_OK, DEVICE_INIT, VIRTIO_MSI_NO_VECTOR,
 };
+use arc_swap::ArcSwap;
+use devices::BusDevice;
+use libc::EFD_NONBLOCK;
+use pci::{
+    BarReprogrammingParams, MsixCap, MsixConfig, PciBarConfiguration, PciBarRegionType,
+    PciCapability, PciCapabilityID, PciClassCode, PciConfiguration, PciDevice, PciDeviceError,
+    PciHeaderType, PciMassStorageSubclass, PciNetworkControllerSubclass, PciSubclass,
+};
+use std::any::Any;
+use std::cmp;
+use std::io::Write;
+use std::result;
+use std::sync::atomic::{AtomicU16, AtomicUsize, Ordering};
+use std::sync::{Arc, Mutex};
+use vm_allocator::SystemAllocator;
+use vm_device::interrupt::{
+    InterruptIndex, InterruptManager, InterruptSourceGroup, MsiIrqGroupConfig,
+};
+use vm_device::{Migratable, MigratableError, Pausable, Snapshotable};
+use vm_memory::{Address, ByteValued, GuestAddress, GuestMemoryMmap, GuestUsize, Le32};
+use vmm_sys_util::{errno::Result, eventfd::EventFd};
 
 #[allow(clippy::enum_variant_names)]
 enum PciCapabilityType {
@@ -49,6 +52,11 @@ enum PciCapabilityType {
     SharedMemoryConfig = 8,
 }
 
+// This offset represents the 2 bytes omitted from the VirtioPciCap structure
+// as they are already handled through add_capability(). These 2 bytes are the
+// fields cap_vndr (1 byte) and cap_next (1 byte) defined in the virtio spec.
+const VIRTIO_PCI_CAP_OFFSET: usize = 2;
+
 #[allow(dead_code)]
 #[repr(packed)]
 #[derive(Clone, Copy, Default)]
@@ -173,6 +181,41 @@ impl VirtioPciCap64 {
     }
 }
 
+#[allow(dead_code)]
+#[repr(packed)]
+#[derive(Clone, Copy, Default)]
+struct VirtioPciCfgCap {
+    cap: VirtioPciCap,
+    pci_cfg_data: [u8; 4],
+}
+// It is safe to implement ByteValued. All members are simple numbers and any value is valid.
+unsafe impl ByteValued for VirtioPciCfgCap {}
+
+impl PciCapability for VirtioPciCfgCap {
+    fn bytes(&self) -> &[u8] {
+        self.as_slice()
+    }
+
+    fn id(&self) -> PciCapabilityID {
+        PciCapabilityID::VendorSpecific
+    }
+}
+
+impl VirtioPciCfgCap {
+    fn new() -> Self {
+        VirtioPciCfgCap {
+            cap: VirtioPciCap::new(PciCapabilityType::PciConfig, 0, 0, 0),
+            ..Default::default()
+        }
+    }
+}
+
+#[derive(Clone, Copy, Default)]
+struct VirtioPciCfgCapInfo {
+    offset: usize,
+    cap: VirtioPciCfgCap,
+}
+
 #[allow(dead_code)]
 #[derive(Copy, Clone)]
 pub enum PciVirtioSubclass {
@@ -232,29 +275,39 @@ pub struct VirtioPciDevice {
 
     // PCI interrupts.
     interrupt_status: Arc<AtomicUsize>,
-    interrupt_cb: Option<Arc<VirtioInterrupt>>,
+    virtio_interrupt: Option<Arc<dyn VirtioInterrupt>>,
+    interrupt_source_group: Arc<Box<dyn InterruptSourceGroup>>,
 
     // virtio queues
     queues: Vec<Queue>,
     queue_evts: Vec<EventFd>,
 
     // Guest memory
-    memory: Option<Arc<RwLock<GuestMemoryMmap>>>,
+    memory: Option<Arc<ArcSwap<GuestMemoryMmap>>>,
 
     // Setting PCI BAR
     settings_bar: u8,
 
     // Whether to use 64-bit bar location or 32-bit
     use_64bit_bar: bool,
+
+    // Add a dedicated structure to hold information about the very specific
+    // virtio-pci capability VIRTIO_PCI_CAP_PCI_CFG. This is needed to support
+    // the legacy/backward compatible mechanism of letting the guest access the
+    // other virtio capabilities without mapping the PCI BARs. This can be
+    // needed when the guest tries to early access the virtio configuration of
+    // a device.
+    cap_pci_cfg_info: VirtioPciCfgCapInfo,
 }
 
 impl VirtioPciDevice {
     /// Constructs a new PCI transport for the given virtio device.
     pub fn new(
-        memory: Arc<RwLock<GuestMemoryMmap>>,
+        memory: Arc<ArcSwap<GuestMemoryMmap>>,
         device: Arc<Mutex<dyn VirtioDevice>>,
         msix_num: u16,
         iommu_mapping_cb: Option<Arc<VirtioIommuRemapping>>,
+        interrupt_manager: &Arc<dyn InterruptManager<GroupConfig = MsiIrqGroupConfig>>,
     ) -> Result<Self> {
         let device_clone = device.clone();
         let locked_device = device_clone.lock().unwrap();
@@ -274,8 +327,16 @@ impl VirtioPciDevice {
 
         let pci_device_id = VIRTIO_PCI_DEVICE_ID_BASE + locked_device.device_type() as u16;
 
+        let interrupt_source_group = interrupt_manager.create_group(MsiIrqGroupConfig {
+            base: 0,
+            count: msix_num as InterruptIndex,
+        })?;
+
         let (msix_config, msix_config_clone) = if msix_num > 0 {
-            let msix_config = Arc::new(Mutex::new(MsixConfig::new(msix_num)));
+            let msix_config = Arc::new(Mutex::new(MsixConfig::new(
+                msix_num,
+                interrupt_source_group.clone(),
+            )));
             let msix_config_clone = msix_config.clone();
             (Some(msix_config), Some(msix_config_clone))
         } else {
@@ -316,7 +377,7 @@ impl VirtioPciDevice {
             msix_config_clone,
         );
 
-        Ok(VirtioPciDevice {
+        let mut virtio_pci_device = VirtioPciDevice {
             configuration,
             common_config: VirtioPciCommonConfig {
                 driver_status: 0,
@@ -331,13 +392,25 @@ impl VirtioPciDevice {
             device,
             device_activated: false,
             interrupt_status: Arc::new(AtomicUsize::new(0)),
-            interrupt_cb: None,
+            virtio_interrupt: None,
             queues,
             queue_evts,
             memory: Some(memory),
             settings_bar: 0,
             use_64bit_bar,
-        })
+            interrupt_source_group,
+            cap_pci_cfg_info: VirtioPciCfgCapInfo::default(),
+        };
+
+        if let Some(msix_config) = &virtio_pci_device.msix_config {
+            virtio_pci_device.virtio_interrupt = Some(Arc::new(VirtioInterruptMsix::new(
+                msix_config.clone(),
+                virtio_pci_device.common_config.msix_config.clone(),
+                virtio_pci_device.interrupt_source_group.clone(),
+            )));
+        }
+
+        Ok(virtio_pci_device)
     }
 
     /// Gets the list of queue events that must be triggered whenever the VM writes to
@@ -361,7 +434,7 @@ impl VirtioPciDevice {
 
     fn are_queues_valid(&self) -> bool {
         if let Some(mem) = self.memory.as_ref() {
-            self.queues.iter().all(|q| q.is_valid(&mem.read().unwrap()))
+            self.queues.iter().all(|q| q.is_valid(mem.load().as_ref()))
         } else {
             false
         }
@@ -418,11 +491,13 @@ impl VirtioPciDevice {
             .add_capability(&notify_cap)
             .map_err(PciDeviceError::CapabilitiesSetup)?;
 
-        //TODO(dgreid) - How will the configuration_cap work?
-        let configuration_cap = VirtioPciCap::new(PciCapabilityType::PciConfig, 0, 0, 0);
-        self.configuration
+        let configuration_cap = VirtioPciCfgCap::new();
+        self.cap_pci_cfg_info.offset = self
+            .configuration
             .add_capability(&configuration_cap)
-            .map_err(PciDeviceError::CapabilitiesSetup)?;
+            .map_err(PciDeviceError::CapabilitiesSetup)?
+            + VIRTIO_PCI_CAP_OFFSET;
+        self.cap_pci_cfg_info.cap = configuration_cap;
 
         if self.msix_config.is_some() {
             let msix_cap = MsixCap::new(
@@ -440,6 +515,49 @@ impl VirtioPciDevice {
         self.settings_bar = settings_bar;
         Ok(())
     }
+
+    fn read_cap_pci_cfg(&mut self, offset: usize, mut data: &mut [u8]) {
+        let cap_slice = self.cap_pci_cfg_info.cap.as_slice();
+        let data_len = data.len();
+        let cap_len = cap_slice.len();
+        if offset + data_len > cap_len {
+            error!("Failed to read cap_pci_cfg from config space");
+            return;
+        }
+
+        if offset < std::mem::size_of::<VirtioPciCap>() {
+            if let Some(end) = offset.checked_add(data_len) {
+                // This write can't fail, offset and end are checked against config_len.
+                data.write_all(&cap_slice[offset..cmp::min(end, cap_len)])
+                    .unwrap();
+            }
+        } else {
+            // Safe since we know self.cap_pci_cfg_info.cap.cap.offset is 32bits long.
+            let bar_offset: u32 =
+                unsafe { std::mem::transmute(self.cap_pci_cfg_info.cap.cap.offset) };
+            self.read_bar(0, bar_offset as u64, data)
+        }
+    }
+
+    fn write_cap_pci_cfg(&mut self, offset: usize, data: &[u8]) {
+        let cap_slice = self.cap_pci_cfg_info.cap.as_mut_slice();
+        let data_len = data.len();
+        let cap_len = cap_slice.len();
+        if offset + data_len > cap_len {
+            error!("Failed to write cap_pci_cfg to config space");
+            return;
+        }
+
+        if offset < std::mem::size_of::<VirtioPciCap>() {
+            let (_, right) = cap_slice.split_at_mut(offset);
+            right[..data_len].copy_from_slice(&data[..]);
+        } else {
+            // Safe since we know self.cap_pci_cfg_info.cap.cap.offset is 32bits long.
+            let bar_offset: u32 =
+                unsafe { std::mem::transmute(self.cap_pci_cfg_info.cap.cap.offset) };
+            self.write_bar(0, bar_offset as u64, data)
+        }
+    }
 }
 
 impl VirtioTransport for VirtioPciDevice {
@@ -458,92 +576,113 @@ impl VirtioTransport for VirtioPciDevice {
     }
 }
 
-impl PciDevice for VirtioPciDevice {
-    fn assign_pin_irq(
-        &mut self,
-        irq_cb: Arc<InterruptDelivery>,
-        irq_num: u32,
-        irq_pin: PciInterruptPin,
-    ) {
-        self.configuration.set_irq(irq_num as u8, irq_pin);
+pub struct VirtioInterruptMsix {
+    msix_config: Arc<Mutex<MsixConfig>>,
+    config_vector: Arc<AtomicU16>,
+    interrupt_source_group: Arc<Box<dyn InterruptSourceGroup>>,
+}
 
-        let interrupt_status = self.interrupt_status.clone();
-        let cb = Arc::new(Box::new(
-            move |int_type: &VirtioInterruptType, _queue: Option<&Queue>| {
-                let param = InterruptParameters { msix: None };
+impl VirtioInterruptMsix {
+    pub fn new(
+        msix_config: Arc<Mutex<MsixConfig>>,
+        config_vector: Arc<AtomicU16>,
+        interrupt_source_group: Arc<Box<dyn InterruptSourceGroup>>,
+    ) -> Self {
+        VirtioInterruptMsix {
+            msix_config,
+            config_vector,
+            interrupt_source_group,
+        }
+    }
+}
 
-                let status = match int_type {
-                    VirtioInterruptType::Config => INTERRUPT_STATUS_CONFIG_CHANGED,
-                    VirtioInterruptType::Queue => INTERRUPT_STATUS_USED_RING,
-                };
-                interrupt_status.fetch_or(status as usize, Ordering::SeqCst);
+impl VirtioInterrupt for VirtioInterruptMsix {
+    fn trigger(
+        &self,
+        int_type: &VirtioInterruptType,
+        queue: Option<&Queue>,
+    ) -> std::result::Result<(), std::io::Error> {
+        let vector = match int_type {
+            VirtioInterruptType::Config => self.config_vector.load(Ordering::SeqCst),
+            VirtioInterruptType::Queue => {
+                if let Some(q) = queue {
+                    q.vector
+                } else {
+                    0
+                }
+            }
+        };
 
-                (irq_cb)(param)
-            },
-        ) as VirtioInterrupt);
+        if vector == VIRTIO_MSI_NO_VECTOR {
+            return Ok(());
+        }
 
-        self.interrupt_cb = Some(cb);
+        let config = &mut self.msix_config.lock().unwrap();
+        let entry = &config.table_entries[vector as usize];
+        // In case the vector control register associated with the entry
+        // has its first bit set, this means the vector is masked and the
+        // device should not inject the interrupt.
+        // Instead, the Pending Bit Array table is updated to reflect there
+        // is a pending interrupt for this specific vector.
+        if config.masked() || entry.masked() {
+            config.set_pba_bit(vector, false);
+            return Ok(());
+        }
+
+        self.interrupt_source_group
+            .trigger(vector as InterruptIndex)
     }
 
-    fn assign_msix(&mut self, msi_cb: Arc<InterruptDelivery>) {
-        if let Some(msix_config) = &self.msix_config {
-            msix_config
-                .lock()
-                .unwrap()
-                .register_interrupt_cb(msi_cb.clone());
+    fn notifier(&self, int_type: &VirtioInterruptType, queue: Option<&Queue>) -> Option<&EventFd> {
+        let vector = match int_type {
+            VirtioInterruptType::Config => self.config_vector.load(Ordering::SeqCst),
+            VirtioInterruptType::Queue => {
+                if let Some(q) = queue {
+                    q.vector
+                } else {
+                    0
+                }
+            }
+        };
 
-            let msix_config_clone = msix_config.clone();
+        self.interrupt_source_group
+            .notifier(vector as InterruptIndex)
+    }
+}
 
-            let common_config_msi_vector = self.common_config.msix_config.clone();
-            let cb = Arc::new(Box::new(
-                move |int_type: &VirtioInterruptType, queue: Option<&Queue>| {
-                    let vector = match int_type {
-                        VirtioInterruptType::Config => {
-                            common_config_msi_vector.load(Ordering::SeqCst)
-                        }
-                        VirtioInterruptType::Queue => {
-                            if let Some(q) = queue {
-                                q.vector
-                            } else {
-                                0
-                            }
-                        }
-                    };
-
-                    let config = &mut msix_config_clone.lock().unwrap();
-                    let entry = &config.table_entries[vector as usize];
-
-                    // If MSI-X interrupts are not enabled for this device, then simply
-                    // ignore the interrupt.
-                    if !config.enabled() {
-                        return Ok(());
-                    }
-
-                    // In case the vector control register associated with the entry
-                    // has its first bit set, this means the vector is masked and the
-                    // device should not inject the interrupt.
-                    // Instead, the Pending Bit Array table is updated to reflect there
-                    // is a pending interrupt for this specific vector.
-                    if config.masked() || entry.masked() {
-                        config.set_pba_bit(vector, false);
-                        return Ok(());
-                    }
-
-                    (msi_cb)(InterruptParameters { msix: Some(entry) })
-                },
-            ) as VirtioInterrupt);
-
-            self.interrupt_cb = Some(cb);
+impl PciDevice for VirtioPciDevice {
+    fn write_config_register(&mut self, reg_idx: usize, offset: u64, data: &[u8]) {
+        // Handle the special case where the capability VIRTIO_PCI_CAP_PCI_CFG
+        // is accessed. This capability has a special meaning as it allows the
+        // guest to access other capabilities without mapping the PCI BAR.
+        let base = reg_idx * 4;
+        if base + offset as usize >= self.cap_pci_cfg_info.offset
+            && base + offset as usize + data.len()
+                <= self.cap_pci_cfg_info.offset + self.cap_pci_cfg_info.cap.bytes().len()
+        {
+            let offset = base + offset as usize - self.cap_pci_cfg_info.offset;
+            self.write_cap_pci_cfg(offset, data);
+        } else {
+            self.configuration
+                .write_config_register(reg_idx, offset, data);
         }
     }
 
-    fn write_config_register(&mut self, reg_idx: usize, offset: u64, data: &[u8]) {
-        self.configuration
-            .write_config_register(reg_idx, offset, data);
-    }
-
-    fn read_config_register(&self, reg_idx: usize) -> u32 {
-        self.configuration.read_reg(reg_idx)
+    fn read_config_register(&mut self, reg_idx: usize) -> u32 {
+        // Handle the special case where the capability VIRTIO_PCI_CAP_PCI_CFG
+        // is accessed. This capability has a special meaning as it allows the
+        // guest to access other capabilities without mapping the PCI BAR.
+        let base = reg_idx * 4;
+        if base >= self.cap_pci_cfg_info.offset
+            && base + 4 <= self.cap_pci_cfg_info.offset + self.cap_pci_cfg_info.cap.bytes().len()
+        {
+            let offset = base - self.cap_pci_cfg_info.offset;
+            let mut data = [0u8; 4];
+            self.read_cap_pci_cfg(offset, &mut data);
+            u32::from_le_bytes(data)
+        } else {
+            self.configuration.read_reg(reg_idx)
+        }
     }
 
     fn detect_bar_reprogramming(
@@ -712,14 +851,14 @@ impl PciDevice for VirtioPciDevice {
         };
 
         if !self.device_activated && self.is_driver_ready() && self.are_queues_valid() {
-            if let Some(interrupt_cb) = self.interrupt_cb.take() {
+            if let Some(virtio_interrupt) = self.virtio_interrupt.take() {
                 if self.memory.is_some() {
                     let mem = self.memory.as_ref().unwrap().clone();
                     let mut device = self.device.lock().unwrap();
                     device
                         .activate(
                             mem,
-                            interrupt_cb,
+                            virtio_interrupt,
                             self.queues.clone(),
                             self.queue_evts.split_off(0),
                         )
@@ -732,9 +871,9 @@ impl PciDevice for VirtioPciDevice {
         // Device has been reset by the driver
         if self.device_activated && self.is_driver_init() {
             let mut device = self.device.lock().unwrap();
-            if let Some((interrupt_cb, mut queue_evts)) = device.reset() {
+            if let Some((virtio_interrupt, mut queue_evts)) = device.reset() {
                 // Upon reset the device returns its interrupt EventFD and it's queue EventFDs
-                self.interrupt_cb = Some(interrupt_cb);
+                self.virtio_interrupt = Some(virtio_interrupt);
                 self.queue_evts.append(&mut queue_evts);
 
                 self.device_activated = false;

vm-virtio/src/vhost_user/blk.rs

@@ -1,41 +1,32 @@
 // Copyright 2019 Intel Corporation. All Rights Reserved.
 // SPDX-License-Identifier: Apache-2.0
 
-use libc;
-use libc::EFD_NONBLOCK;
-use std::cmp;
-use std::io::Write;
-use std::ptr::null;
-use std::result;
-use std::sync::atomic::{AtomicBool, Ordering};
-use std::sync::{Arc, RwLock};
-use std::thread;
-use std::vec::Vec;
-
-use crate::VirtioInterrupt;
-
-use super::Error as DeviceError;
-
-use vm_device::{Migratable, MigratableError, Pausable, Snapshotable};
-use vm_memory::GuestMemoryMmap;
-use vmm_sys_util::eventfd::EventFd;
-
 use super::super::{ActivateError, ActivateResult, Queue, VirtioDevice, VirtioDeviceType};
 use super::handler::*;
 use super::vu_common_ctrl::*;
+use super::Error as DeviceError;
 use super::{Error, Result};
+use crate::block::VirtioBlockConfig;
+use crate::VirtioInterrupt;
+use arc_swap::ArcSwap;
+use libc;
+use libc::EFD_NONBLOCK;
+use std::cmp;
+use std::io::Write;
 use std::mem;
+use std::result;
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::Arc;
+use std::thread;
+use std::vec::Vec;
 use vhost_rs::vhost_user::message::VhostUserConfigFlags;
 use vhost_rs::vhost_user::message::{VhostUserProtocolFeatures, VhostUserVirtioFeatures};
 use vhost_rs::vhost_user::{Master, VhostUserMaster, VhostUserMasterReqHandler};
 use vhost_rs::VhostBackend;
 use virtio_bindings::bindings::virtio_blk::*;
-
-macro_rules! offset_of {
-    ($ty:ty, $field:ident) => {
-        unsafe { &(*(null() as *const $ty)).$field as *const _ as usize }
-    };
-}
+use vm_device::{Migratable, MigratableError, Pausable, Snapshotable};
+use vm_memory::{ByteValued, GuestMemoryMmap};
+use vmm_sys_util::eventfd::EventFd;
 
 struct SlaveReqHandler {}
 impl VhostUserMasterReqHandler for SlaveReqHandler {}
@@ -46,11 +37,11 @@ pub struct Blk {
     pause_evt: Option<EventFd>,
     avail_features: u64,
     acked_features: u64,
-    config_space: Vec<u8>,
+    config: VirtioBlockConfig,
     queue_sizes: Vec<u16>,
     queue_evts: Option<Vec<EventFd>>,
-    interrupt_cb: Option<Arc<VirtioInterrupt>>,
-    epoll_thread: Option<thread::JoinHandle<result::Result<(), DeviceError>>>,
+    interrupt_cb: Option<Arc<dyn VirtioInterrupt>>,
+    epoll_threads: Option<Vec<thread::JoinHandle<result::Result<(), DeviceError>>>>,
     paused: Arc<AtomicBool>,
 }
 
@@ -73,6 +64,10 @@ impl Blk {
             avail_features |= 1 << VIRTIO_BLK_F_CONFIG_WCE;
         }
 
+        if vu_cfg.num_queues > 1 {
+            avail_features |= 1 << VIRTIO_BLK_F_MQ;
+        }
+
         // Set vhost-user owner.
         vhost_user_blk
             .set_owner()
@@ -104,11 +99,20 @@ impl Blk {
                 .set_protocol_features(protocol_features)
                 .map_err(Error::VhostUserSetProtocolFeatures)?;
         }
+        // Get the max queues number from backend, and the queue number set
+        // should be less than this max queue number.
+        let max_queues_num = vhost_user_blk
+            .get_queue_num()
+            .map_err(Error::VhostUserGetQueueMaxNum)?;
 
-        let config_len = mem::size_of::<virtio_blk_config>();
+        if vu_cfg.num_queues > max_queues_num as usize {
+            error!("vhost-user-blk has queue number: {} larger than the max queue number: {} backend allowed\n",
+                vu_cfg.num_queues, max_queues_num);
+            return Err(Error::BadQueueNum);
+        }
+        let config_len = mem::size_of::<VirtioBlockConfig>();
         let config_space: Vec<u8> = vec![0u8; config_len as usize];
-
-        let (_, mut config_space) = vhost_user_blk
+        let (_, config_space) = vhost_user_blk
             .get_config(
                 0,
                 config_len as u32,
@@ -116,12 +120,21 @@ impl Blk {
                 config_space.as_slice(),
             )
             .unwrap();
+        let mut config = VirtioBlockConfig::default();
+        if let Some(backend_config) = VirtioBlockConfig::from_slice(config_space.as_slice()) {
+            config = *backend_config;
+            // Only set num_queues value(u16).
+            config.num_queues = vu_cfg.num_queues as u16;
+        }
 
-        let queue_num_offset = offset_of!(virtio_blk_config, num_queues);
-        // Only set num_queues value(u16).
-        let num_queues_slice = (vu_cfg.num_queues as u16).to_le_bytes();
-        config_space[queue_num_offset..queue_num_offset + mem::size_of::<u16>()]
-            .copy_from_slice(&num_queues_slice);
+        // Send set_vring_base here, since it could tell backends, like SPDK,
+        // how many virt queues to be handled, which backend required to know
+        // at early stage.
+        for i in 0..vu_cfg.num_queues {
+            vhost_user_blk
+                .set_vring_base(i, 0)
+                .map_err(Error::VhostUserSetVringBase)?;
+        }
 
         Ok(Blk {
             vhost_user_blk,
@@ -129,11 +142,11 @@ impl Blk {
             pause_evt: None,
             avail_features,
             acked_features,
-            config_space,
+            config,
             queue_sizes: vec![vu_cfg.queue_size; vu_cfg.num_queues],
             queue_evts: None,
             interrupt_cb: None,
-            epoll_thread: None,
+            epoll_threads: None,
             paused: Arc::new(AtomicBool::new(false)),
         })
     }
@@ -158,27 +171,12 @@ impl VirtioDevice for Blk {
         &self.queue_sizes
     }
 
-    fn features(&self, page: u32) -> u32 {
-        match page {
-            0 => self.avail_features as u32,
-            1 => (self.avail_features >> 32) as u32,
-            _ => {
-                warn!("Received request for unknown features page: {}", page);
-                0u32
-            }
-        }
+    fn features(&self) -> u64 {
+        self.avail_features
     }
 
-    fn ack_features(&mut self, page: u32, value: u32) {
-        let mut v = match page {
-            0 => u64::from(value),
-            1 => u64::from(value) << 32,
-            _ => {
-                warn!("Cannot acknowledge unknown features page: {}", page);
-                0u64
-            }
-        };
-
+    fn ack_features(&mut self, value: u64) {
+        let mut v = value;
         // Check if the guest is ACK'ing a feature that we didn't claim to have.
         let unrequested_features = v & !self.avail_features;
         if unrequested_features != 0 {
@@ -190,40 +188,38 @@ impl VirtioDevice for Blk {
     }
 
     fn read_config(&self, offset: u64, mut data: &mut [u8]) {
-        let config_len = self.config_space.len() as u64;
+        let config_slice = self.config.as_slice();
+        let config_len = config_slice.len() as u64;
         if offset >= config_len {
             error!("Failed to read config space");
             return;
         }
         if let Some(end) = offset.checked_add(data.len() as u64) {
             // This write can't fail, offset and end are checked against config_len.
-            data.write_all(&self.config_space[offset as usize..cmp::min(end, config_len) as usize])
+            data.write_all(&config_slice[offset as usize..cmp::min(end, config_len) as usize])
                 .unwrap();
         }
     }
 
     fn write_config(&mut self, offset: u64, data: &[u8]) {
+        let config_slice = self.config.as_mut_slice();
         let data_len = data.len() as u64;
-        let config_len = self.config_space.len() as u64;
+        let config_len = config_slice.len() as u64;
         if offset + data_len > config_len {
             error!("Failed to write config space");
             return;
         }
-        // In fact, write_config() only handle wce value in vhost-user-blk.
-        // so, we can only set wce value here.
-        if self.config_space[offset as usize] == data[0] {
-            return;
-        }
         self.vhost_user_blk
             .set_config(offset as u32, VhostUserConfigFlags::WRITABLE, data)
             .expect("Failed to set config");
-        self.config_space[offset as usize] = data[0];
+        let (_, right) = config_slice.split_at_mut(offset as usize);
+        right.copy_from_slice(&data[..]);
     }
 
     fn activate(
         &mut self,
-        mem: Arc<RwLock<GuestMemoryMmap>>,
-        interrupt_cb: Arc<VirtioInterrupt>,
+        mem: Arc<ArcSwap<GuestMemoryMmap>>,
+        interrupt_cb: Arc<dyn VirtioInterrupt>,
         queues: Vec<Queue>,
         queue_evts: Vec<EventFd>,
     ) -> ActivateResult {
@@ -258,37 +254,45 @@ impl VirtioDevice for Blk {
         }
         self.queue_evts = Some(tmp_queue_evts);
 
-        let vu_interrupt_list = setup_vhost_user(
+        let mut vu_interrupt_list = setup_vhost_user(
             &mut self.vhost_user_blk,
-            &mem.read().unwrap(),
+            mem.load().as_ref(),
             queues,
             queue_evts,
+            &interrupt_cb,
             self.acked_features,
         )
         .map_err(ActivateError::VhostUserBlkSetup)?;
 
-        let mut handler = VhostUserEpollHandler::<SlaveReqHandler>::new(VhostUserEpollConfig {
-            interrupt_cb,
-            kill_evt,
-            pause_evt,
-            vu_interrupt_list,
-            slave_req_handler: None,
-        });
+        let mut epoll_threads = Vec::new();
+        for _ in 0..vu_interrupt_list.len() {
+            let mut interrupt_list_sub: Vec<(Option<EventFd>, Queue)> = Vec::with_capacity(1);
+            interrupt_list_sub.push(vu_interrupt_list.remove(0));
 
-        let paused = self.paused.clone();
-        thread::Builder::new()
-            .name("vhost_user_blk".to_string())
-            .spawn(move || handler.run(paused))
-            .map(|thread| self.epoll_thread = Some(thread))
-            .map_err(|e| {
-                error!("failed to clone virtio epoll thread: {}", e);
-                ActivateError::BadActivate
-            })?;
+            let mut handler = VhostUserEpollHandler::<SlaveReqHandler>::new(VhostUserEpollConfig {
+                interrupt_cb: interrupt_cb.clone(),
+                kill_evt: kill_evt.try_clone().unwrap(),
+                pause_evt: pause_evt.try_clone().unwrap(),
+                vu_interrupt_list: interrupt_list_sub,
+                slave_req_handler: None,
+            });
+
+            let paused = self.paused.clone();
+            thread::Builder::new()
+                .name("vhost_user_blk".to_string())
+                .spawn(move || handler.run(paused))
+                .map(|thread| epoll_threads.push(thread))
+                .map_err(|e| {
+                    error!("failed to clone virtio epoll thread: {}", e);
+                    ActivateError::BadActivate
+                })?;
+        }
+        self.epoll_threads = Some(epoll_threads);
 
         Ok(())
     }
 
-    fn reset(&mut self) -> Option<(Arc<VirtioInterrupt>, Vec<EventFd>)> {
+    fn reset(&mut self) -> Option<(Arc<dyn VirtioInterrupt>, Vec<EventFd>)> {
         // We first must resume the virtio thread if it was paused.
         if self.pause_evt.take().is_some() {
             self.resume().ok()?;

vm-virtio/src/vhost_user/fs.rs

@@ -9,6 +9,7 @@ use crate::{
     ActivateError, ActivateResult, Queue, VirtioDevice, VirtioDeviceType, VirtioInterrupt,
     VirtioSharedMemoryList, VIRTIO_F_VERSION_1,
 };
+use arc_swap::ArcSwap;
 use libc::{self, EFD_NONBLOCK};
 use std::cmp;
 use std::io;
@@ -16,22 +17,20 @@ use std::io::Write;
 use std::os::unix::io::RawFd;
 use std::result;
 use std::sync::atomic::{AtomicBool, Ordering};
-use std::sync::{Arc, Mutex, RwLock};
+use std::sync::{Arc, Mutex};
 use std::thread;
 use vhost_rs::vhost_user::message::{
     VhostUserFSSlaveMsg, VhostUserProtocolFeatures, VhostUserVirtioFeatures,
+    VHOST_USER_FS_SLAVE_ENTRIES,
 };
 use vhost_rs::vhost_user::{
     HandlerResult, Master, MasterReqHandler, VhostUserMaster, VhostUserMasterReqHandler,
 };
 use vhost_rs::VhostBackend;
 use vm_device::{Migratable, MigratableError, Pausable, Snapshotable};
-use vm_memory::GuestMemoryMmap;
+use vm_memory::{ByteValued, GuestMemoryMmap};
 use vmm_sys_util::eventfd::EventFd;
 
-const CONFIG_SPACE_TAG_SIZE: usize = 36;
-const CONFIG_SPACE_NUM_QUEUES_SIZE: usize = 4;
-const CONFIG_SPACE_SIZE: usize = CONFIG_SPACE_TAG_SIZE + CONFIG_SPACE_NUM_QUEUES_SIZE;
 const NUM_QUEUE_OFFSET: usize = 1;
 
 struct SlaveReqHandler {
@@ -48,24 +47,35 @@ impl VhostUserMasterReqHandler for SlaveReqHandler {
     fn fs_slave_map(&mut self, fs: &VhostUserFSSlaveMsg, fd: RawFd) -> HandlerResult<()> {
         debug!("fs_slave_map");
 
-        let addr = self.mmap_cache_addr + fs.cache_offset[0];
-        let ret = unsafe {
-            libc::mmap(
-                addr as *mut libc::c_void,
-                fs.len[0] as usize,
-                fs.flags[0].bits() as i32,
-                libc::MAP_SHARED | libc::MAP_FIXED,
-                fd,
-                fs.fd_offset[0] as libc::off_t,
-            )
-        };
-        if ret == libc::MAP_FAILED {
-            return Err(io::Error::last_os_error());
-        }
+        for i in 0..VHOST_USER_FS_SLAVE_ENTRIES {
+            // Ignore if the length is 0.
+            if fs.len[i] == 0 {
+                continue;
+            }
 
-        let ret = unsafe { libc::close(fd) };
-        if ret == -1 {
-            return Err(io::Error::last_os_error());
+            if fs.cache_offset[i] > self.cache_size {
+                return Err(io::Error::new(io::ErrorKind::Other, "Wrong offset"));
+            }
+
+            let addr = self.mmap_cache_addr + fs.cache_offset[i];
+            let ret = unsafe {
+                libc::mmap(
+                    addr as *mut libc::c_void,
+                    fs.len[i] as usize,
+                    fs.flags[i].bits() as i32,
+                    libc::MAP_SHARED | libc::MAP_FIXED,
+                    fd,
+                    fs.fd_offset[i] as libc::off_t,
+                )
+            };
+            if ret == libc::MAP_FAILED {
+                return Err(io::Error::last_os_error());
+            }
+
+            let ret = unsafe { libc::close(fd) };
+            if ret == -1 {
+                return Err(io::Error::last_os_error());
+            }
         }
 
         Ok(())
@@ -74,26 +84,38 @@ impl VhostUserMasterReqHandler for SlaveReqHandler {
     fn fs_slave_unmap(&mut self, fs: &VhostUserFSSlaveMsg) -> HandlerResult<()> {
         debug!("fs_slave_unmap");
 
-        let mut len = fs.len[0];
-        // Need to handle a special case where the slave ask for the unmapping
-        // of the entire mapping.
-        if len == 0xffff_ffff_ffff_ffff {
-            len = self.cache_size;
-        }
+        for i in 0..VHOST_USER_FS_SLAVE_ENTRIES {
+            let mut len = fs.len[i];
 
-        let addr = self.mmap_cache_addr + fs.cache_offset[0];
-        let ret = unsafe {
-            libc::mmap(
-                addr as *mut libc::c_void,
-                len as usize,
-                libc::PROT_NONE,
-                libc::MAP_ANONYMOUS | libc::MAP_PRIVATE | libc::MAP_FIXED,
-                -1,
-                0 as libc::off_t,
-            )
-        };
-        if ret == libc::MAP_FAILED {
-            return Err(io::Error::last_os_error());
+            // Ignore if the length is 0.
+            if len == 0 {
+                continue;
+            }
+
+            // Need to handle a special case where the slave ask for the unmapping
+            // of the entire mapping.
+            if len == 0xffff_ffff_ffff_ffff {
+                len = self.cache_size;
+            }
+
+            if fs.cache_offset[i] > self.cache_size {
+                return Err(io::Error::new(io::ErrorKind::Other, "Wrong offset"));
+            }
+
+            let addr = self.mmap_cache_addr + fs.cache_offset[i];
+            let ret = unsafe {
+                libc::mmap(
+                    addr as *mut libc::c_void,
+                    len as usize,
+                    libc::PROT_NONE,
+                    libc::MAP_ANONYMOUS | libc::MAP_PRIVATE | libc::MAP_FIXED,
+                    -1,
+                    0 as libc::off_t,
+                )
+            };
+            if ret == libc::MAP_FAILED {
+                return Err(io::Error::last_os_error());
+            }
         }
 
         Ok(())
@@ -102,30 +124,60 @@ impl VhostUserMasterReqHandler for SlaveReqHandler {
     fn fs_slave_sync(&mut self, fs: &VhostUserFSSlaveMsg) -> HandlerResult<()> {
         debug!("fs_slave_sync");
 
-        let addr = self.mmap_cache_addr + fs.cache_offset[0];
-        let ret =
-            unsafe { libc::msync(addr as *mut libc::c_void, fs.len[0] as usize, libc::MS_SYNC) };
-        if ret == -1 {
-            return Err(io::Error::last_os_error());
+        for i in 0..VHOST_USER_FS_SLAVE_ENTRIES {
+            // Ignore if the length is 0.
+            if fs.len[i] == 0 {
+                continue;
+            }
+
+            if fs.cache_offset[i] > self.cache_size {
+                return Err(io::Error::new(io::ErrorKind::Other, "Wrong offset"));
+            }
+
+            let addr = self.mmap_cache_addr + fs.cache_offset[i];
+            let ret = unsafe {
+                libc::msync(addr as *mut libc::c_void, fs.len[i] as usize, libc::MS_SYNC)
+            };
+            if ret == -1 {
+                return Err(io::Error::last_os_error());
+            }
         }
 
         Ok(())
     }
 }
 
+#[derive(Copy, Clone)]
+#[repr(C, packed)]
+struct VirtioFsConfig {
+    tag: [u8; 36],
+    num_request_queues: u32,
+}
+
+impl Default for VirtioFsConfig {
+    fn default() -> Self {
+        VirtioFsConfig {
+            tag: [0; 36],
+            num_request_queues: 0,
+        }
+    }
+}
+
+unsafe impl ByteValued for VirtioFsConfig {}
+
 pub struct Fs {
     vu: Master,
     queue_sizes: Vec<u16>,
     avail_features: u64,
     acked_features: u64,
-    config_space: Vec<u8>,
+    config: VirtioFsConfig,
     kill_evt: Option<EventFd>,
     pause_evt: Option<EventFd>,
     cache: Option<(VirtioSharedMemoryList, u64)>,
     slave_req_support: bool,
     queue_evts: Option<Vec<EventFd>>,
-    interrupt_cb: Option<Arc<VirtioInterrupt>>,
-    epoll_thread: Option<thread::JoinHandle<result::Result<(), DeviceError>>>,
+    interrupt_cb: Option<Arc<dyn VirtioInterrupt>>,
+    epoll_threads: Option<Vec<thread::JoinHandle<result::Result<(), DeviceError>>>>,
     paused: Arc<AtomicBool>,
 }
 
@@ -190,28 +242,25 @@ impl Fs {
             slave_req_support = true;
         }
 
-        // Create virtio device config space.
-        // First by adding the tag.
-        let mut config_space = tag.to_string().into_bytes();
-        config_space.resize(CONFIG_SPACE_SIZE, 0);
-
-        // And then by copying the number of queues.
-        let num_queues_slice = (req_num_queues as u32).to_le_bytes();
-        config_space[CONFIG_SPACE_TAG_SIZE..CONFIG_SPACE_SIZE].copy_from_slice(&num_queues_slice);
+        // Create virtio-fs device configuration.
+        let mut config = VirtioFsConfig::default();
+        let tag_bytes_vec = tag.to_string().into_bytes();
+        config.tag[..tag_bytes_vec.len()].copy_from_slice(tag_bytes_vec.as_slice());
+        config.num_request_queues = req_num_queues as u32;
 
         Ok(Fs {
             vu: master,
             queue_sizes: vec![queue_size; num_queues],
             avail_features,
             acked_features,
-            config_space,
+            config,
             kill_evt: None,
             pause_evt: None,
             cache,
             slave_req_support,
             queue_evts: None,
             interrupt_cb: None,
-            epoll_thread: None,
+            epoll_threads: None,
             paused: Arc::new(AtomicBool::new(false)),
         })
     }
@@ -235,29 +284,12 @@ impl VirtioDevice for Fs {
         &self.queue_sizes.as_slice()
     }
 
-    fn features(&self, page: u32) -> u32 {
-        match page {
-            // Get the lower 32-bits of the features bitfield.
-            0 => self.avail_features as u32,
-            // Get the upper 32-bits of the features bitfield.
-            1 => (self.avail_features >> 32) as u32,
-            _ => {
-                warn!("fs: Received request for unknown features page: {}", page);
-                0u32
-            }
-        }
+    fn features(&self) -> u64 {
+        self.avail_features
     }
 
-    fn ack_features(&mut self, page: u32, value: u32) {
-        let mut v = match page {
-            0 => u64::from(value),
-            1 => u64::from(value) << 32,
-            _ => {
-                warn!("fs: Cannot acknowledge unknown features page: {}", page);
-                0u64
-            }
-        };
-
+    fn ack_features(&mut self, value: u64) {
+        let mut v = value;
         // Check if the guest is ACK'ing a feature that we didn't claim to have.
         let unrequested_features = v & !self.avail_features;
         if unrequested_features != 0 {
@@ -270,33 +302,35 @@ impl VirtioDevice for Fs {
     }
 
     fn read_config(&self, offset: u64, mut data: &mut [u8]) {
-        let config_len = self.config_space.len() as u64;
+        let config_slice = self.config.as_slice();
+        let config_len = config_slice.len() as u64;
         if offset >= config_len {
             error!("Failed to read config space");
             return;
         }
         if let Some(end) = offset.checked_add(data.len() as u64) {
             // This write can't fail, offset and end are checked against config_len.
-            data.write_all(&self.config_space[offset as usize..cmp::min(end, config_len) as usize])
+            data.write_all(&config_slice[offset as usize..cmp::min(end, config_len) as usize])
                 .unwrap();
         }
     }
 
     fn write_config(&mut self, offset: u64, data: &[u8]) {
+        let config_slice = self.config.as_mut_slice();
         let data_len = data.len() as u64;
-        let config_len = self.config_space.len() as u64;
+        let config_len = config_slice.len() as u64;
         if offset + data_len > config_len {
             error!("Failed to write config space");
             return;
         }
-        let (_, right) = self.config_space.split_at_mut(offset as usize);
+        let (_, right) = config_slice.split_at_mut(offset as usize);
         right.copy_from_slice(&data[..]);
     }
 
     fn activate(
         &mut self,
-        mem: Arc<RwLock<GuestMemoryMmap>>,
-        interrupt_cb: Arc<VirtioInterrupt>,
+        mem: Arc<ArcSwap<GuestMemoryMmap>>,
+        interrupt_cb: Arc<dyn VirtioInterrupt>,
         queues: Vec<Queue>,
         queue_evts: Vec<EventFd>,
     ) -> ActivateResult {
@@ -342,9 +376,10 @@ impl VirtioDevice for Fs {
 
         let vu_call_evt_queue_list = setup_vhost_user(
             &mut self.vu,
-            &mem.read().unwrap(),
+            mem.load().as_ref(),
             queues,
             queue_evts,
+            &interrupt_cb,
             self.acked_features,
         )
         .map_err(ActivateError::VhostUserSetup)?;
@@ -382,19 +417,22 @@ impl VirtioDevice for Fs {
         });
 
         let paused = self.paused.clone();
+        let mut epoll_threads = Vec::new();
         thread::Builder::new()
             .name("virtio_fs".to_string())
             .spawn(move || handler.run(paused))
-            .map(|thread| self.epoll_thread = Some(thread))
+            .map(|thread| epoll_threads.push(thread))
             .map_err(|e| {
                 error!("failed to clone queue EventFd: {}", e);
                 ActivateError::BadActivate
             })?;
 
+        self.epoll_threads = Some(epoll_threads);
+
         Ok(())
     }
 
-    fn reset(&mut self) -> Option<(Arc<VirtioInterrupt>, Vec<EventFd>)> {
+    fn reset(&mut self) -> Option<(Arc<dyn VirtioInterrupt>, Vec<EventFd>)> {
         // We first must resume the virtio thread if it was paused.
         if self.pause_evt.take().is_some() {
             self.resume().ok()?;

vm-virtio/src/vhost_user/handler.rs

@@ -28,10 +28,10 @@ use vhost_rs::vhost_user::{MasterReqHandler, VhostUserMasterReqHandler};
 /// * `kill_evt` - EventFd used to kill the vhost-user device.
 /// * `vu_interrupt_list` - virtqueue and EventFd to signal when buffer used.
 pub struct VhostUserEpollConfig<S: VhostUserMasterReqHandler> {
-    pub interrupt_cb: Arc<VirtioInterrupt>,
+    pub interrupt_cb: Arc<dyn VirtioInterrupt>,
     pub kill_evt: EventFd,
     pub pause_evt: EventFd,
-    pub vu_interrupt_list: Vec<(EventFd, Queue)>,
+    pub vu_interrupt_list: Vec<(Option<EventFd>, Queue)>,
     pub slave_req_handler: Option<MasterReqHandler<S>>,
 }
 
@@ -52,7 +52,9 @@ impl<S: VhostUserMasterReqHandler> VhostUserEpollHandler<S> {
     }
 
     fn signal_used_queue(&self, queue: &Queue) -> Result<()> {
-        (self.vu_epoll_cfg.interrupt_cb)(&VirtioInterruptType::Queue, Some(queue))
+        self.vu_epoll_cfg
+            .interrupt_cb
+            .trigger(&VirtioInterruptType::Queue, Some(queue))
             .map_err(Error::FailedSignalingUsedQueue)
     }
 
@@ -62,14 +64,16 @@ impl<S: VhostUserMasterReqHandler> VhostUserEpollHandler<S> {
 
         for (index, vhost_user_interrupt) in self.vu_epoll_cfg.vu_interrupt_list.iter().enumerate()
         {
-            // Add events
-            epoll::ctl(
-                epoll_fd,
-                epoll::ControlOptions::EPOLL_CTL_ADD,
-                vhost_user_interrupt.0.as_raw_fd(),
-                epoll::Event::new(epoll::Events::EPOLLIN, index as u64),
-            )
-            .map_err(Error::EpollCtl)?;
+            if let Some(eventfd) = &vhost_user_interrupt.0 {
+                // Add events
+                epoll::ctl(
+                    epoll_fd,
+                    epoll::ControlOptions::EPOLL_CTL_ADD,
+                    eventfd.as_raw_fd(),
+                    epoll::Event::new(epoll::Events::EPOLLIN, index as u64),
+                )
+                .map_err(Error::EpollCtl)?;
+            }
         }
 
         let kill_evt_index = self.vu_epoll_cfg.vu_interrupt_list.len();
@@ -134,15 +138,14 @@ impl<S: VhostUserMasterReqHandler> VhostUserEpollHandler<S> {
 
                 match ev_type {
                     x if x < kill_evt_index => {
-                        self.vu_epoll_cfg.vu_interrupt_list[x]
-                            .0
-                            .read()
-                            .map_err(Error::FailedReadingQueue)?;
-                        if let Err(e) =
-                            self.signal_used_queue(&self.vu_epoll_cfg.vu_interrupt_list[x].1)
-                        {
-                            error!("Failed to signal used queue: {:?}", e);
-                            break 'poll;
+                        if let Some(eventfd) = &self.vu_epoll_cfg.vu_interrupt_list[x].0 {
+                            eventfd.read().map_err(Error::FailedReadingQueue)?;
+                            if let Err(e) =
+                                self.signal_used_queue(&self.vu_epoll_cfg.vu_interrupt_list[x].1)
+                            {
+                                error!("Failed to signal used queue: {:?}", e);
+                                break 'poll;
+                            }
                         }
                     }
                     x if kill_evt_index == x => {

vm-virtio/src/vhost_user/mod.rs

@@ -55,6 +55,8 @@ pub enum Error {
     VhostUserConnect(vhost_rs::Error),
     /// Get features failed.
     VhostUserGetFeatures(VhostError),
+    /// Get queue max number failed.
+    VhostUserGetQueueMaxNum(VhostError),
     /// Get protocol features failed.
     VhostUserGetProtocolFeatures(VhostError),
     /// Vhost-user Backend not support vhost-user protocol.

vm-virtio/src/vhost_user/net.rs

@@ -1,33 +1,37 @@
 // Copyright 2019 Intel Corporation. All Rights Reserved.
 // SPDX-License-Identifier: Apache-2.0
 
+use super::super::net_util::{
+    build_net_config_space, CtrlVirtio, NetCtrlEpollHandler, VirtioNetConfig,
+};
+use super::super::Error as CtrlError;
+use super::super::{ActivateError, ActivateResult, Queue, VirtioDevice, VirtioDeviceType};
+use super::handler::*;
+use super::vu_common_ctrl::*;
+use super::Error as DeviceError;
+use super::{Error, Result};
+use crate::VirtioInterrupt;
+use arc_swap::ArcSwap;
 use libc;
 use libc::EFD_NONBLOCK;
+use net_util::MacAddr;
 use std::cmp;
 use std::io::Write;
 use std::result;
 use std::sync::atomic::{AtomicBool, Ordering};
-use std::sync::{Arc, RwLock};
+use std::sync::Arc;
 use std::thread;
 use std::vec::Vec;
-
-use super::Error as DeviceError;
-use crate::VirtioInterrupt;
-use net_util::{MacAddr, MAC_ADDR_LEN};
-
-use vm_device::{Migratable, MigratableError, Pausable, Snapshotable};
-use vm_memory::GuestMemoryMmap;
-use vmm_sys_util::eventfd::EventFd;
-
-use super::super::{ActivateError, ActivateResult, Queue, VirtioDevice, VirtioDeviceType};
-use super::handler::*;
-use super::vu_common_ctrl::*;
-use super::{Error, Result};
 use vhost_rs::vhost_user::message::{VhostUserProtocolFeatures, VhostUserVirtioFeatures};
 use vhost_rs::vhost_user::{Master, VhostUserMaster, VhostUserMasterReqHandler};
 use vhost_rs::VhostBackend;
 use virtio_bindings::bindings::virtio_net;
 use virtio_bindings::bindings::virtio_ring;
+use vm_device::{Migratable, MigratableError, Pausable, Snapshotable};
+use vm_memory::{ByteValued, GuestMemoryMmap};
+use vmm_sys_util::eventfd::EventFd;
+
+const DEFAULT_QUEUE_NUMBER: usize = 2;
 
 struct SlaveReqHandler {}
 impl VhostUserMasterReqHandler for SlaveReqHandler {}
@@ -39,15 +43,17 @@ pub struct Net {
     avail_features: u64,
     acked_features: u64,
     backend_features: u64,
-    config_space: Vec<u8>,
+    config: VirtioNetConfig,
     queue_sizes: Vec<u16>,
     queue_evts: Option<Vec<EventFd>>,
-    interrupt_cb: Option<Arc<VirtioInterrupt>>,
-    epoll_thread: Option<thread::JoinHandle<result::Result<(), DeviceError>>>,
+    interrupt_cb: Option<Arc<dyn VirtioInterrupt>>,
+    epoll_threads: Option<Vec<thread::JoinHandle<result::Result<(), DeviceError>>>>,
+    ctrl_queue_epoll_thread: Option<thread::JoinHandle<result::Result<(), CtrlError>>>,
     paused: Arc<AtomicBool>,
 }
 
 impl Net {
+    /// Create a new vhost-user-net device
     /// Create a new vhost-user-net device
     pub fn new(mac_addr: MacAddr, vu_cfg: VhostUserConfig) -> Result<Net> {
         let mut vhost_user_net = Master::connect(&vu_cfg.sock, vu_cfg.num_queues as u64)
@@ -86,24 +92,45 @@ impl Net {
             .set_features(avail_features)
             .map_err(Error::VhostUserSetFeatures)?;
 
+        let protocol_features;
         let mut acked_features = 0;
         if avail_features & VhostUserVirtioFeatures::PROTOCOL_FEATURES.bits() != 0 {
             acked_features |= VhostUserVirtioFeatures::PROTOCOL_FEATURES.bits();
-            let mut protocol_features = vhost_user_net
+            protocol_features = vhost_user_net
                 .get_protocol_features()
                 .map_err(Error::VhostUserGetProtocolFeatures)?;
-            protocol_features &= VhostUserProtocolFeatures::MQ;
-            vhost_user_net
-                .set_protocol_features(protocol_features)
-                .map_err(Error::VhostUserSetProtocolFeatures)?;
         } else {
             return Err(Error::VhostUserProtocolNotSupport);
         }
 
-        let mut config_space = Vec::with_capacity(MAC_ADDR_LEN);
-        unsafe { config_space.set_len(MAC_ADDR_LEN) }
-        config_space[..].copy_from_slice(mac_addr.get_bytes());
-        avail_features |= 1 << virtio_net::VIRTIO_NET_F_MAC;
+        let max_queue_number =
+            if protocol_features.bits() & VhostUserProtocolFeatures::MQ.bits() != 0 {
+                vhost_user_net
+                    .set_protocol_features(protocol_features & VhostUserProtocolFeatures::MQ)
+                    .map_err(Error::VhostUserSetProtocolFeatures)?;
+                match vhost_user_net.get_queue_num() {
+                    Ok(qn) => qn,
+                    Err(_) => DEFAULT_QUEUE_NUMBER as u64,
+                }
+            } else {
+                DEFAULT_QUEUE_NUMBER as u64
+            };
+        if vu_cfg.num_queues > max_queue_number as usize {
+            error!("vhost-user-net has queue number: {} larger than the max queue number: {} backend allowed\n",
+                vu_cfg.num_queues, max_queue_number);
+            return Err(Error::BadQueueNum);
+        }
+
+        avail_features |= 1 << virtio_net::VIRTIO_NET_F_CTRL_VQ;
+        let queue_num = vu_cfg.num_queues + 1;
+
+        let mut config = VirtioNetConfig::default();
+        build_net_config_space(
+            &mut config,
+            mac_addr,
+            vu_cfg.num_queues,
+            &mut avail_features,
+        );
 
         // Send set_vring_base here, since it could tell backends, like OVS + DPDK,
         // how many virt queues to be handled, which backend required to know at early stage.
@@ -120,11 +147,12 @@ impl Net {
             avail_features,
             acked_features,
             backend_features,
-            config_space,
-            queue_sizes: vec![vu_cfg.queue_size; vu_cfg.num_queues],
+            config,
+            queue_sizes: vec![vu_cfg.queue_size; queue_num],
             queue_evts: None,
             interrupt_cb: None,
-            epoll_thread: None,
+            epoll_threads: None,
+            ctrl_queue_epoll_thread: None,
             paused: Arc::new(AtomicBool::new(false)),
         })
     }
@@ -149,27 +177,12 @@ impl VirtioDevice for Net {
         &self.queue_sizes
     }
 
-    fn features(&self, page: u32) -> u32 {
-        match page {
-            0 => self.avail_features as u32,
-            1 => (self.avail_features >> 32) as u32,
-            _ => {
-                warn!("Received request for unknown features page: {}", page);
-                0u32
-            }
-        }
+    fn features(&self) -> u64 {
+        self.avail_features
     }
 
-    fn ack_features(&mut self, page: u32, value: u32) {
-        let mut v = match page {
-            0 => u64::from(value),
-            1 => u64::from(value) << 32,
-            _ => {
-                warn!("Cannot acknowledge unknown features page: {}", page);
-                0u64
-            }
-        };
-
+    fn ack_features(&mut self, value: u64) {
+        let mut v = value;
         // Check if the guest is ACK'ing a feature that we didn't claim to have.
         let unrequested_features = v & !self.avail_features;
         if unrequested_features != 0 {
@@ -181,36 +194,47 @@ impl VirtioDevice for Net {
     }
 
     fn read_config(&self, offset: u64, mut data: &mut [u8]) {
-        let config_len = self.config_space.len() as u64;
+        let config_slice = self.config.as_slice();
+        let config_len = config_slice.len() as u64;
         if offset >= config_len {
             error!("Failed to read config space");
             return;
         }
         if let Some(end) = offset.checked_add(data.len() as u64) {
             // This write can't fail, offset and end are checked against config_len.
-            data.write_all(&self.config_space[offset as usize..cmp::min(end, config_len) as usize])
+            data.write_all(&config_slice[offset as usize..cmp::min(end, config_len) as usize])
                 .unwrap();
         }
     }
 
     fn write_config(&mut self, offset: u64, data: &[u8]) {
+        let config_slice = self.config.as_mut_slice();
         let data_len = data.len() as u64;
-        let config_len = self.config_space.len() as u64;
+        let config_len = config_slice.len() as u64;
         if offset + data_len > config_len {
             error!("Failed to write config space");
             return;
         }
-        let (_, right) = self.config_space.split_at_mut(offset as usize);
+        let (_, right) = config_slice.split_at_mut(offset as usize);
         right.copy_from_slice(&data[..]);
     }
 
     fn activate(
         &mut self,
-        mem: Arc<RwLock<GuestMemoryMmap>>,
-        interrupt_cb: Arc<VirtioInterrupt>,
-        queues: Vec<Queue>,
-        queue_evts: Vec<EventFd>,
+        mem: Arc<ArcSwap<GuestMemoryMmap>>,
+        interrupt_cb: Arc<dyn VirtioInterrupt>,
+        mut queues: Vec<Queue>,
+        mut queue_evts: Vec<EventFd>,
     ) -> ActivateResult {
+        if queues.len() != self.queue_sizes.len() || queue_evts.len() != self.queue_sizes.len() {
+            error!(
+                "Cannot perform activate. Expected {} queue(s), got {}",
+                self.queue_sizes.len(),
+                queues.len()
+            );
+            return Err(ActivateError::BadActivate);
+        }
+
         let (self_kill_evt, kill_evt) = EventFd::new(EFD_NONBLOCK)
             .and_then(|e| Ok((e.try_clone()?, e)))
             .map_err(|e| {
@@ -242,37 +266,73 @@ impl VirtioDevice for Net {
         }
         self.queue_evts = Some(tmp_queue_evts);
 
-        let vu_interrupt_list = setup_vhost_user(
+        let queue_num = queue_evts.len();
+
+        if (self.acked_features & 1 << virtio_net::VIRTIO_NET_F_CTRL_VQ) != 0 && queue_num % 2 != 0
+        {
+            let cvq_queue = queues.remove(queue_num - 1);
+            let cvq_queue_evt = queue_evts.remove(queue_num - 1);
+
+            let mut ctrl_handler = NetCtrlEpollHandler {
+                mem: mem.clone(),
+                kill_evt: kill_evt.try_clone().unwrap(),
+                pause_evt: pause_evt.try_clone().unwrap(),
+                ctrl_q: CtrlVirtio::new(cvq_queue, cvq_queue_evt),
+                epoll_fd: 0,
+            };
+
+            let paused = self.paused.clone();
+            thread::Builder::new()
+                .name("virtio_net".to_string())
+                .spawn(move || ctrl_handler.run_ctrl(paused))
+                .map(|thread| self.ctrl_queue_epoll_thread = Some(thread))
+                .map_err(|e| {
+                    error!("failed to clone queue EventFd: {}", e);
+                    ActivateError::BadActivate
+                })?;
+        }
+
+        let mut vu_interrupt_list = setup_vhost_user(
             &mut self.vhost_user_net,
-            &mem.read().unwrap(),
+            mem.load().as_ref(),
             queues,
             queue_evts,
+            &interrupt_cb,
             self.acked_features & self.backend_features,
         )
         .map_err(ActivateError::VhostUserNetSetup)?;
 
-        let mut handler = VhostUserEpollHandler::<SlaveReqHandler>::new(VhostUserEpollConfig {
-            interrupt_cb,
-            kill_evt,
-            pause_evt,
-            vu_interrupt_list,
-            slave_req_handler: None,
-        });
+        let mut epoll_threads = Vec::new();
+        for _ in 0..vu_interrupt_list.len() / 2 {
+            let mut interrupt_list_sub: Vec<(Option<EventFd>, Queue)> = Vec::with_capacity(2);
+            interrupt_list_sub.push(vu_interrupt_list.remove(0));
+            interrupt_list_sub.push(vu_interrupt_list.remove(0));
 
-        let paused = self.paused.clone();
-        thread::Builder::new()
-            .name("vhost_user_net".to_string())
-            .spawn(move || handler.run(paused))
-            .map(|thread| self.epoll_thread = Some(thread))
-            .map_err(|e| {
-                error!("failed to clone queue EventFd: {}", e);
-                ActivateError::BadActivate
-            })?;
+            let mut handler = VhostUserEpollHandler::<SlaveReqHandler>::new(VhostUserEpollConfig {
+                interrupt_cb: interrupt_cb.clone(),
+                kill_evt: kill_evt.try_clone().unwrap(),
+                pause_evt: pause_evt.try_clone().unwrap(),
+                vu_interrupt_list: interrupt_list_sub,
+                slave_req_handler: None,
+            });
+
+            let paused = self.paused.clone();
+            thread::Builder::new()
+                .name("vhost_user_net".to_string())
+                .spawn(move || handler.run(paused))
+                .map(|thread| epoll_threads.push(thread))
+                .map_err(|e| {
+                    error!("failed to clone queue EventFd: {}", e);
+                    ActivateError::BadActivate
+                })?;
+        }
+
+        self.epoll_threads = Some(epoll_threads);
 
         Ok(())
     }
 
-    fn reset(&mut self) -> Option<(Arc<VirtioInterrupt>, Vec<EventFd>)> {
+    fn reset(&mut self) -> Option<(Arc<dyn VirtioInterrupt>, Vec<EventFd>)> {
         // We first must resume the virtio thread if it was paused.
         if self.pause_evt.take().is_some() {
             self.resume().ok()?;
@@ -296,6 +356,6 @@ impl VirtioDevice for Net {
     }
 }
 
-virtio_pausable!(Net);
+virtio_ctrl_q_pausable!(Net);
 impl Snapshotable for Net {}
 impl Migratable for Net {}

vm-virtio/src/vhost_user/vu_common_ctrl.rs

@@ -5,9 +5,11 @@ use libc;
 use libc::EFD_NONBLOCK;
 use std::convert::TryInto;
 use std::os::unix::io::AsRawFd;
+use std::sync::Arc;
 use std::vec::Vec;
 
 use crate::queue::Descriptor;
+use crate::{VirtioInterrupt, VirtioInterruptType};
 
 use vm_device::get_host_address_range;
 use vm_memory::{Address, Error as MmapError, GuestMemory, GuestMemoryMmap, GuestMemoryRegion};
@@ -30,7 +32,8 @@ pub fn setup_vhost_user_vring(
     mem: &GuestMemoryMmap,
     queues: Vec<Queue>,
     queue_evts: Vec<EventFd>,
-) -> Result<Vec<(EventFd, Queue)>> {
+    virtio_interrupt: &Arc<dyn VirtioInterrupt>,
+) -> Result<Vec<(Option<EventFd>, Queue)>> {
     let mut regions: Vec<VhostUserMemoryRegionInfo> = Vec::new();
     mem.with_regions_mut(|_, region| {
         let (mmap_handle, mmap_offset) = match region.file_offset() {
@@ -89,10 +92,17 @@ pub fn setup_vhost_user_vring(
         vu.set_vring_base(queue_index, 0u16)
             .map_err(Error::VhostUserSetVringBase)?;
 
-        let vhost_user_interrupt = EventFd::new(EFD_NONBLOCK).map_err(Error::VhostIrqCreate)?;
-        vu.set_vring_call(queue_index, &vhost_user_interrupt)
-            .map_err(Error::VhostUserSetVringCall)?;
-        vu_interrupt_list.push((vhost_user_interrupt, queue));
+        if let Some(eventfd) = virtio_interrupt.notifier(&VirtioInterruptType::Queue, Some(&queue))
+        {
+            vu.set_vring_call(queue_index, &eventfd)
+                .map_err(Error::VhostUserSetVringCall)?;
+            vu_interrupt_list.push((None, queue));
+        } else {
+            let eventfd = EventFd::new(EFD_NONBLOCK).map_err(Error::VhostIrqCreate)?;
+            vu.set_vring_call(queue_index, &eventfd)
+                .map_err(Error::VhostUserSetVringCall)?;
+            vu_interrupt_list.push((Some(eventfd), queue));
+        }
 
         vu.set_vring_kick(queue_index, &queue_evts[queue_index])
             .map_err(Error::VhostUserSetVringKick)?;
@@ -109,13 +119,14 @@ pub fn setup_vhost_user(
     mem: &GuestMemoryMmap,
     queues: Vec<Queue>,
     queue_evts: Vec<EventFd>,
+    virtio_interrupt: &Arc<dyn VirtioInterrupt>,
     acked_features: u64,
-) -> Result<Vec<(EventFd, Queue)>> {
+) -> Result<Vec<(Option<EventFd>, Queue)>> {
     // Set features based on the acked features from the guest driver.
     vu.set_features(acked_features)
         .map_err(Error::VhostUserSetFeatures)?;
 
-    setup_vhost_user_vring(vu, mem, queues, queue_evts)
+    setup_vhost_user_vring(vu, mem, queues, queue_evts, virtio_interrupt)
 }
 
 pub fn reset_vhost_user(vu: &mut Master, num_queues: usize) -> Result<()> {

vm-virtio/src/vsock/csm/connection.rs

@@ -557,7 +557,7 @@ where
     /// Raw data can either be sent straight to the host stream, or to our TX buffer, if the
     /// former fails.
     ///
-    fn send_bytes(&mut self, buf: &[u8]) -> Result<()> {
+    pub fn send_bytes(&mut self, buf: &[u8]) -> Result<()> {
         // If there is data in the TX buffer, that means we're already registered for EPOLLOUT
         // events on the underlying stream. Therefore, there's no point in attempting a write
         // at this point. `self.notify()` will get called when EPOLLOUT arrives, and it will
@@ -592,6 +592,11 @@ where
         Ok(())
     }
 
+    /// Return the connections state.
+    pub fn state(&self) -> ConnState {
+        self.state
+    }
+
     /// Check if the credit information the peer has last received from us is outdated.
     ///
     fn peer_needs_credit_update(&self) -> bool {

vm-virtio/src/vsock/csm/mod.rs

@@ -38,7 +38,7 @@ type Result<T> = std::result::Result<T, Error>;
 
 /// A vsock connection state.
 ///
-#[derive(Debug, PartialEq)]
+#[derive(Clone, Copy, Debug, PartialEq)]
 pub enum ConnState {
     /// The connection has been initiated by the host end, but is yet to be confirmed by the guest.
     LocalInit,

vm-virtio/src/vsock/device.rs

@@ -8,6 +8,13 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the THIRD-PARTY file.
 
+use super::{VsockBackend, VsockPacket};
+use crate::Error as DeviceError;
+use crate::VirtioInterrupt;
+use crate::{
+    ActivateError, ActivateResult, DeviceEventT, Queue, VirtioDevice, VirtioDeviceType,
+    VirtioInterruptType, VIRTIO_F_IN_ORDER, VIRTIO_F_IOMMU_PLATFORM, VIRTIO_F_VERSION_1,
+};
 /// This is the `VirtioDevice` implementation for our vsock device. It handles the virtio-level
 /// device logic: feature negociation, device configuration, and device activation.
 /// The run-time device logic (i.e. event-driven data handling) is implemented by
@@ -27,6 +34,8 @@
 /// - an event queue FD; and
 /// - a backend FD.
 ///
+use arc_swap::ArcSwap;
+use byteorder::{ByteOrder, LittleEndian};
 use epoll;
 use libc::EFD_NONBLOCK;
 use std;
@@ -36,15 +45,6 @@ use std::result;
 use std::sync::atomic::{AtomicBool, Ordering};
 use std::sync::{Arc, RwLock};
 use std::thread;
-
-use super::{VsockBackend, VsockPacket};
-use crate::Error as DeviceError;
-use crate::VirtioInterrupt;
-use crate::{
-    ActivateError, ActivateResult, DeviceEventT, Queue, VirtioDevice, VirtioDeviceType,
-    VirtioInterruptType, VIRTIO_F_IN_ORDER, VIRTIO_F_IOMMU_PLATFORM, VIRTIO_F_VERSION_1,
-};
-use byteorder::{ByteOrder, LittleEndian};
 use vm_device::{Migratable, MigratableError, Pausable, Snapshotable};
 use vm_memory::GuestMemoryMmap;
 use vmm_sys_util::eventfd::EventFd;
@@ -86,12 +86,12 @@ pub const EVENTS_LEN: usize = 6;
 ///   - again, attempt to fetch any incoming packets queued by the backend into virtio RX buffers.
 ///
 pub struct VsockEpollHandler<B: VsockBackend> {
-    pub mem: Arc<RwLock<GuestMemoryMmap>>,
+    pub mem: Arc<ArcSwap<GuestMemoryMmap>>,
     pub queues: Vec<Queue>,
     pub queue_evts: Vec<EventFd>,
     pub kill_evt: EventFd,
     pub pause_evt: EventFd,
-    pub interrupt_cb: Arc<VirtioInterrupt>,
+    pub interrupt_cb: Arc<dyn VirtioInterrupt>,
     pub backend: Arc<RwLock<B>>,
 }
 
@@ -105,10 +105,12 @@ where
     fn signal_used_queue(&self, queue: &Queue) -> result::Result<(), DeviceError> {
         debug!("vsock: raising IRQ");
 
-        (self.interrupt_cb)(&VirtioInterruptType::Queue, Some(queue)).map_err(|e| {
-            error!("Failed to signal used queue: {:?}", e);
-            DeviceError::FailedSignalingUsedQueue(e)
-        })
+        self.interrupt_cb
+            .trigger(&VirtioInterruptType::Queue, Some(queue))
+            .map_err(|e| {
+                error!("Failed to signal used queue: {:?}", e);
+                DeviceError::FailedSignalingUsedQueue(e)
+            })
     }
 
     /// Walk the driver-provided RX queue buffers and attempt to fill them up with any data that we
@@ -119,7 +121,7 @@ where
 
         let mut used_desc_heads = [(0, 0); QUEUE_SIZE as usize];
         let mut used_count = 0;
-        let mem = self.mem.read().unwrap();
+        let mem = self.mem.load();
         for avail_desc in self.queues[0].iter(&mem) {
             let used_len = match VsockPacket::from_rx_virtq_head(&avail_desc) {
                 Ok(mut pkt) => {
@@ -161,7 +163,7 @@ where
 
         let mut used_desc_heads = [(0, 0); QUEUE_SIZE as usize];
         let mut used_count = 0;
-        let mem = self.mem.read().unwrap();
+        let mem = self.mem.load();
         for avail_desc in self.queues[1].iter(&mem) {
             let pkt = match VsockPacket::from_tx_virtq_head(&avail_desc) {
                 Ok(pkt) => pkt,
@@ -380,8 +382,8 @@ pub struct Vsock<B: VsockBackend> {
     avail_features: u64,
     acked_features: u64,
     queue_evts: Option<Vec<EventFd>>,
-    interrupt_cb: Option<Arc<VirtioInterrupt>>,
-    epoll_thread: Option<thread::JoinHandle<result::Result<(), DeviceError>>>,
+    interrupt_cb: Option<Arc<dyn VirtioInterrupt>>,
+    epoll_threads: Option<Vec<thread::JoinHandle<result::Result<(), DeviceError>>>>,
     paused: Arc<AtomicBool>,
 }
 
@@ -407,7 +409,7 @@ where
             acked_features: 0u64,
             queue_evts: None,
             interrupt_cb: None,
-            epoll_thread: None,
+            epoll_threads: None,
             paused: Arc::new(AtomicBool::new(false)),
         })
     }
@@ -437,29 +439,12 @@ where
         QUEUE_SIZES
     }
 
-    fn features(&self, page: u32) -> u32 {
-        match page {
-            // Get the lower 32-bits of the features bitfield.
-            0 => self.avail_features as u32,
-            // Get the upper 32-bits of the features bitfield.
-            1 => (self.avail_features >> 32) as u32,
-            _ => {
-                warn!("Received request for unknown features page.");
-                0u32
-            }
-        }
+    fn features(&self) -> u64 {
+        self.avail_features
     }
 
-    fn ack_features(&mut self, page: u32, value: u32) {
-        let mut v = match page {
-            0 => u64::from(value),
-            1 => u64::from(value) << 32,
-            _ => {
-                warn!("Cannot acknowledge unknown features page.");
-                0u64
-            }
-        };
-
+    fn ack_features(&mut self, value: u64) {
+        let mut v = value;
         // Check if the guest is ACK'ing a feature that we didn't claim to have.
         let unrequested_features = v & !self.avail_features;
         if unrequested_features != 0 {
@@ -496,8 +481,8 @@ where
 
     fn activate(
         &mut self,
-        mem: Arc<RwLock<GuestMemoryMmap>>,
-        interrupt_cb: Arc<VirtioInterrupt>,
+        mem: Arc<ArcSwap<GuestMemoryMmap>>,
+        interrupt_cb: Arc<dyn VirtioInterrupt>,
         queues: Vec<Queue>,
         queue_evts: Vec<EventFd>,
     ) -> ActivateResult {
@@ -552,19 +537,22 @@ where
         };
 
         let paused = self.paused.clone();
+        let mut epoll_threads = Vec::new();
         thread::Builder::new()
             .name("virtio_vsock".to_string())
             .spawn(move || handler.run(paused))
-            .map(|thread| self.epoll_thread = Some(thread))
+            .map(|thread| epoll_threads.push(thread))
             .map_err(|e| {
                 error!("failed to clone the vsock epoll thread: {}", e);
                 ActivateError::BadActivate
             })?;
 
+        self.epoll_threads = Some(epoll_threads);
+
         Ok(())
     }
 
-    fn reset(&mut self) -> Option<(Arc<VirtioInterrupt>, Vec<EventFd>)> {
+    fn reset(&mut self) -> Option<(Arc<dyn VirtioInterrupt>, Vec<EventFd>)> {
         // We first must resume the virtio thread if it was paused.
         if self.pause_evt.take().is_some() {
             self.resume().ok()?;
@@ -583,19 +571,14 @@ where
     }
 }
 
-impl<B> Pausable for Vsock<B>
-where
-    B: VsockBackend + Sync + 'static,
-{
-    virtio_pausable_inner!();
-}
+virtio_pausable!(Vsock, T: 'static + VsockBackend + Sync);
 
 impl<B> Snapshotable for Vsock<B> where B: VsockBackend + Sync + 'static {}
 impl<B> Migratable for Vsock<B> where B: VsockBackend + Sync + 'static {}
 
 #[cfg(test)]
 mod tests {
-    use super::super::tests::TestContext;
+    use super::super::tests::{NoopVirtioInterrupt, TestContext};
     use super::super::*;
     use super::*;
     use crate::vsock::device::{BACKEND_EVENT, EVT_QUEUE_EVENT, RX_QUEUE_EVENT, TX_QUEUE_EVENT};
@@ -619,18 +602,15 @@ mod tests {
             VirtioDeviceType::TYPE_VSOCK as u32
         );
         assert_eq!(ctx.device.queue_max_sizes(), QUEUE_SIZES);
-        assert_eq!(ctx.device.features(0), device_pages[0]);
-        assert_eq!(ctx.device.features(1), device_pages[1]);
-        assert_eq!(ctx.device.features(2), 0);
+        assert_eq!((ctx.device.features() >> (0 * 32)) as u32, device_pages[0]);
+        assert_eq!((ctx.device.features() >> (1 * 32)) as u32, device_pages[1]);
 
         // Ack device features, page 0.
-        ctx.device.ack_features(0, driver_pages[0]);
+        ctx.device
+            .ack_features(u64::from(driver_pages[0]) << (0 * 32));
         // Ack device features, page 1.
-        ctx.device.ack_features(1, driver_pages[1]);
-        // Ack some bogus page (i.e. 2). This should have no side effect.
-        ctx.device.ack_features(2, 0);
-        // Attempt to un-ack the first feature page. This should have no side effect.
-        ctx.device.ack_features(0, !driver_pages[0]);
+        ctx.device
+            .ack_features(u64::from(driver_pages[1]) << (1 * 32));
         // Check that no side effect are present, and that the acked features are exactly the same
         // as the device features.
         assert_eq!(ctx.device.acked_features, device_features & driver_features);
@@ -664,11 +644,8 @@ mod tests {
 
         // Test a bad activation.
         let bad_activate = ctx.device.activate(
-            Arc::new(RwLock::new(ctx.mem.clone())),
-            Arc::new(
-                Box::new(move |_: &VirtioInterruptType, _: Option<&Queue>| Ok(()))
-                    as VirtioInterrupt,
-            ),
+            Arc::new(ArcSwap::from(Arc::new(ctx.mem.clone()))),
+            Arc::new(NoopVirtioInterrupt {}),
             Vec::new(),
             Vec::new(),
         );
@@ -680,11 +657,8 @@ mod tests {
         // Test a correct activation.
         ctx.device
             .activate(
-                Arc::new(RwLock::new(ctx.mem.clone())),
-                Arc::new(
-                    Box::new(move |_: &VirtioInterruptType, _: Option<&Queue>| Ok(()))
-                        as VirtioInterrupt,
-                ),
+                Arc::new(ArcSwap::new(Arc::new(ctx.mem.clone()))),
+                Arc::new(NoopVirtioInterrupt {}),
                 vec![Queue::new(256), Queue::new(256), Queue::new(256)],
                 vec![
                     EventFd::new(EFD_NONBLOCK).unwrap(),

vm-virtio/src/vsock/mod.rs

@@ -17,9 +17,8 @@ pub use self::device::Vsock;
 pub use self::unix::VsockUnixBackend;
 pub use self::unix::VsockUnixError;
 
-use std::os::unix::io::RawFd;
-
 use packet::VsockPacket;
+use std::os::unix::io::RawFd;
 
 mod defs {
 
@@ -158,22 +157,32 @@ pub trait VsockBackend: VsockChannel + VsockEpollListener + Send {}
 
 #[cfg(test)]
 mod tests {
-    use libc::EFD_NONBLOCK;
-
     use super::device::{VsockEpollHandler, RX_QUEUE_EVENT, TX_QUEUE_EVENT};
     use super::packet::VSOCK_PKT_HDR_SIZE;
     use super::*;
-
-    use std::os::unix::io::AsRawFd;
-    use std::sync::atomic::AtomicBool;
-    use std::sync::{Arc, RwLock};
-    use vmm_sys_util::eventfd::EventFd;
-
     use crate::device::{VirtioInterrupt, VirtioInterruptType};
     use crate::queue::tests::VirtQueue as GuestQ;
     use crate::queue::Queue;
     use crate::{VIRTQ_DESC_F_NEXT, VIRTQ_DESC_F_WRITE};
+    use arc_swap::ArcSwap;
+    use libc::EFD_NONBLOCK;
+    use std::os::unix::io::AsRawFd;
+    use std::sync::atomic::AtomicBool;
+    use std::sync::{Arc, RwLock};
     use vm_memory::{GuestAddress, GuestMemoryMmap};
+    use vmm_sys_util::eventfd::EventFd;
+
+    pub struct NoopVirtioInterrupt {}
+
+    impl VirtioInterrupt for NoopVirtioInterrupt {
+        fn trigger(
+            &self,
+            _int_type: &VirtioInterruptType,
+            _queue: Option<&Queue>,
+        ) -> std::result::Result<(), std::io::Error> {
+            Ok(())
+        }
+    }
 
     pub struct TestBackend {
         pub evfd: EventFd,
@@ -255,7 +264,7 @@ mod tests {
             const MEM_SIZE: usize = 1024 * 1024 * 128;
             Self {
                 cid: CID,
-                mem: GuestMemoryMmap::new(&[(GuestAddress(0), MEM_SIZE)]).unwrap(),
+                mem: GuestMemoryMmap::from_ranges(&[(GuestAddress(0), MEM_SIZE)]).unwrap(),
                 mem_size: MEM_SIZE,
                 device: Vsock::new(CID, TestBackend::new(), false).unwrap(),
             }
@@ -294,16 +303,14 @@ mod tests {
                 EventFd::new(EFD_NONBLOCK).unwrap(),
                 EventFd::new(EFD_NONBLOCK).unwrap(),
             ];
-            let interrupt_cb = Arc::new(Box::new(
-                move |_: &VirtioInterruptType, _: Option<&Queue>| Ok(()),
-            ) as VirtioInterrupt);
+            let interrupt_cb = Arc::new(NoopVirtioInterrupt {});
 
             EpollHandlerContext {
                 guest_rxvq,
                 guest_txvq,
                 guest_evvq,
                 handler: VsockEpollHandler {
-                    mem: Arc::new(RwLock::new(self.mem.clone())),
+                    mem: Arc::new(ArcSwap::new(Arc::new(self.mem.clone()))),
                     queues,
                     queue_evts,
                     kill_evt: EventFd::new(EFD_NONBLOCK).unwrap(),

vm-virtio/src/vsock/unix/mod.rs

@@ -28,12 +28,18 @@ mod defs {
 
 #[derive(Debug)]
 pub enum Error {
+    /// Error converting from UTF-8
+    ConvertFromUTF8(std::str::Utf8Error),
     /// Error registering a new epoll-listening FD.
     EpollAdd(std::io::Error),
     /// Error creating an epoll FD.
     EpollFdCreate(std::io::Error),
     /// The host made an invalid vsock port connection request.
     InvalidPortRequest,
+    /// Error parsing integer.
+    ParseInteger(std::num::ParseIntError),
+    /// Error reading stream port.
+    ReadStreamPort(Box<Error>),
     /// Error accepting a new connection from the host-side Unix socket.
     UnixAccept(std::io::Error),
     /// Error binding to the host-side Unix socket.

vm-virtio/src/vsock/unix/muxer.rs

@@ -32,10 +32,11 @@
 ///    mapping `RawFd`s to `EpollListener`s.
 ///
 use std::collections::{HashMap, HashSet};
-use std::io::Read;
+use std::io::{self, Read};
 use std::os::unix::io::{AsRawFd, RawFd};
 use std::os::unix::net::{UnixListener, UnixStream};
 
+use super::super::csm::ConnState;
 use super::super::defs::uapi;
 use super::super::packet::VsockPacket;
 use super::super::{
@@ -284,22 +285,35 @@ impl VsockEpollListener for VsockMuxer {
         debug!("vsock: muxer received kick");
 
         let mut epoll_events = vec![epoll::Event::new(epoll::Events::empty(), 0); 32];
-        match epoll::wait(self.epoll_fd, 0, epoll_events.as_mut_slice()) {
-            Ok(ev_cnt) => {
-                #[allow(clippy::needless_range_loop)]
-                for i in 0..ev_cnt {
-                    self.handle_event(
-                        epoll_events[i].data as RawFd,
-                        // It's ok to unwrap here, since the `epoll_events[i].events` is filled
-                        // in by `epoll::wait()`, and therefore contains only valid epoll
-                        // flags.
-                        epoll::Events::from_bits(epoll_events[i].events).unwrap(),
-                    );
+        'epoll: loop {
+            match epoll::wait(self.epoll_fd, 0, epoll_events.as_mut_slice()) {
+                Ok(ev_cnt) => {
+                    #[allow(clippy::needless_range_loop)]
+                    for i in 0..ev_cnt {
+                        self.handle_event(
+                            epoll_events[i].data as RawFd,
+                            // It's ok to unwrap here, since the `epoll_events[i].events` is filled
+                            // in by `epoll::wait()`, and therefore contains only valid epoll
+                            // flags.
+                            epoll::Events::from_bits(epoll_events[i].events).unwrap(),
+                        );
+                    }
+                }
+                Err(e) => {
+                    if e.kind() == io::ErrorKind::Interrupted {
+                        // It's well defined from the epoll_wait() syscall
+                        // documentation that the epoll loop can be interrupted
+                        // before any of the requested events occurred or the
+                        // timeout expired. In both those cases, epoll_wait()
+                        // returns an error of type EINTR, but this should not
+                        // be considered as a regular error. Instead it is more
+                        // appropriate to retry, by calling into epoll_wait().
+                        continue;
+                    }
+                    warn!("vsock: failed to consume muxer epoll event: {}", e);
                 }
             }
-            Err(e) => {
-                warn!("vsock: failed to consume muxer epoll event: {}", e);
-            }
+            break 'epoll;
         }
     }
 }
@@ -451,7 +465,7 @@ impl VsockMuxer {
         }
 
         let mut word_iter = std::str::from_utf8(&buf[..blen])
-            .map_err(|_| Error::InvalidPortRequest)?
+            .map_err(Error::ConvertFromUTF8)?
             .split_whitespace();
 
         word_iter
@@ -465,8 +479,8 @@ impl VsockMuxer {
                 }
             })
             .and_then(|_| word_iter.next().ok_or(Error::InvalidPortRequest))
-            .and_then(|word| word.parse::<u32>().map_err(|_| Error::InvalidPortRequest))
-            .map_err(|_| Error::InvalidPortRequest)
+            .and_then(|word| word.parse::<u32>().map_err(Error::ParseInteger))
+            .map_err(|e| Error::ReadStreamPort(Box::new(e)))
     }
 
     /// Add a new connection to the active connection pool.
@@ -652,9 +666,20 @@ impl VsockMuxer {
         if let Some(conn) = self.conn_map.get_mut(&key) {
             let had_rx = conn.has_pending_rx();
             let was_expiring = conn.will_expire();
+            let prev_state = conn.state();
 
             mut_fn(conn);
 
+            // If this is a host-initiated connection that has just become established, we'll have
+            // to send an ack message to the host end.
+            if prev_state == ConnState::LocalInit && conn.state() == ConnState::Established {
+                conn.send_bytes(format!("OK {}\n", key.local_port).as_bytes())
+                    .unwrap_or_else(|err| {
+                        conn.kill();
+                        warn!("vsock: unable to ack host connection: {:?}", err);
+                    });
+            }
+
             // If the connection wasn't previously scheduled for RX, add it to our RX queue.
             if !had_rx && conn.has_pending_rx() {
                 self.rxq.push(MuxerRx::ConnRx(key));
@@ -917,6 +942,10 @@ mod tests {
             self.init_pkt(local_port, peer_port, uapi::VSOCK_OP_RESPONSE);
             self.send();
 
+            let mut buf = vec![0u8; 32];
+            let len = stream.read(&mut buf[..]).unwrap();
+            assert_eq!(&buf[..len], format!("OK {}\n", local_port).as_bytes());
+
             (stream, local_port)
         }
     }

vmm/Cargo.toml

@@ -12,29 +12,31 @@ mmio_support = ["vm-virtio/mmio_support"]
 cmos = ["devices/cmos"]
 
 [dependencies]
+arc-swap = "0.4.4"
+clap = "2.33.0"
 acpi_tables = { path = "../acpi_tables", optional = true }
-anyhow = "1.0"
+anyhow = "1.0.26"
 arch = { path = "../arch" }
 devices = { path = "../devices" }
-epoll = ">=4.0.1"
+epoll = "4.1.0"
 kvm-bindings = "0.2.0"
-kvm-ioctls = "0.4.0"
+kvm-ioctls = "0.5.0"
 lazy_static = "1.4.0"
-libc = "0.2.62"
+libc = "0.2.66"
 log = "0.4.8"
 micro_http = { git = "https://github.com/firecracker-microvm/firecracker", branch = "master" }
 net_util = { path = "../net_util" }
 pci = {path = "../pci", optional = true}
 qcow = { path = "../qcow" }
-serde = {version = ">=1.0.27", features = ["rc"] }
-serde_derive = ">=1.0.27"
-serde_json = ">=1.0.9"
+serde = { version = "1.0.104", features = ["rc"] }
+serde_derive = "1.0.104"
+serde_json = "1.0.48"
 vfio = { path = "../vfio", optional = true }
 vm-allocator = { path = "../vm-allocator" }
 vm-device = { path = "../vm-device" }
 vm-virtio = { path = "../vm-virtio" }
-vmm-sys-util = ">=0.3.1"
-signal-hook = "0.1.10"
+vmm-sys-util = "0.4.0"
+signal-hook = "0.1.13"
 
 [dependencies.linux-loader]
 git = "https://github.com/rust-vmm/linux-loader"

vmm/src/acpi.rs

@@ -11,11 +11,11 @@ use vm_memory::{GuestAddress, GuestMemoryMmap};
 
 use vm_memory::{Address, ByteValued, Bytes};
 
-use std::convert::TryInto;
 use std::sync::{Arc, Mutex};
 
 use crate::cpu::CpuManager;
 use crate::device_manager::DeviceManager;
+use crate::memory_manager::MemoryManager;
 use arch::layout;
 
 #[repr(packed)]
@@ -82,12 +82,14 @@ struct IortIdMapping {
 pub fn create_dsdt_table(
     device_manager: &DeviceManager,
     cpu_manager: &Arc<Mutex<CpuManager>>,
+    memory_manager: &Arc<Mutex<MemoryManager>>,
 ) -> SDT {
     // DSDT
     let mut dsdt = SDT::new(*b"DSDT", 36, 6, *b"CLOUDH", *b"CHDSDT  ", 1);
 
     dsdt.append_slice(device_manager.to_aml_bytes().as_slice());
     dsdt.append_slice(cpu_manager.lock().unwrap().to_aml_bytes().as_slice());
+    dsdt.append_slice(memory_manager.lock().unwrap().to_aml_bytes().as_slice());
 
     dsdt
 }
@@ -96,13 +98,14 @@ pub fn create_acpi_tables(
     guest_mem: &GuestMemoryMmap,
     device_manager: &DeviceManager,
     cpu_manager: &Arc<Mutex<CpuManager>>,
+    memory_manager: &Arc<Mutex<MemoryManager>>,
 ) -> GuestAddress {
     // RSDP is at the EBDA
     let rsdp_offset = layout::RSDP_POINTER;
     let mut tables: Vec<u64> = Vec::new();
 
     // DSDT
-    let dsdt = create_dsdt_table(device_manager, cpu_manager);
+    let dsdt = create_dsdt_table(device_manager, cpu_manager, memory_manager);
     let dsdt_offset = rsdp_offset.checked_add(RSDP::len() as u64).unwrap();
     guest_mem
         .write_slice(dsdt.as_slice(), dsdt_offset)
@@ -167,66 +170,6 @@ pub fn create_acpi_tables(
         .expect("Error writing MCFG table");
     tables.push(mcfg_offset.0);
 
-    let (prev_tbl_len, prev_tbl_off) =
-        if let Some((iommu_id, dev_ids)) = &device_manager.virt_iommu() {
-            // IORT
-            let mut iort = SDT::new(*b"IORT", 36, 1, *b"CLOUDH", *b"CHIORT  ", 1);
-            // IORT number of nodes
-            iort.append(2u32);
-            // IORT offset to array of IORT nodes
-            iort.append(48u32);
-            // IORT reserved 4 bytes
-            iort.append(0u32);
-            // IORT paravirtualized IOMMU node
-            iort.append(IortParavirtIommuNode {
-                type_: 128,
-                length: 56,
-                revision: 0,
-                num_id_mappings: 0,
-                ref_id_mappings: 56,
-                device_id: *iommu_id,
-                model: 1,
-                ..Default::default()
-            });
-
-            let num_entries = dev_ids.len();
-            let length: u16 = (36 + (20 * num_entries)).try_into().unwrap();
-
-            // IORT PCI root complex node
-            iort.append(IortPciRootComplexNode {
-                type_: 2,
-                length,
-                revision: 0,
-                num_id_mappings: num_entries as u32,
-                ref_id_mappings: 36,
-                ats_attr: 0,
-                pci_seg_num: 0,
-                mem_addr_size_limit: 255,
-                ..Default::default()
-            });
-
-            for dev_id in dev_ids.iter() {
-                // IORT ID mapping
-                iort.append(IortIdMapping {
-                    input_base: *dev_id,
-                    num_of_ids: 1,
-                    ouput_base: *dev_id,
-                    output_ref: 48,
-                    flags: 0,
-                });
-            }
-
-            let iort_offset = mcfg_offset.checked_add(mcfg.len() as u64).unwrap();
-            guest_mem
-                .write_slice(iort.as_slice(), iort_offset)
-                .expect("Error writing IORT table");
-            tables.push(iort_offset.0);
-
-            (iort.len(), iort_offset)
-        } else {
-            (mcfg.len(), mcfg_offset)
-        };
-
     // XSDT
     let mut xsdt = SDT::new(*b"XSDT", 36, 1, *b"CLOUDH", *b"CHXSDT  ", 1);
     for table in tables {
@@ -234,7 +177,7 @@ pub fn create_acpi_tables(
     }
     xsdt.update_checksum();
 
-    let xsdt_offset = prev_tbl_off.checked_add(prev_tbl_len as u64).unwrap();
+    let xsdt_offset = mcfg_offset.checked_add(mcfg.len() as u64).unwrap();
     guest_mem
         .write_slice(xsdt.as_slice(), xsdt_offset)
         .expect("Error writing XSDT table");

vmm/src/api/http_endpoint.rs

@@ -44,6 +44,9 @@ pub enum HttpError {
     /// Could not act on a VM
     VmAction(ApiError),
 
+    /// Could not resize a VM
+    VmResize(ApiError),
+
     /// Could not shut the VMM down
     VmmShutdown(ApiError),
 
@@ -244,7 +247,7 @@ impl EndpointHandler for VmResize {
 
                         // Call vm_resize()
                         match vm_resize(api_notifier, api_sender, Arc::new(vm_resize_data))
-                            .map_err(HttpError::VmCreate)
+                            .map_err(HttpError::VmResize)
                         {
                             Ok(_) => Response::new(Version::Http11, StatusCode::NoContent),
                             Err(e) => error_response(e, StatusCode::InternalServerError),

vmm/src/api/mod.rs

@@ -115,7 +115,8 @@ pub struct VmmPingResponse {
 
 #[derive(Clone, Deserialize, Serialize)]
 pub struct VmResizeData {
-    pub desired_vcpus: u8,
+    pub desired_vcpus: Option<u8>,
+    pub desired_ram: Option<u64>,
 }
 
 pub enum ApiResponsePayload {

vmm/src/api/openapi/cloud-hypervisor.yaml

@@ -274,9 +274,26 @@ components:
       properties:
         path:
           type: string
+        readonly:
+          type: boolean
+          default: false
         iommu:
           type: boolean
           default: false
+        num_queues:
+          type: integer
+          default: 1
+        queue_size:
+          type: integer
+          default: 128
+        vhost_user:
+          type: boolean
+          default: false
+        vhost_socket:
+          type: string
+        wce:
+          type: boolean
+          default: true
 
     NetConfig:
       type: object
@@ -295,6 +312,17 @@ components:
         iommu:
           type: boolean
           default: false
+        num_queues:
+          type: integer
+          default: 2
+        queue_size:
+          type: integer
+          default: 256
+        vhost_user:
+          type: boolean
+          default: false
+        vhost_socket:
+          type: string
 
     RngConfig:
       required:
@@ -427,11 +455,10 @@ components:
           default: false
 
     VmResize:
-      required:
-      - desired_vcpus
       type: object
       properties:
         desired_vcpus:
           minimum: 1
-          default: 1
+          type: integer
+        desired_ram:
           type: integer