cargo: Update Cargo.lock for the 0.4.0 release

Signed-off-by: Samuel Ortiz <sameo@linux.intel.com>

.travis.yml

@@ -3,17 +3,9 @@ language: rust
 rust:
   - stable
 
-before_script:
-  - rustup component add clippy
-  - rustup component add rustfmt
-
 script:
   - cargo build --release
-  - cargo build --release --no-default-features
+  
-  - cargo test
-  - cargo clippy --all-targets --all-features -- -D warnings
-  - find . -name "*.rs" | xargs rustfmt --check
-
 deploy:
   provider: releases
   api_key: $GITHUB_OAUTH_TOKEN

CONTRIBUTING.md

@@ -51,7 +51,7 @@ Signed-off-by: Rob Bradford <robert.bradford@intel.com>
 Cloud Hypervisor uses the “fork-and-pull” development model. Follow these steps if
 you want to merge your changes to `cloud-hypervisor`:
 
-1. Fork the [cloud-hypervisor](https://github.com/intel/cloud-hypervisor) project
+1. Fork the [cloud-hypervisor](https://github.com/cloud-hypervisor/cloud-hypervisor) project
    into your github organization.
 2. Within your fork, create a branch for your contribution.
 3. [Create a pull request](https://help.github.com/articles/creating-a-pull-request-from-a-fork/)
@@ -65,7 +65,7 @@ you want to merge your changes to `cloud-hypervisor`:
 ## Issue tracking
 
 If you have a problem, please let us know. We recommend using
-[github issues](https://github.com/intel/cloud-hypervisor/issues/new) for formally
+[github issues](https://github.com/cloud-hypervisor/cloud-hypervisor/issues/new) for formally
 reporting and documenting them.
 
 To quickly and informally bring something up to us, you can also reach out on [Slack](https://cloud-hypervisor.slack.com).

Cargo.toml

@@ -1,26 +1,50 @@
 [package]
 name = "cloud-hypervisor"
-version = "0.1.0"
+version = "0.4.0"
 authors = ["The Cloud Hypervisor Authors"]
 edition = "2018"
+default-run = "cloud-hypervisor"
 
 [dependencies]
 clap = "2.33.0"
+epoll = ">=4.0.1"
 lazy_static = "1.4.0"
+libc = "0.2.66"
 log = { version = "0.4.8", features = ["std"] }
+net_gen = { path = "net_gen" }
+net_util = { path = "net_util" }
+qcow = { path = "qcow" }
+vhost_user_backend = { path = "vhost_user_backend"}
+vhost_user_fs = { path = "vhost_user_fs"}
+virtio-bindings = "0.1.0"
 vmm = { path = "vmm" }
+vm-device = { path = "vm-device" }
+vm-memory = { git = "https://github.com/rust-vmm/vm-memory" }
+vmm-sys-util = ">=0.3.1"
+vm-virtio = { path = "vm-virtio" }
 
 [dev-dependencies]
-ssh2 = "0.3.3"
+ssh2 = "0.6.0"
 dirs = "2.0.2"
 credibility = "0.1.3"
 tempdir= "0.3.7"
 lazy_static= "1.4.0"
 
+[dependencies.vhost_rs]
+path = "vhost_rs"
+features = ["vhost-user-slave"]
+
 [features]
-default = []
+default = ["acpi", "pci", "cmos"]
+acpi = ["vmm/acpi"]
+pci = ["vmm/pci_support"]
+mmio = ["vmm/mmio_support"]
+cmos = ["vmm/cmos"]
 
 # Integration tests require a special environment to run in
 integration_tests = []
 
 [workspace]
+members = [
+    "vhost_user_backend",
+]

Jenkinsfile

@@ -1,22 +1,85 @@
-stage ("Builds") {
+pipeline{
-	node ('bionic') {
+	agent none
-		stage ('Checkout') {
+	stages {
-			checkout scm
+		stage ('Master build') {
+			agent { node { label 'master' } }
+			stages {
+				stage ('Check for RFC/WIP builds') {
+					when {
+  						changeRequest comparator: 'REGEXP', title: '.*(rfc|RFC|wip|WIP).*'
+  						beforeAgent true
+					}
+					steps {
+						error("Failing as this is marked as a WIP or RFC PR.")
+					}
+				}
+				stage ('Cancel older builds') {
+					steps {
+						cancelPreviousBuilds()
+					}
+				}
+			}
 		}
-		stage ('Install system packages') {
+		stage ('Worker build') {
-			sh "sudo DEBIAN_FRONTEND=noninteractive apt-get install -yq build-essential mtools libssl-dev pkg-config"
+			agent { node { label 'bionic' } }
-			sh "sudo apt-get install -yq flex bison libelf-dev qemu-utils qemu-system libglib2.0-dev libpixman-1-dev libseccomp-dev"
+			options {
-		}
+				timeout(time: 1, unit: 'HOURS')
-		stage ('Install Rust') {
+			}
-			sh "nohup curl https://sh.rustup.rs -sSf | sh -s -- -y"
+			stages {
-		}
+				stage ('Checkout') {
-		stage ('Run unit tests') {
+					steps {
-			sh "scripts/run_unit_tests.sh"
+						checkout scm
-		}
+					}
-		stage ('Run integration tests') {
+				}
-			sh "sudo mount -t tmpfs tmpfs /tmp"
+				stage ('Install system packages') {
-			sh "scripts/run_integration_tests.sh"
+					steps {
+						sh "sudo DEBIAN_FRONTEND=noninteractive apt-get install -yq build-essential mtools libssl-dev pkg-config"
+							sh "sudo apt-get install -yq flex bison libelf-dev qemu-utils qemu-system libglib2.0-dev libpixman-1-dev libseccomp-dev libcap-ng-dev socat"
+							sh "sudo snap install docker"
+					}
+				}
+				stage ('Install Rust') {
+					steps {
+						sh "nohup curl https://sh.rustup.rs -sSf | sh -s -- -y"
+					}
+				}
+				stage ('Run Cargo tests') {
+					steps {
+						sh "scripts/run_cargo_tests.sh"
+					}
+				}
+				stage ('Run OpenAPI tests') {
+					steps {
+						sh "scripts/run_openapi_tests.sh"
+					}
+				}
+				stage ('Run unit tests') {
+					steps {
+						sh "scripts/run_unit_tests.sh"
+					}
+				}
+				stage ('Run integration tests') {
+					steps {
+						sh "sudo mount -t tmpfs tmpfs /tmp"
+							sh "scripts/run_integration_tests.sh"
+					}
+				}
+			}
 		}
 	}
 }
 
+def cancelPreviousBuilds() {
+	// Check for other instances of this particular build, cancel any that are older than the current one
+	def jobName = env.JOB_NAME
+		def currentBuildNumber = env.BUILD_NUMBER.toInteger()
+		def currentJob = Jenkins.instance.getItemByFullName(jobName)
+
+		// Loop through all instances of this particular job/branch
+		for (def build : currentJob.builds) {
+			if (build.isBuilding() && (build.number.toInteger() < currentBuildNumber)) {
+				echo "Older build still queued. Sending kill signal to build number: ${build.number}"
+				build.doStop()
+			}
+		}
+}

README.md

@@ -1,4 +1,4 @@
-[![Build Status](https://travis-ci.com/intel/cloud-hypervisor.svg?branch=master)](https://travis-ci.com/intel/cloud-hypervisor)
+[![Build Status](https://travis-ci.com/cloud-hypervisor/cloud-hypervisor.svg?branch=master)](https://travis-ci.com/cloud-hypervisor/cloud-hypervisor)
 
 1. [What is Cloud Hypervisor?](#1-what-is-cloud-hypervisor)
    * [Requirements](#requirements)
@@ -14,6 +14,7 @@
 		 - [Disk image](#disk-image)
 		 - [Booting the guest VM](#booting-the-guest-vm)
 3. [Status](#2-status)
+	* [Device Model](#device-model)
 	* [TODO](#todo)
 4. [rust-vmm dependency](#4-rust-vmm-dependency)
 	* [Firecracker and crosvm](#firecracker-and-crosvm)
@@ -73,7 +74,7 @@ First you need to clone and build the cloud-hypervisor repo:
 
 ```shell
 $ pushd $CLOUDH
-$ git clone https://github.com/intel/cloud-hypervisor.git
+$ git clone https://github.com/cloud-hypervisor/cloud-hypervisor.git
 $ cd cloud-hypervisor
 $ cargo build --release
 
@@ -94,7 +95,7 @@ You can run a guest VM by either using an existing cloud image or booting into y
 `cloud-hypervisor` supports booting disk images containing all needed
 components to run cloud workloads, a.k.a. cloud images.  To do that we rely on
 the [Rust Hypervisor
-Firmware](https://github.com/intel/rust-hypervisor-firmware) project to provide
+Firmware](https://github.com/cloud-hypervisor/rust-hypervisor-firmware) project to provide
 an ELF
 formatted KVM firmware for `cloud-hypervisor` to directly boot into.
 
@@ -102,9 +103,9 @@ We need to get the latest `rust-hypervisor-firmware` release and also a working
 
 ```shell
 $ pushd $CLOUDH
-$ wget https://download.clearlinux.org/releases/29160/clear/clear-29160-kvm.img.xz
+$ wget https://download.clearlinux.org/releases/31890/clear/clear-31890-kvm.img.xz
-$ unxz clear-29160-kvm.img.xz
+$ unxz clear-31890-kvm.img.xz
-$ wget https://github.com/intel/rust-hypervisor-firmware/releases/download/0.1.0/hypervisor-fw
+$ wget https://github.com/cloud-hypervisor/rust-hypervisor-firmware/releases/download/0.2.6/hypervisor-fw
 $ popd
 ```
 
@@ -113,15 +114,15 @@ $ pushd $CLOUDH
 $ sudo setcap cap_net_admin+ep ./cloud-hypervisor/target/release/cloud-hypervisor
 $ ./cloud-hypervisor/target/release/cloud-hypervisor \
 	--kernel ./hypervisor-fw \
-	--disk ./clear-29160-kvm.img \
+	--disk path=clear-31890-kvm.img \
-	--cpus 4 \
+	--cpus boot=4 \
 	--memory size=1024M \
 	--net "tap=,mac=,ip=,mask=" \
 	--rng
 $ popd
 ```
 
-Multiple arguments can be given to the `--disk` parameter, currently the firmware requires that the bootable image is on the first disk.
+Multiple arguments can be given to the `--disk` parameter.
 
 ### Custom kernel and disk image
 
@@ -152,8 +153,8 @@ For the disk image, we will use a Clear Linux cloud image that contains a root p
 
 ```shell
 $ pushd $CLOUDH
-$ wget https://download.clearlinux.org/releases/29160/clear/clear-29160-kvm.img.xz
+$ wget https://download.clearlinux.org/releases/31890/clear/clear-31890-kvm.img.xz
-$ unxz clear-29160-kvm.img.xz
+$ unxz clear-31890-kvm.img.xz
 $ popd
 ```
 
@@ -167,9 +168,9 @@ $ pushd $CLOUDH
 $ sudo setcap cap_net_admin+ep ./cloud-hypervisor/target/release/cloud-hypervisor
 $ ./cloud-hypervisor/target/release/cloud-hypervisor \
 	--kernel ./linux-cloud-hypervisor/arch/x86/boot/compressed/vmlinux.bin \
-	--disk ./clear-29160-kvm.img \
+	--disk path=clear-31890-kvm.img \
 	--cmdline "console=hvc0 reboot=k panic=1 nomodules i8042.noaux i8042.nomux i8042.nopnp i8042.dumbkbd root=/dev/vda3" \
-	--cpus 4 \
+	--cpus boot=4 \
 	--memory size=1024M \
 	--net "tap=,mac=,ip=,mask=" \
 	--rng
@@ -187,27 +188,38 @@ $ ./cloud-hypervisor/target/release/cloud-hypervisor \
 	--kernel ./linux-cloud-hypervisor/arch/x86/boot/compressed/vmlinux.bin \
 	--console off \
 	--serial tty \
-	--disk ./clear-29160-kvm.img \
+	--disk path=clear-31890-kvm.img \
 	--cmdline "console=ttyS0 reboot=k panic=1 nomodules i8042.noaux i8042.nomux i8042.nopnp i8042.dumbkbd root=/dev/vda3" \
-	--cpus 4 \
+	--cpus boot=4 \
 	--memory size=1024M \
 	--net "tap=,mac=,ip=,mask=" \
 	--rng
 ```
 
-
 # 3. Status
 
 `cloud-hypervisor` is in a very early, pre-alpha stage. Use at your own risk!
 
-As of 2019/05/12, booting cloud images has only been tested with [Clear Linux images](https://download.clearlinux.org/current/).
+As of 2019-12-12, the following cloud images are supported:
-Direct kernel boot to userspace should work with most rootfs and it's been tested with
+* [Clear Linux](https://download.clearlinux.org/current/) (cloudguest and kvm)
-Clear Linux root partitions, and also basic initrd/initramfs images.
+* [Ubuntu Bionic](https://cloud-images.ubuntu.com/bionic/current/) (cloudimg)
+* [Ubuntu Eoan](https://cloud-images.ubuntu.com/eoan/current/) (cloudimg)
+
+Direct kernel boot to userspace should work with most rootfs.
+
+## Hot Plug
+
+This [document](https://github.com/cloud-hypervisor/cloud-hypervisor/blob/master/docs/hotplug.md) details how to add devices to
+a running VM. Currently only CPU hot plug is supported.
+
+## Device Model
+
+Follow this [documentation](https://github.com/cloud-hypervisor/cloud-hypervisor/blob/master/docs/device_model.md).
 
 ## TODO
 
 We are not tracking the `cloud-hypervisor` TODO list from a specific git tracked file but through
-[github issues](https://github.com/intel/cloud-hypervisor/issues/new) instead.
+[github issues](https://github.com/cloud-hypervisor/cloud-hypervisor/issues/new) instead.
 
 # 4. `rust-vmm` project dependency
 
@@ -255,7 +267,7 @@ etc, are all equal and welcome means of contribution. See the [CONTRIBUTING](CON
 
 ## Join us
 
-Get an [invite to our Slack channel](https://join.slack.com/t/cloud-hypervisor/shared_invite/enQtNjY3MTE3MDkwNDQ4LTc0YzlmYzQxZDkxNDVhYzZjZjA5MTkxMGY3NTI3YzMzYTFkM2IyY2E0YTIxMzkyYTEwYzdlMzBhMWYxYzVmNDI)
+Get an [invite to our Slack channel](https://join.slack.com/t/cloud-hypervisor/shared_invite/enQtNjY3MTE3MDkwNDQ4LWQ1MTA1ZDVmODkwMWQ1MTRhYzk4ZGNlN2UwNTI3ZmFlODU0OTcwOWZjMTkwZDExYWE3YjFmNzgzY2FmNDAyMjI)
 and [join us on Slack](https://cloud-hypervisor.slack.com/).
 
 # 6. Security

acpi_tables/src/lib.rs

@@ -3,6 +3,7 @@
 // SPDX-License-Identifier: Apache-2.0
 //
 
+pub mod aml;
 pub mod rsdp;
 pub mod sdt;
 

acpi_tables/src/rsdp.rs

@@ -36,7 +36,7 @@ impl RSDP {
         };
 
         rsdp.checksum = super::generate_checksum(&rsdp.as_slice()[0..19]);
-        rsdp.extended_checksum = super::generate_checksum(&rsdp.as_slice());;
+        rsdp.extended_checksum = super::generate_checksum(&rsdp.as_slice());
         rsdp
     }
 

acpi_tables/src/sdt.rs

@@ -77,6 +77,14 @@ impl SDT {
         self.write(orig_length, value);
     }
 
+    pub fn append_slice(&mut self, data: &[u8]) {
+        let orig_length = self.data.len();
+        let new_length = orig_length + data.len();
+        self.write_u32(4, new_length as u32);
+        self.data.extend_from_slice(data);
+        self.update_checksum();
+    }
+
     /// Write a value at the given offset
     pub fn write<T>(&mut self, offset: usize, value: T) {
         assert!((offset + (std::mem::size_of::<T>() - 1)) < self.data.len());

arch/Cargo.toml

@@ -4,13 +4,12 @@ version = "0.1.0"
 authors = ["The Chromium OS Authors"]
 
 [features]
-default = ["acpi"]
+default = []
-acpi = ["acpi_tables"]
 
 [dependencies]
 byteorder = "1.3.2"
-kvm-bindings = "0.1.1"
+kvm-bindings = "0.2.0"
-kvm-ioctls = { git = "https://github.com/rust-vmm/kvm-ioctls", branch = "master" }
+kvm-ioctls = "0.4.0"
 libc = "0.2.60"
 
 acpi_tables = { path = "../acpi_tables", optional = true }

arch/src/aarch64/mod.rs

@@ -16,7 +16,7 @@ pub fn configure_system(
     _cmdline_addr: GuestAddress,
     _cmdline_size: usize,
     _num_cpus: u8,
-    _serial_enabled: bool,
+    _rsdp_addr: Option<GuestAddress>,
 ) -> super::Result<()> {
     Ok(())
 }

arch/src/lib.rs

@@ -13,6 +13,7 @@ extern crate byteorder;
 extern crate kvm_bindings;
 extern crate libc;
 
+#[cfg(feature = "acpi")]
 extern crate acpi_tables;
 extern crate arch_gen;
 extern crate kvm_ioctls;
@@ -20,7 +21,6 @@ extern crate linux_loader;
 extern crate vm_memory;
 
 use std::result;
-use vm_memory::GuestAddress;
 
 #[derive(Debug, PartialEq)]
 pub enum Error {
@@ -52,9 +52,6 @@ pub enum RegionType {
     Reserved,
 }
 
-// 1MB.  We don't put anything above here except the kernel itself.
-pub const HIMEM_START: GuestAddress = GuestAddress(0x100000);
-
 #[cfg(target_arch = "aarch64")]
 pub mod aarch64;
 
@@ -69,6 +66,5 @@ pub mod x86_64;
 
 #[cfg(target_arch = "x86_64")]
 pub use x86_64::{
-    arch_memory_regions, configure_system, get_32bit_gap_start as get_reserved_mem_addr,
+    arch_memory_regions, configure_system, layout, layout::CMDLINE_MAX_SIZE, layout::CMDLINE_START,
-    layout::CMDLINE_MAX_SIZE, layout::CMDLINE_START,
 };

arch/src/x86_64/acpi.rs

@@ -1,315 +0,0 @@
-// Copyright © 2019 Intel Corporation
-//
-// SPDX-License-Identifier: Apache-2.0
-//
-use acpi_tables::{
-    rsdp::RSDP,
-    sdt::{GenericAddress, SDT},
-};
-use vm_memory::{GuestAddress, GuestMemoryMmap};
-
-use vm_memory::{Address, ByteValued, Bytes};
-
-#[repr(packed)]
-struct LocalAPIC {
-    pub r#type: u8,
-    pub length: u8,
-    pub processor_id: u8,
-    pub apic_id: u8,
-    pub flags: u32,
-}
-
-#[repr(packed)]
-#[derive(Default)]
-struct IOAPIC {
-    pub r#type: u8,
-    pub length: u8,
-    pub ioapic_id: u8,
-    _reserved: u8,
-    pub apic_address: u32,
-    pub gsi_base: u32,
-}
-
-#[repr(packed)]
-#[derive(Default)]
-struct InterruptSourceOverride {
-    pub r#type: u8,
-    pub length: u8,
-    pub bus: u8,
-    pub source: u8,
-    pub gsi: u32,
-    pub flags: u16,
-}
-
-#[repr(packed)]
-#[derive(Default)]
-struct PCIRangeEntry {
-    pub base_address: u64,
-    pub segment: u16,
-    pub start: u8,
-    pub end: u8,
-    _reserved: u32,
-}
-
-pub fn create_dsdt_table(serial_enabled: bool) -> SDT {
-    /*
-        The hex tables in this file are generated from the ASL below with:
-        "iasl -tc <dsdt.asl>"
-
-        As the output contains a table header that is not required the first 40 bytes
-        should be disregarded.
-    */
-
-    /*
-    Device (_SB.PCI0)
-        {
-            Name (_HID, EisaId ("PNP0A08") /* PCI Express Bus */)  // _HID: Hardware ID
-            Name (_CID, EisaId ("PNP0A03") /* PCI Bus */)  // _CID: Compatible ID
-            Name (_ADR, Zero)  // _ADR: Address
-            Name (_SEG, Zero)  // _SEG: PCI Segment
-            Name (_UID, Zero)  // _UID: Unique ID
-            Name (SUPP, Zero)
-        }
-
-        Scope (_SB.PCI0)
-        {
-            Name (_CRS, ResourceTemplate ()  // _CRS: Current Resource Settings
-            {
-                WordBusNumber (ResourceProducer, MinFixed, MaxFixed, PosDecode,
-                    0x0000,             // Granularity
-                    0x0000,             // Range Minimum
-                    0x00FF,             // Range Maximum
-                    0x0000,             // Translation Offset
-                    0x0100,             // Length
-                    ,, )
-                IO (Decode16,
-                    0x0CF8,             // Range Minimum
-                    0x0CF8,             // Range Maximum
-                    0x01,               // Alignment
-                    0x08,               // Length
-                    )
-                WordIO (ResourceProducer, MinFixed, MaxFixed, PosDecode, EntireRange,
-                    0x0000,             // Granularity
-                    0x0000,             // Range Minimum
-                    0x0CF7,             // Range Maximum
-                    0x0000,             // Translation Offset
-                    0x0CF8,             // Length
-                    ,, , TypeStatic, DenseTranslation)
-                WordIO (ResourceProducer, MinFixed, MaxFixed, PosDecode, EntireRange,
-                    0x0000,             // Granularity
-                    0x0D00,             // Range Minimum
-                    0xFFFF,             // Range Maximum
-                    0x0000,             // Translation Offset
-                    0xF300,             // Length
-                    ,, , TypeStatic, DenseTranslation)
-                DWordMemory (ResourceProducer, PosDecode, MinFixed, MaxFixed, Cacheable, ReadWrite,
-                    0x00000000,         // Granularity
-                    0x000A0000,         // Range Minimum
-                    0x000BFFFF,         // Range Maximum
-                    0x00000000,         // Translation Offset
-                    0x00020000,         // Length
-                    ,, , AddressRangeMemory, TypeStatic)
-                DWordMemory (ResourceProducer, PosDecode, MinFixed, MaxFixed, NonCacheable, ReadWrite,
-                    0x00000000,         // Granularity
-                    0xC0000000,         // Range Minimum
-                    0xFEC00000,         // Range Maximum
-                    0x00000000,         // Translation Offset
-                    0x3EC00001,         // Length
-                    ,, , AddressRangeMemory, TypeStatic)
-                QWordMemory (ResourceProducer, PosDecode, MinFixed, MaxFixed, Cacheable, ReadWrite,
-                    0x0000000000000000, // Granularity
-                    0x0000000800000000, // Range Minimum
-                    0x0000000FFFFFFFFF, // Range Maximum
-                    0x0000000000000000, // Translation Offset
-                    0x0000000800000000, // Length
-                    ,, , AddressRangeMemory, TypeStatic)
-            })
-        }
-    */
-    let pci_dsdt_data = [
-        0x5Bu8, 0x82, 0x36, 0x2E, 0x5F, 0x53, 0x42, 0x5F, 0x50, 0x43, 0x49, 0x30, 0x08, 0x5F, 0x48,
-        0x49, 0x44, 0x0C, 0x41, 0xD0, 0x0A, 0x08, 0x08, 0x5F, 0x43, 0x49, 0x44, 0x0C, 0x41, 0xD0,
-        0x0A, 0x03, 0x08, 0x5F, 0x41, 0x44, 0x52, 0x00, 0x08, 0x5F, 0x53, 0x45, 0x47, 0x00, 0x08,
-        0x5F, 0x55, 0x49, 0x44, 0x00, 0x08, 0x53, 0x55, 0x50, 0x50, 0x00, 0x10, 0x41, 0x0B, 0x2E,
-        0x5F, 0x53, 0x42, 0x5F, 0x50, 0x43, 0x49, 0x30, 0x08, 0x5F, 0x43, 0x52, 0x53, 0x11, 0x40,
-        0x0A, 0x0A, 0x9C, 0x88, 0x0D, 0x00, 0x02, 0x0C, 0x00, 0x00, 0x00, 0x00, 0x00, 0xFF, 0x00,
-        0x00, 0x00, 0x00, 0x01, 0x47, 0x01, 0xF8, 0x0C, 0xF8, 0x0C, 0x01, 0x08, 0x88, 0x0D, 0x00,
-        0x01, 0x0C, 0x03, 0x00, 0x00, 0x00, 0x00, 0xF7, 0x0C, 0x00, 0x00, 0xF8, 0x0C, 0x88, 0x0D,
-        0x00, 0x01, 0x0C, 0x03, 0x00, 0x00, 0x00, 0x0D, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0xF3, 0x87,
-        0x17, 0x00, 0x00, 0x0C, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0A, 0x00, 0xFF, 0xFF,
-        0x0B, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x87, 0x17, 0x00, 0x00, 0x0C,
-        0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xC0, 0x00, 0x00, 0xC0, 0xFE, 0x00, 0x00,
-        0x00, 0x00, 0x01, 0x00, 0xC0, 0x3E, 0x8A, 0x2B, 0x00, 0x00, 0x0C, 0x03, 0x00, 0x00, 0x00,
-        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0xFF, 0xFF,
-        0xFF, 0xFF, 0x0F, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-        0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x79, 0x00,
-    ];
-
-    /*
-    Device (_SB.COM1)
-    {
-        Name (_HID, EisaId ("PNP0501") /* 16550A-compatible COM Serial Port */)  // _HID: Hardware ID
-        Name (_UID, Zero)  // _UID: Unique ID
-        Name (_CRS, ResourceTemplate ()  // _CRS: Current Resource Settings
-        {
-            Interrupt (ResourceConsumer, Edge, ActiveHigh, Exclusive, ,, )
-            {
-                0x00000004,
-            }
-            IO (Decode16,
-                0x03F8,             // Range Minimum
-                0x03F8,             // Range Maximum
-                0x00,               // Alignment
-                0x08,               // Length
-                )
-        })
-    }
-    */
-    let com1_dsdt_data = [
-        0x5Bu8, 0x82, 0x36, 0x2E, 0x5F, 0x53, 0x42, 0x5F, 0x43, 0x4F, 0x4D, 0x31, 0x08, 0x5F, 0x48,
-        0x49, 0x44, 0x0C, 0x41, 0xD0, 0x05, 0x01, 0x08, 0x5F, 0x55, 0x49, 0x44, 0x00, 0x08, 0x5F,
-        0x43, 0x52, 0x53, 0x11, 0x16, 0x0A, 0x13, 0x89, 0x06, 0x00, 0x03, 0x01, 0x04, 0x00, 0x00,
-        0x00, 0x47, 0x01, 0xF8, 0x03, 0xF8, 0x03, 0x00, 0x08, 0x79, 0x00,
-    ];
-
-    /*
-    Name (\_S5, Package (0x01)  // _S5_: S5 System State
-    {
-        0x05
-    })
-    */
-    let s5_sleep_data = [0x08u8, 0x5F, 0x53, 0x35, 0x5F, 0x12, 0x04, 0x01, 0x0A, 0x05];
-
-    // DSDT
-    let mut dsdt = SDT::new(*b"DSDT", 36, 6, *b"CLOUDH", *b"CHDSDT  ", 1);
-    dsdt.append(pci_dsdt_data);
-    if serial_enabled {
-        dsdt.append(com1_dsdt_data);
-    }
-    dsdt.append(s5_sleep_data);
-
-    dsdt
-}
-pub fn create_acpi_tables(
-    guest_mem: &GuestMemoryMmap,
-    num_cpus: u8,
-    serial_enabled: bool,
-) -> GuestAddress {
-    // RSDP is at the EBDA
-    let rsdp_offset = super::EBDA_START;
-    let mut tables: Vec<u64> = Vec::new();
-
-    // DSDT
-    let dsdt = create_dsdt_table(serial_enabled);
-    let dsdt_offset = rsdp_offset.checked_add(RSDP::len() as u64).unwrap();
-    guest_mem
-        .write_slice(dsdt.as_slice(), dsdt_offset)
-        .expect("Error writing DSDT table");
-
-    // FACP aka FADT
-    // Revision 6 of the ACPI FADT table is 276 bytes long
-    let mut facp = SDT::new(*b"FACP", 276, 6, *b"CLOUDH", *b"CHFACP  ", 1);
-
-    // HW_REDUCED_ACPI and RESET_REG_SUP
-    let fadt_flags: u32 = 1 << 20 | 1 << 10;
-    facp.write(112, fadt_flags);
-
-    // RESET_REG
-    facp.write(116, GenericAddress::io_port_address(0x3c0));
-    // RESET_VALUE
-    facp.write(128, 1u8);
-
-    facp.write(131, 3u8); // FADT minor version
-    facp.write(140, dsdt_offset.0); // X_DSDT
-
-    // SLEEP_CONTROL_REG
-    facp.write(244, GenericAddress::io_port_address(0x3c0));
-    // SLEEP_STATUS_REG
-    facp.write(256, GenericAddress::io_port_address(0x3c0));
-
-    facp.write(268, b"CLOUDHYP"); // Hypervisor Vendor Identity
-
-    facp.update_checksum();
-    let facp_offset = dsdt_offset.checked_add(dsdt.len() as u64).unwrap();
-    guest_mem
-        .write_slice(facp.as_slice(), facp_offset)
-        .expect("Error writing FACP table");
-    tables.push(facp_offset.0);
-
-    // MADT
-    let mut madt = SDT::new(*b"APIC", 44, 5, *b"CLOUDH", *b"CHMADT  ", 1);
-    madt.write(36, super::mptable::APIC_DEFAULT_PHYS_BASE);
-
-    for cpu in 0..num_cpus {
-        let lapic = LocalAPIC {
-            r#type: 0,
-            length: 8,
-            processor_id: cpu,
-            apic_id: cpu,
-            flags: 1,
-        };
-        madt.append(lapic);
-    }
-
-    madt.append(IOAPIC {
-        r#type: 1,
-        length: 12,
-        ioapic_id: 0,
-        apic_address: super::mptable::IO_APIC_DEFAULT_PHYS_BASE,
-        gsi_base: 0,
-        ..Default::default()
-    });
-
-    madt.append(InterruptSourceOverride {
-        r#type: 2,
-        length: 10,
-        bus: 0,
-        source: 4,
-        gsi: 4,
-        flags: 0,
-    });
-
-    let madt_offset = facp_offset.checked_add(facp.len() as u64).unwrap();
-    guest_mem
-        .write_slice(madt.as_slice(), madt_offset)
-        .expect("Error writing MADT table");
-    tables.push(madt_offset.0);
-
-    // MCFG
-    let mut mcfg = SDT::new(*b"MCFG", 60, 1, *b"CLOUDH", *b"CHMCFG  ", 1);
-
-    // 32-bit PCI enhanced configuration mechanism
-    mcfg.append(PCIRangeEntry {
-        base_address: super::MEM_32BIT_DEVICES_GAP_SIZE,
-        segment: 0,
-        start: 0,
-        end: 0xff,
-        ..Default::default()
-    });
-
-    let mcfg_offset = madt_offset.checked_add(madt.len() as u64).unwrap();
-    guest_mem
-        .write_slice(mcfg.as_slice(), mcfg_offset)
-        .expect("Error writing MCFG table");
-    tables.push(mcfg_offset.0);
-
-    // XSDT
-    let mut xsdt = SDT::new(*b"XSDT", 36, 1, *b"CLOUDH", *b"CHXSDT  ", 1);
-    for table in tables {
-        xsdt.append(table);
-    }
-    xsdt.update_checksum();
-
-    let xsdt_offset = mcfg_offset.checked_add(mcfg.len() as u64).unwrap();
-    guest_mem
-        .write_slice(xsdt.as_slice(), xsdt_offset)
-        .expect("Error writing XSDT table");
-
-    // RSDP
-    let rsdp = RSDP::new(*b"CLOUDH", xsdt_offset.0);
-    guest_mem
-        .write_slice(rsdp.as_slice(), rsdp_offset)
-        .expect("Error writing RSDP");
-
-    rsdp_offset
-}

arch/src/x86_64/interrupts.rs

@@ -5,7 +5,7 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE-BSD-3-Clause file.
 
-use std::io::{self, Cursor};
+use std::io::Cursor;
 use std::mem;
 use std::result;
 
@@ -16,8 +16,8 @@ use kvm_ioctls;
 
 #[derive(Debug)]
 pub enum Error {
-    GetLapic(io::Error),
+    GetLapic(kvm_ioctls::Error),
-    SetLapic(io::Error),
+    SetLapic(kvm_ioctls::Error),
 }
 
 pub type Result<T> = result::Result<T, Error>;

arch/src/x86_64/layout.rs

@@ -5,21 +5,90 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE-BSD-3-Clause file.
 
-use vm_memory::GuestAddress;
+use vm_memory::{GuestAddress, GuestUsize};
 
-/// Magic addresses externally used to lay out x86_64 VMs.
+/*
+
+Memory layout documentation and constants
+~~~~~~ ~~~~~~ ~~~~~~~~~~~~~ ~~~ ~~~~~~~~~
+
+Constants are in order and grouped by range. Take care to update all references
+when making changes and keep them in order.
+
+*/
+
+// ** Low RAM (start: 0, length: 640KiB) **
+pub const LOW_RAM_START: GuestAddress = GuestAddress(0x0);
+
+// == Fixed addresses within the "Low RAM" range: ==
+
+// Initial GDT/IDT needed to boot kernel
+pub const BOOT_GDT_START: GuestAddress = GuestAddress(0x500);
+pub const BOOT_IDT_START: GuestAddress = GuestAddress(0x520);
+
+/// The 'zero page', a.k.a linux kernel bootparams.
+pub const ZERO_PAGE_START: GuestAddress = GuestAddress(0x7000);
 
 /// Initial stack for the boot CPU.
 pub const BOOT_STACK_START: GuestAddress = GuestAddress(0x8000);
 pub const BOOT_STACK_POINTER: GuestAddress = GuestAddress(0x8ff0);
 
+// Initial pagetables.
+pub const PML4_START: GuestAddress = GuestAddress(0x9000);
+pub const PDPTE_START: GuestAddress = GuestAddress(0xa000);
+pub const PDE_START: GuestAddress = GuestAddress(0xb000);
+
 /// Kernel command line start address.
 pub const CMDLINE_START: GuestAddress = GuestAddress(0x20000);
 /// Kernel command line start address maximum size.
 pub const CMDLINE_MAX_SIZE: usize = 0x10000;
 
-/// Address for the TSS setup.
+// MPTABLE, describing VCPUS.
-pub const KVM_TSS_ADDRESS: GuestAddress = GuestAddress(0xfffbd000);
+pub const MPTABLE_START: GuestAddress = GuestAddress(0x9fc00);
 
-/// The 'zero page', a.k.a linux kernel bootparams.
+// == End of "Low RAM" range. ==
-pub const ZERO_PAGE_START: GuestAddress = GuestAddress(0x7000);
+
+// ** EBDA reserved area (start: 640KiB, length: 384KiB) **
+pub const EBDA_START: GuestAddress = GuestAddress(0xa0000);
+
+// == Fixed constants within the "EBDA" range ==
+
+// ACPI RSDP table
+pub const RSDP_POINTER: GuestAddress = EBDA_START;
+
+// == End of "EBDA" range ==
+
+// ** High RAM (start: 1MiB, length: 3071MiB) **
+pub const HIGH_RAM_START: GuestAddress = GuestAddress(0x100000);
+
+// == No fixed addresses in the "High RAM" range ==
+
+// ** 32-bit reserved area (start: 3GiB, length: 1GiB) **
+pub const MEM_32BIT_RESERVED_START: GuestAddress = GuestAddress(0xc000_0000);
+pub const MEM_32BIT_RESERVED_SIZE: GuestUsize = (1024 << 20);
+
+// == Fixed constants within the "32-bit reserved" range ==
+
+// Sub range: 32-bit PCI devices (start: 3GiB, length: 640Mib)
+pub const MEM_32BIT_DEVICES_START: GuestAddress = MEM_32BIT_RESERVED_START;
+pub const MEM_32BIT_DEVICES_SIZE: GuestUsize = (640 << 20);
+
+// PCI MMCONFIG space (start: after the device space, length: 256MiB)
+pub const PCI_MMCONFIG_START: GuestAddress =
+    GuestAddress(MEM_32BIT_DEVICES_START.0 + MEM_32BIT_DEVICES_SIZE);
+pub const PCI_MMCONFIG_SIZE: GuestUsize = (256 << 20);
+
+// IOAPIC
+pub const IOAPIC_START: GuestAddress = GuestAddress(0xfec0_0000);
+pub const IOAPIC_SIZE: GuestUsize = 0x20;
+
+// APIC
+pub const APIC_START: GuestAddress = GuestAddress(0xfee0_0000);
+
+/// Address for the TSS setup.
+pub const KVM_TSS_ADDRESS: GuestAddress = GuestAddress(0xfffb_d000);
+
+// == End of "32-bit reserved" range. ==
+
+// ** 64-bit RAM start (start: 4GiB, length: varies) **
+pub const RAM_64BIT_START: GuestAddress = GuestAddress(0x1_0000_0000);

arch/src/x86_64/mod.rs

@@ -5,7 +5,6 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE-BSD-3-Clause file.
 
-mod acpi;
 mod gdt;
 pub mod interrupts;
 pub mod layout;
@@ -20,6 +19,7 @@ use vm_memory::{
 };
 
 const E820_RAM: u32 = 1;
+const E820_RESERVED: u32 = 2;
 
 // This is a workaround to the Rust enforcement specifying that any implementation of a foreign
 // trait (in this case `DataInit`) where:
@@ -46,73 +46,53 @@ impl From<Error> for super::Error {
     }
 }
 
-// Where BIOS/VGA magic would live on a real PC.
-const EBDA_START: GuestAddress = GuestAddress(0xa0000);
-const FIRST_ADDR_PAST_32BITS: GuestAddress = GuestAddress(1 << 32);
-
-// Our 32-bit memory gap starts at 3G.
-pub const MEM_32BIT_GAP_START: GuestAddress = GuestAddress(0xc000_0000);
-
-// Our 32-bit memory gap size is 1GB.
-const MEM_32BIT_GAP_SIZE: GuestUsize = (1024 << 20);
-
-// We reserve 768MB in our memory gap for 32-bit devices (e.g. 32-bit PCI BARs).
-const MEM_32BIT_DEVICES_GAP_SIZE: GuestUsize = (768 << 20);
-
 /// Returns a Vec of the valid memory addresses.
 /// These should be used to configure the GuestMemory structure for the platform.
 /// For x86_64 all addresses are valid from the start of the kernel except a
 /// carve out at the end of 32bit address space.
 pub fn arch_memory_regions(size: GuestUsize) -> Vec<(GuestAddress, usize, RegionType)> {
-    let reserved_memory_gap_start = MEM_32BIT_GAP_START
+    let reserved_memory_gap_start = layout::MEM_32BIT_RESERVED_START
-        .checked_add(MEM_32BIT_DEVICES_GAP_SIZE)
+        .checked_add(layout::MEM_32BIT_DEVICES_SIZE)
         .expect("32-bit reserved region is too large");
 
     let requested_memory_size = GuestAddress(size as u64);
     let mut regions = Vec::new();
 
     // case1: guest memory fits before the gap
-    if size as u64 <= MEM_32BIT_GAP_START.raw_value() {
+    if size as u64 <= layout::MEM_32BIT_RESERVED_START.raw_value() {
         regions.push((GuestAddress(0), size as usize, RegionType::Ram));
     // case2: guest memory extends beyond the gap
     } else {
         // push memory before the gap
         regions.push((
             GuestAddress(0),
-            MEM_32BIT_GAP_START.raw_value() as usize,
+            layout::MEM_32BIT_RESERVED_START.raw_value() as usize,
             RegionType::Ram,
         ));
         regions.push((
-            FIRST_ADDR_PAST_32BITS,
+            layout::RAM_64BIT_START,
-            requested_memory_size.unchecked_offset_from(MEM_32BIT_GAP_START) as usize,
+            requested_memory_size.unchecked_offset_from(layout::MEM_32BIT_RESERVED_START) as usize,
             RegionType::Ram,
         ));
     }
 
     // Add the 32-bit device memory hole as a sub region.
     regions.push((
-        MEM_32BIT_GAP_START,
+        layout::MEM_32BIT_RESERVED_START,
-        MEM_32BIT_DEVICES_GAP_SIZE as usize,
+        layout::MEM_32BIT_DEVICES_SIZE as usize,
         RegionType::SubRegion,
     ));
 
     // Add the 32-bit reserved memory hole as a sub region.
     regions.push((
         reserved_memory_gap_start,
-        (MEM_32BIT_GAP_SIZE - MEM_32BIT_DEVICES_GAP_SIZE) as usize,
+        (layout::MEM_32BIT_RESERVED_SIZE - layout::MEM_32BIT_DEVICES_SIZE) as usize,
         RegionType::Reserved,
     ));
 
     regions
 }
 
-/// X86 specific memory hole/memory mapped devices/reserved area.
-pub fn get_32bit_gap_start() -> GuestAddress {
-    FIRST_ADDR_PAST_32BITS
-        .checked_sub(MEM_32BIT_GAP_SIZE)
-        .expect("32-bit hole is too large")
-}
-
 /// Configures the system and should be called once per vm before starting vcpu threads.
 ///
 /// # Arguments
@@ -121,30 +101,27 @@ pub fn get_32bit_gap_start() -> GuestAddress {
 /// * `cmdline_addr` - Address in `guest_mem` where the kernel command line was loaded.
 /// * `cmdline_size` - Size of the kernel command line in bytes including the null terminator.
 /// * `num_cpus` - Number of virtual CPUs the guest will have.
+#[allow(clippy::too_many_arguments)]
 pub fn configure_system(
     guest_mem: &GuestMemoryMmap,
     cmdline_addr: GuestAddress,
     cmdline_size: usize,
     num_cpus: u8,
     setup_hdr: Option<setup_header>,
-    serial_enabled: bool,
+    rsdp_addr: Option<GuestAddress>,
 ) -> super::Result<()> {
     const KERNEL_BOOT_FLAG_MAGIC: u16 = 0xaa55;
     const KERNEL_HDR_MAGIC: u32 = 0x53726448;
     const KERNEL_LOADER_OTHER: u8 = 0xff;
     const KERNEL_MIN_ALIGNMENT_BYTES: u32 = 0x1000000; // Must be non-zero.
-    let first_addr_past_32bits = FIRST_ADDR_PAST_32BITS;
-    let end_32bit_gap_start = get_32bit_gap_start();
-
-    let himem_start = super::HIMEM_START;
 
     // Note that this puts the mptable at the last 1k of Linux's 640k base RAM
     mptable::setup_mptable(guest_mem, num_cpus).map_err(Error::MpTableSetup)?;
 
     let mut params: BootParamsWrapper = BootParamsWrapper(boot_params::default());
 
-    if setup_hdr.is_some() {
+    if let Some(hdr) = setup_hdr {
-        params.0.hdr = setup_hdr.unwrap();
+        params.0.hdr = hdr;
         params.0.hdr.cmd_line_ptr = cmdline_addr.raw_value() as u32;
         params.0.hdr.cmdline_size = cmdline_size as u32;
     } else {
@@ -156,36 +133,41 @@ pub fn configure_system(
         params.0.hdr.kernel_alignment = KERNEL_MIN_ALIGNMENT_BYTES;
     };
 
-    add_e820_entry(&mut params.0, 0, EBDA_START.raw_value(), E820_RAM)?;
+    add_e820_entry(&mut params.0, 0, layout::EBDA_START.raw_value(), E820_RAM)?;
 
     let mem_end = guest_mem.end_addr();
-    if mem_end < end_32bit_gap_start {
+    if mem_end < layout::MEM_32BIT_RESERVED_START {
         add_e820_entry(
             &mut params.0,
-            himem_start.raw_value(),
+            layout::HIGH_RAM_START.raw_value(),
-            mem_end.unchecked_offset_from(himem_start),
+            mem_end.unchecked_offset_from(layout::HIGH_RAM_START) + 1,
             E820_RAM,
         )?;
     } else {
         add_e820_entry(
             &mut params.0,
-            himem_start.raw_value(),
+            layout::HIGH_RAM_START.raw_value(),
-            end_32bit_gap_start.unchecked_offset_from(himem_start),
+            layout::MEM_32BIT_RESERVED_START.unchecked_offset_from(layout::HIGH_RAM_START),
             E820_RAM,
         )?;
-        if mem_end > first_addr_past_32bits {
+        if mem_end > layout::RAM_64BIT_START {
             add_e820_entry(
                 &mut params.0,
-                first_addr_past_32bits.raw_value(),
+                layout::RAM_64BIT_START.raw_value(),
-                mem_end.unchecked_offset_from(first_addr_past_32bits),
+                mem_end.unchecked_offset_from(layout::RAM_64BIT_START) + 1,
                 E820_RAM,
             )?;
         }
     }
 
-    #[cfg(feature = "acpi")]
+    add_e820_entry(
-    {
+        &mut params.0,
-        let rsdp_addr = acpi::create_acpi_tables(guest_mem, num_cpus, serial_enabled);
+        layout::PCI_MMCONFIG_START.0,
+        layout::PCI_MMCONFIG_SIZE,
+        E820_RESERVED,
+    )?;
+
+    if let Some(rsdp_addr) = rsdp_addr {
         params.0.acpi_rsdp_addr = rsdp_addr.0;
     }
 
@@ -241,21 +223,11 @@ mod tests {
         assert_eq!(GuestAddress(1 << 32), regions[1].0);
     }
 
-    #[test]
-    fn test_32bit_gap() {
-        assert_eq!(
-            get_32bit_gap_start(),
-            FIRST_ADDR_PAST_32BITS
-                .checked_sub(MEM_32BIT_GAP_SIZE as u64)
-                .expect("32-bit hole is too large")
-        );
-    }
-
     #[test]
     fn test_system_configuration() {
         let no_vcpus = 4;
         let gm = GuestMemoryMmap::new(&vec![(GuestAddress(0), 0x10000)]).unwrap();
-        let config_err = configure_system(&gm, GuestAddress(0), 0, 1, None, false);
+        let config_err = configure_system(&gm, GuestAddress(0), 0, 1, None, None);
         assert!(config_err.is_err());
         assert_eq!(
             config_err.unwrap_err(),
@@ -273,7 +245,7 @@ mod tests {
             .map(|r| (r.0, r.1))
             .collect();
         let gm = GuestMemoryMmap::new(&ram_regions).unwrap();
-        configure_system(&gm, GuestAddress(0), 0, no_vcpus, None, false).unwrap();
+        configure_system(&gm, GuestAddress(0), 0, no_vcpus, None, None).unwrap();
 
         // Now assigning some memory that is equal to the start of the 32bit memory hole.
         let mem_size = 3328 << 20;
@@ -284,7 +256,7 @@ mod tests {
             .map(|r| (r.0, r.1))
             .collect();
         let gm = GuestMemoryMmap::new(&ram_regions).unwrap();
-        configure_system(&gm, GuestAddress(0), 0, no_vcpus, None, false).unwrap();
+        configure_system(&gm, GuestAddress(0), 0, no_vcpus, None, None).unwrap();
 
         // Now assigning some memory that falls after the 32bit memory hole.
         let mem_size = 3330 << 20;
@@ -295,7 +267,7 @@ mod tests {
             .map(|r| (r.0, r.1))
             .collect();
         let gm = GuestMemoryMmap::new(&ram_regions).unwrap();
-        configure_system(&gm, GuestAddress(0), 0, no_vcpus, None, false).unwrap();
+        configure_system(&gm, GuestAddress(0), 0, no_vcpus, None, None).unwrap();
     }
 
     #[test]

arch/src/x86_64/mptable.rs

@@ -13,7 +13,8 @@ use std::slice;
 use libc::c_char;
 
 use arch_gen::x86::mpspec;
-use vm_memory::{Address, ByteValued, Bytes, GuestAddress, GuestMemory, GuestMemoryMmap};
+use layout::{APIC_START, IOAPIC_START, MPTABLE_START};
+use vm_memory::{Address, ByteValued, Bytes, GuestMemory, GuestMemoryMmap};
 
 // This is a workaround to the Rust enforcement specifying that any implementation of a foreign
 // trait (in this case `ByteValued`) where:
@@ -44,9 +45,6 @@ unsafe impl ByteValued for MpcTableWrapper {}
 unsafe impl ByteValued for MpcLintsrcWrapper {}
 unsafe impl ByteValued for MpfIntelWrapper {}
 
-// MPTABLE, describing VCPUS.
-const MPTABLE_START: GuestAddress = GuestAddress(0x9fc00);
-
 #[derive(Debug, PartialEq)]
 pub enum Error {
     /// There was too little guest memory to store the entire MP table.
@@ -92,8 +90,6 @@ const MPC_SPEC: i8 = 4;
 const MPC_OEM: [c_char; 8] = char_array!(c_char; 'F', 'C', ' ', ' ', ' ', ' ', ' ', ' ');
 const MPC_PRODUCT_ID: [c_char; 12] = ['0' as c_char; 12];
 const BUS_TYPE_ISA: [u8; 6] = char_array!(u8; 'I', 'S', 'A', ' ', ' ', ' ');
-pub const IO_APIC_DEFAULT_PHYS_BASE: u32 = 0xfec00000; // source: linux/arch/x86/include/asm/apicdef.h
-pub const APIC_DEFAULT_PHYS_BASE: u32 = 0xfee00000; // source: linux/arch/x86/include/asm/apicdef.h
 const APIC_VERSION: u8 = 0x14;
 const CPU_STEPPING: u32 = 0x600;
 const CPU_FEATURE_APIC: u32 = 0x200;
@@ -208,7 +204,7 @@ pub fn setup_mptable(mem: &GuestMemoryMmap, num_cpus: u8) -> Result<()> {
         mpc_ioapic.0.apicid = ioapicid;
         mpc_ioapic.0.apicver = APIC_VERSION;
         mpc_ioapic.0.flags = mpspec::MPC_APIC_USABLE as u8;
-        mpc_ioapic.0.apicaddr = IO_APIC_DEFAULT_PHYS_BASE;
+        mpc_ioapic.0.apicaddr = IOAPIC_START.0 as u32;
         mem.write_obj(mpc_ioapic, base_mp)
             .map_err(|_| Error::WriteMpcIoapic)?;
         base_mp = base_mp.unchecked_add(size as u64);
@@ -271,7 +267,7 @@ pub fn setup_mptable(mem: &GuestMemoryMmap, num_cpus: u8) -> Result<()> {
         mpc_table.0.spec = MPC_SPEC;
         mpc_table.0.oem = MPC_OEM;
         mpc_table.0.productid = MPC_PRODUCT_ID;
-        mpc_table.0.lapic = APIC_DEFAULT_PHYS_BASE;
+        mpc_table.0.lapic = APIC_START.0 as u32;
         checksum = checksum.wrapping_add(compute_checksum(&mpc_table.0));
         mpc_table.0.checksum = (!checksum).wrapping_add(1) as i8;
         mem.write_obj(mpc_table, table_base)
@@ -284,7 +280,7 @@ pub fn setup_mptable(mem: &GuestMemoryMmap, num_cpus: u8) -> Result<()> {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use vm_memory::GuestUsize;
+    use vm_memory::{GuestAddress, GuestUsize};
 
     fn table_entry_size(type_: u8) -> usize {
         match type_ as u32 {

arch/src/x86_64/regs.rs

@@ -5,18 +5,14 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE-BSD-3-Clause file.
 
-use std::{io, mem, result};
+use std::{mem, result};
 
 use super::gdt::{gdt_entry, kvm_segment_from_gdt};
 use arch_gen::x86::msr_index;
-use kvm_bindings::{kvm_fpu, kvm_msr_entry, kvm_msrs, kvm_regs, kvm_sregs};
+use kvm_bindings::{kvm_fpu, kvm_msr_entry, kvm_regs, kvm_sregs, Msrs};
 use kvm_ioctls::VcpuFd;
-use vm_memory::{Address, Bytes, GuestAddress, GuestMemory, GuestMemoryMmap};
+use layout::{BOOT_GDT_START, BOOT_IDT_START, PDE_START, PDPTE_START, PML4_START};
-
+use vm_memory::{Address, Bytes, GuestMemory, GuestMemoryMmap};
-// Initial pagetables.
-const PML4_START: GuestAddress = GuestAddress(0x9000);
-const PDPTE_START: GuestAddress = GuestAddress(0xa000);
-const PDE_START: GuestAddress = GuestAddress(0xb000);
 
 // MTRR constants
 const MTRR_ENABLE: u64 = 0x800; // IA32_MTRR_DEF_TYPE MSR: E (MTRRs enabled) flag, bit 11
@@ -25,15 +21,15 @@ const MTRR_MEM_TYPE_WB: u64 = 0x6;
 #[derive(Debug)]
 pub enum Error {
     /// Failed to get SREGs for this CPU.
-    GetStatusRegisters(io::Error),
+    GetStatusRegisters(kvm_ioctls::Error),
     /// Failed to set base registers for this CPU.
-    SetBaseRegisters(io::Error),
+    SetBaseRegisters(kvm_ioctls::Error),
     /// Failed to configure the FPU.
-    SetFPURegisters(io::Error),
+    SetFPURegisters(kvm_ioctls::Error),
     /// Setting up MSRs failed.
-    SetModelSpecificRegisters(io::Error),
+    SetModelSpecificRegisters(kvm_ioctls::Error),
     /// Failed to set SREGs for this CPU.
-    SetStatusRegisters(io::Error),
+    SetStatusRegisters(kvm_ioctls::Error),
     /// Writing the GDT to RAM failed.
     WriteGDT,
     /// Writing the IDT to RAM failed.
@@ -69,26 +65,10 @@ pub fn setup_fpu(vcpu: &VcpuFd) -> Result<()> {
 ///
 /// * `vcpu` - Structure for the VCPU that holds the VCPU's fd.
 pub fn setup_msrs(vcpu: &VcpuFd) -> Result<()> {
-    let entry_vec = create_msr_entries();
+    vcpu.set_msrs(&create_msr_entries())
-    let vec_size_bytes =
+        .map_err(Error::SetModelSpecificRegisters)?;
-        mem::size_of::<kvm_msrs>() + (entry_vec.len() * mem::size_of::<kvm_msr_entry>());
-    let vec: Vec<u8> = Vec::with_capacity(vec_size_bytes);
-    let msrs: &mut kvm_msrs = unsafe {
-        // Converting the vector's memory to a struct is unsafe.  Carefully using the read-only
-        // vector to size and set the members ensures no out-of-bounds errors below.
-        &mut *(vec.as_ptr() as *mut kvm_msrs)
-    };
 
-    unsafe {
+    Ok(())
-        // Mapping the unsized array to a slice is unsafe because the length isn't known.
-        // Providing the length used to create the struct guarantees the entire slice is valid.
-        let entries: &mut [kvm_msr_entry] = msrs.entries.as_mut_slice(entry_vec.len());
-        entries.copy_from_slice(&entry_vec);
-    }
-    msrs.nmsrs = entry_vec.len() as u32;
-
-    vcpu.set_msrs(msrs)
-        .map_err(Error::SetModelSpecificRegisters)
 }
 
 /// Configure base registers for a given CPU.
@@ -127,9 +107,6 @@ pub fn setup_sregs(mem: &GuestMemoryMmap, vcpu: &VcpuFd) -> Result<()> {
     vcpu.set_sregs(&sregs).map_err(Error::SetStatusRegisters)
 }
 
-const BOOT_GDT_OFFSET: GuestAddress = GuestAddress(0x500);
-const BOOT_IDT_OFFSET: GuestAddress = GuestAddress(0x520);
-
 const BOOT_GDT_MAX: usize = 4;
 
 const EFER_LMA: u64 = 0x400;
@@ -140,7 +117,7 @@ const X86_CR0_PG: u64 = 0x80000000;
 const X86_CR4_PAE: u64 = 0x20;
 
 fn write_gdt_table(table: &[u64], guest_mem: &GuestMemoryMmap) -> Result<()> {
-    let boot_gdt_addr = BOOT_GDT_OFFSET;
+    let boot_gdt_addr = BOOT_GDT_START;
     for (index, entry) in table.iter().enumerate() {
         let addr = guest_mem
             .checked_offset(boot_gdt_addr, index * mem::size_of::<u64>())
@@ -153,7 +130,7 @@ fn write_gdt_table(table: &[u64], guest_mem: &GuestMemoryMmap) -> Result<()> {
 }
 
 fn write_idt_value(val: u64, guest_mem: &GuestMemoryMmap) -> Result<()> {
-    let boot_idt_addr = BOOT_IDT_OFFSET;
+    let boot_idt_addr = BOOT_IDT_START;
     guest_mem
         .write_obj(val, boot_idt_addr)
         .map_err(|_| Error::WriteIDT)
@@ -173,11 +150,11 @@ fn configure_segments_and_sregs(mem: &GuestMemoryMmap, sregs: &mut kvm_sregs) ->
 
     // Write segments
     write_gdt_table(&gdt_table[..], mem)?;
-    sregs.gdt.base = BOOT_GDT_OFFSET.raw_value();
+    sregs.gdt.base = BOOT_GDT_START.raw_value();
     sregs.gdt.limit = mem::size_of_val(&gdt_table) as u16 - 1;
 
     write_idt_value(0, mem)?;
-    sregs.idt.base = BOOT_IDT_OFFSET.raw_value();
+    sregs.idt.base = BOOT_IDT_START.raw_value();
     sregs.idt.limit = mem::size_of::<u64>() as u16 - 1;
 
     sregs.cs = code_seg;
@@ -218,7 +195,7 @@ fn setup_page_tables(mem: &GuestMemoryMmap, sregs: &mut kvm_sregs) -> Result<()>
     Ok(())
 }
 
-fn create_msr_entries() -> Vec<kvm_msr_entry> {
+fn create_msr_entries() -> Msrs {
     let mut entries = Vec::<kvm_msr_entry>::new();
 
     entries.push(kvm_msr_entry {
@@ -279,7 +256,7 @@ fn create_msr_entries() -> Vec<kvm_msr_entry> {
         ..Default::default()
     });
 
-    entries
+    Msrs::from_entries(&entries)
 }
 
 #[cfg(test)]
@@ -305,20 +282,20 @@ mod tests {
         let gm = create_guest_mem();
         configure_segments_and_sregs(&gm, &mut sregs).unwrap();
 
-        assert_eq!(0x0, read_u64(&gm, BOOT_GDT_OFFSET));
+        assert_eq!(0x0, read_u64(&gm, BOOT_GDT_START));
         assert_eq!(
             0xaf9b000000ffff,
-            read_u64(&gm, BOOT_GDT_OFFSET.unchecked_add(8))
+            read_u64(&gm, BOOT_GDT_START.unchecked_add(8))
         );
         assert_eq!(
             0xcf93000000ffff,
-            read_u64(&gm, BOOT_GDT_OFFSET.unchecked_add(16))
+            read_u64(&gm, BOOT_GDT_START.unchecked_add(16))
         );
         assert_eq!(
             0x8f8b000000ffff,
-            read_u64(&gm, BOOT_GDT_OFFSET.unchecked_add(24))
+            read_u64(&gm, BOOT_GDT_START.unchecked_add(24))
         );
-        assert_eq!(0x0, read_u64(&gm, BOOT_IDT_OFFSET));
+        assert_eq!(0x0, read_u64(&gm, BOOT_IDT_START));
 
         assert_eq!(0, sregs.cs.base);
         assert_eq!(0xfffff, sregs.ds.limit);
@@ -384,24 +361,11 @@ mod tests {
 
         // This test will check against the last MSR entry configured (the tenth one).
         // See create_msr_entries for details.
-        let test_kvm_msrs_entry = [kvm_msr_entry {
+        let mut msrs = Msrs::from_entries(&[kvm_msr_entry {
             index: msr_index::MSR_IA32_MISC_ENABLE,
             ..Default::default()
-        }];
+        }]);
-        let vec_size_bytes = mem::size_of::<kvm_msrs>() + mem::size_of::<kvm_msr_entry>();
-        let vec: Vec<u8> = Vec::with_capacity(vec_size_bytes);
-        let mut msrs: &mut kvm_msrs = unsafe {
-            // Converting the vector's memory to a struct is unsafe.  Carefully using the read-only
-            // vector to size and set the members ensures no out-of-bounds errors below.
-            &mut *(vec.as_ptr() as *mut kvm_msrs)
-        };
 
-        unsafe {
-            let entries: &mut [kvm_msr_entry] = msrs.entries.as_mut_slice(1);
-            entries.copy_from_slice(&test_kvm_msrs_entry);
-        }
-
-        msrs.nmsrs = 1;
         // get_msrs returns the number of msrs that it succeed in reading. We only want to read 1
         // in this test case scenario.
         let read_msrs = vcpu.get_msrs(&mut msrs).unwrap();
@@ -411,9 +375,7 @@ mod tests {
         // tenth one (i.e the one with index msr_index::MSR_IA32_MISC_ENABLE has the data we
         // expect.
         let entry_vec = create_msr_entries();
-        unsafe {
+        assert_eq!(entry_vec.as_slice()[9], msrs.as_slice()[0]);
-            assert_eq!(entry_vec[9], msrs.entries.as_slice(1)[0]);
-        }
     }
 
     #[test]

devices/Cargo.toml

@@ -5,17 +5,18 @@ authors = ["The Chromium OS Authors"]
 
 [dependencies]
 byteorder = "1.3.2"
-epoll = "4.1.0"
+epoll = ">=4.0.1"
-kvm-bindings = "0.1.1"
+kvm-bindings = "0.2.0"
-kvm-ioctls = { git = "https://github.com/rust-vmm/kvm-ioctls", branch = "master" }
+kvm-ioctls = "0.4.0"
 libc = "0.2.60"
 log = "0.4.8"
 vm-memory = { git = "https://github.com/rust-vmm/vm-memory" }
-vmm-sys-util = { git = "https://github.com/rust-vmm/vmm-sys-util" }
+vmm-sys-util = ">=0.3.1"
 
 [dev-dependencies]
 tempfile = "3.1.0"
 
 [features]
-default = ["acpi"]
+default = []
-acpi = []
+acpi = []
+cmos = []

devices/src/acpi.rs

@@ -5,6 +5,8 @@
 
 use vmm_sys_util::eventfd::EventFd;
 use BusDevice;
+use HotPlugNotificationType;
+use Interrupt;
 
 /// A device for handling ACPI shutdown and reboot
 pub struct AcpiShutdownDevice {
@@ -50,3 +52,43 @@ impl BusDevice for AcpiShutdownDevice {
         }
     }
 }
+
+/// A device for handling ACPI GED event generation
+pub struct AcpiGEDDevice {
+    interrupt: Box<dyn Interrupt>,
+    notification_type: HotPlugNotificationType,
+    ged_irq: u32,
+}
+
+impl AcpiGEDDevice {
+    pub fn new(interrupt: Box<dyn Interrupt>, ged_irq: u32) -> AcpiGEDDevice {
+        AcpiGEDDevice {
+            interrupt,
+            notification_type: HotPlugNotificationType::NoDevicesChanged,
+            ged_irq,
+        }
+    }
+
+    pub fn notify(
+        &mut self,
+        notification_type: HotPlugNotificationType,
+    ) -> Result<(), std::io::Error> {
+        self.notification_type = notification_type;
+        self.interrupt.deliver()
+    }
+
+    pub fn irq(&self) -> u32 {
+        self.ged_irq
+    }
+}
+
+// I/O port reports what type of notification was made
+impl BusDevice for AcpiGEDDevice {
+    // Spec has all fields as zero
+    fn read(&mut self, _base: u64, _offset: u64, data: &mut [u8]) {
+        data[0] = self.notification_type as u8;
+        self.notification_type = HotPlugNotificationType::NoDevicesChanged;
+    }
+
+    fn write(&mut self, _base: u64, _offset: u64, _data: &[u8]) {}
+}

devices/src/bus.rs

@@ -9,8 +9,8 @@
 
 use std::cmp::{Ord, Ordering, PartialEq, PartialOrd};
 use std::collections::btree_map::BTreeMap;
-use std::result;
+use std::sync::{Arc, Mutex, RwLock};
-use std::sync::{Arc, Mutex};
+use std::{convert, error, fmt, io, result};
 
 /// Trait for devices that respond to reads or writes in an arbitrary address space.
 ///
@@ -30,10 +30,28 @@ pub trait BusDevice: Send {
 pub enum Error {
     /// The insertion failed because the new device overlapped with an old device.
     Overlap,
+    /// Failed to operate on zero sized range.
+    ZeroSizedRange,
+    /// Failed to find address range.
+    MissingAddressRange,
 }
 
 pub type Result<T> = result::Result<T, Error>;
 
+impl fmt::Display for Error {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(f, "bus_error: {:?}", self)
+    }
+}
+
+impl error::Error for Error {}
+
+impl convert::From<Error> for io::Error {
+    fn from(e: Error) -> Self {
+        io::Error::new(io::ErrorKind::Other, e)
+    }
+}
+
 /// Holds a base and length representing the address space occupied by a `BusDevice`.
 ///
 /// * base - The address at which the range start.
@@ -75,29 +93,30 @@ impl PartialOrd for BusRange {
 ///
 /// This doesn't have any restrictions on what kind of device or address space this applies to. The
 /// only restriction is that no two devices can overlap in this address space.
-#[derive(Clone, Default)]
+#[derive(Default)]
 pub struct Bus {
-    devices: BTreeMap<BusRange, Arc<Mutex<dyn BusDevice>>>,
+    devices: RwLock<BTreeMap<BusRange, Arc<Mutex<dyn BusDevice>>>>,
 }
 
 impl Bus {
     /// Constructs an a bus with an empty address space.
     pub fn new() -> Bus {
         Bus {
-            devices: BTreeMap::new(),
+            devices: RwLock::new(BTreeMap::new()),
         }
     }
 
-    fn first_before(&self, addr: u64) -> Option<(BusRange, &Mutex<dyn BusDevice>)> {
+    fn first_before(&self, addr: u64) -> Option<(BusRange, Arc<Mutex<dyn BusDevice>>)> {
-        let (range, dev) = self
+        let devices = self.devices.read().unwrap();
-            .devices
+        let (range, dev) = devices
             .range(..=BusRange { base: addr, len: 1 })
             .rev()
             .next()?;
-        Some((*range, dev))
+        Some((*range, dev.clone()))
     }
 
-    pub fn resolve(&self, addr: u64) -> Option<(u64, u64, &Mutex<dyn BusDevice>)> {
+    #[allow(clippy::type_complexity)]
+    pub fn resolve(&self, addr: u64) -> Option<(u64, u64, Arc<Mutex<dyn BusDevice>>)> {
         if let Some((range, dev)) = self.first_before(addr) {
             let offset = addr - range.base;
             if offset < range.len {
@@ -108,14 +127,16 @@ impl Bus {
     }
 
     /// Puts the given device at the given address space.
-    pub fn insert(&mut self, device: Arc<Mutex<dyn BusDevice>>, base: u64, len: u64) -> Result<()> {
+    pub fn insert(&self, device: Arc<Mutex<dyn BusDevice>>, base: u64, len: u64) -> Result<()> {
         if len == 0 {
-            return Err(Error::Overlap);
+            return Err(Error::ZeroSizedRange);
         }
 
         // Reject all cases where the new device's range overlaps with an existing device.
         if self
             .devices
+            .read()
+            .unwrap()
             .iter()
             .any(|(range, _dev)| range.overlaps(base, len))
         {
@@ -124,6 +145,8 @@ impl Bus {
 
         if self
             .devices
+            .write()
+            .unwrap()
             .insert(BusRange { base, len }, device)
             .is_some()
         {
@@ -133,6 +156,43 @@ impl Bus {
         Ok(())
     }
 
+    /// Removes the device at the given address space range.
+    pub fn remove(&self, base: u64, len: u64) -> Result<()> {
+        if len == 0 {
+            return Err(Error::ZeroSizedRange);
+        }
+
+        let bus_range = BusRange { base, len };
+
+        if self.devices.write().unwrap().remove(&bus_range).is_none() {
+            return Err(Error::MissingAddressRange);
+        }
+
+        Ok(())
+    }
+
+    /// Updates the address range for an existing device.
+    pub fn update_range(
+        &self,
+        old_base: u64,
+        old_len: u64,
+        new_base: u64,
+        new_len: u64,
+    ) -> Result<()> {
+        // Retrieve the device corresponding to the range
+        let device = if let Some((_, _, dev)) = self.resolve(old_base) {
+            dev.clone()
+        } else {
+            return Err(Error::MissingAddressRange);
+        };
+
+        // Remove the old address range
+        self.remove(old_base, old_len)?;
+
+        // Insert the new address range
+        self.insert(device, new_base, new_len)
+    }
+
     /// Reads data from the device that owns the range containing `addr` and puts it into `data`.
     ///
     /// Returns true on success, otherwise `data` is untouched.
@@ -188,7 +248,7 @@ mod tests {
 
     #[test]
     fn bus_insert() {
-        let mut bus = Bus::new();
+        let bus = Bus::new();
         let dummy = Arc::new(Mutex::new(DummyDevice));
         assert!(bus.insert(dummy.clone(), 0x10, 0).is_err());
         assert!(bus.insert(dummy.clone(), 0x10, 0x10).is_ok());
@@ -209,7 +269,7 @@ mod tests {
 
     #[test]
     fn bus_read_write() {
-        let mut bus = Bus::new();
+        let bus = Bus::new();
         let dummy = Arc::new(Mutex::new(DummyDevice));
         assert!(bus.insert(dummy.clone(), 0x10, 0x10).is_ok());
         assert!(bus.read(0x10, &mut [0, 0, 0, 0]));
@@ -226,7 +286,7 @@ mod tests {
 
     #[test]
     fn bus_read_write_values() {
-        let mut bus = Bus::new();
+        let bus = Bus::new();
         let dummy = Arc::new(Mutex::new(ConstantDevice));
         assert!(bus.insert(dummy.clone(), 0x10, 0x10).is_ok());
 
@@ -240,7 +300,7 @@ mod tests {
     }
 
     #[test]
-    fn busrange_cmp_and_clone() {
+    fn busrange_cmp() {
         let range = BusRange { base: 0x10, len: 2 };
         assert_eq!(range, BusRange { base: 0x10, len: 3 });
         assert_eq!(range, BusRange { base: 0x10, len: 2 });
@@ -250,17 +310,14 @@ mod tests {
 
         assert_eq!(range, range.clone());
 
-        let mut bus = Bus::new();
+        let bus = Bus::new();
         let mut data = [1, 2, 3, 4];
         assert!(bus
             .insert(Arc::new(Mutex::new(DummyDevice)), 0x10, 0x10)
             .is_ok());
         assert!(bus.write(0x10, &mut data));
-        let bus_clone = bus.clone();
         assert!(bus.read(0x10, &mut data));
         assert_eq!(data, [1, 2, 3, 4]);
-        assert!(bus_clone.read(0x10, &mut data));
-        assert_eq!(data, [1, 2, 3, 4]);
     }
 
     #[test]

devices/src/ioapic.rs

@@ -13,13 +13,14 @@ use crate::BusDevice;
 use byteorder::{ByteOrder, LittleEndian};
 use kvm_bindings::kvm_msi;
 use kvm_ioctls::VmFd;
+use std::result;
 use std::sync::Arc;
-use std::{io, result};
+use vm_memory::GuestAddress;
 
 #[derive(Debug)]
 pub enum Error {
     /// Failed to send an interrupt.
-    InterruptFailed(io::Error),
+    InterruptFailed(kvm_ioctls::Error),
     /// Invalid destination mode.
     InvalidDestinationMode,
     /// Invalid trigger mode.
@@ -156,6 +157,7 @@ pub struct Ioapic {
     reg_sel: u32,
     reg_entries: [RedirectionTableEntry; NUM_IOAPIC_PINS],
     vm_fd: Arc<VmFd>,
+    apic_address: GuestAddress,
 }
 
 impl BusDevice for Ioapic {
@@ -194,12 +196,13 @@ impl BusDevice for Ioapic {
 }
 
 impl Ioapic {
-    pub fn new(vm_fd: Arc<VmFd>) -> Ioapic {
+    pub fn new(vm_fd: Arc<VmFd>, apic_address: GuestAddress) -> Ioapic {
         Ioapic {
             id: 0,
             reg_sel: 0,
             reg_entries: [0; NUM_IOAPIC_PINS],
             vm_fd,
+            apic_address,
         }
     }
 
@@ -239,7 +242,7 @@ impl Ioapic {
         let redirection_hint: u8 = 1;
 
         // Generate MSI message address
-        let address_lo: u32 = 0xfee0_0000
+        let address_lo: u32 = self.apic_address.0 as u32
             | u32::from(destination_id) << 12
             | u32::from(redirection_hint) << 3
             | u32::from(destination_mode) << 2;

devices/src/legacy/cmos.rs

@@ -0,0 +1,115 @@
+// Copyright 2017 The Chromium OS Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+use libc::{gmtime_r, time, time_t, tm};
+use std::cmp::min;
+use std::mem;
+
+use crate::BusDevice;
+
+const INDEX_MASK: u8 = 0x7f;
+const INDEX_OFFSET: u64 = 0x0;
+const DATA_OFFSET: u64 = 0x1;
+const DATA_LEN: usize = 128;
+
+/// A CMOS/RTC device commonly seen on x86 I/O port 0x70/0x71.
+pub struct Cmos {
+    index: u8,
+    data: [u8; DATA_LEN],
+}
+
+impl Cmos {
+    /// Constructs a CMOS/RTC device with initial data.
+    /// `mem_below_4g` is the size of memory in bytes below the 32-bit gap.
+    /// `mem_above_4g` is the size of memory in bytes above the 32-bit gap.
+    pub fn new(mem_below_4g: u64, mem_above_4g: u64) -> Cmos {
+        let mut data = [0u8; DATA_LEN];
+
+        // Extended memory from 16 MB to 4 GB in units of 64 KB
+        let ext_mem = min(
+            0xFFFF,
+            mem_below_4g.saturating_sub(16 * 1024 * 1024) / (64 * 1024),
+        );
+        data[0x34] = ext_mem as u8;
+        data[0x35] = (ext_mem >> 8) as u8;
+
+        // High memory (> 4GB) in units of 64 KB
+        let high_mem = min(0x00FF_FFFF, mem_above_4g / (64 * 1024));
+        data[0x5b] = high_mem as u8;
+        data[0x5c] = (high_mem >> 8) as u8;
+        data[0x5d] = (high_mem >> 16) as u8;
+
+        Cmos { index: 0, data }
+    }
+}
+
+impl BusDevice for Cmos {
+    fn write(&mut self, _base: u64, offset: u64, data: &[u8]) {
+        if data.len() != 1 {
+            return;
+        }
+
+        match offset {
+            INDEX_OFFSET => self.index = data[0] & INDEX_MASK,
+            DATA_OFFSET => self.data[self.index as usize] = data[0],
+            o => panic!("bad write offset on CMOS device: {}", o),
+        }
+    }
+
+    fn read(&mut self, _base: u64, offset: u64, data: &mut [u8]) {
+        fn to_bcd(v: u8) -> u8 {
+            assert!(v < 100);
+            ((v / 10) << 4) | (v % 10)
+        }
+
+        if data.len() != 1 {
+            return;
+        }
+
+        data[0] = match offset {
+            INDEX_OFFSET => self.index,
+            DATA_OFFSET => {
+                let seconds;
+                let minutes;
+                let hours;
+                let week_day;
+                let day;
+                let month;
+                let year;
+                // The time and gmtime_r calls are safe as long as the structs they are given are
+                // large enough, and neither of them fail. It is safe to zero initialize the tm
+                // struct because it contains only plain data.
+                unsafe {
+                    let mut tm: tm = mem::zeroed();
+                    let mut now: time_t = 0;
+                    time(&mut now as *mut _);
+                    gmtime_r(&now, &mut tm as *mut _);
+                    // The following lines of code are safe but depend on tm being in scope.
+                    seconds = tm.tm_sec;
+                    minutes = tm.tm_min;
+                    hours = tm.tm_hour;
+                    week_day = tm.tm_wday + 1;
+                    day = tm.tm_mday;
+                    month = tm.tm_mon + 1;
+                    year = tm.tm_year;
+                };
+                match self.index {
+                    0x00 => to_bcd(seconds as u8),
+                    0x02 => to_bcd(minutes as u8),
+                    0x04 => to_bcd(hours as u8),
+                    0x06 => to_bcd(week_day as u8),
+                    0x07 => to_bcd(day as u8),
+                    0x08 => to_bcd(month as u8),
+                    0x09 => to_bcd((year % 100) as u8),
+                    0x32 => to_bcd(((year + 1900) / 100) as u8),
+                    _ => {
+                        // self.index is always guaranteed to be in range via INDEX_MASK.
+                        self.data[(self.index & INDEX_MASK) as usize]
+                    }
+                }
+            }
+            o => panic!("bad read offset on CMOS device: {}", o),
+        }
+    }
+}

devices/src/legacy/mod.rs

@@ -5,8 +5,12 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE-BSD-3-Clause file.
 
+#[cfg(feature = "cmos")]
+mod cmos;
 mod i8042;
 mod serial;
 
+#[cfg(feature = "cmos")]
+pub use self::cmos::Cmos;
 pub use self::i8042::I8042Device;
 pub use self::serial::Serial;

devices/src/lib.rs

@@ -26,7 +26,7 @@ pub mod ioapic;
 pub mod legacy;
 
 #[cfg(feature = "acpi")]
-pub use self::acpi::AcpiShutdownDevice;
+pub use self::acpi::{AcpiGEDDevice, AcpiShutdownDevice};
 pub use self::bus::{Bus, BusDevice, Error as BusError};
 
 pub type DeviceEventT = u16;
@@ -67,6 +67,12 @@ pub enum Error {
     IoError(io::Error),
 }
 
-pub trait Interrupt: Send {
+pub trait Interrupt: Send + Sync {
     fn deliver(&self) -> result::Result<(), std::io::Error>;
 }
+
+#[derive(Clone, Copy)]
+pub enum HotPlugNotificationType {
+    NoDevicesChanged,
+    CPUDevicesChanged,
+}

docs/custom-image.md

@@ -0,0 +1,96 @@
+# How to create a custom Clear Linux image
+
+In the context of adding more utility to the cloudguest image being used
+for integration testing, this is a quick guide on how to achieve the creation
+of your own Clear Linux image using the official Clear Linux tooling.
+
+## Prepare the environment
+
+From the host, the goal is run a Clear Linux VM that will allow us to build
+the custom image we want.
+
+```bash
+# Get latest CL version:
+IMG_VERSION=$(curl https://download.clearlinux.org/latest)
+# Get latest clear-kvm image:
+wget -P $HOME/workloads/ https://download.clearlinux.org/current/clear-${IMG_VERSION}-kvm.img.xz
+# Extract the image
+unxz $HOME/workloads/clear-${IMG_VERSION}-kvm.img.xz
+# Make sure cloud-hypervisor binary has CAP_NET_ADMIN capability set
+sudo setcap cap_net_admin+ep cloud-hypervisor
+# Boot cloud-hypervisor VM with the downloaded image
+./cloud-hypervisor -v --kernel $HOME/workloads/vmlinux --disk path=clear-${IMG_VERSION}-kvm.img --cmdline "console=ttyS0 console=hvc0 reboot=k panic=1 nomodules root=/dev/vda3 rw" --cpus 1 --memory size=4G --net tap=,mac=
+# Setup connectivity
+# First make sure to enable IP forwarding (disabled on Linux by default)
+sudo bash -c "echo 1 > /proc/sys/net/ipv4/ip_forward"
+# Retrieve the interface name and the gateway IP
+IFACE=$(ip route | grep default | awk -F 'dev' '{print $2}' | awk -F ' ' '{print $1}')
+GW=$(ip route | grep vmtap0 | awk -F ' ' '{print $1}')
+# Create a new masquerade rule to tag the packets going out
+sudo iptables -t nat -A POSTROUTING -s ${GW} -o ${IFACE} -j MASQUERADE
+```
+
+## Create the image
+
+From the guest, we can now create the image.
+
+```bash
+# Setup connectivity
+sudo ip addr add 192.168.249.2/24 dev enp0s3
+sudo ip route add default via 192.168.249.1
+# Install necessary bundles
+sudo swupd bundle-add clr-installer
+sudo swupd bundle-add os-installer
+# Download and update cloudguest image configuration
+wget https://download.clearlinux.org/current/config/image/cloudguest.yaml
+sed -i '/size: \"864M\"/d' cloudguest.yaml
+sed -i 's/\"800M\"/\"2G\"/g' cloudguest.yaml
+sed -i 's/bootloader,/bootloader,\n    iperf,/g' cloudguest.yaml
+sed -i 's/systemd-networkd-autostart/sysadmin-basic,\n    systemd-networkd-autostart/g' cloudguest.yaml
+# Create the custom cloudguest image
+clr-installer -c cloudguest.yaml
+# Make the guest accessible through ssh
+sudo mkdir -p /etc/ssh
+sudo bash -c "echo 'PermitRootLogin yes' >> /etc/ssh/sshd_config"
+```
+
+### Retrieve the image
+
+Once the new image has been created and the guest is accessible through
+`ssh`, it is time to retrieve the image from the host.
+
+```bash
+# Retrieve new image (this is a raw image)
+scp root@192.168.249.2:cloudguest.img .
+mv cloudguest.img clear-cloudguest-raw.img
+# Create the QCOW image from the RAW image
+qemu-img convert -p -f raw -O qcow2 clear-cloudguest-raw.img clear-cloudguest.img
+# Compress the QCOW image
+xz -k -T $(nproc) clear-cloudguest.img
+```
+
+## Switch CI to use the new image
+
+### Upload to Azure storage
+
+The next step is to update the image stored as part of the Azure storage
+account, replacing it with the newly created image. This will make this
+new image available from the integration tests.
+This is usually achieved through the web interface.
+
+### Update integration tests
+
+Last step is about updating the integration tests to work with this new image.
+The key point is to identify the UUID of this new image so that it can be used
+directly from the tests.
+
+Proceed as follow to determine this UUID:
+
+```bash
+# Mount the image
+sudo mount -o loop,offset=$((2048 * 512)) clear-cloudguest-raw.img /mnt/
+# Identify UUID
+sudo cat /mnt/loader/entries/Clear-linux-kvm-*.conf | grep "root=PARTUUID="
+# Unmount the image
+sudo umount /mnt
+```

docs/debug-port.md

@@ -43,7 +43,7 @@ to easily grep for the tracing logs (e.g.
 ```
 ./target/debug/cloud-hypervisor \
     --kernel ~/rust-hypervisor-firmware/target/target/release/hypervisor-fw \
-    --disk ~/hypervisor/images/clear-30080-kvm.img \
+    --disk path=~/hypervisor/images/clear-30080-kvm.img \
     --cpus 4 \
     --memory size=1024M \
     --rng \

docs/device_model.md

@@ -0,0 +1,208 @@
+# Device Model
+
+This document describes the device model supported by `cloud-hypervisor`.
+
+## Summary
+
+| Device | Build configurable | Enabled by default | Runtime configurable |
+| :----: | :----: | :----: | :----: |
+| Serial port | :negative_squared_cross_mark: | :negative_squared_cross_mark: | :heavy_check_mark: |
+| RTC/CMOS | :heavy_check_mark: | :heavy_check_mark: | :negative_squared_cross_mark: |
+| I/O APIC | :negative_squared_cross_mark: | :negative_squared_cross_mark: | :heavy_check_mark: |
+| i8042 shutdown/reboot | :negative_squared_cross_mark: | :negative_squared_cross_mark: | :negative_squared_cross_mark: |
+| ACPI shutdown/reboot | :negative_squared_cross_mark: | :heavy_check_mark: | :negative_squared_cross_mark: |
+| virtio-blk | :negative_squared_cross_mark: | :negative_squared_cross_mark: | :heavy_check_mark: |
+| virtio-console | :negative_squared_cross_mark: | :negative_squared_cross_mark: | :heavy_check_mark: |
+| virtio-iommu | :negative_squared_cross_mark: | :negative_squared_cross_mark: | :heavy_check_mark: |
+| virtio-net | :negative_squared_cross_mark: | :negative_squared_cross_mark: | :heavy_check_mark: |
+| virtio-pmem | :negative_squared_cross_mark: | :negative_squared_cross_mark: | :heavy_check_mark: |
+| virtio-rng | :negative_squared_cross_mark: | :negative_squared_cross_mark: | :heavy_check_mark: |
+| virtio-vsock | :negative_squared_cross_mark: | :negative_squared_cross_mark: | :heavy_check_mark: |
+| vhost-user-blk | :negative_squared_cross_mark: | :negative_squared_cross_mark: | :heavy_check_mark: |
+| vhost-user-fs | :negative_squared_cross_mark: | :negative_squared_cross_mark: | :heavy_check_mark: |
+| vhost-user-net | :negative_squared_cross_mark: | :negative_squared_cross_mark: | :heavy_check_mark: |
+| VFIO | :heavy_check_mark: | :negative_squared_cross_mark: | :heavy_check_mark: |
+
+## Legacy devices
+
+### Serial port
+
+Simple emulation of a serial port by reading and writing to specific port I/O
+addresses. Used as the default console for Linux when booting with the option
+`console=ttyS0`, the serial port can be very useful to gather early logs from
+the operating system booted inside the VM.
+
+This device is always built-in, and it is disabled by default. It can be
+enabled with the `--serial` option, as long as its parameter is not `off`.
+
+### RTC/CMOS
+
+For environments such as Windows or EFI which cannot rely on KVM clock, the
+emulation of this legacy device makes the platform usable.
+
+This device is built-in by default, but it can be compiled out with Rust
+features. When compiled in, it is always enabled, and cannot be disabled
+from the command line.
+
+### I/O APIC
+
+`cloud-hypervisor` supports a so-called split IRQ chip implementation by
+implementing support for the [IOAPIC](https://wiki.osdev.org/IOAPIC).
+By moving part of the IRQ chip implementation from kernel space to user space,
+the IRQ chip emulation does not always run in a fully privileged mode.
+
+The device is always built-in, and it is enabled depending on the presence of
+the serial port. If the serial port is disabled, and because no other device
+would require pin based interrupts (INTx), the I/O APIC is disabled.
+
+### i8042
+
+Simplified PS/2 port since it supports only one key to trigger a reboot or
+shutdown, depending on the ACPI support.
+
+This device is always built-in, but it is disabled by default. Because ACPI is
+enabled by default, the handling of reboot/shutdown goes through the dedicated
+ACPI device. In case ACPI is disabled, this device is enabled to bring to the
+VM some reboot/shutdown support.
+
+### ACPI device
+
+This is a dedicated device for handling ACPI shutdown and reboot when ACPI is
+enabled.
+
+This device is always built-in, and it is enabled by default since the ACPI
+feature is enabled by default.
+
+## Virtio devices
+
+For all virtio devices listed below, both `virtio-mmio` and `virtio-pci`
+transport layers are supported, `virtio-pci` being the default.
+
+Both `virtio-mmio` and `virtio-pci` can be compiled out. `virtio-pci` is
+built-in by default, and enabled by default. If both transport layers were
+built at the same time, `virtio-pci` would be the default transport layer.
+
+### virtio-block
+
+The `virtio-blk` device exposes a block device to the guest. This device is
+usually used to boot the operating system running in the VM.
+
+This device is always built-in, and it is enabled based on the presence of the
+flag `--disk`.
+
+### virtio-console
+
+`cloud-hypervisor` exposes a `virtio-console` device to the guest. Although
+using this device as a guest console can potentially cut some early boot
+messages, it can reduce the guest boot time and provides a complete console
+implementation.
+
+This device is always built-in, and it is enabled by default to provide a guest
+console. It can be disabled, switching back to the legacy serial port by
+selecting `--serial tty --console off` from the command line.
+
+### virtio-iommu
+
+As we want to improve our nested guests support, we added support for exposing
+a [paravirtualized IOMMU](https://github.com/cloud-hypervisor/cloud-hypervisor/blob/master/docs/iommu.md)
+device through virtio. This allows for a safer nested virtio and directly
+assigned devices support.
+
+This device is always built-in, and it is enabled based on the presence of the
+parameter `iommu=on` in any of the virtio or VFIO devices. If at least one of
+these devices needs to be connected to the paravirtualized IOMMU, the
+`virtio-iommu` device will be created.
+
+### virtio-net
+
+The `virtio-net` device provides network connectivity for the guest, as it
+creates a network interface connected to a TAP interface automatically created
+by the `cloud-hypervisor` on the host.
+
+This device is always built-in, and it is enabled based on the presence of the
+flag `--net`.
+
+### virtio-pmem
+
+The `virtio-pmem` implementation emulates a virtual persistent memory device
+that `cloud-hypervisor` can e.g. boot from. Booting from a `virtio-pmem` device
+allows to bypass the guest page cache and improve the guest memory footprint.
+
+This device is always built-in, and it is enabled based on the presence of the
+flag `--pmem`.
+
+### virtio-rng
+
+A VM does not generate entropy like a real machine would, which is an issue
+when workloads running in the guest need random numbers to be generated. The
+`virtio-rng` device provides entropy to the guest by relying on the generator
+that can be found on the host. By default, the chosen source of entropy is
+`/dev/urandom`.
+
+This device is always built-in, and it is always enabled. The `--rng` flag can
+be used to change the source of entropy.
+
+### virtio-vsock
+
+In order to more efficiently and securely communicate between host and guest,
+we added a hybrid implementation of the [VSOCK](http://man7.org/linux/man-pages/man7/vsock.7.html)
+socket address family over virtio.
+Credits go to the [Firecracker](https://github.com/firecracker-microvm/firecracker/blob/master/docs/vsock.md)
+project as our implementation is a copy of theirs.
+
+This device is always built-in, and it is enabled based on the presence of the
+flag `--vsock`.
+
+## Vhost-user devices
+
+Vhost-user devices are virtio backends running outside of the VMM, as its own
+separate process. They are usually used to bring more flexibility and increased
+isolation.
+
+### vhost-user-blk
+
+As part of the general effort to offload paravirtualized I/O to external
+processes, we added support for vhost-user-blk backends. This enables
+`cloud-hypervisor` users to plug a `vhost-user` based block device (e.g. SPDK)
+into the VMM as their virtio block backend.
+
+This device is always built-in, and it is enabled based on the presence of the
+flag `--vhost-user-blk`.
+
+### vhost-user-fs
+
+`cloud-hypervisor` supports the [virtio-fs](https://virtio-fs.gitlab.io/)
+shared file system, allowing for an efficient and reliable way of sharing
+a filesystem between the host and the cloud-hypervisor guest.
+
+See our [filesystem sharing](https://github.com/cloud-hypervisor/cloud-hypervisor/blob/master/docs/fs.md)
+documentation for more details on how to use virtio-fs with cloud-hypervisor.
+
+This device is always built-in, and it is enabled based on the presence of the
+flag `--fs`.
+
+### vhost-user-net
+
+As part of the general effort to offload paravirtualized I/O to external
+processes, we added support for [vhost-user-net](https://access.redhat.com/solutions/3394851)
+backends. This enables `cloud-hypervisor` users to plug a `vhost-user` based
+networking device (e.g. DPDK) into the VMM as their virtio network backend.
+
+This device is always built-in, and it is enabled based on the presence of the
+flag `--vhost-user-net`.
+
+## VFIO
+
+VFIO (Virtual Function I/O) is a kernel framework that exposes direct device
+access to userspace. `cloud-hypervisor` uses VFIO to directly assign host
+physical devices into its guest.
+
+See our [VFIO documentation](https://github.com/cloud-hypervisor/cloud-hypervisor/blob/master/docs/vfio.md)
+for more details on how to directly assign host devices to `cloud-hypervisor`
+guests.
+
+Because VFIO implies `vfio-pci` in the `cloud-hypervisor` context, the VFIO
+support is built-in when the `pci` feature is selected. And because the `pci`
+feature is built-in by default, VFIO support is also built-in by default.
+When VFIO support is built-in, a physical device can be passed through, using
+the flag `--device` in order to enable the VFIO code.

docs/fs.md

@@ -10,11 +10,12 @@ __virtio-fs__, also known as __vhost-user-fs__ is a virtual device defined by th
 
 This virtual device relies on the _vhost-user_ protocol, which assumes the backend (device emulation) is handled by a dedicated process running on the host. This daemon is called __virtiofsd__ and needs to be present on the host.
 
-_Install virtiofsd_
+_Build virtiofsd_
 ```bash
-VIRTIOFSD_URL="$(curl --silent https://api.github.com/repos/intel/nemu/releases/latest | grep "browser_download_url" | grep "virtiofsd-x86_64" | grep -o 'https://.*[^ "]')"
+git clone --depth 1 "https://github.com/sboeuf/qemu.git" -b "virtio-fs" $VIRTIOFSD_DIR
-wget --quiet $VIRTIOFSD_URL -O "virtiofsd"
+cd $VIRTIOFSD_DIR
-chmod +x "virtiofsd"
+./configure --prefix=$PWD --target-list=x86_64-softmmu
+make virtiofsd -j `nproc`
 sudo setcap cap_sys_admin+epi "virtiofsd"
 ```
 _Create shared directory_
@@ -27,15 +28,18 @@ _Run virtiofsd_
     -d \
     -o vhost_user_socket=/tmp/virtiofs \
     -o source=/tmp/shared_dir \
-    -o cache=none
+    -o cache=always
 ```
-The `cache=none` option here is an important one as it tells the daemon not to try any memory mapping of the files, but instead to use the _virtqueues_ to convey the files content. The support for the memory mapping of the files will be added later.
+
+The `cache=always` option should be the default when using `virtiofsd` with the __cloud-hypervisor__ VMM. This allows the daemon to memory map the shared files, which gives better I/O performance.
+
+The `cache=none` option is another way to run the daemon but because the _virtqueues_ are used to convey the files content in this case, the I/O performance is impacted.
 
 ### The kernel
 
 In order to leverage __virtio-fs__ support from within the guest, and because the code has not been merged in upstream Linux kernel yet, it is required to build a custom kernel embedding the patches.
 
-The following branch `virtio-pmem_and_virtio-fs` on the repository https://github.com/sboeuf/linux.git includes all the needed patches to support __virtio-fs__.
+The following branch `virtio-fs-virtio-iommu` on the repository https://github.com/cloud-hypervisor/linux.git includes all the needed patches to support __virtio-fs__.
 
 Make sure to build a kernel out of this branch that can be then used to boot the VM.
 
@@ -53,18 +57,34 @@ Assuming you have `clear-kvm.img` and `custom-vmlinux.bin` on your system, here
 ./cloud-hypervisor \
     --cpus 4 \
     --memory "size=512,file=/dev/shm" \
-    --disk clear-kvm.img \
+    --disk path=clear-kvm.img \
     --kernel custom-vmlinux.bin \
     --cmdline "console=ttyS0 reboot=k panic=1 nomodules root=/dev/vda3" \ 
     --fs tag=virtiofs,sock=/tmp/virtiofs,num_queues=1,queue_size=512
 ```
 
+By default, DAX is enabled with a cache window of 8GiB. You can specify a custom size (let's say 4GiB for this example) for the cache by explicitly setting DAX and the cache size:
+
+```bash
+--fs tag=virtiofs,sock=/tmp/virtiofs,num_queues=1,queue_size=512,dax=on,cache_size=4G
+
+```
+
+In case you don't want to use a shared window of cache to pass the shared files content, this means you will have to explicitly disable DAX with `dax=off`. Note that in this case, the `cache_size` parameter will be ignored.
+
+```bash
+--fs tag=virtiofs,sock=/tmp/virtiofs,num_queues=1,queue_size=512,dax=off
+
+```
+
 ### Mount the shared directory
 The last step is to mount the shared directory inside the guest, using the `virtio_fs` filesystem type.
 ```bash
 mkdir mount_dir
 mount \
-    -t virtio_fs /dev/null mount_dir/ \
+    -t virtio_fs virtiofs mount_dir/ \
-    -o tag=virtiofs,rootmode=040000,user_id=0,group_id=0
+    -o rootmode=040000,user_id=0,group_id=0,dax
 ```
-The `tag` needs to be consistent with what has been provided through the __cloud-hypervisor__ command line.
+The `tag` needs to be consistent with what has been provided through the __cloud-hypervisor__ command line, which happens to be `virtiofs` in this example.
+
+The `dax` option must be removed in case the shared cache region is not enabled from the VMM.

docs/hotplug.md

@@ -0,0 +1,57 @@
+# Cloud Hypervisor Hot Plug
+
+Currently Cloud Hypervisor only support hot plugging of CPU devices.
+
+## Kernel support
+
+For hotplug on Cloud Hypervisor ACPI GED support is needed. This can either be achieved by turning on `CONFIG_ACPI_REDUCED_HARDWARE_ONLY` 
+or by using this kernel patch (available in 5.5rc1 and later): https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/patch/drivers/acpi/Makefile?id=ac36d37e943635fc072e9d4f47e40a48fbcdb3f0
+
+This patch is integrated into the Clear Linux KVM and cloudguest images.
+
+## CPU Hot Plug
+
+Extra vCPUs can be added (but not removed [1]) from a running Cloud Hypervisor instance. This is controlled by two mechanisms:
+
+1. Specifying a number of maximum potential vCPUs that is greater than the number of default (boot) vCPUs.
+2. Making a HTTP API request to the VM to ask for the additional vCPUs to be added.
+
+To use CPU hotplug start the VM with the number of max vCPUs greater than the number of boot vCPUs, e.g.
+
+```shell
+$ pushd $CLOUDH
+$ sudo setcap cap_net_admin+ep ./cloud-hypervisor/target/release/cloud-hypervisor
+$ ./cloud-hypervisor/target/release/cloud-hypervisor \
+	--kernel ./hypervisor-fw \
+	--disk path=clear-31890-kvm.img \
+	--cpus boot=4,max=8 \
+	--memory size=1024M \
+	--net "tap=,mac=,ip=,mask=" \
+	--rng \
+    --api-socket=/tmp/ch-socket
+$ popd
+```
+
+Notice the addition of `--api-socket=/tmp/ch-socket` and a `max` parameter on `--cpus boot=4.max=8`.
+
+To ask the VMM to add additional vCPUs then use the resize API:
+
+```shell
+curl -H "Accept: application/json" -H "Content-Type: application/json" -i -XPUT --unix-socket /tmp/ch-socket -d "{ \"desired_vcpus\":8}" http://localhost/api/v1/vm.resize
+```
+
+The extra vCPU threads will be created and advertised to the running kernel. The kernel does not bring up the CPUs immediately and instead the user must "on-line" them from inside the VM:
+
+```shell
+root@ch-guest ~ # lscpu | grep list:
+On-line CPU(s) list:             0-3
+Off-line CPU(s) list:            4-7
+root@ch-guest ~ # echo 1 | tee /sys/devices/system/cpu/cpu[4,5,6,7]/online
+1
+root@ch-guest ~ # lscpu | grep list:
+On-line CPU(s) list:             0-7
+```
+
+After a reboot the added CPUs will remain.
+
+[1]: It is not currently possible to remove CPUs after they are added however CPU hot unplug is included in our roadmap for a future version.

docs/iommu.md

@@ -0,0 +1,209 @@
+# Virtual IOMMU
+
+## Rationales
+
+Having the possibility to expose a virtual IOMMU to the guest can be
+interesting to support specific use cases. That being said, it is always
+important to keep in mind a virtual IOMMU can impact the performance of the
+attached devices, which is the reason why one should be careful when enabling
+this feature.
+
+### Protect nested virtual machines
+
+The first reason why one might want to expose a virtual IOMMU to the guest is
+to increase the security regarding the memory accesses performed by the virtual
+devices (VIRTIO devices), on behalf of the guest drivers.
+
+With a virtual IOMMU, the VMM stands between the guest driver and its device
+counterpart, validating and translating every address before to try accessing
+the guest memory. This is standard interposition that is performed here by the
+VMM.
+
+The increased security does not apply for a simple case where we have one VM
+per VMM. Because the guest cannot be trusted, as we always consider it could
+be malicious and gain unauthorized privileges inside the VM, preventing some
+devices from accessing the entire guest memory is pointless.
+
+But let's take the interesting case of nested virtualization, and let's assume
+we have a VMM running a first layer VM. This L1 guest is fully trusted as the
+user intends to run multiple VMs from this L1. We can end up with multiple L2
+VMs running on a single L1 VM. In this particular case, and without exposing a
+virtual IOMMU to the L1 guest, it would be possible for any L2 guest to use the
+device implementation from the host VMM to access the entire guest L1 memory.
+The virtual IOMMU prevents from this kind of trouble as it will validate the
+addresses the device is authorized to access.
+
+### Achieve VFIO nested
+
+Another reason for having a virtual IOMMU is to allow passing physical devices
+from the host through multiple layers of virtualization. Let's take as example
+a system with a physical IOMMU running a VM with a virtual IOMMU. The
+implementation of the virtual IOMMU is responsible for updating the physical
+DMA Remapping table (DMAR) everytime the DMA mapping changes. This must happen
+through the VFIO framework on the host as this is the only userspace interface
+to interact with a physical IOMMU.
+
+Relying on this update mechanism, it is possible to attach physical devices to
+the virtual IOMMU, which allows these devices to be passed from L1 to another
+layer of virtualization.
+
+## Why virtio-iommu?
+
+The Cloud Hypervisor project decided to implement the brand new virtio-iommu
+device in order to provide a virtual IOMMU to its users. The reason being the
+simplicity brought by the paravirtualization solution. By having one side
+handled from the guest itself, it removes the complexity of trapping memory
+page accesses and shadowing them. This is why the project will not try to
+implement a full emulation of a physical IOMMU.
+
+## Pre-requisites
+
+### Kernel
+
+Since virtio-iommu has landed partially into the version 5.3 of the Linux
+kernel, a special branch is needed to get things working with Cloud Hypervisor.
+By partially, we are talking about x86 specifically, as it is already fully
+functional for ARM architectures.
+
+## Usage
+
+In order to expose a virtual IOMMU to the guest, it is required to create a
+virtio-iommu device and expose it through the ACPI IORT table. This can be
+simply achieved by attaching at least one device to the virtual IOMMU.
+
+The way to expose to the guest a specific device as sitting behind this IOMMU
+is to explicitly tag it from the command line with the option `iommu=on`.
+
+Not all devices support this extra option, and the default value will always
+be `off` since we want to avoid the performance impact for most users who don't
+need this.
+
+Refer to the command line `--help` to find out which device support to be
+attached to the virtual IOMMU.
+
+Below is a simple example exposing the `virtio-blk` device as attached to the
+virtual IOMMU:
+
+```bash
+./cloud-hypervisor \
+    --cpus 1 \
+    --memory size=512M \
+    --disk path=clear-kvm.img,iommu=on \
+    --kernel custom-bzImage \
+    --cmdline "console=ttyS0 root=/dev/vda3" \
+```
+
+From a guest perspective, it is easy to verify if the device is protected by
+the virtual IOMMU. Check the directories listed under
+`/sys/kernel/iommu_groups`:
+
+```bash
+ls /sys/kernel/iommu_groups
+0
+```
+
+In this case, only one IOMMU group should be created. Under this group, it is
+possible to find out the b/d/f of the device(s) part of this group.
+
+```bash
+ls /sys/kernel/iommu_groups/0/devices/
+0000:00:03.0
+```
+
+And you can validate the device is the one we expect running `lspci`:
+
+```bash
+lspci
+00:00.0 Host bridge: Intel Corporation Device 0d57
+00:01.0 Unassigned class [ffff]: Red Hat, Inc. Device 1057
+00:02.0 Unassigned class [ffff]: Red Hat, Inc. Virtio console
+00:03.0 Mass storage controller: Red Hat, Inc. Virtio block device
+00:04.0 Unassigned class [ffff]: Red Hat, Inc. Virtio RNG
+```
+
+## Faster mappings
+
+By default, the guest memory is mapped with 4k pages and no huge pages, which
+causes the virtual IOMMU device to be asked for 4k mappings only. This
+configuration slows down the setup of the physical IOMMU as an important number
+of requests need to be issued in order to create large mappings.
+
+One use case is even more impacted by the slowdown, the nested VFIO case. When
+passing a device through a L2 guest, the VFIO driver running in L1 will update
+the DMAR entries for the specific device. Because VFIO pins the entire guest
+memory, this means the entire mapping of the L2 guest need to be stored into
+multiple 4k mappings. Obviously, the bigger the L2 guest RAM is, the longer the
+update of the mappings will last. There is an additional problem happening in
+this case, if the L2 guest RAM is quite large, it will require a large number
+of mappings, which might exceed the VFIO limit set on the host. The default
+value is 65536, which can simply be reached with a 256MiB sized RAM.
+
+The way to solve both problems, the slowdown and the limit being exceeded, is
+to reduce the amount of requests to describe those same large mappings. This
+can be achieved by using 2MiB pages, known as huge pages. By seeing the guest
+RAM as larger pages, and because the virtual IOMMU device supports it, the
+guest will require less mappings, which will prevent the limit from being
+exceeded, but also will take less time to process them on the host. That's
+how using huge pages as much as possible can speed up VM boot time.
+
+### Basic usage
+
+Let's look at an example of how to run a guest with huge pages.
+
+First, make sure your system has enough pages to cover the entire guest RAM:
+```bash
+# This example creates 4096 hugepages
+echo 4096 > /proc/sys/vm/nr_hugepages
+```
+
+Next step is simply to create the VM. Two things are important, first we want
+the VM RAM to be mapped on huge pages by backing it with `/dev/hugepages`. And
+second thing, we need to create some huge pages in the guest itself so they can
+be consumed.
+
+```bash
+./cloud-hypervisor \
+    --cpus 1 \
+    --memory size=8G,file=/dev/hugepages \
+    --disk path=clear-kvm.img \
+    --kernel custom-bzImage \
+    --cmdline "console=ttyS0 root=/dev/vda3 hugepagesz=2M hugepages=2048" \
+    --net tap=,mac=,iommu=on
+```
+
+### Nested usage
+
+Let's now look at the specific example of nested virtualization. In order to
+reach optimized performances, the L2 guest also need to be mapped based on
+huge pages. Here is how to achieve this, assuming the physical device you are
+passing through is `0000:00:01.0`.
+
+```bash
+./cloud-hypervisor \
+    --cpus 1 \
+    --memory size=8G,file=/dev/hugepages \
+    --disk path=clear-kvm.img \
+    --kernel custom-bzImage \
+    --cmdline "console=ttyS0 root=/dev/vda3 kvm-intel.nested=1 vfio_iommu_type1.allow_unsafe_interrupts rw hugepagesz=2M hugepages=2048" \
+    --device path=/sys/bus/pci/devices/0000:00:01.0,iommu=on
+```
+
+Once the L1 VM is running, unbind the device from the default driver in the
+guest, and bind it to VFIO (it should appear as `0000:00:04.0`).
+
+```bash
+echo 0000:00:04.0 > /sys/bus/pci/devices/0000\:00\:04.0/driver/unbind
+echo 8086 1502 > /sys/bus/pci/drivers/vfio-pci/new_id
+```
+
+Last thing is to start the L2 guest with the huge pages memory backend.
+
+```bash
+./cloud-hypervisor \
+    --cpus 1 \
+    --memory size=4G,file=/dev/hugepages \
+    --disk path=clear-kvm.img \
+    --kernel custom-bzImage \
+    --cmdline "console=ttyS0 root=/dev/vda3" \
+    --device path=/sys/bus/pci/devices/0000:00:04.0
+```

docs/networking.md

@@ -1,6 +1,6 @@
 # How to use networking
 
-cloud-hypervisor can emulate one or more virtual network interfaces, represented at the hypervisor host by [tap devices](https://www.kernel.org/doc/Documentation/networking/tuntap.txt"). This guide briefly describes, in a manual and distribution neutral way, how to setup and use networking with cloud-hypevisor.
+cloud-hypervisor can emulate one or more virtual network interfaces, represented at the hypervisor host by [tap devices](https://www.kernel.org/doc/Documentation/networking/tuntap.txt). This guide briefly describes, in a manual and distribution neutral way, how to setup and use networking with cloud-hypevisor.
 
 ## Start cloud-hypervisor with net devices
 
@@ -10,7 +10,7 @@ Use one `--net` command-line argument from cloud-hypervisor to specify the emula
 ./cloud-hypervisor \
     --cpus 4 \
     --memory "size=512M" \
-    --disk my-root-disk.img \
+    --disk path=my-root-disk.img \
     --kernel my-vmlinux.bin \
     --cmdline "console=ttyS0 reboot=k panic=1 nomodules root=/dev/vda3" \
     --net tap=ich0,mac=a4:a1:c2:00:00:01,ip=192.168.4.2,mask=255.255.255.0 \

docs/vfio.md

@@ -66,13 +66,13 @@ takes the device's sysfs path as an argument. In our example it is
 ```
 ./target/debug/cloud-hypervisor \
     --kernel ~/vmlinux \
-    --disk ~/clear-29160-kvm.img \
+    --disk path=~/clear-29160-kvm.img \
     --console off \
     --serial tty \
     --cmdline "console=ttyS0 reboot=k panic=1 nomodules i8042.noaux i8042.nomux i8042.nopnp i8042.dumbkbd root=/dev/vda3" \
     --cpus 4 \
     --memory size=512M \
-    --device /sys/bus/pci/devices/0000:01:00.0/
+    --device path=/sys/bus/pci/devices/0000:01:00.0/
 ```
 
 The guest kernel will then detect the card reader on its PCI bus and provided

net_gen/Cargo.toml

@@ -4,4 +4,4 @@ version = "0.1.0"
 authors = ["The Chromium OS Authors"]
 
 [dependencies]
-vmm-sys-util = { git = "https://github.com/rust-vmm/vmm-sys-util" }
+vmm-sys-util = ">=0.3.1"

net_util/Cargo.toml

@@ -7,9 +7,9 @@ authors = ["The Chromium OS Authors"]
 libc = "0.2.60"
 rand = "0.7.0"
 serde = "1.0.98"
+vmm-sys-util = ">=0.3.1"
 
 net_gen = { path = "../net_gen" }
-vmm-sys-util = { git = "https://github.com/rust-vmm/vmm-sys-util" }
 
 [dev-dependencies]
 lazy_static = "1.3.0"

net_util/src/mac.rs

@@ -6,6 +6,7 @@
 // found in the THIRD-PARTY file.
 
 use rand::Rng;
+use std::fmt;
 use std::result::Result;
 
 use serde::de::{Deserialize, Deserializer, Error};
@@ -66,14 +67,6 @@ impl MacAddr {
         &self.bytes
     }
 
-    pub fn to_string(self) -> String {
-        let b = &self.bytes;
-        format!(
-            "{:02x}:{:02x}:{:02x}:{:02x}:{:02x}:{:02x}",
-            b[0], b[1], b[2], b[3], b[4], b[5]
-        )
-    }
-
     pub fn local_random() -> MacAddr {
         // Generate a fully random MAC
         let mut random_bytes = rand::thread_rng().gen::<[u8; MAC_ADDR_LEN]>();
@@ -87,6 +80,17 @@ impl MacAddr {
     }
 }
 
+impl fmt::Display for MacAddr {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        let b = &self.bytes;
+        write!(
+            f,
+            "{:02x}:{:02x}:{:02x}:{:02x}:{:02x}:{:02x}",
+            b[0], b[1], b[2], b[3], b[4], b[5]
+        )
+    }
+}
+
 impl Serialize for MacAddr {
     fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
     where

net_util/src/tap.rs

@@ -49,6 +49,15 @@ impl PartialEq for Tap {
     }
 }
 
+impl std::clone::Clone for Tap {
+    fn clone(&self) -> Self {
+        Tap {
+            tap_file: self.tap_file.try_clone().unwrap(),
+            if_name: self.if_name,
+        }
+    }
+}
+
 // Returns a byte vector representing the contents of a null terminated C string which
 // contains if_name.
 fn build_terminated_if_name(if_name: &str) -> Result<Vec<u8>> {

pci/Cargo.toml

@@ -8,9 +8,7 @@ edition = "2018"
 vm-allocator = { path = "../vm-allocator" }
 byteorder = "1.3.2"
 devices = { path = "../devices" }
-kvm-bindings = "0.1.1"
-kvm-ioctls = { git = "https://github.com/rust-vmm/kvm-ioctls", branch = "master" }
 libc = "0.2.60"
 log = "0.4.8"
 vm-memory = { git = "https://github.com/rust-vmm/vm-memory" }
-vmm-sys-util = { git = "https://github.com/rust-vmm/vmm-sys-util" }
+vmm-sys-util = ">=0.3.1"

pci/src/bus.rs

@@ -5,12 +5,13 @@
 use crate::configuration::{
     PciBarRegionType, PciBridgeSubclass, PciClassCode, PciConfiguration, PciHeaderType,
 };
-use crate::device::{Error as PciDeviceError, PciDevice};
+use crate::device::{DeviceRelocation, Error as PciDeviceError, PciDevice};
 use byteorder::{ByteOrder, LittleEndian};
 use devices::BusDevice;
 use std;
-use std::sync::Arc;
+use std::any::Any;
-use std::sync::Mutex;
+use std::ops::DerefMut;
+use std::sync::{Arc, Mutex, Weak};
 use vm_memory::{Address, GuestAddress, GuestUsize};
 
 const VENDOR_ID_INTEL: u16 = 0x8086;
@@ -69,32 +70,36 @@ impl PciDevice for PciRoot {
     fn read_config_register(&self, reg_idx: usize) -> u32 {
         self.config.read_reg(reg_idx)
     }
+
+    fn as_any(&mut self) -> &mut dyn Any {
+        self
+    }
 }
 
-pub struct PciConfigIo {
+pub struct PciBus {
     /// Devices attached to this bus.
     /// Device 0 is host bridge.
     devices: Vec<Arc<Mutex<dyn PciDevice>>>,
-    /// Config space register.
+    device_reloc: Weak<dyn DeviceRelocation>,
-    config_address: u32,
 }
 
-impl PciConfigIo {
+impl PciBus {
-    pub fn new(pci_root: PciRoot) -> Self {
+    pub fn new(pci_root: PciRoot, device_reloc: Weak<dyn DeviceRelocation>) -> Self {
         let mut devices: Vec<Arc<Mutex<dyn PciDevice>>> = Vec::new();
+
         devices.push(Arc::new(Mutex::new(pci_root)));
 
-        PciConfigIo {
+        PciBus {
             devices,
-            config_address: 0,
+            device_reloc,
         }
     }
 
     pub fn register_mapping(
         &self,
         dev: Arc<Mutex<dyn BusDevice>>,
-        io_bus: &mut devices::Bus,
+        io_bus: &devices::Bus,
-        mmio_bus: &mut devices::Bus,
+        mmio_bus: &devices::Bus,
         bars: Vec<(GuestAddress, GuestUsize, PciBarRegionType)>,
     ) -> Result<()> {
         for (address, size, type_) in bars {
@@ -119,6 +124,25 @@ impl PciConfigIo {
         Ok(())
     }
 
+    pub fn next_device_id(&self) -> u32 {
+        self.devices.len() as u32
+    }
+}
+
+pub struct PciConfigIo {
+    /// Config space register.
+    config_address: u32,
+    pci_bus: Arc<Mutex<PciBus>>,
+}
+
+impl PciConfigIo {
+    pub fn new(pci_bus: Arc<Mutex<PciBus>>) -> Self {
+        PciConfigIo {
+            pci_bus,
+            config_address: 0,
+        }
+    }
+
     pub fn config_space_read(&self) -> u32 {
         let enabled = (self.config_address & 0x8000_0000) != 0;
         if !enabled {
@@ -138,9 +162,14 @@ impl PciConfigIo {
             return 0xffff_ffff;
         }
 
-        self.devices.get(device).map_or(0xffff_ffff, |d| {
+        self.pci_bus
-            d.lock().unwrap().read_config_register(register)
+            .lock()
-        })
+            .unwrap()
+            .devices
+            .get(device)
+            .map_or(0xffff_ffff, |d| {
+                d.lock().unwrap().read_config_register(register)
+            })
     }
 
     pub fn config_space_write(&mut self, offset: u64, data: &[u8]) {
@@ -161,10 +190,26 @@ impl PciConfigIo {
             return;
         }
 
-        if let Some(d) = self.devices.get(device) {
+        let pci_bus = self.pci_bus.lock().unwrap();
-            d.lock()
+        if let Some(d) = pci_bus.devices.get(device) {
-                .unwrap()
+            let mut device = d.lock().unwrap();
-                .write_config_register(register, offset, data);
+
+            // Find out if one of the device's BAR is being reprogrammed, and
+            // reprogram it if needed.
+            if let Some(params) = device.detect_bar_reprogramming(register, data) {
+                if let Err(e) = pci_bus.device_reloc.upgrade().unwrap().move_bar(
+                    params.old_base,
+                    params.new_base,
+                    params.len,
+                    device.deref_mut(),
+                    params.region_type,
+                ) {
+                    error!("Failed moving device BAR: {}", e);
+                }
+            }
+
+            // Update the register value
+            device.write_config_register(register, offset, data);
         }
     }
 
@@ -223,17 +268,12 @@ impl BusDevice for PciConfigIo {
 
 /// Emulates PCI memory-mapped configuration access mechanism.
 pub struct PciConfigMmio {
-    /// Devices attached to this bus.
+    pci_bus: Arc<Mutex<PciBus>>,
-    /// Device 0 is host bridge.
-    devices: Vec<Arc<Mutex<dyn PciDevice>>>,
 }
 
 impl PciConfigMmio {
-    pub fn new(pci_root: PciRoot) -> Self {
+    pub fn new(pci_bus: Arc<Mutex<PciBus>>) -> Self {
-        let mut devices: Vec<Arc<Mutex<dyn PciDevice>>> = Vec::new();
+        PciConfigMmio { pci_bus }
-
-        devices.push(Arc::new(Mutex::new(pci_root)));
-        PciConfigMmio { devices }
     }
 
     fn config_space_read(&self, config_address: u32) -> u32 {
@@ -244,9 +284,14 @@ impl PciConfigMmio {
             return 0xffff_ffff;
         }
 
-        self.devices.get(device).map_or(0xffff_ffff, |d| {
+        self.pci_bus
-            d.lock().unwrap().read_config_register(register)
+            .lock()
-        })
+            .unwrap()
+            .devices
+            .get(device)
+            .map_or(0xffff_ffff, |d| {
+                d.lock().unwrap().read_config_register(register)
+            })
     }
 
     fn config_space_write(&mut self, config_address: u32, offset: u64, data: &[u8]) {
@@ -261,10 +306,26 @@ impl PciConfigMmio {
             return;
         }
 
-        if let Some(d) = self.devices.get(device) {
+        let pci_bus = self.pci_bus.lock().unwrap();
-            d.lock()
+        if let Some(d) = pci_bus.devices.get(device) {
-                .unwrap()
+            let mut device = d.lock().unwrap();
-                .write_config_register(register, offset, data);
+
+            // Find out if one of the device's BAR is being reprogrammed, and
+            // reprogram it if needed.
+            if let Some(params) = device.detect_bar_reprogramming(register, data) {
+                if let Err(e) = pci_bus.device_reloc.upgrade().unwrap().move_bar(
+                    params.old_base,
+                    params.new_base,
+                    params.len,
+                    device.deref_mut(),
+                    params.region_type,
+                ) {
+                    error!("Failed moving device BAR: {}", e);
+                }
+            }
+
+            // Update the register value
+            device.write_config_register(register, offset, data);
         }
     }
 }

pci/src/configuration.rs

@@ -4,6 +4,7 @@
 
 use std::sync::{Arc, Mutex};
 
+use crate::device::BarReprogrammingParams;
 use crate::{MsixConfig, PciInterruptPin};
 use byteorder::{ByteOrder, LittleEndian};
 use std::fmt::{self, Display};
@@ -249,8 +250,11 @@ pub trait PciCapability {
 pub struct PciConfiguration {
     registers: [u32; NUM_CONFIGURATION_REGISTERS],
     writable_bits: [u32; NUM_CONFIGURATION_REGISTERS], // writable bits for each register.
+    bar_addr: [u32; NUM_BAR_REGS],
     bar_size: [u32; NUM_BAR_REGS],
     bar_used: [bool; NUM_BAR_REGS],
+    bar_type: [Option<PciBarRegionType>; NUM_BAR_REGS],
+    rom_bar_addr: u32,
     rom_bar_size: u32,
     rom_bar_used: bool,
     // Contains the byte offset and size of the last capability.
@@ -260,7 +264,7 @@ pub struct PciConfiguration {
 }
 
 /// See pci_regs.h in kernel
-#[derive(Copy, Clone)]
+#[derive(Copy, Clone, PartialEq)]
 pub enum PciBarRegionType {
     Memory32BitRegion = 0,
     IORegion = 0x01,
@@ -343,6 +347,7 @@ impl PciConfiguration {
     ) -> Self {
         let mut registers = [0u32; NUM_CONFIGURATION_REGISTERS];
         let mut writable_bits = [0u32; NUM_CONFIGURATION_REGISTERS];
+        let bar_addr = [0u32; NUM_BAR_REGS];
         let bar_size = [0u32; NUM_BAR_REGS];
         registers[0] = u32::from(device_id) << 16 | u32::from(vendor_id);
         // TODO(dverkamp): Status should be write-1-to-clear
@@ -372,8 +377,11 @@ impl PciConfiguration {
         PciConfiguration {
             registers,
             writable_bits,
+            bar_addr,
             bar_size,
             bar_used: [false; NUM_BAR_REGS],
+            bar_type: [None; NUM_BAR_REGS],
+            rom_bar_addr: 0,
             rom_bar_size: 0,
             rom_bar_used: false,
             last_capability: None,
@@ -390,15 +398,15 @@ impl PciConfiguration {
     /// Writes a 32bit register to `reg_idx` in the register map.
     pub fn write_reg(&mut self, reg_idx: usize, value: u32) {
         let mut mask = self.writable_bits[reg_idx];
-        if reg_idx >= BAR0_REG
+
-            && reg_idx < BAR0_REG + NUM_BAR_REGS
+        if value == 0xffff_ffff {
-            && (value & BAR_MEM_ADDR_MASK) == BAR_MEM_ADDR_MASK
+            if reg_idx >= BAR0_REG && reg_idx < BAR0_REG + NUM_BAR_REGS {
-        {
+                // Handle very specific case where the BAR is being written with
-            // Handle very specific case where the BAR is being written with
+                // all 1's to retrieve the BAR size on next BAR reading.
-            // all 1's to retrieve the BAR size on next BAR reading.
+                mask = self.bar_size[reg_idx - 4];
-            mask = self.bar_size[reg_idx - 4];
+            } else if reg_idx == ROM_BAR_REG {
-        } else if reg_idx == ROM_BAR_REG && (value & ROM_BAR_ADDR_MASK) == ROM_BAR_ADDR_MASK {
+                mask = self.rom_bar_size;
-            mask = self.rom_bar_size;
+            }
         }
 
         if let Some(r) = self.registers.get_mut(reg_idx) {
@@ -497,6 +505,7 @@ impl PciConfiguration {
 
                 self.registers[bar_idx + 1] = (config.addr >> 32) as u32;
                 self.writable_bits[bar_idx + 1] = 0xffff_ffff;
+                self.bar_addr[config.reg_idx + 1] = self.registers[bar_idx + 1];
                 self.bar_size[config.reg_idx + 1] = (config.size >> 32) as u32;
                 self.bar_used[config.reg_idx + 1] = true;
             }
@@ -512,8 +521,10 @@ impl PciConfiguration {
 
         self.registers[bar_idx] = ((config.addr as u32) & mask) | lower_bits;
         self.writable_bits[bar_idx] = mask;
+        self.bar_addr[config.reg_idx] = self.registers[bar_idx];
         self.bar_size[config.reg_idx] = config.size as u32;
         self.bar_used[config.reg_idx] = true;
+        self.bar_type[config.reg_idx] = Some(config.region_type);
         Ok(config.reg_idx)
     }
 
@@ -542,24 +553,25 @@ impl PciConfiguration {
 
         self.registers[config.reg_idx] = (config.addr as u32) | active;
         self.writable_bits[config.reg_idx] = ROM_BAR_ADDR_MASK;
+        self.rom_bar_addr = self.registers[config.reg_idx];
         self.rom_bar_size = config.size as u32;
         self.rom_bar_used = true;
         Ok(config.reg_idx)
     }
 
-    /// Returns the address of the given 32 bits BAR region.
+    /// Returns the address of the given BAR region.
-    pub fn get_bar32_addr(&self, bar_num: usize) -> u32 {
+    pub fn get_bar_addr(&self, bar_num: usize) -> u64 {
         let bar_idx = BAR0_REG + bar_num;
 
-        self.registers[bar_idx] & BAR_MEM_ADDR_MASK
+        let mut addr = u64::from(self.bar_addr[bar_num] & self.writable_bits[bar_idx]);
-    }
 
-    /// Returns the address of the given 64 bits BAR region.
+        if let Some(bar_type) = self.bar_type[bar_num] {
-    pub fn get_bar64_addr(&self, bar_num: usize) -> u64 {
+            if bar_type == PciBarRegionType::Memory64BitRegion {
-        let bar_idx = BAR0_REG + bar_num;
+                addr |= u64::from(self.bar_addr[bar_num + 1]) << 32;
+            }
+        }
 
-        u64::from(self.registers[bar_idx] & BAR_MEM_ADDR_MASK)
+        addr
-            | (u64::from(self.registers[bar_idx + 1]) << 32)
     }
 
     /// Configures the IRQ line and pin used by this device.
@@ -645,6 +657,101 @@ impl PciConfiguration {
     pub fn read_config_register(&self, reg_idx: usize) -> u32 {
         self.read_reg(reg_idx)
     }
+
+    pub fn detect_bar_reprogramming(
+        &mut self,
+        reg_idx: usize,
+        data: &[u8],
+    ) -> Option<BarReprogrammingParams> {
+        if data.len() != 4 {
+            return None;
+        }
+
+        let value = LittleEndian::read_u32(data);
+
+        if value == 0xffff_ffff {
+            return None;
+        }
+
+        let mask = self.writable_bits[reg_idx];
+        if reg_idx >= BAR0_REG && reg_idx < BAR0_REG + NUM_BAR_REGS {
+            let bar_idx = reg_idx - 4;
+            if (value & mask) != (self.bar_addr[bar_idx] & mask) {
+                // Handle special case where the address being written is
+                // different from the address initially provided. This is a
+                // BAR reprogramming case which needs to be properly caught.
+                if let Some(bar_type) = self.bar_type[bar_idx] {
+                    match bar_type {
+                        PciBarRegionType::Memory64BitRegion => {}
+                        _ => {
+                            debug!(
+                                "DETECT BAR REPROG: current 0x{:x}, new 0x{:x}",
+                                self.registers[reg_idx], value
+                            );
+                            let old_base = u64::from(self.bar_addr[bar_idx] & mask);
+                            let new_base = u64::from(value & mask);
+                            let len = u64::from(self.bar_size[bar_idx]);
+                            let region_type = bar_type;
+
+                            self.bar_addr[bar_idx] = value;
+
+                            return Some(BarReprogrammingParams {
+                                old_base,
+                                new_base,
+                                len,
+                                region_type,
+                            });
+                        }
+                    }
+                } else if (reg_idx > BAR0_REG)
+                    && (self.registers[reg_idx - 1] & self.writable_bits[reg_idx - 1])
+                        != (self.bar_addr[bar_idx - 1] & self.writable_bits[reg_idx - 1])
+                {
+                    debug!(
+                        "DETECT BAR REPROG: current 0x{:x}, new 0x{:x}",
+                        self.registers[reg_idx], value
+                    );
+                    let old_base = u64::from(self.bar_addr[bar_idx] & mask) << 32
+                        | u64::from(self.bar_addr[bar_idx - 1] & self.writable_bits[reg_idx - 1]);
+                    let new_base = u64::from(value & mask) << 32
+                        | u64::from(self.registers[reg_idx - 1] & self.writable_bits[reg_idx - 1]);
+                    let len = u64::from(self.bar_size[bar_idx]) << 32
+                        | u64::from(self.bar_size[bar_idx - 1]);
+                    let region_type = PciBarRegionType::Memory64BitRegion;
+
+                    self.bar_addr[bar_idx] = value;
+                    self.bar_addr[bar_idx - 1] = self.registers[reg_idx - 1];
+
+                    return Some(BarReprogrammingParams {
+                        old_base,
+                        new_base,
+                        len,
+                        region_type,
+                    });
+                }
+            }
+        } else if reg_idx == ROM_BAR_REG && (value & mask) != (self.rom_bar_addr & mask) {
+            debug!(
+                "DETECT ROM BAR REPROG: current 0x{:x}, new 0x{:x}",
+                self.registers[reg_idx], value
+            );
+            let old_base = u64::from(self.rom_bar_addr & mask);
+            let new_base = u64::from(value & mask);
+            let len = u64::from(self.rom_bar_size);
+            let region_type = PciBarRegionType::Memory32BitRegion;
+
+            self.rom_bar_addr = value;
+
+            return Some(BarReprogrammingParams {
+                old_base,
+                new_base,
+                len,
+                region_type,
+            });
+        }
+
+        None
+    }
 }
 
 impl Default for PciBarConfiguration {

pci/src/device.rs

@@ -6,19 +6,19 @@ use crate::configuration::{self, PciBarRegionType};
 use crate::msix::MsixTableEntry;
 use crate::PciInterruptPin;
 use devices::BusDevice;
-use std;
+use std::any::Any;
 use std::fmt::{self, Display};
 use std::sync::Arc;
+use std::{self, io, result};
 use vm_allocator::SystemAllocator;
 use vm_memory::{GuestAddress, GuestUsize};
-use vmm_sys_util::eventfd::EventFd;
 
 pub struct InterruptParameters<'a> {
     pub msix: Option<&'a MsixTableEntry>,
 }
 
 pub type InterruptDelivery =
-    Box<dyn Fn(InterruptParameters) -> std::result::Result<(), std::io::Error> + Send + Sync>;
+    Box<dyn Fn(InterruptParameters) -> result::Result<(), io::Error> + Send + Sync>;
 
 #[derive(Debug)]
 pub enum Error {
@@ -47,6 +47,14 @@ impl Display for Error {
     }
 }
 
+#[derive(Clone, Copy)]
+pub struct BarReprogrammingParams {
+    pub old_base: u64,
+    pub new_base: u64,
+    pub len: u64,
+    pub region_type: PciBarRegionType,
+}
+
 pub trait PciDevice: BusDevice {
     /// Assign a legacy PCI IRQ to this device.
     /// The device may write to `irq_evt` to trigger an interrupt.
@@ -70,11 +78,6 @@ pub trait PciDevice: BusDevice {
         Ok(Vec::new())
     }
 
-    /// Gets a list of ioeventfds that should be registered with the running VM. The list is
-    /// returned as a Vec of (eventfd, addr, datamatch) tuples.
-    fn ioeventfds(&self) -> Vec<(&EventFd, u64, u64)> {
-        Vec::new()
-    }
     /// Sets a register in the configuration space.
     /// * `reg_idx` - The index of the config register to modify.
     /// * `offset` - Offset in to the register.
@@ -82,6 +85,14 @@ pub trait PciDevice: BusDevice {
     /// Gets a register from the configuration space.
     /// * `reg_idx` - The index of the config register to read.
     fn read_config_register(&self, reg_idx: usize) -> u32;
+    /// Detects if a BAR is being reprogrammed.
+    fn detect_bar_reprogramming(
+        &mut self,
+        _reg_idx: usize,
+        _data: &[u8],
+    ) -> Option<BarReprogrammingParams> {
+        None
+    }
     /// Reads from a BAR region mapped in to the device.
     /// * `addr` - The guest address inside the BAR.
     /// * `data` - Filled with the data from `addr`.
@@ -90,4 +101,26 @@ pub trait PciDevice: BusDevice {
     /// * `addr` - The guest address inside the BAR.
     /// * `data` - The data to write.
     fn write_bar(&mut self, _base: u64, _offset: u64, _data: &[u8]) {}
+    /// Relocates the BAR to a different address in guest address space.
+    fn move_bar(&mut self, _old_base: u64, _new_base: u64) -> result::Result<(), io::Error> {
+        Ok(())
+    }
+    /// Provides a mutable reference to the Any trait. This is useful to let
+    /// the caller have access to the underlying type behind the trait.
+    fn as_any(&mut self) -> &mut dyn Any;
+}
+
+/// This trait defines a set of functions which can be triggered whenever a
+/// PCI device is modified in any way.
+pub trait DeviceRelocation: Send + Sync {
+    /// The BAR needs to be moved to a different location in the guest address
+    /// space. This follows a decision from the software running in the guest.
+    fn move_bar(
+        &self,
+        old_base: u64,
+        new_base: u64,
+        len: u64,
+        pci_dev: &mut dyn PciDevice,
+        region_type: PciBarRegionType,
+    ) -> result::Result<(), io::Error>;
 }

pci/src/lib.rs

@@ -6,7 +6,6 @@
 #[macro_use]
 extern crate log;
 extern crate devices;
-extern crate kvm_ioctls;
 extern crate vm_memory;
 extern crate vmm_sys_util;
 
@@ -16,14 +15,15 @@ mod device;
 mod msi;
 mod msix;
 
-pub use self::bus::{PciConfigIo, PciConfigMmio, PciRoot, PciRootError};
+pub use self::bus::{PciBus, PciConfigIo, PciConfigMmio, PciRoot, PciRootError};
 pub use self::configuration::{
     PciBarConfiguration, PciBarPrefetchable, PciBarRegionType, PciCapability, PciCapabilityID,
     PciClassCode, PciConfiguration, PciHeaderType, PciMassStorageSubclass,
     PciNetworkControllerSubclass, PciProgrammingInterface, PciSerialBusSubClass, PciSubclass,
 };
 pub use self::device::{
-    Error as PciDeviceError, InterruptDelivery, InterruptParameters, PciDevice,
+    BarReprogrammingParams, DeviceRelocation, Error as PciDeviceError, InterruptDelivery,
+    InterruptParameters, PciDevice,
 };
 pub use self::msi::MsiCap;
 pub use self::msix::{MsixCap, MsixConfig, MsixTableEntry, MSIX_TABLE_ENTRY_SIZE};

pci/src/msi.rs

@@ -132,7 +132,7 @@ impl MsiCap {
             4 => {
                 let value = LittleEndian::read_u32(data);
                 match offset {
-                    MSI_MSG_CTL_OFFSET => {
+                    0x0 => {
                         self.msg_ctl = (self.msg_ctl & !(MSI_CTL_ENABLE | MSI_CTL_MULTI_MSG_ENABLE))
                             | ((value >> 16) as u16 & (MSI_CTL_ENABLE | MSI_CTL_MULTI_MSG_ENABLE))
                     }

qcow/Cargo.toml

@@ -13,7 +13,7 @@ byteorder = "1.3.2"
 libc = "0.2.60"
 log = "0.4.8"
 remain = "0.1.3"
-vmm-sys-util = { git = "https://github.com/rust-vmm/vmm-sys-util" }
+vmm-sys-util = ">=0.3.1"
 
 [dev-dependencies]
 tempfile = "3.1.0"

release-notes.md

@@ -1,23 +1,190 @@
+- [v0.4.0](#v040)
+    + [Dynamic virtual CPUs addition](#dynamic-virtual-cpus-addition)
+    + [Programmatic firmware tables generation](#programmatic-firmware-tables-generation)
+    + [Filesystem and block devices vhost-user backends](#filesystem-and-block-devices-vhost-user-backends)
+    + [Guest pause and resume](#guest-pause-and-resume)
+    + [Userspace IOAPIC by default](#userspace-ioapic-by-default)
+    + [PCI BAR reprogramming](#pci-bar-reprogramming)
+    + [New `cloud-hypervisor` organization](#new--cloud-hypervisor--organization)
+    + [Contributors](#contributors)
+- [v0.3.0](#v030)
+    + [Block device offloading](#block-device-offloading)
+    + [Network device backend](#network-device-backend)
+    + [Virtual sockets](#virtual-sockets)
+    + [HTTP based API](#http-based-api)
+    + [Memory mapped virtio transport](#memory-mapped-virtio-transport)
+    + [Paravirtualized IOMMU](#paravirtualized-iommu)
+    + [Ubuntu 19.10](#ubuntu-1910)
+    + [Guest large memory](#guest-large-memory)
 - [v0.2.0](#v020)
-	- [Network device offloading](#network-device-offloading)
+    + [Network device offloading](#network-device-offloading)
-	- [Minimal hardware-reduced ACPI](#minimal-hardware-reduced-acpi)
+    + [Minimal hardware-reduced ACPI](#minimal-hardware-reduced-acpi)
-	- [Debug I/O port](#debug-io-port)
+    + [Debug I/O port](#debug-i-o-port)
-	- [Improved direct device assignment](#improved-direct-device-assignment)
+    + [Improved direct device assignment](#improved-direct-device-assignment)
-	- [Improved shared filesystem](#improved-shared-filesystem)
+    + [Improved shared filesystem](#improved-shared-filesystem)
-	- [Ubuntu bionic based CI](#ubuntu-bionic-based-ci)
+    + [Ubuntu bionic based CI](#ubuntu-bionic-based-ci)
 - [v0.1.0](#v010)
-	- [Shared filesystem](#shared-filesystem)
+    + [Shared filesystem](#shared-filesystem)
-	- [Initial direct device assignment support](#initial-direct-device-assignment-support)
+    + [Initial direct device assignment support](#initial-direct-device-assignment-support)
-	- [Userspace IOAPIC](#userspace-ioapic)
+    + [Userspace IOAPIC](#userspace-ioapic)
-	- [Virtual persistent memory](#virtual-persistent-memory)
+    + [Virtual persistent memory](#virtual-persistent-memory)
-	- [Linux kernel bzImage](#linux-kernel-bzimage)
+    + [Linux kernel bzImage](#linux-kernel-bzimage)
-	- [Console over virtio](#console-over-virtio)
+    + [Console over virtio](#console-over-virtio)
-	- [Unit testing](#unit-testing)
+    + [Unit testing](#unit-testing)
-	- [Integration tests parallelization](#integration-tests-parallelization)
+    + [Integration tests parallelization](#integration-tests-parallelization)
+
+# v0.4.0
+
+This release has been tracked through the [0.4.0 project](https://github.com/cloud-hypervisor/cloud-hypervisor/projects/4).
+
+Highlights for `cloud-hypervisor` version 0.4.0 include:
+
+### Dynamic virtual CPUs addition
+
+As a way to vertically scale Cloud-Hypervisor guests, we now support dynamically
+adding virtual CPUs to the guests, a mechanism also known as CPU hot plug.
+Through hardware-reduced ACPI notifications, Cloud Hypervisor can now add CPUs
+to an already running guest and the high level operations for that process are
+documented [here](https://github.com/cloud-hypervisor/cloud-hypervisor/blob/master/docs/hotplug.md)
+
+During the next release cycles we are planning to extend Cloud Hypervisor
+hot plug framework to other resources, namely PCI devices and memory.
+
+### Programmatic firmware tables generation
+
+As part of the CPU hot plug feature enablement, and as a requirement for hot
+plugging other resources like devices or RAM, we added support for
+programmatically generating the needed ACPI tables. Through a dedicated
+`acpi-tables` crate, we now have a flexible and clean way of generating those
+tables based on the VMM device model and topology.
+
+### Filesystem and block devices vhost-user backends
+
+Our objective of running all Cloud Hypervisor paravirtualized I/O to a
+vhost-user based framework is getting closer as we've added Rust based
+implementations for vhost-user-blk and virtiofs backends. Together with the
+vhost-user-net backend that came with the 0.3.0 release, this will form the
+default Cloud Hypervisor I/O architecture.
+
+### Guest pause and resume
+
+As an initial requiremnt for enabling live migration, we added support for
+pausing and resuming any VMM components. As an intermediate step towards live
+migration, the upcoming guest snapshotting feature will be based on the pause
+and resume capabilities.
+
+### Userspace IOAPIC by default
+
+As a way to simplify our device manager implementation, but also in order to
+stay away from privileged rings as often as possible, any device that relies on
+pin based interrupts will be using the userspace IOAPIC implementation by
+default.
+
+### PCI BAR reprogramming
+
+In order to allow for a more flexible device model, and also support guests
+that would want to move PCI devices, we added support for PCI devices BAR
+reprogramming.
+
+### New `cloud-hypervisor` organization
+
+As we wanted to be more flexible on how we manage the Cloud Hypervisor project,
+we decided to move it under a [dedicated GitHub organization](https://github.com/cloud-hypervisor).
+Together with the [cloud-hypervisor](https://github.com/cloud-hypervisor/cloud-hypervisor)
+project, this new organization also now hosts our [kernel](https://github.com/cloud-hypervisor/linux)
+and [firmware](https://github.com/cloud-hypervisor/rust-hypervisor-firmware)
+repositories. We may also use it to host any rust-vmm that we'd need to
+temporarily fork.
+Thanks to GitHub's seamless repository redirections, the move is completely
+transparent to all Cloud Hypervisor contributors, users and followers.
+
+### Contributors
+
+Many thanks to everyone that contributed to the 0.4.0 release:
+
+* Cathy Zhang <cathy.zhang@intel.com>
+* Emin Ghuliev <drmint80@gmail.com>
+* Jose Carlos Venegas Munoz <jose.carlos.venegas.munoz@intel.com>
+* Qiu Wenbo <qiuwenbo@phytium.com.cn>
+* Rob Bradford <robert.bradford@intel.com>
+* Samuel Ortiz <sameo@linux.intel.com>
+* Sebastien Boeuf <sebastien.boeuf@intel.com>
+* Sergio Lopez <slp@redhat.com>
+* Wu Zongyong <wuzongyong@linux.alibaba.com>
+
+# v0.3.0
+
+This release has been tracked through the [0.3.0 project](https://github.com/cloud-hypervisor/cloud-hypervisor/projects/3).
+
+Highlights for `cloud-hypervisor` version 0.3.0 include:
+
+### Block device offloading
+
+We continue to work on offloading paravirtualized I/O to external processes,
+and we added support for
+[vhost-user-blk](https://access.redhat.com/solutions/3394851) backends.
+This enables `cloud-hypervisor` users to plug a `vhost-user` based block device
+like [SPDK](https://spdk.io)) into the VMM as their paravirtualized storage
+backend.
+
+### Network device backend
+
+The previous release provided support for
+[vhost-user-net](https://access.redhat.com/solutions/3394851) backends. Now we
+also provide a TAP based vhost-user-net backend, implemented in Rust. Together
+with the vhost-user-net device implementation, this will eventually become the
+Cloud Hypervisor default paravirtualized networking architecture.
+
+### Virtual sockets
+
+In order to more efficiently and securely communicate between host and guest,
+we added an hybrid implementation of the
+[VSOCK](http://man7.org/linux/man-pages/man7/vsock.7.html) socket address
+family over virtio. Credits go to the
+[Firecracker](https://github.com/firecracker-microvm/firecracker/blob/master/docs/vsock.md)
+project as our implementation is a copy of theirs.
+
+### HTTP based API
+
+In anticipation of the need to support asynchronous operations to Cloud
+Hypervisor guests (e.g. resources hotplug and guest migration), we added a HTTP
+based API to the VMM. The API will be more extensively documented during the
+next release cycle.
+
+### Memory mapped virtio transport
+
+In order to support potential PCI-free use cases, we added support for the
+[virtio MMIO](https://docs.oasis-open.org/virtio/virtio/v1.1/cs01/virtio-v1.1-cs01.html#x1-1440002)
+transport layer. This will allow us to support simple, minimal guest
+configurations that do not require a PCI bus emulation.
+
+### Paravirtualized IOMMU
+
+As we want to improve our nested guests support, we added support for exposing
+a [paravirtualized IOMMU](https://github.com/cloud-hypervisor/cloud-hypervisor/blob/master/docs/iommu.md)
+device through virtio. This allows for a safer nested virtio and directly
+assigned devices support.
+
+To add the IOMMU support, we had to make some CLI changes for Cloud Hypervisor
+users to be able to specify if devices had to be handled through this virtual
+IOMMU or not. In particular, the `--disk` option now expects disk paths to be
+prefixed with a `path=` string, and supports an optional `iommu=[on|off]`
+setting.
+
+### Ubuntu 19.10
+
+With the latest [hypervisor firmware](https://github.com/cloud-hypervisor/rust-hypervisor-firmware),
+we can now support the latest
+[Ubuntu 19.10 (Eoan Ermine)](http://releases.ubuntu.com/19.10/) cloud images.
+
+### Large memory guests
+
+After simplifying and changing our guest address space handling, we can now
+support guests with large amount of memory (more than 64GB).
 
 # v0.2.0
 
-This release has been tracked through the [0.2.0 project](https://github.com/intel/cloud-hypervisor/projects/2).
+This release has been tracked through the [0.2.0 project](https://github.com/cloud-hypervisor/cloud-hypervisor/projects/2).
 
 Highlights for `cloud-hypervisor` version 0.2.0 include:
 
@@ -45,7 +212,7 @@ Based on the Firecracker idea of using a dedicated I/O port to measure guest
 boot times, we added support for logging guest events through the
 [0x80](https://www.intel.com/content/www/us/en/support/articles/000005500/boards-and-kits.html)
 PC debug port. This allows, among other things, for granular guest boot time
-measurements. See our [debug port documentation](https://github.com/intel/cloud-hypervisor/blob/master/docs/debug-port.md)
+measurements. See our [debug port documentation](https://github.com/cloud-hypervisor/cloud-hypervisor/blob/master/docs/debug-port.md)
 for more details.
 
 ### Improved direct device assignment
@@ -65,13 +232,13 @@ memory footprint.
 
 ### Ubuntu bionic based CI
 
-Thanks to our [simple KVM firmware](https://github.com/intel/rust-hypervisor-firmware)
+Thanks to our [simple KVM firmware](https://github.com/cloud-hypervisor/rust-hypervisor-firmware)
 improvements, we are now able to boot Ubuntu bionic images. We added those to
 our CI pipeline.
 
 # v0.1.0
 
-This release has been tracked through the [0.1.0 project](https://github.com/intel/cloud-hypervisor/projects/1).
+This release has been tracked through the [0.1.0 project](https://github.com/cloud-hypervisor/cloud-hypervisor/projects/1).
 
 Highlights for `cloud-hypervisor` version 0.1.0 include:
 
@@ -81,7 +248,7 @@ We added support for the [virtio-fs](https://virtio-fs.gitlab.io/) shared file
 system, allowing for an efficient and reliable way of sharing a filesystem
 between the host and the `cloud-hypervisor` guest.
 
-See our [filesystem sharing](https://github.com/intel/cloud-hypervisor/blob/master/docs/fs.md)
+See our [filesystem sharing](https://github.com/cloud-hypervisor/cloud-hypervisor/blob/master/docs/fs.md)
 documentation for more details on how to use virtio-fs with `cloud-hypervisor`.
 
 ### Initial direct device assignment support
@@ -90,7 +257,7 @@ VFIO (Virtual Function I/O) is a kernel framework that exposes direct device
 access to userspace. `cloud-hypervisor` uses VFIO to directly assign host
 physical devices into its guest.
 
-See our [VFIO](https://github.com/intel/cloud-hypervisor/blob/master/docs/vfio.md)
+See our [VFIO](https://github.com/cloud-hypervisor/cloud-hypervisor/blob/master/docs/vfio.md)
 documentation for more detail on how to directly assign host devices to
 `cloud-hypervisor` guests.
 

resources/linux-5.0-config

@@ -166,10 +166,11 @@ CONFIG_ANON_INODES=y
 CONFIG_SYSCTL_EXCEPTION_TRACE=y
 CONFIG_HAVE_PCSPKR_PLATFORM=y
 CONFIG_BPF=y
-# CONFIG_EXPERT is not set
+CONFIG_EXPERT=y
 CONFIG_MULTIUSER=y
 CONFIG_SGETMASK_SYSCALL=y
 CONFIG_SYSFS_SYSCALL=y
+# CONFIG_SYSCTL_SYSCALL is not set
 CONFIG_FHANDLE=y
 CONFIG_POSIX_TIMERS=y
 CONFIG_PRINTK=y
@@ -196,8 +197,10 @@ CONFIG_BPF_SYSCALL=y
 CONFIG_USERFAULTFD=y
 CONFIG_ARCH_HAS_MEMBARRIER_SYNC_CORE=y
 CONFIG_RSEQ=y
+# CONFIG_DEBUG_RSEQ is not set
 # CONFIG_EMBEDDED is not set
 CONFIG_HAVE_PERF_EVENTS=y
+# CONFIG_PC104 is not set
 
 #
 # Kernel Performance Events And Counters
@@ -206,9 +209,11 @@ CONFIG_PERF_EVENTS=y
 # CONFIG_DEBUG_PERF_USE_VMALLOC is not set
 CONFIG_VM_EVENT_COUNTERS=y
 CONFIG_SLUB_DEBUG=y
+# CONFIG_SLUB_MEMCG_SYSFS_ON is not set
 # CONFIG_COMPAT_BRK is not set
 # CONFIG_SLAB is not set
 CONFIG_SLUB=y
+# CONFIG_SLOB is not set
 CONFIG_SLAB_MERGE_DEFAULT=y
 # CONFIG_SLAB_FREELIST_RANDOM is not set
 CONFIG_SLAB_FREELIST_HARDENED=y
@@ -295,6 +300,7 @@ CONFIG_X86_CMPXCHG64=y
 CONFIG_X86_CMOV=y
 CONFIG_X86_MINIMUM_CPU_FAMILY=64
 CONFIG_X86_DEBUGCTLMSR=y
+# CONFIG_PROCESSOR_SELECT is not set
 CONFIG_CPU_SUP_INTEL=y
 CONFIG_CPU_SUP_AMD=y
 CONFIG_CPU_SUP_HYGON=y
@@ -454,6 +460,7 @@ CONFIG_ACPI_HOTPLUG_IOAPIC=y
 # CONFIG_ACPI_HED is not set
 # CONFIG_ACPI_CUSTOM_METHOD is not set
 # CONFIG_ACPI_BGRT is not set
+CONFIG_ACPI_REDUCED_HARDWARE_ONLY=y
 # CONFIG_ACPI_NFIT is not set
 CONFIG_HAVE_ACPI_APEI=y
 CONFIG_HAVE_ACPI_APEI_NMI=y
@@ -509,6 +516,8 @@ CONFIG_INTEL_IDLE=y
 CONFIG_PCI_DIRECT=y
 CONFIG_PCI_MMCONFIG=y
 CONFIG_MMCONF_FAM10H=y
+# CONFIG_PCI_CNB20LE_QUIRK is not set
+# CONFIG_ISA_BUS is not set
 CONFIG_ISA_DMA_API=y
 CONFIG_AMD_NB=y
 # CONFIG_X86_SYSFB is not set
@@ -1127,7 +1136,6 @@ CONFIG_NET_CORE=y
 # CONFIG_MACVLAN is not set
 # CONFIG_VXLAN is not set
 # CONFIG_GENEVE is not set
-# CONFIG_GTP is not set
 # CONFIG_MACSEC is not set
 # CONFIG_NETCONSOLE is not set
 CONFIG_TUN=y
@@ -1238,7 +1246,6 @@ CONFIG_UNIX98_PTYS=y
 # CONFIG_NOZOMI is not set
 # CONFIG_N_GSM is not set
 # CONFIG_TRACE_SINK is not set
-# CONFIG_LDISC_AUTOLOAD is not set
 CONFIG_DEVMEM=y
 # CONFIG_DEVKMEM is not set
 
@@ -1280,6 +1287,7 @@ CONFIG_SERIAL_ARC_NR_PORTS=1
 # CONFIG_SERIAL_FSL_LPUART is not set
 CONFIG_SERIAL_DEV_BUS=y
 CONFIG_SERIAL_DEV_CTRL_TTYPORT=y
+# CONFIG_TTY_PRINTK is not set
 CONFIG_HVC_DRIVER=y
 CONFIG_VIRTIO_CONSOLE=y
 # CONFIG_IPMI_HANDLER is not set
@@ -1290,6 +1298,7 @@ CONFIG_HW_RANDOM_AMD=y
 CONFIG_HW_RANDOM_VIA=y
 CONFIG_HW_RANDOM_VIRTIO=y
 CONFIG_NVRAM=y
+# CONFIG_R3964 is not set
 # CONFIG_APPLICOM is not set
 # CONFIG_MWAVE is not set
 CONFIG_RAW_DRIVER=y
@@ -2385,3 +2394,4 @@ CONFIG_OPTIMIZE_INLINING=y
 # CONFIG_PUNIT_ATOM_DEBUG is not set
 # CONFIG_UNWINDER_ORC is not set
 CONFIG_UNWINDER_FRAME_POINTER=y
+# CONFIG_UNWINDER_GUESS is not set

resources/linux-virtio-fs-virtio-iommu-config

@@ -4,10 +4,10 @@
 #
 
 #
-# Compiler: gcc (Ubuntu 7.4.0-1ubuntu1~18.04.1) 7.4.0
+# Compiler: gcc (Ubuntu 8.3.0-6ubuntu1) 8.3.0
 #
 CONFIG_CC_IS_GCC=y
-CONFIG_GCC_VERSION=70400
+CONFIG_GCC_VERSION=80300
 CONFIG_CLANG_VERSION=0
 CONFIG_CC_CAN_LINK=y
 CONFIG_CC_HAS_ASM_GOTO=y
@@ -61,6 +61,7 @@ CONFIG_IRQ_DOMAIN=y
 CONFIG_IRQ_DOMAIN_HIERARCHY=y
 CONFIG_GENERIC_MSI_IRQ=y
 CONFIG_GENERIC_MSI_IRQ_DOMAIN=y
+CONFIG_IRQ_MSI_IOMMU=y
 CONFIG_GENERIC_IRQ_MATRIX_ALLOCATOR=y
 CONFIG_GENERIC_IRQ_RESERVATION_MODE=y
 CONFIG_IRQ_FORCED_THREADING=y
@@ -182,10 +183,11 @@ CONFIG_SYSCTL=y
 CONFIG_SYSCTL_EXCEPTION_TRACE=y
 CONFIG_HAVE_PCSPKR_PLATFORM=y
 CONFIG_BPF=y
-# CONFIG_EXPERT is not set
+CONFIG_EXPERT=y
 CONFIG_MULTIUSER=y
 CONFIG_SGETMASK_SYSCALL=y
 CONFIG_SYSFS_SYSCALL=y
+# CONFIG_SYSCTL_SYSCALL is not set
 CONFIG_FHANDLE=y
 CONFIG_POSIX_TIMERS=y
 CONFIG_PRINTK=y
@@ -213,8 +215,10 @@ CONFIG_BPF_SYSCALL=y
 CONFIG_USERFAULTFD=y
 CONFIG_ARCH_HAS_MEMBARRIER_SYNC_CORE=y
 CONFIG_RSEQ=y
+# CONFIG_DEBUG_RSEQ is not set
 # CONFIG_EMBEDDED is not set
 CONFIG_HAVE_PERF_EVENTS=y
+# CONFIG_PC104 is not set
 
 #
 # Kernel Performance Events And Counters
@@ -225,9 +229,11 @@ CONFIG_PERF_EVENTS=y
 
 CONFIG_VM_EVENT_COUNTERS=y
 CONFIG_SLUB_DEBUG=y
+# CONFIG_SLUB_MEMCG_SYSFS_ON is not set
 # CONFIG_COMPAT_BRK is not set
 # CONFIG_SLAB is not set
 CONFIG_SLUB=y
+# CONFIG_SLOB is not set
 CONFIG_SLAB_MERGE_DEFAULT=y
 # CONFIG_SLAB_FREELIST_RANDOM is not set
 CONFIG_SLAB_FREELIST_HARDENED=y
@@ -313,6 +319,7 @@ CONFIG_X86_CMPXCHG64=y
 CONFIG_X86_CMOV=y
 CONFIG_X86_MINIMUM_CPU_FAMILY=64
 CONFIG_X86_DEBUGCTLMSR=y
+# CONFIG_PROCESSOR_SELECT is not set
 CONFIG_CPU_SUP_INTEL=y
 CONFIG_CPU_SUP_AMD=y
 CONFIG_CPU_SUP_HYGON=y
@@ -477,6 +484,7 @@ CONFIG_ACPI_HOTPLUG_IOAPIC=y
 # CONFIG_ACPI_HED is not set
 # CONFIG_ACPI_CUSTOM_METHOD is not set
 # CONFIG_ACPI_BGRT is not set
+CONFIG_ACPI_REDUCED_HARDWARE_ONLY=y
 # CONFIG_ACPI_NFIT is not set
 # CONFIG_ACPI_HMAT is not set
 CONFIG_HAVE_ACPI_APEI=y
@@ -485,6 +493,7 @@ CONFIG_HAVE_ACPI_APEI_NMI=y
 # CONFIG_DPTF_POWER is not set
 # CONFIG_PMIC_OPREGION is not set
 # CONFIG_ACPI_CONFIGFS is not set
+CONFIG_ACPI_IORT=y
 CONFIG_X86_PM_TIMER=y
 # CONFIG_SFI is not set
 
@@ -538,6 +547,8 @@ CONFIG_INTEL_IDLE=y
 CONFIG_PCI_DIRECT=y
 CONFIG_PCI_MMCONFIG=y
 CONFIG_MMCONF_FAM10H=y
+# CONFIG_PCI_CNB20LE_QUIRK is not set
+# CONFIG_ISA_BUS is not set
 CONFIG_ISA_DMA_API=y
 CONFIG_AMD_NB=y
 # CONFIG_X86_SYSFB is not set
@@ -584,7 +595,25 @@ CONFIG_EFI_EARLYCON=y
 # end of Firmware Drivers
 
 CONFIG_HAVE_KVM=y
-# CONFIG_VIRTUALIZATION is not set
+CONFIG_HAVE_KVM_IRQCHIP=y
+CONFIG_HAVE_KVM_IRQFD=y
+CONFIG_HAVE_KVM_IRQ_ROUTING=y
+CONFIG_HAVE_KVM_EVENTFD=y
+CONFIG_KVM_MMIO=y
+CONFIG_KVM_ASYNC_PF=y
+CONFIG_HAVE_KVM_MSI=y
+CONFIG_HAVE_KVM_CPU_RELAX_INTERCEPT=y
+CONFIG_KVM_VFIO=y
+CONFIG_KVM_GENERIC_DIRTYLOG_READ_PROTECT=y
+CONFIG_HAVE_KVM_IRQ_BYPASS=y
+CONFIG_HAVE_KVM_NO_POLL=y
+CONFIG_VIRTUALIZATION=y
+CONFIG_KVM=y
+CONFIG_KVM_INTEL=y
+# CONFIG_KVM_AMD is not set
+# CONFIG_VHOST_NET is not set
+# CONFIG_VHOST_VSOCK is not set
+# CONFIG_VHOST_CROSS_ENDIAN_LEGACY is not set
 
 #
 # General architecture-dependent options
@@ -598,6 +627,7 @@ CONFIG_JUMP_LABEL=y
 # CONFIG_STATIC_KEYS_SELFTEST is not set
 CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS=y
 CONFIG_ARCH_USE_BUILTIN_BSWAP=y
+CONFIG_USER_RETURN_NOTIFIER=y
 CONFIG_HAVE_IOREMAP_PROT=y
 CONFIG_HAVE_KPROBES=y
 CONFIG_HAVE_KRETPROBES=y
@@ -733,6 +763,7 @@ CONFIG_BLK_PM=y
 # CONFIG_IOSCHED_BFQ is not set
 # end of IO Schedulers
 
+CONFIG_PREEMPT_NOTIFIERS=y
 CONFIG_INLINE_SPIN_UNLOCK_IRQ=y
 CONFIG_INLINE_READ_UNLOCK=y
 CONFIG_INLINE_READ_UNLOCK_IRQ=y
@@ -789,6 +820,7 @@ CONFIG_CONTIG_ALLOC=y
 CONFIG_PHYS_ADDR_T_64BIT=y
 CONFIG_BOUNCE=y
 CONFIG_VIRT_TO_BUS=y
+CONFIG_MMU_NOTIFIER=y
 CONFIG_KSM=y
 CONFIG_DEFAULT_MMAP_MIN_ADDR=4096
 CONFIG_TRANSPARENT_HUGEPAGE=y
@@ -1372,6 +1404,7 @@ CONFIG_SERIAL_ARC_NR_PORTS=1
 
 CONFIG_SERIAL_DEV_BUS=y
 CONFIG_SERIAL_DEV_CTRL_TTYPORT=y
+# CONFIG_TTY_PRINTK is not set
 CONFIG_HVC_DRIVER=y
 CONFIG_VIRTIO_CONSOLE=y
 # CONFIG_IPMI_HANDLER is not set
@@ -1750,6 +1783,17 @@ CONFIG_UIO_DMEM_GENIRQ=y
 # CONFIG_UIO_NETX is not set
 # CONFIG_UIO_PRUSS is not set
 # CONFIG_UIO_MF624 is not set
+CONFIG_VFIO_IOMMU_TYPE1=y
+CONFIG_VFIO_VIRQFD=y
+CONFIG_VFIO=y
+# CONFIG_VFIO_NOIOMMU is not set
+CONFIG_VFIO_PCI=y
+# CONFIG_VFIO_PCI_VGA is not set
+CONFIG_VFIO_PCI_MMAP=y
+CONFIG_VFIO_PCI_INTX=y
+# CONFIG_VFIO_PCI_IGD is not set
+# CONFIG_VFIO_MDEV is not set
+CONFIG_IRQ_BYPASS_MANAGER=y
 # CONFIG_VIRT_DRIVERS is not set
 CONFIG_VIRTIO=y
 CONFIG_VIRTIO_MENU=y
@@ -1794,6 +1838,8 @@ CONFIG_CLKBLD_I8253=y
 CONFIG_MAILBOX=y
 CONFIG_PCC=y
 # CONFIG_ALTERA_MBOX is not set
+CONFIG_IOMMU_IOVA=y
+CONFIG_IOMMU_API=y
 CONFIG_IOMMU_SUPPORT=y
 
 #
@@ -1802,9 +1848,12 @@ CONFIG_IOMMU_SUPPORT=y
 # end of Generic IOMMU Pagetable Support
 
 # CONFIG_IOMMU_DEBUGFS is not set
+# CONFIG_IOMMU_DEFAULT_PASSTHROUGH is not set
+CONFIG_IOMMU_DMA=y
 # CONFIG_AMD_IOMMU is not set
 # CONFIG_INTEL_IOMMU is not set
 # CONFIG_IRQ_REMAP is not set
+CONFIG_VIRTIO_IOMMU=y
 
 #
 # Remoteproc drivers
@@ -2368,6 +2417,7 @@ CONFIG_ZLIB_DEFLATE=y
 CONFIG_LZO_COMPRESS=y
 CONFIG_LZO_DECOMPRESS=y
 # CONFIG_XZ_DEC is not set
+CONFIG_INTERVAL_TREE=y
 CONFIG_XARRAY_MULTI=y
 CONFIG_ASSOCIATIVE_ARRAY=y
 CONFIG_HAS_IOMEM=y
@@ -2608,4 +2658,5 @@ CONFIG_IO_DELAY_0X80=y
 # CONFIG_PUNIT_ATOM_DEBUG is not set
 # CONFIG_UNWINDER_ORC is not set
 CONFIG_UNWINDER_FRAME_POINTER=y
+# CONFIG_UNWINDER_GUESS is not set
 # end of Kernel hacking

scripts/run_cargo_tests.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+set -e
+set -x
+
+source $HOME/.cargo/env
+
+# Install cargo components
+rustup component add clippy
+rustup component add rustfmt
+cargo install --force cargo-audit
+
+# Run cargo builds and checks
+cargo rustc --bin cloud-hypervisor -- -D warnings
+cargo rustc --bin vhost_user_net -- -D warnings
+cargo test
+cargo audit
+cargo rustc --bin cloud-hypervisor --no-default-features --features "pci,acpi"  -- -D warnings
+cargo rustc --bin vhost_user_net --no-default-features --features "pci,acpi"  -- -D warnings
+cargo clippy --all-targets --all-features -- -D warnings
+cargo rustc --bin cloud-hypervisor --no-default-features --features "pci"  -- -D warnings
+cargo rustc --bin vhost_user_net --no-default-features --features "pci"  -- -D warnings
+cargo rustc --bin cloud-hypervisor --no-default-features --features "mmio"  -- -D warnings
+cargo rustc --bin vhost_user_net --no-default-features --features "mmio"  -- -D warnings
+find . \( -name "*.rs" ! -wholename "*/out/*.rs" \) | xargs rustfmt --check
+cargo build --release

scripts/run_integration_tests.sh

@@ -6,7 +6,7 @@ source $HOME/.cargo/env
 WORKLOADS_DIR="$HOME/workloads"
 mkdir -p "$WORKLOADS_DIR"
 
-FW_URL=$(curl --silent https://api.github.com/repos/intel/rust-hypervisor-firmware/releases/latest | grep "browser_download_url" | grep -o 'https://.*[^ "]')
+FW_URL=$(curl --silent https://api.github.com/repos/cloud-hypervisor/rust-hypervisor-firmware/releases/latest | grep "browser_download_url" | grep -o 'https://.*[^ "]')
 FW="$WORKLOADS_DIR/hypervisor-fw"
 if [ ! -f "$FW" ]; then
     pushd $WORKLOADS_DIR
@@ -14,25 +14,16 @@ if [ ! -f "$FW" ]; then
     popd
 fi
 
-OVMF_URL="https://cdn.download.clearlinux.org/image/OVMF.fd"
+CLEAR_OS_IMAGE_NAME="clear-31310-cloudguest.img"
-OVMF="$WORKLOADS_DIR/OVMF.fd"
+CLEAR_OS_IMAGE_URL="https://cloudhypervisorstorage.blob.core.windows.net/images/$CLEAR_OS_IMAGE_NAME"
-if [ ! -f "$OVMF" ]; then
-    pushd $WORKLOADS_DIR
-    wget --quiet $OVMF_URL
-    popd
-fi
-
-CLEAR_OS_IMAGE_NAME="clear-29810-cloud.img"
-CLEAR_OS_IMAGE_URL="https://cloudhypervisorstorage.blob.core.windows.net/images/$CLEAR_OS_IMAGE_NAME.xz"
 CLEAR_OS_IMAGE="$WORKLOADS_DIR/$CLEAR_OS_IMAGE_NAME"
 if [ ! -f "$CLEAR_OS_IMAGE" ]; then
     pushd $WORKLOADS_DIR
     wget --quiet $CLEAR_OS_IMAGE_URL
-    unxz $CLEAR_OS_IMAGE_NAME.xz
     popd
 fi
 
-CLEAR_OS_RAW_IMAGE_NAME="clear-29810-cloud-raw.img"
+CLEAR_OS_RAW_IMAGE_NAME="clear-31310-cloudguest-raw.img"
 CLEAR_OS_RAW_IMAGE="$WORKLOADS_DIR/$CLEAR_OS_RAW_IMAGE_NAME"
 if [ ! -f "$CLEAR_OS_RAW_IMAGE" ]; then
     pushd $WORKLOADS_DIR
@@ -58,6 +49,24 @@ if [ ! -f "$BIONIC_OS_RAW_IMAGE" ]; then
 fi
 
 
+EOAN_OS_IMAGE_NAME="eoan-server-cloudimg-amd64.img"
+EOAN_OS_IMAGE_URL="https://cloudhypervisorstorage.blob.core.windows.net/images/$EOAN_OS_IMAGE_NAME"
+EOAN_OS_IMAGE="$WORKLOADS_DIR/$EOAN_OS_IMAGE_NAME"
+if [ ! -f "$EOAN_OS_IMAGE" ]; then
+    pushd $WORKLOADS_DIR
+    wget --quiet $EOAN_OS_IMAGE_URL
+    popd
+fi
+
+EOAN_OS_RAW_IMAGE_NAME="eoan-server-cloudimg-amd64-raw.img"
+EOAN_OS_RAW_IMAGE="$WORKLOADS_DIR/$EOAN_OS_RAW_IMAGE_NAME"
+if [ ! -f "$EOAN_OS_RAW_IMAGE" ]; then
+    pushd $WORKLOADS_DIR
+    qemu-img convert -p -f qcow2 -O raw $EOAN_OS_IMAGE_NAME $EOAN_OS_RAW_IMAGE_NAME
+    popd
+fi
+
+
 # Build custom kernel based on virtio-pmem and virtio-fs upstream patches
 VMLINUX_IMAGE="$WORKLOADS_DIR/vmlinux"
 BZIMAGE_IMAGE="$WORKLOADS_DIR/bzImage"
@@ -67,9 +76,9 @@ LINUX_CUSTOM_DIR="linux-custom"
 if [ ! -f "$VMLINUX_IMAGE" ]; then
     SRCDIR=$PWD
     pushd $WORKLOADS_DIR
-    git clone --depth 1 "https://github.com/sboeuf/linux.git" -b "virtio-fs" $LINUX_CUSTOM_DIR
+    git clone --depth 1 "https://github.com/cloud-hypervisor/linux.git" -b "virtio-fs-virtio-iommu" $LINUX_CUSTOM_DIR
     pushd $LINUX_CUSTOM_DIR
-    cp $SRCDIR/resources/linux-virtio-fs-config .config
+    cp $SRCDIR/resources/linux-virtio-fs-virtio-iommu-config .config
     make bzImage -j `nproc`
     cp vmlinux $VMLINUX_IMAGE
     cp arch/x86/boot/bzImage $BZIMAGE_IMAGE
@@ -79,22 +88,34 @@ if [ ! -f "$VMLINUX_IMAGE" ]; then
 fi
 
 VIRTIOFSD="$WORKLOADS_DIR/virtiofsd"
-VUBRIDGE="$WORKLOADS_DIR/vubridge"
 QEMU_DIR="qemu_build"
-if [ ! -f "$VIRTIOFSD" ] || [ ! -f "$VUBRIDGE" ]; then
+if [ ! -f "$VIRTIOFSD" ]; then
     pushd $WORKLOADS_DIR
-    git clone --depth 1 "https://github.com/sboeuf/qemu.git" -b "virtio-fs" $QEMU_DIR
+    git clone --depth 1 "https://gitlab.com/virtio-fs/qemu.git" -b "virtio-fs-dev" $QEMU_DIR
     pushd $QEMU_DIR
     ./configure --prefix=$PWD --target-list=x86_64-softmmu
-    make virtiofsd tests/vhost-user-bridge -j `nproc`
+    make virtiofsd -j `nproc`
     cp virtiofsd $VIRTIOFSD
-    cp tests/vhost-user-bridge $VUBRIDGE
     popd
     rm -rf $QEMU_DIR
     sudo setcap cap_dac_override,cap_sys_admin+epi "virtiofsd"
     popd
 fi
 
+BLK_IMAGE="$WORKLOADS_DIR/blk.img"
+MNT_DIR="mount_image"
+if [ ! -f "$BLK_IMAGE" ]; then
+   pushd $WORKLOADS_DIR
+   fallocate -l 16M $BLK_IMAGE
+   mkfs.ext4 -j $BLK_IMAGE
+   mkdir $MNT_DIR
+   sudo mount -t ext4 $BLK_IMAGE $MNT_DIR
+   sudo bash -c "echo bar > $MNT_DIR/foo"
+   sudo umount $BLK_IMAGE
+   rm -r $MNT_DIR
+   popd
+fi
+
 SHARED_DIR="$WORKLOADS_DIR/shared_dir"
 if [ ! -d "$SHARED_DIR" ]; then
     mkdir -p $SHARED_DIR
@@ -111,10 +132,10 @@ if [ ! -d "$VFIO_DIR" ]; then
 fi
 
 # VFIO test network setup.
-# We reserve a different IP class for it: 172.16.0.0/24.
+# We reserve a different IP class for it: 172.17.0.0/24.
 sudo ip link add name vfio-br0 type bridge
 sudo ip link set vfio-br0 up
-sudo ip addr add 172.16.0.1/24 dev vfio-br0
+sudo ip addr add 172.17.0.1/24 dev vfio-br0
 
 sudo ip tuntap add vfio-tap0 mode tap
 sudo ip link set vfio-tap0 master vfio-br0
@@ -124,24 +145,52 @@ sudo ip tuntap add vfio-tap1 mode tap
 sudo ip link set vfio-tap1 master vfio-br0
 sudo ip link set vfio-tap1 up
 
-cargo build
+sudo ip tuntap add vfio-tap2 mode tap
-sudo setcap cap_net_admin+ep target/debug/cloud-hypervisor
+sudo ip link set vfio-tap2 master vfio-br0
+sudo ip link set vfio-tap2 up
+
+cargo build --release
+sudo setcap cap_net_admin+ep target/release/cloud-hypervisor
+sudo setcap cap_net_admin+ep target/release/vhost_user_net
+sudo setcap cap_dac_override,cap_sys_admin+epi target/release/vhost_user_fs
 
 # We always copy a fresh version of our binary for our L2 guest.
-cp target/debug/cloud-hypervisor $VFIO_DIR
+cp target/release/cloud-hypervisor $VFIO_DIR
-# We need qemu to have NET_ADMIN as well.
+
-sudo setcap cap_net_admin+ep /usr/bin/qemu-system-x86_64
+# Enable KSM with some reasonable parameters so that it won't take too long
+# for the memory to be merged between two processes.
+sudo bash -c "echo 10000 > /sys/kernel/mm/ksm/pages_to_scan"
+sudo bash -c "echo 10 > /sys/kernel/mm/ksm/sleep_millisecs"
+sudo bash -c "echo 1 > /sys/kernel/mm/ksm/run"
+
+# Ensure test binary has the same caps as the cloud-hypervisor one
+cargo test --no-run --features "integration_tests" -- --nocapture
+ls target/debug/deps/cloud_hypervisor-* | xargs -n 1 sudo setcap cap_net_admin+ep
 
 sudo adduser $USER kvm
 newgrp kvm << EOF
 export RUST_BACKTRACE=1
-cargo test --features "integration_tests"
+cargo test --features "integration_tests" -- --nocapture
 EOF
 RES=$?
 
+if [ $RES -eq 0 ]; then
+    # virtio-mmio based testing
+    cargo build --release --no-default-features --features "mmio"
+    sudo setcap cap_net_admin+ep target/release/cloud-hypervisor
+
+    newgrp kvm << EOF
+export RUST_BACKTRACE=1
+cargo test --features "integration_tests,mmio" -- --nocapture
+EOF
+
+    RES=$?
+fi
+
 # Tear VFIO test network down
 sudo ip link del vfio-br0
 sudo ip link del vfio-tap0
 sudo ip link del vfio-tap1
+sudo ip link del vfio-tap2
 
 exit $RES

scripts/run_openapi_tests.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+set -e
+set -x
+
+sudo docker run --rm -v ${PWD}:/local openapitools/openapi-generator-cli validate -i /local/vmm/src/api/openapi/cloud-hypervisor.yaml

src/bin/vhost_user_blk.rs

@@ -0,0 +1,300 @@
+// Copyright 2019 Red Hat, Inc. All Rights Reserved.
+//
+// Portions Copyright 2019 Intel Corporation. All Rights Reserved.
+//
+// Portions Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+//
+// Portions Copyright 2017 The Chromium OS Authors. All rights reserved.
+//
+// SPDX-License-Identifier: (Apache-2.0 AND BSD-3-Clause)
+
+#[macro_use(crate_version, crate_authors)]
+extern crate clap;
+extern crate log;
+extern crate vhost_rs;
+extern crate vhost_user_backend;
+extern crate vm_virtio;
+
+use clap::{App, Arg};
+use epoll;
+use log::*;
+use std::fs::File;
+use std::fs::OpenOptions;
+use std::io::Read;
+use std::io::{Seek, SeekFrom, Write};
+use std::mem;
+use std::path::PathBuf;
+use std::process;
+use std::slice;
+use std::sync::{Arc, RwLock};
+use std::vec::Vec;
+
+use qcow::{self, ImageType, QcowFile};
+
+use vhost_rs::vhost_user::message::*;
+use vhost_user_backend::{VhostUserBackend, VhostUserDaemon, Vring, VringWorker};
+
+use virtio_bindings::bindings::virtio_blk::*;
+use vm_memory::{Bytes, GuestMemoryError, GuestMemoryMmap};
+use vm_virtio::block::{build_disk_image_id, Request};
+
+const QUEUE_SIZE: usize = 1024;
+const NUM_QUEUES: usize = 1;
+const SECTOR_SHIFT: u8 = 9;
+const SECTOR_SIZE: u64 = (0x01 as u64) << SECTOR_SHIFT;
+const BLK_SIZE: u32 = 512;
+
+trait DiskFile: Read + Seek + Write + Send + Sync {}
+impl<D: Read + Seek + Write + Send + Sync> DiskFile for D {}
+
+pub type Result<T> = std::result::Result<T, Error>;
+pub type VhostUserBackendResult<T> = std::result::Result<T, std::io::Error>;
+
+#[derive(Debug)]
+pub enum Error {
+    /// Failed to detect image type.
+    DetectImageType,
+    /// Bad memory address.
+    GuestMemory(GuestMemoryError),
+    /// Can't open image file.
+    OpenImage,
+    /// Failed to parse image parameter.
+    ParseImageParam,
+    /// Failed to parse sock parameter.
+    ParseSockParam,
+}
+
+struct VhostUserBlkBackend {
+    mem: Option<GuestMemoryMmap>,
+    vring_worker: Option<Arc<VringWorker>>,
+    disk_image: Box<dyn DiskFile>,
+    disk_image_id: Vec<u8>,
+    disk_nsectors: u64,
+    config: virtio_blk_config,
+}
+
+impl VhostUserBlkBackend {
+    pub fn new(image_path: String) -> Result<Self> {
+        let raw_img: File = OpenOptions::new()
+            .read(true)
+            .write(true)
+            .open(&image_path)
+            .unwrap();
+
+        let image_id = build_disk_image_id(&PathBuf::from(&image_path));
+        let image_type = qcow::detect_image_type(&raw_img).unwrap();
+        let mut image = match image_type {
+            ImageType::Raw => Box::new(vm_virtio::RawFile::new(raw_img)) as Box<dyn DiskFile>,
+            ImageType::Qcow2 => Box::new(QcowFile::from(raw_img).unwrap()) as Box<dyn DiskFile>,
+        };
+
+        let nsectors = (image.seek(SeekFrom::End(0)).unwrap() as u64) / SECTOR_SIZE;
+        let mut config = virtio_blk_config::default();
+
+        config.capacity = nsectors;
+        config.blk_size = BLK_SIZE;
+        config.size_max = 65535;
+        config.seg_max = 128 - 2;
+        config.min_io_size = 1;
+        config.opt_io_size = 1;
+        config.num_queues = 1;
+
+        Ok(VhostUserBlkBackend {
+            mem: None,
+            vring_worker: None,
+            disk_image: image,
+            disk_image_id: image_id,
+            disk_nsectors: nsectors,
+            config,
+        })
+    }
+
+    pub fn process_queue(&mut self, vring: &mut Vring) -> bool {
+        let mut used_any = false;
+        let mem = match self.mem.as_ref() {
+            Some(m) => m,
+            None => return false,
+        };
+
+        while let Some(head) = vring.mut_queue().iter(mem).next() {
+            debug!("got an element in the queue");
+            let len;
+            match Request::parse(&head, mem) {
+                Ok(request) => {
+                    debug!("element is a valid request");
+                    let status = match request.execute(
+                        &mut self.disk_image,
+                        self.disk_nsectors,
+                        mem,
+                        &self.disk_image_id,
+                    ) {
+                        Ok(l) => {
+                            len = l;
+                            VIRTIO_BLK_S_OK
+                        }
+                        Err(e) => {
+                            len = 1;
+                            e.status()
+                        }
+                    };
+                    mem.write_obj(status, request.status_addr).unwrap();
+                }
+                Err(err) => {
+                    error!("failed to parse available descriptor chain: {:?}", err);
+                    len = 0;
+                }
+            }
+            vring.mut_queue().add_used(mem, head.index, len);
+            used_any = true;
+        }
+
+        used_any
+    }
+}
+
+impl VhostUserBackend for VhostUserBlkBackend {
+    fn num_queues(&self) -> usize {
+        NUM_QUEUES
+    }
+
+    fn max_queue_size(&self) -> usize {
+        QUEUE_SIZE
+    }
+
+    fn features(&self) -> u64 {
+        1 << VIRTIO_BLK_F_MQ
+            | 1 << VIRTIO_F_VERSION_1
+            | VhostUserVirtioFeatures::PROTOCOL_FEATURES.bits()
+    }
+
+    fn protocol_features(&self) -> VhostUserProtocolFeatures {
+        VhostUserProtocolFeatures::CONFIG
+    }
+
+    fn update_memory(&mut self, mem: GuestMemoryMmap) -> VhostUserBackendResult<()> {
+        self.mem = Some(mem);
+        Ok(())
+    }
+
+    fn handle_event(
+        &mut self,
+        device_event: u16,
+        evset: epoll::Events,
+        vrings: &[Arc<RwLock<Vring>>],
+    ) -> VhostUserBackendResult<bool> {
+        if evset != epoll::Events::EPOLLIN {
+            warn!("invalid events operation");
+            return Ok(false);
+        }
+
+        debug!("event received: {:?}", device_event);
+
+        let mut vring = vrings[0].write().unwrap();
+        if self.process_queue(&mut vring) {
+            debug!("signalling queue");
+            vring.signal_used_queue().unwrap();
+        }
+
+        Ok(false)
+    }
+
+    fn get_config(&self, _offset: u32, _size: u32) -> Vec<u8> {
+        // self.config is a statically allocated virtio_blk_config
+        let buf = unsafe {
+            slice::from_raw_parts(
+                &self.config as *const virtio_blk_config as *const _,
+                mem::size_of::<virtio_blk_config>(),
+            )
+        };
+
+        buf.to_vec()
+    }
+}
+
+pub struct VhostUserBlkBackendConfig<'a> {
+    pub image: &'a str,
+    pub sock: &'a str,
+}
+
+impl<'a> VhostUserBlkBackendConfig<'a> {
+    pub fn parse(backend: &'a str) -> Result<Self> {
+        let params_list: Vec<&str> = backend.split(',').collect();
+
+        let mut image: &str = "";
+        let mut sock: &str = "";
+
+        for param in params_list.iter() {
+            if param.starts_with("image=") {
+                image = &param[6..];
+            } else if param.starts_with("sock=") {
+                sock = &param[5..];
+            }
+        }
+
+        if image.is_empty() {
+            return Err(Error::ParseImageParam);
+        }
+        if sock.is_empty() {
+            return Err(Error::ParseSockParam);
+        }
+
+        Ok(VhostUserBlkBackendConfig { image, sock })
+    }
+}
+
+fn main() {
+    let cmd_arguments = App::new("vhost-user-blk backend")
+        .version(crate_version!())
+        .author(crate_authors!())
+        .about("Launch a vhost-user-blk backend.")
+        .arg(
+            Arg::with_name("backend")
+                .long("backend")
+                .help(
+                    "Backend parameters \"image=<image_path>,\
+                     sock=<socket_path>\"",
+                )
+                .takes_value(true)
+                .min_values(1),
+        )
+        .get_matches();
+
+    let vhost_user_blk_backend = cmd_arguments.value_of("backend").unwrap();
+
+    let backend_config = match VhostUserBlkBackendConfig::parse(vhost_user_blk_backend) {
+        Ok(config) => config,
+        Err(e) => {
+            println!("Failed parsing parameters {:?}", e);
+            process::exit(1);
+        }
+    };
+
+    let blk_backend = Arc::new(RwLock::new(
+        VhostUserBlkBackend::new(backend_config.image.to_string()).unwrap(),
+    ));
+
+    debug!("blk_backend is created!\n");
+
+    let name = "vhost-user-blk-backend";
+    let mut blk_daemon = VhostUserDaemon::new(
+        name.to_string(),
+        backend_config.sock.to_string(),
+        blk_backend.clone(),
+    )
+    .unwrap();
+    debug!("blk_daemon is created!\n");
+
+    let vring_worker = blk_daemon.get_vring_worker();
+
+    blk_backend.write().unwrap().vring_worker = Some(vring_worker);
+
+    if let Err(e) = blk_daemon.start() {
+        println!(
+            "failed to start daemon for vhost-user-blk with error: {:?}\n",
+            e
+        );
+        process::exit(1);
+    }
+
+    blk_daemon.wait().unwrap();
+}

src/bin/vhost_user_fs.rs

@@ -0,0 +1,246 @@
+// Copyright 2019 Intel Corporation. All Rights Reserved.
+//
+// SPDX-License-Identifier: (Apache-2.0 AND BSD-3-Clause)
+
+#[macro_use(crate_version, crate_authors)]
+extern crate clap;
+extern crate log;
+extern crate vhost_rs;
+extern crate vhost_user_backend;
+extern crate vm_virtio;
+
+use clap::{App, Arg};
+use epoll;
+use libc::EFD_NONBLOCK;
+use log::*;
+use std::os::unix::io::AsRawFd;
+use std::sync::{Arc, RwLock};
+use std::{convert, error, fmt, io, process};
+
+use vhost_rs::vhost_user::message::*;
+use vhost_user_backend::{VhostUserBackend, VhostUserDaemon, Vring};
+use vhost_user_fs::descriptor_utils::{Reader, Writer};
+use vhost_user_fs::filesystem::FileSystem;
+use vhost_user_fs::passthrough::{self, PassthroughFs};
+use vhost_user_fs::server::Server;
+use vhost_user_fs::Error as VhostUserFsError;
+use virtio_bindings::bindings::virtio_net::*;
+use vm_memory::GuestMemoryMmap;
+use vmm_sys_util::eventfd::EventFd;
+
+const QUEUE_SIZE: usize = 1024;
+const NUM_QUEUES: usize = 2;
+
+// The guest queued an available buffer for the high priority queue.
+const HIPRIO_QUEUE_EVENT: u16 = 0;
+// The guest queued an available buffer for the request queue.
+const REQ_QUEUE_EVENT: u16 = 1;
+// The device has been dropped.
+const KILL_EVENT: u16 = 2;
+
+type Result<T> = std::result::Result<T, Error>;
+type VhostUserBackendResult<T> = std::result::Result<T, std::io::Error>;
+
+#[derive(Debug)]
+enum Error {
+    /// Failed to create kill eventfd.
+    CreateKillEventFd,
+    /// Failed to handle event other than input event.
+    HandleEventNotEpollIn,
+    /// Failed to handle unknown event.
+    HandleEventUnknownEvent,
+    /// No memory configured.
+    NoMemoryConfigured,
+    /// Processing queue failed.
+    ProcessQueue(VhostUserFsError),
+}
+
+impl fmt::Display for Error {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "vhost_user_fs_error: {:?}", self)
+    }
+}
+
+impl error::Error for Error {}
+
+impl convert::From<Error> for io::Error {
+    fn from(e: Error) -> Self {
+        io::Error::new(io::ErrorKind::Other, e)
+    }
+}
+
+struct VhostUserFsBackend<F: FileSystem + Send + Sync + 'static> {
+    mem: Option<GuestMemoryMmap>,
+    kill_evt: EventFd,
+    server: Arc<Server<F>>,
+}
+
+impl<F: FileSystem + Send + Sync + 'static> Clone for VhostUserFsBackend<F> {
+    fn clone(&self) -> Self {
+        VhostUserFsBackend {
+            mem: self.mem.clone(),
+            kill_evt: self.kill_evt.try_clone().unwrap(),
+            server: self.server.clone(),
+        }
+    }
+}
+
+impl<F: FileSystem + Send + Sync + 'static> VhostUserFsBackend<F> {
+    fn new(fs: F) -> Result<Self> {
+        Ok(VhostUserFsBackend {
+            mem: None,
+            kill_evt: EventFd::new(EFD_NONBLOCK).map_err(|_| Error::CreateKillEventFd)?,
+            server: Arc::new(Server::new(fs)),
+        })
+    }
+
+    fn process_queue(&mut self, vring: &mut Vring) -> Result<()> {
+        let mem = self.mem.as_ref().ok_or(Error::NoMemoryConfigured)?;
+
+        let mut used_desc_heads = [(0, 0); QUEUE_SIZE];
+        let mut used_count = 0;
+        while let Some(avail_desc) = vring.mut_queue().iter(&mem).next() {
+            let head_index = avail_desc.index;
+            let reader = Reader::new(mem, avail_desc.clone()).unwrap();
+            let writer = Writer::new(mem, avail_desc.clone()).unwrap();
+
+            let total = self
+                .server
+                .handle_message(reader, writer)
+                .map_err(Error::ProcessQueue)?;
+
+            used_desc_heads[used_count] = (head_index, total);
+            used_count += 1;
+        }
+
+        if used_count > 0 {
+            for &(desc_index, _) in &used_desc_heads[..used_count] {
+                vring.mut_queue().add_used(&mem, desc_index, 0);
+            }
+            vring.signal_used_queue().unwrap();
+        }
+
+        Ok(())
+    }
+}
+
+impl<F: FileSystem + Send + Sync + 'static> VhostUserBackend for VhostUserFsBackend<F> {
+    fn num_queues(&self) -> usize {
+        NUM_QUEUES
+    }
+
+    fn max_queue_size(&self) -> usize {
+        QUEUE_SIZE
+    }
+
+    fn features(&self) -> u64 {
+        1 << VIRTIO_F_VERSION_1 | VhostUserVirtioFeatures::PROTOCOL_FEATURES.bits()
+    }
+
+    fn protocol_features(&self) -> VhostUserProtocolFeatures {
+        VhostUserProtocolFeatures::all()
+    }
+
+    fn update_memory(&mut self, mem: GuestMemoryMmap) -> VhostUserBackendResult<()> {
+        self.mem = Some(mem);
+        Ok(())
+    }
+
+    fn handle_event(
+        &mut self,
+        device_event: u16,
+        evset: epoll::Events,
+        vrings: &[Arc<RwLock<Vring>>],
+    ) -> VhostUserBackendResult<bool> {
+        if evset != epoll::Events::EPOLLIN {
+            return Err(Error::HandleEventNotEpollIn.into());
+        }
+
+        match device_event {
+            HIPRIO_QUEUE_EVENT => {
+                debug!("HIPRIO_QUEUE_EVENT");
+            }
+            REQ_QUEUE_EVENT => {
+                debug!("REQ_QUEUE_EVENT");
+                let mut vring = vrings[1].write().unwrap();
+                self.process_queue(&mut vring)?;
+            }
+            KILL_EVENT => {
+                debug!("KILL_EVENT");
+                self.kill_evt.read().unwrap();
+                return Ok(true);
+            }
+            _ => return Err(Error::HandleEventUnknownEvent.into()),
+        }
+
+        Ok(false)
+    }
+}
+
+fn main() {
+    let cmd_arguments = App::new("vhost-user-fs backend")
+        .version(crate_version!())
+        .author(crate_authors!())
+        .about("Launch a vhost-user-fs backend.")
+        .arg(
+            Arg::with_name("shared-dir")
+                .long("shared-dir")
+                .help("Shared directory path")
+                .takes_value(true)
+                .min_values(1),
+        )
+        .arg(
+            Arg::with_name("sock")
+                .long("sock")
+                .help("vhost-user socket path")
+                .takes_value(true)
+                .min_values(1),
+        )
+        .get_matches();
+
+    // Retrieve arguments
+    let shared_dir = cmd_arguments
+        .value_of("shared-dir")
+        .expect("Failed to retrieve shared directory path");
+    let sock = cmd_arguments
+        .value_of("sock")
+        .expect("Failed to retrieve vhost-user socket path");
+
+    // Convert into appropriate types
+    let sock = String::from(sock);
+
+    let fs_cfg = passthrough::Config {
+        root_dir: shared_dir.to_string(),
+        ..Default::default()
+    };
+    let fs = PassthroughFs::new(fs_cfg).unwrap();
+    let fs_backend = Arc::new(RwLock::new(VhostUserFsBackend::new(fs).unwrap()));
+
+    let mut daemon = VhostUserDaemon::new(
+        String::from("vhost-user-fs-backend"),
+        sock,
+        fs_backend.clone(),
+    )
+    .unwrap();
+
+    let vring_worker = daemon.get_vring_worker();
+
+    if let Err(e) = vring_worker.register_listener(
+        fs_backend.read().unwrap().kill_evt.as_raw_fd(),
+        epoll::Events::EPOLLIN,
+        u64::from(KILL_EVENT),
+    ) {
+        println!("Failed to register listener for kill event: {:?}", e);
+        process::exit(1);
+    }
+
+    if let Err(e) = daemon.start() {
+        println!("Failed to start daemon: {:?}", e);
+        process::exit(1);
+    }
+
+    if let Err(e) = daemon.wait() {
+        println!("Waiting for daemon failed: {:?}", e);
+        process::exit(1);
+    }
+}

src/bin/vhost_user_net.rs

@@ -0,0 +1,601 @@
+// Copyright 2019 Intel Corporation. All Rights Reserved.
+//
+// Portions Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+//
+// Portions Copyright 2017 The Chromium OS Authors. All rights reserved.
+//
+// SPDX-License-Identifier: (Apache-2.0 AND BSD-3-Clause)
+
+#[macro_use(crate_version, crate_authors)]
+extern crate clap;
+extern crate log;
+extern crate net_util;
+extern crate vhost_rs;
+extern crate vhost_user_backend;
+extern crate vm_virtio;
+
+use clap::{App, Arg};
+use epoll;
+use libc::{self, EAGAIN, EFD_NONBLOCK};
+use log::*;
+use std::cmp;
+use std::fmt;
+use std::io::Read;
+use std::io::{self, Write};
+use std::mem;
+use std::net::Ipv4Addr;
+use std::os::unix::io::AsRawFd;
+use std::process;
+use std::sync::{Arc, RwLock};
+use std::vec::Vec;
+
+use vhost_rs::vhost_user::message::*;
+use vhost_rs::vhost_user::Error as VhostUserError;
+use vhost_user_backend::{VhostUserBackend, VhostUserDaemon, Vring, VringWorker};
+
+use net_gen;
+
+use net_util::{Tap, TapError};
+use virtio_bindings::bindings::virtio_net::*;
+use vm_memory::{Bytes, GuestAddress, GuestMemoryMmap};
+use vmm_sys_util::eventfd::EventFd;
+
+/// The maximum buffer size when segmentation offload is enabled. This
+/// includes the 12-byte virtio net header.
+/// http://docs.oasis-open.org/virtio/virtio/v1.0/virtio-v1.0.html#x1-1740003
+const MAX_BUFFER_SIZE: usize = 65562;
+const QUEUE_SIZE: usize = 1024;
+const NUM_QUEUES: usize = 2;
+
+// The guest has made a buffer available to receive a frame into.
+const RX_QUEUE_EVENT: u16 = 0;
+// The transmit queue has a frame that is ready to send from the guest.
+const TX_QUEUE_EVENT: u16 = 1;
+// A frame is available for reading from the tap device to receive in the guest.
+const RX_TAP_EVENT: u16 = 2;
+// The device has been dropped.
+const KILL_EVENT: u16 = 3;
+
+pub type VhostUserResult<T> = std::result::Result<T, VhostUserError>;
+pub type Result<T> = std::result::Result<T, Error>;
+pub type VhostUserBackendResult<T> = std::result::Result<T, std::io::Error>;
+
+#[derive(Debug)]
+pub enum Error {
+    /// Failed to activate device.
+    BadActivate,
+    /// Failed to create kill eventfd
+    CreateKillEventFd,
+    /// Failed to add event.
+    EpollCtl(io::Error),
+    /// Fail to wait event.
+    EpollWait(io::Error),
+    /// Failed to create EventFd.
+    EpollCreateFd,
+    /// Failed to read Tap.
+    FailedReadTap,
+    /// Failed to signal used queue.
+    FailedSignalingUsedQueue,
+    /// Failed to handle event other than input event.
+    HandleEventNotEpollIn,
+    /// Failed to handle unknown event.
+    HandleEventUnknownEvent,
+    /// Invalid vring address.
+    InvalidVringAddr,
+    /// No vring call fd to notify.
+    NoVringCallFdNotify,
+    /// No memory configured.
+    NoMemoryConfigured,
+    /// Failed to parse sock parameter.
+    ParseSockParam,
+    /// Failed to parse ip parameter.
+    ParseIpParam,
+    /// Failed to parse mask parameter.
+    ParseMaskParam,
+    /// Open tap device failed.
+    TapOpen(TapError),
+    /// Setting tap IP failed.
+    TapSetIp(TapError),
+    /// Setting tap netmask failed.
+    TapSetNetmask(TapError),
+    /// Setting tap interface offload flags failed.
+    TapSetOffload(TapError),
+    /// Setting vnet header size failed.
+    TapSetVnetHdrSize(TapError),
+    /// Enabling tap interface failed.
+    TapEnable(TapError),
+}
+
+impl fmt::Display for Error {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "vhost_user_net_error: {:?}", self)
+    }
+}
+
+impl std::error::Error for Error {}
+
+impl std::convert::From<Error> for std::io::Error {
+    fn from(e: Error) -> Self {
+        std::io::Error::new(io::ErrorKind::Other, e)
+    }
+}
+
+#[derive(Clone)]
+struct TxVirtio {
+    iovec: Vec<(GuestAddress, usize)>,
+    frame_buf: [u8; MAX_BUFFER_SIZE],
+}
+
+impl TxVirtio {
+    fn new() -> Self {
+        TxVirtio {
+            iovec: Vec::new(),
+            frame_buf: [0u8; MAX_BUFFER_SIZE],
+        }
+    }
+}
+
+#[derive(Clone)]
+struct RxVirtio {
+    deferred_frame: bool,
+    deferred_irqs: bool,
+    bytes_read: usize,
+    frame_buf: [u8; MAX_BUFFER_SIZE],
+}
+
+impl RxVirtio {
+    fn new() -> Self {
+        RxVirtio {
+            deferred_frame: false,
+            deferred_irqs: false,
+            bytes_read: 0,
+            frame_buf: [0u8; MAX_BUFFER_SIZE],
+        }
+    }
+}
+
+fn vnet_hdr_len() -> usize {
+    mem::size_of::<virtio_net_hdr_v1>()
+}
+
+struct VhostUserNetBackend {
+    mem: Option<GuestMemoryMmap>,
+    vring_worker: Option<Arc<VringWorker>>,
+    kill_evt: EventFd,
+    tap: Tap,
+    rx: RxVirtio,
+    tx: TxVirtio,
+    rx_tap_listening: bool,
+}
+
+impl std::clone::Clone for VhostUserNetBackend {
+    fn clone(&self) -> Self {
+        VhostUserNetBackend {
+            mem: self.mem.clone(),
+            vring_worker: self.vring_worker.clone(),
+            kill_evt: self.kill_evt.try_clone().unwrap(),
+            tap: self.tap.clone(),
+            rx: self.rx.clone(),
+            tx: self.tx.clone(),
+            rx_tap_listening: self.rx_tap_listening,
+        }
+    }
+}
+
+impl VhostUserNetBackend {
+    /// Create a new virtio network device with the given TAP interface.
+    pub fn new_with_tap(tap: Tap) -> Result<Self> {
+        // Set offload flags to match the virtio features below.
+        tap.set_offload(
+            net_gen::TUN_F_CSUM | net_gen::TUN_F_UFO | net_gen::TUN_F_TSO4 | net_gen::TUN_F_TSO6,
+        )
+        .map_err(Error::TapSetOffload)?;
+
+        let vnet_hdr_size = vnet_hdr_len() as i32;
+        tap.set_vnet_hdr_size(vnet_hdr_size)
+            .map_err(Error::TapSetVnetHdrSize)?;
+
+        let rx = RxVirtio::new();
+        let tx = TxVirtio::new();
+
+        Ok(VhostUserNetBackend {
+            mem: None,
+            vring_worker: None,
+            kill_evt: EventFd::new(EFD_NONBLOCK).map_err(|_| Error::CreateKillEventFd)?,
+            tap,
+            rx,
+            tx,
+            rx_tap_listening: false,
+        })
+    }
+
+    /// Create a new virtio network device with the given IP address and
+    /// netmask.
+    pub fn new(ip_addr: Ipv4Addr, netmask: Ipv4Addr) -> Result<Self> {
+        let tap = Tap::new().map_err(Error::TapOpen)?;
+        tap.set_ip_addr(ip_addr).map_err(Error::TapSetIp)?;
+        tap.set_netmask(netmask).map_err(Error::TapSetNetmask)?;
+        tap.enable().map_err(Error::TapEnable)?;
+
+        Self::new_with_tap(tap)
+    }
+
+    // Copies a single frame from `self.rx.frame_buf` into the guest. Returns true
+    // if a buffer was used, and false if the frame must be deferred until a buffer
+    // is made available by the driver.
+    fn rx_single_frame(&mut self, vring: &mut Vring) -> Result<bool> {
+        let mem = self.mem.as_ref().ok_or(Error::NoMemoryConfigured)?;
+
+        let mut next_desc = vring.mut_queue().iter(&mem).next();
+
+        if next_desc.is_none() {
+            // Queue has no available descriptors
+            if self.rx_tap_listening {
+                self.vring_worker
+                    .as_ref()
+                    .unwrap()
+                    .unregister_listener(
+                        self.tap.as_raw_fd(),
+                        epoll::Events::EPOLLIN,
+                        u64::from(RX_TAP_EVENT),
+                    )
+                    .unwrap();
+                self.rx_tap_listening = false;
+            }
+            return Ok(false);
+        }
+
+        // We just checked that the head descriptor exists.
+        let head_index = next_desc.as_ref().unwrap().index;
+        let mut write_count = 0;
+
+        // Copy from frame into buffer, which may span multiple descriptors.
+        loop {
+            match next_desc {
+                Some(desc) => {
+                    if !desc.is_write_only() {
+                        break;
+                    }
+                    let limit = cmp::min(write_count + desc.len as usize, self.rx.bytes_read);
+                    let source_slice = &self.rx.frame_buf[write_count..limit];
+                    let write_result = mem.write_slice(source_slice, desc.addr);
+
+                    match write_result {
+                        Ok(_) => {
+                            write_count = limit;
+                        }
+                        Err(e) => {
+                            error!("Failed to write slice: {:?}", e);
+                            break;
+                        }
+                    };
+
+                    if write_count >= self.rx.bytes_read {
+                        break;
+                    }
+                    next_desc = desc.next_descriptor();
+                }
+                None => {
+                    warn!("Receiving buffer is too small to hold frame of current size");
+                    break;
+                }
+            }
+        }
+
+        vring
+            .mut_queue()
+            .add_used(&mem, head_index, write_count as u32);
+
+        // Mark that we have at least one pending packet and we need to interrupt the guest.
+        self.rx.deferred_irqs = true;
+
+        Ok(write_count >= self.rx.bytes_read)
+    }
+
+    fn process_rx(&mut self, vring: &mut Vring) -> Result<()> {
+        // Read as many frames as possible.
+        loop {
+            match self.read_tap() {
+                Ok(count) => {
+                    self.rx.bytes_read = count;
+                    if !self.rx_single_frame(vring)? {
+                        self.rx.deferred_frame = true;
+                        break;
+                    }
+                }
+                Err(e) => {
+                    // The tap device is non-blocking, so any error aside from EAGAIN is
+                    // unexpected.
+                    match e.raw_os_error() {
+                        Some(err) if err == EAGAIN => (),
+                        _ => {
+                            error!("Failed to read tap: {:?}", e);
+                            return Err(Error::FailedReadTap);
+                        }
+                    };
+                    break;
+                }
+            }
+        }
+        if self.rx.deferred_irqs {
+            self.rx.deferred_irqs = false;
+            vring.signal_used_queue().unwrap();
+            Ok(())
+        } else {
+            Ok(())
+        }
+    }
+
+    fn resume_rx(&mut self, vring: &mut Vring) -> Result<()> {
+        if self.rx.deferred_frame {
+            if self.rx_single_frame(vring)? {
+                self.rx.deferred_frame = false;
+                // process_rx() was interrupted possibly before consuming all
+                // packets in the tap; try continuing now.
+                self.process_rx(vring)
+            } else if self.rx.deferred_irqs {
+                self.rx.deferred_irqs = false;
+                vring.signal_used_queue().unwrap();
+                Ok(())
+            } else {
+                Ok(())
+            }
+        } else {
+            Ok(())
+        }
+    }
+
+    fn process_tx(&mut self, vring: &mut Vring) -> Result<()> {
+        let mem = self.mem.as_ref().ok_or(Error::NoMemoryConfigured)?;
+
+        let mut used_desc_heads = [(0, 0); QUEUE_SIZE];
+        let mut used_count = 0;
+        while let Some(avail_desc) = vring.mut_queue().iter(&mem).next() {
+            let head_index = avail_desc.index;
+            let mut read_count = 0;
+            let mut next_desc = Some(avail_desc);
+
+            self.tx.iovec.clear();
+            while let Some(desc) = next_desc {
+                if desc.is_write_only() {
+                    break;
+                }
+                self.tx.iovec.push((desc.addr, desc.len as usize));
+                read_count += desc.len as usize;
+                next_desc = desc.next_descriptor();
+            }
+            used_desc_heads[used_count] = (head_index, read_count);
+            used_count += 1;
+            read_count = 0;
+            // Copy buffer from across multiple descriptors.
+            // TODO(performance - Issue #420): change this to use `writev()` instead of `write()`
+            // and get rid of the intermediate buffer.
+            for (desc_addr, desc_len) in self.tx.iovec.drain(..) {
+                let limit = cmp::min((read_count + desc_len) as usize, self.tx.frame_buf.len());
+
+                let read_result = mem.read_slice(
+                    &mut self.tx.frame_buf[read_count..limit as usize],
+                    desc_addr,
+                );
+                match read_result {
+                    Ok(_) => {
+                        // Increment by number of bytes actually read
+                        read_count += limit - read_count;
+                    }
+                    Err(e) => {
+                        error!("Failed to read slice: {:?}", e);
+                        break;
+                    }
+                }
+            }
+
+            let write_result = self.tap.write(&self.tx.frame_buf[..read_count as usize]);
+            match write_result {
+                Ok(_) => {}
+                Err(e) => {
+                    error!("net: tx: error failed to write to tap: {}", e);
+                }
+            };
+        }
+
+        if used_count > 0 {
+            for &(desc_index, _) in &used_desc_heads[..used_count] {
+                vring.mut_queue().add_used(&mem, desc_index, 0);
+            }
+            vring.signal_used_queue().unwrap();
+        }
+
+        Ok(())
+    }
+
+    fn read_tap(&mut self) -> io::Result<usize> {
+        self.tap.read(&mut self.rx.frame_buf)
+    }
+}
+
+impl VhostUserBackend for VhostUserNetBackend {
+    fn num_queues(&self) -> usize {
+        NUM_QUEUES
+    }
+
+    fn max_queue_size(&self) -> usize {
+        QUEUE_SIZE
+    }
+
+    fn features(&self) -> u64 {
+        1 << VIRTIO_NET_F_GUEST_CSUM
+            | 1 << VIRTIO_NET_F_CSUM
+            | 1 << VIRTIO_NET_F_GUEST_TSO4
+            | 1 << VIRTIO_NET_F_GUEST_UFO
+            | 1 << VIRTIO_NET_F_HOST_TSO4
+            | 1 << VIRTIO_NET_F_HOST_UFO
+            | 1 << VIRTIO_F_VERSION_1
+            | VhostUserVirtioFeatures::PROTOCOL_FEATURES.bits()
+    }
+
+    fn protocol_features(&self) -> VhostUserProtocolFeatures {
+        VhostUserProtocolFeatures::all()
+    }
+
+    fn update_memory(&mut self, mem: GuestMemoryMmap) -> VhostUserBackendResult<()> {
+        self.mem = Some(mem);
+        Ok(())
+    }
+
+    fn handle_event(
+        &mut self,
+        device_event: u16,
+        evset: epoll::Events,
+        vrings: &[Arc<RwLock<Vring>>],
+    ) -> VhostUserBackendResult<bool> {
+        if evset != epoll::Events::EPOLLIN {
+            return Err(Error::HandleEventNotEpollIn.into());
+        }
+
+        match device_event {
+            RX_QUEUE_EVENT => {
+                let mut vring = vrings[0].write().unwrap();
+                self.resume_rx(&mut vring)?;
+
+                if !self.rx_tap_listening {
+                    self.vring_worker.as_ref().unwrap().register_listener(
+                        self.tap.as_raw_fd(),
+                        epoll::Events::EPOLLIN,
+                        u64::from(RX_TAP_EVENT),
+                    )?;
+                    self.rx_tap_listening = true;
+                }
+            }
+            TX_QUEUE_EVENT => {
+                let mut vring = vrings[1].write().unwrap();
+                self.process_tx(&mut vring)?;
+            }
+            RX_TAP_EVENT => {
+                let mut vring = vrings[0].write().unwrap();
+                if self.rx.deferred_frame
+                // Process a deferred frame first if available. Don't read from tap again
+                // until we manage to receive this deferred frame.
+                {
+                    if self.rx_single_frame(&mut vring)? {
+                        self.rx.deferred_frame = false;
+                        self.process_rx(&mut vring)?;
+                    } else if self.rx.deferred_irqs {
+                        self.rx.deferred_irqs = false;
+                        vring.signal_used_queue()?;
+                    }
+                } else {
+                    self.process_rx(&mut vring)?;
+                }
+            }
+            KILL_EVENT => {
+                self.kill_evt.read().unwrap();
+                return Ok(true);
+            }
+            _ => return Err(Error::HandleEventUnknownEvent.into()),
+        }
+
+        Ok(false)
+    }
+}
+
+pub struct VhostUserNetBackendConfig<'a> {
+    pub ip: Ipv4Addr,
+    pub mask: Ipv4Addr,
+    pub sock: &'a str,
+}
+
+impl<'a> VhostUserNetBackendConfig<'a> {
+    pub fn parse(backend: &'a str) -> Result<Self> {
+        let params_list: Vec<&str> = backend.split(',').collect();
+
+        let mut ip_str: &str = "";
+        let mut mask_str: &str = "";
+        let mut sock: &str = "";
+
+        for param in params_list.iter() {
+            if param.starts_with("ip=") {
+                ip_str = &param[3..];
+            } else if param.starts_with("mask=") {
+                mask_str = &param[5..];
+            } else if param.starts_with("sock=") {
+                sock = &param[5..];
+            }
+        }
+
+        let mut ip: Ipv4Addr = Ipv4Addr::new(192, 168, 100, 1);
+        let mut mask: Ipv4Addr = Ipv4Addr::new(255, 255, 255, 0);
+
+        if sock.is_empty() {
+            return Err(Error::ParseSockParam);
+        }
+        if !ip_str.is_empty() {
+            ip = ip_str.parse().map_err(|_| Error::ParseIpParam)?;
+        }
+        if !mask_str.is_empty() {
+            mask = mask_str.parse().map_err(|_| Error::ParseMaskParam)?;
+        }
+
+        Ok(VhostUserNetBackendConfig { ip, mask, sock })
+    }
+}
+
+fn main() {
+    let cmd_arguments = App::new("vhost-user-net backend")
+        .version(crate_version!())
+        .author(crate_authors!())
+        .about("Launch a vhost-user-net backend.")
+        .arg(
+            Arg::with_name("backend")
+                .long("backend")
+                .help(
+                    "Backend parameters \"ip=<ip_addr>,\
+                     mask=<net_mask>,sock=<socket_path>\"",
+                )
+                .takes_value(true)
+                .min_values(1),
+        )
+        .get_matches();
+
+    let vhost_user_net_backend = cmd_arguments.value_of("backend").unwrap();
+
+    let backend_config = match VhostUserNetBackendConfig::parse(vhost_user_net_backend) {
+        Ok(config) => config,
+        Err(e) => {
+            println!("Failed parsing parameters {:?}", e);
+            process::exit(1);
+        }
+    };
+
+    let net_backend = Arc::new(RwLock::new(
+        VhostUserNetBackend::new(backend_config.ip, backend_config.mask).unwrap(),
+    ));
+    let name = "vhost-user-net-backend";
+    let mut net_daemon = VhostUserDaemon::new(
+        name.to_string(),
+        backend_config.sock.to_string(),
+        net_backend.clone(),
+    )
+    .unwrap();
+    let vring_worker = net_daemon.get_vring_worker();
+
+    if let Err(e) = vring_worker.register_listener(
+        net_backend.read().unwrap().kill_evt.as_raw_fd(),
+        epoll::Events::EPOLLIN,
+        u64::from(KILL_EVENT),
+    ) {
+        println!("failed to register listener for kill event: {:?}", e);
+        process::exit(1);
+    }
+
+    net_backend.write().unwrap().vring_worker = Some(vring_worker);
+
+    if let Err(e) = net_daemon.start() {
+        println!(
+            "failed to start daemon for vhost-user-net with error: {:?}",
+            e
+        );
+        process::exit(1);
+    }
+
+    net_daemon.wait().unwrap();
+}

test_data/cloud-init/clear/openstack/latest/user_data

@@ -17,7 +17,7 @@ write_files:
         Gateway=192.168.2.1
 
   -
-    path: /etc/systemd/network/00-static-l2.network
+    path: /etc/systemd/network/00-static-l2-1.network
     permissions: 0644
     content: |
         [Match]
@@ -27,6 +27,17 @@ write_files:
         Address=192.168.2.3/24
         Gateway=192.168.2.1
 
+  -
+    path: /etc/systemd/network/00-static-l2-2.network
+    permissions: 0644
+    content: |
+        [Match]
+        MACAddress=de:ad:be:ef:34:56
+
+        [Network]
+        Address=192.168.2.4/24
+        Gateway=192.168.2.1
+
   -
     path: /etc/systemd/system/vfio.service
     permissions: 0644
@@ -47,10 +58,10 @@ write_files:
     content: |
         #!/bin/bash
 
-        mount -t 9p -o trans=virtio cloud_hypervisor /mnt -oversion=9p2000.L,posixacl,cache=loose
+        mount -t virtio_fs virtiofs /mnt -o rootmode=040000,user_id=0,group_id=0,dax
-        modprobe vfio_iommu_type1 allow_unsafe_interrupts
+        bash -c "echo 0000:00:05.0 > /sys/bus/pci/devices/0000\:00\:05.0/driver/unbind"
-        modprobe vfio_pci
+        bash -c "echo 1af4 1041 > /sys/bus/pci/drivers/vfio-pci/new_id"
-        bash -c "echo 0000:00:03.0 > /sys/bus/pci/devices/0000\:00\:03.0/driver/unbind"
+        bash -c "echo 0000:00:06.0 > /sys/bus/pci/devices/0000\:00\:06.0/driver/unbind"
         bash -c "echo 1af4 1041 > /sys/bus/pci/drivers/vfio-pci/new_id"
 
-        /mnt/cloud-hypervisor --console off --serial tty --kernel /mnt/vmlinux --cmdline "console=ttyS0 reboot=k panic=1 nomodules i8042.noaux i8042.nomux i8042.nopnp i8042.dumbkbd root=/dev/vda2 VFIOTAG" --disk /mnt/clear-29810-cloud.img /mnt/cloudinit.img --cpus 1 --memory size=512M --rng --device /sys/bus/pci/devices/0000:00:03.0/
+        /mnt/cloud-hypervisor --kernel /mnt/vmlinux --cmdline "console=hvc0 reboot=k panic=1 nomodules i8042.noaux i8042.nomux i8042.nopnp i8042.dumbkbd root=/dev/vda2 VFIOTAG" --disk path=/mnt/clear-31310-cloudguest.img path=/mnt/cloudinit.img --cpus 1 --memory size=512M --rng --device path=/sys/bus/pci/devices/0000:00:05.0/ path=/sys/bus/pci/devices/0000:00:06.0/

vfio-bindings/Cargo.toml

@@ -1,12 +0,0 @@
-[package]
-name = "vfio-bindings"
-version = "0.0.1"
-authors = ["The Cloud Hypervisor Authors"]
-edition = "2018"
-
-[dependencies]
-vmm-sys-util = { git = "https://github.com/rust-vmm/vmm-sys-util" }
-
-[features]
-default = ["v5_0_0"]
-v5_0_0 = []

vfio-bindings/src/lib.rs

@@ -1,16 +0,0 @@
-// Copyright 2017 The Chromium OS Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE-BSD-3-Clause file.
-
-#![allow(non_upper_case_globals)]
-#![allow(non_camel_case_types)]
-#![allow(non_snake_case)]
-
-// generated with bindgen linux/uapi/linux/vfio.h --constified-enum '*' --with-derive-default
-#[cfg(feature = "v5_0_0")]
-mod v5_0_0;
-
-pub mod bindings {
-    #[cfg(feature = "v5_0_0")]
-    pub use super::v5_0_0::*;
-}

vfio-bindings/src/v5_0_0/mod.rs

@@ -1,6 +0,0 @@
-#![allow(clippy::all)]
-#![allow(non_upper_case_globals)]
-#![allow(non_camel_case_types)]
-#![allow(non_snake_case)]
-
-pub mod vfio;

vfio/Cargo.toml

@@ -6,14 +6,15 @@ authors = ["The Cloud Hypervisor Authors"]
 [dependencies]
 byteorder = "1.3.2"
 devices = { path = "../devices" }
-kvm-bindings = "0.1.1"
+kvm-bindings = "0.2.0"
-kvm-ioctls = { git = "https://github.com/rust-vmm/kvm-ioctls", branch = "master" }
+kvm-ioctls = "0.4.0"
 libc = "0.2.60"
 log = "0.4.8"
 pci = { path = "../pci" }
-vfio-bindings = { path = "../vfio-bindings" }
+vfio-bindings = "0.1.0"
 vm-allocator = { path = "../vm-allocator" }
-vmm-sys-util = { git = "https://github.com/rust-vmm/vmm-sys-util" }
+vm-device = { path = "../vm-device" }
+vmm-sys-util = ">=0.3.1"
 
 [dependencies.vm-memory]
 git = "https://github.com/rust-vmm/vm-memory"

vfio/src/lib.rs

@@ -14,6 +14,7 @@ extern crate log;
 extern crate pci;
 extern crate vfio_bindings;
 extern crate vm_allocator;
+extern crate vm_device;
 extern crate vm_memory;
 #[macro_use]
 extern crate vmm_sys_util;
@@ -24,7 +25,7 @@ mod vfio_pci;
 
 use std::mem::size_of;
 
-pub use vfio_device::{VfioDevice, VfioError};
+pub use vfio_device::{VfioContainer, VfioDevice, VfioDmaMapping, VfioError};
 pub use vfio_pci::{VfioPciDevice, VfioPciError};
 
 // Returns a `Vec<T>` with a size in bytes at least as large as `size_in_bytes`.

vfio/src/vfio_device.rs

@@ -6,6 +6,7 @@ use crate::vec_with_array_field;
 use byteorder::{ByteOrder, LittleEndian};
 use kvm_ioctls::*;
 use std::collections::HashMap;
+use std::convert::TryInto;
 use std::ffi::CString;
 use std::fmt;
 use std::fs::{File, OpenOptions};
@@ -14,11 +15,13 @@ use std::mem;
 use std::os::unix::io::{AsRawFd, FromRawFd, RawFd};
 use std::os::unix::prelude::FileExt;
 use std::path::{Path, PathBuf};
+use std::result;
 use std::sync::{Arc, RwLock};
 use std::u32;
 use vfio_bindings::bindings::vfio::*;
 use vfio_ioctls::*;
-use vm_memory::{Address, GuestMemory, GuestMemoryMmap, GuestMemoryRegion};
+use vm_device::{get_host_address_range, ExternalDmaMapping};
+use vm_memory::{Address, GuestAddress, GuestMemory, GuestMemoryMmap, GuestMemoryRegion};
 use vmm_sys_util::eventfd::EventFd;
 use vmm_sys_util::ioctl::*;
 
@@ -36,7 +39,7 @@ pub enum VfioError {
     UnsetContainer,
     ContainerSetIOMMU,
     GroupGetDeviceFD,
-    KvmSetDeviceAttr(io::Error),
+    KvmSetDeviceAttr(kvm_ioctls::Error),
     VfioDeviceGetInfo,
     VfioDeviceGetRegionInfo,
     InvalidPath,
@@ -103,7 +106,7 @@ struct vfio_region_info_with_cap {
     cap_info: __IncompleteArrayField<u8>,
 }
 
-struct VfioContainer {
+pub struct VfioContainer {
     container: File,
 }
 
@@ -151,7 +154,7 @@ impl VfioContainer {
         Ok(())
     }
 
-    fn vfio_dma_map(&self, iova: u64, size: u64, user_addr: u64) -> Result<()> {
+    pub fn vfio_dma_map(&self, iova: u64, size: u64, user_addr: u64) -> Result<()> {
         let dma_map = vfio_iommu_type1_dma_map {
             argsz: mem::size_of::<vfio_iommu_type1_dma_map>() as u32,
             flags: VFIO_DMA_MAP_FLAG_READ | VFIO_DMA_MAP_FLAG_WRITE,
@@ -170,7 +173,7 @@ impl VfioContainer {
         Ok(())
     }
 
-    fn vfio_dma_unmap(&self, iova: u64, size: u64) -> Result<()> {
+    pub fn vfio_dma_unmap(&self, iova: u64, size: u64) -> Result<()> {
         let mut dma_unmap = vfio_iommu_type1_dma_unmap {
             argsz: mem::size_of::<vfio_iommu_type1_dma_unmap>() as u32,
             flags: 0,
@@ -198,7 +201,7 @@ impl AsRawFd for VfioContainer {
 struct VfioGroup {
     group: File,
     device: Arc<DeviceFd>,
-    container: VfioContainer,
+    container: Arc<VfioContainer>,
 }
 
 impl VfioGroup {
@@ -225,7 +228,7 @@ impl VfioGroup {
             return Err(VfioError::GroupViable);
         }
 
-        let container = VfioContainer::new()?;
+        let container = Arc::new(VfioContainer::new()?);
         if container.get_api_version() as u32 != VFIO_API_VERSION {
             return Err(VfioError::VfioApiVersion);
         }
@@ -266,7 +269,7 @@ impl VfioGroup {
             .map_err(VfioError::KvmSetDeviceAttr)
     }
 
-    fn kvm_device_del_group(&self) -> std::result::Result<(), io::Error> {
+    fn kvm_device_del_group(&self) -> std::result::Result<(), kvm_ioctls::Error> {
         let group_fd = self.as_raw_fd();
         let group_fd_ptr = &group_fd as *const i32;
         let dev_attr = kvm_bindings::kvm_device_attr {
@@ -506,6 +509,67 @@ impl VfioDeviceInfo {
     }
 }
 
+/// This structure implements the ExternalDmaMapping trait. It is meant to
+/// be used when the caller tries to provide a way to update the mappings
+/// associated with a specific VFIO container.
+pub struct VfioDmaMapping {
+    container: Arc<VfioContainer>,
+    memory: Arc<RwLock<GuestMemoryMmap>>,
+}
+
+impl VfioDmaMapping {
+    pub fn new(container: Arc<VfioContainer>, memory: Arc<RwLock<GuestMemoryMmap>>) -> Self {
+        VfioDmaMapping { container, memory }
+    }
+}
+
+impl ExternalDmaMapping for VfioDmaMapping {
+    fn map(&self, iova: u64, gpa: u64, size: u64) -> result::Result<(), io::Error> {
+        let user_addr = if let Some(addr) = get_host_address_range(
+            &self.memory.read().unwrap(),
+            GuestAddress(gpa),
+            size.try_into().unwrap(),
+        ) {
+            addr as u64
+        } else {
+            return Err(io::Error::new(
+                io::ErrorKind::Other,
+                format!(
+                    "failed to convert guest address 0x{:x} into \
+                     host user virtual address",
+                    gpa
+                ),
+            ));
+        };
+
+        self.container
+            .vfio_dma_map(iova, size, user_addr)
+            .map_err(|e| {
+                io::Error::new(
+                    io::ErrorKind::Other,
+                    format!(
+                        "failed to map memory for VFIO container, \
+                         iova 0x{:x}, gpa 0x{:x}, size 0x{:x}: {:?}",
+                        iova, gpa, size, e
+                    ),
+                )
+            })
+    }
+
+    fn unmap(&self, iova: u64, size: u64) -> result::Result<(), io::Error> {
+        self.container.vfio_dma_unmap(iova, size).map_err(|e| {
+            io::Error::new(
+                io::ErrorKind::Other,
+                format!(
+                    "failed to unmap memory for VFIO container, \
+                     iova 0x{:x}, size 0x{:x}: {:?}",
+                    iova, size, e
+                ),
+            )
+        })
+    }
+}
+
 /// Vfio device for exposing regions which could be read/write to kernel vfio device.
 pub struct VfioDevice {
     device: File,
@@ -514,6 +578,7 @@ pub struct VfioDevice {
     regions: Vec<VfioRegion>,
     irqs: HashMap<u32, VfioIrq>,
     mem: Arc<RwLock<GuestMemoryMmap>>,
+    iommu_attached: bool,
 }
 
 impl VfioDevice {
@@ -524,6 +589,7 @@ impl VfioDevice {
         sysfspath: &Path,
         device_fd: Arc<DeviceFd>,
         mem: Arc<RwLock<GuestMemoryMmap>>,
+        iommu_attached: bool,
     ) -> Result<Self> {
         let uuid_path: PathBuf = [sysfspath, Path::new("iommu_group")].iter().collect();
         let group_path = uuid_path.read_link().map_err(|_| VfioError::InvalidPath)?;
@@ -545,6 +611,7 @@ impl VfioDevice {
             regions,
             irqs,
             mem,
+            iommu_attached,
         })
     }
 
@@ -625,10 +692,10 @@ impl VfioDevice {
 
         let mut irq_set = vec_with_array_field::<vfio_irq_set, u32>(0);
         irq_set[0].argsz = mem::size_of::<vfio_irq_set>() as u32;
-        irq_set[0].flags = VFIO_IRQ_SET_ACTION_MASK;
+        irq_set[0].flags = VFIO_IRQ_SET_ACTION_TRIGGER | VFIO_IRQ_SET_DATA_NONE;
         irq_set[0].index = irq_index;
         irq_set[0].start = 0;
-        irq_set[0].count = irq.count;
+        irq_set[0].count = 0;
 
         // Safe as we are the owner of self and irq_set which are valid value
         let ret = unsafe { ioctl_with_ref(self, VFIO_DEVICE_SET_IRQS(), &irq_set[0]) };
@@ -762,6 +829,10 @@ impl VfioDevice {
         }
     }
 
+    pub fn get_container(&self) -> Arc<VfioContainer> {
+        self.group.container.clone()
+    }
+
     fn vfio_dma_map(&self, iova: u64, size: u64, user_addr: u64) -> Result<()> {
         self.group.container.vfio_dma_map(iova, size, user_addr)
     }
@@ -773,22 +844,26 @@ impl VfioDevice {
     /// Add all guest memory regions into vfio container's iommu table,
     /// then vfio kernel driver could access guest memory from gfn
     pub fn setup_dma_map(&self) -> Result<()> {
-        self.mem.read().unwrap().with_regions(|_index, region| {
+        if !self.iommu_attached {
-            self.vfio_dma_map(
+            self.mem.read().unwrap().with_regions(|_index, region| {
-                region.start_addr().raw_value(),
+                self.vfio_dma_map(
-                region.len() as u64,
+                    region.start_addr().raw_value(),
-                region.as_ptr() as u64,
+                    region.len() as u64,
-            )
+                    region.as_ptr() as u64,
-        })?;
+                )
+            })?;
+        }
         Ok(())
     }
 
     /// remove all guest memory regions from vfio containers iommu table
     /// then vfio kernel driver couldn't access this guest memory
     pub fn unset_dma_map(&self) -> Result<()> {
-        self.mem.read().unwrap().with_regions(|_index, region| {
+        if !self.iommu_attached {
-            self.vfio_dma_unmap(region.start_addr().raw_value(), region.len() as u64)
+            self.mem.read().unwrap().with_regions(|_index, region| {
-        })?;
+                self.vfio_dma_unmap(region.start_addr().raw_value(), region.len() as u64)
+            })?;
+        }
         Ok(())
     }
 

vfio/src/vfio_pci.rs

@@ -16,14 +16,16 @@ use kvm_bindings::{
 };
 use kvm_ioctls::*;
 use pci::{
-    MsiCap, MsixCap, MsixConfig, PciBarConfiguration, PciBarRegionType, PciCapabilityID,
+    BarReprogrammingParams, MsiCap, MsixCap, MsixConfig, PciBarConfiguration, PciBarRegionType,
-    PciClassCode, PciConfiguration, PciDevice, PciDeviceError, PciHeaderType, PciSubclass,
+    PciCapabilityID, PciClassCode, PciConfiguration, PciDevice, PciDeviceError, PciHeaderType,
-    MSIX_TABLE_ENTRY_SIZE,
+    PciSubclass, MSIX_TABLE_ENTRY_SIZE,
 };
+use std::any::Any;
+use std::collections::HashMap;
 use std::os::unix::io::AsRawFd;
 use std::ptr::null_mut;
-use std::sync::Arc;
+use std::sync::{Arc, Mutex};
-use std::{fmt, io};
+use std::{fmt, io, result};
 use vfio_bindings::bindings::vfio::*;
 use vm_allocator::SystemAllocator;
 use vm_memory::{Address, GuestAddress, GuestUsize};
@@ -33,10 +35,12 @@ use vmm_sys_util::eventfd::EventFd;
 pub enum VfioPciError {
     AllocateGsi,
     EventFd(io::Error),
-    IrqFd(io::Error),
+    IrqFd(kvm_ioctls::Error),
     NewVfioPciDevice,
-    MapRegionGuest(io::Error),
+    MapRegionGuest(kvm_ioctls::Error),
-    SetGsiRouting(io::Error),
+    SetGsiRouting(kvm_ioctls::Error),
+    MsiNotConfigured,
+    MsixNotConfigured,
 }
 pub type Result<T> = std::result::Result<T, VfioPciError>;
 
@@ -51,6 +55,8 @@ impl fmt::Display for VfioPciError {
                 write!(f, "failed to map VFIO PCI region into guest: {}", e)
             }
             VfioPciError::SetGsiRouting(e) => write!(f, "failed to set GSI routes for KVM: {}", e),
+            VfioPciError::MsiNotConfigured => write!(f, "MSI interrupt not yet configured"),
+            VfioPciError::MsixNotConfigured => write!(f, "MSI-X interrupt not yet configured"),
         }
     }
 }
@@ -181,22 +187,6 @@ impl Interrupt {
         None
     }
 
-    fn msix_enabled(&self) -> bool {
-        if let Some(msix) = &self.msix {
-            return msix.cap.enabled();
-        }
-
-        false
-    }
-
-    fn msix_function_masked(&self) -> bool {
-        if let Some(msix) = &self.msix {
-            return msix.cap.masked();
-        }
-
-        false
-    }
-
     fn msix_table_accessed(&self, bar_index: u32, offset: u64) -> bool {
         if let Some(msix) = &self.msix {
             return msix.table_accessed(bar_index, offset);
@@ -220,33 +210,27 @@ impl Interrupt {
     }
 }
 
-#[derive(Copy, Clone, Default)]
-struct MsiVector {
-    msg_addr_lo: u32,
-    msg_addr_hi: u32,
-    msg_data: u32,
-    masked: bool,
-}
-
 struct InterruptRoute {
     gsi: u32,
     irq_fd: EventFd,
-    msi_vector: MsiVector,
 }
 
 impl InterruptRoute {
-    fn new(vm: &Arc<VmFd>, allocator: &mut SystemAllocator, msi_vector: MsiVector) -> Result<Self> {
+    fn new(allocator: &mut SystemAllocator) -> Result<Self> {
         let irq_fd = EventFd::new(libc::EFD_NONBLOCK).map_err(VfioPciError::EventFd)?;
         let gsi = allocator.allocate_gsi().ok_or(VfioPciError::AllocateGsi)?;
 
-        vm.register_irqfd(irq_fd.as_raw_fd(), gsi)
+        Ok(InterruptRoute { gsi, irq_fd })
-            .map_err(VfioPciError::IrqFd)?;
+    }
 
-        Ok(InterruptRoute {
+    fn enable(&self, vm: &Arc<VmFd>) -> Result<()> {
-            gsi,
+        vm.register_irqfd(&self.irq_fd, self.gsi)
-            irq_fd,
+            .map_err(VfioPciError::IrqFd)
-            msi_vector,
+    }
-        })
+
+    fn disable(&self, vm: &Arc<VmFd>) -> Result<()> {
+        vm.unregister_irqfd(&self.irq_fd, self.gsi)
+            .map_err(VfioPciError::IrqFd)
     }
 }
 
@@ -255,6 +239,9 @@ struct MmioRegion {
     start: GuestAddress,
     length: GuestUsize,
     index: u32,
+    mem_slot: Option<u32>,
+    host_addr: Option<u64>,
+    mmap_size: Option<usize>,
 }
 
 struct VfioPciConfig {
@@ -311,6 +298,7 @@ pub struct VfioPciDevice {
     mmio_regions: Vec<MmioRegion>,
     interrupt: Interrupt,
     interrupt_routes: Vec<InterruptRoute>,
+    gsi_msi_routes: Arc<Mutex<HashMap<u32, kvm_irq_routing_entry>>>,
 }
 
 impl VfioPciDevice {
@@ -319,6 +307,7 @@ impl VfioPciDevice {
         vm_fd: &Arc<VmFd>,
         allocator: &mut SystemAllocator,
         device: VfioDevice,
+        gsi_msi_routes: Arc<Mutex<HashMap<u32, kvm_irq_routing_entry>>>,
     ) -> Result<Self> {
         let device = Arc::new(device);
         device.reset();
@@ -348,6 +337,7 @@ impl VfioPciDevice {
                 msix: None,
             },
             interrupt_routes: Vec::new(),
+            gsi_msi_routes,
         };
 
         vfio_pci_device.parse_capabilities();
@@ -356,43 +346,36 @@ impl VfioPciDevice {
         // The MSI vectors will be filled when the guest driver programs the device.
         let max_interrupts = vfio_pci_device.device.max_interrupts();
         for _ in 0..max_interrupts {
-            let msi_vector: MsiVector = Default::default();
+            let route = InterruptRoute::new(allocator)?;
-            let route = InterruptRoute::new(vm_fd, allocator, msi_vector)?;
             vfio_pci_device.interrupt_routes.push(route);
         }
 
         Ok(vfio_pci_device)
     }
 
-    fn irq_fds(&self) -> Result<Vec<&EventFd>> {
+    fn enable_irq_fds(&self) -> Result<Vec<&EventFd>> {
         let mut irq_fds: Vec<&EventFd> = Vec::new();
 
         for r in &self.interrupt_routes {
+            r.enable(&self.vm_fd)?;
             irq_fds.push(&r.irq_fd);
         }
 
         Ok(irq_fds)
     }
 
+    fn disable_irq_fds(&self) -> Result<()> {
+        for r in &self.interrupt_routes {
+            r.disable(&self.vm_fd)?;
+        }
+
+        Ok(())
+    }
+
     fn set_kvm_routes(&self) -> Result<()> {
         let mut entry_vec: Vec<kvm_irq_routing_entry> = Vec::new();
-        for route in self.interrupt_routes.iter() {
+        for (_, entry) in self.gsi_msi_routes.lock().unwrap().iter() {
-            // Do not add masked vectors to the GSI mapping
+            entry_vec.push(*entry);
-            if route.msi_vector.masked {
-                continue;
-            }
-
-            let mut entry = kvm_irq_routing_entry {
-                gsi: route.gsi,
-                type_: KVM_IRQ_ROUTING_MSI,
-                ..Default::default()
-            };
-
-            entry.u.msi.address_lo = route.msi_vector.msg_addr_lo;
-            entry.u.msi.address_hi = route.msi_vector.msg_addr_hi;
-            entry.u.msi.data = route.msi_vector.msg_data;
-
-            entry_vec.push(entry);
         }
 
         let mut irq_routing =
@@ -478,60 +461,99 @@ impl VfioPciDevice {
         }
     }
 
-    fn update_msi_interrupt_routes(&mut self, msi: &VfioMsi) -> Result<()> {
+    fn update_msi_interrupt_routes(&self, msi: &VfioMsi) -> Result<()> {
-        let num_vectors = msi.cap.num_enabled_vectors();
-        for (idx, route) in self.interrupt_routes.iter_mut().enumerate() {
-            // Mask the MSI vector if the amount of vectors supported by the
-            // guest OS does not match the expected amount. This is related
-            // to "Multiple Message Capable" and "Multiple Message Enable"
-            // fields from the "Message Control" register.
-            if idx >= num_vectors {
-                route.msi_vector.masked = true;
-                continue;
-            }
-
-            route.msi_vector.msg_addr_lo = msi.cap.msg_addr_lo;
-            route.msi_vector.msg_addr_hi = msi.cap.msg_addr_hi;
-            route.msi_vector.msg_data = u32::from(msi.cap.msg_data) | (idx as u32);
-            route.msi_vector.masked = msi.cap.vector_masked(idx);
-        }
-
-        // Check if we need to update KVM GSI mapping, based on the status of
-        // the "MSI Enable" bit.
         if msi.cap.enabled() {
-            return self.set_kvm_routes();
+            let mut gsi_msi_routes = self.gsi_msi_routes.lock().unwrap();
+
+            for (idx, route) in self.interrupt_routes.iter().enumerate() {
+                // Ignore MSI vector if the amount of vectors supported by the
+                // guest OS does not match the expected amount. This is related
+                // to "Multiple Message Capable" and "Multiple Message Enable"
+                // fields from the "Message Control" register.
+                if idx >= msi.cap.num_enabled_vectors() {
+                    continue;
+                }
+
+                // Ignore MSI vector if masked.
+                if msi.cap.vector_masked(idx) {
+                    continue;
+                }
+
+                let mut entry = kvm_irq_routing_entry {
+                    gsi: route.gsi,
+                    type_: KVM_IRQ_ROUTING_MSI,
+                    ..Default::default()
+                };
+
+                entry.u.msi.address_lo = msi.cap.msg_addr_lo;
+                entry.u.msi.address_hi = msi.cap.msg_addr_hi;
+                entry.u.msi.data = u32::from(msi.cap.msg_data) | (idx as u32);
+
+                gsi_msi_routes.insert(route.gsi, entry);
+            }
+        } else {
+            let mut gsi_msi_routes = self.gsi_msi_routes.lock().unwrap();
+
+            for route in self.interrupt_routes.iter() {
+                gsi_msi_routes.remove(&route.gsi);
+            }
         }
 
-        Ok(())
+        self.set_kvm_routes()
+    }
+
+    fn update_msix_interrupt_routes(&self, msix: &VfioMsix) -> Result<()> {
+        if msix.cap.enabled() && !msix.cap.masked() {
+            let mut gsi_msi_routes = self.gsi_msi_routes.lock().unwrap();
+
+            for (idx, table_entry) in msix.bar.table_entries.iter().enumerate() {
+                // Ignore MSI-X vector if masked.
+                if table_entry.masked() {
+                    continue;
+                }
+
+                let gsi = self.interrupt_routes[idx].gsi;
+
+                let mut entry = kvm_irq_routing_entry {
+                    gsi,
+                    type_: KVM_IRQ_ROUTING_MSI,
+                    ..Default::default()
+                };
+
+                entry.u.msi.address_lo = table_entry.msg_addr_lo;
+                entry.u.msi.address_hi = table_entry.msg_addr_hi;
+                entry.u.msi.data = table_entry.msg_data;
+
+                gsi_msi_routes.insert(gsi, entry);
+            }
+        } else {
+            let mut gsi_msi_routes = self.gsi_msi_routes.lock().unwrap();
+
+            for route in self.interrupt_routes.iter() {
+                gsi_msi_routes.remove(&route.gsi);
+            }
+        }
+
+        self.set_kvm_routes()
     }
 
     fn read_msix_table(&mut self, offset: u64, data: &mut [u8]) {
         self.interrupt.msix_read_table(offset, data);
     }
 
-    fn update_msix_table(&mut self, offset: u64, data: &[u8]) -> Result<()> {
+    fn write_msix_table(&mut self, offset: u64, data: &[u8]) {
         self.interrupt.msix_write_table(offset, data);
 
-        if self.interrupt.msix_enabled() && !self.interrupt.msix_function_masked() {
+        if let Some(msix) = &self.interrupt.msix {
-            // Fill tables
+            if let Err(e) = self.update_msix_interrupt_routes(&msix) {
-            if let Some(msix) = &self.interrupt.msix {
+                error!("Could not update MSI-X interrupt routes: {}", e);
-                for (idx, entry) in msix.bar.table_entries.iter().enumerate() {
-                    self.interrupt_routes[idx].msi_vector.msg_addr_lo = entry.msg_addr_lo;
-                    self.interrupt_routes[idx].msi_vector.msg_addr_hi = entry.msg_addr_hi;
-                    self.interrupt_routes[idx].msi_vector.msg_data = entry.msg_data;
-                    self.interrupt_routes[idx].msi_vector.masked = entry.masked();
-                }
             }
-
-            return self.set_kvm_routes();
         }
-
-        Ok(())
     }
 
     fn update_msi_capabilities(&mut self, offset: u64, data: &[u8]) -> Result<()> {
         match self.interrupt.update_msi(offset, data) {
-            Some(InterruptUpdateAction::EnableMsi) => match self.irq_fds() {
+            Some(InterruptUpdateAction::EnableMsi) => match self.enable_irq_fds() {
                 Ok(fds) => {
                     if let Err(e) = self.device.enable_msi(fds) {
                         warn!("Could not enable MSI: {}", e);
@@ -543,24 +565,28 @@ impl VfioPciDevice {
                 if let Err(e) = self.device.disable_msi() {
                     warn!("Could not disable MSI: {}", e);
                 }
+                if let Err(e) = self.disable_irq_fds() {
+                    warn!("Could not disable MSI: {}", e);
+                }
             }
             _ => {}
         }
 
-        // Update the interrupt_routes table now that the MSI cache has been
+        // Update the gsi_msi_routes table now that the MSI cache has been
         // updated. The point is to always update the table based on latest
         // changes to the cache, and based on the state of masking flags, the
         // KVM GSI routes should be configured.
-        if let Some(msi) = self.interrupt.msi {
+        if let Some(msi) = &self.interrupt.msi {
             return self.update_msi_interrupt_routes(&msi);
         }
 
-        Ok(())
+        // If the code reach this point, something went wrong.
+        Err(VfioPciError::MsiNotConfigured)
     }
 
-    fn update_msix_capabilities(&mut self, offset: u64, data: &[u8]) {
+    fn update_msix_capabilities(&mut self, offset: u64, data: &[u8]) -> Result<()> {
         match self.interrupt.update_msix(offset, data) {
-            Some(InterruptUpdateAction::EnableMsix) => match self.irq_fds() {
+            Some(InterruptUpdateAction::EnableMsix) => match self.enable_irq_fds() {
                 Ok(fds) => {
                     if let Err(e) = self.device.enable_msix(fds) {
                         warn!("Could not enable MSI-X: {}", e);
@@ -570,11 +596,23 @@ impl VfioPciDevice {
             },
             Some(InterruptUpdateAction::DisableMsix) => {
                 if let Err(e) = self.device.disable_msix() {
+                    warn!("Could not disable MSI-X: {}", e);
+                }
+                if let Err(e) = self.disable_irq_fds() {
                     warn!("Could not disable MSI: {}", e);
                 }
             }
             _ => {}
         }
+
+        // Update the gsi_msi_routes table because the state of the enable bit
+        // changed.
+        if let Some(msix) = &self.interrupt.msix {
+            return self.update_msix_interrupt_routes(&msix);
+        }
+
+        // If the code reach this point, something went wrong.
+        Err(VfioPciError::MsixNotConfigured)
     }
 
     fn find_region(&self, addr: u64) -> Option<MmioRegion> {
@@ -603,7 +641,7 @@ impl VfioPciDevice {
         let fd = self.device.as_raw_fd();
         let mut new_mem_slot = mem_slot;
 
-        for region in self.mmio_regions.iter() {
+        for region in self.mmio_regions.iter_mut() {
             // We want to skip the mapping of the BAR containing the MSI-X
             // table even if it is mappable. The reason is we need to trap
             // any access to the MSI-X table and update the GSI routing
@@ -658,22 +696,48 @@ impl VfioPciDevice {
                     vm.set_user_memory_region(mem_region)
                         .map_err(VfioPciError::MapRegionGuest)?;
                 }
+
+                // Update the region with memory mapped info.
+                region.mem_slot = Some(new_mem_slot);
+                region.host_addr = Some(host_addr as u64);
+                region.mmap_size = Some(mmap_size as usize);
+
                 new_mem_slot += 1;
             }
         }
 
         Ok(new_mem_slot)
     }
+
+    pub fn unmap_mmio_regions(&mut self) {
+        for region in self.mmio_regions.iter() {
+            if let (Some(addr), Some(size)) = (region.host_addr, region.mmap_size) {
+                let ret = unsafe { libc::munmap(addr as *mut libc::c_void, size) };
+                if ret != 0 {
+                    error!(
+                        "Could not unmap regions, error:{}",
+                        io::Error::last_os_error()
+                    );
+                }
+            }
+        }
+    }
 }
 
 impl Drop for VfioPciDevice {
     fn drop(&mut self) {
-        if self.interrupt.msi.is_some() && self.device.disable_msi().is_err() {
+        self.unmap_mmio_regions();
-            error!("Could not disable MSI");
+
+        if let Some(msix) = &self.interrupt.msix {
+            if msix.cap.enabled() && self.device.disable_msix().is_err() {
+                error!("Could not disable MSI-X");
+            }
         }
 
-        if self.interrupt.msix.is_some() && self.device.disable_msix().is_err() {
+        if let Some(msi) = &self.interrupt.msi {
-            error!("Could not disable MSI-X");
+            if msi.cap.enabled() && self.device.disable_msi().is_err() {
+                error!("Could not disable MSI");
+            }
         }
 
         if self.device.unset_dma_map().is_err() {
@@ -870,6 +934,9 @@ impl PciDevice for VfioPciDevice {
                 start: bar_addr,
                 length: region_size,
                 index: bar_id as u32,
+                mem_slot: None,
+                host_addr: None,
+                mmap_size: None,
             });
 
             bar_id += 1;
@@ -901,8 +968,6 @@ impl PciDevice for VfioPciDevice {
         }
 
         let reg = (reg_idx * PCI_CONFIG_REGISTER_SIZE) as u64;
-        self.device
-            .region_write(VFIO_PCI_CONFIG_REGION_INDEX, data, reg + offset);
 
         // If the MSI or MSI-X capabilities are accessed, we need to
         // update our local cache accordingly.
@@ -917,11 +982,24 @@ impl PciDevice for VfioPciDevice {
                     }
                 }
                 PciCapabilityID::MSIX => {
-                    self.update_msix_capabilities(cap_offset, data);
+                    if let Err(e) = self.update_msix_capabilities(cap_offset, data) {
+                        error!("Could not update MSI-X capabilities: {}", e);
+                    }
                 }
                 _ => {}
             }
         }
+
+        // Make sure to write to the device's PCI config space after MSI/MSI-X
+        // interrupts have been enabled/disabled. In case of MSI, when the
+        // interrupts are enabled through VFIO (using VFIO_DEVICE_SET_IRQS),
+        // the MSI Enable bit in the MSI capability structure found in the PCI
+        // config space is disabled by default. That's why when the guest is
+        // enabling this bit, we first need to enable the MSI interrupts with
+        // VFIO through VFIO_DEVICE_SET_IRQS ioctl, and only after we can write
+        // to the device region to update the MSI Enable bit.
+        self.device
+            .region_write(VFIO_PCI_CONFIG_REGION_INDEX, data, reg + offset);
     }
 
     fn read_config_register(&self, reg_idx: usize) -> u32 {
@@ -958,6 +1036,14 @@ impl PciDevice for VfioPciDevice {
             & mask
     }
 
+    fn detect_bar_reprogramming(
+        &mut self,
+        reg_idx: usize,
+        data: &[u8],
+    ) -> Option<BarReprogrammingParams> {
+        self.configuration.detect_bar_reprogramming(reg_idx, data)
+    }
+
     fn read_bar(&mut self, base: u64, offset: u64, data: &mut [u8]) {
         let addr = base + offset;
         if let Some(region) = self.find_region(addr) {
@@ -978,12 +1064,60 @@ impl PciDevice for VfioPciDevice {
 
             // If the MSI-X table is written to, we need to update our cache.
             if self.interrupt.msix_table_accessed(region.index, offset) {
-                if let Err(e) = self.update_msix_table(offset, data) {
+                self.write_msix_table(offset, data);
-                    error!("Could not update MSI-X table: {}", e);
-                }
             } else {
                 self.device.region_write(region.index, data, offset);
             }
         }
     }
+
+    fn move_bar(&mut self, old_base: u64, new_base: u64) -> result::Result<(), io::Error> {
+        for region in self.mmio_regions.iter_mut() {
+            if region.start.raw_value() == old_base {
+                region.start = GuestAddress(new_base);
+
+                if let Some(mem_slot) = region.mem_slot {
+                    if let Some(host_addr) = region.host_addr {
+                        let (mmap_offset, mmap_size) = self.device.get_region_mmap(region.index);
+
+                        // Remove old region from KVM
+                        let old_mem_region = kvm_userspace_memory_region {
+                            slot: mem_slot,
+                            guest_phys_addr: old_base + mmap_offset,
+                            memory_size: 0,
+                            userspace_addr: host_addr,
+                            flags: 0,
+                        };
+                        // Safe because the guest regions are guaranteed not to overlap.
+                        unsafe {
+                            self.vm_fd
+                                .set_user_memory_region(old_mem_region)
+                                .map_err(|e| io::Error::from_raw_os_error(e.errno()))?;
+                        }
+
+                        // Insert new region to KVM
+                        let new_mem_region = kvm_userspace_memory_region {
+                            slot: mem_slot,
+                            guest_phys_addr: new_base + mmap_offset,
+                            memory_size: mmap_size as u64,
+                            userspace_addr: host_addr,
+                            flags: 0,
+                        };
+                        // Safe because the guest regions are guaranteed not to overlap.
+                        unsafe {
+                            self.vm_fd
+                                .set_user_memory_region(new_mem_region)
+                                .map_err(|e| io::Error::from_raw_os_error(e.errno()))?;
+                        }
+                    }
+                }
+            }
+        }
+
+        Ok(())
+    }
+
+    fn as_any(&mut self) -> &mut dyn Any {
+        self
+    }
 }

vhost_rs/Cargo.toml

@@ -15,7 +15,7 @@ vhost-user-slave = []
 [dependencies]
 bitflags = "1.1.0"
 libc = "0.2.60"
-vmm-sys-util = { git = "https://github.com/rust-vmm/vmm-sys-util" }
+vmm-sys-util = ">=0.3.1"
 
 [dependencies.vm-memory]
 git = "https://github.com/rust-vmm/vm-memory"

vhost_rs/src/vhost_user/connection.rs

@@ -197,45 +197,6 @@ impl<R: Req> Endpoint<R> {
         Ok(())
     }
 
-    /// Send a message with header, body and config info. Optional file descriptors may be attached to
-    /// the message.
-    ///
-    /// # Return:
-    /// * - number of bytes sent on success
-    /// * - SocketRetry: temporary error caused by signals or short of resources.
-    /// * - SocketBroken: the underline socket is broken.
-    /// * - SocketError: other socket related errors.
-    /// * - PartialMessage: received a partial message.
-    pub fn send_config_message(
-        &mut self,
-        hdr: &VhostUserMsgHeader<R>,
-        user_config: &VhostUserConfig,
-        buf: &mut [u8],
-        fds: Option<&[RawFd]>,
-    ) -> Result<()> {
-        // Safe because there can't be other mutable referance to hdr and body.
-        let iovs = unsafe {
-            [
-                slice::from_raw_parts(
-                    hdr as *const VhostUserMsgHeader<R> as *const u8,
-                    mem::size_of::<VhostUserMsgHeader<R>>(),
-                ),
-                slice::from_raw_parts(
-                    user_config as *const VhostUserConfig as *const u8,
-                    mem::size_of::<VhostUserConfig>(),
-                ),
-                slice::from_raw_parts(buf.as_ptr() as *const u8, buf.len()),
-            ]
-        };
-        let bytes = self.send_iovec(&iovs[..], fds)?;
-        let total =
-            mem::size_of::<VhostUserMsgHeader<R>>() + mem::size_of::<VhostUserConfig>() + buf.len();
-        if bytes != total {
-            return Err(Error::PartialMessage);
-        }
-        Ok(())
-    }
-
     /// Send a message with header, body and payload. Optional file descriptors
     /// may also be attached to the message.
     ///

vhost_rs/src/vhost_user/master.rs

@@ -40,11 +40,12 @@ pub trait VhostUserMaster: VhostBackend {
         offset: u32,
         size: u32,
         flags: VhostUserConfigFlags,
-    ) -> Result<Vec<u8>>;
+        buf: &[u8],
+    ) -> Result<(VhostUserConfig, VhostUserConfigPayload)>;
 
     /// Change the virtio device configuration space. It also can be used for live migration on the
     /// destination host to set readonly configuration space fields.
-    fn set_config(&mut self, offset: u32, buf: &[u8], flags: VhostUserConfigFlags) -> Result<()>;
+    fn set_config(&mut self, offset: u32, flags: VhostUserConfigFlags, buf: &[u8]) -> Result<()>;
 
     /// Setup slave communication channel.
     fn set_slave_request_fd(&mut self, fd: RawFd) -> Result<()>;
@@ -166,9 +167,12 @@ impl VhostBackend for Master {
         node.wait_for_ack(&hdr).map_err(|e| e.into())
     }
 
+    // Clippy doesn't seem to know that if let with && is still experimental
+    #[allow(clippy::unnecessary_unwrap)]
     fn set_log_base(&mut self, base: u64, fd: Option<RawFd>) -> Result<()> {
         let mut node = self.node.lock().unwrap();
         let val = VhostUserU64::new(base);
+
         if node.acked_protocol_features & VhostUserProtocolFeatures::LOG_SHMFD.bits() != 0
             && fd.is_some()
         {
@@ -344,7 +348,8 @@ impl VhostUserMaster for Master {
         offset: u32,
         size: u32,
         flags: VhostUserConfigFlags,
-    ) -> Result<Vec<u8>> {
+        buf: &[u8],
+    ) -> Result<(VhostUserConfig, VhostUserConfigPayload)> {
         let body = VhostUserConfig::new(offset, size, flags);
         if !body.is_valid() {
             return error_code(VhostUserError::InvalidParam);
@@ -356,24 +361,24 @@ impl VhostUserMaster for Master {
             return error_code(VhostUserError::InvalidOperation);
         }
 
-        // TODO: vhost-user spec states that:
+        // vhost-user spec states that:
         // "Master payload: virtio device config space"
-        // But what content should the payload contains for a get_config() request?
+        // "Slave payload: virtio device config space"
-        // So current implementation doesn't conform to the spec.
+        let hdr = node.send_request_with_payload(MasterReq::GET_CONFIG, &body, buf, None)?;
-        let hdr = node.send_request_with_config_body(MasterReq::GET_CONFIG, &body, None)?;
+        let (body_reply, buf_reply, rfds) =
-        let (reply, buf, rfds) = node.recv_reply_with_payload::<VhostUserConfig>(&hdr)?;
+            node.recv_reply_with_payload::<VhostUserConfig>(&hdr)?;
         if rfds.is_some() {
             Endpoint::<MasterReq>::close_rfds(rfds);
             return error_code(VhostUserError::InvalidMessage);
-        } else if reply.size == 0 {
+        } else if body_reply.size == 0 {
             return error_code(VhostUserError::SlaveInternalError);
-        } else if reply.size != body.size || reply.size as usize != buf.len() {
+        } else if body_reply.size != body.size || body_reply.size as usize != buf.len() {
             return error_code(VhostUserError::InvalidMessage);
         }
-        Ok(buf)
+        Ok((body_reply, buf_reply))
     }
 
-    fn set_config(&mut self, offset: u32, buf: &[u8], flags: VhostUserConfigFlags) -> Result<()> {
+    fn set_config(&mut self, offset: u32, flags: VhostUserConfigFlags, buf: &[u8]) -> Result<()> {
         if buf.len() > MAX_MSG_SIZE {
             return error_code(VhostUserError::InvalidParam);
         }
@@ -480,27 +485,6 @@ impl MasterInternal {
         Ok(hdr)
     }
 
-    fn send_request_with_config_body(
-        &mut self,
-        code: MasterReq,
-        user_config: &VhostUserConfig,
-        fds: Option<&[RawFd]>,
-    ) -> VhostUserResult<VhostUserMsgHeader<MasterReq>> {
-        if mem::size_of::<VhostUserConfig>() + user_config.size as usize > MAX_MSG_SIZE {
-            return Err(VhostUserError::InvalidParam);
-        }
-        self.check_state()?;
-
-        let hdr = Self::new_request_header(
-            code,
-            mem::size_of::<VhostUserConfig>() as u32 + user_config.size,
-        );
-        let mut buf = vec![0; user_config.size as usize];
-        self.main_sock
-            .send_config_message(&hdr, user_config, &mut buf, fds)?;
-        Ok(hdr)
-    }
-
     fn send_request_with_payload<T: Sized, P: Sized>(
         &mut self,
         code: MasterReq,
@@ -571,7 +555,7 @@ impl MasterInternal {
         }
         self.check_state()?;
 
-        let mut buf = vec![0; MAX_MSG_SIZE - mem::size_of::<T>()];
+        let mut buf: Vec<u8> = vec![0; hdr.get_size() as usize - mem::size_of::<T>()];
         let (reply, body, bytes, rfds) = self.main_sock.recv_payload_into_buf::<T>(&mut buf)?;
         if !reply.is_reply_for(hdr)
             || reply.get_size() as usize != mem::size_of::<T>() + bytes

vhost_rs/src/vhost_user/message.rs

@@ -346,6 +346,8 @@ bitflags! {
         const SLAVE_SEND_FD = 0x0000_0400;
         /// Allow the slave to register a host notifier.
         const HOST_NOTIFIER = 0x0000_0800;
+        /// Support inflight shmfd.
+        const INFLIGHT_SHMFD = 0x0000_1000;
     }
 }
 

vhost_rs/src/vhost_user/slave_req_handler.rs

@@ -265,7 +265,6 @@ impl<S: VhostUserSlaveReqHandler> SlaveReqHandler<S> {
                 if self.acked_protocol_features & VhostUserProtocolFeatures::CONFIG.bits() == 0 {
                     return Err(Error::InvalidOperation);
                 }
-                self.check_request_size(&hdr, size, mem::size_of::<VhostUserConfig>())?;
                 self.get_config(&hdr, &buf)?;
             }
             MasterReq::SET_CONFIG => {
@@ -341,6 +340,10 @@ impl<S: VhostUserSlaveReqHandler> SlaveReqHandler<S> {
         if !msg.is_valid() {
             return Err(Error::InvalidMessage);
         }
+        let payload_offset = mem::size_of::<VhostUserConfig>();
+        if buf.len() - payload_offset != msg.size as usize {
+            return Err(Error::InvalidMessage);
+        }
         let flags = match VhostUserConfigFlags::from_bits(msg.flags) {
             Some(val) => val,
             None => return Err(Error::InvalidMessage),
@@ -519,6 +522,7 @@ impl<S: VhostUserSlaveReqHandler> SlaveReqHandler<S> {
     fn new_reply_header<T: Sized>(
         &self,
         req: &VhostUserMsgHeader<MasterReq>,
+        payload_size: usize,
     ) -> Result<VhostUserMsgHeader<MasterReq>> {
         if mem::size_of::<T>() > MAX_MSG_SIZE {
             return Err(Error::InvalidParam);
@@ -527,7 +531,7 @@ impl<S: VhostUserSlaveReqHandler> SlaveReqHandler<S> {
         Ok(VhostUserMsgHeader::new(
             req.get_code(),
             VhostUserHeaderFlag::REPLY.bits(),
-            mem::size_of::<T>() as u32,
+            (mem::size_of::<T>() + payload_size) as u32,
         ))
     }
 
@@ -537,7 +541,7 @@ impl<S: VhostUserSlaveReqHandler> SlaveReqHandler<S> {
         res: Result<()>,
     ) -> Result<()> {
         if self.reply_ack_enabled {
-            let hdr = self.new_reply_header::<VhostUserU64>(req)?;
+            let hdr = self.new_reply_header::<VhostUserU64>(req, 0)?;
             let val = match res {
                 Ok(_) => 0,
                 Err(_) => 1,
@@ -553,7 +557,7 @@ impl<S: VhostUserSlaveReqHandler> SlaveReqHandler<S> {
         req: &VhostUserMsgHeader<MasterReq>,
         msg: &T,
     ) -> Result<()> {
-        let hdr = self.new_reply_header::<T>(req)?;
+        let hdr = self.new_reply_header::<T>(req, 0)?;
         self.main_sock.send_message(&hdr, msg, None)?;
         Ok(())
     }
@@ -568,7 +572,7 @@ impl<S: VhostUserSlaveReqHandler> SlaveReqHandler<S> {
         T: Sized,
         P: Sized,
     {
-        let hdr = self.new_reply_header::<T>(req)?;
+        let hdr = self.new_reply_header::<T>(req, payload.len())?;
         self.main_sock
             .send_message_with_payload(&hdr, msg, payload, None)?;
         Ok(())

vhost_user_backend/Cargo.toml

@@ -0,0 +1,22 @@
+[package]
+name = "vhost_user_backend"
+version = "0.1.0"
+authors = ["The Cloud Hypervisor Authors"]
+edition = "2018"
+
+[features]
+default = []
+pci_support = ["vm-virtio/pci_support"]
+mmio_support = ["vm-virtio/mmio_support"]
+
+[dependencies]
+epoll = ">=4.0.1"
+libc = "0.2.66"
+vm-memory = { git = "https://github.com/rust-vmm/vm-memory" }
+vm-virtio = { path = "../vm-virtio" }
+vmm-sys-util = ">=0.3.1"
+
+[dependencies.vhost_rs]
+path = "../vhost_rs"
+features = ["vhost-user-slave"]
+

vhost_user_backend/src/lib.rs

@@ -0,0 +1,738 @@
+// Copyright 2019 Intel Corporation. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+//
+// Copyright 2019 Alibaba Cloud Computing. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+use std::error;
+use std::fs::File;
+use std::io;
+use std::num::Wrapping;
+use std::os::unix::io::{AsRawFd, FromRawFd, RawFd};
+use std::result;
+use std::sync::{Arc, Mutex, RwLock};
+use std::thread;
+use vhost_rs::vhost_user::message::{
+    VhostUserConfigFlags, VhostUserMemoryRegion, VhostUserProtocolFeatures,
+    VhostUserVirtioFeatures, VhostUserVringAddrFlags, VhostUserVringState,
+};
+use vhost_rs::vhost_user::{
+    Error as VhostUserError, Result as VhostUserResult, SlaveListener, VhostUserSlaveReqHandler,
+};
+use vm_memory::guest_memory::FileOffset;
+use vm_memory::{GuestAddress, GuestMemoryMmap};
+use vm_virtio::Queue;
+use vmm_sys_util::eventfd::EventFd;
+
+#[derive(Debug)]
+/// Errors related to vhost-user daemon.
+pub enum Error {
+    /// Failed to create a new vhost-user handler.
+    NewVhostUserHandler(VhostUserHandlerError),
+    /// Failed creating vhost-user slave listener.
+    CreateSlaveListener(VhostUserError),
+    /// Failed creating vhost-user slave handler.
+    CreateSlaveReqHandler(VhostUserError),
+    /// Failed starting daemon thread.
+    StartDaemon(io::Error),
+    /// Failed waiting for daemon thread.
+    WaitDaemon(std::boxed::Box<dyn std::any::Any + std::marker::Send>),
+    /// Failed handling a vhost-user request.
+    HandleRequest(VhostUserError),
+    /// Failed to process queue.
+    ProcessQueue(VringEpollHandlerError),
+    /// Failed to register listener.
+    RegisterListener(io::Error),
+    /// Failed to unregister listener.
+    UnregisterListener(io::Error),
+}
+
+/// Result of vhost-user daemon operations.
+pub type Result<T> = result::Result<T, Error>;
+
+/// This trait must be implemented by the caller in order to provide backend
+/// specific implementation.
+pub trait VhostUserBackend: Send + Sync + 'static {
+    /// Number of queues.
+    fn num_queues(&self) -> usize;
+
+    /// Depth of each queue.
+    fn max_queue_size(&self) -> usize;
+
+    /// Virtio features.
+    fn features(&self) -> u64;
+
+    /// Virtio protocol features.
+    fn protocol_features(&self) -> VhostUserProtocolFeatures;
+
+    /// Update guest memory regions.
+    fn update_memory(&mut self, mem: GuestMemoryMmap) -> result::Result<(), io::Error>;
+
+    /// This function gets called if the backend registered some additional
+    /// listeners onto specific file descriptors. The library can handle
+    /// virtqueues on its own, but does not know what to do with events
+    /// happening on custom listeners.
+    fn handle_event(
+        &mut self,
+        device_event: u16,
+        evset: epoll::Events,
+        vrings: &[Arc<RwLock<Vring>>],
+    ) -> result::Result<bool, io::Error>;
+
+    /// Get virtio device configuration.
+    /// A default implementation is provided as we cannot expect all backends
+    /// to implement this function.
+    fn get_config(&self, _offset: u32, _size: u32) -> Vec<u8> {
+        Vec::new()
+    }
+
+    /// Set virtio device configuration.
+    /// A default implementation is provided as we cannot expect all backends
+    /// to implement this function.
+    fn set_config(&mut self, _offset: u32, _buf: &[u8]) -> result::Result<(), io::Error> {
+        Ok(())
+    }
+}
+
+/// This structure is the public API the backend is allowed to interact with
+/// in order to run a fully functional vhost-user daemon.
+pub struct VhostUserDaemon<S: VhostUserBackend> {
+    name: String,
+    sock_path: String,
+    handler: Arc<Mutex<VhostUserHandler<S>>>,
+    main_thread: Option<thread::JoinHandle<Result<()>>>,
+}
+
+impl<S: VhostUserBackend> VhostUserDaemon<S> {
+    /// Create the daemon instance, providing the backend implementation of
+    /// VhostUserBackend.
+    /// Under the hood, this will start a dedicated thread responsible for
+    /// listening onto registered event. Those events can be vring events or
+    /// custom events from the backend, but they get to be registered later
+    /// during the sequence.
+    pub fn new(name: String, sock_path: String, backend: Arc<RwLock<S>>) -> Result<Self> {
+        let handler = Arc::new(Mutex::new(
+            VhostUserHandler::new(backend).map_err(Error::NewVhostUserHandler)?,
+        ));
+
+        Ok(VhostUserDaemon {
+            name,
+            sock_path,
+            handler,
+            main_thread: None,
+        })
+    }
+
+    /// Connect to the vhost-user socket and run a dedicated thread handling
+    /// all requests coming through this socket. This runs in an infinite loop
+    /// that should be terminating once the other end of the socket (the VMM)
+    /// disconnects.
+    pub fn start(&mut self) -> Result<()> {
+        let mut slave_listener =
+            SlaveListener::new(self.sock_path.as_str(), true, self.handler.clone())
+                .map_err(Error::CreateSlaveListener)?;
+        let mut slave_handler = slave_listener
+            .accept()
+            .map_err(Error::CreateSlaveReqHandler)?
+            .unwrap();
+        let handle = thread::Builder::new()
+            .name(self.name.clone())
+            .spawn(move || loop {
+                slave_handler
+                    .handle_request()
+                    .map_err(Error::HandleRequest)?;
+            })
+            .map_err(Error::StartDaemon)?;
+
+        self.main_thread = Some(handle);
+
+        Ok(())
+    }
+
+    /// Wait for the thread handling the vhost-user socket connection to
+    /// terminate.
+    pub fn wait(&mut self) -> Result<()> {
+        if let Some(handle) = self.main_thread.take() {
+            let _ = handle.join().map_err(Error::WaitDaemon)?;
+        }
+        Ok(())
+    }
+
+    /// Retrieve the vring worker. This is necessary to perform further
+    /// actions like registering and unregistering some extra event file
+    /// descriptors.
+    pub fn get_vring_worker(&self) -> Arc<VringWorker> {
+        self.handler.lock().unwrap().get_vring_worker()
+    }
+}
+
+struct AddrMapping {
+    vmm_addr: u64,
+    size: u64,
+    gpa_base: u64,
+}
+
+struct Memory {
+    mappings: Vec<AddrMapping>,
+}
+
+pub struct Vring {
+    queue: Queue,
+    kick: Option<EventFd>,
+    call: Option<EventFd>,
+    err: Option<EventFd>,
+    enabled: bool,
+}
+
+impl Vring {
+    fn new(max_queue_size: u16) -> Self {
+        Vring {
+            queue: Queue::new(max_queue_size),
+            kick: None,
+            call: None,
+            err: None,
+            enabled: false,
+        }
+    }
+
+    pub fn mut_queue(&mut self) -> &mut Queue {
+        &mut self.queue
+    }
+
+    pub fn signal_used_queue(&self) -> result::Result<(), io::Error> {
+        if let Some(call) = self.call.as_ref() {
+            return call.write(1);
+        }
+
+        Ok(())
+    }
+}
+
+#[derive(Debug)]
+/// Errors related to vring epoll handler.
+pub enum VringEpollHandlerError {
+    /// Failed to process the queue from the backend.
+    ProcessQueueBackendProcessing(io::Error),
+    /// Failed to signal used queue.
+    SignalUsedQueue(io::Error),
+    /// Failed to read the event from kick EventFd.
+    HandleEventReadKick(io::Error),
+    /// Failed to handle the event from the backend.
+    HandleEventBackendHandling(io::Error),
+}
+
+/// Result of vring epoll handler operations.
+type VringEpollHandlerResult<T> = std::result::Result<T, VringEpollHandlerError>;
+
+struct VringEpollHandler<S: VhostUserBackend> {
+    backend: Arc<RwLock<S>>,
+    vrings: Vec<Arc<RwLock<Vring>>>,
+}
+
+impl<S: VhostUserBackend> VringEpollHandler<S> {
+    fn handle_event(
+        &self,
+        device_event: u16,
+        evset: epoll::Events,
+    ) -> VringEpollHandlerResult<bool> {
+        let num_queues = self.vrings.len();
+        if (device_event as usize) < num_queues {
+            if let Some(kick) = &self.vrings[device_event as usize].read().unwrap().kick {
+                kick.read()
+                    .map_err(VringEpollHandlerError::HandleEventReadKick)?;
+            }
+
+            // If the vring is not enabled, it should not be processed.
+            // The event is only read to be discarded.
+            if !self.vrings[device_event as usize].read().unwrap().enabled {
+                return Ok(false);
+            }
+        }
+
+        self.backend
+            .write()
+            .unwrap()
+            .handle_event(device_event, evset, &self.vrings)
+            .map_err(VringEpollHandlerError::HandleEventBackendHandling)
+    }
+}
+
+#[derive(Debug)]
+/// Errors related to vring worker.
+enum VringWorkerError {
+    /// Failed while waiting for events.
+    EpollWait(io::Error),
+}
+
+/// Result of vring worker operations.
+type VringWorkerResult<T> = std::result::Result<T, VringWorkerError>;
+
+pub struct VringWorker {
+    epoll_fd: RawFd,
+}
+
+impl VringWorker {
+    fn run<S: VhostUserBackend>(&self, handler: VringEpollHandler<S>) -> VringWorkerResult<()> {
+        const EPOLL_EVENTS_LEN: usize = 100;
+        let mut events = vec![epoll::Event::new(epoll::Events::empty(), 0); EPOLL_EVENTS_LEN];
+
+        'epoll: loop {
+            let num_events = match epoll::wait(self.epoll_fd, -1, &mut events[..]) {
+                Ok(res) => res,
+                Err(e) => {
+                    if e.kind() == io::ErrorKind::Interrupted {
+                        // It's well defined from the epoll_wait() syscall
+                        // documentation that the epoll loop can be interrupted
+                        // before any of the requested events occurred or the
+                        // timeout expired. In both those cases, epoll_wait()
+                        // returns an error of type EINTR, but this should not
+                        // be considered as a regular error. Instead it is more
+                        // appropriate to retry, by calling into epoll_wait().
+                        continue;
+                    }
+                    return Err(VringWorkerError::EpollWait(e));
+                }
+            };
+
+            for event in events.iter().take(num_events) {
+                let evset = match epoll::Events::from_bits(event.events) {
+                    Some(evset) => evset,
+                    None => {
+                        let evbits = event.events;
+                        println!("epoll: ignoring unknown event set: 0x{:x}", evbits);
+                        continue;
+                    }
+                };
+
+                let ev_type = event.data as u16;
+
+                if let Err(e) = handler.handle_event(ev_type, evset) {
+                    println!(
+                        "vring handler handle event {} with error {:?}\n",
+                        ev_type, e
+                    );
+                    break 'epoll;
+                }
+            }
+        }
+
+        Ok(())
+    }
+
+    /// Register a custom event only meaningful to the caller. When this event
+    /// is later triggered, and because only the caller knows what to do about
+    /// it, the backend implementation of `handle_event` will be called.
+    /// This lets entire control to the caller about what needs to be done for
+    /// this special event, without forcing it to run its own dedicated epoll
+    /// loop for it.
+    pub fn register_listener(
+        &self,
+        fd: RawFd,
+        ev_type: epoll::Events,
+        data: u64,
+    ) -> result::Result<(), io::Error> {
+        epoll::ctl(
+            self.epoll_fd,
+            epoll::ControlOptions::EPOLL_CTL_ADD,
+            fd,
+            epoll::Event::new(ev_type, data),
+        )
+    }
+
+    /// Unregister a custom event. If the custom event is triggered after this
+    /// function has been called, nothing will happen as it will be removed
+    /// from the list of file descriptors the epoll loop is listening to.
+    pub fn unregister_listener(
+        &self,
+        fd: RawFd,
+        ev_type: epoll::Events,
+        data: u64,
+    ) -> result::Result<(), io::Error> {
+        epoll::ctl(
+            self.epoll_fd,
+            epoll::ControlOptions::EPOLL_CTL_DEL,
+            fd,
+            epoll::Event::new(ev_type, data),
+        )
+    }
+}
+
+#[derive(Debug)]
+/// Errors related to vhost-user handler.
+pub enum VhostUserHandlerError {
+    /// Failed to create epoll file descriptor.
+    EpollCreateFd(io::Error),
+    /// Failed to spawn vring worker.
+    SpawnVringWorker(io::Error),
+    /// Could not find the mapping from memory regions.
+    MissingMemoryMapping,
+}
+
+impl std::fmt::Display for VhostUserHandlerError {
+    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
+        match self {
+            VhostUserHandlerError::EpollCreateFd(e) => write!(f, "failed creating epoll fd: {}", e),
+            VhostUserHandlerError::SpawnVringWorker(e) => {
+                write!(f, "failed spawning the vring worker: {}", e)
+            }
+            VhostUserHandlerError::MissingMemoryMapping => write!(f, "Missing memory mapping"),
+        }
+    }
+}
+
+impl error::Error for VhostUserHandlerError {}
+
+/// Result of vhost-user handler operations.
+type VhostUserHandlerResult<T> = std::result::Result<T, VhostUserHandlerError>;
+
+struct VhostUserHandler<S: VhostUserBackend> {
+    backend: Arc<RwLock<S>>,
+    worker: Arc<VringWorker>,
+    owned: bool,
+    features_acked: bool,
+    acked_features: u64,
+    acked_protocol_features: u64,
+    num_queues: usize,
+    max_queue_size: usize,
+    memory: Option<Memory>,
+    vrings: Vec<Arc<RwLock<Vring>>>,
+}
+
+impl<S: VhostUserBackend> VhostUserHandler<S> {
+    fn new(backend: Arc<RwLock<S>>) -> VhostUserHandlerResult<Self> {
+        let num_queues = backend.read().unwrap().num_queues();
+        let max_queue_size = backend.read().unwrap().max_queue_size();
+
+        let mut vrings: Vec<Arc<RwLock<Vring>>> = Vec::new();
+        for _ in 0..num_queues {
+            let vring = Arc::new(RwLock::new(Vring::new(max_queue_size as u16)));
+            vrings.push(vring);
+        }
+
+        // Create the epoll file descriptor
+        let epoll_fd = epoll::create(true).map_err(VhostUserHandlerError::EpollCreateFd)?;
+
+        let vring_handler = VringEpollHandler {
+            backend: backend.clone(),
+            vrings: vrings.clone(),
+        };
+        let vring_worker = Arc::new(VringWorker { epoll_fd });
+        let worker = vring_worker.clone();
+
+        thread::Builder::new()
+            .name("vring_worker".to_string())
+            .spawn(move || vring_worker.run(vring_handler))
+            .map_err(VhostUserHandlerError::SpawnVringWorker)?;
+
+        Ok(VhostUserHandler {
+            backend,
+            worker,
+            owned: false,
+            features_acked: false,
+            acked_features: 0,
+            acked_protocol_features: 0,
+            num_queues,
+            max_queue_size,
+            memory: None,
+            vrings,
+        })
+    }
+
+    fn get_vring_worker(&self) -> Arc<VringWorker> {
+        self.worker.clone()
+    }
+
+    fn vmm_va_to_gpa(&self, vmm_va: u64) -> VhostUserHandlerResult<u64> {
+        if let Some(memory) = &self.memory {
+            for mapping in memory.mappings.iter() {
+                if vmm_va >= mapping.vmm_addr && vmm_va < mapping.vmm_addr + mapping.size {
+                    return Ok(vmm_va - mapping.vmm_addr + mapping.gpa_base);
+                }
+            }
+        }
+
+        Err(VhostUserHandlerError::MissingMemoryMapping)
+    }
+}
+
+impl<S: VhostUserBackend> VhostUserSlaveReqHandler for VhostUserHandler<S> {
+    fn set_owner(&mut self) -> VhostUserResult<()> {
+        if self.owned {
+            return Err(VhostUserError::InvalidOperation);
+        }
+        self.owned = true;
+        Ok(())
+    }
+
+    fn reset_owner(&mut self) -> VhostUserResult<()> {
+        self.owned = false;
+        self.features_acked = false;
+        self.acked_features = 0;
+        self.acked_protocol_features = 0;
+        Ok(())
+    }
+
+    fn get_features(&mut self) -> VhostUserResult<u64> {
+        Ok(self.backend.read().unwrap().features())
+    }
+
+    fn set_features(&mut self, features: u64) -> VhostUserResult<()> {
+        if (features & !self.backend.read().unwrap().features()) != 0 {
+            return Err(VhostUserError::InvalidParam);
+        }
+
+        self.acked_features = features;
+        self.features_acked = true;
+
+        // If VHOST_USER_F_PROTOCOL_FEATURES has not been negotiated,
+        // the ring is initialized in an enabled state.
+        // If VHOST_USER_F_PROTOCOL_FEATURES has been negotiated,
+        // the ring is initialized in a disabled state. Client must not
+        // pass data to/from the backend until ring is enabled by
+        // VHOST_USER_SET_VRING_ENABLE with parameter 1, or after it has
+        // been disabled by VHOST_USER_SET_VRING_ENABLE with parameter 0.
+        let vring_enabled =
+            self.acked_features & VhostUserVirtioFeatures::PROTOCOL_FEATURES.bits() == 0;
+        for vring in self.vrings.iter_mut() {
+            vring.write().unwrap().enabled = vring_enabled;
+        }
+
+        Ok(())
+    }
+
+    fn get_protocol_features(&mut self) -> VhostUserResult<VhostUserProtocolFeatures> {
+        Ok(self.backend.read().unwrap().protocol_features())
+    }
+
+    fn set_protocol_features(&mut self, features: u64) -> VhostUserResult<()> {
+        // Note: slave that reported VHOST_USER_F_PROTOCOL_FEATURES must
+        // support this message even before VHOST_USER_SET_FEATURES was
+        // called.
+        self.acked_protocol_features = features;
+        Ok(())
+    }
+
+    fn set_mem_table(
+        &mut self,
+        ctx: &[VhostUserMemoryRegion],
+        fds: &[RawFd],
+    ) -> VhostUserResult<()> {
+        // We need to create tuple of ranges from the list of VhostUserMemoryRegion
+        // that we get from the caller.
+        let mut regions: Vec<(GuestAddress, usize, Option<FileOffset>)> = Vec::new();
+        let mut mappings: Vec<AddrMapping> = Vec::new();
+
+        for (idx, region) in ctx.iter().enumerate() {
+            let g_addr = GuestAddress(region.guest_phys_addr);
+            let len = region.memory_size as usize;
+            let file = unsafe { File::from_raw_fd(fds[idx]) };
+            let f_off = FileOffset::new(file, region.mmap_offset);
+
+            regions.push((g_addr, len, Some(f_off)));
+            mappings.push(AddrMapping {
+                vmm_addr: region.user_addr,
+                size: region.memory_size,
+                gpa_base: region.guest_phys_addr,
+            });
+        }
+
+        let mem = GuestMemoryMmap::with_files(regions).map_err(|e| {
+            VhostUserError::ReqHandlerError(io::Error::new(io::ErrorKind::Other, e))
+        })?;
+        self.backend
+            .write()
+            .unwrap()
+            .update_memory(mem)
+            .map_err(|e| {
+                VhostUserError::ReqHandlerError(io::Error::new(io::ErrorKind::Other, e))
+            })?;
+        self.memory = Some(Memory { mappings });
+
+        Ok(())
+    }
+
+    fn get_queue_num(&mut self) -> VhostUserResult<u64> {
+        Ok(self.num_queues as u64)
+    }
+
+    fn set_vring_num(&mut self, index: u32, num: u32) -> VhostUserResult<()> {
+        if index as usize >= self.num_queues || num == 0 || num as usize > self.max_queue_size {
+            return Err(VhostUserError::InvalidParam);
+        }
+        self.vrings[index as usize].write().unwrap().queue.size = num as u16;
+        Ok(())
+    }
+
+    fn set_vring_addr(
+        &mut self,
+        index: u32,
+        _flags: VhostUserVringAddrFlags,
+        descriptor: u64,
+        used: u64,
+        available: u64,
+        _log: u64,
+    ) -> VhostUserResult<()> {
+        if index as usize >= self.num_queues {
+            return Err(VhostUserError::InvalidParam);
+        }
+
+        if self.memory.is_some() {
+            let desc_table = self.vmm_va_to_gpa(descriptor).map_err(|e| {
+                VhostUserError::ReqHandlerError(io::Error::new(io::ErrorKind::Other, e))
+            })?;
+            let avail_ring = self.vmm_va_to_gpa(available).map_err(|e| {
+                VhostUserError::ReqHandlerError(io::Error::new(io::ErrorKind::Other, e))
+            })?;
+            let used_ring = self.vmm_va_to_gpa(used).map_err(|e| {
+                VhostUserError::ReqHandlerError(io::Error::new(io::ErrorKind::Other, e))
+            })?;
+            self.vrings[index as usize]
+                .write()
+                .unwrap()
+                .queue
+                .desc_table = GuestAddress(desc_table);
+            self.vrings[index as usize]
+                .write()
+                .unwrap()
+                .queue
+                .avail_ring = GuestAddress(avail_ring);
+            self.vrings[index as usize].write().unwrap().queue.used_ring = GuestAddress(used_ring);
+            Ok(())
+        } else {
+            Err(VhostUserError::InvalidParam)
+        }
+    }
+
+    fn set_vring_base(&mut self, index: u32, base: u32) -> VhostUserResult<()> {
+        self.vrings[index as usize]
+            .write()
+            .unwrap()
+            .queue
+            .next_avail = Wrapping(base as u16);
+        self.vrings[index as usize].write().unwrap().queue.next_used = Wrapping(base as u16);
+        Ok(())
+    }
+
+    fn get_vring_base(&mut self, index: u32) -> VhostUserResult<VhostUserVringState> {
+        if index as usize >= self.num_queues {
+            return Err(VhostUserError::InvalidParam);
+        }
+        // Quote from vhost-user specification:
+        // Client must start ring upon receiving a kick (that is, detecting
+        // that file descriptor is readable) on the descriptor specified by
+        // VHOST_USER_SET_VRING_KICK, and stop ring upon receiving
+        // VHOST_USER_GET_VRING_BASE.
+        self.vrings[index as usize].write().unwrap().queue.ready = false;
+        if let Some(fd) = self.vrings[index as usize].read().unwrap().kick.as_ref() {
+            self.worker
+                .unregister_listener(fd.as_raw_fd(), epoll::Events::EPOLLIN, u64::from(index))
+                .map_err(VhostUserError::ReqHandlerError)?;
+        }
+
+        let next_avail = self.vrings[index as usize]
+            .read()
+            .unwrap()
+            .queue
+            .next_avail
+            .0 as u16;
+
+        Ok(VhostUserVringState::new(index, u32::from(next_avail)))
+    }
+
+    fn set_vring_kick(&mut self, index: u8, fd: Option<RawFd>) -> VhostUserResult<()> {
+        if index as usize >= self.num_queues {
+            return Err(VhostUserError::InvalidParam);
+        }
+
+        if let Some(kick) = self.vrings[index as usize].write().unwrap().kick.take() {
+            // Close file descriptor set by previous operations.
+            let _ = unsafe { libc::close(kick.as_raw_fd()) };
+        }
+        self.vrings[index as usize].write().unwrap().kick =
+            fd.map(|x| unsafe { EventFd::from_raw_fd(x) });
+
+        // Quote from vhost-user specification:
+        // Client must start ring upon receiving a kick (that is, detecting
+        // that file descriptor is readable) on the descriptor specified by
+        // VHOST_USER_SET_VRING_KICK, and stop ring upon receiving
+        // VHOST_USER_GET_VRING_BASE.
+        self.vrings[index as usize].write().unwrap().queue.ready = true;
+        if let Some(fd) = self.vrings[index as usize].read().unwrap().kick.as_ref() {
+            self.worker
+                .register_listener(fd.as_raw_fd(), epoll::Events::EPOLLIN, u64::from(index))
+                .map_err(VhostUserError::ReqHandlerError)?;
+        }
+
+        Ok(())
+    }
+
+    fn set_vring_call(&mut self, index: u8, fd: Option<RawFd>) -> VhostUserResult<()> {
+        if index as usize >= self.num_queues {
+            return Err(VhostUserError::InvalidParam);
+        }
+
+        if let Some(call) = self.vrings[index as usize].write().unwrap().call.take() {
+            // Close file descriptor set by previous operations.
+            let _ = unsafe { libc::close(call.as_raw_fd()) };
+        }
+        self.vrings[index as usize].write().unwrap().call =
+            fd.map(|x| unsafe { EventFd::from_raw_fd(x) });
+
+        Ok(())
+    }
+
+    fn set_vring_err(&mut self, index: u8, fd: Option<RawFd>) -> VhostUserResult<()> {
+        if index as usize >= self.num_queues {
+            return Err(VhostUserError::InvalidParam);
+        }
+
+        if let Some(err) = self.vrings[index as usize].write().unwrap().err.take() {
+            // Close file descriptor set by previous operations.
+            let _ = unsafe { libc::close(err.as_raw_fd()) };
+        }
+        self.vrings[index as usize].write().unwrap().err =
+            fd.map(|x| unsafe { EventFd::from_raw_fd(x) });
+
+        Ok(())
+    }
+
+    fn set_vring_enable(&mut self, index: u32, enable: bool) -> VhostUserResult<()> {
+        // This request should be handled only when VHOST_USER_F_PROTOCOL_FEATURES
+        // has been negotiated.
+        if self.acked_features & VhostUserVirtioFeatures::PROTOCOL_FEATURES.bits() == 0 {
+            return Err(VhostUserError::InvalidOperation);
+        } else if index as usize >= self.num_queues {
+            return Err(VhostUserError::InvalidParam);
+        }
+
+        // Slave must not pass data to/from the backend until ring is
+        // enabled by VHOST_USER_SET_VRING_ENABLE with parameter 1,
+        // or after it has been disabled by VHOST_USER_SET_VRING_ENABLE
+        // with parameter 0.
+        self.vrings[index as usize].write().unwrap().enabled = enable;
+
+        Ok(())
+    }
+
+    fn get_config(
+        &mut self,
+        offset: u32,
+        size: u32,
+        _flags: VhostUserConfigFlags,
+    ) -> VhostUserResult<Vec<u8>> {
+        Ok(self.backend.read().unwrap().get_config(offset, size))
+    }
+
+    fn set_config(
+        &mut self,
+        offset: u32,
+        buf: &[u8],
+        _flags: VhostUserConfigFlags,
+    ) -> VhostUserResult<()> {
+        self.backend
+            .write()
+            .unwrap()
+            .set_config(offset, buf)
+            .map_err(VhostUserError::ReqHandlerError)
+    }
+}

vhost_user_fs/Cargo.toml

@@ -0,0 +1,12 @@
+[package]
+name = "vhost_user_fs"
+version = "0.1.0"
+authors = ["The Cloud Hypervisor Authors"]
+edition = "2018"
+
+[dependencies]
+bitflags = "1.1.0"
+libc = "0.2.65"
+log = "0.4.8"
+vm-memory = { git = "https://github.com/rust-vmm/vm-memory" }
+vm-virtio = { path = "../vm-virtio" }

vhost_user_fs/src/descriptor_utils.rs

@@ -0,0 +1,985 @@
+// Copyright 2019 The Chromium OS Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+use std::cmp;
+use std::collections::VecDeque;
+use std::fmt::{self, Display};
+use std::io::{self, Read, Write};
+use std::mem::{size_of, MaybeUninit};
+use std::ops::Deref;
+use std::ptr::copy_nonoverlapping;
+use std::result;
+
+use vm_memory::{
+    Address, ByteValued, Bytes, GuestAddress, GuestMemory, GuestMemoryError, GuestMemoryMmap,
+    GuestMemoryRegion, Le16, Le32, Le64, VolatileMemory, VolatileMemoryError, VolatileSlice,
+};
+use vm_virtio::DescriptorChain;
+
+use crate::file_traits::{FileReadWriteAtVolatile, FileReadWriteVolatile};
+
+#[derive(Debug)]
+pub enum Error {
+    DescriptorChainOverflow,
+    FindMemoryRegion,
+    GuestMemoryError(GuestMemoryError),
+    InvalidChain,
+    IoError(io::Error),
+    SplitOutOfBounds(usize),
+    VolatileMemoryError(VolatileMemoryError),
+}
+
+impl Display for Error {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        use self::Error::*;
+
+        match self {
+            DescriptorChainOverflow => write!(
+                f,
+                "the combined length of all the buffers in a `DescriptorChain` would overflow"
+            ),
+            FindMemoryRegion => write!(f, "no memory region for this address range"),
+            GuestMemoryError(e) => write!(f, "descriptor guest memory error: {}", e),
+            InvalidChain => write!(f, "invalid descriptor chain"),
+            IoError(e) => write!(f, "descriptor I/O error: {}", e),
+            SplitOutOfBounds(off) => write!(f, "`DescriptorChain` split is out of bounds: {}", off),
+            VolatileMemoryError(e) => write!(f, "volatile memory error: {}", e),
+        }
+    }
+}
+
+pub type Result<T> = result::Result<T, Error>;
+
+impl std::error::Error for Error {}
+
+#[derive(Clone)]
+struct DescriptorChainConsumer<'a> {
+    buffers: VecDeque<VolatileSlice<'a>>,
+    bytes_consumed: usize,
+}
+
+impl<'a> DescriptorChainConsumer<'a> {
+    fn available_bytes(&self) -> usize {
+        // This is guaranteed not to overflow because the total length of the chain
+        // is checked during all creations of `DescriptorChainConsumer` (see
+        // `Reader::new()` and `Writer::new()`).
+        self.buffers
+            .iter()
+            .fold(0usize, |count, vs| count + vs.len() as usize)
+    }
+
+    fn bytes_consumed(&self) -> usize {
+        self.bytes_consumed
+    }
+
+    /// Consumes at most `count` bytes from the `DescriptorChain`. Callers must provide a function
+    /// that takes a `&[VolatileSlice]` and returns the total number of bytes consumed. This
+    /// function guarantees that the combined length of all the slices in the `&[VolatileSlice]` is
+    /// less than or equal to `count`.
+    ///
+    /// # Errors
+    ///
+    /// If the provided function returns any error then no bytes are consumed from the buffer and
+    /// the error is returned to the caller.
+    fn consume<F>(&mut self, count: usize, f: F) -> io::Result<usize>
+    where
+        F: FnOnce(&[VolatileSlice]) -> io::Result<usize>,
+    {
+        let mut buflen = 0;
+        let mut bufs = Vec::with_capacity(self.buffers.len());
+        for &vs in &self.buffers {
+            if buflen >= count {
+                break;
+            }
+
+            bufs.push(vs);
+
+            let rem = count - buflen;
+            if rem < vs.len() {
+                buflen += rem;
+            } else {
+                buflen += vs.len() as usize;
+            }
+        }
+
+        if bufs.is_empty() {
+            return Ok(0);
+        }
+
+        let bytes_consumed = f(&*bufs)?;
+
+        // This can happen if a driver tricks a device into reading/writing more data than
+        // fits in a `usize`.
+        let total_bytes_consumed =
+            self.bytes_consumed
+                .checked_add(bytes_consumed)
+                .ok_or_else(|| {
+                    io::Error::new(io::ErrorKind::InvalidData, Error::DescriptorChainOverflow)
+                })?;
+
+        let mut rem = bytes_consumed;
+        while let Some(vs) = self.buffers.pop_front() {
+            if rem < vs.len() {
+                // Split the slice and push the remainder back into the buffer list. Safe because we
+                // know that `rem` is not out of bounds due to the check and we checked the bounds
+                // on `vs` when we added it to the buffer list.
+                self.buffers.push_front(vs.offset(rem).unwrap());
+                break;
+            }
+
+            // No need for checked math because we know that `vs.size() <= rem`.
+            rem -= vs.len();
+        }
+
+        self.bytes_consumed = total_bytes_consumed;
+
+        Ok(bytes_consumed)
+    }
+
+    fn split_at(&mut self, offset: usize) -> Result<DescriptorChainConsumer<'a>> {
+        let mut rem = offset;
+        let pos = self.buffers.iter().position(|vs| {
+            if rem < vs.len() {
+                true
+            } else {
+                rem -= vs.len();
+                false
+            }
+        });
+
+        if let Some(at) = pos {
+            let mut other = self.buffers.split_off(at);
+
+            if rem > 0 {
+                // There must be at least one element in `other` because we checked
+                // its `size` value in the call to `position` above.
+                let front = other.pop_front().expect("empty VecDeque after split");
+                self.buffers
+                    .push_back(front.offset(rem).map_err(Error::VolatileMemoryError)?);
+                other.push_front(front.offset(rem).map_err(Error::VolatileMemoryError)?);
+            }
+
+            Ok(DescriptorChainConsumer {
+                buffers: other,
+                bytes_consumed: 0,
+            })
+        } else if rem == 0 {
+            Ok(DescriptorChainConsumer {
+                buffers: VecDeque::new(),
+                bytes_consumed: 0,
+            })
+        } else {
+            Err(Error::SplitOutOfBounds(offset))
+        }
+    }
+}
+
+/// Provides high-level interface over the sequence of memory regions
+/// defined by readable descriptors in the descriptor chain.
+///
+/// Note that virtio spec requires driver to place any device-writable
+/// descriptors after any device-readable descriptors (2.6.4.2 in Virtio Spec v1.1).
+/// Reader will skip iterating over descriptor chain when first writable
+/// descriptor is encountered.
+#[derive(Clone)]
+pub struct Reader<'a> {
+    buffer: DescriptorChainConsumer<'a>,
+}
+
+impl<'a> Reader<'a> {
+    /// Construct a new Reader wrapper over `desc_chain`.
+    pub fn new(mem: &'a GuestMemoryMmap, desc_chain: DescriptorChain<'a>) -> Result<Reader<'a>> {
+        let mut total_len: usize = 0;
+        let buffers = desc_chain
+            .into_iter()
+            .readable()
+            .map(|desc| {
+                // Verify that summing the descriptor sizes does not overflow.
+                // This can happen if a driver tricks a device into reading more data than
+                // fits in a `usize`.
+                total_len = total_len
+                    .checked_add(desc.len as usize)
+                    .ok_or(Error::DescriptorChainOverflow)?;
+
+                let region = mem.find_region(desc.addr).ok_or(Error::FindMemoryRegion)?;
+                let offset = desc
+                    .addr
+                    .checked_sub(region.start_addr().raw_value())
+                    .unwrap();
+                region
+                    .deref()
+                    .get_slice(offset.raw_value() as usize, desc.len as usize)
+                    .map_err(Error::VolatileMemoryError)
+            })
+            .collect::<Result<VecDeque<VolatileSlice<'a>>>>()?;
+        Ok(Reader {
+            buffer: DescriptorChainConsumer {
+                buffers,
+                bytes_consumed: 0,
+            },
+        })
+    }
+
+    /// Reads an object from the descriptor chain buffer.
+    pub fn read_obj<T: ByteValued>(&mut self) -> io::Result<T> {
+        let mut obj = MaybeUninit::<T>::uninit();
+
+        // Safe because `MaybeUninit` guarantees that the pointer is valid for
+        // `size_of::<T>()` bytes.
+        let buf = unsafe {
+            ::std::slice::from_raw_parts_mut(obj.as_mut_ptr() as *mut u8, size_of::<T>())
+        };
+
+        self.read_exact(buf)?;
+
+        // Safe because any type that implements `ByteValued` can be considered initialized
+        // even if it is filled with random data.
+        Ok(unsafe { obj.assume_init() })
+    }
+
+    /// Reads data from the descriptor chain buffer into a file descriptor.
+    /// Returns the number of bytes read from the descriptor chain buffer.
+    /// The number of bytes read can be less than `count` if there isn't
+    /// enough data in the descriptor chain buffer.
+    pub fn read_to<F: FileReadWriteVolatile>(
+        &mut self,
+        mut dst: F,
+        count: usize,
+    ) -> io::Result<usize> {
+        self.buffer
+            .consume(count, |bufs| dst.write_vectored_volatile(bufs))
+    }
+
+    /// Reads data from the descriptor chain buffer into a File at offset `off`.
+    /// Returns the number of bytes read from the descriptor chain buffer.
+    /// The number of bytes read can be less than `count` if there isn't
+    /// enough data in the descriptor chain buffer.
+    pub fn read_to_at<F: FileReadWriteAtVolatile>(
+        &mut self,
+        mut dst: F,
+        count: usize,
+        off: u64,
+    ) -> io::Result<usize> {
+        self.buffer
+            .consume(count, |bufs| dst.write_vectored_at_volatile(bufs, off))
+    }
+
+    pub fn read_exact_to<F: FileReadWriteVolatile>(
+        &mut self,
+        mut dst: F,
+        mut count: usize,
+    ) -> io::Result<()> {
+        while count > 0 {
+            match self.read_to(&mut dst, count) {
+                Ok(0) => {
+                    return Err(io::Error::new(
+                        io::ErrorKind::UnexpectedEof,
+                        "failed to fill whole buffer",
+                    ))
+                }
+                Ok(n) => count -= n,
+                Err(ref e) if e.kind() == io::ErrorKind::Interrupted => {}
+                Err(e) => return Err(e),
+            }
+        }
+
+        Ok(())
+    }
+
+    /// Returns number of bytes available for reading.  May return an error if the combined
+    /// lengths of all the buffers in the DescriptorChain would cause an integer overflow.
+    pub fn available_bytes(&self) -> usize {
+        self.buffer.available_bytes()
+    }
+
+    /// Returns number of bytes already read from the descriptor chain buffer.
+    pub fn bytes_read(&self) -> usize {
+        self.buffer.bytes_consumed()
+    }
+
+    /// Splits this `Reader` into two at the given offset in the `DescriptorChain` buffer.
+    /// After the split, `self` will be able to read up to `offset` bytes while the returned
+    /// `Reader` can read up to `available_bytes() - offset` bytes.  Returns an error if
+    /// `offset > self.available_bytes()`.
+    pub fn split_at(&mut self, offset: usize) -> Result<Reader<'a>> {
+        self.buffer.split_at(offset).map(|buffer| Reader { buffer })
+    }
+}
+
+impl<'a> io::Read for Reader<'a> {
+    fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
+        self.buffer.consume(buf.len(), |bufs| {
+            let mut rem = buf;
+            let mut total = 0;
+            for vs in bufs {
+                let copy_len = cmp::min(rem.len(), vs.len());
+
+                // Safe because we have already verified that `vs` points to valid memory.
+                unsafe {
+                    copy_nonoverlapping(vs.as_ptr() as *const u8, rem.as_mut_ptr(), copy_len);
+                }
+                rem = &mut rem[copy_len..];
+                total += copy_len;
+            }
+            Ok(total)
+        })
+    }
+}
+
+/// Provides high-level interface over the sequence of memory regions
+/// defined by writable descriptors in the descriptor chain.
+///
+/// Note that virtio spec requires driver to place any device-writable
+/// descriptors after any device-readable descriptors (2.6.4.2 in Virtio Spec v1.1).
+/// Writer will start iterating the descriptors from the first writable one and will
+/// assume that all following descriptors are writable.
+#[derive(Clone)]
+pub struct Writer<'a> {
+    buffer: DescriptorChainConsumer<'a>,
+}
+
+impl<'a> Writer<'a> {
+    /// Construct a new Writer wrapper over `desc_chain`.
+    pub fn new(mem: &'a GuestMemoryMmap, desc_chain: DescriptorChain<'a>) -> Result<Writer<'a>> {
+        let mut total_len: usize = 0;
+        let buffers = desc_chain
+            .into_iter()
+            .writable()
+            .map(|desc| {
+                // Verify that summing the descriptor sizes does not overflow.
+                // This can happen if a driver tricks a device into writing more data than
+                // fits in a `usize`.
+                total_len = total_len
+                    .checked_add(desc.len as usize)
+                    .ok_or(Error::DescriptorChainOverflow)?;
+
+                let region = mem.find_region(desc.addr).ok_or(Error::FindMemoryRegion)?;
+                let offset = desc
+                    .addr
+                    .checked_sub(region.start_addr().raw_value())
+                    .unwrap();
+                region
+                    .deref()
+                    .get_slice(offset.raw_value() as usize, desc.len as usize)
+                    .map_err(Error::VolatileMemoryError)
+            })
+            .collect::<Result<VecDeque<VolatileSlice<'a>>>>()?;
+
+        Ok(Writer {
+            buffer: DescriptorChainConsumer {
+                buffers,
+                bytes_consumed: 0,
+            },
+        })
+    }
+
+    /// Writes an object to the descriptor chain buffer.
+    pub fn write_obj<T: ByteValued>(&mut self, val: T) -> io::Result<()> {
+        self.write_all(val.as_slice())
+    }
+
+    /// Returns number of bytes available for writing.  May return an error if the combined
+    /// lengths of all the buffers in the DescriptorChain would cause an overflow.
+    pub fn available_bytes(&self) -> usize {
+        self.buffer.available_bytes()
+    }
+
+    /// Writes data to the descriptor chain buffer from a file descriptor.
+    /// Returns the number of bytes written to the descriptor chain buffer.
+    /// The number of bytes written can be less than `count` if
+    /// there isn't enough data in the descriptor chain buffer.
+    pub fn write_from<F: FileReadWriteVolatile>(
+        &mut self,
+        mut src: F,
+        count: usize,
+    ) -> io::Result<usize> {
+        self.buffer
+            .consume(count, |bufs| src.read_vectored_volatile(bufs))
+    }
+
+    /// Writes data to the descriptor chain buffer from a File at offset `off`.
+    /// Returns the number of bytes written to the descriptor chain buffer.
+    /// The number of bytes written can be less than `count` if
+    /// there isn't enough data in the descriptor chain buffer.
+    pub fn write_from_at<F: FileReadWriteAtVolatile>(
+        &mut self,
+        mut src: F,
+        count: usize,
+        off: u64,
+    ) -> io::Result<usize> {
+        self.buffer
+            .consume(count, |bufs| src.read_vectored_at_volatile(bufs, off))
+    }
+
+    pub fn write_all_from<F: FileReadWriteVolatile>(
+        &mut self,
+        mut src: F,
+        mut count: usize,
+    ) -> io::Result<()> {
+        while count > 0 {
+            match self.write_from(&mut src, count) {
+                Ok(0) => {
+                    return Err(io::Error::new(
+                        io::ErrorKind::WriteZero,
+                        "failed to write whole buffer",
+                    ))
+                }
+                Ok(n) => count -= n,
+                Err(ref e) if e.kind() == io::ErrorKind::Interrupted => {}
+                Err(e) => return Err(e),
+            }
+        }
+
+        Ok(())
+    }
+
+    /// Returns number of bytes already written to the descriptor chain buffer.
+    pub fn bytes_written(&self) -> usize {
+        self.buffer.bytes_consumed()
+    }
+
+    /// Splits this `Writer` into two at the given offset in the `DescriptorChain` buffer.
+    /// After the split, `self` will be able to write up to `offset` bytes while the returned
+    /// `Writer` can write up to `available_bytes() - offset` bytes.  Returns an error if
+    /// `offset > self.available_bytes()`.
+    pub fn split_at(&mut self, offset: usize) -> Result<Writer<'a>> {
+        self.buffer.split_at(offset).map(|buffer| Writer { buffer })
+    }
+}
+
+impl<'a> io::Write for Writer<'a> {
+    fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
+        self.buffer.consume(buf.len(), |bufs| {
+            let mut rem = buf;
+            let mut total = 0;
+            for vs in bufs {
+                let copy_len = cmp::min(rem.len(), vs.len());
+
+                // Safe because we have already verified that `vs` points to valid memory.
+                unsafe {
+                    copy_nonoverlapping(rem.as_ptr(), vs.as_ptr(), copy_len);
+                }
+                rem = &rem[copy_len..];
+                total += copy_len;
+            }
+            Ok(total)
+        })
+    }
+
+    fn flush(&mut self) -> io::Result<()> {
+        // Nothing to flush since the writes go straight into the buffer.
+        Ok(())
+    }
+}
+
+const VIRTQ_DESC_F_NEXT: u16 = 0x1;
+const VIRTQ_DESC_F_WRITE: u16 = 0x2;
+
+#[derive(Copy, Clone, PartialEq, Eq)]
+pub enum DescriptorType {
+    Readable,
+    Writable,
+}
+
+#[derive(Copy, Clone, Debug, Default)]
+#[repr(C)]
+struct virtq_desc {
+    addr: Le64,
+    len: Le32,
+    flags: Le16,
+    next: Le16,
+}
+
+// Safe because it only has data and has no implicit padding.
+unsafe impl ByteValued for virtq_desc {}
+
+/// Test utility function to create a descriptor chain in guest memory.
+pub fn create_descriptor_chain(
+    memory: &GuestMemoryMmap,
+    descriptor_array_addr: GuestAddress,
+    mut buffers_start_addr: GuestAddress,
+    descriptors: Vec<(DescriptorType, u32)>,
+    spaces_between_regions: u32,
+) -> Result<DescriptorChain> {
+    let descriptors_len = descriptors.len();
+    for (index, (type_, size)) in descriptors.into_iter().enumerate() {
+        let mut flags = 0;
+        if let DescriptorType::Writable = type_ {
+            flags |= VIRTQ_DESC_F_WRITE;
+        }
+        if index + 1 < descriptors_len {
+            flags |= VIRTQ_DESC_F_NEXT;
+        }
+
+        let index = index as u16;
+        let desc = virtq_desc {
+            addr: buffers_start_addr.raw_value().into(),
+            len: size.into(),
+            flags: flags.into(),
+            next: (index + 1).into(),
+        };
+
+        let offset = size + spaces_between_regions;
+        buffers_start_addr = buffers_start_addr
+            .checked_add(u64::from(offset))
+            .ok_or(Error::InvalidChain)?;
+
+        let _ = memory.write_obj(
+            desc,
+            descriptor_array_addr
+                .checked_add(u64::from(index) * std::mem::size_of::<virtq_desc>() as u64)
+                .ok_or(Error::InvalidChain)?,
+        );
+    }
+
+    DescriptorChain::checked_new(memory, descriptor_array_addr, 0x100, 0, None)
+        .ok_or(Error::InvalidChain)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn reader_test_simple_chain() {
+        use DescriptorType::*;
+
+        let memory_start_addr = GuestAddress(0x0);
+        let memory = GuestMemoryMmap::new(&vec![(memory_start_addr, 0x10000)]).unwrap();
+
+        let chain = create_descriptor_chain(
+            &memory,
+            GuestAddress(0x0),
+            GuestAddress(0x100),
+            vec![
+                (Readable, 8),
+                (Readable, 16),
+                (Readable, 18),
+                (Readable, 64),
+            ],
+            0,
+        )
+        .expect("create_descriptor_chain failed");
+        let mut reader = Reader::new(&memory, chain).expect("failed to create Reader");
+        assert_eq!(reader.available_bytes(), 106);
+        assert_eq!(reader.bytes_read(), 0);
+
+        let mut buffer = [0 as u8; 64];
+        if let Err(_) = reader.read_exact(&mut buffer) {
+            panic!("read_exact should not fail here");
+        }
+
+        assert_eq!(reader.available_bytes(), 42);
+        assert_eq!(reader.bytes_read(), 64);
+
+        match reader.read(&mut buffer) {
+            Err(_) => panic!("read should not fail here"),
+            Ok(length) => assert_eq!(length, 42),
+        }
+
+        assert_eq!(reader.available_bytes(), 0);
+        assert_eq!(reader.bytes_read(), 106);
+    }
+
+    #[test]
+    fn writer_test_simple_chain() {
+        use DescriptorType::*;
+
+        let memory_start_addr = GuestAddress(0x0);
+        let memory = GuestMemoryMmap::new(&vec![(memory_start_addr, 0x10000)]).unwrap();
+
+        let chain = create_descriptor_chain(
+            &memory,
+            GuestAddress(0x0),
+            GuestAddress(0x100),
+            vec![
+                (Writable, 8),
+                (Writable, 16),
+                (Writable, 18),
+                (Writable, 64),
+            ],
+            0,
+        )
+        .expect("create_descriptor_chain failed");
+        let mut writer = Writer::new(&memory, chain).expect("failed to create Writer");
+        assert_eq!(writer.available_bytes(), 106);
+        assert_eq!(writer.bytes_written(), 0);
+
+        let mut buffer = [0 as u8; 64];
+        if let Err(_) = writer.write_all(&mut buffer) {
+            panic!("write_all should not fail here");
+        }
+
+        assert_eq!(writer.available_bytes(), 42);
+        assert_eq!(writer.bytes_written(), 64);
+
+        match writer.write(&mut buffer) {
+            Err(_) => panic!("write should not fail here"),
+            Ok(length) => assert_eq!(length, 42),
+        }
+
+        assert_eq!(writer.available_bytes(), 0);
+        assert_eq!(writer.bytes_written(), 106);
+    }
+
+    #[test]
+    fn reader_test_incompatible_chain() {
+        use DescriptorType::*;
+
+        let memory_start_addr = GuestAddress(0x0);
+        let memory = GuestMemoryMmap::new(&vec![(memory_start_addr, 0x10000)]).unwrap();
+
+        let chain = create_descriptor_chain(
+            &memory,
+            GuestAddress(0x0),
+            GuestAddress(0x100),
+            vec![(Writable, 8)],
+            0,
+        )
+        .expect("create_descriptor_chain failed");
+        let mut reader = Reader::new(&memory, chain).expect("failed to create Reader");
+        assert_eq!(reader.available_bytes(), 0);
+        assert_eq!(reader.bytes_read(), 0);
+
+        assert!(reader.read_obj::<u8>().is_err());
+
+        assert_eq!(reader.available_bytes(), 0);
+        assert_eq!(reader.bytes_read(), 0);
+    }
+
+    #[test]
+    fn writer_test_incompatible_chain() {
+        use DescriptorType::*;
+
+        let memory_start_addr = GuestAddress(0x0);
+        let memory = GuestMemoryMmap::new(&vec![(memory_start_addr, 0x10000)]).unwrap();
+
+        let chain = create_descriptor_chain(
+            &memory,
+            GuestAddress(0x0),
+            GuestAddress(0x100),
+            vec![(Readable, 8)],
+            0,
+        )
+        .expect("create_descriptor_chain failed");
+        let mut writer = Writer::new(&memory, chain).expect("failed to create Writer");
+        assert_eq!(writer.available_bytes(), 0);
+        assert_eq!(writer.bytes_written(), 0);
+
+        assert!(writer.write_obj(0u8).is_err());
+
+        assert_eq!(writer.available_bytes(), 0);
+        assert_eq!(writer.bytes_written(), 0);
+    }
+
+    #[test]
+    fn reader_writer_shared_chain() {
+        use DescriptorType::*;
+
+        let memory_start_addr = GuestAddress(0x0);
+        let memory = GuestMemoryMmap::new(&vec![(memory_start_addr, 0x10000)]).unwrap();
+
+        let chain = create_descriptor_chain(
+            &memory,
+            GuestAddress(0x0),
+            GuestAddress(0x100),
+            vec![
+                (Readable, 16),
+                (Readable, 16),
+                (Readable, 96),
+                (Writable, 64),
+                (Writable, 1),
+                (Writable, 3),
+            ],
+            0,
+        )
+        .expect("create_descriptor_chain failed");
+        let mut reader = Reader::new(&memory, chain.clone()).expect("failed to create Reader");
+        let mut writer = Writer::new(&memory, chain).expect("failed to create Writer");
+
+        assert_eq!(reader.bytes_read(), 0);
+        assert_eq!(writer.bytes_written(), 0);
+
+        let mut buffer = Vec::with_capacity(200);
+
+        assert_eq!(
+            reader
+                .read_to_end(&mut buffer)
+                .expect("read should not fail here"),
+            128
+        );
+
+        // The writable descriptors are only 68 bytes long.
+        writer
+            .write_all(&buffer[..68])
+            .expect("write should not fail here");
+
+        assert_eq!(reader.available_bytes(), 0);
+        assert_eq!(reader.bytes_read(), 128);
+        assert_eq!(writer.available_bytes(), 0);
+        assert_eq!(writer.bytes_written(), 68);
+    }
+
+    #[test]
+    fn reader_writer_shattered_object() {
+        use DescriptorType::*;
+
+        let memory_start_addr = GuestAddress(0x0);
+        let memory = GuestMemoryMmap::new(&vec![(memory_start_addr, 0x10000)]).unwrap();
+
+        let secret: Le32 = 0x12345678.into();
+
+        // Create a descriptor chain with memory regions that are properly separated.
+        let chain_writer = create_descriptor_chain(
+            &memory,
+            GuestAddress(0x0),
+            GuestAddress(0x100),
+            vec![(Writable, 1), (Writable, 1), (Writable, 1), (Writable, 1)],
+            123,
+        )
+        .expect("create_descriptor_chain failed");
+        let mut writer = Writer::new(&memory, chain_writer).expect("failed to create Writer");
+        if let Err(_) = writer.write_obj(secret) {
+            panic!("write_obj should not fail here");
+        }
+
+        // Now create new descriptor chain pointing to the same memory and try to read it.
+        let chain_reader = create_descriptor_chain(
+            &memory,
+            GuestAddress(0x0),
+            GuestAddress(0x100),
+            vec![(Readable, 1), (Readable, 1), (Readable, 1), (Readable, 1)],
+            123,
+        )
+        .expect("create_descriptor_chain failed");
+        let mut reader = Reader::new(&memory, chain_reader).expect("failed to create Reader");
+        match reader.read_obj::<Le32>() {
+            Err(_) => panic!("read_obj should not fail here"),
+            Ok(read_secret) => assert_eq!(read_secret, secret),
+        }
+    }
+
+    #[test]
+    fn reader_unexpected_eof() {
+        use DescriptorType::*;
+
+        let memory_start_addr = GuestAddress(0x0);
+        let memory = GuestMemoryMmap::new(&vec![(memory_start_addr, 0x10000)]).unwrap();
+
+        let chain = create_descriptor_chain(
+            &memory,
+            GuestAddress(0x0),
+            GuestAddress(0x100),
+            vec![(Readable, 256), (Readable, 256)],
+            0,
+        )
+        .expect("create_descriptor_chain failed");
+
+        let mut reader = Reader::new(&memory, chain).expect("failed to create Reader");
+
+        let mut buf = Vec::with_capacity(1024);
+        buf.resize(1024, 0);
+
+        assert_eq!(
+            reader
+                .read_exact(&mut buf[..])
+                .expect_err("read more bytes than available")
+                .kind(),
+            io::ErrorKind::UnexpectedEof
+        );
+    }
+
+    #[test]
+    fn split_border() {
+        use DescriptorType::*;
+
+        let memory_start_addr = GuestAddress(0x0);
+        let memory = GuestMemoryMmap::new(&vec![(memory_start_addr, 0x10000)]).unwrap();
+
+        let chain = create_descriptor_chain(
+            &memory,
+            GuestAddress(0x0),
+            GuestAddress(0x100),
+            vec![
+                (Readable, 16),
+                (Readable, 16),
+                (Readable, 96),
+                (Writable, 64),
+                (Writable, 1),
+                (Writable, 3),
+            ],
+            0,
+        )
+        .expect("create_descriptor_chain failed");
+        let mut reader = Reader::new(&memory, chain).expect("failed to create Reader");
+
+        let other = reader.split_at(32).expect("failed to split Reader");
+        assert_eq!(reader.available_bytes(), 32);
+        assert_eq!(other.available_bytes(), 96);
+    }
+
+    #[test]
+    fn split_middle() {
+        use DescriptorType::*;
+
+        let memory_start_addr = GuestAddress(0x0);
+        let memory = GuestMemoryMmap::new(&vec![(memory_start_addr, 0x10000)]).unwrap();
+
+        let chain = create_descriptor_chain(
+            &memory,
+            GuestAddress(0x0),
+            GuestAddress(0x100),
+            vec![
+                (Readable, 16),
+                (Readable, 16),
+                (Readable, 96),
+                (Writable, 64),
+                (Writable, 1),
+                (Writable, 3),
+            ],
+            0,
+        )
+        .expect("create_descriptor_chain failed");
+        let mut reader = Reader::new(&memory, chain).expect("failed to create Reader");
+
+        let other = reader.split_at(24).expect("failed to split Reader");
+        assert_eq!(reader.available_bytes(), 24);
+        assert_eq!(other.available_bytes(), 104);
+    }
+
+    #[test]
+    fn split_end() {
+        use DescriptorType::*;
+
+        let memory_start_addr = GuestAddress(0x0);
+        let memory = GuestMemoryMmap::new(&vec![(memory_start_addr, 0x10000)]).unwrap();
+
+        let chain = create_descriptor_chain(
+            &memory,
+            GuestAddress(0x0),
+            GuestAddress(0x100),
+            vec![
+                (Readable, 16),
+                (Readable, 16),
+                (Readable, 96),
+                (Writable, 64),
+                (Writable, 1),
+                (Writable, 3),
+            ],
+            0,
+        )
+        .expect("create_descriptor_chain failed");
+        let mut reader = Reader::new(&memory, chain).expect("failed to create Reader");
+
+        let other = reader.split_at(128).expect("failed to split Reader");
+        assert_eq!(reader.available_bytes(), 128);
+        assert_eq!(other.available_bytes(), 0);
+    }
+
+    #[test]
+    fn split_beginning() {
+        use DescriptorType::*;
+
+        let memory_start_addr = GuestAddress(0x0);
+        let memory = GuestMemoryMmap::new(&vec![(memory_start_addr, 0x10000)]).unwrap();
+
+        let chain = create_descriptor_chain(
+            &memory,
+            GuestAddress(0x0),
+            GuestAddress(0x100),
+            vec![
+                (Readable, 16),
+                (Readable, 16),
+                (Readable, 96),
+                (Writable, 64),
+                (Writable, 1),
+                (Writable, 3),
+            ],
+            0,
+        )
+        .expect("create_descriptor_chain failed");
+        let mut reader = Reader::new(&memory, chain).expect("failed to create Reader");
+
+        let other = reader.split_at(0).expect("failed to split Reader");
+        assert_eq!(reader.available_bytes(), 0);
+        assert_eq!(other.available_bytes(), 128);
+    }
+
+    #[test]
+    fn split_outofbounds() {
+        use DescriptorType::*;
+
+        let memory_start_addr = GuestAddress(0x0);
+        let memory = GuestMemoryMmap::new(&vec![(memory_start_addr, 0x10000)]).unwrap();
+
+        let chain = create_descriptor_chain(
+            &memory,
+            GuestAddress(0x0),
+            GuestAddress(0x100),
+            vec![
+                (Readable, 16),
+                (Readable, 16),
+                (Readable, 96),
+                (Writable, 64),
+                (Writable, 1),
+                (Writable, 3),
+            ],
+            0,
+        )
+        .expect("create_descriptor_chain failed");
+        let mut reader = Reader::new(&memory, chain).expect("failed to create Reader");
+
+        if let Ok(_) = reader.split_at(256) {
+            panic!("successfully split Reader with out of bounds offset");
+        }
+    }
+
+    #[test]
+    fn read_full() {
+        use DescriptorType::*;
+
+        let memory_start_addr = GuestAddress(0x0);
+        let memory = GuestMemoryMmap::new(&vec![(memory_start_addr, 0x10000)]).unwrap();
+
+        let chain = create_descriptor_chain(
+            &memory,
+            GuestAddress(0x0),
+            GuestAddress(0x100),
+            vec![(Readable, 16), (Readable, 16), (Readable, 16)],
+            0,
+        )
+        .expect("create_descriptor_chain failed");
+        let mut reader = Reader::new(&memory, chain).expect("failed to create Reader");
+
+        let mut buf = vec![0u8; 64];
+        assert_eq!(
+            reader.read(&mut buf[..]).expect("failed to read to buffer"),
+            48
+        );
+    }
+
+    #[test]
+    fn write_full() {
+        use DescriptorType::*;
+
+        let memory_start_addr = GuestAddress(0x0);
+        let memory = GuestMemoryMmap::new(&vec![(memory_start_addr, 0x10000)]).unwrap();
+
+        let chain = create_descriptor_chain(
+            &memory,
+            GuestAddress(0x0),
+            GuestAddress(0x100),
+            vec![(Writable, 16), (Writable, 16), (Writable, 16)],
+            0,
+        )
+        .expect("create_descriptor_chain failed");
+        let mut writer = Writer::new(&memory, chain).expect("failed to create Writer");
+
+        let buf = vec![0xdeu8; 64];
+        assert_eq!(
+            writer.write(&buf[..]).expect("failed to write from buffer"),
+            48
+        );
+    }
+}

vhost_user_fs/src/file_traits.rs

@@ -0,0 +1,409 @@
+// Copyright 2018 The Chromium OS Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+use std::fs::File;
+use std::io::{Error, ErrorKind, Result};
+use std::os::unix::io::AsRawFd;
+
+use vm_memory::VolatileSlice;
+
+use libc::{
+    c_int, c_void, off64_t, pread64, preadv64, pwrite64, pwritev64, read, readv, size_t, write,
+    writev,
+};
+
+/// A trait for setting the size of a file.
+/// This is equivalent to File's `set_len` method, but
+/// wrapped in a trait so that it can be implemented for
+/// other types.
+pub trait FileSetLen {
+    // Set the size of this file.
+    // This is the moral equivalent of `ftruncate()`.
+    fn set_len(&self, _len: u64) -> Result<()>;
+}
+
+impl FileSetLen for File {
+    fn set_len(&self, len: u64) -> Result<()> {
+        File::set_len(self, len)
+    }
+}
+
+/// A trait similar to `Read` and `Write`, but uses volatile memory as buffers.
+pub trait FileReadWriteVolatile {
+    /// Read bytes from this file into the given slice, returning the number of bytes read on
+    /// success.
+    fn read_volatile(&mut self, slice: VolatileSlice) -> Result<usize>;
+
+    /// Like `read_volatile`, except it reads to a slice of buffers. Data is copied to fill each
+    /// buffer in order, with the final buffer written to possibly being only partially filled. This
+    /// method must behave as a single call to `read_volatile` with the buffers concatenated would.
+    /// The default implementation calls `read_volatile` with either the first nonempty buffer
+    /// provided, or returns `Ok(0)` if none exists.
+    fn read_vectored_volatile(&mut self, bufs: &[VolatileSlice]) -> Result<usize> {
+        bufs.iter()
+            .find(|b| !b.is_empty())
+            .map(|&b| self.read_volatile(b))
+            .unwrap_or(Ok(0))
+    }
+
+    /// Reads bytes from this into the given slice until all bytes in the slice are written, or an
+    /// error is returned.
+    fn read_exact_volatile(&mut self, mut slice: VolatileSlice) -> Result<()> {
+        while !slice.is_empty() {
+            let bytes_read = self.read_volatile(slice)?;
+            if bytes_read == 0 {
+                return Err(Error::from(ErrorKind::UnexpectedEof));
+            }
+            // Will panic if read_volatile read more bytes than we gave it, which would be worthy of
+            // a panic.
+            slice = slice.offset(bytes_read).unwrap();
+        }
+        Ok(())
+    }
+
+    /// Write bytes from the slice to the given file, returning the number of bytes written on
+    /// success.
+    fn write_volatile(&mut self, slice: VolatileSlice) -> Result<usize>;
+
+    /// Like `write_volatile`, except that it writes from a slice of buffers. Data is copied from
+    /// each buffer in order, with the final buffer read from possibly being only partially
+    /// consumed. This method must behave as a call to `write_volatile` with the buffers
+    /// concatenated would. The default implementation calls `write_volatile` with either the first
+    /// nonempty buffer provided, or returns `Ok(0)` if none exists.
+    fn write_vectored_volatile(&mut self, bufs: &[VolatileSlice]) -> Result<usize> {
+        bufs.iter()
+            .find(|b| !b.is_empty())
+            .map(|&b| self.write_volatile(b))
+            .unwrap_or(Ok(0))
+    }
+
+    /// Write bytes from the slice to the given file until all the bytes from the slice have been
+    /// written, or an error is returned.
+    fn write_all_volatile(&mut self, mut slice: VolatileSlice) -> Result<()> {
+        while !slice.is_empty() {
+            let bytes_written = self.write_volatile(slice)?;
+            if bytes_written == 0 {
+                return Err(Error::from(ErrorKind::WriteZero));
+            }
+            // Will panic if read_volatile read more bytes than we gave it, which would be worthy of
+            // a panic.
+            slice = slice.offset(bytes_written).unwrap();
+        }
+        Ok(())
+    }
+}
+
+impl<'a, T: FileReadWriteVolatile + ?Sized> FileReadWriteVolatile for &'a mut T {
+    fn read_volatile(&mut self, slice: VolatileSlice) -> Result<usize> {
+        (**self).read_volatile(slice)
+    }
+
+    fn read_vectored_volatile(&mut self, bufs: &[VolatileSlice]) -> Result<usize> {
+        (**self).read_vectored_volatile(bufs)
+    }
+
+    fn read_exact_volatile(&mut self, slice: VolatileSlice) -> Result<()> {
+        (**self).read_exact_volatile(slice)
+    }
+
+    fn write_volatile(&mut self, slice: VolatileSlice) -> Result<usize> {
+        (**self).write_volatile(slice)
+    }
+
+    fn write_vectored_volatile(&mut self, bufs: &[VolatileSlice]) -> Result<usize> {
+        (**self).write_vectored_volatile(bufs)
+    }
+
+    fn write_all_volatile(&mut self, slice: VolatileSlice) -> Result<()> {
+        (**self).write_all_volatile(slice)
+    }
+}
+
+/// A trait similar to the unix `ReadExt` and `WriteExt` traits, but for volatile memory.
+pub trait FileReadWriteAtVolatile {
+    /// Reads bytes from this file at `offset` into the given slice, returning the number of bytes
+    /// read on success.
+    fn read_at_volatile(&mut self, slice: VolatileSlice, offset: u64) -> Result<usize>;
+
+    /// Like `read_at_volatile`, except it reads to a slice of buffers. Data is copied to fill each
+    /// buffer in order, with the final buffer written to possibly being only partially filled. This
+    /// method must behave as a single call to `read_at_volatile` with the buffers concatenated
+    /// would. The default implementation calls `read_at_volatile` with either the first nonempty
+    /// buffer provided, or returns `Ok(0)` if none exists.
+    fn read_vectored_at_volatile(&mut self, bufs: &[VolatileSlice], offset: u64) -> Result<usize> {
+        if let Some(&slice) = bufs.first() {
+            self.read_at_volatile(slice, offset)
+        } else {
+            Ok(0)
+        }
+    }
+
+    /// Reads bytes from this file at `offset` into the given slice until all bytes in the slice are
+    /// read, or an error is returned.
+    fn read_exact_at_volatile(&mut self, mut slice: VolatileSlice, mut offset: u64) -> Result<()> {
+        while !slice.is_empty() {
+            match self.read_at_volatile(slice, offset) {
+                Ok(0) => return Err(Error::from(ErrorKind::UnexpectedEof)),
+                Ok(n) => {
+                    slice = slice.offset(n).unwrap();
+                    offset = offset.checked_add(n as u64).unwrap();
+                }
+                Err(ref e) if e.kind() == ErrorKind::Interrupted => {}
+                Err(e) => return Err(e),
+            }
+        }
+        Ok(())
+    }
+
+    /// Writes bytes from this file at `offset` into the given slice, returning the number of bytes
+    /// written on success.
+    fn write_at_volatile(&mut self, slice: VolatileSlice, offset: u64) -> Result<usize>;
+
+    /// Like `write_at_at_volatile`, except that it writes from a slice of buffers. Data is copied
+    /// from each buffer in order, with the final buffer read from possibly being only partially
+    /// consumed. This method must behave as a call to `write_at_volatile` with the buffers
+    /// concatenated would. The default implementation calls `write_at_volatile` with either the
+    /// first nonempty buffer provided, or returns `Ok(0)` if none exists.
+    fn write_vectored_at_volatile(&mut self, bufs: &[VolatileSlice], offset: u64) -> Result<usize> {
+        if let Some(&slice) = bufs.first() {
+            self.write_at_volatile(slice, offset)
+        } else {
+            Ok(0)
+        }
+    }
+
+    /// Writes bytes from this file at `offset` into the given slice until all bytes in the slice
+    /// are written, or an error is returned.
+    fn write_all_at_volatile(&mut self, mut slice: VolatileSlice, mut offset: u64) -> Result<()> {
+        while !slice.is_empty() {
+            match self.write_at_volatile(slice, offset) {
+                Ok(0) => return Err(Error::from(ErrorKind::WriteZero)),
+                Ok(n) => {
+                    slice = slice.offset(n).unwrap();
+                    offset = offset.checked_add(n as u64).unwrap();
+                }
+                Err(ref e) if e.kind() == ErrorKind::Interrupted => {}
+                Err(e) => return Err(e),
+            }
+        }
+        Ok(())
+    }
+}
+
+impl<'a, T: FileReadWriteAtVolatile + ?Sized> FileReadWriteAtVolatile for &'a mut T {
+    fn read_at_volatile(&mut self, slice: VolatileSlice, offset: u64) -> Result<usize> {
+        (**self).read_at_volatile(slice, offset)
+    }
+
+    fn read_vectored_at_volatile(&mut self, bufs: &[VolatileSlice], offset: u64) -> Result<usize> {
+        (**self).read_vectored_at_volatile(bufs, offset)
+    }
+
+    fn read_exact_at_volatile(&mut self, slice: VolatileSlice, offset: u64) -> Result<()> {
+        (**self).read_exact_at_volatile(slice, offset)
+    }
+
+    fn write_at_volatile(&mut self, slice: VolatileSlice, offset: u64) -> Result<usize> {
+        (**self).write_at_volatile(slice, offset)
+    }
+
+    fn write_vectored_at_volatile(&mut self, bufs: &[VolatileSlice], offset: u64) -> Result<usize> {
+        (**self).write_vectored_at_volatile(bufs, offset)
+    }
+
+    fn write_all_at_volatile(&mut self, slice: VolatileSlice, offset: u64) -> Result<()> {
+        (**self).write_all_at_volatile(slice, offset)
+    }
+}
+
+macro_rules! volatile_impl {
+    ($ty:ty) => {
+        impl FileReadWriteVolatile for $ty {
+            fn read_volatile(&mut self, slice: VolatileSlice) -> Result<usize> {
+                // Safe because only bytes inside the slice are accessed and the kernel is expected
+                // to handle arbitrary memory for I/O.
+                let ret =
+                    unsafe { read(self.as_raw_fd(), slice.as_ptr() as *mut c_void, slice.len()) };
+                if ret >= 0 {
+                    Ok(ret as usize)
+                } else {
+                    Err(Error::last_os_error())
+                }
+            }
+
+            fn read_vectored_volatile(&mut self, bufs: &[VolatileSlice]) -> Result<usize> {
+                let iovecs: Vec<libc::iovec> = bufs
+                    .iter()
+                    .map(|s| libc::iovec {
+                        iov_base: s.as_ptr() as *mut c_void,
+                        iov_len: s.len() as size_t,
+                    })
+                    .collect();
+
+                if iovecs.is_empty() {
+                    return Ok(0);
+                }
+
+                // Safe because only bytes inside the buffers are accessed and the kernel is
+                // expected to handle arbitrary memory for I/O.
+                let ret = unsafe { readv(self.as_raw_fd(), &iovecs[0], iovecs.len() as c_int) };
+                if ret >= 0 {
+                    Ok(ret as usize)
+                } else {
+                    Err(Error::last_os_error())
+                }
+            }
+
+            fn write_volatile(&mut self, slice: VolatileSlice) -> Result<usize> {
+                // Safe because only bytes inside the slice are accessed and the kernel is expected
+                // to handle arbitrary memory for I/O.
+                let ret = unsafe {
+                    write(
+                        self.as_raw_fd(),
+                        slice.as_ptr() as *const c_void,
+                        slice.len(),
+                    )
+                };
+                if ret >= 0 {
+                    Ok(ret as usize)
+                } else {
+                    Err(Error::last_os_error())
+                }
+            }
+
+            fn write_vectored_volatile(&mut self, bufs: &[VolatileSlice]) -> Result<usize> {
+                let iovecs: Vec<libc::iovec> = bufs
+                    .iter()
+                    .map(|s| libc::iovec {
+                        iov_base: s.as_ptr() as *mut c_void,
+                        iov_len: s.len() as size_t,
+                    })
+                    .collect();
+
+                if iovecs.is_empty() {
+                    return Ok(0);
+                }
+
+                // Safe because only bytes inside the buffers are accessed and the kernel is
+                // expected to handle arbitrary memory for I/O.
+                let ret = unsafe { writev(self.as_raw_fd(), &iovecs[0], iovecs.len() as c_int) };
+                if ret >= 0 {
+                    Ok(ret as usize)
+                } else {
+                    Err(Error::last_os_error())
+                }
+            }
+        }
+
+        impl FileReadWriteAtVolatile for $ty {
+            fn read_at_volatile(&mut self, slice: VolatileSlice, offset: u64) -> Result<usize> {
+                // Safe because only bytes inside the slice are accessed and the kernel is expected
+                // to handle arbitrary memory for I/O.
+                let ret = unsafe {
+                    pread64(
+                        self.as_raw_fd(),
+                        slice.as_ptr() as *mut c_void,
+                        slice.len(),
+                        offset as off64_t,
+                    )
+                };
+
+                if ret >= 0 {
+                    Ok(ret as usize)
+                } else {
+                    Err(Error::last_os_error())
+                }
+            }
+
+            fn read_vectored_at_volatile(
+                &mut self,
+                bufs: &[VolatileSlice],
+                offset: u64,
+            ) -> Result<usize> {
+                let iovecs: Vec<libc::iovec> = bufs
+                    .iter()
+                    .map(|s| libc::iovec {
+                        iov_base: s.as_ptr() as *mut c_void,
+                        iov_len: s.len() as size_t,
+                    })
+                    .collect();
+
+                if iovecs.is_empty() {
+                    return Ok(0);
+                }
+
+                // Safe because only bytes inside the buffers are accessed and the kernel is
+                // expected to handle arbitrary memory for I/O.
+                let ret = unsafe {
+                    preadv64(
+                        self.as_raw_fd(),
+                        &iovecs[0],
+                        iovecs.len() as c_int,
+                        offset as off64_t,
+                    )
+                };
+                if ret >= 0 {
+                    Ok(ret as usize)
+                } else {
+                    Err(Error::last_os_error())
+                }
+            }
+
+            fn write_at_volatile(&mut self, slice: VolatileSlice, offset: u64) -> Result<usize> {
+                // Safe because only bytes inside the slice are accessed and the kernel is expected
+                // to handle arbitrary memory for I/O.
+                let ret = unsafe {
+                    pwrite64(
+                        self.as_raw_fd(),
+                        slice.as_ptr() as *const c_void,
+                        slice.len(),
+                        offset as off64_t,
+                    )
+                };
+
+                if ret >= 0 {
+                    Ok(ret as usize)
+                } else {
+                    Err(Error::last_os_error())
+                }
+            }
+
+            fn write_vectored_at_volatile(
+                &mut self,
+                bufs: &[VolatileSlice],
+                offset: u64,
+            ) -> Result<usize> {
+                let iovecs: Vec<libc::iovec> = bufs
+                    .iter()
+                    .map(|s| libc::iovec {
+                        iov_base: s.as_ptr() as *mut c_void,
+                        iov_len: s.len() as size_t,
+                    })
+                    .collect();
+
+                if iovecs.is_empty() {
+                    return Ok(0);
+                }
+
+                // Safe because only bytes inside the buffers are accessed and the kernel is
+                // expected to handle arbitrary memory for I/O.
+                let ret = unsafe {
+                    pwritev64(
+                        self.as_raw_fd(),
+                        &iovecs[0],
+                        iovecs.len() as c_int,
+                        offset as off64_t,
+                    )
+                };
+                if ret >= 0 {
+                    Ok(ret as usize)
+                } else {
+                    Err(Error::last_os_error())
+                }
+            }
+        }
+    };
+}
+
+volatile_impl!(File);

vhost_user_fs/src/lib.rs

@@ -0,0 +1,57 @@
+// Copyright © 2019 Intel Corporation
+//
+// SPDX-License-Identifier: Apache-2.0 AND BSD-3-Clause
+
+#[macro_use]
+extern crate log;
+
+pub mod descriptor_utils;
+pub mod file_traits;
+pub mod filesystem;
+pub mod fuse;
+pub mod multikey;
+pub mod passthrough;
+pub mod server;
+
+use std::ffi::FromBytesWithNulError;
+use std::{error, fmt, io};
+
+#[derive(Debug)]
+pub enum Error {
+    /// Failed to decode protocol messages.
+    DecodeMessage(io::Error),
+    /// Failed to encode protocol messages.
+    EncodeMessage(io::Error),
+    /// One or more parameters are missing.
+    MissingParameter,
+    /// A C string parameter is invalid.
+    InvalidCString(FromBytesWithNulError),
+    /// The `len` field of the header is too small.
+    InvalidHeaderLength,
+    /// The `size` field of the `SetxattrIn` message does not match the length
+    /// of the decoded value.
+    InvalidXattrSize((u32, usize)),
+}
+
+impl error::Error for Error {}
+
+impl fmt::Display for Error {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        use Error::*;
+        match self {
+            DecodeMessage(err) => write!(f, "failed to decode fuse message: {}", err),
+            EncodeMessage(err) => write!(f, "failed to encode fuse message: {}", err),
+            MissingParameter => write!(f, "one or more parameters are missing"),
+            InvalidHeaderLength => write!(f, "the `len` field of the header is too small"),
+            InvalidCString(err) => write!(f, "a c string parameter is invalid: {}", err),
+            InvalidXattrSize((size, len)) => write!(
+                f,
+                "The `size` field of the `SetxattrIn` message does not match the length of the\
+                 decoded value: size = {}, value.len() = {}",
+                size, len
+            ),
+        }
+    }
+}
+
+pub type Result<T> = ::std::result::Result<T, Error>;

vhost_user_fs/src/multikey.rs

@@ -0,0 +1,274 @@
+// Copyright 2019 The Chromium OS Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+use std::borrow::Borrow;
+use std::collections::BTreeMap;
+
+/// A BTreeMap that supports 2 types of keys per value. All the usual restrictions and warnings for
+/// `std::collections::BTreeMap` also apply to this struct. Additionally, there is a 1:1
+/// relationship between the 2 key types. In other words, for each `K1` in the map, there is exactly
+/// one `K2` in the map and vice versa.
+#[derive(Default)]
+pub struct MultikeyBTreeMap<K1, K2, V>
+where
+    K1: Ord,
+    K2: Ord,
+{
+    // We need to keep a copy of the second key in the main map so that we can remove entries using
+    // just the main key. Otherwise we would require the caller to provide both keys when calling
+    // `remove`.
+    main: BTreeMap<K1, (K2, V)>,
+    alt: BTreeMap<K2, K1>,
+}
+
+impl<K1, K2, V> MultikeyBTreeMap<K1, K2, V>
+where
+    K1: Clone + Ord,
+    K2: Clone + Ord,
+{
+    /// Create a new empty MultikeyBTreeMap.
+    pub fn new() -> Self {
+        MultikeyBTreeMap {
+            main: BTreeMap::default(),
+            alt: BTreeMap::default(),
+        }
+    }
+
+    /// Returns a reference to the value corresponding to the key.
+    ///
+    /// The key may be any borrowed form of `K1``, but the ordering on the borrowed form must match
+    /// the ordering on `K1`.
+    pub fn get<Q>(&self, key: &Q) -> Option<&V>
+    where
+        K1: Borrow<Q>,
+        Q: Ord + ?Sized,
+    {
+        self.main.get(key).map(|(_, v)| v)
+    }
+
+    /// Returns a reference to the value corresponding to the alternate key.
+    ///
+    /// The key may be any borrowed form of the `K2``, but the ordering on the borrowed form must
+    /// match the ordering on `K2`.
+    ///
+    /// Note that this method performs 2 lookups: one to get the main key and another to get the
+    /// value associated with that key. For best performance callers should prefer the `get` method
+    /// over this method whenever possible as `get` only needs to perform one lookup.
+    pub fn get_alt<Q2>(&self, key: &Q2) -> Option<&V>
+    where
+        K2: Borrow<Q2>,
+        Q2: Ord + ?Sized,
+    {
+        if let Some(k) = self.alt.get(key) {
+            self.get(k)
+        } else {
+            None
+        }
+    }
+
+    /// Inserts a new entry into the map with the given keys and value.
+    ///
+    /// Returns `None` if the map did not have an entry with `k1` or `k2` present. If exactly one
+    /// key was present, then the value associated with that key is updated, the other key is
+    /// removed, and the old value is returned. If **both** keys were present then the value
+    /// associated with the main key is updated, the value associated with the alternate key is
+    /// removed, and the old value associated with the main key is returned.
+    pub fn insert(&mut self, k1: K1, k2: K2, v: V) -> Option<V> {
+        let oldval = if let Some(oldkey) = self.alt.insert(k2.clone(), k1.clone()) {
+            self.main.remove(&oldkey)
+        } else {
+            None
+        };
+        self.main
+            .insert(k1, (k2.clone(), v))
+            .or(oldval)
+            .map(|(oldk2, v)| {
+                if oldk2 != k2 {
+                    self.alt.remove(&oldk2);
+                }
+                v
+            })
+    }
+
+    /// Remove a key from the map, returning the value associated with that key if it was previously
+    /// in the map.
+    ///
+    /// The key may be any borrowed form of `K1``, but the ordering on the borrowed form must match
+    /// the ordering on `K1`.
+    pub fn remove<Q>(&mut self, key: &Q) -> Option<V>
+    where
+        K1: Borrow<Q>,
+        Q: Ord + ?Sized,
+    {
+        self.main.remove(key).map(|(k2, v)| {
+            self.alt.remove(&k2);
+            v
+        })
+    }
+
+    /// Clears the map, removing all values.
+    pub fn clear(&mut self) {
+        self.alt.clear();
+        self.main.clear()
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+
+    #[test]
+    fn get() {
+        let mut m = MultikeyBTreeMap::<u64, i64, u32>::new();
+
+        let k1 = 0xc6c8_f5e0_b13e_ed40;
+        let k2 = 0x1a04_ce4b_8329_14fe;
+        let val = 0xf4e3_c360;
+        assert!(m.insert(k1, k2, val).is_none());
+
+        assert_eq!(*m.get(&k1).expect("failed to look up main key"), val);
+        assert_eq!(*m.get_alt(&k2).expect("failed to look up alt key"), val);
+    }
+
+    #[test]
+    fn update_main_key() {
+        let mut m = MultikeyBTreeMap::<u64, i64, u32>::new();
+
+        let k1 = 0xc6c8_f5e0_b13e_ed40;
+        let k2 = 0x1a04_ce4b_8329_14fe;
+        let val = 0xf4e3_c360;
+        assert!(m.insert(k1, k2, val).is_none());
+
+        let new_k1 = 0x3add_f8f8_c7c5_df5e;
+        let val2 = 0x7389_f8a7;
+        assert_eq!(
+            m.insert(new_k1, k2, val2)
+                .expect("failed to update main key"),
+            val
+        );
+
+        assert!(m.get(&k1).is_none());
+        assert_eq!(*m.get(&new_k1).expect("failed to look up main key"), val2);
+        assert_eq!(*m.get_alt(&k2).expect("failed to look up alt key"), val2);
+    }
+
+    #[test]
+    fn update_alt_key() {
+        let mut m = MultikeyBTreeMap::<u64, i64, u32>::new();
+
+        let k1 = 0xc6c8_f5e0_b13e_ed40;
+        let k2 = 0x1a04_ce4b_8329_14fe;
+        let val = 0xf4e3_c360;
+        assert!(m.insert(k1, k2, val).is_none());
+
+        let new_k2 = 0x6825_a60b_61ac_b333;
+        let val2 = 0xbb14_8f2c;
+        assert_eq!(
+            m.insert(k1, new_k2, val2)
+                .expect("failed to update alt key"),
+            val
+        );
+
+        assert!(m.get_alt(&k2).is_none());
+        assert_eq!(*m.get(&k1).expect("failed to look up main key"), val2);
+        assert_eq!(
+            *m.get_alt(&new_k2).expect("failed to look up alt key"),
+            val2
+        );
+    }
+
+    #[test]
+    fn update_value() {
+        let mut m = MultikeyBTreeMap::<u64, i64, u32>::new();
+
+        let k1 = 0xc6c8_f5e0_b13e_ed40;
+        let k2 = 0x1a04_ce4b_8329_14fe;
+        let val = 0xf4e3_c360;
+        assert!(m.insert(k1, k2, val).is_none());
+
+        let val2 = 0xe42d_79ba;
+        assert_eq!(
+            m.insert(k1, k2, val2).expect("failed to update alt key"),
+            val
+        );
+
+        assert_eq!(*m.get(&k1).expect("failed to look up main key"), val2);
+        assert_eq!(*m.get_alt(&k2).expect("failed to look up alt key"), val2);
+    }
+
+    #[test]
+    fn update_both_keys_main() {
+        let mut m = MultikeyBTreeMap::<u64, i64, u32>::new();
+
+        let k1 = 0xc6c8_f5e0_b13e_ed40;
+        let k2 = 0x1a04_ce4b_8329_14fe;
+        let val = 0xf4e3_c360;
+        assert!(m.insert(k1, k2, val).is_none());
+
+        let new_k1 = 0xc980_587a_24b3_ae30;
+        let new_k2 = 0x2773_c5ee_8239_45a2;
+        let val2 = 0x31f4_33f9;
+        assert!(m.insert(new_k1, new_k2, val2).is_none());
+
+        let val3 = 0x8da1_9cf7;
+        assert_eq!(
+            m.insert(k1, new_k2, val3)
+                .expect("failed to update main key"),
+            val
+        );
+
+        // Both new_k1 and k2 should now be gone from the map.
+        assert!(m.get(&new_k1).is_none());
+        assert!(m.get_alt(&k2).is_none());
+
+        assert_eq!(*m.get(&k1).expect("failed to look up main key"), val3);
+        assert_eq!(
+            *m.get_alt(&new_k2).expect("failed to look up alt key"),
+            val3
+        );
+    }
+
+    #[test]
+    fn update_both_keys_alt() {
+        let mut m = MultikeyBTreeMap::<u64, i64, u32>::new();
+
+        let k1 = 0xc6c8_f5e0_b13e_ed40;
+        let k2 = 0x1a04_ce4b_8329_14fe;
+        let val = 0xf4e3_c360;
+        assert!(m.insert(k1, k2, val).is_none());
+
+        let new_k1 = 0xc980_587a_24b3_ae30;
+        let new_k2 = 0x2773_c5ee_8239_45a2;
+        let val2 = 0x31f4_33f9;
+        assert!(m.insert(new_k1, new_k2, val2).is_none());
+
+        let val3 = 0x8da1_9cf7;
+        assert_eq!(
+            m.insert(new_k1, k2, val3)
+                .expect("failed to update main key"),
+            val2
+        );
+
+        // Both k1 and new_k2 should now be gone from the map.
+        assert!(m.get(&k1).is_none());
+        assert!(m.get_alt(&new_k2).is_none());
+
+        assert_eq!(*m.get(&new_k1).expect("failed to look up main key"), val3);
+        assert_eq!(*m.get_alt(&k2).expect("failed to look up alt key"), val3);
+    }
+
+    #[test]
+    fn remove() {
+        let mut m = MultikeyBTreeMap::<u64, i64, u32>::new();
+
+        let k1 = 0xc6c8_f5e0_b13e_ed40;
+        let k2 = 0x1a04_ce4b_8329_14fe;
+        let val = 0xf4e3_c360;
+        assert!(m.insert(k1, k2, val).is_none());
+
+        assert_eq!(m.remove(&k1).expect("failed to remove entry"), val);
+        assert!(m.get(&k1).is_none());
+        assert!(m.get_alt(&k2).is_none());
+    }
+}

virtio-bindings/Cargo.toml

@@ -1,6 +0,0 @@
-[package]
-name = "virtio-bindings"
-version = "0.1.0"
-authors = ["The Chromium OS Authors"]
-
-[dependencies]

virtio-bindings/patches/0001-virtio_gen-remove-derive-Debug-from-packed-struct.patch

@@ -1,31 +0,0 @@
-From 67247b7abc56a11d8cae7eb354994d818fb72e93 Mon Sep 17 00:00:00 2001
-From: Andreea Florescu <fandree@amazon.com>
-Date: Tue, 15 Jan 2019 18:30:07 +0200
-Subject: [PATCH] virtio_gen: remove derive Debug from packed struct
-
-Bindgen automatically adds derive debug on virtio_net_ctrl_mac, a packed
-structure. This generates a warning while building.
-
-Manually remove the Debug derive.
-
-Signed-off-by: Andreea Florescu <fandree@amazon.com>
----
- virtio_gen/src/virtio_net.rs | 2 +-
- 1 file changed, 1 insertion(+), 1 deletion(-)
-
-diff --git a/virtio_gen/src/virtio_net.rs b/virtio_gen/src/virtio_net.rs
-index 0b68d09..a1c9dca 100644
---- a/virtio_gen/src/virtio_net.rs
-+++ b/virtio_gen/src/virtio_net.rs
-@@ -681,7 +681,7 @@ fn bindgen_test_layout_virtio_net_ctrl_hdr() {
- }
- pub type virtio_net_ctrl_ack = __u8;
- #[repr(C, packed)]
--#[derive(Debug, Default)]
-+#[derive(Default)]
- pub struct virtio_net_ctrl_mac {
-     pub entries: __virtio32,
-     pub macs: __IncompleteArrayField<[__u8; 6usize]>,
--- 
-2.7.4
-

virtio-bindings/src/lib.rs

@@ -1,15 +0,0 @@
-// Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
-// SPDX-License-Identifier: Apache-2.0
-//
-// Portions Copyright 2017 The Chromium OS Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE-BSD-3-Clause file.
-
-#![allow(clippy::all)]
-#![allow(non_upper_case_globals)]
-#![allow(non_camel_case_types)]
-#![allow(non_snake_case)]
-
-pub mod virtio_blk;
-pub mod virtio_net;
-pub mod virtio_ring;

virtio-bindings/src/virtio_blk.rs

@@ -1,486 +0,0 @@
-// Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
-// SPDX-License-Identifier: Apache-2.0
-
-/* automatically generated by rust-bindgen */
-
-pub const __BITS_PER_LONG: u32 = 64;
-pub const __FD_SETSIZE: u32 = 1024;
-pub const VIRTIO_ID_NET: u32 = 1;
-pub const VIRTIO_ID_BLOCK: u32 = 2;
-pub const VIRTIO_ID_CONSOLE: u32 = 3;
-pub const VIRTIO_ID_RNG: u32 = 4;
-pub const VIRTIO_ID_BALLOON: u32 = 5;
-pub const VIRTIO_ID_RPMSG: u32 = 7;
-pub const VIRTIO_ID_SCSI: u32 = 8;
-pub const VIRTIO_ID_9P: u32 = 9;
-pub const VIRTIO_ID_RPROC_SERIAL: u32 = 11;
-pub const VIRTIO_ID_CAIF: u32 = 12;
-pub const VIRTIO_ID_GPU: u32 = 16;
-pub const VIRTIO_ID_INPUT: u32 = 18;
-pub const VIRTIO_CONFIG_S_ACKNOWLEDGE: u32 = 1;
-pub const VIRTIO_CONFIG_S_DRIVER: u32 = 2;
-pub const VIRTIO_CONFIG_S_DRIVER_OK: u32 = 4;
-pub const VIRTIO_CONFIG_S_FEATURES_OK: u32 = 8;
-pub const VIRTIO_CONFIG_S_FAILED: u32 = 128;
-pub const VIRTIO_TRANSPORT_F_START: u32 = 28;
-pub const VIRTIO_TRANSPORT_F_END: u32 = 33;
-pub const VIRTIO_F_NOTIFY_ON_EMPTY: u32 = 24;
-pub const VIRTIO_F_ANY_LAYOUT: u32 = 27;
-pub const VIRTIO_F_VERSION_1: u32 = 32;
-pub const VIRTIO_BLK_F_SIZE_MAX: u32 = 1;
-pub const VIRTIO_BLK_F_SEG_MAX: u32 = 2;
-pub const VIRTIO_BLK_F_GEOMETRY: u32 = 4;
-pub const VIRTIO_BLK_F_RO: u32 = 5;
-pub const VIRTIO_BLK_F_BLK_SIZE: u32 = 6;
-pub const VIRTIO_BLK_F_TOPOLOGY: u32 = 10;
-pub const VIRTIO_BLK_F_MQ: u32 = 12;
-pub const VIRTIO_BLK_F_BARRIER: u32 = 0;
-pub const VIRTIO_BLK_F_SCSI: u32 = 7;
-pub const VIRTIO_BLK_F_FLUSH: u32 = 9;
-pub const VIRTIO_BLK_F_CONFIG_WCE: u32 = 11;
-pub const VIRTIO_BLK_F_WCE: u32 = 9;
-pub const VIRTIO_BLK_ID_BYTES: u32 = 20;
-pub const VIRTIO_BLK_T_IN: u32 = 0;
-pub const VIRTIO_BLK_T_OUT: u32 = 1;
-pub const VIRTIO_BLK_T_SCSI_CMD: u32 = 2;
-pub const VIRTIO_BLK_T_FLUSH: u32 = 4;
-pub const VIRTIO_BLK_T_GET_ID: u32 = 8;
-pub const VIRTIO_BLK_T_BARRIER: u32 = 2147483648;
-pub const VIRTIO_BLK_S_OK: u32 = 0;
-pub const VIRTIO_BLK_S_IOERR: u32 = 1;
-pub const VIRTIO_BLK_S_UNSUPP: u32 = 2;
-pub type __s8 = ::std::os::raw::c_schar;
-pub type __u8 = ::std::os::raw::c_uchar;
-pub type __s16 = ::std::os::raw::c_short;
-pub type __u16 = ::std::os::raw::c_ushort;
-pub type __s32 = ::std::os::raw::c_int;
-pub type __u32 = ::std::os::raw::c_uint;
-pub type __s64 = ::std::os::raw::c_longlong;
-pub type __u64 = ::std::os::raw::c_ulonglong;
-#[repr(C)]
-#[derive(Debug, Default, Copy, Clone, PartialEq)]
-pub struct __kernel_fd_set {
-    pub fds_bits: [::std::os::raw::c_ulong; 16usize],
-}
-#[test]
-fn bindgen_test_layout___kernel_fd_set() {
-    assert_eq!(
-        ::std::mem::size_of::<__kernel_fd_set>(),
-        128usize,
-        concat!("Size of: ", stringify!(__kernel_fd_set))
-    );
-    assert_eq!(
-        ::std::mem::align_of::<__kernel_fd_set>(),
-        8usize,
-        concat!("Alignment of ", stringify!(__kernel_fd_set))
-    );
-    assert_eq!(
-        unsafe { &(*(::std::ptr::null::<__kernel_fd_set>())).fds_bits as *const _ as usize },
-        0usize,
-        concat!(
-            "Offset of field: ",
-            stringify!(__kernel_fd_set),
-            "::",
-            stringify!(fds_bits)
-        )
-    );
-}
-pub type __kernel_sighandler_t =
-    ::std::option::Option<unsafe extern "C" fn(arg1: ::std::os::raw::c_int)>;
-pub type __kernel_key_t = ::std::os::raw::c_int;
-pub type __kernel_mqd_t = ::std::os::raw::c_int;
-pub type __kernel_old_uid_t = ::std::os::raw::c_ushort;
-pub type __kernel_old_gid_t = ::std::os::raw::c_ushort;
-pub type __kernel_old_dev_t = ::std::os::raw::c_ulong;
-pub type __kernel_long_t = ::std::os::raw::c_long;
-pub type __kernel_ulong_t = ::std::os::raw::c_ulong;
-pub type __kernel_ino_t = __kernel_ulong_t;
-pub type __kernel_mode_t = ::std::os::raw::c_uint;
-pub type __kernel_pid_t = ::std::os::raw::c_int;
-pub type __kernel_ipc_pid_t = ::std::os::raw::c_int;
-pub type __kernel_uid_t = ::std::os::raw::c_uint;
-pub type __kernel_gid_t = ::std::os::raw::c_uint;
-pub type __kernel_suseconds_t = __kernel_long_t;
-pub type __kernel_daddr_t = ::std::os::raw::c_int;
-pub type __kernel_uid32_t = ::std::os::raw::c_uint;
-pub type __kernel_gid32_t = ::std::os::raw::c_uint;
-pub type __kernel_size_t = __kernel_ulong_t;
-pub type __kernel_ssize_t = __kernel_long_t;
-pub type __kernel_ptrdiff_t = __kernel_long_t;
-#[repr(C)]
-#[derive(Debug, Default, Copy, Clone, PartialEq)]
-pub struct __kernel_fsid_t {
-    pub val: [::std::os::raw::c_int; 2usize],
-}
-#[test]
-fn bindgen_test_layout___kernel_fsid_t() {
-    assert_eq!(
-        ::std::mem::size_of::<__kernel_fsid_t>(),
-        8usize,
-        concat!("Size of: ", stringify!(__kernel_fsid_t))
-    );
-    assert_eq!(
-        ::std::mem::align_of::<__kernel_fsid_t>(),
-        4usize,
-        concat!("Alignment of ", stringify!(__kernel_fsid_t))
-    );
-    assert_eq!(
-        unsafe { &(*(::std::ptr::null::<__kernel_fsid_t>())).val as *const _ as usize },
-        0usize,
-        concat!(
-            "Offset of field: ",
-            stringify!(__kernel_fsid_t),
-            "::",
-            stringify!(val)
-        )
-    );
-}
-pub type __kernel_off_t = __kernel_long_t;
-pub type __kernel_loff_t = ::std::os::raw::c_longlong;
-pub type __kernel_time_t = __kernel_long_t;
-pub type __kernel_clock_t = __kernel_long_t;
-pub type __kernel_timer_t = ::std::os::raw::c_int;
-pub type __kernel_clockid_t = ::std::os::raw::c_int;
-pub type __kernel_caddr_t = *mut ::std::os::raw::c_char;
-pub type __kernel_uid16_t = ::std::os::raw::c_ushort;
-pub type __kernel_gid16_t = ::std::os::raw::c_ushort;
-pub type __le16 = __u16;
-pub type __be16 = __u16;
-pub type __le32 = __u32;
-pub type __be32 = __u32;
-pub type __le64 = __u64;
-pub type __be64 = __u64;
-pub type __sum16 = __u16;
-pub type __wsum = __u32;
-pub type __virtio16 = __u16;
-pub type __virtio32 = __u32;
-pub type __virtio64 = __u64;
-#[repr(C, packed)]
-#[derive(Debug, Default, Copy, Clone, PartialEq)]
-pub struct virtio_blk_config {
-    pub capacity: __u64,
-    pub size_max: __u32,
-    pub seg_max: __u32,
-    pub geometry: virtio_blk_config_virtio_blk_geometry,
-    pub blk_size: __u32,
-    pub physical_block_exp: __u8,
-    pub alignment_offset: __u8,
-    pub min_io_size: __u16,
-    pub opt_io_size: __u32,
-    pub wce: __u8,
-    pub unused: __u8,
-    pub num_queues: __u16,
-}
-#[repr(C)]
-#[derive(Debug, Default, Copy, Clone, PartialEq)]
-pub struct virtio_blk_config_virtio_blk_geometry {
-    pub cylinders: __u16,
-    pub heads: __u8,
-    pub sectors: __u8,
-}
-#[test]
-fn bindgen_test_layout_virtio_blk_config_virtio_blk_geometry() {
-    assert_eq!(
-        ::std::mem::size_of::<virtio_blk_config_virtio_blk_geometry>(),
-        4usize,
-        concat!(
-            "Size of: ",
-            stringify!(virtio_blk_config_virtio_blk_geometry)
-        )
-    );
-    assert_eq!(
-        ::std::mem::align_of::<virtio_blk_config_virtio_blk_geometry>(),
-        2usize,
-        concat!(
-            "Alignment of ",
-            stringify!(virtio_blk_config_virtio_blk_geometry)
-        )
-    );
-    assert_eq!(
-        unsafe {
-            &(*(::std::ptr::null::<virtio_blk_config_virtio_blk_geometry>())).cylinders as *const _
-                as usize
-        },
-        0usize,
-        concat!(
-            "Offset of field: ",
-            stringify!(virtio_blk_config_virtio_blk_geometry),
-            "::",
-            stringify!(cylinders)
-        )
-    );
-    assert_eq!(
-        unsafe {
-            &(*(::std::ptr::null::<virtio_blk_config_virtio_blk_geometry>())).heads as *const _
-                as usize
-        },
-        2usize,
-        concat!(
-            "Offset of field: ",
-            stringify!(virtio_blk_config_virtio_blk_geometry),
-            "::",
-            stringify!(heads)
-        )
-    );
-    assert_eq!(
-        unsafe {
-            &(*(::std::ptr::null::<virtio_blk_config_virtio_blk_geometry>())).sectors as *const _
-                as usize
-        },
-        3usize,
-        concat!(
-            "Offset of field: ",
-            stringify!(virtio_blk_config_virtio_blk_geometry),
-            "::",
-            stringify!(sectors)
-        )
-    );
-}
-#[test]
-fn bindgen_test_layout_virtio_blk_config() {
-    assert_eq!(
-        ::std::mem::size_of::<virtio_blk_config>(),
-        36usize,
-        concat!("Size of: ", stringify!(virtio_blk_config))
-    );
-    assert_eq!(
-        ::std::mem::align_of::<virtio_blk_config>(),
-        1usize,
-        concat!("Alignment of ", stringify!(virtio_blk_config))
-    );
-    assert_eq!(
-        unsafe { &(*(::std::ptr::null::<virtio_blk_config>())).capacity as *const _ as usize },
-        0usize,
-        concat!(
-            "Offset of field: ",
-            stringify!(virtio_blk_config),
-            "::",
-            stringify!(capacity)
-        )
-    );
-    assert_eq!(
-        unsafe { &(*(::std::ptr::null::<virtio_blk_config>())).size_max as *const _ as usize },
-        8usize,
-        concat!(
-            "Offset of field: ",
-            stringify!(virtio_blk_config),
-            "::",
-            stringify!(size_max)
-        )
-    );
-    assert_eq!(
-        unsafe { &(*(::std::ptr::null::<virtio_blk_config>())).seg_max as *const _ as usize },
-        12usize,
-        concat!(
-            "Offset of field: ",
-            stringify!(virtio_blk_config),
-            "::",
-            stringify!(seg_max)
-        )
-    );
-    assert_eq!(
-        unsafe { &(*(::std::ptr::null::<virtio_blk_config>())).geometry as *const _ as usize },
-        16usize,
-        concat!(
-            "Offset of field: ",
-            stringify!(virtio_blk_config),
-            "::",
-            stringify!(geometry)
-        )
-    );
-    assert_eq!(
-        unsafe { &(*(::std::ptr::null::<virtio_blk_config>())).blk_size as *const _ as usize },
-        20usize,
-        concat!(
-            "Offset of field: ",
-            stringify!(virtio_blk_config),
-            "::",
-            stringify!(blk_size)
-        )
-    );
-    assert_eq!(
-        unsafe {
-            &(*(::std::ptr::null::<virtio_blk_config>())).physical_block_exp as *const _ as usize
-        },
-        24usize,
-        concat!(
-            "Offset of field: ",
-            stringify!(virtio_blk_config),
-            "::",
-            stringify!(physical_block_exp)
-        )
-    );
-    assert_eq!(
-        unsafe {
-            &(*(::std::ptr::null::<virtio_blk_config>())).alignment_offset as *const _ as usize
-        },
-        25usize,
-        concat!(
-            "Offset of field: ",
-            stringify!(virtio_blk_config),
-            "::",
-            stringify!(alignment_offset)
-        )
-    );
-    assert_eq!(
-        unsafe { &(*(::std::ptr::null::<virtio_blk_config>())).min_io_size as *const _ as usize },
-        26usize,
-        concat!(
-            "Offset of field: ",
-            stringify!(virtio_blk_config),
-            "::",
-            stringify!(min_io_size)
-        )
-    );
-    assert_eq!(
-        unsafe { &(*(::std::ptr::null::<virtio_blk_config>())).opt_io_size as *const _ as usize },
-        28usize,
-        concat!(
-            "Offset of field: ",
-            stringify!(virtio_blk_config),
-            "::",
-            stringify!(opt_io_size)
-        )
-    );
-    assert_eq!(
-        unsafe { &(*(::std::ptr::null::<virtio_blk_config>())).wce as *const _ as usize },
-        32usize,
-        concat!(
-            "Offset of field: ",
-            stringify!(virtio_blk_config),
-            "::",
-            stringify!(wce)
-        )
-    );
-    assert_eq!(
-        unsafe { &(*(::std::ptr::null::<virtio_blk_config>())).unused as *const _ as usize },
-        33usize,
-        concat!(
-            "Offset of field: ",
-            stringify!(virtio_blk_config),
-            "::",
-            stringify!(unused)
-        )
-    );
-    assert_eq!(
-        unsafe { &(*(::std::ptr::null::<virtio_blk_config>())).num_queues as *const _ as usize },
-        34usize,
-        concat!(
-            "Offset of field: ",
-            stringify!(virtio_blk_config),
-            "::",
-            stringify!(num_queues)
-        )
-    );
-}
-#[repr(C)]
-#[derive(Debug, Default, Copy, Clone, PartialEq)]
-pub struct virtio_blk_outhdr {
-    pub type_: __virtio32,
-    pub ioprio: __virtio32,
-    pub sector: __virtio64,
-}
-#[test]
-fn bindgen_test_layout_virtio_blk_outhdr() {
-    assert_eq!(
-        ::std::mem::size_of::<virtio_blk_outhdr>(),
-        16usize,
-        concat!("Size of: ", stringify!(virtio_blk_outhdr))
-    );
-    assert_eq!(
-        ::std::mem::align_of::<virtio_blk_outhdr>(),
-        8usize,
-        concat!("Alignment of ", stringify!(virtio_blk_outhdr))
-    );
-    assert_eq!(
-        unsafe { &(*(::std::ptr::null::<virtio_blk_outhdr>())).type_ as *const _ as usize },
-        0usize,
-        concat!(
-            "Offset of field: ",
-            stringify!(virtio_blk_outhdr),
-            "::",
-            stringify!(type_)
-        )
-    );
-    assert_eq!(
-        unsafe { &(*(::std::ptr::null::<virtio_blk_outhdr>())).ioprio as *const _ as usize },
-        4usize,
-        concat!(
-            "Offset of field: ",
-            stringify!(virtio_blk_outhdr),
-            "::",
-            stringify!(ioprio)
-        )
-    );
-    assert_eq!(
-        unsafe { &(*(::std::ptr::null::<virtio_blk_outhdr>())).sector as *const _ as usize },
-        8usize,
-        concat!(
-            "Offset of field: ",
-            stringify!(virtio_blk_outhdr),
-            "::",
-            stringify!(sector)
-        )
-    );
-}
-#[repr(C)]
-#[derive(Debug, Default, Copy, Clone, PartialEq)]
-pub struct virtio_scsi_inhdr {
-    pub errors: __virtio32,
-    pub data_len: __virtio32,
-    pub sense_len: __virtio32,
-    pub residual: __virtio32,
-}
-#[test]
-fn bindgen_test_layout_virtio_scsi_inhdr() {
-    assert_eq!(
-        ::std::mem::size_of::<virtio_scsi_inhdr>(),
-        16usize,
-        concat!("Size of: ", stringify!(virtio_scsi_inhdr))
-    );
-    assert_eq!(
-        ::std::mem::align_of::<virtio_scsi_inhdr>(),
-        4usize,
-        concat!("Alignment of ", stringify!(virtio_scsi_inhdr))
-    );
-    assert_eq!(
-        unsafe { &(*(::std::ptr::null::<virtio_scsi_inhdr>())).errors as *const _ as usize },
-        0usize,
-        concat!(
-            "Offset of field: ",
-            stringify!(virtio_scsi_inhdr),
-            "::",
-            stringify!(errors)
-        )
-    );
-    assert_eq!(
-        unsafe { &(*(::std::ptr::null::<virtio_scsi_inhdr>())).data_len as *const _ as usize },
-        4usize,
-        concat!(
-            "Offset of field: ",
-            stringify!(virtio_scsi_inhdr),
-            "::",
-            stringify!(data_len)
-        )
-    );
-    assert_eq!(
-        unsafe { &(*(::std::ptr::null::<virtio_scsi_inhdr>())).sense_len as *const _ as usize },
-        8usize,
-        concat!(
-            "Offset of field: ",
-            stringify!(virtio_scsi_inhdr),
-            "::",
-            stringify!(sense_len)
-        )
-    );
-    assert_eq!(
-        unsafe { &(*(::std::ptr::null::<virtio_scsi_inhdr>())).residual as *const _ as usize },
-        12usize,
-        concat!(
-            "Offset of field: ",
-            stringify!(virtio_scsi_inhdr),
-            "::",
-            stringify!(residual)
-        )
-    );
-}

virtio-bindings/src/virtio_net.rs

@@ -1,734 +0,0 @@
-// Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
-// SPDX-License-Identifier: Apache-2.0
-
-/* automatically generated by rust-bindgen */
-
-#[repr(C)]
-#[derive(Default)]
-pub struct __IncompleteArrayField<T>(::std::marker::PhantomData<T>, [T; 0]);
-impl<T> __IncompleteArrayField<T> {
-    #[inline]
-    pub fn new() -> Self {
-        __IncompleteArrayField(::std::marker::PhantomData, [])
-    }
-    #[inline]
-    pub unsafe fn as_ptr(&self) -> *const T {
-        ::std::mem::transmute(self)
-    }
-    #[inline]
-    pub unsafe fn as_mut_ptr(&mut self) -> *mut T {
-        ::std::mem::transmute(self)
-    }
-    #[inline]
-    pub unsafe fn as_slice(&self, len: usize) -> &[T] {
-        ::std::slice::from_raw_parts(self.as_ptr(), len)
-    }
-    #[inline]
-    pub unsafe fn as_mut_slice(&mut self, len: usize) -> &mut [T] {
-        ::std::slice::from_raw_parts_mut(self.as_mut_ptr(), len)
-    }
-}
-impl<T> ::std::fmt::Debug for __IncompleteArrayField<T> {
-    fn fmt(&self, fmt: &mut ::std::fmt::Formatter<'_>) -> ::std::fmt::Result {
-        fmt.write_str("__IncompleteArrayField")
-    }
-}
-impl<T> ::std::clone::Clone for __IncompleteArrayField<T> {
-    #[inline]
-    fn clone(&self) -> Self {
-        Self::new()
-    }
-}
-pub const __BITS_PER_LONG: u32 = 64;
-pub const __FD_SETSIZE: u32 = 1024;
-pub const VIRTIO_ID_NET: u32 = 1;
-pub const VIRTIO_ID_BLOCK: u32 = 2;
-pub const VIRTIO_ID_CONSOLE: u32 = 3;
-pub const VIRTIO_ID_RNG: u32 = 4;
-pub const VIRTIO_ID_BALLOON: u32 = 5;
-pub const VIRTIO_ID_RPMSG: u32 = 7;
-pub const VIRTIO_ID_SCSI: u32 = 8;
-pub const VIRTIO_ID_9P: u32 = 9;
-pub const VIRTIO_ID_RPROC_SERIAL: u32 = 11;
-pub const VIRTIO_ID_CAIF: u32 = 12;
-pub const VIRTIO_ID_GPU: u32 = 16;
-pub const VIRTIO_ID_INPUT: u32 = 18;
-pub const VIRTIO_CONFIG_S_ACKNOWLEDGE: u32 = 1;
-pub const VIRTIO_CONFIG_S_DRIVER: u32 = 2;
-pub const VIRTIO_CONFIG_S_DRIVER_OK: u32 = 4;
-pub const VIRTIO_CONFIG_S_FEATURES_OK: u32 = 8;
-pub const VIRTIO_CONFIG_S_FAILED: u32 = 128;
-pub const VIRTIO_TRANSPORT_F_START: u32 = 28;
-pub const VIRTIO_TRANSPORT_F_END: u32 = 33;
-pub const VIRTIO_F_NOTIFY_ON_EMPTY: u32 = 24;
-pub const VIRTIO_F_ANY_LAYOUT: u32 = 27;
-pub const VIRTIO_F_VERSION_1: u32 = 32;
-pub const ETH_ALEN: u32 = 6;
-pub const ETH_TLEN: u32 = 2;
-pub const ETH_HLEN: u32 = 14;
-pub const ETH_ZLEN: u32 = 60;
-pub const ETH_DATA_LEN: u32 = 1500;
-pub const ETH_FRAME_LEN: u32 = 1514;
-pub const ETH_FCS_LEN: u32 = 4;
-pub const ETH_P_LOOP: u32 = 96;
-pub const ETH_P_PUP: u32 = 512;
-pub const ETH_P_PUPAT: u32 = 513;
-pub const ETH_P_TSN: u32 = 8944;
-pub const ETH_P_IP: u32 = 2048;
-pub const ETH_P_X25: u32 = 2053;
-pub const ETH_P_ARP: u32 = 2054;
-pub const ETH_P_BPQ: u32 = 2303;
-pub const ETH_P_IEEEPUP: u32 = 2560;
-pub const ETH_P_IEEEPUPAT: u32 = 2561;
-pub const ETH_P_BATMAN: u32 = 17157;
-pub const ETH_P_DEC: u32 = 24576;
-pub const ETH_P_DNA_DL: u32 = 24577;
-pub const ETH_P_DNA_RC: u32 = 24578;
-pub const ETH_P_DNA_RT: u32 = 24579;
-pub const ETH_P_LAT: u32 = 24580;
-pub const ETH_P_DIAG: u32 = 24581;
-pub const ETH_P_CUST: u32 = 24582;
-pub const ETH_P_SCA: u32 = 24583;
-pub const ETH_P_TEB: u32 = 25944;
-pub const ETH_P_RARP: u32 = 32821;
-pub const ETH_P_ATALK: u32 = 32923;
-pub const ETH_P_AARP: u32 = 33011;
-pub const ETH_P_8021Q: u32 = 33024;
-pub const ETH_P_IPX: u32 = 33079;
-pub const ETH_P_IPV6: u32 = 34525;
-pub const ETH_P_PAUSE: u32 = 34824;
-pub const ETH_P_SLOW: u32 = 34825;
-pub const ETH_P_WCCP: u32 = 34878;
-pub const ETH_P_MPLS_UC: u32 = 34887;
-pub const ETH_P_MPLS_MC: u32 = 34888;
-pub const ETH_P_ATMMPOA: u32 = 34892;
-pub const ETH_P_PPP_DISC: u32 = 34915;
-pub const ETH_P_PPP_SES: u32 = 34916;
-pub const ETH_P_LINK_CTL: u32 = 34924;
-pub const ETH_P_ATMFATE: u32 = 34948;
-pub const ETH_P_PAE: u32 = 34958;
-pub const ETH_P_AOE: u32 = 34978;
-pub const ETH_P_8021AD: u32 = 34984;
-pub const ETH_P_802_EX1: u32 = 34997;
-pub const ETH_P_TIPC: u32 = 35018;
-pub const ETH_P_8021AH: u32 = 35047;
-pub const ETH_P_MVRP: u32 = 35061;
-pub const ETH_P_1588: u32 = 35063;
-pub const ETH_P_PRP: u32 = 35067;
-pub const ETH_P_FCOE: u32 = 35078;
-pub const ETH_P_TDLS: u32 = 35085;
-pub const ETH_P_FIP: u32 = 35092;
-pub const ETH_P_80221: u32 = 35095;
-pub const ETH_P_LOOPBACK: u32 = 36864;
-pub const ETH_P_QINQ1: u32 = 37120;
-pub const ETH_P_QINQ2: u32 = 37376;
-pub const ETH_P_QINQ3: u32 = 37632;
-pub const ETH_P_EDSA: u32 = 56026;
-pub const ETH_P_AF_IUCV: u32 = 64507;
-pub const ETH_P_802_3_MIN: u32 = 1536;
-pub const ETH_P_802_3: u32 = 1;
-pub const ETH_P_AX25: u32 = 2;
-pub const ETH_P_ALL: u32 = 3;
-pub const ETH_P_802_2: u32 = 4;
-pub const ETH_P_SNAP: u32 = 5;
-pub const ETH_P_DDCMP: u32 = 6;
-pub const ETH_P_WAN_PPP: u32 = 7;
-pub const ETH_P_PPP_MP: u32 = 8;
-pub const ETH_P_LOCALTALK: u32 = 9;
-pub const ETH_P_CAN: u32 = 12;
-pub const ETH_P_CANFD: u32 = 13;
-pub const ETH_P_PPPTALK: u32 = 16;
-pub const ETH_P_TR_802_2: u32 = 17;
-pub const ETH_P_MOBITEX: u32 = 21;
-pub const ETH_P_CONTROL: u32 = 22;
-pub const ETH_P_IRDA: u32 = 23;
-pub const ETH_P_ECONET: u32 = 24;
-pub const ETH_P_HDLC: u32 = 25;
-pub const ETH_P_ARCNET: u32 = 26;
-pub const ETH_P_DSA: u32 = 27;
-pub const ETH_P_TRAILER: u32 = 28;
-pub const ETH_P_PHONET: u32 = 245;
-pub const ETH_P_IEEE802154: u32 = 246;
-pub const ETH_P_CAIF: u32 = 247;
-pub const ETH_P_XDSA: u32 = 248;
-pub const VIRTIO_NET_F_CSUM: u32 = 0;
-pub const VIRTIO_NET_F_GUEST_CSUM: u32 = 1;
-pub const VIRTIO_NET_F_CTRL_GUEST_OFFLOADS: u32 = 2;
-pub const VIRTIO_NET_F_MTU: u32 = 3;
-pub const VIRTIO_NET_F_MAC: u32 = 5;
-pub const VIRTIO_NET_F_GUEST_TSO4: u32 = 7;
-pub const VIRTIO_NET_F_GUEST_TSO6: u32 = 8;
-pub const VIRTIO_NET_F_GUEST_ECN: u32 = 9;
-pub const VIRTIO_NET_F_GUEST_UFO: u32 = 10;
-pub const VIRTIO_NET_F_HOST_TSO4: u32 = 11;
-pub const VIRTIO_NET_F_HOST_TSO6: u32 = 12;
-pub const VIRTIO_NET_F_HOST_ECN: u32 = 13;
-pub const VIRTIO_NET_F_HOST_UFO: u32 = 14;
-pub const VIRTIO_NET_F_MRG_RXBUF: u32 = 15;
-pub const VIRTIO_NET_F_STATUS: u32 = 16;
-pub const VIRTIO_NET_F_CTRL_VQ: u32 = 17;
-pub const VIRTIO_NET_F_CTRL_RX: u32 = 18;
-pub const VIRTIO_NET_F_CTRL_VLAN: u32 = 19;
-pub const VIRTIO_NET_F_CTRL_RX_EXTRA: u32 = 20;
-pub const VIRTIO_NET_F_GUEST_ANNOUNCE: u32 = 21;
-pub const VIRTIO_NET_F_MQ: u32 = 22;
-pub const VIRTIO_NET_F_CTRL_MAC_ADDR: u32 = 23;
-pub const VIRTIO_NET_F_GSO: u32 = 6;
-pub const VIRTIO_NET_S_LINK_UP: u32 = 1;
-pub const VIRTIO_NET_S_ANNOUNCE: u32 = 2;
-pub const VIRTIO_NET_HDR_F_NEEDS_CSUM: u32 = 1;
-pub const VIRTIO_NET_HDR_F_DATA_VALID: u32 = 2;
-pub const VIRTIO_NET_HDR_GSO_NONE: u32 = 0;
-pub const VIRTIO_NET_HDR_GSO_TCPV4: u32 = 1;
-pub const VIRTIO_NET_HDR_GSO_UDP: u32 = 3;
-pub const VIRTIO_NET_HDR_GSO_TCPV6: u32 = 4;
-pub const VIRTIO_NET_HDR_GSO_ECN: u32 = 128;
-pub const VIRTIO_NET_OK: u32 = 0;
-pub const VIRTIO_NET_ERR: u32 = 1;
-pub const VIRTIO_NET_CTRL_RX: u32 = 0;
-pub const VIRTIO_NET_CTRL_RX_PROMISC: u32 = 0;
-pub const VIRTIO_NET_CTRL_RX_ALLMULTI: u32 = 1;
-pub const VIRTIO_NET_CTRL_RX_ALLUNI: u32 = 2;
-pub const VIRTIO_NET_CTRL_RX_NOMULTI: u32 = 3;
-pub const VIRTIO_NET_CTRL_RX_NOUNI: u32 = 4;
-pub const VIRTIO_NET_CTRL_RX_NOBCAST: u32 = 5;
-pub const VIRTIO_NET_CTRL_MAC: u32 = 1;
-pub const VIRTIO_NET_CTRL_MAC_TABLE_SET: u32 = 0;
-pub const VIRTIO_NET_CTRL_MAC_ADDR_SET: u32 = 1;
-pub const VIRTIO_NET_CTRL_VLAN: u32 = 2;
-pub const VIRTIO_NET_CTRL_VLAN_ADD: u32 = 0;
-pub const VIRTIO_NET_CTRL_VLAN_DEL: u32 = 1;
-pub const VIRTIO_NET_CTRL_ANNOUNCE: u32 = 3;
-pub const VIRTIO_NET_CTRL_ANNOUNCE_ACK: u32 = 0;
-pub const VIRTIO_NET_CTRL_MQ: u32 = 4;
-pub const VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET: u32 = 0;
-pub const VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MIN: u32 = 1;
-pub const VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX: u32 = 32768;
-pub const VIRTIO_NET_CTRL_GUEST_OFFLOADS: u32 = 5;
-pub const VIRTIO_NET_CTRL_GUEST_OFFLOADS_SET: u32 = 0;
-pub type __s8 = ::std::os::raw::c_schar;
-pub type __u8 = ::std::os::raw::c_uchar;
-pub type __s16 = ::std::os::raw::c_short;
-pub type __u16 = ::std::os::raw::c_ushort;
-pub type __s32 = ::std::os::raw::c_int;
-pub type __u32 = ::std::os::raw::c_uint;
-pub type __s64 = ::std::os::raw::c_longlong;
-pub type __u64 = ::std::os::raw::c_ulonglong;
-#[repr(C)]
-#[derive(Debug, Default, Copy, Clone, PartialEq)]
-pub struct __kernel_fd_set {
-    pub fds_bits: [::std::os::raw::c_ulong; 16usize],
-}
-#[test]
-fn bindgen_test_layout___kernel_fd_set() {
-    assert_eq!(
-        ::std::mem::size_of::<__kernel_fd_set>(),
-        128usize,
-        concat!("Size of: ", stringify!(__kernel_fd_set))
-    );
-    assert_eq!(
-        ::std::mem::align_of::<__kernel_fd_set>(),
-        8usize,
-        concat!("Alignment of ", stringify!(__kernel_fd_set))
-    );
-    assert_eq!(
-        unsafe { &(*(::std::ptr::null::<__kernel_fd_set>())).fds_bits as *const _ as usize },
-        0usize,
-        concat!(
-            "Offset of field: ",
-            stringify!(__kernel_fd_set),
-            "::",
-            stringify!(fds_bits)
-        )
-    );
-}
-pub type __kernel_sighandler_t =
-    ::std::option::Option<unsafe extern "C" fn(arg1: ::std::os::raw::c_int)>;
-pub type __kernel_key_t = ::std::os::raw::c_int;
-pub type __kernel_mqd_t = ::std::os::raw::c_int;
-pub type __kernel_old_uid_t = ::std::os::raw::c_ushort;
-pub type __kernel_old_gid_t = ::std::os::raw::c_ushort;
-pub type __kernel_old_dev_t = ::std::os::raw::c_ulong;
-pub type __kernel_long_t = ::std::os::raw::c_long;
-pub type __kernel_ulong_t = ::std::os::raw::c_ulong;
-pub type __kernel_ino_t = __kernel_ulong_t;
-pub type __kernel_mode_t = ::std::os::raw::c_uint;
-pub type __kernel_pid_t = ::std::os::raw::c_int;
-pub type __kernel_ipc_pid_t = ::std::os::raw::c_int;
-pub type __kernel_uid_t = ::std::os::raw::c_uint;
-pub type __kernel_gid_t = ::std::os::raw::c_uint;
-pub type __kernel_suseconds_t = __kernel_long_t;
-pub type __kernel_daddr_t = ::std::os::raw::c_int;
-pub type __kernel_uid32_t = ::std::os::raw::c_uint;
-pub type __kernel_gid32_t = ::std::os::raw::c_uint;
-pub type __kernel_size_t = __kernel_ulong_t;
-pub type __kernel_ssize_t = __kernel_long_t;
-pub type __kernel_ptrdiff_t = __kernel_long_t;
-#[repr(C)]
-#[derive(Debug, Default, Copy, Clone, PartialEq)]
-pub struct __kernel_fsid_t {
-    pub val: [::std::os::raw::c_int; 2usize],
-}
-#[test]
-fn bindgen_test_layout___kernel_fsid_t() {
-    assert_eq!(
-        ::std::mem::size_of::<__kernel_fsid_t>(),
-        8usize,
-        concat!("Size of: ", stringify!(__kernel_fsid_t))
-    );
-    assert_eq!(
-        ::std::mem::align_of::<__kernel_fsid_t>(),
-        4usize,
-        concat!("Alignment of ", stringify!(__kernel_fsid_t))
-    );
-    assert_eq!(
-        unsafe { &(*(::std::ptr::null::<__kernel_fsid_t>())).val as *const _ as usize },
-        0usize,
-        concat!(
-            "Offset of field: ",
-            stringify!(__kernel_fsid_t),
-            "::",
-            stringify!(val)
-        )
-    );
-}
-pub type __kernel_off_t = __kernel_long_t;
-pub type __kernel_loff_t = ::std::os::raw::c_longlong;
-pub type __kernel_time_t = __kernel_long_t;
-pub type __kernel_clock_t = __kernel_long_t;
-pub type __kernel_timer_t = ::std::os::raw::c_int;
-pub type __kernel_clockid_t = ::std::os::raw::c_int;
-pub type __kernel_caddr_t = *mut ::std::os::raw::c_char;
-pub type __kernel_uid16_t = ::std::os::raw::c_ushort;
-pub type __kernel_gid16_t = ::std::os::raw::c_ushort;
-pub type __le16 = __u16;
-pub type __be16 = __u16;
-pub type __le32 = __u32;
-pub type __be32 = __u32;
-pub type __le64 = __u64;
-pub type __be64 = __u64;
-pub type __sum16 = __u16;
-pub type __wsum = __u32;
-pub type __virtio16 = __u16;
-pub type __virtio32 = __u32;
-pub type __virtio64 = __u64;
-#[repr(C, packed)]
-#[derive(Debug, Default, Copy, Clone, PartialEq)]
-pub struct ethhdr {
-    pub h_dest: [::std::os::raw::c_uchar; 6usize],
-    pub h_source: [::std::os::raw::c_uchar; 6usize],
-    pub h_proto: __be16,
-}
-#[test]
-fn bindgen_test_layout_ethhdr() {
-    assert_eq!(
-        ::std::mem::size_of::<ethhdr>(),
-        14usize,
-        concat!("Size of: ", stringify!(ethhdr))
-    );
-    assert_eq!(
-        ::std::mem::align_of::<ethhdr>(),
-        1usize,
-        concat!("Alignment of ", stringify!(ethhdr))
-    );
-    assert_eq!(
-        unsafe { &(*(::std::ptr::null::<ethhdr>())).h_dest as *const _ as usize },
-        0usize,
-        concat!(
-            "Offset of field: ",
-            stringify!(ethhdr),
-            "::",
-            stringify!(h_dest)
-        )
-    );
-    assert_eq!(
-        unsafe { &(*(::std::ptr::null::<ethhdr>())).h_source as *const _ as usize },
-        6usize,
-        concat!(
-            "Offset of field: ",
-            stringify!(ethhdr),
-            "::",
-            stringify!(h_source)
-        )
-    );
-    assert_eq!(
-        unsafe { &(*(::std::ptr::null::<ethhdr>())).h_proto as *const _ as usize },
-        12usize,
-        concat!(
-            "Offset of field: ",
-            stringify!(ethhdr),
-            "::",
-            stringify!(h_proto)
-        )
-    );
-}
-#[repr(C, packed)]
-#[derive(Debug, Default, Copy, Clone, PartialEq)]
-pub struct virtio_net_config {
-    pub mac: [__u8; 6usize],
-    pub status: __u16,
-    pub max_virtqueue_pairs: __u16,
-    pub mtu: __u16,
-}
-#[test]
-fn bindgen_test_layout_virtio_net_config() {
-    assert_eq!(
-        ::std::mem::size_of::<virtio_net_config>(),
-        12usize,
-        concat!("Size of: ", stringify!(virtio_net_config))
-    );
-    assert_eq!(
-        ::std::mem::align_of::<virtio_net_config>(),
-        1usize,
-        concat!("Alignment of ", stringify!(virtio_net_config))
-    );
-    assert_eq!(
-        unsafe { &(*(::std::ptr::null::<virtio_net_config>())).mac as *const _ as usize },
-        0usize,
-        concat!(
-            "Offset of field: ",
-            stringify!(virtio_net_config),
-            "::",
-            stringify!(mac)
-        )
-    );
-    assert_eq!(
-        unsafe { &(*(::std::ptr::null::<virtio_net_config>())).status as *const _ as usize },
-        6usize,
-        concat!(
-            "Offset of field: ",
-            stringify!(virtio_net_config),
-            "::",
-            stringify!(status)
-        )
-    );
-    assert_eq!(
-        unsafe {
-            &(*(::std::ptr::null::<virtio_net_config>())).max_virtqueue_pairs as *const _ as usize
-        },
-        8usize,
-        concat!(
-            "Offset of field: ",
-            stringify!(virtio_net_config),
-            "::",
-            stringify!(max_virtqueue_pairs)
-        )
-    );
-    assert_eq!(
-        unsafe { &(*(::std::ptr::null::<virtio_net_config>())).mtu as *const _ as usize },
-        10usize,
-        concat!(
-            "Offset of field: ",
-            stringify!(virtio_net_config),
-            "::",
-            stringify!(mtu)
-        )
-    );
-}
-#[repr(C)]
-#[derive(Debug, Default, Copy, Clone, PartialEq)]
-pub struct virtio_net_hdr_v1 {
-    pub flags: __u8,
-    pub gso_type: __u8,
-    pub hdr_len: __virtio16,
-    pub gso_size: __virtio16,
-    pub csum_start: __virtio16,
-    pub csum_offset: __virtio16,
-    pub num_buffers: __virtio16,
-}
-#[test]
-fn bindgen_test_layout_virtio_net_hdr_v1() {
-    assert_eq!(
-        ::std::mem::size_of::<virtio_net_hdr_v1>(),
-        12usize,
-        concat!("Size of: ", stringify!(virtio_net_hdr_v1))
-    );
-    assert_eq!(
-        ::std::mem::align_of::<virtio_net_hdr_v1>(),
-        2usize,
-        concat!("Alignment of ", stringify!(virtio_net_hdr_v1))
-    );
-    assert_eq!(
-        unsafe { &(*(::std::ptr::null::<virtio_net_hdr_v1>())).flags as *const _ as usize },
-        0usize,
-        concat!(
-            "Offset of field: ",
-            stringify!(virtio_net_hdr_v1),
-            "::",
-            stringify!(flags)
-        )
-    );
-    assert_eq!(
-        unsafe { &(*(::std::ptr::null::<virtio_net_hdr_v1>())).gso_type as *const _ as usize },
-        1usize,
-        concat!(
-            "Offset of field: ",
-            stringify!(virtio_net_hdr_v1),
-            "::",
-            stringify!(gso_type)
-        )
-    );
-    assert_eq!(
-        unsafe { &(*(::std::ptr::null::<virtio_net_hdr_v1>())).hdr_len as *const _ as usize },
-        2usize,
-        concat!(
-            "Offset of field: ",
-            stringify!(virtio_net_hdr_v1),
-            "::",
-            stringify!(hdr_len)
-        )
-    );
-    assert_eq!(
-        unsafe { &(*(::std::ptr::null::<virtio_net_hdr_v1>())).gso_size as *const _ as usize },
-        4usize,
-        concat!(
-            "Offset of field: ",
-            stringify!(virtio_net_hdr_v1),
-            "::",
-            stringify!(gso_size)
-        )
-    );
-    assert_eq!(
-        unsafe { &(*(::std::ptr::null::<virtio_net_hdr_v1>())).csum_start as *const _ as usize },
-        6usize,
-        concat!(
-            "Offset of field: ",
-            stringify!(virtio_net_hdr_v1),
-            "::",
-            stringify!(csum_start)
-        )
-    );
-    assert_eq!(
-        unsafe { &(*(::std::ptr::null::<virtio_net_hdr_v1>())).csum_offset as *const _ as usize },
-        8usize,
-        concat!(
-            "Offset of field: ",
-            stringify!(virtio_net_hdr_v1),
-            "::",
-            stringify!(csum_offset)
-        )
-    );
-    assert_eq!(
-        unsafe { &(*(::std::ptr::null::<virtio_net_hdr_v1>())).num_buffers as *const _ as usize },
-        10usize,
-        concat!(
-            "Offset of field: ",
-            stringify!(virtio_net_hdr_v1),
-            "::",
-            stringify!(num_buffers)
-        )
-    );
-}
-#[repr(C)]
-#[derive(Debug, Default, Copy, Clone, PartialEq)]
-pub struct virtio_net_hdr {
-    pub flags: __u8,
-    pub gso_type: __u8,
-    pub hdr_len: __virtio16,
-    pub gso_size: __virtio16,
-    pub csum_start: __virtio16,
-    pub csum_offset: __virtio16,
-}
-#[test]
-fn bindgen_test_layout_virtio_net_hdr() {
-    assert_eq!(
-        ::std::mem::size_of::<virtio_net_hdr>(),
-        10usize,
-        concat!("Size of: ", stringify!(virtio_net_hdr))
-    );
-    assert_eq!(
-        ::std::mem::align_of::<virtio_net_hdr>(),
-        2usize,
-        concat!("Alignment of ", stringify!(virtio_net_hdr))
-    );
-    assert_eq!(
-        unsafe { &(*(::std::ptr::null::<virtio_net_hdr>())).flags as *const _ as usize },
-        0usize,
-        concat!(
-            "Offset of field: ",
-            stringify!(virtio_net_hdr),
-            "::",
-            stringify!(flags)
-        )
-    );
-    assert_eq!(
-        unsafe { &(*(::std::ptr::null::<virtio_net_hdr>())).gso_type as *const _ as usize },
-        1usize,
-        concat!(
-            "Offset of field: ",
-            stringify!(virtio_net_hdr),
-            "::",
-            stringify!(gso_type)
-        )
-    );
-    assert_eq!(
-        unsafe { &(*(::std::ptr::null::<virtio_net_hdr>())).hdr_len as *const _ as usize },
-        2usize,
-        concat!(
-            "Offset of field: ",
-            stringify!(virtio_net_hdr),
-            "::",
-            stringify!(hdr_len)
-        )
-    );
-    assert_eq!(
-        unsafe { &(*(::std::ptr::null::<virtio_net_hdr>())).gso_size as *const _ as usize },
-        4usize,
-        concat!(
-            "Offset of field: ",
-            stringify!(virtio_net_hdr),
-            "::",
-            stringify!(gso_size)
-        )
-    );
-    assert_eq!(
-        unsafe { &(*(::std::ptr::null::<virtio_net_hdr>())).csum_start as *const _ as usize },
-        6usize,
-        concat!(
-            "Offset of field: ",
-            stringify!(virtio_net_hdr),
-            "::",
-            stringify!(csum_start)
-        )
-    );
-    assert_eq!(
-        unsafe { &(*(::std::ptr::null::<virtio_net_hdr>())).csum_offset as *const _ as usize },
-        8usize,
-        concat!(
-            "Offset of field: ",
-            stringify!(virtio_net_hdr),
-            "::",
-            stringify!(csum_offset)
-        )
-    );
-}
-#[repr(C)]
-#[derive(Debug, Default, Copy, Clone, PartialEq)]
-pub struct virtio_net_hdr_mrg_rxbuf {
-    pub hdr: virtio_net_hdr,
-    pub num_buffers: __virtio16,
-}
-#[test]
-fn bindgen_test_layout_virtio_net_hdr_mrg_rxbuf() {
-    assert_eq!(
-        ::std::mem::size_of::<virtio_net_hdr_mrg_rxbuf>(),
-        12usize,
-        concat!("Size of: ", stringify!(virtio_net_hdr_mrg_rxbuf))
-    );
-    assert_eq!(
-        ::std::mem::align_of::<virtio_net_hdr_mrg_rxbuf>(),
-        2usize,
-        concat!("Alignment of ", stringify!(virtio_net_hdr_mrg_rxbuf))
-    );
-    assert_eq!(
-        unsafe { &(*(::std::ptr::null::<virtio_net_hdr_mrg_rxbuf>())).hdr as *const _ as usize },
-        0usize,
-        concat!(
-            "Offset of field: ",
-            stringify!(virtio_net_hdr_mrg_rxbuf),
-            "::",
-            stringify!(hdr)
-        )
-    );
-    assert_eq!(
-        unsafe {
-            &(*(::std::ptr::null::<virtio_net_hdr_mrg_rxbuf>())).num_buffers as *const _ as usize
-        },
-        10usize,
-        concat!(
-            "Offset of field: ",
-            stringify!(virtio_net_hdr_mrg_rxbuf),
-            "::",
-            stringify!(num_buffers)
-        )
-    );
-}
-#[repr(C, packed)]
-#[derive(Debug, Default, Copy, Clone, PartialEq)]
-pub struct virtio_net_ctrl_hdr {
-    pub class: __u8,
-    pub cmd: __u8,
-}
-#[test]
-fn bindgen_test_layout_virtio_net_ctrl_hdr() {
-    assert_eq!(
-        ::std::mem::size_of::<virtio_net_ctrl_hdr>(),
-        2usize,
-        concat!("Size of: ", stringify!(virtio_net_ctrl_hdr))
-    );
-    assert_eq!(
-        ::std::mem::align_of::<virtio_net_ctrl_hdr>(),
-        1usize,
-        concat!("Alignment of ", stringify!(virtio_net_ctrl_hdr))
-    );
-    assert_eq!(
-        unsafe { &(*(::std::ptr::null::<virtio_net_ctrl_hdr>())).class as *const _ as usize },
-        0usize,
-        concat!(
-            "Offset of field: ",
-            stringify!(virtio_net_ctrl_hdr),
-            "::",
-            stringify!(class)
-        )
-    );
-    assert_eq!(
-        unsafe { &(*(::std::ptr::null::<virtio_net_ctrl_hdr>())).cmd as *const _ as usize },
-        1usize,
-        concat!(
-            "Offset of field: ",
-            stringify!(virtio_net_ctrl_hdr),
-            "::",
-            stringify!(cmd)
-        )
-    );
-}
-pub type virtio_net_ctrl_ack = __u8;
-#[repr(C, packed)]
-#[derive(Default)]
-pub struct virtio_net_ctrl_mac {
-    pub entries: __virtio32,
-    pub macs: __IncompleteArrayField<[__u8; 6usize]>,
-}
-#[test]
-fn bindgen_test_layout_virtio_net_ctrl_mac() {
-    assert_eq!(
-        ::std::mem::size_of::<virtio_net_ctrl_mac>(),
-        4usize,
-        concat!("Size of: ", stringify!(virtio_net_ctrl_mac))
-    );
-    assert_eq!(
-        ::std::mem::align_of::<virtio_net_ctrl_mac>(),
-        1usize,
-        concat!("Alignment of ", stringify!(virtio_net_ctrl_mac))
-    );
-}
-#[repr(C)]
-#[derive(Debug, Default, Copy, Clone, PartialEq)]
-pub struct virtio_net_ctrl_mq {
-    pub virtqueue_pairs: __virtio16,
-}
-#[test]
-fn bindgen_test_layout_virtio_net_ctrl_mq() {
-    assert_eq!(
-        ::std::mem::size_of::<virtio_net_ctrl_mq>(),
-        2usize,
-        concat!("Size of: ", stringify!(virtio_net_ctrl_mq))
-    );
-    assert_eq!(
-        ::std::mem::align_of::<virtio_net_ctrl_mq>(),
-        2usize,
-        concat!("Alignment of ", stringify!(virtio_net_ctrl_mq))
-    );
-    assert_eq!(
-        unsafe {
-            &(*(::std::ptr::null::<virtio_net_ctrl_mq>())).virtqueue_pairs as *const _ as usize
-        },
-        0usize,
-        concat!(
-            "Offset of field: ",
-            stringify!(virtio_net_ctrl_mq),
-            "::",
-            stringify!(virtqueue_pairs)
-        )
-    );
-}

virtio-bindings/src/virtio_ring.rs

@@ -1,453 +0,0 @@
-// Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
-// SPDX-License-Identifier: Apache-2.0
-
-/* automatically generated by rust-bindgen */
-
-#[repr(C)]
-#[derive(Default)]
-pub struct __IncompleteArrayField<T>(::std::marker::PhantomData<T>, [T; 0]);
-impl<T> __IncompleteArrayField<T> {
-    #[inline]
-    pub fn new() -> Self {
-        __IncompleteArrayField(::std::marker::PhantomData, [])
-    }
-    #[inline]
-    pub unsafe fn as_ptr(&self) -> *const T {
-        ::std::mem::transmute(self)
-    }
-    #[inline]
-    pub unsafe fn as_mut_ptr(&mut self) -> *mut T {
-        ::std::mem::transmute(self)
-    }
-    #[inline]
-    pub unsafe fn as_slice(&self, len: usize) -> &[T] {
-        ::std::slice::from_raw_parts(self.as_ptr(), len)
-    }
-    #[inline]
-    pub unsafe fn as_mut_slice(&mut self, len: usize) -> &mut [T] {
-        ::std::slice::from_raw_parts_mut(self.as_mut_ptr(), len)
-    }
-}
-impl<T> ::std::fmt::Debug for __IncompleteArrayField<T> {
-    fn fmt(&self, fmt: &mut ::std::fmt::Formatter<'_>) -> ::std::fmt::Result {
-        fmt.write_str("__IncompleteArrayField")
-    }
-}
-impl<T> ::std::clone::Clone for __IncompleteArrayField<T> {
-    #[inline]
-    fn clone(&self) -> Self {
-        Self::new()
-    }
-}
-pub const _STDINT_H: u32 = 1;
-pub const _FEATURES_H: u32 = 1;
-pub const _DEFAULT_SOURCE: u32 = 1;
-pub const __USE_ISOC11: u32 = 1;
-pub const __USE_ISOC99: u32 = 1;
-pub const __USE_ISOC95: u32 = 1;
-pub const __USE_POSIX_IMPLICITLY: u32 = 1;
-pub const _POSIX_SOURCE: u32 = 1;
-pub const _POSIX_C_SOURCE: u32 = 200809;
-pub const __USE_POSIX: u32 = 1;
-pub const __USE_POSIX2: u32 = 1;
-pub const __USE_POSIX199309: u32 = 1;
-pub const __USE_POSIX199506: u32 = 1;
-pub const __USE_XOPEN2K: u32 = 1;
-pub const __USE_XOPEN2K8: u32 = 1;
-pub const _ATFILE_SOURCE: u32 = 1;
-pub const __USE_MISC: u32 = 1;
-pub const __USE_ATFILE: u32 = 1;
-pub const __USE_FORTIFY_LEVEL: u32 = 0;
-pub const _STDC_PREDEF_H: u32 = 1;
-pub const __STDC_IEC_559__: u32 = 1;
-pub const __STDC_IEC_559_COMPLEX__: u32 = 1;
-pub const __STDC_ISO_10646__: u32 = 201505;
-pub const __STDC_NO_THREADS__: u32 = 1;
-pub const __GNU_LIBRARY__: u32 = 6;
-pub const __GLIBC__: u32 = 2;
-pub const __GLIBC_MINOR__: u32 = 23;
-pub const _SYS_CDEFS_H: u32 = 1;
-pub const __WORDSIZE: u32 = 64;
-pub const __WORDSIZE_TIME64_COMPAT32: u32 = 1;
-pub const __SYSCALL_WORDSIZE: u32 = 64;
-pub const _BITS_WCHAR_H: u32 = 1;
-pub const INT8_MIN: i32 = -128;
-pub const INT16_MIN: i32 = -32768;
-pub const INT32_MIN: i32 = -2147483648;
-pub const INT8_MAX: u32 = 127;
-pub const INT16_MAX: u32 = 32767;
-pub const INT32_MAX: u32 = 2147483647;
-pub const UINT8_MAX: u32 = 255;
-pub const UINT16_MAX: u32 = 65535;
-pub const UINT32_MAX: u32 = 4294967295;
-pub const INT_LEAST8_MIN: i32 = -128;
-pub const INT_LEAST16_MIN: i32 = -32768;
-pub const INT_LEAST32_MIN: i32 = -2147483648;
-pub const INT_LEAST8_MAX: u32 = 127;
-pub const INT_LEAST16_MAX: u32 = 32767;
-pub const INT_LEAST32_MAX: u32 = 2147483647;
-pub const UINT_LEAST8_MAX: u32 = 255;
-pub const UINT_LEAST16_MAX: u32 = 65535;
-pub const UINT_LEAST32_MAX: u32 = 4294967295;
-pub const INT_FAST8_MIN: i32 = -128;
-pub const INT_FAST16_MIN: i64 = -9223372036854775808;
-pub const INT_FAST32_MIN: i64 = -9223372036854775808;
-pub const INT_FAST8_MAX: u32 = 127;
-pub const INT_FAST16_MAX: u64 = 9223372036854775807;
-pub const INT_FAST32_MAX: u64 = 9223372036854775807;
-pub const UINT_FAST8_MAX: u32 = 255;
-pub const UINT_FAST16_MAX: i32 = -1;
-pub const UINT_FAST32_MAX: i32 = -1;
-pub const INTPTR_MIN: i64 = -9223372036854775808;
-pub const INTPTR_MAX: u64 = 9223372036854775807;
-pub const UINTPTR_MAX: i32 = -1;
-pub const PTRDIFF_MIN: i64 = -9223372036854775808;
-pub const PTRDIFF_MAX: u64 = 9223372036854775807;
-pub const SIG_ATOMIC_MIN: i32 = -2147483648;
-pub const SIG_ATOMIC_MAX: u32 = 2147483647;
-pub const SIZE_MAX: i32 = -1;
-pub const WINT_MIN: u32 = 0;
-pub const WINT_MAX: u32 = 4294967295;
-pub const __BITS_PER_LONG: u32 = 64;
-pub const __FD_SETSIZE: u32 = 1024;
-pub const VRING_DESC_F_NEXT: u32 = 1;
-pub const VRING_DESC_F_WRITE: u32 = 2;
-pub const VRING_DESC_F_INDIRECT: u32 = 4;
-pub const VRING_USED_F_NO_NOTIFY: u32 = 1;
-pub const VRING_AVAIL_F_NO_INTERRUPT: u32 = 1;
-pub const VIRTIO_RING_F_INDIRECT_DESC: u32 = 28;
-pub const VIRTIO_RING_F_EVENT_IDX: u32 = 29;
-pub const VRING_AVAIL_ALIGN_SIZE: u32 = 2;
-pub const VRING_USED_ALIGN_SIZE: u32 = 4;
-pub const VRING_DESC_ALIGN_SIZE: u32 = 16;
-pub type int_least8_t = ::std::os::raw::c_schar;
-pub type int_least16_t = ::std::os::raw::c_short;
-pub type int_least32_t = ::std::os::raw::c_int;
-pub type int_least64_t = ::std::os::raw::c_long;
-pub type uint_least8_t = ::std::os::raw::c_uchar;
-pub type uint_least16_t = ::std::os::raw::c_ushort;
-pub type uint_least32_t = ::std::os::raw::c_uint;
-pub type uint_least64_t = ::std::os::raw::c_ulong;
-pub type int_fast8_t = ::std::os::raw::c_schar;
-pub type int_fast16_t = ::std::os::raw::c_long;
-pub type int_fast32_t = ::std::os::raw::c_long;
-pub type int_fast64_t = ::std::os::raw::c_long;
-pub type uint_fast8_t = ::std::os::raw::c_uchar;
-pub type uint_fast16_t = ::std::os::raw::c_ulong;
-pub type uint_fast32_t = ::std::os::raw::c_ulong;
-pub type uint_fast64_t = ::std::os::raw::c_ulong;
-pub type intmax_t = ::std::os::raw::c_long;
-pub type uintmax_t = ::std::os::raw::c_ulong;
-pub type __s8 = ::std::os::raw::c_schar;
-pub type __u8 = ::std::os::raw::c_uchar;
-pub type __s16 = ::std::os::raw::c_short;
-pub type __u16 = ::std::os::raw::c_ushort;
-pub type __s32 = ::std::os::raw::c_int;
-pub type __u32 = ::std::os::raw::c_uint;
-pub type __s64 = ::std::os::raw::c_longlong;
-pub type __u64 = ::std::os::raw::c_ulonglong;
-#[repr(C)]
-#[derive(Debug, Default, Copy, Clone, PartialEq)]
-pub struct __kernel_fd_set {
-    pub fds_bits: [::std::os::raw::c_ulong; 16usize],
-}
-#[test]
-fn bindgen_test_layout___kernel_fd_set() {
-    assert_eq!(
-        ::std::mem::size_of::<__kernel_fd_set>(),
-        128usize,
-        concat!("Size of: ", stringify!(__kernel_fd_set))
-    );
-    assert_eq!(
-        ::std::mem::align_of::<__kernel_fd_set>(),
-        8usize,
-        concat!("Alignment of ", stringify!(__kernel_fd_set))
-    );
-    assert_eq!(
-        unsafe { &(*(::std::ptr::null::<__kernel_fd_set>())).fds_bits as *const _ as usize },
-        0usize,
-        concat!(
-            "Offset of field: ",
-            stringify!(__kernel_fd_set),
-            "::",
-            stringify!(fds_bits)
-        )
-    );
-}
-pub type __kernel_sighandler_t =
-    ::std::option::Option<unsafe extern "C" fn(arg1: ::std::os::raw::c_int)>;
-pub type __kernel_key_t = ::std::os::raw::c_int;
-pub type __kernel_mqd_t = ::std::os::raw::c_int;
-pub type __kernel_old_uid_t = ::std::os::raw::c_ushort;
-pub type __kernel_old_gid_t = ::std::os::raw::c_ushort;
-pub type __kernel_old_dev_t = ::std::os::raw::c_ulong;
-pub type __kernel_long_t = ::std::os::raw::c_long;
-pub type __kernel_ulong_t = ::std::os::raw::c_ulong;
-pub type __kernel_ino_t = __kernel_ulong_t;
-pub type __kernel_mode_t = ::std::os::raw::c_uint;
-pub type __kernel_pid_t = ::std::os::raw::c_int;
-pub type __kernel_ipc_pid_t = ::std::os::raw::c_int;
-pub type __kernel_uid_t = ::std::os::raw::c_uint;
-pub type __kernel_gid_t = ::std::os::raw::c_uint;
-pub type __kernel_suseconds_t = __kernel_long_t;
-pub type __kernel_daddr_t = ::std::os::raw::c_int;
-pub type __kernel_uid32_t = ::std::os::raw::c_uint;
-pub type __kernel_gid32_t = ::std::os::raw::c_uint;
-pub type __kernel_size_t = __kernel_ulong_t;
-pub type __kernel_ssize_t = __kernel_long_t;
-pub type __kernel_ptrdiff_t = __kernel_long_t;
-#[repr(C)]
-#[derive(Debug, Default, Copy, Clone, PartialEq)]
-pub struct __kernel_fsid_t {
-    pub val: [::std::os::raw::c_int; 2usize],
-}
-#[test]
-fn bindgen_test_layout___kernel_fsid_t() {
-    assert_eq!(
-        ::std::mem::size_of::<__kernel_fsid_t>(),
-        8usize,
-        concat!("Size of: ", stringify!(__kernel_fsid_t))
-    );
-    assert_eq!(
-        ::std::mem::align_of::<__kernel_fsid_t>(),
-        4usize,
-        concat!("Alignment of ", stringify!(__kernel_fsid_t))
-    );
-    assert_eq!(
-        unsafe { &(*(::std::ptr::null::<__kernel_fsid_t>())).val as *const _ as usize },
-        0usize,
-        concat!(
-            "Offset of field: ",
-            stringify!(__kernel_fsid_t),
-            "::",
-            stringify!(val)
-        )
-    );
-}
-pub type __kernel_off_t = __kernel_long_t;
-pub type __kernel_loff_t = ::std::os::raw::c_longlong;
-pub type __kernel_time_t = __kernel_long_t;
-pub type __kernel_clock_t = __kernel_long_t;
-pub type __kernel_timer_t = ::std::os::raw::c_int;
-pub type __kernel_clockid_t = ::std::os::raw::c_int;
-pub type __kernel_caddr_t = *mut ::std::os::raw::c_char;
-pub type __kernel_uid16_t = ::std::os::raw::c_ushort;
-pub type __kernel_gid16_t = ::std::os::raw::c_ushort;
-pub type __le16 = __u16;
-pub type __be16 = __u16;
-pub type __le32 = __u32;
-pub type __be32 = __u32;
-pub type __le64 = __u64;
-pub type __be64 = __u64;
-pub type __sum16 = __u16;
-pub type __wsum = __u32;
-pub type __virtio16 = __u16;
-pub type __virtio32 = __u32;
-pub type __virtio64 = __u64;
-#[repr(C)]
-#[derive(Debug, Default, Copy, Clone, PartialEq)]
-pub struct vring_desc {
-    pub addr: __virtio64,
-    pub len: __virtio32,
-    pub flags: __virtio16,
-    pub next: __virtio16,
-}
-#[test]
-fn bindgen_test_layout_vring_desc() {
-    assert_eq!(
-        ::std::mem::size_of::<vring_desc>(),
-        16usize,
-        concat!("Size of: ", stringify!(vring_desc))
-    );
-    assert_eq!(
-        ::std::mem::align_of::<vring_desc>(),
-        8usize,
-        concat!("Alignment of ", stringify!(vring_desc))
-    );
-    assert_eq!(
-        unsafe { &(*(::std::ptr::null::<vring_desc>())).addr as *const _ as usize },
-        0usize,
-        concat!(
-            "Offset of field: ",
-            stringify!(vring_desc),
-            "::",
-            stringify!(addr)
-        )
-    );
-    assert_eq!(
-        unsafe { &(*(::std::ptr::null::<vring_desc>())).len as *const _ as usize },
-        8usize,
-        concat!(
-            "Offset of field: ",
-            stringify!(vring_desc),
-            "::",
-            stringify!(len)
-        )
-    );
-    assert_eq!(
-        unsafe { &(*(::std::ptr::null::<vring_desc>())).flags as *const _ as usize },
-        12usize,
-        concat!(
-            "Offset of field: ",
-            stringify!(vring_desc),
-            "::",
-            stringify!(flags)
-        )
-    );
-    assert_eq!(
-        unsafe { &(*(::std::ptr::null::<vring_desc>())).next as *const _ as usize },
-        14usize,
-        concat!(
-            "Offset of field: ",
-            stringify!(vring_desc),
-            "::",
-            stringify!(next)
-        )
-    );
-}
-#[repr(C)]
-#[derive(Debug, Default)]
-pub struct vring_avail {
-    pub flags: __virtio16,
-    pub idx: __virtio16,
-    pub ring: __IncompleteArrayField<__virtio16>,
-}
-#[test]
-fn bindgen_test_layout_vring_avail() {
-    assert_eq!(
-        ::std::mem::size_of::<vring_avail>(),
-        4usize,
-        concat!("Size of: ", stringify!(vring_avail))
-    );
-    assert_eq!(
-        ::std::mem::align_of::<vring_avail>(),
-        2usize,
-        concat!("Alignment of ", stringify!(vring_avail))
-    );
-}
-#[repr(C)]
-#[derive(Debug, Default, Copy, Clone, PartialEq)]
-pub struct vring_used_elem {
-    pub id: __virtio32,
-    pub len: __virtio32,
-}
-#[test]
-fn bindgen_test_layout_vring_used_elem() {
-    assert_eq!(
-        ::std::mem::size_of::<vring_used_elem>(),
-        8usize,
-        concat!("Size of: ", stringify!(vring_used_elem))
-    );
-    assert_eq!(
-        ::std::mem::align_of::<vring_used_elem>(),
-        4usize,
-        concat!("Alignment of ", stringify!(vring_used_elem))
-    );
-    assert_eq!(
-        unsafe { &(*(::std::ptr::null::<vring_used_elem>())).id as *const _ as usize },
-        0usize,
-        concat!(
-            "Offset of field: ",
-            stringify!(vring_used_elem),
-            "::",
-            stringify!(id)
-        )
-    );
-    assert_eq!(
-        unsafe { &(*(::std::ptr::null::<vring_used_elem>())).len as *const _ as usize },
-        4usize,
-        concat!(
-            "Offset of field: ",
-            stringify!(vring_used_elem),
-            "::",
-            stringify!(len)
-        )
-    );
-}
-#[repr(C)]
-#[derive(Debug, Default)]
-pub struct vring_used {
-    pub flags: __virtio16,
-    pub idx: __virtio16,
-    pub ring: __IncompleteArrayField<vring_used_elem>,
-    pub __bindgen_align: [u32; 0usize],
-}
-#[test]
-fn bindgen_test_layout_vring_used() {
-    assert_eq!(
-        ::std::mem::size_of::<vring_used>(),
-        4usize,
-        concat!("Size of: ", stringify!(vring_used))
-    );
-    assert_eq!(
-        ::std::mem::align_of::<vring_used>(),
-        4usize,
-        concat!("Alignment of ", stringify!(vring_used))
-    );
-}
-#[repr(C)]
-#[derive(Debug, Copy, Clone, PartialEq)]
-pub struct vring {
-    pub num: ::std::os::raw::c_uint,
-    pub desc: *mut vring_desc,
-    pub avail: *mut vring_avail,
-    pub used: *mut vring_used,
-}
-#[test]
-fn bindgen_test_layout_vring() {
-    assert_eq!(
-        ::std::mem::size_of::<vring>(),
-        32usize,
-        concat!("Size of: ", stringify!(vring))
-    );
-    assert_eq!(
-        ::std::mem::align_of::<vring>(),
-        8usize,
-        concat!("Alignment of ", stringify!(vring))
-    );
-    assert_eq!(
-        unsafe { &(*(::std::ptr::null::<vring>())).num as *const _ as usize },
-        0usize,
-        concat!(
-            "Offset of field: ",
-            stringify!(vring),
-            "::",
-            stringify!(num)
-        )
-    );
-    assert_eq!(
-        unsafe { &(*(::std::ptr::null::<vring>())).desc as *const _ as usize },
-        8usize,
-        concat!(
-            "Offset of field: ",
-            stringify!(vring),
-            "::",
-            stringify!(desc)
-        )
-    );
-    assert_eq!(
-        unsafe { &(*(::std::ptr::null::<vring>())).avail as *const _ as usize },
-        16usize,
-        concat!(
-            "Offset of field: ",
-            stringify!(vring),
-            "::",
-            stringify!(avail)
-        )
-    );
-    assert_eq!(
-        unsafe { &(*(::std::ptr::null::<vring>())).used as *const _ as usize },
-        24usize,
-        concat!(
-            "Offset of field: ",
-            stringify!(vring),
-            "::",
-            stringify!(used)
-        )
-    );
-}
-impl Default for vring {
-    fn default() -> Self {
-        unsafe { ::std::mem::zeroed() }
-    }
-}

vm-device/Cargo.toml

@@ -0,0 +1,16 @@
+[package]
+name = "vm-device"
+version = "0.1.0"
+authors = ["The Cloud Hypervisor Authors"]
+edition = "2018"
+
+[dependencies]
+anyhow = "1.0"
+thiserror = "1.0"
+serde = {version = ">=1.0.27", features = ["rc"] }
+serde_derive = ">=1.0.27"
+serde_json = ">=1.0.9"
+
+[dependencies.vm-memory]
+git = "https://github.com/rust-vmm/vm-memory"
+features = ["backend-mmap"]

vm-device/src/lib.rs

@@ -0,0 +1,114 @@
+extern crate serde;
+extern crate thiserror;
+extern crate vm_memory;
+
+use vm_memory::{
+    Address, GuestAddress, GuestMemory, GuestMemoryMmap, GuestMemoryRegion, GuestRegionMmap,
+    MemoryRegionAddress,
+};
+
+use thiserror::Error;
+
+/// Trait meant for triggering the DMA mapping update related to an external
+/// device not managed fully through virtio. It is dedicated to virtio-iommu
+/// in order to trigger the map update anytime the mapping is updated from the
+/// guest.
+pub trait ExternalDmaMapping: Send + Sync {
+    /// Map a memory range
+    fn map(&self, iova: u64, gpa: u64, size: u64) -> std::result::Result<(), std::io::Error>;
+
+    /// Unmap a memory range
+    fn unmap(&self, iova: u64, size: u64) -> std::result::Result<(), std::io::Error>;
+}
+
+#[derive(Error, Debug)]
+pub enum MigratableError {
+    #[error("Failed to pause migratable component: {0}")]
+    Pause(#[source] anyhow::Error),
+
+    #[error("Failed to resume migratable component: {0}")]
+    Resume(#[source] anyhow::Error),
+}
+
+/// A Pausable component can be paused and resumed.
+pub trait Pausable {
+    /// Pause the component.
+    fn pause(&mut self) -> std::result::Result<(), MigratableError>;
+
+    /// Resume the component.
+    fn resume(&mut self) -> std::result::Result<(), MigratableError>;
+}
+
+/// A snapshotable component can be snapshoted.
+pub trait Snapshotable {}
+
+/// Trait to be implemented by any component (device, CPU, RAM, etc) that
+/// can be migrated.
+/// All migratable components are paused before being snapshotted, and then
+/// eventually resumed. Thus any Migratable component must be both Pausable
+/// and Snapshotable.
+pub trait Migratable: Pausable + Snapshotable {}
+
+fn get_region_host_address_range(
+    region: &GuestRegionMmap,
+    addr: MemoryRegionAddress,
+    size: usize,
+) -> Option<*mut u8> {
+    region.check_address(addr).and_then(|addr| {
+        region
+            .checked_offset(addr, size)
+            .map(|_| region.as_ptr().wrapping_offset(addr.raw_value() as isize))
+    })
+}
+
+/// Convert an absolute address into an address space (GuestMemory)
+/// to a host pointer and verify that the provided size define a valid
+/// range within a single memory region.
+/// Return None if it is out of bounds or if addr+size overlaps a single region.
+///
+/// This is a temporary vm-memory wrapper.
+pub fn get_host_address_range(
+    mem: &GuestMemoryMmap,
+    addr: GuestAddress,
+    size: usize,
+) -> Option<*mut u8> {
+    mem.to_region_addr(addr)
+        .and_then(|(r, addr)| get_region_host_address_range(r, addr, size))
+}
+
+#[cfg(test)]
+mod tests {
+
+    use super::*;
+    use vm_memory::{GuestAddress, GuestMemoryMmap};
+
+    #[test]
+    fn test_get_host_address_range() {
+        let start_addr1 = GuestAddress(0x0);
+        let start_addr2 = GuestAddress(0x1000);
+        let guest_mem =
+            GuestMemoryMmap::new(&[(start_addr1, 0x400), (start_addr2, 0x400)]).unwrap();
+
+        assert!(get_host_address_range(&guest_mem, GuestAddress(0x600), 0x100).is_none());
+
+        // Overlapping range
+        assert!(get_host_address_range(&guest_mem, GuestAddress(0x1000), 0x500).is_none());
+
+        // Overlapping range
+        assert!(get_host_address_range(&guest_mem, GuestAddress(0x1200), 0x500).is_none());
+
+        let ptr = get_host_address_range(&guest_mem, GuestAddress(0x1000), 0x100).unwrap();
+
+        let ptr0 = get_host_address_range(&guest_mem, GuestAddress(0x1100), 0x100).unwrap();
+
+        let ptr1 = guest_mem.get_host_address(GuestAddress(0x1200)).unwrap();
+        assert_eq!(
+            ptr,
+            guest_mem
+                .find_region(GuestAddress(0x1100))
+                .unwrap()
+                .as_ptr()
+        );
+        assert_eq!(unsafe { ptr0.offset(0x100) }, ptr1);
+    }
+}

vm-virtio/Cargo.toml

@@ -4,19 +4,25 @@ version = "0.1.0"
 authors = ["Samuel Ortiz <sameo@linux.intel.com>"]
 edition = "2018"
 
+[features]
+default = []
+pci_support = ["pci"]
+mmio_support = []
+
 [dependencies]
 byteorder = "1.3.2"
 devices = { path = "../devices" }
-epoll = "4.1.0"
+epoll = ">=4.0.1"
 libc = "0.2.60"
 log = "0.4.8"
 net_gen = { path = "../net_gen" }
 net_util = { path = "../net_util" }
-pci = { path = "../pci" }
+pci = { path = "../pci", optional = true }
 tempfile = "3.1.0"
-virtio-bindings = { path = "../virtio-bindings" }
+virtio-bindings = { git = "https://github.com/rust-vmm/virtio-bindings", version = "0.1", features = ["virtio-v5_0_0"]}
 vm-allocator = { path = "../vm-allocator" }
-vmm-sys-util = { git = "https://github.com/rust-vmm/vmm-sys-util" }
+vm-device = { path = "../vm-device" }
+vmm-sys-util = ">=0.3.1"
 
 [dependencies.vhost_rs]
 path = "../vhost_rs"

vm-virtio/src/block.rs

@@ -17,6 +17,7 @@ use std::os::linux::fs::MetadataExt;
 use std::os::unix::io::AsRawFd;
 use std::path::PathBuf;
 use std::result;
+use std::sync::atomic::{AtomicBool, Ordering};
 use std::sync::{Arc, RwLock};
 use std::thread;
 
@@ -26,7 +27,8 @@ use super::{
     VirtioDeviceType, VirtioInterruptType,
 };
 use crate::VirtioInterrupt;
-use virtio_bindings::virtio_blk::*;
+use virtio_bindings::bindings::virtio_blk::*;
+use vm_device::{Migratable, MigratableError, Pausable, Snapshotable};
 use vm_memory::{Bytes, GuestAddress, GuestMemory, GuestMemoryError, GuestMemoryMmap};
 use vmm_sys_util::eventfd::EventFd;
 
@@ -43,9 +45,11 @@ const QUEUE_AVAIL_EVENT: DeviceEventT = 0;
 pub const KILL_EVENT: DeviceEventT = 1;
 // Number of DeviceEventT events supported by this implementation.
 pub const BLOCK_EVENTS_COUNT: usize = 2;
+// The device should be paused.
+const PAUSE_EVENT: DeviceEventT = 3;
 
 #[derive(Debug)]
-enum Error {
+pub enum Error {
     /// Guest gave us bad memory addresses.
     GuestMemory(GuestMemoryError),
     /// Guest gave us offsets that would have overflowed a usize.
@@ -65,7 +69,7 @@ enum Error {
 }
 
 #[derive(Debug)]
-enum ExecuteError {
+pub enum ExecuteError {
     BadRequest(Error),
     Flush(io::Error),
     Read(GuestMemoryError),
@@ -75,7 +79,7 @@ enum ExecuteError {
 }
 
 impl ExecuteError {
-    fn status(&self) -> u32 {
+    pub fn status(&self) -> u32 {
         match *self {
             ExecuteError::BadRequest(_) => VIRTIO_BLK_S_IOERR,
             ExecuteError::Flush(_) => VIRTIO_BLK_S_IOERR,
@@ -131,7 +135,7 @@ impl Clone for RawFile {
 }
 
 #[derive(Clone, Copy, Debug, PartialEq)]
-enum RequestType {
+pub enum RequestType {
     In,
     Out,
     Flush,
@@ -139,7 +143,7 @@ enum RequestType {
     Unsupported(u32),
 }
 
-fn request_type(
+pub fn request_type(
     mem: &GuestMemoryMmap,
     desc_addr: GuestAddress,
 ) -> result::Result<RequestType, Error> {
@@ -179,7 +183,7 @@ fn build_device_id(disk_path: &PathBuf) -> result::Result<String, Error> {
     Ok(device_id)
 }
 
-fn build_disk_image_id(disk_path: &PathBuf) -> Vec<u8> {
+pub fn build_disk_image_id(disk_path: &PathBuf) -> Vec<u8> {
     let mut default_disk_image_id = vec![0; VIRTIO_BLK_ID_BYTES as usize];
     match build_device_id(disk_path) {
         Err(_) => {
@@ -196,16 +200,16 @@ fn build_disk_image_id(disk_path: &PathBuf) -> Vec<u8> {
     default_disk_image_id
 }
 
-struct Request {
+pub struct Request {
     request_type: RequestType,
     sector: u64,
     data_addr: GuestAddress,
     data_len: u32,
-    status_addr: GuestAddress,
+    pub status_addr: GuestAddress,
 }
 
 impl Request {
-    fn parse(
+    pub fn parse(
         avail_desc: &DescriptorChain,
         mem: &GuestMemoryMmap,
     ) -> result::Result<Request, Error> {
@@ -269,7 +273,7 @@ impl Request {
     }
 
     #[allow(clippy::ptr_arg)]
-    fn execute<T: Seek + Read + Write>(
+    pub fn execute<T: Seek + Read + Write>(
         &self,
         disk: &mut T,
         disk_nsectors: u64,
@@ -326,6 +330,8 @@ struct BlockEpollHandler<T: DiskFile> {
     disk_nsectors: u64,
     interrupt_cb: Arc<VirtioInterrupt>,
     disk_image_id: Vec<u8>,
+    kill_evt: EventFd,
+    pause_evt: EventFd,
 }
 
 impl<T: DiskFile> BlockEpollHandler<T> {
@@ -399,7 +405,11 @@ impl<T: DiskFile> BlockEpollHandler<T> {
         Ok(())
     }
 
-    fn run(&mut self, queue_evt: EventFd, kill_evt: EventFd) -> result::Result<(), DeviceError> {
+    fn run(
+        &mut self,
+        queue_evt: EventFd,
+        paused: Arc<AtomicBool>,
+    ) -> result::Result<(), DeviceError> {
         // Create the epoll file descriptor
         let epoll_fd = epoll::create(true).map_err(DeviceError::EpollCreateFd)?;
 
@@ -414,10 +424,17 @@ impl<T: DiskFile> BlockEpollHandler<T> {
         epoll::ctl(
             epoll_fd,
             epoll::ControlOptions::EPOLL_CTL_ADD,
-            kill_evt.as_raw_fd(),
+            self.kill_evt.as_raw_fd(),
             epoll::Event::new(epoll::Events::EPOLLIN, u64::from(KILL_EVENT)),
         )
         .map_err(DeviceError::EpollCtl)?;
+        epoll::ctl(
+            epoll_fd,
+            epoll::ControlOptions::EPOLL_CTL_ADD,
+            self.pause_evt.as_raw_fd(),
+            epoll::Event::new(epoll::Events::EPOLLIN, u64::from(PAUSE_EVENT)),
+        )
+        .map_err(DeviceError::EpollCtl)?;
 
         const EPOLL_EVENTS_LEN: usize = 100;
         let mut events = vec![epoll::Event::new(epoll::Events::empty(), 0); EPOLL_EVENTS_LEN];
@@ -459,6 +476,15 @@ impl<T: DiskFile> BlockEpollHandler<T> {
                         debug!("KILL_EVENT received, stopping epoll loop");
                         break 'epoll;
                     }
+                    PAUSE_EVENT => {
+                        debug!("PAUSE_EVENT received, pausing virtio-block epoll loop");
+                        // We loop here to handle spurious park() returns.
+                        // Until we have not resumed, the paused boolean will
+                        // be true.
+                        while paused.load(Ordering::SeqCst) {
+                            thread::park();
+                        }
+                    }
                     _ => {
                         error!("Unknown event for virtio-block");
                     }
@@ -481,6 +507,9 @@ pub struct Block<T: DiskFile> {
     config_space: Vec<u8>,
     queue_evt: Option<EventFd>,
     interrupt_cb: Option<Arc<VirtioInterrupt>>,
+    epoll_thread: Option<thread::JoinHandle<result::Result<(), DeviceError>>>,
+    pause_evt: Option<EventFd>,
+    paused: Arc<AtomicBool>,
 }
 
 pub fn build_config_space(disk_size: u64) -> Vec<u8> {
@@ -503,6 +532,7 @@ impl<T: DiskFile> Block<T> {
         mut disk_image: T,
         disk_path: PathBuf,
         is_disk_read_only: bool,
+        iommu: bool,
     ) -> io::Result<Block<T>> {
         let disk_size = disk_image.seek(SeekFrom::End(0))? as u64;
         if disk_size % SECTOR_SIZE != 0 {
@@ -515,6 +545,10 @@ impl<T: DiskFile> Block<T> {
 
         let mut avail_features = (1u64 << VIRTIO_F_VERSION_1) | (1u64 << VIRTIO_BLK_F_FLUSH);
 
+        if iommu {
+            avail_features |= 1u64 << VIRTIO_F_IOMMU_PLATFORM;
+        }
+
         if is_disk_read_only {
             avail_features |= 1u64 << VIRTIO_BLK_F_RO;
         };
@@ -529,6 +563,9 @@ impl<T: DiskFile> Block<T> {
             config_space: build_config_space(disk_size),
             queue_evt: None,
             interrupt_cb: None,
+            epoll_thread: None,
+            pause_evt: None,
+            paused: Arc::new(AtomicBool::new(false)),
         })
     }
 }
@@ -625,16 +662,23 @@ impl<T: 'static + DiskFile + Send> VirtioDevice for Block<T> {
             return Err(ActivateError::BadActivate);
         }
 
-        let (self_kill_evt, kill_evt) =
+        let (self_kill_evt, kill_evt) = EventFd::new(EFD_NONBLOCK)
-            match EventFd::new(EFD_NONBLOCK).and_then(|e| Ok((e.try_clone()?, e))) {
+            .and_then(|e| Ok((e.try_clone()?, e)))
-                Ok(v) => v,
+            .map_err(|e| {
-                Err(e) => {
+                error!("failed creating kill EventFd pair: {}", e);
-                    error!("failed creating kill EventFd pair: {}", e);
+                ActivateError::BadActivate
-                    return Err(ActivateError::BadActivate);
+            })?;
-                }
+
-            };
         self.kill_evt = Some(self_kill_evt);
 
+        let (self_pause_evt, pause_evt) = EventFd::new(EFD_NONBLOCK)
+            .and_then(|e| Ok((e.try_clone()?, e)))
+            .map_err(|e| {
+                error!("failed creating pause EventFd pair: {}", e);
+                ActivateError::BadActivate
+            })?;
+        self.pause_evt = Some(self_pause_evt);
+
         if let Some(disk_image) = self.disk_image.clone() {
             let disk_image_id = build_disk_image_id(&self.disk_path);
 
@@ -658,16 +702,19 @@ impl<T: 'static + DiskFile + Send> VirtioDevice for Block<T> {
                 disk_nsectors: self.disk_nsectors,
                 interrupt_cb,
                 disk_image_id,
+                kill_evt,
+                pause_evt,
             };
 
-            let worker_result = thread::Builder::new()
+            let paused = self.paused.clone();
+            thread::Builder::new()
                 .name("virtio_blk".to_string())
-                .spawn(move || handler.run(queue_evt, kill_evt));
+                .spawn(move || handler.run(queue_evt, paused))
-
+                .map(|thread| self.epoll_thread = Some(thread))
-            if let Err(e) = worker_result {
+                .map_err(|e| {
-                error!("failed to spawn virtio_blk worker: {}", e);
+                    error!("failed to clone the virtio-blk epoll thread: {}", e);
-                return Err(ActivateError::BadActivate);
+                    ActivateError::BadActivate
-            }
+                })?;
 
             return Ok(());
         }
@@ -675,6 +722,11 @@ impl<T: 'static + DiskFile + Send> VirtioDevice for Block<T> {
     }
 
     fn reset(&mut self) -> Option<(Arc<VirtioInterrupt>, Vec<EventFd>)> {
+        // We first must resume the virtio thread if it was paused.
+        if self.pause_evt.take().is_some() {
+            self.resume().ok()?;
+        }
+
         if let Some(kill_evt) = self.kill_evt.take() {
             // Ignore the result because there is nothing we can do about it.
             let _ = kill_evt.write(1);
@@ -687,3 +739,10 @@ impl<T: 'static + DiskFile + Send> VirtioDevice for Block<T> {
         ))
     }
 }
+
+impl<T: 'static + DiskFile + Send> Pausable for Block<T> {
+    virtio_pausable_inner!();
+}
+
+impl<T: 'static + DiskFile + Send> Snapshotable for Block<T> {}
+impl<T: 'static + DiskFile + Send> Migratable for Block<T> {}

vm-virtio/src/console.rs

@@ -8,6 +8,7 @@ use std::cmp;
 use std::collections::VecDeque;
 use std::io;
 use std::io::Write;
+use std::ops::DerefMut;
 use std::os::unix::io::AsRawFd;
 use std::result;
 use std::sync::{Arc, Mutex, RwLock};
@@ -16,10 +17,11 @@ use std::thread;
 use super::Error as DeviceError;
 use super::{
     ActivateError, ActivateResult, DeviceEventT, Queue, VirtioDevice, VirtioDeviceType,
-    VirtioInterruptType, VIRTIO_F_VERSION_1,
+    VirtioInterruptType, VIRTIO_F_IOMMU_PLATFORM, VIRTIO_F_VERSION_1,
 };
 use crate::VirtioInterrupt;
-use std::sync::atomic::{AtomicU64, Ordering};
+use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
+use vm_device::{Migratable, MigratableError, Pausable, Snapshotable};
 use vm_memory::{ByteValued, Bytes, GuestMemoryMmap};
 use vmm_sys_util::eventfd::EventFd;
 
@@ -36,6 +38,8 @@ const INPUT_EVENT: DeviceEventT = 2;
 const KILL_EVENT: DeviceEventT = 3;
 // Console configuration change event is triggered.
 const CONFIG_EVENT: DeviceEventT = 4;
+// The device should be paused.
+const PAUSE_EVENT: DeviceEventT = 5;
 
 //Console size feature bit
 const VIRTIO_CONSOLE_F_SIZE: u64 = 0;
@@ -57,12 +61,13 @@ struct ConsoleEpollHandler {
     mem: Arc<RwLock<GuestMemoryMmap>>,
     interrupt_cb: Arc<VirtioInterrupt>,
     in_buffer: Arc<Mutex<VecDeque<u8>>>,
-    out: Box<dyn io::Write + Send>,
+    out: Arc<Mutex<Box<dyn io::Write + Send + Sync + 'static>>>,
     input_queue_evt: EventFd,
     output_queue_evt: EventFd,
     input_evt: EventFd,
     config_evt: EventFd,
     kill_evt: EventFd,
+    pause_evt: EventFd,
 }
 
 impl ConsoleEpollHandler {
@@ -130,8 +135,13 @@ impl ConsoleEpollHandler {
         let mem = self.mem.read().unwrap();
         for avail_desc in trans_queue.iter(&mem) {
             let len;
-            let _ = mem.write_to(avail_desc.addr, &mut self.out, avail_desc.len as usize);
+            let mut out = self.out.lock().unwrap();
-            let _ = self.out.flush();
+            let _ = mem.write_to(
+                avail_desc.addr,
+                &mut out.deref_mut(),
+                avail_desc.len as usize,
+            );
+            let _ = out.flush();
 
             len = avail_desc.len;
             used_desc_heads[used_count] = (avail_desc.index, len);
@@ -151,7 +161,7 @@ impl ConsoleEpollHandler {
         })
     }
 
-    fn run(&mut self) -> result::Result<(), DeviceError> {
+    fn run(&mut self, paused: Arc<AtomicBool>) -> result::Result<(), DeviceError> {
         // Create the epoll file descriptor
         let epoll_fd = epoll::create(true).map_err(DeviceError::EpollCreateFd)?;
 
@@ -192,6 +202,13 @@ impl ConsoleEpollHandler {
             epoll::Event::new(epoll::Events::EPOLLIN, u64::from(KILL_EVENT)),
         )
         .map_err(DeviceError::EpollCtl)?;
+        epoll::ctl(
+            epoll_fd,
+            epoll::ControlOptions::EPOLL_CTL_ADD,
+            self.pause_evt.as_raw_fd(),
+            epoll::Event::new(epoll::Events::EPOLLIN, u64::from(PAUSE_EVENT)),
+        )
+        .map_err(DeviceError::EpollCtl)?;
 
         const EPOLL_EVENTS_LEN: usize = 100;
         let mut events = vec![epoll::Event::new(epoll::Events::empty(), 0); EPOLL_EVENTS_LEN];
@@ -258,6 +275,15 @@ impl ConsoleEpollHandler {
                         debug!("KILL_EVENT received, stopping epoll loop");
                         break 'epoll;
                     }
+                    PAUSE_EVENT => {
+                        debug!("PAUSE_EVENT received, pausing virtio-console epoll loop");
+                        // We loop here to handle spurious park() returns.
+                        // Until we have not resumed, the paused boolean will
+                        // be true.
+                        while paused.load(Ordering::SeqCst) {
+                            thread::park();
+                        }
+                    }
                     _ => {
                         error!("Unknown event for virtio-console");
                     }
@@ -317,21 +343,31 @@ impl VirtioConsoleConfig {
 /// Virtio device for exposing console to the guest OS through virtio.
 pub struct Console {
     kill_evt: Option<EventFd>,
+    pause_evt: Option<EventFd>,
     avail_features: u64,
     acked_features: u64,
     config: Arc<Mutex<VirtioConsoleConfig>>,
     input: Arc<ConsoleInput>,
-    out: Option<Box<dyn io::Write + Send>>,
+    out: Arc<Mutex<Box<dyn io::Write + Send + Sync + 'static>>>,
+    queue_evts: Option<Vec<EventFd>>,
+    interrupt_cb: Option<Arc<VirtioInterrupt>>,
+    epoll_thread: Option<thread::JoinHandle<result::Result<(), DeviceError>>>,
+    paused: Arc<AtomicBool>,
 }
 
 impl Console {
     /// Create a new virtio console device that gets random data from /dev/urandom.
     pub fn new(
-        out: Option<Box<dyn io::Write + Send>>,
+        out: Box<dyn io::Write + Send + Sync + 'static>,
         cols: u16,
         rows: u16,
+        iommu: bool,
     ) -> io::Result<(Console, Arc<ConsoleInput>)> {
-        let avail_features = 1u64 << VIRTIO_F_VERSION_1 | 1u64 << VIRTIO_CONSOLE_F_SIZE;
+        let mut avail_features = 1u64 << VIRTIO_F_VERSION_1 | 1u64 << VIRTIO_CONSOLE_F_SIZE;
+
+        if iommu {
+            avail_features |= 1u64 << VIRTIO_F_IOMMU_PLATFORM;
+        }
 
         let input_evt = EventFd::new(EFD_NONBLOCK).unwrap();
         let config_evt = EventFd::new(EFD_NONBLOCK).unwrap();
@@ -347,11 +383,16 @@ impl Console {
         Ok((
             Console {
                 kill_evt: None,
+                pause_evt: None,
                 avail_features,
                 acked_features: 0u64,
                 config: console_config,
                 input: console_input.clone(),
-                out,
+                out: Arc::new(Mutex::new(out)),
+                queue_evts: None,
+                interrupt_cb: None,
+                epoll_thread: None,
+                paused: Arc::new(AtomicBool::new(false)),
             },
             console_input,
         ))
@@ -446,16 +487,38 @@ impl VirtioDevice for Console {
             return Err(ActivateError::BadActivate);
         }
 
-        let (self_kill_evt, kill_evt) =
+        let (self_kill_evt, kill_evt) = EventFd::new(EFD_NONBLOCK)
-            match EventFd::new(EFD_NONBLOCK).and_then(|e| Ok((e.try_clone()?, e))) {
+            .and_then(|e| Ok((e.try_clone()?, e)))
-                Ok(v) => v,
+            .map_err(|e| {
-                Err(e) => {
+                error!("failed creating kill EventFd pair: {}", e);
-                    error!("failed creating kill EventFd pair: {}", e);
+                ActivateError::BadActivate
-                    return Err(ActivateError::BadActivate);
+            })?;
-                }
+
-            };
         self.kill_evt = Some(self_kill_evt);
 
+        let (self_pause_evt, pause_evt) = EventFd::new(EFD_NONBLOCK)
+            .and_then(|e| Ok((e.try_clone()?, e)))
+            .map_err(|e| {
+                error!("failed creating pause EventFd pair: {}", e);
+                ActivateError::BadActivate
+            })?;
+        self.pause_evt = Some(self_pause_evt);
+
+        // Save the interrupt EventFD as we need to return it on reset
+        // but clone it to pass into the thread.
+        self.interrupt_cb = Some(interrupt_cb.clone());
+
+        let mut tmp_queue_evts: Vec<EventFd> = Vec::new();
+        for queue_evt in queue_evts.iter() {
+            // Save the queue EventFD as we need to return it on reset
+            // but clone it to pass into the thread.
+            tmp_queue_evts.push(queue_evt.try_clone().map_err(|e| {
+                error!("failed to clone queue EventFd: {}", e);
+                ActivateError::BadActivate
+            })?);
+        }
+        self.queue_evts = Some(tmp_queue_evts);
+
         self.input
             .acked_features
             .store(self.acked_features, Ordering::Relaxed);
@@ -466,31 +529,52 @@ impl VirtioDevice for Console {
             }
         }
 
-        if let Some(out) = self.out.take() {
+        let mut handler = ConsoleEpollHandler {
-            let mut handler = ConsoleEpollHandler {
+            queues,
-                queues,
+            mem,
-                mem,
+            interrupt_cb,
-                interrupt_cb,
+            in_buffer: self.input.in_buffer.clone(),
-                in_buffer: self.input.in_buffer.clone(),
+            out: self.out.clone(),
-                out,
+            input_queue_evt: queue_evts.remove(0),
-                input_queue_evt: queue_evts.remove(0),
+            output_queue_evt: queue_evts.remove(0),
-                output_queue_evt: queue_evts.remove(0),
+            input_evt: self.input.input_evt.try_clone().unwrap(),
-                input_evt: self.input.input_evt.try_clone().unwrap(),
+            config_evt: self.input.config_evt.try_clone().unwrap(),
-                config_evt: self.input.config_evt.try_clone().unwrap(),
+            kill_evt,
-                kill_evt,
+            pause_evt,
-            };
+        };
 
-            let worker_result = thread::Builder::new()
+        let paused = self.paused.clone();
-                .name("virtio_console".to_string())
+        thread::Builder::new()
-                .spawn(move || handler.run());
+            .name("virtio_console".to_string())
+            .spawn(move || handler.run(paused))
+            .map(|thread| self.epoll_thread = Some(thread))
+            .map_err(|e| {
+                error!("failed to clone the virtio-console epoll thread: {}", e);
+                ActivateError::BadActivate
+            })?;
 
-            if let Err(e) = worker_result {
+        Ok(())
-                error!("failed to spawn virtio_console worker: {}", e);
+    }
-                return Err(ActivateError::BadActivate);;
-            }
 
-            return Ok(());
+    fn reset(&mut self) -> Option<(Arc<VirtioInterrupt>, Vec<EventFd>)> {
+        // We first must resume the virtio thread if it was paused.
+        if self.pause_evt.take().is_some() {
+            self.resume().ok()?;
         }
-        Err(ActivateError::BadActivate)
+
+        if let Some(kill_evt) = self.kill_evt.take() {
+            // Ignore the result because there is nothing we can do about it.
+            let _ = kill_evt.write(1);
+        }
+
+        // Return the interrupt and queue EventFDs
+        Some((
+            self.interrupt_cb.take().unwrap(),
+            self.queue_evts.take().unwrap(),
+        ))
     }
 }
+
+virtio_pausable!(Console);
+impl Snapshotable for Console {}
+impl Migratable for Console {}

vm-virtio/src/device.rs

@@ -22,6 +22,9 @@ pub type VirtioInterrupt = Box<
         + Sync,
 >;
 
+pub type VirtioIommuRemapping =
+    Box<dyn Fn(u64) -> std::result::Result<u64, std::io::Error> + Send + Sync>;
+
 #[derive(Clone)]
 pub struct VirtioSharedMemory {
     pub offset: u64,
@@ -83,4 +86,60 @@ pub trait VirtioDevice: Send {
     fn get_shm_regions(&self) -> Option<VirtioSharedMemoryList> {
         None
     }
+
+    fn iommu_translate(&self, addr: u64) -> u64 {
+        addr
+    }
+}
+
+/// Trait providing address translation the same way a physical DMA remapping
+/// table would provide translation between an IOVA and a physical address.
+/// The goal of this trait is to be used by virtio devices to perform the
+/// address translation before they try to read from the guest physical address.
+/// On the other side, the implementation itself should be provided by the code
+/// emulating the IOMMU for the guest.
+pub trait DmaRemapping: Send + Sync {
+    fn translate(&self, id: u32, addr: u64) -> std::result::Result<u64, std::io::Error>;
+}
+
+#[macro_export]
+macro_rules! virtio_pausable_inner {
+    () => {
+        fn pause(&mut self) -> result::Result<(), MigratableError> {
+            debug!(
+                "Pausing virtio-{}",
+                VirtioDeviceType::from(self.device_type())
+            );
+            self.paused.store(true, Ordering::SeqCst);
+            if let Some(pause_evt) = &self.pause_evt {
+                pause_evt
+                    .write(1)
+                    .map_err(|e| MigratableError::Pause(e.into()))?;
+            }
+
+            Ok(())
+        }
+
+        fn resume(&mut self) -> result::Result<(), MigratableError> {
+            debug!(
+                "Resuming virtio-{}",
+                VirtioDeviceType::from(self.device_type())
+            );
+            self.paused.store(false, Ordering::SeqCst);
+            if let Some(epoll_thread) = &self.epoll_thread {
+                epoll_thread.thread().unpark();
+            }
+
+            Ok(())
+        }
+    }
+}
+
+#[macro_export]
+macro_rules! virtio_pausable {
+    ($name:ident) => {
+        impl Pausable for $name {
+            virtio_pausable_inner!();
+        }
+    };
 }

vm-virtio/src/iommu.rs

@@ -0,0 +1,957 @@
+// Copyright © 2019 Intel Corporation
+//
+// SPDX-License-Identifier: Apache-2.0 AND BSD-3-Clause
+
+use epoll;
+use libc::EFD_NONBLOCK;
+use std::cmp;
+use std::collections::BTreeMap;
+use std::fmt::{self, Display};
+use std::io::{self, Write};
+use std::mem::size_of;
+use std::ops::Bound::Included;
+use std::os::unix::io::AsRawFd;
+use std::result;
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::{Arc, RwLock};
+use std::thread;
+
+use super::Error as DeviceError;
+use super::{
+    ActivateError, ActivateResult, DescriptorChain, DeviceEventT, Queue, VirtioDevice,
+    VirtioDeviceType, VIRTIO_F_VERSION_1,
+};
+use crate::{DmaRemapping, VirtioInterrupt, VirtioInterruptType};
+use vm_device::{ExternalDmaMapping, Migratable, MigratableError, Pausable, Snapshotable};
+use vm_memory::{Address, ByteValued, Bytes, GuestAddress, GuestMemoryError, GuestMemoryMmap};
+use vmm_sys_util::eventfd::EventFd;
+
+/// Queues sizes
+const QUEUE_SIZE: u16 = 256;
+const NUM_QUEUES: usize = 2;
+const QUEUE_SIZES: &[u16] = &[QUEUE_SIZE; NUM_QUEUES];
+
+/// New descriptors are pending on the request queue.
+/// "requestq" is meant to be used anytime an action is required to be
+/// performed on behalf of the guest driver.
+const REQUEST_Q_EVENT: DeviceEventT = 0;
+/// New descriptors are pending on the event queue.
+/// "eventq" lets the device report any fault or other asynchronous event to
+/// the guest driver.
+const EVENT_Q_EVENT: DeviceEventT = 1;
+/// The device has been dropped.
+const KILL_EVENT: DeviceEventT = 2;
+/// The device should be paused.
+const PAUSE_EVENT: DeviceEventT = 3;
+
+/// Virtio IOMMU features
+#[allow(unused)]
+const VIRTIO_IOMMU_F_INPUT_RANGE: u32 = 0;
+#[allow(unused)]
+const VIRTIO_IOMMU_F_DOMAIN_BITS: u32 = 1;
+#[allow(unused)]
+const VIRTIO_IOMMU_F_MAP_UNMAP: u32 = 2;
+#[allow(unused)]
+const VIRTIO_IOMMU_F_BYPASS: u32 = 3;
+#[allow(unused)]
+const VIRTIO_IOMMU_F_PROBE: u32 = 4;
+
+// Support 2MiB and 4KiB page sizes.
+#[allow(unused)]
+const VIRTIO_IOMMU_PAGE_SIZE_MASK: u64 = (2 << 20) | (4 << 10);
+
+#[allow(unused)]
+#[derive(Copy, Clone, Debug, Default)]
+#[repr(packed)]
+struct VirtioIommuRange {
+    start: u64,
+    end: u64,
+}
+
+unsafe impl ByteValued for VirtioIommuRange {}
+
+#[derive(Copy, Clone, Debug, Default)]
+#[repr(packed)]
+struct VirtioIommuConfig {
+    page_size_mask: u64,
+    input_range: VirtioIommuRange,
+    domain_bits: u8,
+    padding: [u8; 3],
+    probe_size: u32,
+}
+
+unsafe impl ByteValued for VirtioIommuConfig {}
+
+/// Virtio IOMMU request type
+const VIRTIO_IOMMU_T_ATTACH: u8 = 1;
+const VIRTIO_IOMMU_T_DETACH: u8 = 2;
+const VIRTIO_IOMMU_T_MAP: u8 = 3;
+const VIRTIO_IOMMU_T_UNMAP: u8 = 4;
+const VIRTIO_IOMMU_T_PROBE: u8 = 5;
+
+#[allow(unused)]
+#[derive(Copy, Clone, Debug, Default)]
+#[repr(packed)]
+struct VirtioIommuReqHead {
+    type_: u8,
+    reserved: [u8; 3],
+}
+
+unsafe impl ByteValued for VirtioIommuReqHead {}
+
+/// Virtio IOMMU request status
+const VIRTIO_IOMMU_S_OK: u8 = 0;
+#[allow(unused)]
+const VIRTIO_IOMMU_S_IOERR: u8 = 1;
+#[allow(unused)]
+const VIRTIO_IOMMU_S_UNSUPP: u8 = 2;
+#[allow(unused)]
+const VIRTIO_IOMMU_S_DEVERR: u8 = 3;
+#[allow(unused)]
+const VIRTIO_IOMMU_S_INVAL: u8 = 4;
+#[allow(unused)]
+const VIRTIO_IOMMU_S_RANGE: u8 = 5;
+#[allow(unused)]
+const VIRTIO_IOMMU_S_NOENT: u8 = 6;
+#[allow(unused)]
+const VIRTIO_IOMMU_S_FAULT: u8 = 7;
+
+#[allow(unused)]
+#[derive(Copy, Clone, Debug, Default)]
+#[repr(packed)]
+struct VirtioIommuReqTail {
+    status: u8,
+    reserved: [u8; 3],
+}
+
+unsafe impl ByteValued for VirtioIommuReqTail {}
+
+/// ATTACH request
+#[allow(unused)]
+#[derive(Copy, Clone, Debug, Default)]
+#[repr(packed)]
+struct VirtioIommuReqAttach {
+    domain: u32,
+    endpoint: u32,
+    reserved: [u8; 8],
+}
+
+unsafe impl ByteValued for VirtioIommuReqAttach {}
+
+/// DETACH request
+#[allow(unused)]
+#[derive(Copy, Clone, Debug, Default)]
+#[repr(packed)]
+struct VirtioIommuReqDetach {
+    domain: u32,
+    endpoint: u32,
+    reserved: [u8; 8],
+}
+
+unsafe impl ByteValued for VirtioIommuReqDetach {}
+
+/// Virtio IOMMU request MAP flags
+#[allow(unused)]
+const VIRTIO_IOMMU_MAP_F_READ: u32 = 1;
+#[allow(unused)]
+const VIRTIO_IOMMU_MAP_F_WRITE: u32 = 1 << 1;
+#[allow(unused)]
+const VIRTIO_IOMMU_MAP_F_EXEC: u32 = 1 << 2;
+#[allow(unused)]
+const VIRTIO_IOMMU_MAP_F_MMIO: u32 = 1 << 3;
+
+/// MAP request
+#[allow(unused)]
+#[derive(Copy, Clone, Debug, Default)]
+#[repr(packed)]
+struct VirtioIommuReqMap {
+    domain: u32,
+    virt_start: u64,
+    virt_end: u64,
+    phys_start: u64,
+    flags: u32,
+}
+
+unsafe impl ByteValued for VirtioIommuReqMap {}
+
+/// UNMAP request
+#[allow(unused)]
+#[derive(Copy, Clone, Debug, Default)]
+#[repr(packed)]
+struct VirtioIommuReqUnmap {
+    domain: u32,
+    virt_start: u64,
+    virt_end: u64,
+    reserved: [u8; 4],
+}
+
+unsafe impl ByteValued for VirtioIommuReqUnmap {}
+
+/// Virtio IOMMU request PROBE types
+#[allow(unused)]
+const VIRTIO_IOMMU_PROBE_T_MASK: u32 = 0xfff;
+#[allow(unused)]
+const VIRTIO_IOMMU_PROBE_T_NONE: u32 = 0;
+#[allow(unused)]
+const VIRTIO_IOMMU_PROBE_T_RESV_MEM: u32 = 1;
+
+/// PROBE request
+#[allow(unused)]
+#[derive(Copy, Clone, Debug, Default)]
+#[repr(packed)]
+struct VirtioIommuReqProbe {
+    endpoint: u32,
+    reserved: [u64; 8],
+}
+
+unsafe impl ByteValued for VirtioIommuReqProbe {}
+
+#[allow(unused)]
+#[derive(Copy, Clone, Debug, Default)]
+#[repr(packed)]
+struct VirtioIommuProbeProperty {
+    type_: u16,
+    length: u16,
+}
+
+unsafe impl ByteValued for VirtioIommuProbeProperty {}
+
+/// Virtio IOMMU request PROBE property RESV_MEM subtypes
+#[allow(unused)]
+const VIRTIO_IOMMU_RESV_MEM_T_RESERVED: u32 = 0;
+#[allow(unused)]
+const VIRTIO_IOMMU_RESV_MEM_T_MSI: u32 = 1;
+
+#[allow(unused)]
+#[derive(Copy, Clone, Debug, Default)]
+#[repr(packed)]
+struct VirtioIommuProbeResvMem {
+    head: VirtioIommuProbeProperty,
+    subtype: u8,
+    reserved: [u8; 3],
+    start: u64,
+    end: u64,
+}
+
+unsafe impl ByteValued for VirtioIommuProbeResvMem {}
+
+/// Virtio IOMMU fault flags
+#[allow(unused)]
+const VIRTIO_IOMMU_FAULT_F_READ: u32 = 1;
+#[allow(unused)]
+const VIRTIO_IOMMU_FAULT_F_WRITE: u32 = 1 << 1;
+#[allow(unused)]
+const VIRTIO_IOMMU_FAULT_F_EXEC: u32 = 1 << 2;
+#[allow(unused)]
+const VIRTIO_IOMMU_FAULT_F_ADDRESS: u32 = 1 << 8;
+
+/// Virtio IOMMU fault reasons
+#[allow(unused)]
+const VIRTIO_IOMMU_FAULT_R_UNKNOWN: u32 = 0;
+#[allow(unused)]
+const VIRTIO_IOMMU_FAULT_R_DOMAIN: u32 = 1;
+#[allow(unused)]
+const VIRTIO_IOMMU_FAULT_R_MAPPING: u32 = 2;
+
+/// Fault reporting through eventq
+#[allow(unused)]
+#[derive(Copy, Clone, Debug, Default)]
+#[repr(packed)]
+struct VirtioIommuFault {
+    reason: u8,
+    reserved: [u8; 3],
+    flags: u32,
+    endpoint: u32,
+    reserved1: u32,
+    address: u64,
+}
+
+unsafe impl ByteValued for VirtioIommuFault {}
+
+#[derive(Debug)]
+enum Error {
+    /// Guest gave us bad memory addresses.
+    GuestMemory(GuestMemoryError),
+    /// Guest gave us a write only descriptor that protocol says to read from.
+    UnexpectedWriteOnlyDescriptor,
+    /// Guest gave us a read only descriptor that protocol says to write to.
+    UnexpectedReadOnlyDescriptor,
+    /// Guest gave us too few descriptors in a descriptor chain.
+    DescriptorChainTooShort,
+    /// Guest gave us a buffer that was too short to use.
+    BufferLengthTooSmall,
+    /// Guest sent us invalid request.
+    InvalidRequest,
+    /// Guest sent us invalid ATTACH request.
+    InvalidAttachRequest,
+    /// Guest sent us invalid DETACH request.
+    InvalidDetachRequest,
+    /// Guest sent us invalid MAP request.
+    InvalidMapRequest,
+    /// Guest sent us invalid UNMAP request.
+    InvalidUnmapRequest,
+    /// Guest sent us invalid PROBE request.
+    InvalidProbeRequest,
+    /// Failed to performing external mapping.
+    ExternalMapping(io::Error),
+    /// Failed to performing external unmapping.
+    ExternalUnmapping(io::Error),
+}
+
+impl Display for Error {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        use self::Error::*;
+
+        match self {
+            BufferLengthTooSmall => write!(f, "buffer length too small"),
+            DescriptorChainTooShort => write!(f, "descriptor chain too short"),
+            GuestMemory(e) => write!(f, "bad guest memory address: {}", e),
+            InvalidRequest => write!(f, "invalid request"),
+            InvalidAttachRequest => write!(f, "invalid attach request"),
+            InvalidDetachRequest => write!(f, "invalid detach request"),
+            InvalidMapRequest => write!(f, "invalid map request"),
+            InvalidUnmapRequest => write!(f, "invalid unmap request"),
+            InvalidProbeRequest => write!(f, "invalid probe request"),
+            UnexpectedReadOnlyDescriptor => write!(f, "unexpected read-only descriptor"),
+            UnexpectedWriteOnlyDescriptor => write!(f, "unexpected write-only descriptor"),
+            ExternalMapping(e) => write!(f, "failed performing external mapping: {}", e),
+            ExternalUnmapping(e) => write!(f, "failed performing external unmapping: {}", e),
+        }
+    }
+}
+
+#[derive(Debug, PartialEq)]
+enum RequestType {
+    Attach,
+    Detach,
+    Map,
+    Unmap,
+    Probe,
+}
+
+struct Request {
+    #[allow(unused)]
+    type_: RequestType,
+    status_addr: GuestAddress,
+}
+
+impl Request {
+    // Parse the available vring buffer. Based on the hashmap table of external
+    // mappings required from various devices such as VFIO or vhost-user ones,
+    // this function might update the hashmap table of external mappings per
+    // domain.
+    // Basically, the VMM knows about the device_id <=> mapping relationship
+    // before running the VM, but at runtime, a new domain <=> mapping hashmap
+    // is created based on the information provided from the guest driver for
+    // virtio-iommu (giving the link device_id <=> domain).
+    fn parse(
+        avail_desc: &DescriptorChain,
+        mem: &GuestMemoryMmap,
+        mapping: &Arc<IommuMapping>,
+        ext_mapping: &BTreeMap<u32, Arc<dyn ExternalDmaMapping>>,
+        ext_domain_mapping: &mut BTreeMap<u32, Arc<dyn ExternalDmaMapping>>,
+    ) -> result::Result<Request, Error> {
+        // The head contains the request type which MUST be readable.
+        if avail_desc.is_write_only() {
+            return Err(Error::UnexpectedWriteOnlyDescriptor);
+        }
+
+        if (avail_desc.len as usize) < size_of::<VirtioIommuReqHead>() {
+            return Err(Error::InvalidRequest);
+        }
+
+        let req_head: VirtioIommuReqHead =
+            mem.read_obj(avail_desc.addr).map_err(Error::GuestMemory)?;
+        let req_offset = size_of::<VirtioIommuReqHead>();
+        let desc_size_left = (avail_desc.len as usize) - req_offset;
+        let req_addr = if let Some(addr) = avail_desc.addr.checked_add(req_offset as u64) {
+            addr
+        } else {
+            return Err(Error::InvalidRequest);
+        };
+
+        let request_type = match req_head.type_ {
+            VIRTIO_IOMMU_T_ATTACH => {
+                if desc_size_left != size_of::<VirtioIommuReqAttach>() {
+                    return Err(Error::InvalidAttachRequest);
+                }
+
+                let req: VirtioIommuReqAttach = mem
+                    .read_obj(req_addr as GuestAddress)
+                    .map_err(Error::GuestMemory)?;
+                debug!("Attach request {:?}", req);
+
+                // Copy the value to use it as a proper reference.
+                let domain = req.domain;
+                let endpoint = req.endpoint;
+
+                // Add endpoint associated with specific domain
+                mapping.endpoints.write().unwrap().insert(endpoint, domain);
+
+                // If the endpoint is part of the list of devices with an
+                // external mapping, insert a new entry for the corresponding
+                // domain, with the same reference to the trait.
+                if let Some(map) = ext_mapping.get(&endpoint) {
+                    ext_domain_mapping.insert(domain, map.clone());
+                }
+
+                // Add new domain with no mapping if the entry didn't exist yet
+                let mut mappings = mapping.mappings.write().unwrap();
+                if !mappings.contains_key(&domain) {
+                    mappings.insert(domain, BTreeMap::new());
+                }
+
+                RequestType::Attach
+            }
+            VIRTIO_IOMMU_T_DETACH => {
+                if desc_size_left != size_of::<VirtioIommuReqDetach>() {
+                    return Err(Error::InvalidDetachRequest);
+                }
+
+                let req: VirtioIommuReqDetach = mem
+                    .read_obj(req_addr as GuestAddress)
+                    .map_err(Error::GuestMemory)?;
+                debug!("Detach request {:?}", req);
+
+                // Copy the value to use it as a proper reference.
+                let domain = req.domain;
+                let endpoint = req.endpoint;
+
+                // If the endpoint is part of the list of devices with an
+                // external mapping, remove the entry for the corresponding
+                // domain.
+                if ext_mapping.contains_key(&endpoint) {
+                    ext_domain_mapping.remove(&domain);
+                }
+
+                // Remove endpoint associated with specific domain
+                mapping.endpoints.write().unwrap().remove(&endpoint);
+
+                RequestType::Detach
+            }
+            VIRTIO_IOMMU_T_MAP => {
+                if desc_size_left != size_of::<VirtioIommuReqMap>() {
+                    return Err(Error::InvalidMapRequest);
+                }
+
+                let req: VirtioIommuReqMap = mem
+                    .read_obj(req_addr as GuestAddress)
+                    .map_err(Error::GuestMemory)?;
+                debug!("Map request {:?}", req);
+
+                // Copy the value to use it as a proper reference.
+                let domain = req.domain;
+
+                // Trigger external mapping if necessary.
+                if let Some(ext_map) = ext_domain_mapping.get(&domain) {
+                    let size = req.virt_end - req.virt_start + 1;
+                    ext_map
+                        .map(req.virt_start, req.phys_start, size)
+                        .map_err(Error::ExternalMapping)?;
+                }
+
+                // Add new mapping associated with the domain
+                if let Some(entry) = mapping.mappings.write().unwrap().get_mut(&domain) {
+                    entry.insert(
+                        req.virt_start,
+                        Mapping {
+                            gpa: req.phys_start,
+                            size: req.virt_end - req.virt_start + 1,
+                        },
+                    );
+                } else {
+                    return Err(Error::InvalidMapRequest);
+                }
+
+                RequestType::Map
+            }
+            VIRTIO_IOMMU_T_UNMAP => {
+                if desc_size_left != size_of::<VirtioIommuReqUnmap>() {
+                    return Err(Error::InvalidUnmapRequest);
+                }
+
+                let req: VirtioIommuReqUnmap = mem
+                    .read_obj(req_addr as GuestAddress)
+                    .map_err(Error::GuestMemory)?;
+                debug!("Unmap request {:?}", req);
+
+                // Copy the value to use it as a proper reference.
+                let domain = req.domain;
+                let virt_start = req.virt_start;
+
+                // Trigger external unmapping if necessary.
+                if let Some(ext_map) = ext_domain_mapping.get(&domain) {
+                    let size = req.virt_end - virt_start + 1;
+                    ext_map
+                        .unmap(virt_start, size)
+                        .map_err(Error::ExternalUnmapping)?;
+                }
+
+                // Add new mapping associated with the domain
+                if let Some(entry) = mapping.mappings.write().unwrap().get_mut(&domain) {
+                    entry.remove(&virt_start);
+                }
+
+                RequestType::Unmap
+            }
+            VIRTIO_IOMMU_T_PROBE => {
+                if desc_size_left != size_of::<VirtioIommuReqProbe>() {
+                    return Err(Error::InvalidProbeRequest);
+                }
+
+                let req: VirtioIommuReqProbe = mem
+                    .read_obj(req_addr as GuestAddress)
+                    .map_err(Error::GuestMemory)?;
+                debug!("Probe request {:?}", req);
+
+                RequestType::Probe
+            }
+            _ => return Err(Error::InvalidRequest),
+        };
+
+        let status_desc = avail_desc
+            .next_descriptor()
+            .ok_or(Error::DescriptorChainTooShort)?;
+
+        // The status MUST always be writable
+        if !status_desc.is_write_only() {
+            return Err(Error::UnexpectedReadOnlyDescriptor);
+        }
+
+        if (status_desc.len as usize) < size_of::<VirtioIommuReqTail>() {
+            return Err(Error::BufferLengthTooSmall);
+        }
+
+        Ok(Request {
+            type_: request_type,
+            status_addr: status_desc.addr,
+        })
+    }
+}
+
+struct IommuEpollHandler {
+    queues: Vec<Queue>,
+    mem: Arc<RwLock<GuestMemoryMmap>>,
+    interrupt_cb: Arc<VirtioInterrupt>,
+    queue_evts: Vec<EventFd>,
+    kill_evt: EventFd,
+    pause_evt: EventFd,
+    mapping: Arc<IommuMapping>,
+    ext_mapping: BTreeMap<u32, Arc<dyn ExternalDmaMapping>>,
+    ext_domain_mapping: BTreeMap<u32, Arc<dyn ExternalDmaMapping>>,
+}
+
+impl IommuEpollHandler {
+    fn request_queue(&mut self) -> bool {
+        let mut used_desc_heads = [(0, 0); QUEUE_SIZE as usize];
+        let mut used_count = 0;
+        let mem = self.mem.read().unwrap();
+        for avail_desc in self.queues[0].iter(&mem) {
+            let len = match Request::parse(
+                &avail_desc,
+                &mem,
+                &self.mapping,
+                &self.ext_mapping,
+                &mut self.ext_domain_mapping,
+            ) {
+                Ok(ref req) => {
+                    let reply = VirtioIommuReqTail {
+                        status: VIRTIO_IOMMU_S_OK,
+                        ..Default::default()
+                    };
+
+                    match mem.write_obj(reply, req.status_addr) {
+                        Ok(_) => size_of::<VirtioIommuReqTail>() as u32,
+                        Err(e) => {
+                            error!("bad guest memory address: {}", e);
+                            0
+                        }
+                    }
+                }
+                Err(e) => {
+                    error!("Failed to parse available descriptor chain: {:?}", e);
+                    0
+                }
+            };
+
+            used_desc_heads[used_count] = (avail_desc.index, len);
+            used_count += 1;
+        }
+
+        for &(desc_index, len) in &used_desc_heads[..used_count] {
+            self.queues[0].add_used(&mem, desc_index, len);
+        }
+        used_count > 0
+    }
+
+    fn event_queue(&mut self) -> bool {
+        false
+    }
+
+    fn signal_used_queue(&self, queue: &Queue) -> result::Result<(), DeviceError> {
+        (self.interrupt_cb)(&VirtioInterruptType::Queue, Some(queue)).map_err(|e| {
+            error!("Failed to signal used queue: {:?}", e);
+            DeviceError::FailedSignalingUsedQueue(e)
+        })
+    }
+
+    fn run(&mut self, paused: Arc<AtomicBool>) -> result::Result<(), DeviceError> {
+        // Create the epoll file descriptor
+        let epoll_fd = epoll::create(true).map_err(DeviceError::EpollCreateFd)?;
+
+        // Add events
+        epoll::ctl(
+            epoll_fd,
+            epoll::ControlOptions::EPOLL_CTL_ADD,
+            self.queue_evts[0].as_raw_fd(),
+            epoll::Event::new(epoll::Events::EPOLLIN, u64::from(REQUEST_Q_EVENT)),
+        )
+        .map_err(DeviceError::EpollCtl)?;
+        epoll::ctl(
+            epoll_fd,
+            epoll::ControlOptions::EPOLL_CTL_ADD,
+            self.queue_evts[1].as_raw_fd(),
+            epoll::Event::new(epoll::Events::EPOLLIN, u64::from(EVENT_Q_EVENT)),
+        )
+        .map_err(DeviceError::EpollCtl)?;
+        epoll::ctl(
+            epoll_fd,
+            epoll::ControlOptions::EPOLL_CTL_ADD,
+            self.kill_evt.as_raw_fd(),
+            epoll::Event::new(epoll::Events::EPOLLIN, u64::from(KILL_EVENT)),
+        )
+        .map_err(DeviceError::EpollCtl)?;
+        epoll::ctl(
+            epoll_fd,
+            epoll::ControlOptions::EPOLL_CTL_ADD,
+            self.pause_evt.as_raw_fd(),
+            epoll::Event::new(epoll::Events::EPOLLIN, u64::from(PAUSE_EVENT)),
+        )
+        .map_err(DeviceError::EpollCtl)?;
+
+        const EPOLL_EVENTS_LEN: usize = 100;
+        let mut events = vec![epoll::Event::new(epoll::Events::empty(), 0); EPOLL_EVENTS_LEN];
+
+        'epoll: loop {
+            let num_events = match epoll::wait(epoll_fd, -1, &mut events[..]) {
+                Ok(res) => res,
+                Err(e) => {
+                    if e.kind() == io::ErrorKind::Interrupted {
+                        // It's well defined from the epoll_wait() syscall
+                        // documentation that the epoll loop can be interrupted
+                        // before any of the requested events occurred or the
+                        // timeout expired. In both those cases, epoll_wait()
+                        // returns an error of type EINTR, but this should not
+                        // be considered as a regular error. Instead it is more
+                        // appropriate to retry, by calling into epoll_wait().
+                        continue;
+                    }
+                    return Err(DeviceError::EpollWait(e));
+                }
+            };
+
+            for event in events.iter().take(num_events) {
+                let ev_type = event.data as u16;
+
+                match ev_type {
+                    REQUEST_Q_EVENT => {
+                        if let Err(e) = self.queue_evts[0].read() {
+                            error!("Failed to get queue event: {:?}", e);
+                            break 'epoll;
+                        } else if self.request_queue() {
+                            if let Err(e) = self.signal_used_queue(&self.queues[0]) {
+                                error!("Failed to signal used queue: {:?}", e);
+                                break 'epoll;
+                            }
+                        }
+                    }
+                    EVENT_Q_EVENT => {
+                        if let Err(e) = self.queue_evts[1].read() {
+                            error!("Failed to get queue event: {:?}", e);
+                            break 'epoll;
+                        } else if self.event_queue() {
+                            if let Err(e) = self.signal_used_queue(&self.queues[1]) {
+                                error!("Failed to signal used queue: {:?}", e);
+                                break 'epoll;
+                            }
+                        }
+                    }
+                    KILL_EVENT => {
+                        debug!("kill_evt received, stopping epoll loop");
+                        break 'epoll;
+                    }
+                    PAUSE_EVENT => {
+                        debug!("PAUSE_EVENT received, pausing virtio-iommu epoll loop");
+                        // We loop here to handle spurious park() returns.
+                        // Until we have not resumed, the paused boolean will
+                        // be true.
+                        while paused.load(Ordering::SeqCst) {
+                            thread::park();
+                        }
+                    }
+                    _ => {
+                        error!("Unknown event for virtio-iommu");
+                        break 'epoll;
+                    }
+                }
+            }
+
+            info!("Exit epoll loop");
+        }
+
+        Ok(())
+    }
+}
+
+#[derive(Clone, Copy)]
+struct Mapping {
+    gpa: u64,
+    size: u64,
+}
+
+pub struct IommuMapping {
+    // Domain related to an endpoint.
+    endpoints: Arc<RwLock<BTreeMap<u32, u32>>>,
+    // List of mappings per domain.
+    mappings: Arc<RwLock<BTreeMap<u32, BTreeMap<u64, Mapping>>>>,
+}
+
+impl DmaRemapping for IommuMapping {
+    fn translate(&self, id: u32, addr: u64) -> std::result::Result<u64, std::io::Error> {
+        debug!("Translate addr 0x{:x}", addr);
+        if let Some(domain) = self.endpoints.read().unwrap().get(&id) {
+            if let Some(mapping) = self.mappings.read().unwrap().get(domain) {
+                let range_start = if VIRTIO_IOMMU_PAGE_SIZE_MASK > addr {
+                    0
+                } else {
+                    addr - VIRTIO_IOMMU_PAGE_SIZE_MASK
+                };
+                for (&key, &value) in mapping.range((Included(&range_start), Included(&addr))) {
+                    if addr >= key && addr < key + value.size {
+                        let new_addr = addr - key + value.gpa;
+                        debug!("Into new_addr 0x{:x}", new_addr);
+                        return Ok(new_addr);
+                    }
+                }
+            }
+        }
+
+        debug!("Into same addr...");
+        Ok(addr)
+    }
+}
+
+pub struct Iommu {
+    kill_evt: Option<EventFd>,
+    pause_evt: Option<EventFd>,
+    avail_features: u64,
+    acked_features: u64,
+    config: VirtioIommuConfig,
+    mapping: Arc<IommuMapping>,
+    ext_mapping: BTreeMap<u32, Arc<dyn ExternalDmaMapping>>,
+    queue_evts: Option<Vec<EventFd>>,
+    interrupt_cb: Option<Arc<VirtioInterrupt>>,
+    epoll_thread: Option<thread::JoinHandle<result::Result<(), DeviceError>>>,
+    paused: Arc<AtomicBool>,
+}
+
+impl Iommu {
+    pub fn new() -> io::Result<(Self, Arc<IommuMapping>)> {
+        let config = VirtioIommuConfig {
+            page_size_mask: VIRTIO_IOMMU_PAGE_SIZE_MASK,
+            ..Default::default()
+        };
+
+        let mapping = Arc::new(IommuMapping {
+            endpoints: Arc::new(RwLock::new(BTreeMap::new())),
+            mappings: Arc::new(RwLock::new(BTreeMap::new())),
+        });
+
+        Ok((
+            Iommu {
+                kill_evt: None,
+                pause_evt: None,
+                avail_features: 1u64 << VIRTIO_F_VERSION_1 | 1u64 << VIRTIO_IOMMU_F_MAP_UNMAP,
+                acked_features: 0u64,
+                config,
+                mapping: mapping.clone(),
+                ext_mapping: BTreeMap::new(),
+                queue_evts: None,
+                interrupt_cb: None,
+                epoll_thread: None,
+                paused: Arc::new(AtomicBool::new(false)),
+            },
+            mapping,
+        ))
+    }
+
+    pub fn add_external_mapping(&mut self, device_id: u32, mapping: Arc<dyn ExternalDmaMapping>) {
+        self.ext_mapping.insert(device_id, mapping);
+    }
+}
+
+impl Drop for Iommu {
+    fn drop(&mut self) {
+        if let Some(kill_evt) = self.kill_evt.take() {
+            // Ignore the result because there is nothing we can do about it.
+            let _ = kill_evt.write(1);
+        }
+    }
+}
+
+impl VirtioDevice for Iommu {
+    fn device_type(&self) -> u32 {
+        VirtioDeviceType::TYPE_IOMMU as u32
+    }
+
+    fn queue_max_sizes(&self) -> &[u16] {
+        QUEUE_SIZES
+    }
+
+    fn features(&self, page: u32) -> u32 {
+        match page {
+            // Get the lower 32-bits of the features bitfield.
+            0 => self.avail_features as u32,
+            // Get the upper 32-bits of the features bitfield.
+            1 => (self.avail_features >> 32) as u32,
+            _ => {
+                warn!("Received request for unknown features page.");
+                0u32
+            }
+        }
+    }
+
+    fn ack_features(&mut self, page: u32, value: u32) {
+        let mut v = match page {
+            0 => u64::from(value),
+            1 => u64::from(value) << 32,
+            _ => {
+                warn!("Cannot acknowledge unknown features page.");
+                0u64
+            }
+        };
+
+        // Check if the guest is ACK'ing a feature that we didn't claim to have.
+        let unrequested_features = v & !self.avail_features;
+        if unrequested_features != 0 {
+            warn!("Received acknowledge request for unknown feature.");
+
+            // Don't count these features as acked.
+            v &= !unrequested_features;
+        }
+        self.acked_features |= v;
+    }
+
+    fn read_config(&self, offset: u64, mut data: &mut [u8]) {
+        let config_slice = self.config.as_slice();
+        let config_len = config_slice.len() as u64;
+        if offset >= config_len {
+            error!("Failed to read config space");
+            return;
+        }
+
+        if let Some(end) = offset.checked_add(data.len() as u64) {
+            // This write can't fail, offset and end are checked against config_len.
+            data.write_all(&config_slice[offset as usize..cmp::min(end, config_len) as usize])
+                .unwrap();
+        }
+    }
+
+    fn write_config(&mut self, _offset: u64, _data: &[u8]) {
+        warn!("virtio-iommu device configuration is read-only");
+    }
+
+    fn activate(
+        &mut self,
+        mem: Arc<RwLock<GuestMemoryMmap>>,
+        interrupt_cb: Arc<VirtioInterrupt>,
+        queues: Vec<Queue>,
+        queue_evts: Vec<EventFd>,
+    ) -> ActivateResult {
+        if queues.len() != NUM_QUEUES || queue_evts.len() != NUM_QUEUES {
+            error!(
+                "Cannot perform activate. Expected {} queue(s), got {}",
+                NUM_QUEUES,
+                queues.len()
+            );
+            return Err(ActivateError::BadActivate);
+        }
+
+        let (self_kill_evt, kill_evt) = EventFd::new(EFD_NONBLOCK)
+            .and_then(|e| Ok((e.try_clone()?, e)))
+            .map_err(|e| {
+                error!("failed creating kill EventFd pair: {}", e);
+                ActivateError::BadActivate
+            })?;
+        self.kill_evt = Some(self_kill_evt);
+
+        let (self_pause_evt, pause_evt) = EventFd::new(EFD_NONBLOCK)
+            .and_then(|e| Ok((e.try_clone()?, e)))
+            .map_err(|e| {
+                error!("failed creating pause EventFd pair: {}", e);
+                ActivateError::BadActivate
+            })?;
+        self.pause_evt = Some(self_pause_evt);
+
+        // Save the interrupt EventFD as we need to return it on reset
+        // but clone it to pass into the thread.
+        self.interrupt_cb = Some(interrupt_cb.clone());
+
+        let mut tmp_queue_evts: Vec<EventFd> = Vec::new();
+        for queue_evt in queue_evts.iter() {
+            // Save the queue EventFD as we need to return it on reset
+            // but clone it to pass into the thread.
+            tmp_queue_evts.push(queue_evt.try_clone().map_err(|e| {
+                error!("failed to clone queue EventFd: {}", e);
+                ActivateError::BadActivate
+            })?);
+        }
+        self.queue_evts = Some(tmp_queue_evts);
+
+        let mut handler = IommuEpollHandler {
+            queues,
+            mem,
+            interrupt_cb,
+            queue_evts,
+            kill_evt,
+            pause_evt,
+            mapping: self.mapping.clone(),
+            ext_mapping: self.ext_mapping.clone(),
+            ext_domain_mapping: BTreeMap::new(),
+        };
+
+        let paused = self.paused.clone();
+        thread::Builder::new()
+            .name("virtio_iommu".to_string())
+            .spawn(move || handler.run(paused))
+            .map(|thread| self.epoll_thread = Some(thread))
+            .map_err(|e| {
+                error!("failed to clone the virtio-iommu epoll thread: {}", e);
+                ActivateError::BadActivate
+            })?;
+
+        Ok(())
+    }
+
+    fn reset(&mut self) -> Option<(Arc<VirtioInterrupt>, Vec<EventFd>)> {
+        // We first must resume the virtio thread if it was paused.
+        if self.pause_evt.take().is_some() {
+            self.resume().ok()?;
+        }
+
+        if let Some(kill_evt) = self.kill_evt.take() {
+            // Ignore the result because there is nothing we can do about it.
+            let _ = kill_evt.write(1);
+        }
+
+        // Return the interrupt and queue EventFDs
+        Some((
+            self.interrupt_cb.take().unwrap(),
+            self.queue_evts.take().unwrap(),
+        ))
+    }
+}
+
+virtio_pausable!(Iommu);
+impl Snapshotable for Iommu {}
+impl Migratable for Iommu {}

vm-virtio/src/lib.rs

@@ -12,21 +12,26 @@
 extern crate epoll;
 #[macro_use]
 extern crate log;
+#[cfg(feature = "pci_support")]
 extern crate pci;
 extern crate vhost_rs;
 extern crate virtio_bindings;
+extern crate vm_device;
 extern crate vm_memory;
 
 use std::fmt;
 use std::io;
 
-mod block;
+#[macro_use]
-mod console;
 mod device;
+pub mod block;
+mod console;
+mod iommu;
 pub mod net;
 mod pmem;
 mod queue;
 mod rng;
+pub mod vsock;
 
 pub mod transport;
 pub mod vhost_user;
@@ -34,10 +39,12 @@ pub mod vhost_user;
 pub use self::block::*;
 pub use self::console::*;
 pub use self::device::*;
+pub use self::iommu::*;
 pub use self::net::*;
 pub use self::pmem::*;
 pub use self::queue::*;
 pub use self::rng::*;
+pub use self::vsock::*;
 
 const DEVICE_INIT: u32 = 0x00;
 const DEVICE_ACKNOWLEDGE: u32 = 0x01;
@@ -47,6 +54,8 @@ const DEVICE_FEATURES_OK: u32 = 0x08;
 const DEVICE_FAILED: u32 = 0x80;
 
 const VIRTIO_F_VERSION_1: u32 = 32;
+const VIRTIO_F_IOMMU_PLATFORM: u32 = 33;
+const VIRTIO_F_IN_ORDER: u32 = 35;
 
 // Types taken from linux/virtio_ids.h
 #[derive(Copy, Clone)]
@@ -63,6 +72,7 @@ enum VirtioDeviceType {
     TYPE_GPU = 16,
     TYPE_INPUT = 18,
     TYPE_VSOCK = 19,
+    TYPE_IOMMU = 23,
     TYPE_FS = 26,
     TYPE_PMEM = 27,
     TYPE_UNKNOWN = 0xFF,
@@ -79,6 +89,7 @@ impl From<u32> for VirtioDeviceType {
             16 => VirtioDeviceType::TYPE_GPU,
             18 => VirtioDeviceType::TYPE_INPUT,
             19 => VirtioDeviceType::TYPE_VSOCK,
+            23 => VirtioDeviceType::TYPE_IOMMU,
             26 => VirtioDeviceType::TYPE_FS,
             27 => VirtioDeviceType::TYPE_PMEM,
             _ => VirtioDeviceType::TYPE_UNKNOWN,
@@ -126,6 +137,10 @@ pub enum ActivateError {
     VhostUserSetup(vhost_user::Error),
     /// Failed to setup vhost-user daemon.
     VhostUserNetSetup(vhost_user::Error),
+    /// Failed to setup vhost-user daemon.
+    VhostUserBlkSetup(vhost_user::Error),
+    /// Failed to reset vhost-user daemon.
+    VhostUserReset(vhost_user::Error),
 }
 
 pub type ActivateResult = std::result::Result<(), ActivateError>;

vm-virtio/src/net.rs

@@ -15,6 +15,7 @@ use std::mem;
 use std::net::Ipv4Addr;
 use std::os::unix::io::{AsRawFd, RawFd};
 use std::result;
+use std::sync::atomic::{AtomicBool, Ordering};
 use std::sync::{Arc, RwLock};
 use std::thread;
 use std::vec::Vec;
@@ -28,7 +29,8 @@ use super::{
 };
 use crate::VirtioInterrupt;
 use net_util::{MacAddr, Tap, TapError, MAC_ADDR_LEN};
-use virtio_bindings::virtio_net::*;
+use virtio_bindings::bindings::virtio_net::*;
+use vm_device::{Migratable, MigratableError, Pausable, Snapshotable};
 use vm_memory::{Bytes, GuestAddress, GuestMemoryMmap};
 use vmm_sys_util::eventfd::EventFd;
 
@@ -50,6 +52,8 @@ const TX_QUEUE_EVENT: DeviceEventT = 2;
 pub const KILL_EVENT: DeviceEventT = 3;
 // Number of DeviceEventT events supported by this implementation.
 pub const NET_EVENTS_COUNT: usize = 4;
+// The device should be paused.
+const PAUSE_EVENT: DeviceEventT = 5;
 
 #[derive(Debug)]
 pub enum Error {
@@ -121,6 +125,7 @@ struct NetEpollHandler {
     tx: TxVirtio,
     interrupt_cb: Arc<VirtioInterrupt>,
     kill_evt: EventFd,
+    pause_evt: EventFd,
     epoll_fd: RawFd,
     rx_tap_listening: bool,
 }
@@ -323,7 +328,7 @@ impl NetEpollHandler {
         Ok(())
     }
 
-    fn run(&mut self) -> result::Result<(), DeviceError> {
+    fn run(&mut self, paused: Arc<AtomicBool>) -> result::Result<(), DeviceError> {
         // Create the epoll file descriptor
         self.epoll_fd = epoll::create(true).map_err(DeviceError::EpollCreateFd)?;
         // Add events
@@ -351,6 +356,13 @@ impl NetEpollHandler {
             epoll::Event::new(epoll::Events::EPOLLIN, u64::from(KILL_EVENT)),
         )
         .map_err(DeviceError::EpollCtl)?;
+        epoll::ctl(
+            self.epoll_fd,
+            epoll::ControlOptions::EPOLL_CTL_ADD,
+            self.pause_evt.as_raw_fd(),
+            epoll::Event::new(epoll::Events::EPOLLIN, u64::from(PAUSE_EVENT)),
+        )
+        .map_err(DeviceError::EpollCtl)?;
 
         const EPOLL_EVENTS_LEN: usize = 100;
         let mut events = vec![epoll::Event::new(epoll::Events::empty(), 0); EPOLL_EVENTS_LEN];
@@ -420,6 +432,15 @@ impl NetEpollHandler {
                         debug!("KILL_EVENT received, stopping epoll loop");
                         break 'epoll;
                     }
+                    PAUSE_EVENT => {
+                        debug!("PAUSE_EVENT received, pausing virtio-net epoll loop");
+                        // We loop here to handle spurious park() returns.
+                        // Until we have not resumed, the paused boolean will
+                        // be true.
+                        while paused.load(Ordering::SeqCst) {
+                            thread::park();
+                        }
+                    }
                     _ => {
                         error!("Unknown event for virtio-net");
                     }
@@ -433,17 +454,22 @@ impl NetEpollHandler {
 
 pub struct Net {
     kill_evt: Option<EventFd>,
+    pause_evt: Option<EventFd>,
     tap: Option<Tap>,
     avail_features: u64,
     acked_features: u64,
     // The config space will only consist of the MAC address specified by the user,
     // or nothing, if no such address if provided.
     config_space: Vec<u8>,
+    queue_evts: Option<Vec<EventFd>>,
+    interrupt_cb: Option<Arc<VirtioInterrupt>>,
+    epoll_thread: Option<thread::JoinHandle<result::Result<(), DeviceError>>>,
+    paused: Arc<AtomicBool>,
 }
 
 impl Net {
     /// Create a new virtio network device with the given TAP interface.
-    pub fn new_with_tap(tap: Tap, guest_mac: Option<&MacAddr>) -> Result<Self> {
+    pub fn new_with_tap(tap: Tap, guest_mac: Option<&MacAddr>, iommu: bool) -> Result<Self> {
         // Set offload flags to match the virtio features below.
         tap.set_offload(
             net_gen::TUN_F_CSUM | net_gen::TUN_F_UFO | net_gen::TUN_F_TSO4 | net_gen::TUN_F_TSO6,
@@ -462,6 +488,10 @@ impl Net {
             | 1 << VIRTIO_NET_F_HOST_UFO
             | 1 << VIRTIO_F_VERSION_1;
 
+        if iommu {
+            avail_features |= 1u64 << VIRTIO_F_IOMMU_PLATFORM;
+        }
+
         let mut config_space;
         if let Some(mac) = guest_mac {
             config_space = Vec::with_capacity(MAC_ADDR_LEN);
@@ -477,22 +507,32 @@ impl Net {
 
         Ok(Net {
             kill_evt: None,
+            pause_evt: None,
             tap: Some(tap),
             avail_features,
             acked_features: 0u64,
             config_space,
+            queue_evts: None,
+            interrupt_cb: None,
+            epoll_thread: None,
+            paused: Arc::new(AtomicBool::new(false)),
         })
     }
 
     /// Create a new virtio network device with the given IP address and
     /// netmask.
-    pub fn new(ip_addr: Ipv4Addr, netmask: Ipv4Addr, guest_mac: Option<&MacAddr>) -> Result<Self> {
+    pub fn new(
+        ip_addr: Ipv4Addr,
+        netmask: Ipv4Addr,
+        guest_mac: Option<&MacAddr>,
+        iommu: bool,
+    ) -> Result<Self> {
         let tap = Tap::new().map_err(Error::TapOpen)?;
         tap.set_ip_addr(ip_addr).map_err(Error::TapSetIp)?;
         tap.set_netmask(netmask).map_err(Error::TapSetNetmask)?;
         tap.enable().map_err(Error::TapEnable)?;
 
-        Self::new_with_tap(tap, guest_mac)
+        Self::new_with_tap(tap, guest_mac, iommu)
     }
 }
 
@@ -587,17 +627,38 @@ impl VirtioDevice for Net {
             return Err(ActivateError::BadActivate);
         }
 
-        let (self_kill_evt, kill_evt) =
+        let (self_kill_evt, kill_evt) = EventFd::new(EFD_NONBLOCK)
-            match EventFd::new(EFD_NONBLOCK).and_then(|e| Ok((e.try_clone()?, e))) {
+            .and_then(|e| Ok((e.try_clone()?, e)))
-                Ok(v) => v,
+            .map_err(|e| {
-                Err(e) => {
+                error!("failed creating kill EventFd pair: {}", e);
-                    error!("failed creating kill EventFd pair: {}", e);
+                ActivateError::BadActivate
-                    return Err(ActivateError::BadActivate);
+            })?;
-                }
-            };
         self.kill_evt = Some(self_kill_evt);
 
-        if let Some(tap) = self.tap.take() {
+        let (self_pause_evt, pause_evt) = EventFd::new(EFD_NONBLOCK)
+            .and_then(|e| Ok((e.try_clone()?, e)))
+            .map_err(|e| {
+                error!("failed creating pause EventFd pair: {}", e);
+                ActivateError::BadActivate
+            })?;
+        self.pause_evt = Some(self_pause_evt);
+
+        if let Some(tap) = self.tap.clone() {
+            // Save the interrupt EventFD as we need to return it on reset
+            // but clone it to pass into the thread.
+            self.interrupt_cb = Some(interrupt_cb.clone());
+
+            let mut tmp_queue_evts: Vec<EventFd> = Vec::new();
+            for queue_evt in queue_evts.iter() {
+                // Save the queue EventFD as we need to return it on reset
+                // but clone it to pass into the thread.
+                tmp_queue_evts.push(queue_evt.try_clone().map_err(|e| {
+                    error!("failed to clone queue EventFd: {}", e);
+                    ActivateError::BadActivate
+                })?);
+            }
+            self.queue_evts = Some(tmp_queue_evts);
+
             let rx_queue = queues.remove(0);
             let tx_queue = queues.remove(0);
             let rx_queue_evt = queue_evts.remove(0);
@@ -609,21 +670,45 @@ impl VirtioDevice for Net {
                 tx: TxVirtio::new(tx_queue, tx_queue_evt),
                 interrupt_cb,
                 kill_evt,
+                pause_evt,
                 epoll_fd: 0,
                 rx_tap_listening: false,
             };
 
-            let worker_result = thread::Builder::new()
+            let paused = self.paused.clone();
+            thread::Builder::new()
                 .name("virtio_net".to_string())
-                .spawn(move || handler.run());
+                .spawn(move || handler.run(paused))
-
+                .map(|thread| self.epoll_thread = Some(thread))
-            if let Err(e) = worker_result {
+                .map_err(|e| {
-                error!("failed to spawn virtio_blk worker: {}", e);
+                    error!("failed to clone the virtio-net epoll thread: {}", e);
-                return Err(ActivateError::BadActivate);
+                    ActivateError::BadActivate
-            }
+                })?;
 
             return Ok(());
         }
         Err(ActivateError::BadActivate)
     }
+
+    fn reset(&mut self) -> Option<(Arc<VirtioInterrupt>, Vec<EventFd>)> {
+        // We first must resume the virtio thread if it was paused.
+        if self.pause_evt.take().is_some() {
+            self.resume().ok()?;
+        }
+
+        if let Some(kill_evt) = self.kill_evt.take() {
+            // Ignore the result because there is nothing we can do about it.
+            let _ = kill_evt.write(1);
+        }
+
+        // Return the interrupt and queue EventFDs
+        Some((
+            self.interrupt_cb.take().unwrap(),
+            self.queue_evts.take().unwrap(),
+        ))
+    }
 }
+
+virtio_pausable!(Net);
+impl Snapshotable for Net {}
+impl Migratable for Net {}

vm-virtio/src/pmem.rs

@@ -15,15 +15,17 @@ use std::io::{self, Write};
 use std::mem::size_of;
 use std::os::unix::io::AsRawFd;
 use std::result;
+use std::sync::atomic::{AtomicBool, Ordering};
 use std::sync::{Arc, RwLock};
 use std::thread;
 
 use super::Error as DeviceError;
 use super::{
     ActivateError, ActivateResult, DescriptorChain, DeviceEventT, Queue, VirtioDevice,
-    VirtioDeviceType, VIRTIO_F_VERSION_1,
+    VirtioDeviceType, VIRTIO_F_IOMMU_PLATFORM, VIRTIO_F_VERSION_1,
 };
 use crate::{VirtioInterrupt, VirtioInterruptType};
+use vm_device::{Migratable, MigratableError, Pausable, Snapshotable};
 use vm_memory::{
     Address, ByteValued, Bytes, GuestAddress, GuestMemoryError, GuestMemoryMmap, GuestUsize,
 };
@@ -41,6 +43,8 @@ const VIRTIO_PMEM_RESP_TYPE_EIO: u32 = 1;
 const QUEUE_AVAIL_EVENT: DeviceEventT = 0;
 // The device has been dropped.
 const KILL_EVENT: DeviceEventT = 1;
+// The device should be paused.
+const PAUSE_EVENT: DeviceEventT = 2;
 
 #[derive(Copy, Clone, Debug, Default)]
 #[repr(C)]
@@ -159,6 +163,7 @@ struct PmemEpollHandler {
     interrupt_cb: Arc<VirtioInterrupt>,
     queue_evt: EventFd,
     kill_evt: EventFd,
+    pause_evt: EventFd,
 }
 
 impl PmemEpollHandler {
@@ -214,7 +219,7 @@ impl PmemEpollHandler {
         })
     }
 
-    fn run(&mut self) -> result::Result<(), DeviceError> {
+    fn run(&mut self, paused: Arc<AtomicBool>) -> result::Result<(), DeviceError> {
         // Create the epoll file descriptor
         let epoll_fd = epoll::create(true).map_err(DeviceError::EpollCreateFd)?;
 
@@ -234,6 +239,14 @@ impl PmemEpollHandler {
         )
         .map_err(DeviceError::EpollCtl)?;
 
+        epoll::ctl(
+            epoll_fd,
+            epoll::ControlOptions::EPOLL_CTL_ADD,
+            self.pause_evt.as_raw_fd(),
+            epoll::Event::new(epoll::Events::EPOLLIN, u64::from(PAUSE_EVENT)),
+        )
+        .map_err(DeviceError::EpollCtl)?;
+
         const EPOLL_EVENTS_LEN: usize = 100;
         let mut events = vec![epoll::Event::new(epoll::Events::empty(), 0); EPOLL_EVENTS_LEN];
 
@@ -274,6 +287,15 @@ impl PmemEpollHandler {
                         debug!("kill_evt received, stopping epoll loop");
                         break 'epoll;
                     }
+                    PAUSE_EVENT => {
+                        debug!("PAUSE_EVENT received, pausing virtio-pmem epoll loop");
+                        // We loop here to handle spurious park() returns.
+                        // Until we have not resumed, the paused boolean will
+                        // be true.
+                        while paused.load(Ordering::SeqCst) {
+                            thread::park();
+                        }
+                    }
                     _ => {
                         error!("Unknown event for virtio-block");
                     }
@@ -287,25 +309,41 @@ impl PmemEpollHandler {
 
 pub struct Pmem {
     kill_evt: Option<EventFd>,
+    pause_evt: Option<EventFd>,
     disk: Option<File>,
     avail_features: u64,
     acked_features: u64,
     config: VirtioPmemConfig,
+    queue_evts: Option<Vec<EventFd>>,
+    interrupt_cb: Option<Arc<VirtioInterrupt>>,
+    epoll_thread: Option<thread::JoinHandle<result::Result<(), DeviceError>>>,
+    paused: Arc<AtomicBool>,
 }
 
 impl Pmem {
-    pub fn new(disk: File, addr: GuestAddress, size: GuestUsize) -> io::Result<Pmem> {
+    pub fn new(disk: File, addr: GuestAddress, size: GuestUsize, iommu: bool) -> io::Result<Pmem> {
         let config = VirtioPmemConfig {
             start: addr.raw_value().to_le(),
             size: size.to_le(),
         };
 
+        let mut avail_features = 1u64 << VIRTIO_F_VERSION_1;
+
+        if iommu {
+            avail_features |= 1u64 << VIRTIO_F_IOMMU_PLATFORM;
+        }
+
         Ok(Pmem {
             kill_evt: None,
+            pause_evt: None,
             disk: Some(disk),
-            avail_features: 1u64 << VIRTIO_F_VERSION_1,
+            avail_features,
             acked_features: 0u64,
             config,
+            queue_evts: None,
+            interrupt_cb: None,
+            epoll_thread: None,
+            paused: Arc::new(AtomicBool::new(false)),
         })
     }
 }
@@ -397,17 +435,42 @@ impl VirtioDevice for Pmem {
             return Err(ActivateError::BadActivate);
         }
 
-        let (self_kill_evt, kill_evt) =
+        let (self_kill_evt, kill_evt) = EventFd::new(EFD_NONBLOCK)
-            match EventFd::new(EFD_NONBLOCK).and_then(|e| Ok((e.try_clone()?, e))) {
+            .and_then(|e| Ok((e.try_clone()?, e)))
-                Ok(v) => v,
+            .map_err(|e| {
-                Err(e) => {
+                error!("failed creating kill EventFd pair: {}", e);
-                    error!("failed creating kill EventFd pair: {}", e);
+                ActivateError::BadActivate
-                    return Err(ActivateError::BadActivate);
+            })?;
-                }
-            };
         self.kill_evt = Some(self_kill_evt);
 
-        if let Some(disk) = self.disk.take() {
+        let (self_pause_evt, pause_evt) = EventFd::new(EFD_NONBLOCK)
+            .and_then(|e| Ok((e.try_clone()?, e)))
+            .map_err(|e| {
+                error!("failed creating pause EventFd pair: {}", e);
+                ActivateError::BadActivate
+            })?;
+        self.pause_evt = Some(self_pause_evt);
+
+        // Save the interrupt EventFD as we need to return it on reset
+        // but clone it to pass into the thread.
+        self.interrupt_cb = Some(interrupt_cb.clone());
+
+        let mut tmp_queue_evts: Vec<EventFd> = Vec::new();
+        for queue_evt in queue_evts.iter() {
+            // Save the queue EventFD as we need to return it on reset
+            // but clone it to pass into the thread.
+            tmp_queue_evts.push(queue_evt.try_clone().map_err(|e| {
+                error!("failed to clone queue EventFd: {}", e);
+                ActivateError::BadActivate
+            })?);
+        }
+        self.queue_evts = Some(tmp_queue_evts);
+
+        if let Some(disk) = self.disk.as_ref() {
+            let disk = disk.try_clone().map_err(|e| {
+                error!("failed cloning pmem disk: {}", e);
+                ActivateError::BadActivate
+            })?;
             let mut handler = PmemEpollHandler {
                 queue: queues.remove(0),
                 mem,
@@ -415,19 +478,43 @@ impl VirtioDevice for Pmem {
                 interrupt_cb,
                 queue_evt: queue_evts.remove(0),
                 kill_evt,
+                pause_evt,
             };
 
-            let worker_result = thread::Builder::new()
+            let paused = self.paused.clone();
+            thread::Builder::new()
                 .name("virtio_pmem".to_string())
-                .spawn(move || handler.run());
+                .spawn(move || handler.run(paused))
-
+                .map(|thread| self.epoll_thread = Some(thread))
-            if let Err(e) = worker_result {
+                .map_err(|e| {
-                error!("failed to spawn virtio_pmem worker: {}", e);
+                    error!("failed to clone virtio-pmem epoll thread: {}", e);
-                return Err(ActivateError::BadActivate);;
+                    ActivateError::BadActivate
-            }
+                })?;
 
             return Ok(());
         }
         Err(ActivateError::BadActivate)
     }
+
+    fn reset(&mut self) -> Option<(Arc<VirtioInterrupt>, Vec<EventFd>)> {
+        // We first must resume the virtio thread if it was paused.
+        if self.pause_evt.take().is_some() {
+            self.resume().ok()?;
+        }
+
+        if let Some(kill_evt) = self.kill_evt.take() {
+            // Ignore the result because there is nothing we can do about it.
+            let _ = kill_evt.write(1);
+        }
+
+        // Return the interrupt and queue EventFDs
+        Some((
+            self.interrupt_cb.take().unwrap(),
+            self.queue_evts.take().unwrap(),
+        ))
+    }
 }
+
+virtio_pausable!(Pmem);
+impl Snapshotable for Pmem {}
+impl Migratable for Pmem {}

vm-virtio/src/queue.rs

@@ -11,7 +11,9 @@
 use std::cmp::min;
 use std::num::Wrapping;
 use std::sync::atomic::{fence, Ordering};
+use std::sync::Arc;
 
+use crate::device::VirtioIommuRemapping;
 use vm_memory::{
     Address, ByteValued, Bytes, GuestAddress, GuestMemory, GuestMemoryMmap, GuestUsize,
 };
@@ -27,10 +29,41 @@ pub(super) const VIRTQ_DESC_F_WRITE: u16 = 0x2;
 // The Virtio Spec 1.0 defines the alignment of VirtIO descriptor is 16 bytes,
 // which fulfills the explicit constraint of GuestMemoryMmap::read_obj().
 
+/// An iterator over a single descriptor chain.  Not to be confused with AvailIter,
+/// which iterates over the descriptor chain heads in a queue.
+pub struct DescIter<'a> {
+    next: Option<DescriptorChain<'a>>,
+}
+
+impl<'a> DescIter<'a> {
+    /// Returns an iterator that only yields the readable descriptors in the chain.
+    pub fn readable(self) -> impl Iterator<Item = DescriptorChain<'a>> {
+        self.filter(|d| !d.is_write_only())
+    }
+
+    /// Returns an iterator that only yields the writable descriptors in the chain.
+    pub fn writable(self) -> impl Iterator<Item = DescriptorChain<'a>> {
+        self.filter(DescriptorChain::is_write_only)
+    }
+}
+
+impl<'a> Iterator for DescIter<'a> {
+    type Item = DescriptorChain<'a>;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        if let Some(current) = self.next.take() {
+            self.next = current.next_descriptor();
+            Some(current)
+        } else {
+            None
+        }
+    }
+}
+
 /// A virtio descriptor constraints with C representive.
 #[repr(C)]
 #[derive(Default, Clone, Copy)]
-struct Descriptor {
+pub struct Descriptor {
     addr: u64,
     len: u32,
     flags: u16,
@@ -40,11 +73,15 @@ struct Descriptor {
 unsafe impl ByteValued for Descriptor {}
 
 /// A virtio descriptor chain.
+#[derive(Clone)]
 pub struct DescriptorChain<'a> {
-    mem: &'a GuestMemoryMmap,
     desc_table: GuestAddress,
     queue_size: u16,
     ttl: u16, // used to prevent infinite chain cycles
+    iommu_mapping_cb: Option<Arc<VirtioIommuRemapping>>,
+
+    /// Reference to guest memory
+    pub mem: &'a GuestMemoryMmap,
 
     /// Index into the descriptor table
     pub index: u16,
@@ -64,11 +101,12 @@ pub struct DescriptorChain<'a> {
 }
 
 impl<'a> DescriptorChain<'a> {
-    fn checked_new(
+    pub fn checked_new(
         mem: &GuestMemoryMmap,
         desc_table: GuestAddress,
         queue_size: u16,
         index: u16,
+        iommu_mapping_cb: Option<Arc<VirtioIommuRemapping>>,
     ) -> Option<DescriptorChain> {
         if index >= queue_size {
             return None;
@@ -89,16 +127,25 @@ impl<'a> DescriptorChain<'a> {
                 return None;
             }
         };
+
+        // Translate address if necessary
+        let desc_addr = if let Some(iommu_mapping_cb) = &iommu_mapping_cb {
+            (iommu_mapping_cb)(desc.addr).unwrap()
+        } else {
+            desc.addr
+        };
+
         let chain = DescriptorChain {
             mem,
             desc_table,
             queue_size,
             ttl: queue_size,
             index,
-            addr: GuestAddress(desc.addr),
+            addr: GuestAddress(desc_addr),
             len: desc.len,
             flags: desc.flags,
             next: desc.next,
+            iommu_mapping_cb,
         };
 
         if chain.is_valid() {
@@ -135,18 +182,32 @@ impl<'a> DescriptorChain<'a> {
     /// the head of the next _available_ descriptor chain.
     pub fn next_descriptor(&self) -> Option<DescriptorChain<'a>> {
         if self.has_next() {
-            DescriptorChain::checked_new(self.mem, self.desc_table, self.queue_size, self.next).map(
+            DescriptorChain::checked_new(
-                |mut c| {
+                self.mem,
-                    c.ttl = self.ttl - 1;
+                self.desc_table,
-                    c
+                self.queue_size,
-                },
+                self.next,
+                self.iommu_mapping_cb.clone(),
             )
+            .map(|mut c| {
+                c.ttl = self.ttl - 1;
+                c
+            })
         } else {
             None
         }
     }
 }
 
+impl<'a> IntoIterator for DescriptorChain<'a> {
+    type Item = DescriptorChain<'a>;
+    type IntoIter = DescIter<'a>;
+
+    fn into_iter(self) -> Self::IntoIter {
+        DescIter { next: Some(self) }
+    }
+}
+
 /// Consuming iterator over all available descriptor chain heads in the queue.
 pub struct AvailIter<'a, 'b> {
     mem: &'a GuestMemoryMmap,
@@ -156,6 +217,7 @@ pub struct AvailIter<'a, 'b> {
     last_index: Wrapping<u16>,
     queue_size: u16,
     next_avail: &'b mut Wrapping<u16>,
+    iommu_mapping_cb: Option<Arc<VirtioIommuRemapping>>,
 }
 
 impl<'a, 'b> AvailIter<'a, 'b> {
@@ -168,6 +230,7 @@ impl<'a, 'b> AvailIter<'a, 'b> {
             last_index: Wrapping(0),
             queue_size: 0,
             next_avail: q_next_avail,
+            iommu_mapping_cb: None,
         }
     }
 }
@@ -197,8 +260,13 @@ impl<'a, 'b> Iterator for AvailIter<'a, 'b> {
 
         self.next_index += Wrapping(1);
 
-        let ret =
+        let ret = DescriptorChain::checked_new(
-            DescriptorChain::checked_new(self.mem, self.desc_table, self.queue_size, desc_index);
+            self.mem,
+            self.desc_table,
+            self.queue_size,
+            desc_index,
+            self.iommu_mapping_cb.clone(),
+        );
         if ret.is_some() {
             *self.next_avail += Wrapping(1);
         }
@@ -230,8 +298,10 @@ pub struct Queue {
     /// Guest physical address of the used ring
     pub used_ring: GuestAddress,
 
-    next_avail: Wrapping<u16>,
+    pub next_avail: Wrapping<u16>,
-    next_used: Wrapping<u16>,
+    pub next_used: Wrapping<u16>,
+
+    pub iommu_mapping_cb: Option<Arc<VirtioIommuRemapping>>,
 }
 
 impl Queue {
@@ -247,6 +317,7 @@ impl Queue {
             used_ring: GuestAddress(0),
             next_avail: Wrapping(0),
             next_used: Wrapping(0),
+            iommu_mapping_cb: None,
         }
     }
 
@@ -254,6 +325,26 @@ impl Queue {
         self.max_size
     }
 
+    pub fn enable(&mut self, set: bool) {
+        self.ready = set;
+
+        if set {
+            // Translate address of descriptor table and vrings.
+            if let Some(iommu_mapping_cb) = &self.iommu_mapping_cb {
+                self.desc_table =
+                    GuestAddress((iommu_mapping_cb)(self.desc_table.raw_value()).unwrap());
+                self.avail_ring =
+                    GuestAddress((iommu_mapping_cb)(self.avail_ring.raw_value()).unwrap());
+                self.used_ring =
+                    GuestAddress((iommu_mapping_cb)(self.used_ring.raw_value()).unwrap());
+            }
+        } else {
+            self.desc_table = GuestAddress(0);
+            self.avail_ring = GuestAddress(0);
+            self.used_ring = GuestAddress(0);
+        }
+    }
+
     /// Return the actual size of the queue, as the driver may not set up a
     /// queue as big as the device allows.
     pub fn actual_size(&self) -> u16 {
@@ -352,6 +443,7 @@ impl Queue {
             last_index: Wrapping(last_index),
             queue_size,
             next_avail: &mut self.next_avail,
+            iommu_mapping_cb: self.iommu_mapping_cb.clone(),
         }
     }
 
@@ -645,14 +737,16 @@ pub(crate) mod tests {
         assert!(vq.end().0 < 0x1000);
 
         // index >= queue_size
-        assert!(DescriptorChain::checked_new(m, vq.start(), 16, 16).is_none());
+        assert!(DescriptorChain::checked_new(m, vq.start(), 16, 16, None).is_none());
 
         // desc_table address is way off
-        assert!(DescriptorChain::checked_new(m, GuestAddress(0x00ff_ffff_ffff), 16, 0).is_none());
+        assert!(
+            DescriptorChain::checked_new(m, GuestAddress(0x00ff_ffff_ffff), 16, 0, None).is_none()
+        );
 
         // the addr field of the descriptor is way off
         vq.dtable[0].addr.set(0x0fff_ffff_ffff);
-        assert!(DescriptorChain::checked_new(m, vq.start(), 16, 0).is_none());
+        assert!(DescriptorChain::checked_new(m, vq.start(), 16, 0, None).is_none());
 
         // let's create some invalid chains
 
@@ -661,7 +755,7 @@ pub(crate) mod tests {
             vq.dtable[0].addr.set(0x1000);
             // ...but the length is too large
             vq.dtable[0].len.set(0xffff_ffff);
-            assert!(DescriptorChain::checked_new(m, vq.start(), 16, 0).is_none());
+            assert!(DescriptorChain::checked_new(m, vq.start(), 16, 0, None).is_none());
         }
 
         {
@@ -671,7 +765,7 @@ pub(crate) mod tests {
             //..but the the index of the next descriptor is too large
             vq.dtable[0].next.set(16);
 
-            assert!(DescriptorChain::checked_new(m, vq.start(), 16, 0).is_none());
+            assert!(DescriptorChain::checked_new(m, vq.start(), 16, 0, None).is_none());
         }
 
         // finally, let's test an ok chain
@@ -680,7 +774,7 @@ pub(crate) mod tests {
             vq.dtable[0].next.set(1);
             vq.dtable[1].set(0x2000, 0x1000, 0, 0);
 
-            let c = DescriptorChain::checked_new(m, vq.start(), 16, 0).unwrap();
+            let c = DescriptorChain::checked_new(m, vq.start(), 16, 0, None).unwrap();
 
             assert_eq!(c.mem as *const GuestMemoryMmap, m as *const GuestMemoryMmap);
             assert_eq!(c.desc_table, vq.start());

vm-virtio/src/rng.rs

@@ -9,15 +9,17 @@ use std::fs::File;
 use std::io;
 use std::os::unix::io::AsRawFd;
 use std::result;
+use std::sync::atomic::{AtomicBool, Ordering};
 use std::sync::{Arc, RwLock};
 use std::thread;
 
 use super::Error as DeviceError;
 use super::{
     ActivateError, ActivateResult, DeviceEventT, Queue, VirtioDevice, VirtioDeviceType,
-    VIRTIO_F_VERSION_1,
+    VIRTIO_F_IOMMU_PLATFORM, VIRTIO_F_VERSION_1,
 };
 use crate::{VirtioInterrupt, VirtioInterruptType};
+use vm_device::{Migratable, MigratableError, Pausable, Snapshotable};
 use vm_memory::{Bytes, GuestMemoryMmap};
 use vmm_sys_util::eventfd::EventFd;
 
@@ -29,6 +31,8 @@ const QUEUE_SIZES: &[u16] = &[QUEUE_SIZE];
 const QUEUE_AVAIL_EVENT: DeviceEventT = 0;
 // The device has been dropped.
 const KILL_EVENT: DeviceEventT = 1;
+// The device should be paused.
+const PAUSE_EVENT: DeviceEventT = 2;
 
 struct RngEpollHandler {
     queues: Vec<Queue>,
@@ -37,6 +41,7 @@ struct RngEpollHandler {
     interrupt_cb: Arc<VirtioInterrupt>,
     queue_evt: EventFd,
     kill_evt: EventFd,
+    pause_evt: EventFd,
 }
 
 impl RngEpollHandler {
@@ -81,7 +86,7 @@ impl RngEpollHandler {
         })
     }
 
-    fn run(&mut self) -> result::Result<(), DeviceError> {
+    fn run(&mut self, paused: Arc<AtomicBool>) -> result::Result<(), DeviceError> {
         // Create the epoll file descriptor
         let epoll_fd = epoll::create(true).map_err(DeviceError::EpollCreateFd)?;
 
@@ -100,6 +105,13 @@ impl RngEpollHandler {
             epoll::Event::new(epoll::Events::EPOLLIN, u64::from(KILL_EVENT)),
         )
         .map_err(DeviceError::EpollCtl)?;
+        epoll::ctl(
+            epoll_fd,
+            epoll::ControlOptions::EPOLL_CTL_ADD,
+            self.pause_evt.as_raw_fd(),
+            epoll::Event::new(epoll::Events::EPOLLIN, u64::from(PAUSE_EVENT)),
+        )
+        .map_err(DeviceError::EpollCtl)?;
 
         const EPOLL_EVENTS_LEN: usize = 100;
         let mut events = vec![epoll::Event::new(epoll::Events::empty(), 0); EPOLL_EVENTS_LEN];
@@ -141,6 +153,15 @@ impl RngEpollHandler {
                         debug!("KILL_EVENT received, stopping epoll loop");
                         break 'epoll;
                     }
+                    PAUSE_EVENT => {
+                        debug!("PAUSE_EVENT received, pausing virtio-rng epoll loop");
+                        // We loop here to handle spurious park() returns.
+                        // Until we have not resumed, the paused boolean will
+                        // be true.
+                        while paused.load(Ordering::SeqCst) {
+                            thread::park();
+                        }
+                    }
                     _ => {
                         error!("Unknown event for virtio-block");
                     }
@@ -155,22 +176,36 @@ impl RngEpollHandler {
 /// Virtio device for exposing entropy to the guest OS through virtio.
 pub struct Rng {
     kill_evt: Option<EventFd>,
+    pause_evt: Option<EventFd>,
     random_file: Option<File>,
     avail_features: u64,
     acked_features: u64,
+    queue_evts: Option<Vec<EventFd>>,
+    interrupt_cb: Option<Arc<VirtioInterrupt>>,
+    epoll_thread: Option<thread::JoinHandle<result::Result<(), DeviceError>>>,
+    paused: Arc<AtomicBool>,
 }
 
 impl Rng {
     /// Create a new virtio rng device that gets random data from /dev/urandom.
-    pub fn new(path: &str) -> io::Result<Rng> {
+    pub fn new(path: &str, iommu: bool) -> io::Result<Rng> {
         let random_file = File::open(path)?;
-        let avail_features = 1u64 << VIRTIO_F_VERSION_1;
+        let mut avail_features = 1u64 << VIRTIO_F_VERSION_1;
+
+        if iommu {
+            avail_features |= 1u64 << VIRTIO_F_IOMMU_PLATFORM;
+        }
 
         Ok(Rng {
             kill_evt: None,
+            pause_evt: None,
             random_file: Some(random_file),
             avail_features,
             acked_features: 0u64,
+            queue_evts: None,
+            interrupt_cb: None,
+            epoll_thread: None,
+            paused: Arc::new(AtomicBool::new(false)),
         })
     }
 }
@@ -251,17 +286,42 @@ impl VirtioDevice for Rng {
             return Err(ActivateError::BadActivate);
         }
 
-        let (self_kill_evt, kill_evt) =
+        let (self_kill_evt, kill_evt) = EventFd::new(EFD_NONBLOCK)
-            match EventFd::new(EFD_NONBLOCK).and_then(|e| Ok((e.try_clone()?, e))) {
+            .and_then(|e| Ok((e.try_clone()?, e)))
-                Ok(v) => v,
+            .map_err(|e| {
-                Err(e) => {
+                error!("failed creating kill EventFd pair: {}", e);
-                    error!("failed creating kill EventFd pair: {}", e);
+                ActivateError::BadActivate
-                    return Err(ActivateError::BadActivate);
+            })?;
-                }
-            };
         self.kill_evt = Some(self_kill_evt);
 
-        if let Some(random_file) = self.random_file.take() {
+        let (self_pause_evt, pause_evt) = EventFd::new(EFD_NONBLOCK)
+            .and_then(|e| Ok((e.try_clone()?, e)))
+            .map_err(|e| {
+                error!("failed creating pause EventFd pair: {}", e);
+                ActivateError::BadActivate
+            })?;
+        self.pause_evt = Some(self_pause_evt);
+
+        // Save the interrupt EventFD as we need to return it on reset
+        // but clone it to pass into the thread.
+        self.interrupt_cb = Some(interrupt_cb.clone());
+
+        let mut tmp_queue_evts: Vec<EventFd> = Vec::new();
+        for queue_evt in queue_evts.iter() {
+            // Save the queue EventFD as we need to return it on reset
+            // but clone it to pass into the thread.
+            tmp_queue_evts.push(queue_evt.try_clone().map_err(|e| {
+                error!("failed to clone queue EventFd: {}", e);
+                ActivateError::BadActivate
+            })?);
+        }
+        self.queue_evts = Some(tmp_queue_evts);
+
+        if let Some(file) = self.random_file.as_ref() {
+            let random_file = file.try_clone().map_err(|e| {
+                error!("failed cloning rng source: {}", e);
+                ActivateError::BadActivate
+            })?;
             let mut handler = RngEpollHandler {
                 queues,
                 mem,
@@ -269,19 +329,44 @@ impl VirtioDevice for Rng {
                 interrupt_cb,
                 queue_evt: queue_evts.remove(0),
                 kill_evt,
+                pause_evt,
             };
 
-            let worker_result = thread::Builder::new()
+            let paused = self.paused.clone();
+            thread::Builder::new()
                 .name("virtio_rng".to_string())
-                .spawn(move || handler.run());
+                .spawn(move || handler.run(paused))
-
+                .map(|thread| self.epoll_thread = Some(thread))
-            if let Err(e) = worker_result {
+                .map_err(|e| {
-                error!("failed to spawn virtio_rng worker: {}", e);
+                    error!("failed to clone the virtio-rng epoll thread: {}", e);
-                return Err(ActivateError::BadActivate);;
+                    ActivateError::BadActivate
-            }
+                })?;
 
             return Ok(());
         }
         Err(ActivateError::BadActivate)
     }
+
+    fn reset(&mut self) -> Option<(Arc<VirtioInterrupt>, Vec<EventFd>)> {
+        // We first must resume the virtio thread if it was paused.
+        if self.pause_evt.take().is_some() {
+            self.resume().ok()?;
+        }
+
+        // Then kill it.
+        if let Some(kill_evt) = self.kill_evt.take() {
+            // Ignore the result because there is nothing we can do about it.
+            let _ = kill_evt.write(1);
+        }
+
+        // Return the interrupt and queue EventFDs
+        Some((
+            self.interrupt_cb.take().unwrap(),
+            self.queue_evts.take().unwrap(),
+        ))
+    }
 }
+
+virtio_pausable!(Rng);
+impl Snapshotable for Rng {}
+impl Migratable for Rng {}

vm-virtio/src/transport/mmio.rs

@@ -0,0 +1,292 @@
+// Copyright 2017 The Chromium OS Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+use std::result;
+use std::sync::atomic::{AtomicUsize, Ordering};
+use std::sync::{Arc, Mutex, RwLock};
+
+use byteorder::{ByteOrder, LittleEndian};
+use libc::EFD_NONBLOCK;
+
+use crate::transport::{VirtioTransport, NOTIFY_REG_OFFSET};
+use crate::{
+    Queue, VirtioDevice, VirtioInterrupt, VirtioInterruptType, DEVICE_ACKNOWLEDGE, DEVICE_DRIVER,
+    DEVICE_DRIVER_OK, DEVICE_FAILED, DEVICE_FEATURES_OK, DEVICE_INIT,
+    INTERRUPT_STATUS_CONFIG_CHANGED, INTERRUPT_STATUS_USED_RING,
+};
+use devices::{BusDevice, Interrupt};
+use vm_device::{Migratable, MigratableError, Pausable, Snapshotable};
+use vm_memory::{GuestAddress, GuestMemoryMmap};
+use vmm_sys_util::{errno::Result, eventfd::EventFd};
+
+const VENDOR_ID: u32 = 0;
+
+const MMIO_MAGIC_VALUE: u32 = 0x7472_6976;
+const MMIO_VERSION: u32 = 2;
+
+/// Implements the
+/// [MMIO](http://docs.oasis-open.org/virtio/virtio/v1.0/cs04/virtio-v1.0-cs04.html#x1-1090002)
+/// transport for virtio devices.
+///
+/// This requires 3 points of installation to work with a VM:
+///
+/// 1. Mmio reads and writes must be sent to this device at what is referred to here as MMIO base.
+/// 1. `Mmio::queue_evts` must be installed at `virtio::NOTIFY_REG_OFFSET` offset from the MMIO
+/// base. Each event in the array must be signaled if the index is written at that offset.
+/// 1. `Mmio::interrupt_evt` must signal an interrupt that the guest driver is listening to when it
+/// is written to.
+///
+/// Typically one page (4096 bytes) of MMIO address space is sufficient to handle this transport
+/// and inner virtio device.
+pub struct MmioDevice {
+    device: Arc<Mutex<dyn VirtioDevice>>,
+    device_activated: bool,
+
+    features_select: u32,
+    acked_features_select: u32,
+    queue_select: u32,
+    interrupt_status: Arc<AtomicUsize>,
+    interrupt_cb: Option<Arc<VirtioInterrupt>>,
+    driver_status: u32,
+    config_generation: u32,
+    queues: Vec<Queue>,
+    queue_evts: Vec<EventFd>,
+    mem: Option<Arc<RwLock<GuestMemoryMmap>>>,
+}
+
+impl MmioDevice {
+    /// Constructs a new MMIO transport for the given virtio device.
+    pub fn new(
+        mem: Arc<RwLock<GuestMemoryMmap>>,
+        device: Arc<Mutex<dyn VirtioDevice>>,
+    ) -> Result<MmioDevice> {
+        let device_clone = device.clone();
+        let locked_device = device_clone.lock().unwrap();
+        let mut queue_evts = Vec::new();
+        for _ in locked_device.queue_max_sizes().iter() {
+            queue_evts.push(EventFd::new(EFD_NONBLOCK)?)
+        }
+        let queues = locked_device
+            .queue_max_sizes()
+            .iter()
+            .map(|&s| Queue::new(s))
+            .collect();
+        Ok(MmioDevice {
+            device,
+            device_activated: false,
+            features_select: 0,
+            acked_features_select: 0,
+            queue_select: 0,
+            interrupt_status: Arc::new(AtomicUsize::new(0)),
+            interrupt_cb: None,
+            driver_status: DEVICE_INIT,
+            config_generation: 0,
+            queues,
+            queue_evts,
+            mem: Some(mem),
+        })
+    }
+
+    /// Gets the list of queue events that must be triggered whenever the VM writes to
+    /// `virtio::NOTIFY_REG_OFFSET` past the MMIO base. Each event must be triggered when the
+    /// value being written equals the index of the event in this list.
+    fn queue_evts(&self) -> &[EventFd] {
+        self.queue_evts.as_slice()
+    }
+
+    fn is_driver_ready(&self) -> bool {
+        let ready_bits = DEVICE_ACKNOWLEDGE | DEVICE_DRIVER | DEVICE_DRIVER_OK | DEVICE_FEATURES_OK;
+        self.driver_status == ready_bits && self.driver_status & DEVICE_FAILED == 0
+    }
+
+    fn are_queues_valid(&self) -> bool {
+        if let Some(mem) = self.mem.as_ref() {
+            self.queues.iter().all(|q| q.is_valid(&mem.read().unwrap()))
+        } else {
+            false
+        }
+    }
+
+    fn with_queue<U, F>(&self, d: U, f: F) -> U
+    where
+        F: FnOnce(&Queue) -> U,
+    {
+        match self.queues.get(self.queue_select as usize) {
+            Some(queue) => f(queue),
+            None => d,
+        }
+    }
+
+    fn with_queue_mut<F: FnOnce(&mut Queue)>(&mut self, f: F) -> bool {
+        if let Some(queue) = self.queues.get_mut(self.queue_select as usize) {
+            f(queue);
+            true
+        } else {
+            false
+        }
+    }
+
+    pub fn assign_interrupt(&mut self, interrupt: Box<dyn Interrupt>) {
+        let interrupt_status = self.interrupt_status.clone();
+        let cb = Arc::new(Box::new(
+            move |int_type: &VirtioInterruptType, _queue: Option<&Queue>| {
+                let status = match int_type {
+                    VirtioInterruptType::Config => INTERRUPT_STATUS_CONFIG_CHANGED,
+                    VirtioInterruptType::Queue => INTERRUPT_STATUS_USED_RING,
+                };
+                interrupt_status.fetch_or(status as usize, Ordering::SeqCst);
+
+                interrupt.deliver()
+            },
+        ) as VirtioInterrupt);
+
+        self.interrupt_cb = Some(cb);
+    }
+}
+
+impl VirtioTransport for MmioDevice {
+    fn ioeventfds(&self, base_addr: u64) -> Vec<(&EventFd, u64)> {
+        let notify_base = base_addr + u64::from(NOTIFY_REG_OFFSET);
+        self.queue_evts()
+            .iter()
+            .map(|event| (event, notify_base))
+            .collect()
+    }
+}
+
+impl BusDevice for MmioDevice {
+    fn read(&mut self, _base: u64, offset: u64, data: &mut [u8]) {
+        match offset {
+            0x00..=0xff if data.len() == 4 => {
+                let v = match offset {
+                    0x0 => MMIO_MAGIC_VALUE,
+                    0x04 => MMIO_VERSION,
+                    0x08 => self.device.lock().unwrap().device_type(),
+                    0x0c => VENDOR_ID, // vendor id
+                    0x10 => {
+                        self.device.lock().unwrap().features(self.features_select)
+                            | if self.features_select == 1 { 0x1 } else { 0x0 }
+                    }
+                    0x34 => self.with_queue(0, |q| u32::from(q.get_max_size())),
+                    0x44 => self.with_queue(0, |q| q.ready as u32),
+                    0x60 => self.interrupt_status.load(Ordering::SeqCst) as u32,
+                    0x70 => self.driver_status,
+                    0xfc => self.config_generation,
+                    _ => {
+                        warn!("unknown virtio mmio register read: 0x{:x}", offset);
+                        return;
+                    }
+                };
+                LittleEndian::write_u32(data, v);
+            }
+            0x100..=0xfff => self
+                .device
+                .lock()
+                .unwrap()
+                .read_config(offset - 0x100, data),
+            _ => {
+                warn!(
+                    "invalid virtio mmio read: 0x{:x}:0x{:x}",
+                    offset,
+                    data.len()
+                );
+            }
+        };
+    }
+
+    fn write(&mut self, _base: u64, offset: u64, data: &[u8]) {
+        fn hi(v: &mut GuestAddress, x: u32) {
+            *v = (*v & 0xffff_ffff) | (u64::from(x) << 32)
+        }
+
+        fn lo(v: &mut GuestAddress, x: u32) {
+            *v = (*v & !0xffff_ffff) | u64::from(x)
+        }
+
+        let mut mut_q = false;
+        match offset {
+            0x00..=0xff if data.len() == 4 => {
+                let v = LittleEndian::read_u32(data);
+                match offset {
+                    0x14 => self.features_select = v,
+                    0x20 => self
+                        .device
+                        .lock()
+                        .unwrap()
+                        .ack_features(self.acked_features_select, v),
+                    0x24 => self.acked_features_select = v,
+                    0x30 => self.queue_select = v,
+                    0x38 => mut_q = self.with_queue_mut(|q| q.size = v as u16),
+                    0x44 => mut_q = self.with_queue_mut(|q| q.ready = v == 1),
+                    0x64 => {
+                        self.interrupt_status
+                            .fetch_and(!(v as usize), Ordering::SeqCst);
+                    }
+                    0x70 => self.driver_status = v,
+                    0x80 => mut_q = self.with_queue_mut(|q| lo(&mut q.desc_table, v)),
+                    0x84 => mut_q = self.with_queue_mut(|q| hi(&mut q.desc_table, v)),
+                    0x90 => mut_q = self.with_queue_mut(|q| lo(&mut q.avail_ring, v)),
+                    0x94 => mut_q = self.with_queue_mut(|q| hi(&mut q.avail_ring, v)),
+                    0xa0 => mut_q = self.with_queue_mut(|q| lo(&mut q.used_ring, v)),
+                    0xa4 => mut_q = self.with_queue_mut(|q| hi(&mut q.used_ring, v)),
+                    _ => {
+                        warn!("unknown virtio mmio register write: 0x{:x}", offset);
+                        return;
+                    }
+                }
+            }
+            0x100..=0xfff => {
+                return self
+                    .device
+                    .lock()
+                    .unwrap()
+                    .write_config(offset - 0x100, data)
+            }
+            _ => {
+                warn!(
+                    "invalid virtio mmio write: 0x{:x}:0x{:x}",
+                    offset,
+                    data.len()
+                );
+                return;
+            }
+        }
+
+        if self.device_activated && mut_q {
+            warn!("virtio queue was changed after device was activated");
+        }
+
+        if !self.device_activated && self.is_driver_ready() && self.are_queues_valid() {
+            if let Some(interrupt_cb) = self.interrupt_cb.take() {
+                if self.mem.is_some() {
+                    let mem = self.mem.as_ref().unwrap().clone();
+                    self.device
+                        .lock()
+                        .unwrap()
+                        .activate(
+                            mem,
+                            interrupt_cb,
+                            self.queues.clone(),
+                            self.queue_evts.split_off(0),
+                        )
+                        .expect("Failed to activate device");
+                    self.device_activated = true;
+                }
+            }
+        }
+    }
+}
+
+impl Pausable for MmioDevice {
+    fn pause(&mut self) -> result::Result<(), MigratableError> {
+        Ok(())
+    }
+
+    fn resume(&mut self) -> result::Result<(), MigratableError> {
+        Ok(())
+    }
+}
+
+impl Snapshotable for MmioDevice {}
+impl Migratable for MmioDevice {}

vm-virtio/src/transport/mod.rs

@@ -2,8 +2,23 @@
 //
 // SPDX-License-Identifier: Apache-2.0
 
+use vmm_sys_util::eventfd::EventFd;
+#[cfg(feature = "pci_support")]
 mod pci_common_config;
+#[cfg(feature = "pci_support")]
 mod pci_device;
-
+#[cfg(feature = "pci_support")]
 pub use pci_common_config::VirtioPciCommonConfig;
+#[cfg(feature = "pci_support")]
 pub use pci_device::VirtioPciDevice;
+
+#[cfg(feature = "mmio_support")]
+mod mmio;
+#[cfg(feature = "mmio_support")]
+pub use mmio::MmioDevice;
+#[cfg(feature = "mmio_support")]
+pub const NOTIFY_REG_OFFSET: u32 = 0x50;
+
+pub trait VirtioTransport {
+    fn ioeventfds(&self, base_addr: u64) -> Vec<(&EventFd, u64)>;
+}