build, release-notes: Document 0.8.0 release

Update the build version and the release notes for the 0.8.0 release.

Signed-off-by: Rob Bradford <robert.bradford@intel.com>

.github/workflows/build.yaml

@@ -1,4 +1,4 @@
-name: Cloud Hypervisor Actions
+name: Cloud Hypervisor Build
 on: [pull_request, create]
 
 jobs:
@@ -12,6 +12,9 @@ jobs:
           - stable
           - beta
           - nightly
+        target:
+          - x86_64-unknown-linux-gnu
+          - x86_64-unknown-linux-musl
     steps:
       - name: Code checkout
         uses: actions/checkout@v2
@@ -19,52 +22,8 @@ jobs:
         uses: actions-rs/toolchain@v1
         with:
             toolchain: ${{ matrix.rust }}
+            target: ${{ matrix.target }}
             override: true
 
       - name: Build
-        run: cargo build --release
-
-  release:
-    if: github.event_name == 'create' && github.event.ref_type == 'tag'
-    name: Release
-    runs-on: ubuntu-latest
-    steps:
-      - name: Code checkout
-        uses: actions/checkout@v2
-      - name: Install Rust toolchain (stable)
-        uses: actions-rs/toolchain@v1
-        with:
-            toolchain: stable
-            override: true
-      - name: Build
-        run: cargo build --release
-      - name: Create Release
-        id: create_release
-        uses: actions/create-release@v1
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        with:
-          tag_name: ${{ github.ref }}
-          release_name: ${{ github.ref }}
-          draft: true
-          prerelease: true
-      - name: Upload cloud-hypervisor
-        id: upload-release-cloud-hypervisor
-        uses: actions/upload-release-asset@v1
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        with:
-          upload_url: ${{ steps.create_release.outputs.upload_url }}
-          asset_path: target/release/cloud-hypervisor
-          asset_name: cloud-hypervisor
-          asset_content_type: application/octet-stream
-      - name: Upload ch-remote
-        id: upload-release-ch-remote
-        uses: actions/upload-release-asset@v1
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        with:
-          upload_url: ${{ steps.create_release.outputs.upload_url }}
-          asset_path: target/release/ch-remote
-          asset_name: ch-remote
-          asset_content_type: application/octet-stream
+        run: cargo build --release --target=${{ matrix.target }}

.github/workflows/cross-build.yaml

@@ -0,0 +1,34 @@
+name: Cloud Hypervisor Cross Build
+on: [pull_request, create]
+
+jobs:
+  build:
+    if: github.event_name == 'pull_request'
+    name: Build
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        rust:
+          - stable
+        target:
+          - aarch64-unknown-linux-gnu
+          - aarch64-unknown-linux-musl
+    steps:
+      - name: Code checkout
+        uses: actions/checkout@v2
+      - name: Install Rust toolchain (${{ matrix.rust }})
+        uses: actions-rs/toolchain@v1
+        with:
+            toolchain: ${{ matrix.rust }}
+            target: ${{ matrix.target }}
+            override: true
+      - name: Install arm64 libfdt
+        run: wget http://ftp.us.debian.org/debian/pool/main/d/device-tree-compiler/libfdt-dev_1.6.0-1_arm64.deb && dpkg-deb -xv libfdt-dev_1.6.0-1_arm64.deb ./tlibfdtdev && sudo mkdir /tmmmp && mkdir target && mkdir target/debug && mkdir target/debug/deps && sudo cp ./tlibfdtdev/usr/lib/aarch64-linux-gnu/libfdt.a target/debug/deps/libfdt.a && echo "libfdt installed"
+      - name: Disable "with-serde" in kvm-bindings
+        run: sed -i 's/"with-serde",\ //g' vmm/Cargo.toml
+      - name: Build
+        uses: actions-rs/cargo@v1
+        with:
+          use-cross: true
+          command: build
+          args: --target=${{ matrix.target }} --no-default-features --features "mmio"

.github/workflows/release.yaml

@@ -0,0 +1,67 @@
+name: Cloud Hypervisor Release
+on: [create]
+
+jobs:
+  release:
+    if: github.event_name == 'create' && github.event.ref_type == 'tag'
+    name: Release
+    runs-on: ubuntu-latest
+    steps:
+      - name: Code checkout
+        uses: actions/checkout@v2
+      - name: Install Rust toolchain (x86_64-unknown-linux-gnu)
+        uses: actions-rs/toolchain@v1
+        with:
+            toolchain: stable
+            target: x86_64-unknown-linux-gnu
+      - name: Install Rust toolchain (x86_64-unknown-linux-musl)
+        uses: actions-rs/toolchain@v1
+        with:
+            toolchain: stable
+            target: x86_64-unknown-linux-musl
+      - name: Build
+        run: cargo build --release --target=x86_64-unknown-linux-gnu
+      - name: Static Build
+        run: cargo build --release --target=x86_64-unknown-linux-musl
+      - name: Strip cloud-hypervisor binaries
+        run: strip target/*/release/cloud-hypervisor
+      - name: Create Release
+        id: create_release
+        uses: actions/create-release@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          tag_name: ${{ github.ref }}
+          release_name: ${{ github.ref }}
+          draft: true
+          prerelease: true
+      - name: Upload cloud-hypervisor
+        id: upload-release-cloud-hypervisor
+        uses: actions/upload-release-asset@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          upload_url: ${{ steps.create_release.outputs.upload_url }}
+          asset_path: target/x86_64-unknown-linux-gnu/release/cloud-hypervisor
+          asset_name: cloud-hypervisor
+          asset_content_type: application/octet-stream
+      - name: Upload static cloud-hypervisor
+        id: upload-release-static-cloud-hypervisor
+        uses: actions/upload-release-asset@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          upload_url: ${{ steps.create_release.outputs.upload_url }}
+          asset_path: target/x86_64-unknown-linux-musl/release/cloud-hypervisor
+          asset_name: cloud-hypervisor-static
+          asset_content_type: application/octet-stream
+      - name: Upload ch-remote
+        id: upload-release-ch-remote
+        uses: actions/upload-release-asset@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          upload_url: ${{ steps.create_release.outputs.upload_url }}
+          asset_path: target/x86_64-unknown-linux-gnu/release/ch-remote
+          asset_name: ch-remote
+          asset_content_type: application/octet-stream

.gitignore

@@ -1,3 +1,4 @@
+/build
 /target
 **/*.rs.bk
 **/Cargo.lock

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "cloud-hypervisor"
-version = "0.6.0"
+version = "0.8.0"
 authors = ["The Cloud Hypervisor Authors"]
 edition = "2018"
 default-run = "cloud-hypervisor"
@@ -8,31 +8,38 @@ build = "build.rs"
 
 [dependencies]
 arc-swap = ">=0.4.4"
-clap = { version = "2.33.0", features=["wrap_help"] }
+clap = { version = "2.33.1", features=["wrap_help"] }
 epoll = ">=4.0.1"
-futures = { version = "0.3.4", features = ["thread-pool"] }
+futures = { version = "0.3.5", features = ["thread-pool"] }
 lazy_static = "1.4.0"
-libc = "0.2.68"
+libc = "0.2.71"
 log = { version = "0.4.8", features = ["std"] }
+seccomp = { git = "https://github.com/firecracker-microvm/firecracker", tag = "v0.21.1" }
 serde_json = ">=1.0.9"
 vhost_user_backend = { path = "vhost_user_backend"}
 vhost_user_block = { path = "vhost_user_block"}
 vhost_user_fs = { path = "vhost_user_fs"}
 vhost_user_net = { path = "vhost_user_net"}
-virtio-bindings = "0.1.0"
+virtio-bindings = { version = "0.1", features = ["virtio-v5_0_0"]}
 vmm = { path = "vmm" }
 vm-device = { path = "vm-device" }
-vm-memory = "0.1.0"
-vmm-sys-util = "0.4.0"
+vm-memory = "0.2.1"
+vmm-sys-util = "0.6.1"
 vm-virtio = { path = "vm-virtio" }
 vhost_rs = { git = "https://github.com/cloud-hypervisor/vhost", branch = "dragonball", package = "vhost", features = ["vhost-user-slave"] }
 
+[patch.crates-io]
+vm-memory = { git = "https://github.com/cloud-hypervisor/vm-memory", branch = "ch" }
+
 [dev-dependencies]
-ssh2 = "0.8.0"
+ssh2 = "0.8.1"
 dirs = "2.0.2"
 credibility = "0.1.3"
 tempdir= "0.3.7"
 lazy_static= "1.4.0"
+tempfile = "3.1.0"
+serde_json = ">=1.0.9"
+net_util = { path = "net_util" }
 
 [features]
 default = ["acpi", "pci", "cmos"]
@@ -40,6 +47,7 @@ acpi = ["vmm/acpi"]
 pci = ["vmm/pci_support"]
 mmio = ["vmm/mmio_support"]
 cmos = ["vmm/cmos"]
+fwdebug = ["vmm/fwdebug"]
 
 # Integration tests require a special environment to run in
 integration_tests = []
@@ -53,11 +61,11 @@ members = [
     "vmm",
     "vm-virtio",
     "vm-device",
+    "vm-migration",
     "vhost_user_block",
     "vhost_user_backend",
     "vhost_user_fs",
     "vhost_user_net",
-    "vfio",
     "net_util",
     "acpi_tables",
     "arch_gen",

Jenkinsfile

@@ -67,6 +67,29 @@ pipeline{
 						}
 					}
 				}
+				stage ('Worker build (musl)') {
+					agent { node { label 'bionic' } }
+					options {
+						timeout(time: 1, unit: 'HOURS')
+					}
+					stages {
+						stage ('Checkout') {
+							steps {
+								checkout scm
+							}
+						}
+						stage ('Run unit tests for musl') {
+							steps {
+								sh "scripts/dev_cli.sh tests --unit --libc musl"
+							}
+						}
+						stage ('Run integration tests for musl') {
+							steps {
+								sh "scripts/dev_cli.sh tests --integration --libc musl"
+							}
+						}
+					}
+				}
 			}
 		}
 	}

README.md

@@ -229,10 +229,10 @@ $ ./cloud-hypervisor/target/release/cloud-hypervisor \
 
 `cloud-hypervisor` is in a very early, pre-alpha stage. Use at your own risk!
 
-As of 2019-12-12, the following cloud images are supported:
+As of 2020-04-23, the following cloud images are supported:
 * [Clear Linux](https://download.clearlinux.org/current/) (cloudguest and kvm)
 * [Ubuntu Bionic](https://cloud-images.ubuntu.com/bionic/current/) (cloudimg)
-* [Ubuntu Eoan](https://cloud-images.ubuntu.com/eoan/current/) (cloudimg)
+* [Ubuntu Focal](https://cloud-images.ubuntu.com/focal/current/) (cloudimg)
 
 Direct kernel boot to userspace should work with most rootfs.
 

acpi_tables/Cargo.toml

@@ -5,5 +5,5 @@ authors = ["The Cloud Hypervisor Authors"]
 edition = "2018"
 
 [dependencies]
-vm-memory = "0.1.0"
+vm-memory = "0.2.1"
 

arch/Cargo.toml

@@ -5,13 +5,14 @@ authors = ["The Chromium OS Authors"]
 
 [features]
 default = []
+acpi = ["acpi_tables"]
 
 [dependencies]
 byteorder = "1.3.4"
-kvm-bindings = "0.2.0"
-kvm-ioctls = "0.5.0"
-libc = "0.2.68"
-vm-memory = { version = "0.1.0", features = ["backend-mmap"] }
+kvm-bindings = { git = "https://github.com/cloud-hypervisor/kvm-bindings", branch = "ch" }
+kvm-ioctls = { git = "https://github.com/cloud-hypervisor/kvm-ioctls", branch = "ch" }
+libc = "0.2.71"
+vm-memory = { version = "0.2.1", features = ["backend-mmap"] }
 
 acpi_tables = { path = "../acpi_tables", optional = true }
 arch_gen = { path = "../arch_gen" }

arch/src/aarch64/fdt.rs

@@ -0,0 +1,610 @@
+// Copyright 2020 Arm Limited (or its affiliates). All rights reserved.
+// Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+//
+// Portions Copyright 2017 The Chromium OS Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the THIRD-PARTY file.
+
+use libc::{c_char, c_int, c_void};
+use std::collections::HashMap;
+use std::ffi::{CStr, CString, NulError};
+use std::fmt::Debug;
+use std::ptr::null;
+use std::{io, result};
+
+use super::super::DeviceType;
+use super::super::InitramfsConfig;
+use super::get_fdt_addr;
+use super::gic::GICDevice;
+use super::layout::FDT_MAX_SIZE;
+use crate::aarch64::fdt::Error::CstringFDTTransform;
+use vm_memory::{Address, Bytes, GuestAddress, GuestMemory, GuestMemoryError, GuestMemoryMmap};
+
+// This is a value for uniquely identifying the FDT node declaring the interrupt controller.
+const GIC_PHANDLE: u32 = 1;
+// This is a value for uniquely identifying the FDT node containing the clock definition.
+const CLOCK_PHANDLE: u32 = 2;
+// Read the documentation specified when appending the root node to the FDT.
+const ADDRESS_CELLS: u32 = 0x2;
+const SIZE_CELLS: u32 = 0x2;
+
+// As per kvm tool and
+// https://www.kernel.org/doc/Documentation/devicetree/bindings/interrupt-controller/arm%2Cgic.txt
+// Look for "The 1st cell..."
+const GIC_FDT_IRQ_TYPE_SPI: u32 = 0;
+const GIC_FDT_IRQ_TYPE_PPI: u32 = 1;
+
+// From https://elixir.bootlin.com/linux/v4.9.62/source/include/dt-bindings/interrupt-controller/irq.h#L17
+const IRQ_TYPE_EDGE_RISING: u32 = 1;
+const IRQ_TYPE_LEVEL_HI: u32 = 4;
+
+// This links to libfdt which handles the creation of the binary blob
+// flattened device tree (fdt) that is passed to the kernel and indicates
+// the hardware configuration of the machine.
+#[link(name = "fdt")]
+extern "C" {
+    fn fdt_create(buf: *mut c_void, bufsize: c_int) -> c_int;
+    fn fdt_finish_reservemap(fdt: *mut c_void) -> c_int;
+    fn fdt_begin_node(fdt: *mut c_void, name: *const c_char) -> c_int;
+    fn fdt_property(fdt: *mut c_void, name: *const c_char, val: *const c_void, len: c_int)
+        -> c_int;
+    fn fdt_end_node(fdt: *mut c_void) -> c_int;
+    fn fdt_open_into(fdt: *const c_void, buf: *mut c_void, bufsize: c_int) -> c_int;
+    fn fdt_finish(fdt: *const c_void) -> c_int;
+    fn fdt_pack(fdt: *mut c_void) -> c_int;
+}
+
+/// Trait for devices to be added to the Flattened Device Tree.
+pub trait DeviceInfoForFDT {
+    /// Returns the address where this device will be loaded.
+    fn addr(&self) -> u64;
+    /// Returns the associated interrupt for this device.
+    fn irq(&self) -> u32;
+    /// Returns the amount of memory that needs to be reserved for this device.
+    fn length(&self) -> u64;
+}
+
+/// Errors thrown while configuring the Flattened Device Tree for aarch64.
+#[derive(Debug)]
+pub enum Error {
+    /// Failed to append node to the FDT.
+    AppendFDTNode(io::Error),
+    /// Failed to append a property to the FDT.
+    AppendFDTProperty(io::Error),
+    /// Syscall for creating FDT failed.
+    CreateFDT(io::Error),
+    /// Failed to obtain a C style string.
+    CstringFDTTransform(NulError),
+    /// Failure in calling syscall for terminating this FDT.
+    FinishFDTReserveMap(io::Error),
+    /// Failure in writing FDT in memory.
+    WriteFDTToMemory(GuestMemoryError),
+}
+type Result<T> = result::Result<T, Error>;
+
+/// Creates the flattened device tree for this aarch64 VM.
+pub fn create_fdt<T: DeviceInfoForFDT + Clone + Debug>(
+    guest_mem: &GuestMemoryMmap,
+    cmdline: &CStr,
+    vcpu_mpidr: Vec<u64>,
+    device_info: &HashMap<(DeviceType, String), T>,
+    gic_device: &Box<dyn GICDevice>,
+    initrd: &Option<InitramfsConfig>,
+) -> Result<Vec<u8>> {
+    // Alocate stuff necessary for the holding the blob.
+    let mut fdt = vec![0; FDT_MAX_SIZE];
+
+    allocate_fdt(&mut fdt)?;
+
+    // For an explanation why these nodes were introduced in the blob take a look at
+    // https://github.com/torvalds/linux/blob/master/Documentation/devicetree/booting-without-of.txt#L845
+    // Look for "Required nodes and properties".
+
+    // Header or the root node as per above mentioned documentation.
+    append_begin_node(&mut fdt, "")?;
+    append_property_string(&mut fdt, "compatible", "linux,dummy-virt")?;
+    // For info on #address-cells and size-cells read "Note about cells and address representation"
+    // from the above mentioned txt file.
+    append_property_u32(&mut fdt, "#address-cells", ADDRESS_CELLS)?;
+    append_property_u32(&mut fdt, "#size-cells", SIZE_CELLS)?;
+    // This is not mandatory but we use it to point the root node to the node
+    // containing description of the interrupt controller for this VM.
+    append_property_u32(&mut fdt, "interrupt-parent", GIC_PHANDLE)?;
+    create_cpu_nodes(&mut fdt, &vcpu_mpidr)?;
+    create_memory_node(&mut fdt, guest_mem)?;
+    create_chosen_node(&mut fdt, cmdline, initrd)?;
+    create_gic_node(&mut fdt, gic_device)?;
+    create_timer_node(&mut fdt)?;
+    create_clock_node(&mut fdt)?;
+    create_psci_node(&mut fdt)?;
+    create_devices_node(&mut fdt, device_info)?;
+
+    // End Header node.
+    append_end_node(&mut fdt)?;
+
+    // Allocate another buffer so we can format and then write fdt to guest.
+    let mut fdt_final = vec![0; FDT_MAX_SIZE];
+    finish_fdt(&mut fdt, &mut fdt_final)?;
+
+    // Write FDT to memory.
+    let fdt_address = GuestAddress(get_fdt_addr(&guest_mem));
+    guest_mem
+        .write_slice(fdt_final.as_slice(), fdt_address)
+        .map_err(Error::WriteFDTToMemory)?;
+    Ok(fdt_final)
+}
+
+// Following are auxiliary functions for allocating and finishing the FDT.
+fn allocate_fdt(fdt: &mut Vec<u8>) -> Result<()> {
+    // Safe since we allocated this array with FDT_MAX_SIZE.
+    let mut fdt_ret = unsafe { fdt_create(fdt.as_mut_ptr() as *mut c_void, FDT_MAX_SIZE as c_int) };
+
+    if fdt_ret != 0 {
+        return Err(Error::CreateFDT(io::Error::last_os_error()));
+    }
+
+    // The flattened device trees created with fdt_create() contains a list of
+    // reserved memory areas. We need to call `fdt_finish_reservemap` so as to make sure that there is a
+    // terminator in the reservemap list and whatever happened to be at the
+    // start of the FDT data section would end up being interpreted as
+    // reservemap entries.
+    // Safe since we previously allocated this array.
+    fdt_ret = unsafe { fdt_finish_reservemap(fdt.as_mut_ptr() as *mut c_void) };
+    if fdt_ret != 0 {
+        return Err(Error::FinishFDTReserveMap(io::Error::last_os_error()));
+    }
+    Ok(())
+}
+
+fn finish_fdt(from_fdt: &mut Vec<u8>, to_fdt: &mut Vec<u8>) -> Result<()> {
+    // Safe since we allocated `fdt_final` and previously passed in its size.
+    let mut fdt_ret = unsafe { fdt_finish(from_fdt.as_mut_ptr() as *mut c_void) };
+    if fdt_ret != 0 {
+        return Err(Error::FinishFDTReserveMap(io::Error::last_os_error()));
+    }
+
+    // Safe because we allocated both arrays with the correct size.
+    fdt_ret = unsafe {
+        fdt_open_into(
+            from_fdt.as_mut_ptr() as *mut c_void,
+            to_fdt.as_mut_ptr() as *mut c_void,
+            FDT_MAX_SIZE as i32,
+        )
+    };
+    if fdt_ret != 0 {
+        return Err(Error::FinishFDTReserveMap(io::Error::last_os_error()));
+    }
+
+    // Safe since we allocated `to_fdt`.
+    fdt_ret = unsafe { fdt_pack(to_fdt.as_mut_ptr() as *mut c_void) };
+    if fdt_ret != 0 {
+        return Err(Error::FinishFDTReserveMap(io::Error::last_os_error()));
+    }
+    Ok(())
+}
+
+// Following are auxiliary functions for appending nodes to FDT.
+fn append_begin_node(fdt: &mut Vec<u8>, name: &str) -> Result<()> {
+    let cstr_name = CString::new(name).map_err(CstringFDTTransform)?;
+
+    // Safe because we allocated fdt and converted name to a CString
+    let fdt_ret = unsafe { fdt_begin_node(fdt.as_mut_ptr() as *mut c_void, cstr_name.as_ptr()) };
+    if fdt_ret != 0 {
+        return Err(Error::AppendFDTNode(io::Error::last_os_error()));
+    }
+    Ok(())
+}
+
+fn append_end_node(fdt: &mut Vec<u8>) -> Result<()> {
+    // Safe because we allocated fdt.
+    let fdt_ret = unsafe { fdt_end_node(fdt.as_mut_ptr() as *mut c_void) };
+    if fdt_ret != 0 {
+        return Err(Error::AppendFDTNode(io::Error::last_os_error()));
+    }
+    Ok(())
+}
+
+// Following are auxiliary functions for appending property nodes to the nodes of the FDT.
+fn append_property_u32(fdt: &mut Vec<u8>, name: &str, val: u32) -> Result<()> {
+    append_property(fdt, name, &to_be32(val))
+}
+
+fn append_property_u64(fdt: &mut Vec<u8>, name: &str, val: u64) -> Result<()> {
+    append_property(fdt, name, &to_be64(val))
+}
+
+fn append_property_string(fdt: &mut Vec<u8>, name: &str, value: &str) -> Result<()> {
+    let cstr_value = CString::new(value).map_err(CstringFDTTransform)?;
+    append_property_cstring(fdt, name, &cstr_value)
+}
+
+fn append_property_cstring(fdt: &mut Vec<u8>, name: &str, cstr_value: &CStr) -> Result<()> {
+    let value_bytes = cstr_value.to_bytes_with_nul();
+    let cstr_name = CString::new(name).map_err(CstringFDTTransform)?;
+    // Safe because we allocated fdt, converted name and value to CStrings
+    let fdt_ret = unsafe {
+        fdt_property(
+            fdt.as_mut_ptr() as *mut c_void,
+            cstr_name.as_ptr(),
+            value_bytes.as_ptr() as *mut c_void,
+            value_bytes.len() as i32,
+        )
+    };
+    if fdt_ret != 0 {
+        return Err(Error::AppendFDTProperty(io::Error::last_os_error()));
+    }
+    Ok(())
+}
+
+fn append_property_null(fdt: &mut Vec<u8>, name: &str) -> Result<()> {
+    let cstr_name = CString::new(name).map_err(CstringFDTTransform)?;
+
+    // Safe because we allocated fdt, converted name to a CString
+    let fdt_ret = unsafe {
+        fdt_property(
+            fdt.as_mut_ptr() as *mut c_void,
+            cstr_name.as_ptr(),
+            null(),
+            0,
+        )
+    };
+    if fdt_ret != 0 {
+        return Err(Error::AppendFDTProperty(io::Error::last_os_error()));
+    }
+    Ok(())
+}
+
+fn append_property(fdt: &mut Vec<u8>, name: &str, val: &[u8]) -> Result<()> {
+    let cstr_name = CString::new(name).map_err(CstringFDTTransform)?;
+    let val_ptr = val.as_ptr() as *const c_void;
+
+    // Safe because we allocated fdt and converted name to a CString
+    let fdt_ret = unsafe {
+        fdt_property(
+            fdt.as_mut_ptr() as *mut c_void,
+            cstr_name.as_ptr(),
+            val_ptr,
+            val.len() as i32,
+        )
+    };
+    if fdt_ret != 0 {
+        return Err(Error::AppendFDTProperty(io::Error::last_os_error()));
+    }
+    Ok(())
+}
+
+// Auxiliary functions for writing u32/u64 numbers in big endian order.
+fn to_be32(input: u32) -> [u8; 4] {
+    u32::to_be_bytes(input)
+}
+
+fn to_be64(input: u64) -> [u8; 8] {
+    u64::to_be_bytes(input)
+}
+
+// Helper functions for generating a properly formatted byte vector using 32-bit/64-bit cells.
+fn generate_prop32(cells: &[u32]) -> Vec<u8> {
+    let mut ret: Vec<u8> = Vec::new();
+    for &e in cells {
+        ret.extend(to_be32(e).iter());
+    }
+    ret
+}
+
+fn generate_prop64(cells: &[u64]) -> Vec<u8> {
+    let mut ret: Vec<u8> = Vec::new();
+    for &e in cells {
+        ret.extend(to_be64(e).iter());
+    }
+    ret
+}
+
+// Following are the auxiliary function for creating the different nodes that we append to our FDT.
+fn create_cpu_nodes(fdt: &mut Vec<u8>, vcpu_mpidr: &Vec<u64>) -> Result<()> {
+    // See https://github.com/torvalds/linux/blob/master/Documentation/devicetree/bindings/arm/cpus.yaml.
+    append_begin_node(fdt, "cpus")?;
+    // As per documentation, on ARM v8 64-bit systems value should be set to 2.
+    append_property_u32(fdt, "#address-cells", 0x02)?;
+    append_property_u32(fdt, "#size-cells", 0x0)?;
+    let num_cpus = vcpu_mpidr.len();
+
+    for cpu_index in 0..num_cpus {
+        let cpu_name = format!("cpu@{:x}", cpu_index);
+        append_begin_node(fdt, &cpu_name)?;
+        append_property_string(fdt, "device_type", "cpu")?;
+        append_property_string(fdt, "compatible", "arm,arm-v8")?;
+        if num_cpus > 1 {
+            // This is required on armv8 64-bit. See aforementioned documentation.
+            append_property_string(fdt, "enable-method", "psci")?;
+        }
+        // Set the field to first 24 bits of the MPIDR - Multiprocessor Affinity Register.
+        // See http://infocenter.arm.com/help/index.jsp?topic=/com.arm.doc.ddi0488c/BABHBJCI.html.
+        append_property_u64(fdt, "reg", vcpu_mpidr[cpu_index] & 0x7FFFFF)?;
+        append_end_node(fdt)?;
+    }
+    append_end_node(fdt)?;
+    Ok(())
+}
+
+fn create_memory_node(fdt: &mut Vec<u8>, guest_mem: &GuestMemoryMmap) -> Result<()> {
+    let mem_size = guest_mem.last_addr().raw_value() - super::layout::RAM_64BIT_START + 1;
+    // See https://github.com/torvalds/linux/blob/master/Documentation/devicetree/booting-without-of.txt#L960
+    // for an explanation of this.
+    let mem_reg_prop = generate_prop64(&[super::layout::RAM_64BIT_START as u64, mem_size as u64]);
+
+    append_begin_node(fdt, "memory")?;
+    append_property_string(fdt, "device_type", "memory")?;
+    append_property(fdt, "reg", &mem_reg_prop)?;
+    append_end_node(fdt)?;
+    Ok(())
+}
+
+fn create_chosen_node(
+    fdt: &mut Vec<u8>,
+    cmdline: &CStr,
+    initrd: &Option<InitramfsConfig>,
+) -> Result<()> {
+    append_begin_node(fdt, "chosen")?;
+    append_property_cstring(fdt, "bootargs", cmdline)?;
+
+    if let Some(initrd_config) = initrd {
+        append_property_u64(
+            fdt,
+            "linux,initrd-start",
+            initrd_config.address.raw_value() as u64,
+        )?;
+        append_property_u64(
+            fdt,
+            "linux,initrd-end",
+            initrd_config.address.raw_value() + initrd_config.size as u64,
+        )?;
+    }
+
+    append_end_node(fdt)?;
+
+    Ok(())
+}
+
+fn create_gic_node(fdt: &mut Vec<u8>, gic_device: &Box<dyn GICDevice>) -> Result<()> {
+    let gic_reg_prop = generate_prop64(gic_device.device_properties());
+
+    append_begin_node(fdt, "intc")?;
+    append_property_string(fdt, "compatible", gic_device.fdt_compatibility())?;
+    append_property_null(fdt, "interrupt-controller")?;
+    // "interrupt-cells" field specifies the number of cells needed to encode an
+    // interrupt source. The type shall be a <u32> and the value shall be 3 if no PPI affinity description
+    // is required.
+    append_property_u32(fdt, "#interrupt-cells", 3)?;
+    append_property(fdt, "reg", &gic_reg_prop)?;
+    append_property_u32(fdt, "phandle", GIC_PHANDLE)?;
+    append_property_u32(fdt, "#address-cells", 2)?;
+    append_property_u32(fdt, "#size-cells", 2)?;
+    append_property_null(fdt, "ranges")?;
+    let gic_intr = [
+        GIC_FDT_IRQ_TYPE_PPI,
+        gic_device.fdt_maint_irq(),
+        IRQ_TYPE_LEVEL_HI,
+    ];
+    let gic_intr_prop = generate_prop32(&gic_intr);
+
+    append_property(fdt, "interrupts", &gic_intr_prop)?;
+    append_end_node(fdt)?;
+
+    Ok(())
+}
+
+fn create_clock_node(fdt: &mut Vec<u8>) -> Result<()> {
+    // The Advanced Peripheral Bus (APB) is part of the Advanced Microcontroller Bus Architecture
+    // (AMBA) protocol family. It defines a low-cost interface that is optimized for minimal power
+    // consumption and reduced interface complexity.
+    // PCLK is the clock source and this node defines exactly the clock for the APB.
+    append_begin_node(fdt, "apb-pclk")?;
+    append_property_string(fdt, "compatible", "fixed-clock")?;
+    append_property_u32(fdt, "#clock-cells", 0x0)?;
+    append_property_u32(fdt, "clock-frequency", 24000000)?;
+    append_property_string(fdt, "clock-output-names", "clk24mhz")?;
+    append_property_u32(fdt, "phandle", CLOCK_PHANDLE)?;
+    append_end_node(fdt)?;
+
+    Ok(())
+}
+
+fn create_timer_node(fdt: &mut Vec<u8>) -> Result<()> {
+    // See
+    // https://github.com/torvalds/linux/blob/master/Documentation/devicetree/bindings/interrupt-controller/arch_timer.txt
+    // These are fixed interrupt numbers for the timer device.
+    let irqs = [13, 14, 11, 10];
+    let compatible = "arm,armv8-timer";
+
+    let mut timer_reg_cells: Vec<u32> = Vec::new();
+    for &irq in irqs.iter() {
+        timer_reg_cells.push(GIC_FDT_IRQ_TYPE_PPI);
+        timer_reg_cells.push(irq);
+        timer_reg_cells.push(IRQ_TYPE_LEVEL_HI);
+    }
+    let timer_reg_prop = generate_prop32(timer_reg_cells.as_slice());
+
+    append_begin_node(fdt, "timer")?;
+    append_property_string(fdt, "compatible", compatible)?;
+    append_property_null(fdt, "always-on")?;
+    append_property(fdt, "interrupts", &timer_reg_prop)?;
+    append_end_node(fdt)?;
+
+    Ok(())
+}
+
+fn create_psci_node(fdt: &mut Vec<u8>) -> Result<()> {
+    let compatible = "arm,psci-0.2";
+    append_begin_node(fdt, "psci")?;
+    append_property_string(fdt, "compatible", compatible)?;
+    // Two methods available: hvc and smc.
+    // As per documentation, PSCI calls between a guest and hypervisor may use the HVC conduit instead of SMC.
+    // So, since we are using kvm, we need to use hvc.
+    append_property_string(fdt, "method", "hvc")?;
+    append_end_node(fdt)?;
+
+    Ok(())
+}
+
+fn create_virtio_node<T: DeviceInfoForFDT + Clone + Debug>(
+    fdt: &mut Vec<u8>,
+    dev_info: &T,
+) -> Result<()> {
+    let device_reg_prop = generate_prop64(&[dev_info.addr(), dev_info.length()]);
+    let irq = generate_prop32(&[GIC_FDT_IRQ_TYPE_SPI, dev_info.irq(), IRQ_TYPE_EDGE_RISING]);
+
+    append_begin_node(fdt, &format!("virtio_mmio@{:x}", dev_info.addr()))?;
+    append_property_string(fdt, "compatible", "virtio,mmio")?;
+    append_property(fdt, "reg", &device_reg_prop)?;
+    append_property(fdt, "interrupts", &irq)?;
+    append_property_u32(fdt, "interrupt-parent", GIC_PHANDLE)?;
+    append_end_node(fdt)?;
+
+    Ok(())
+}
+
+fn create_serial_node<T: DeviceInfoForFDT + Clone + Debug>(
+    fdt: &mut Vec<u8>,
+    dev_info: &T,
+) -> Result<()> {
+    let serial_reg_prop = generate_prop64(&[dev_info.addr(), dev_info.length()]);
+    let irq = generate_prop32(&[GIC_FDT_IRQ_TYPE_SPI, dev_info.irq(), IRQ_TYPE_EDGE_RISING]);
+
+    append_begin_node(fdt, &format!("uart@{:x}", dev_info.addr()))?;
+    append_property_string(fdt, "compatible", "ns16550a")?;
+    append_property(fdt, "reg", &serial_reg_prop)?;
+    append_property_u32(fdt, "clocks", CLOCK_PHANDLE)?;
+    append_property_string(fdt, "clock-names", "apb_pclk")?;
+    append_property(fdt, "interrupts", &irq)?;
+    append_end_node(fdt)?;
+
+    Ok(())
+}
+
+fn create_rtc_node<T: DeviceInfoForFDT + Clone + Debug>(
+    fdt: &mut Vec<u8>,
+    dev_info: &T,
+) -> Result<()> {
+    let compatible = b"arm,pl031\0arm,primecell\0";
+    let rtc_reg_prop = generate_prop64(&[dev_info.addr(), dev_info.length()]);
+    let irq = generate_prop32(&[GIC_FDT_IRQ_TYPE_SPI, dev_info.irq(), IRQ_TYPE_LEVEL_HI]);
+    append_begin_node(fdt, &format!("rtc@{:x}", dev_info.addr()))?;
+    append_property(fdt, "compatible", compatible)?;
+    append_property(fdt, "reg", &rtc_reg_prop)?;
+    append_property(fdt, "interrupts", &irq)?;
+    append_property_u32(fdt, "clocks", CLOCK_PHANDLE)?;
+    append_property_string(fdt, "clock-names", "apb_pclk")?;
+    append_end_node(fdt)?;
+
+    Ok(())
+}
+
+fn create_devices_node<T: DeviceInfoForFDT + Clone + Debug>(
+    fdt: &mut Vec<u8>,
+    dev_info: &HashMap<(DeviceType, String), T>,
+) -> Result<()> {
+    // Create one temp Vec to store all virtio devices
+    let mut ordered_virtio_device: Vec<&T> = Vec::new();
+
+    for ((device_type, _device_id), info) in dev_info {
+        match device_type {
+            DeviceType::RTC => create_rtc_node(fdt, info)?,
+            DeviceType::Serial => create_serial_node(fdt, info)?,
+            DeviceType::Virtio(_) => {
+                ordered_virtio_device.push(info);
+            }
+        }
+    }
+
+    // Sort out virtio devices by address from low to high and insert them into fdt table.
+    ordered_virtio_device.sort_by(|a, b| a.addr().cmp(&b.addr()));
+    for ordered_device_info in ordered_virtio_device.drain(..) {
+        create_virtio_node(fdt, ordered_device_info)?;
+    }
+
+    Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::aarch64::gic::create_gic;
+    use crate::aarch64::layout;
+    use kvm_ioctls::Kvm;
+
+    const LEN: u64 = 4096;
+
+    #[derive(Clone, Debug)]
+    pub struct MMIODeviceInfo {
+        addr: u64,
+        irq: u32,
+    }
+
+    impl DeviceInfoForFDT for MMIODeviceInfo {
+        fn addr(&self) -> u64 {
+            self.addr
+        }
+        fn irq(&self) -> u32 {
+            self.irq
+        }
+        fn length(&self) -> u64 {
+            LEN
+        }
+    }
+
+    // The `load` function from the `device_tree` will mistakenly check the actual size
+    // of the buffer with the allocated size. This works around that.
+    fn set_size(buf: &mut [u8], pos: usize, val: usize) {
+        buf[pos] = ((val >> 24) & 0xff) as u8;
+        buf[pos + 1] = ((val >> 16) & 0xff) as u8;
+        buf[pos + 2] = ((val >> 8) & 0xff) as u8;
+        buf[pos + 3] = (val & 0xff) as u8;
+    }
+
+    #[test]
+    fn test_create_fdt_with_devices() {
+        let mut regions = Vec::new();
+        regions.push((
+            GuestAddress(layout::RAM_64BIT_START),
+            (layout::FDT_MAX_SIZE + 0x1000) as usize,
+        ));
+        let mem = GuestMemoryMmap::from_ranges(&regions).expect("Cannot initialize memory");
+
+        let dev_info: HashMap<(DeviceType, std::string::String), MMIODeviceInfo> = [
+            (
+                (DeviceType::Serial, DeviceType::Serial.to_string()),
+                MMIODeviceInfo { addr: 0x00, irq: 1 },
+            ),
+            (
+                (DeviceType::Virtio(1), "virtio".to_string()),
+                MMIODeviceInfo {
+                    addr: 0x00 + LEN,
+                    irq: 2,
+                },
+            ),
+            (
+                (DeviceType::RTC, "rtc".to_string()),
+                MMIODeviceInfo {
+                    addr: 0x00 + 2 * LEN,
+                    irq: 3,
+                },
+            ),
+        ]
+        .iter()
+        .cloned()
+        .collect();
+        let kvm = Kvm::new().unwrap();
+        let vm = kvm.create_vm().unwrap();
+        let gic = create_gic(&vm, 1).unwrap();
+        assert!(create_fdt(
+            &mem,
+            &CString::new("console=tty0").unwrap(),
+            vec![0],
+            &dev_info,
+            &gic,
+            &None,
+        )
+        .is_ok())
+    }
+}

arch/src/aarch64/gic.rs

@@ -0,0 +1,159 @@
+// Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+use std::{boxed::Box, result};
+
+use kvm_ioctls::{DeviceFd, VmFd};
+
+use super::gicv2::GICv2;
+use super::gicv3::GICv3;
+
+/// Errors thrown while setting up the GIC.
+#[derive(Debug)]
+pub enum Error {
+    /// Error while calling KVM ioctl for setting up the global interrupt controller.
+    CreateGIC(kvm_ioctls::Error),
+    /// Error while setting device attributes for the GIC.
+    SetDeviceAttribute(kvm_ioctls::Error),
+}
+type Result<T> = result::Result<T, Error>;
+
+/// Trait for GIC devices.
+pub trait GICDevice: Send + Sync {
+    /// Returns the file descriptor of the GIC device
+    fn device_fd(&self) -> &DeviceFd;
+
+    /// Returns an array with GIC device properties
+    fn device_properties(&self) -> &[u64];
+
+    /// Returns the number of vCPUs this GIC handles
+    fn vcpu_count(&self) -> u64;
+
+    /// Returns the fdt compatibility property of the device
+    fn fdt_compatibility(&self) -> &str;
+
+    /// Returns the maint_irq fdt property of the device
+    fn fdt_maint_irq(&self) -> u32;
+
+    /// Returns the GIC version of the device
+    fn version() -> u32
+    where
+        Self: Sized;
+
+    /// Create the GIC device object
+    fn create_device(fd: DeviceFd, vcpu_count: u64) -> Box<dyn GICDevice>
+    where
+        Self: Sized;
+
+    /// Setup the device-specific attributes
+    fn init_device_attributes(gic_device: &Box<dyn GICDevice>) -> Result<()>
+    where
+        Self: Sized;
+
+    /// Initialize a GIC device
+    fn init_device(vm: &VmFd) -> Result<DeviceFd>
+    where
+        Self: Sized,
+    {
+        let mut gic_device = kvm_bindings::kvm_create_device {
+            type_: Self::version(),
+            fd: 0,
+            flags: 0,
+        };
+
+        vm.create_device(&mut gic_device).map_err(Error::CreateGIC)
+    }
+
+    /// Set a GIC device attribute
+    fn set_device_attribute(
+        fd: &DeviceFd,
+        group: u32,
+        attr: u64,
+        addr: u64,
+        flags: u32,
+    ) -> Result<()>
+    where
+        Self: Sized,
+    {
+        let attr = kvm_bindings::kvm_device_attr {
+            group: group,
+            attr: attr,
+            addr: addr,
+            flags: flags,
+        };
+        fd.set_device_attr(&attr)
+            .map_err(Error::SetDeviceAttribute)?;
+
+        Ok(())
+    }
+
+    /// Finalize the setup of a GIC device
+    fn finalize_device(gic_device: &Box<dyn GICDevice>) -> Result<()>
+    where
+        Self: Sized,
+    {
+        /* We need to tell the kernel how many irqs to support with this vgic.
+         * See the `layout` module for details.
+         */
+        let nr_irqs: u32 = super::layout::IRQ_MAX - super::layout::IRQ_BASE + 1;
+        let nr_irqs_ptr = &nr_irqs as *const u32;
+        Self::set_device_attribute(
+            gic_device.device_fd(),
+            kvm_bindings::KVM_DEV_ARM_VGIC_GRP_NR_IRQS,
+            0,
+            nr_irqs_ptr as u64,
+            0,
+        )?;
+
+        /* Finalize the GIC.
+         * See https://code.woboq.org/linux/linux/virt/kvm/arm/vgic/vgic-kvm-device.c.html#211.
+         */
+        Self::set_device_attribute(
+            gic_device.device_fd(),
+            kvm_bindings::KVM_DEV_ARM_VGIC_GRP_CTRL,
+            u64::from(kvm_bindings::KVM_DEV_ARM_VGIC_CTRL_INIT),
+            0,
+            0,
+        )?;
+
+        Ok(())
+    }
+
+    /// Method to initialize the GIC device
+    fn new(vm: &VmFd, vcpu_count: u64) -> Result<Box<dyn GICDevice>>
+    where
+        Self: Sized,
+    {
+        let vgic_fd = Self::init_device(vm)?;
+
+        let device = Self::create_device(vgic_fd, vcpu_count);
+
+        Self::init_device_attributes(&device)?;
+
+        Self::finalize_device(&device)?;
+
+        Ok(device)
+    }
+}
+
+/// Create a GIC device.
+///
+/// It will try to create by default a GICv3 device. If that fails it will try
+/// to fall-back to a GICv2 device.
+pub fn create_gic(vm: &VmFd, vcpu_count: u64) -> Result<Box<dyn GICDevice>> {
+    GICv3::new(vm, vcpu_count).or_else(|_| GICv2::new(vm, vcpu_count))
+}
+
+#[cfg(test)]
+mod tests {
+
+    use super::*;
+    use kvm_ioctls::Kvm;
+
+    #[test]
+    fn test_create_gic() {
+        let kvm = Kvm::new().unwrap();
+        let vm = kvm.create_vm().unwrap();
+        assert!(create_gic(&vm, 1).is_ok());
+    }
+}

arch/src/aarch64/gicv2.rs

@@ -0,0 +1,114 @@
+// Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+use std::{boxed::Box, result};
+
+use kvm_ioctls::DeviceFd;
+
+use super::gic::{Error, GICDevice};
+
+type Result<T> = result::Result<T, Error>;
+
+/// Represent a GIC v2 device
+pub struct GICv2 {
+    /// The file descriptor for the KVM device
+    fd: DeviceFd,
+
+    /// GIC device properties, to be used for setting up the fdt entry
+    properties: [u64; 4],
+
+    /// Number of CPUs handled by the device
+    vcpu_count: u64,
+}
+
+impl GICv2 {
+    // Unfortunately bindgen omits defines that are based on other defines.
+    // See arch/arm64/include/uapi/asm/kvm.h file from the linux kernel.
+    const KVM_VGIC_V2_DIST_SIZE: u64 = 0x1000;
+    const KVM_VGIC_V2_CPU_SIZE: u64 = 0x2000;
+
+    // Device trees specific constants
+    const ARCH_GIC_V2_MAINT_IRQ: u32 = 8;
+
+    /// Get the address of the GICv2 distributor.
+    const fn get_dist_addr() -> u64 {
+        super::layout::MAPPED_IO_START - GICv2::KVM_VGIC_V2_DIST_SIZE
+    }
+
+    /// Get the size of the GIC_v2 distributor.
+    const fn get_dist_size() -> u64 {
+        GICv2::KVM_VGIC_V2_DIST_SIZE
+    }
+
+    /// Get the address of the GIC_v2 CPU.
+    const fn get_cpu_addr() -> u64 {
+        GICv2::get_dist_addr() - GICv2::KVM_VGIC_V2_CPU_SIZE
+    }
+
+    /// Get the size of the GIC_v2 CPU.
+    const fn get_cpu_size() -> u64 {
+        GICv2::KVM_VGIC_V2_CPU_SIZE
+    }
+}
+
+impl GICDevice for GICv2 {
+    fn version() -> u32 {
+        kvm_bindings::kvm_device_type_KVM_DEV_TYPE_ARM_VGIC_V2
+    }
+
+    fn device_fd(&self) -> &DeviceFd {
+        &self.fd
+    }
+
+    fn device_properties(&self) -> &[u64] {
+        &self.properties
+    }
+
+    fn vcpu_count(&self) -> u64 {
+        self.vcpu_count
+    }
+
+    fn fdt_compatibility(&self) -> &str {
+        "arm,gic-400"
+    }
+
+    fn fdt_maint_irq(&self) -> u32 {
+        GICv2::ARCH_GIC_V2_MAINT_IRQ
+    }
+
+    fn create_device(fd: DeviceFd, vcpu_count: u64) -> Box<dyn GICDevice> {
+        Box::new(GICv2 {
+            fd: fd,
+            properties: [
+                GICv2::get_dist_addr(),
+                GICv2::get_dist_size(),
+                GICv2::get_cpu_addr(),
+                GICv2::get_cpu_size(),
+            ],
+            vcpu_count: vcpu_count,
+        })
+    }
+
+    fn init_device_attributes(gic_device: &Box<dyn GICDevice>) -> Result<()> {
+        /* Setting up the distributor attribute.
+        We are placing the GIC below 1GB so we need to substract the size of the distributor. */
+        Self::set_device_attribute(
+            &gic_device.device_fd(),
+            kvm_bindings::KVM_DEV_ARM_VGIC_GRP_ADDR,
+            u64::from(kvm_bindings::KVM_VGIC_V2_ADDR_TYPE_DIST),
+            &GICv2::get_dist_addr() as *const u64 as u64,
+            0,
+        )?;
+
+        /* Setting up the CPU attribute. */
+        Self::set_device_attribute(
+            &gic_device.device_fd(),
+            kvm_bindings::KVM_DEV_ARM_VGIC_GRP_ADDR,
+            u64::from(kvm_bindings::KVM_VGIC_V2_ADDR_TYPE_CPU),
+            &GICv2::get_cpu_addr() as *const u64 as u64,
+            0,
+        )?;
+
+        Ok(())
+    }
+}

arch/src/aarch64/gicv3.rs

@@ -0,0 +1,117 @@
+// Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+use std::{boxed::Box, result};
+
+use kvm_ioctls::DeviceFd;
+
+use super::gic::{Error, GICDevice};
+
+type Result<T> = result::Result<T, Error>;
+
+pub struct GICv3 {
+    /// The file descriptor for the KVM device
+    fd: DeviceFd,
+
+    /// GIC device properties, to be used for setting up the fdt entry
+    properties: [u64; 4],
+
+    /// Number of CPUs handled by the device
+    vcpu_count: u64,
+}
+
+impl GICv3 {
+    // Unfortunately bindgen omits defines that are based on other defines.
+    // See arch/arm64/include/uapi/asm/kvm.h file from the linux kernel.
+    const SZ_64K: u64 = 0x0001_0000;
+    const KVM_VGIC_V3_DIST_SIZE: u64 = GICv3::SZ_64K;
+    const KVM_VGIC_V3_REDIST_SIZE: u64 = (2 * GICv3::SZ_64K);
+
+    // Device trees specific constants
+    const ARCH_GIC_V3_MAINT_IRQ: u32 = 9;
+
+    /// Get the address of the GIC distributor.
+    fn get_dist_addr() -> u64 {
+        super::layout::MAPPED_IO_START - GICv3::KVM_VGIC_V3_DIST_SIZE
+    }
+
+    /// Get the size of the GIC distributor.
+    fn get_dist_size() -> u64 {
+        GICv3::KVM_VGIC_V3_DIST_SIZE
+    }
+
+    /// Get the address of the GIC redistributors.
+    fn get_redists_addr(vcpu_count: u64) -> u64 {
+        GICv3::get_dist_addr() - GICv3::get_redists_size(vcpu_count)
+    }
+
+    /// Get the size of the GIC redistributors.
+    fn get_redists_size(vcpu_count: u64) -> u64 {
+        vcpu_count * GICv3::KVM_VGIC_V3_REDIST_SIZE
+    }
+}
+
+impl GICDevice for GICv3 {
+    fn version() -> u32 {
+        kvm_bindings::kvm_device_type_KVM_DEV_TYPE_ARM_VGIC_V3
+    }
+
+    fn device_fd(&self) -> &DeviceFd {
+        &self.fd
+    }
+
+    fn device_properties(&self) -> &[u64] {
+        &self.properties
+    }
+
+    fn vcpu_count(&self) -> u64 {
+        self.vcpu_count
+    }
+
+    fn fdt_compatibility(&self) -> &str {
+        "arm,gic-v3"
+    }
+
+    fn fdt_maint_irq(&self) -> u32 {
+        GICv3::ARCH_GIC_V3_MAINT_IRQ
+    }
+
+    fn create_device(fd: DeviceFd, vcpu_count: u64) -> Box<dyn GICDevice> {
+        Box::new(GICv3 {
+            fd: fd,
+            properties: [
+                GICv3::get_dist_addr(),
+                GICv3::get_dist_size(),
+                GICv3::get_redists_addr(vcpu_count),
+                GICv3::get_redists_size(vcpu_count),
+            ],
+            vcpu_count: vcpu_count,
+        })
+    }
+
+    fn init_device_attributes(gic_device: &Box<dyn GICDevice>) -> Result<()> {
+        /* Setting up the distributor attribute.
+         We are placing the GIC below 1GB so we need to substract the size of the distributor.
+        */
+        Self::set_device_attribute(
+            &gic_device.device_fd(),
+            kvm_bindings::KVM_DEV_ARM_VGIC_GRP_ADDR,
+            u64::from(kvm_bindings::KVM_VGIC_V3_ADDR_TYPE_DIST),
+            &GICv3::get_dist_addr() as *const u64 as u64,
+            0,
+        )?;
+
+        /* Setting up the redistributors' attribute.
+        We are calculating here the start of the redistributors address. We have one per CPU.
+        */
+        Self::set_device_attribute(
+            &gic_device.device_fd(),
+            kvm_bindings::KVM_DEV_ARM_VGIC_GRP_ADDR,
+            u64::from(kvm_bindings::KVM_VGIC_V3_ADDR_TYPE_REDIST),
+            &GICv3::get_redists_addr(u64::from(gic_device.vcpu_count())) as *const u64 as u64,
+            0,
+        )?;
+
+        Ok(())
+    }
+}

arch/src/aarch64/layout.rs

@@ -1,7 +1,86 @@
+// Copyright 2020 Arm Limited (or its affiliates). All rights reserved.
 // Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
 // SPDX-License-Identifier: Apache-2.0
 
-/// Kernel command line start address.
-pub const CMDLINE_START: usize = 0x0;
-/// Kernel command line start address maximum size.
-pub const CMDLINE_MAX_SIZE: usize = 0x0;
+//
+// Memory layout of Aarch64 guest:
+//
+// Physical  +---------------------------------------------------------------+
+// address   |                                                               |
+// end       |                                                               |
+//           ~                   ~                       ~                   ~
+//           |                                                               |
+//           |                      Highmem PCI MMIO space                   |
+//           |                                                               |
+// RAM end   +---------------------------------------------------------------+
+// (dynamic, |                                                               |
+// including |                                                               |
+// hotplug   ~                   ~                       ~                   ~
+// memory)   |                                                               |
+//           |                            DRAM                               |
+//           |                                                               |
+// 2GB       +---------------------------------------------------------------+
+//           |                                                               |
+//           |                           Reserved                            |
+//           |                                                               |
+// 1G+256M   +---------------------------------------------------------------+
+//           |                                                               |
+//           |                        PCI MMCONFIG space                     |
+//           |                                                               |
+// 1GB       +---------------------------------------------------------------+
+//           |                                                               |
+//           |                           PCI MMIO space                      |
+//           |                                                               |
+// 256 M     +---------------------------------------------------------------|
+//           |                                                               |
+//           |                        Legacy devices space                   |
+//           |                                                               |
+// 144 M     +---------------------------------------------------------------|
+//           |                                                               |
+//           |                    Reserverd (now GIC is here)                |
+//           |                                                               |
+// 0GB       +---------------------------------------------------------------+
+//
+//
+
+use vm_memory::{GuestAddress, GuestUsize};
+
+/// Below this address will reside the GIC, above this address will reside the MMIO devices.
+pub const MAPPED_IO_START: u64 = 0x0900_0000;
+
+/// Space 0x0900_0000 ~ 0x1000_0000 is reserved for legacy devices.
+pub const LEGACY_SERIAL_MAPPED_IO_START: u64 = 0x0900_0000;
+pub const LEGACY_RTC_MAPPED_IO_START: u64 = 0x0901_0000;
+
+/// Legacy space will be allocated at once whiling setting up legacy devices.
+pub const LEGACY_DEVICES_MAPPED_IO_SIZE: u64 = 0x0700_0000;
+
+/// Starting from 0x1000_0000 (256MiB), the 768MiB (ends at 1 GiB) is used for PCIE MMIO
+pub const MEM_32BIT_DEVICES_START: GuestAddress = GuestAddress(0x1000_0000);
+pub const MEM_32BIT_DEVICES_SIZE: u64 = 0x3000_0000;
+
+/// PCI MMCONFIG space (start: after the device space at 1 GiB, length: 256MiB)
+pub const PCI_MMCONFIG_START: GuestAddress = GuestAddress(0x4000_0000);
+pub const PCI_MMCONFIG_SIZE: GuestUsize = 256 << 20;
+
+/// Start of RAM on 64 bit ARM.
+pub const RAM_64BIT_START: u64 = 0x8000_0000;
+
+/// Kernel command line maximum size.
+/// As per `arch/arm64/include/uapi/asm/setup.h`.
+pub const CMDLINE_MAX_SIZE: usize = 2048;
+
+/// Maximum size of the device tree blob as specified in https://www.kernel.org/doc/Documentation/arm64/booting.txt.
+pub const FDT_MAX_SIZE: usize = 0x20_0000;
+
+// As per virt/kvm/arm/vgic/vgic-kvm-device.c we need
+// the number of interrupts our GIC will support to be:
+// * bigger than 32
+// * less than 1023 and
+// * a multiple of 32.
+// We are setting up our interrupt controller to support a maximum of 128 interrupts.
+/// First usable interrupt on aarch64.
+pub const IRQ_BASE: u32 = 32;
+
+/// Last usable interrupt on aarch64.
+pub const IRQ_MAX: u32 = 159;

arch/src/aarch64/mod.rs

@@ -1,27 +1,251 @@
+// Copyright 2020 Arm Limited (or its affiliates). All rights reserved.
 // Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
 // SPDX-License-Identifier: Apache-2.0
 
+/// Module for the flattened device tree.
+pub mod fdt;
+/// Module for the global interrupt controller configuration.
+pub mod gic;
+mod gicv2;
+mod gicv3;
+/// Layout for this aarch64 system.
 pub mod layout;
+/// Logic for configuring aarch64 registers.
+pub mod regs;
 
-use memory_model::{GuestAddress, GuestMemory};
+pub use self::fdt::DeviceInfoForFDT;
+use crate::DeviceType;
+use crate::RegionType;
+use kvm_ioctls::*;
+use std::collections::HashMap;
+use std::ffi::CStr;
+use std::fmt::Debug;
+use vm_memory::{
+    Address, GuestAddress, GuestAddressSpace, GuestMemory, GuestMemoryAtomic, GuestMemoryMmap,
+    GuestUsize,
+};
 
-/// Stub function that needs to be implemented when aarch64 functionality is added.
-pub fn arch_memory_regions(size: usize) -> Vec<(GuestAddress, usize, RegionType)> {
-    vec![(GuestAddress(0), size, RegionType::Ram)]
+/// Errors thrown while configuring aarch64 system.
+#[derive(Debug)]
+pub enum Error {
+    /// Failed to create a FDT.
+    SetupFDT(fdt::Error),
+
+    /// Failed to create a GIC.
+    SetupGIC(gic::Error),
+
+    /// Failed to compute the initrd address.
+    InitrdAddress,
+
+    /// Error configuring the general purpose registers
+    REGSConfiguration(regs::Error),
+
+    /// Error fetching prefered target
+    VcpuArmPreferredTarget(kvm_ioctls::Error),
+
+    /// Error doing Vcpu Init on Arm.
+    VcpuArmInit(kvm_ioctls::Error),
 }
 
-/// Stub function that needs to be implemented when aarch64 functionality is added.
-pub fn configure_system(
-    _guest_mem: &GuestMemory,
-    _cmdline_addr: GuestAddress,
-    _cmdline_size: usize,
-    _num_cpus: u8,
-    _rsdp_addr: Option<GuestAddress>,
+impl From<Error> for super::Error {
+    fn from(e: Error) -> super::Error {
+        super::Error::AArch64Setup(e)
+    }
+}
+
+#[derive(Debug, Copy, Clone)]
+/// Specifies the entry point address where the guest must start
+/// executing code.
+pub struct EntryPoint {
+    /// Address in guest memory where the guest must start execution
+    pub entry_addr: GuestAddress,
+}
+
+/// Configure the specified VCPU, and return its MPIDR.
+pub fn configure_vcpu(
+    fd: &VcpuFd,
+    id: u8,
+    vm_fd: &VmFd,
+    kernel_entry_point: Option<EntryPoint>,
+    vm_memory: &GuestMemoryAtomic<GuestMemoryMmap>,
+) -> super::Result<u64> {
+    let mut kvi: kvm_bindings::kvm_vcpu_init = kvm_bindings::kvm_vcpu_init::default();
+
+    // This reads back the kernel's preferred target type.
+    vm_fd
+        .get_preferred_target(&mut kvi)
+        .map_err(Error::VcpuArmPreferredTarget)?;
+    // We already checked that the capability is supported.
+    kvi.features[0] |= 1 << kvm_bindings::KVM_ARM_VCPU_PSCI_0_2;
+    // Non-boot cpus are powered off initially.
+    if id > 0 {
+        kvi.features[0] |= 1 << kvm_bindings::KVM_ARM_VCPU_POWER_OFF;
+    }
+
+    fd.vcpu_init(&kvi).map_err(Error::VcpuArmInit)?;
+    if let Some(kernel_entry_point) = kernel_entry_point {
+        regs::setup_regs(
+            fd,
+            id,
+            kernel_entry_point.entry_addr.raw_value(),
+            &vm_memory.memory(),
+        )
+        .map_err(Error::REGSConfiguration)?;
+    }
+
+    let mpidr = regs::read_mpidr(fd).map_err(Error::REGSConfiguration)?;
+    Ok(mpidr)
+}
+
+pub fn arch_memory_regions(size: GuestUsize) -> Vec<(GuestAddress, usize, RegionType)> {
+    let mut regions = Vec::new();
+    // 0 ~ 256 MiB: Reserved
+    regions.push((
+        GuestAddress(0),
+        layout::MEM_32BIT_DEVICES_START.0 as usize,
+        RegionType::Reserved,
+    ));
+
+    // 256 MiB ~ 1 G: MMIO space
+    regions.push((
+        layout::MEM_32BIT_DEVICES_START,
+        layout::MEM_32BIT_DEVICES_SIZE as usize,
+        RegionType::SubRegion,
+    ));
+
+    // 1G  ~ 2G: reserved. The leading 256M for PCIe MMCONFIG space
+    regions.push((
+        layout::PCI_MMCONFIG_START,
+        (layout::RAM_64BIT_START - layout::PCI_MMCONFIG_START.0) as usize,
+        RegionType::Reserved,
+    ));
+
+    regions.push((
+        GuestAddress(layout::RAM_64BIT_START),
+        size as usize,
+        RegionType::Ram,
+    ));
+
+    regions
+}
+
+/// Configures the system and should be called once per vm before starting vcpu threads.
+///
+/// # Arguments
+///
+/// * `guest_mem` - The memory to be used by the guest.
+/// * `num_cpus` - Number of virtual CPUs the guest will have.
+#[allow(clippy::too_many_arguments)]
+#[allow(unused_variables)]
+pub fn configure_system<T: DeviceInfoForFDT + Clone + Debug>(
+    vm_fd: &VmFd,
+    guest_mem: &GuestMemoryMmap,
+    cmdline_cstring: &CStr,
+    vcpu_count: u64,
+    vcpu_mpidr: Vec<u64>,
+    device_info: &HashMap<(DeviceType, String), T>,
+    initrd: &Option<super::InitramfsConfig>,
 ) -> super::Result<()> {
+    let gic_device = gic::create_gic(vm_fd, vcpu_count).map_err(Error::SetupGIC)?;
+
+    let dtb = fdt::create_fdt(
+        guest_mem,
+        cmdline_cstring,
+        vcpu_mpidr,
+        device_info,
+        &gic_device,
+        initrd,
+    )
+    .map_err(Error::SetupFDT)?;
+
     Ok(())
 }
 
-/// Stub function that needs to be implemented when aarch64 functionality is added.
-pub fn get_reserved_mem_addr() -> usize {
-    0
+/// Returns the memory address where the kernel could be loaded.
+pub fn get_kernel_start() -> u64 {
+    layout::RAM_64BIT_START
+}
+
+// Auxiliary function to get the address where the device tree blob is loaded.
+fn get_fdt_addr(mem: &GuestMemoryMmap) -> u64 {
+    // If the memory allocated is smaller than the size allocated for the FDT,
+    // we return the start of the DRAM so that
+    // we allow the code to try and load the FDT.
+
+    if let Some(addr) = mem.last_addr().checked_sub(layout::FDT_MAX_SIZE as u64 - 1) {
+        if mem.address_in_range(addr) {
+            return addr.raw_value();
+        }
+    }
+
+    layout::RAM_64BIT_START
+}
+
+pub fn get_host_cpu_phys_bits() -> u8 {
+    // The value returned here is used to determine the physical address space size
+    // for a VM (IPA size).
+    // In recent kernel versions, the maxium IPA size supported by the host can be
+    // known by querying cap KVM_CAP_ARM_VM_IPA_SIZE. And the IPA size for a
+    // guest can be configured smaller.
+    // But in Cloud-Hypervisor we simply use the maxium value for the VM.
+    // Reference https://lwn.net/Articles/766767/.
+    //
+    // The correct way to query KVM_CAP_ARM_VM_IPA_SIZE is via rust-vmm/kvm-ioctls,
+    // which wraps all IOCTL's and provides easy interface to user hypervisors.
+    // For now the cap hasn't been supported. A separate patch will be submitted to
+    // rust-vmm to add it.
+    // So a hardcoded value is used here as a temporary solution.
+    // It will be replace once rust-vmm/kvm-ioctls is ready.
+    //
+    40
+}
+
+pub fn check_required_kvm_extensions(kvm: &Kvm) -> super::Result<()> {
+    if !kvm.check_extension(Cap::SignalMsi) {
+        return Err(super::Error::CapabilityMissing(Cap::SignalMsi));
+    }
+    Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_arch_memory_regions_dram() {
+        let regions = arch_memory_regions((1usize << 32) as u64); //4GB
+        assert_eq!(4, regions.len());
+        assert_eq!(GuestAddress(layout::RAM_64BIT_START), regions[3].0);
+        assert_eq!(1usize << 32, regions[3].1);
+        assert_eq!(RegionType::Ram, regions[3].2);
+    }
+
+    #[test]
+    fn test_get_fdt_addr() {
+        let mut regions = Vec::new();
+
+        regions.push((
+            GuestAddress(layout::RAM_64BIT_START),
+            (layout::FDT_MAX_SIZE - 0x1000) as usize,
+        ));
+        let mem = GuestMemoryMmap::from_ranges(&regions).expect("Cannot initialize memory");
+        assert_eq!(get_fdt_addr(&mem), layout::RAM_64BIT_START);
+        regions.clear();
+
+        regions.push((
+            GuestAddress(layout::RAM_64BIT_START),
+            (layout::FDT_MAX_SIZE) as usize,
+        ));
+        let mem = GuestMemoryMmap::from_ranges(&regions).expect("Cannot initialize memory");
+        assert_eq!(get_fdt_addr(&mem), layout::RAM_64BIT_START);
+        regions.clear();
+
+        regions.push((
+            GuestAddress(layout::RAM_64BIT_START),
+            (layout::FDT_MAX_SIZE + 0x1000) as usize,
+        ));
+        let mem = GuestMemoryMmap::from_ranges(&regions).expect("Cannot initialize memory");
+        assert_eq!(get_fdt_addr(&mem), 0x1000 + layout::RAM_64BIT_START);
+        regions.clear();
+    }
 }

arch/src/aarch64/regs.rs

@@ -0,0 +1,198 @@
+// Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+//
+// Portions Copyright 2017 The Chromium OS Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the THIRD-PARTY file.
+
+use std::{mem, result};
+
+use super::get_fdt_addr;
+use kvm_bindings::{
+    user_pt_regs, KVM_REG_ARM64, KVM_REG_ARM64_SYSREG, KVM_REG_ARM64_SYSREG_CRM_MASK,
+    KVM_REG_ARM64_SYSREG_CRM_SHIFT, KVM_REG_ARM64_SYSREG_CRN_MASK, KVM_REG_ARM64_SYSREG_CRN_SHIFT,
+    KVM_REG_ARM64_SYSREG_OP0_MASK, KVM_REG_ARM64_SYSREG_OP0_SHIFT, KVM_REG_ARM64_SYSREG_OP1_MASK,
+    KVM_REG_ARM64_SYSREG_OP1_SHIFT, KVM_REG_ARM64_SYSREG_OP2_MASK, KVM_REG_ARM64_SYSREG_OP2_SHIFT,
+    KVM_REG_ARM_CORE, KVM_REG_SIZE_U64,
+};
+use kvm_ioctls::VcpuFd;
+
+use vm_memory::GuestMemoryMmap;
+
+/// Errors thrown while setting aarch64 registers.
+#[derive(Debug)]
+pub enum Error {
+    /// Failed to set core register (PC, PSTATE or general purpose ones).
+    SetCoreRegister(kvm_ioctls::Error),
+    /// Failed to get a system register.
+    GetSysRegister(kvm_ioctls::Error),
+}
+type Result<T> = result::Result<T, Error>;
+
+#[allow(non_upper_case_globals)]
+// PSR (Processor State Register) bits.
+// Taken from arch/arm64/include/uapi/asm/ptrace.h.
+const PSR_MODE_EL1h: u64 = 0x0000_0005;
+const PSR_F_BIT: u64 = 0x0000_0040;
+const PSR_I_BIT: u64 = 0x0000_0080;
+const PSR_A_BIT: u64 = 0x0000_0100;
+const PSR_D_BIT: u64 = 0x0000_0200;
+// Taken from arch/arm64/kvm/inject_fault.c.
+const PSTATE_FAULT_BITS_64: u64 = PSR_MODE_EL1h | PSR_A_BIT | PSR_F_BIT | PSR_I_BIT | PSR_D_BIT;
+
+// Following are macros that help with getting the ID of a aarch64 core register.
+// The core register are represented by the user_pt_regs structure. Look for it in
+// arch/arm64/include/uapi/asm/ptrace.h.
+
+// This macro gets the offset of a structure (i.e `str`) member (i.e `field`) without having
+// an instance of that structure.
+// It uses a null pointer to retrieve the offset to the field.
+// Inspired by C solution: `#define offsetof(str, f) ((size_t)(&((str *)0)->f))`.
+// Doing `offset__of!(user_pt_regs, pstate)` in our rust code will trigger the following:
+// unsafe { &(*(0 as *const user_pt_regs)).pstate as *const _ as usize }
+// The dereference expression produces an lvalue, but that lvalue is not actually read from,
+// we're just doing pointer math on it, so in theory, it should safe.
+macro_rules! offset__of {
+    ($str:ty, $field:ident) => {
+        unsafe { &(*(0 as *const $str)).$field as *const _ as usize }
+    };
+}
+
+macro_rules! arm64_core_reg {
+    ($reg: tt) => {
+        // As per `kvm_arm_copy_reg_indices`, the id of a core register can be obtained like this:
+        // `const u64 core_reg = KVM_REG_ARM64 | KVM_REG_SIZE_U64 | KVM_REG_ARM_CORE | i`, where i is obtained with:
+        // `for (i = 0; i < sizeof(struct kvm_regs) / sizeof(__u32); i++) {`
+        // We are using here `user_pt_regs` since this structure contains the core register and it is at
+        // the start of `kvm_regs`.
+        // struct kvm_regs {
+        //	struct user_pt_regs regs;	/* sp = sp_el0 */
+        //
+        //	__u64	sp_el1;
+        //	__u64	elr_el1;
+        //
+        //	__u64	spsr[KVM_NR_SPSR];
+        //
+        //	struct user_fpsimd_state fp_regs;
+        //};
+        // struct user_pt_regs {
+        //	__u64		regs[31];
+        //	__u64		sp;
+        //	__u64		pc;
+        //	__u64		pstate;
+        //};
+        // In our implementation we need: pc, pstate and user_pt_regs->regs[0].
+        KVM_REG_ARM64 as u64
+            | KVM_REG_SIZE_U64 as u64
+            | u64::from(KVM_REG_ARM_CORE)
+            | ((offset__of!(user_pt_regs, $reg) / mem::size_of::<u32>()) as u64)
+    };
+}
+
+// This macro computes the ID of a specific ARM64 system register similar to how
+// the kernel C macro does.
+// https://elixir.bootlin.com/linux/v4.20.17/source/arch/arm64/include/uapi/asm/kvm.h#L203
+macro_rules! arm64_sys_reg {
+    ($name: tt, $op0: tt, $op1: tt, $crn: tt, $crm: tt, $op2: tt) => {
+        const $name: u64 = KVM_REG_ARM64 as u64
+            | KVM_REG_SIZE_U64 as u64
+            | KVM_REG_ARM64_SYSREG as u64
+            | ((($op0 as u64) << KVM_REG_ARM64_SYSREG_OP0_SHIFT)
+                & KVM_REG_ARM64_SYSREG_OP0_MASK as u64)
+            | ((($op1 as u64) << KVM_REG_ARM64_SYSREG_OP1_SHIFT)
+                & KVM_REG_ARM64_SYSREG_OP1_MASK as u64)
+            | ((($crn as u64) << KVM_REG_ARM64_SYSREG_CRN_SHIFT)
+                & KVM_REG_ARM64_SYSREG_CRN_MASK as u64)
+            | ((($crm as u64) << KVM_REG_ARM64_SYSREG_CRM_SHIFT)
+                & KVM_REG_ARM64_SYSREG_CRM_MASK as u64)
+            | ((($op2 as u64) << KVM_REG_ARM64_SYSREG_OP2_SHIFT)
+                & KVM_REG_ARM64_SYSREG_OP2_MASK as u64);
+    };
+}
+
+// Constant imported from the Linux kernel:
+// https://elixir.bootlin.com/linux/v4.20.17/source/arch/arm64/include/asm/sysreg.h#L135
+arm64_sys_reg!(MPIDR_EL1, 3, 0, 0, 0, 5);
+
+/// Configure core registers for a given CPU.
+///
+/// # Arguments
+///
+/// * `vcpu` - Structure for the VCPU that holds the VCPU's fd.
+/// * `cpu_id` - Index of current vcpu.
+/// * `boot_ip` - Starting instruction pointer.
+/// * `mem` - Reserved DRAM for current VM.
+pub fn setup_regs(vcpu: &VcpuFd, cpu_id: u8, boot_ip: u64, mem: &GuestMemoryMmap) -> Result<()> {
+    // Get the register index of the PSTATE (Processor State) register.
+    vcpu.set_one_reg(arm64_core_reg!(pstate), PSTATE_FAULT_BITS_64)
+        .map_err(Error::SetCoreRegister)?;
+
+    // Other vCPUs are powered off initially awaiting PSCI wakeup.
+    if cpu_id == 0 {
+        // Setting the PC (Processor Counter) to the current program address (kernel address).
+        vcpu.set_one_reg(arm64_core_reg!(pc), boot_ip)
+            .map_err(Error::SetCoreRegister)?;
+
+        // Last mandatory thing to set -> the address pointing to the FDT (also called DTB).
+        // "The device tree blob (dtb) must be placed on an 8-byte boundary and must
+        // not exceed 2 megabytes in size." -> https://www.kernel.org/doc/Documentation/arm64/booting.txt.
+        // We are choosing to place it the end of DRAM. See `get_fdt_addr`.
+        vcpu.set_one_reg(arm64_core_reg!(regs), get_fdt_addr(mem) as u64)
+            .map_err(Error::SetCoreRegister)?;
+    }
+    Ok(())
+}
+
+/// Read the MPIDR - Multiprocessor Affinity Register.
+///
+/// # Arguments
+///
+/// * `vcpu` - Structure for the VCPU that holds the VCPU's fd.
+pub fn read_mpidr(vcpu: &VcpuFd) -> Result<u64> {
+    vcpu.get_one_reg(MPIDR_EL1).map_err(Error::GetSysRegister)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::aarch64::layout;
+    use kvm_ioctls::Kvm;
+    use vm_memory::{GuestAddress, GuestMemoryMmap};
+
+    #[test]
+    fn test_setup_regs() {
+        let kvm = Kvm::new().unwrap();
+        let vm = kvm.create_vm().unwrap();
+        let vcpu = vm.create_vcpu(0).unwrap();
+        let mut regions = Vec::new();
+        regions.push((
+            GuestAddress(layout::RAM_64BIT_START),
+            (layout::FDT_MAX_SIZE + 0x1000) as usize,
+        ));
+        let mem = GuestMemoryMmap::from_ranges(&regions).expect("Cannot initialize memory");
+
+        match setup_regs(&vcpu, 0, 0x0, &mem).unwrap_err() {
+            Error::SetCoreRegister(ref e) => assert_eq!(e.errno(), libc::ENOEXEC),
+            _ => panic!("Expected to receive Error::SetCoreRegister"),
+        }
+        let mut kvi: kvm_bindings::kvm_vcpu_init = kvm_bindings::kvm_vcpu_init::default();
+        vm.get_preferred_target(&mut kvi).unwrap();
+        vcpu.vcpu_init(&kvi).unwrap();
+
+        assert!(setup_regs(&vcpu, 0, 0x0, &mem).is_ok());
+    }
+    #[test]
+    fn test_read_mpidr() {
+        let kvm = Kvm::new().unwrap();
+        let vm = kvm.create_vm().unwrap();
+        let vcpu = vm.create_vcpu(0).unwrap();
+        let mut kvi: kvm_bindings::kvm_vcpu_init = kvm_bindings::kvm_vcpu_init::default();
+        vm.get_preferred_target(&mut kvi).unwrap();
+
+        // Must fail when vcpu is not initialized yet.
+        assert!(read_mpidr(&vcpu).is_err());
+
+        vcpu.vcpu_init(&kvi).unwrap();
+        assert_eq!(read_mpidr(&vcpu).unwrap(), 0x80000000);
+    }
+}

arch/src/lib.rs

@@ -1,8 +1,11 @@
+// Copyright 2020 Arm Limited (or its affiliates). All rights reserved.
 // Copyright © 2020, Oracle and/or its affiliates.
 //
 // Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
 // SPDX-License-Identifier: Apache-2.0
 
+//! Implements platform specific functionality.
+//! Supported platforms: x86_64, aarch64.
 #![allow(
     clippy::unreadable_literal,
     clippy::redundant_static_lifetimes,
@@ -13,22 +16,29 @@
 
 extern crate byteorder;
 extern crate kvm_bindings;
+extern crate kvm_ioctls;
 extern crate libc;
 
+extern crate vm_memory;
+
 #[cfg(feature = "acpi")]
 extern crate acpi_tables;
 extern crate arch_gen;
-extern crate kvm_ioctls;
 extern crate linux_loader;
-extern crate vm_memory;
 
+use kvm_ioctls::*;
+use std::fmt;
 use std::result;
 
+/// Type for returning error code.
 #[derive(Debug)]
 pub enum Error {
     #[cfg(target_arch = "x86_64")]
     /// X86_64 specific error triggered during system configuration.
     X86_64Setup(x86_64::Error),
+    #[cfg(target_arch = "aarch64")]
+    /// AArch64 specific error triggered during system configuration.
+    AArch64Setup(aarch64::Error),
     /// The zero page extends past the end of guest_mem.
     ZeroPagePastRamEnd,
     /// Error writing the zero page of guest memory.
@@ -41,10 +51,21 @@ pub enum Error {
     StartInfoPastRamEnd,
     /// Error writing hvm_start_info to guest memory.
     StartInfoSetup,
+    /// Failed to compute initramfs address.
+    InitramfsAddress,
+    /// Error writing module entry to guest memory.
+    ModlistSetup(vm_memory::GuestMemoryError),
+    /// RSDP Beyond Guest Memory
+    RSDPPastRamEnd,
+    /// Capability missing
+    CapabilityMissing(Cap),
 }
+
+/// Type for returning public functions outcome.
 pub type Result<T> = result::Result<T, Error>;
 
-#[derive(PartialEq)]
+/// Type for memory region types.
+#[derive(PartialEq, Debug)]
 pub enum RegionType {
     /// RAM type
     Ram,
@@ -62,13 +83,15 @@ pub enum RegionType {
     Reserved,
 }
 
+/// Module for aarch64 related functionality.
 #[cfg(target_arch = "aarch64")]
 pub mod aarch64;
 
 #[cfg(target_arch = "aarch64")]
 pub use aarch64::{
-    arch_memory_regions, configure_system, get_reserved_mem_addr, layout::CMDLINE_MAX_SIZE,
-    layout::CMDLINE_START,
+    arch_memory_regions, check_required_kvm_extensions, configure_system, configure_vcpu,
+    fdt::DeviceInfoForFDT, get_host_cpu_phys_bits, get_kernel_start, layout,
+    layout::CMDLINE_MAX_SIZE, layout::IRQ_BASE, layout::IRQ_MAX, EntryPoint,
 };
 
 #[cfg(target_arch = "x86_64")]
@@ -76,6 +99,66 @@ pub mod x86_64;
 
 #[cfg(target_arch = "x86_64")]
 pub use x86_64::{
-    arch_memory_regions, configure_system, layout, layout::CMDLINE_MAX_SIZE, layout::CMDLINE_START,
-    BootProtocol, EntryPoint,
+    arch_memory_regions, check_required_kvm_extensions, configure_system, configure_vcpu,
+    get_host_cpu_phys_bits, initramfs_load_addr, layout, layout::CMDLINE_MAX_SIZE,
+    layout::CMDLINE_START, regs, BootProtocol, CpuidPatch, CpuidReg, EntryPoint,
 };
+
+/// Safe wrapper for `sysconf(_SC_PAGESIZE)`.
+#[cfg(target_arch = "x86_64")]
+#[inline(always)]
+fn pagesize() -> usize {
+    // Trivially safe
+    unsafe { libc::sysconf(libc::_SC_PAGESIZE) as usize }
+}
+
+/// Type for passing information about the initramfs in the guest memory.
+pub struct InitramfsConfig {
+    /// Load address of initramfs in guest memory
+    pub address: vm_memory::GuestAddress,
+    /// Size of initramfs in guest memory
+    pub size: usize,
+}
+
+/// Types of devices that can get attached to this platform.
+#[derive(Clone, Debug, PartialEq, Eq, Hash, Copy)]
+pub enum DeviceType {
+    /// Device Type: Virtio.
+    Virtio(u32),
+    /// Device Type: Serial.
+    #[cfg(target_arch = "aarch64")]
+    Serial,
+    /// Device Type: RTC.
+    #[cfg(target_arch = "aarch64")]
+    RTC,
+}
+
+/// Default (smallest) memory page size for the supported architectures.
+pub const PAGE_SIZE: usize = 4096;
+
+impl fmt::Display for DeviceType {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(f, "{:?}", self)
+    }
+}
+
+/// Structure to describe MMIO device information
+#[derive(Clone, Debug)]
+#[cfg(target_arch = "aarch64")]
+pub struct MMIODeviceInfo {
+    pub addr: u64,
+    pub irq: u32,
+}
+
+#[cfg(target_arch = "aarch64")]
+impl DeviceInfoForFDT for MMIODeviceInfo {
+    fn addr(&self) -> u64 {
+        self.addr
+    }
+    fn irq(&self) -> u32 {
+        self.irq
+    }
+    fn length(&self) -> u64 {
+        4096 as u64
+    }
+}

arch/src/x86_64/layout.rs

@@ -31,6 +31,10 @@ pub const BOOT_IDT_START: GuestAddress = GuestAddress(0x520);
 /// Address for the hvm_start_info struct used in PVH boot
 pub const PVH_INFO_START: GuestAddress = GuestAddress(0x6000);
 
+/// Starting address of array of modules of hvm_modlist_entry type.
+/// Used to enable initrd support using the PVH boot ABI.
+pub const MODLIST_START: GuestAddress = GuestAddress(0x6040);
+
 /// Address of memory map table used in PVH boot. Can overlap
 /// with the zero page address since they are mutually exclusive.
 pub const MEMMAP_START: GuestAddress = GuestAddress(0x7000);
@@ -43,9 +47,10 @@ pub const BOOT_STACK_START: GuestAddress = GuestAddress(0x8000);
 pub const BOOT_STACK_POINTER: GuestAddress = GuestAddress(0x8ff0);
 
 // Initial pagetables.
-pub const PML4_START: GuestAddress = GuestAddress(0x9000);
-pub const PDPTE_START: GuestAddress = GuestAddress(0xa000);
-pub const PDE_START: GuestAddress = GuestAddress(0xb000);
+pub const PML5_START: GuestAddress = GuestAddress(0x9000);
+pub const PML4_START: GuestAddress = GuestAddress(0xa000);
+pub const PDPTE_START: GuestAddress = GuestAddress(0xb000);
+pub const PDE_START: GuestAddress = GuestAddress(0xc000);
 
 /// Kernel command line start address.
 pub const CMDLINE_START: GuestAddress = GuestAddress(0x20000);
@@ -74,18 +79,18 @@ pub const HIGH_RAM_START: GuestAddress = GuestAddress(0x100000);
 
 // ** 32-bit reserved area (start: 3GiB, length: 1GiB) **
 pub const MEM_32BIT_RESERVED_START: GuestAddress = GuestAddress(0xc000_0000);
-pub const MEM_32BIT_RESERVED_SIZE: GuestUsize = (1024 << 20);
+pub const MEM_32BIT_RESERVED_SIZE: GuestUsize = 1024 << 20;
 
 // == Fixed constants within the "32-bit reserved" range ==
 
 // Sub range: 32-bit PCI devices (start: 3GiB, length: 640Mib)
 pub const MEM_32BIT_DEVICES_START: GuestAddress = MEM_32BIT_RESERVED_START;
-pub const MEM_32BIT_DEVICES_SIZE: GuestUsize = (640 << 20);
+pub const MEM_32BIT_DEVICES_SIZE: GuestUsize = 640 << 20;
 
 // PCI MMCONFIG space (start: after the device space, length: 256MiB)
 pub const PCI_MMCONFIG_START: GuestAddress =
     GuestAddress(MEM_32BIT_DEVICES_START.0 + MEM_32BIT_DEVICES_SIZE);
-pub const PCI_MMCONFIG_SIZE: GuestUsize = (256 << 20);
+pub const PCI_MMCONFIG_SIZE: GuestUsize = 256 << 20;
 
 // IOAPIC
 pub const IOAPIC_START: GuestAddress = GuestAddress(0xfec0_0000);

arch/src/x86_64/mod.rs

@@ -10,15 +10,21 @@
 mod gdt;
 pub mod interrupts;
 pub mod layout;
+#[cfg(not(feature = "acpi"))]
 mod mptable;
 pub mod regs;
-
+use crate::InitramfsConfig;
 use crate::RegionType;
+use kvm_bindings::CpuId;
+use kvm_ioctls::*;
 use linux_loader::loader::bootparam::{boot_params, setup_header};
-use linux_loader::loader::start_info::{hvm_memmap_table_entry, hvm_start_info};
+use linux_loader::loader::elf::start_info::{
+    hvm_memmap_table_entry, hvm_modlist_entry, hvm_start_info,
+};
 use std::mem;
 use vm_memory::{
-    Address, ByteValued, Bytes, GuestAddress, GuestMemory, GuestMemoryMmap, GuestUsize,
+    Address, ByteValued, Bytes, GuestAddress, GuestAddressSpace, GuestMemory, GuestMemoryAtomic,
+    GuestMemoryMmap, GuestMemoryRegion, GuestUsize,
 };
 
 #[derive(Debug, Copy, Clone)]
@@ -45,6 +51,8 @@ pub struct EntryPoint {
     pub entry_addr: GuestAddress,
     /// Specifies which boot protocol to use
     pub protocol: BootProtocol,
+    /// This field is used for bzImage to fill zero page
+    pub setup_header: Option<setup_header>,
 }
 
 const E820_RAM: u32 = 1;
@@ -66,6 +74,11 @@ struct MemmapTableEntryWrapper(hvm_memmap_table_entry);
 
 unsafe impl ByteValued for MemmapTableEntryWrapper {}
 
+#[derive(Copy, Clone, Default)]
+struct ModlistEntryWrapper(hvm_modlist_entry);
+
+unsafe impl ByteValued for ModlistEntryWrapper {}
+
 // This is a workaround to the Rust enforcement specifying that any implementation of a foreign
 // trait (in this case `DataInit`) where:
 // *    the type that is implementing the trait is foreign or
@@ -81,8 +94,27 @@ unsafe impl ByteValued for BootParamsWrapper {}
 pub enum Error {
     /// Invalid e820 setup params.
     E820Configuration,
+    #[cfg(not(feature = "acpi"))]
     /// Error writing MP table to memory.
     MpTableSetup(mptable::Error),
+
+    /// Error configuring the general purpose registers
+    REGSConfiguration(regs::Error),
+
+    /// Error configuring the special registers
+    SREGSConfiguration(regs::Error),
+
+    /// Error configuring the floating point related registers
+    FPUConfiguration(regs::Error),
+
+    /// Error configuring the MSR registers
+    MSRSConfiguration(regs::Error),
+
+    /// The call to KVM_SET_CPUID2 failed.
+    SetSupportedCpusFailed(kvm_ioctls::Error),
+
+    /// Cannot set the local interruption due to bad configuration.
+    LocalIntConfiguration(interrupts::Error),
 }
 
 impl From<Error> for super::Error {
@@ -91,6 +123,113 @@ impl From<Error> for super::Error {
     }
 }
 
+#[allow(dead_code)]
+#[derive(Copy, Clone)]
+pub enum CpuidReg {
+    EAX,
+    EBX,
+    ECX,
+    EDX,
+}
+
+pub struct CpuidPatch {
+    pub function: u32,
+    pub index: u32,
+    pub flags_bit: Option<u8>,
+    pub eax_bit: Option<u8>,
+    pub ebx_bit: Option<u8>,
+    pub ecx_bit: Option<u8>,
+    pub edx_bit: Option<u8>,
+}
+
+impl CpuidPatch {
+    pub fn set_cpuid_reg(
+        cpuid: &mut CpuId,
+        function: u32,
+        index: Option<u32>,
+        reg: CpuidReg,
+        value: u32,
+    ) {
+        let entries = cpuid.as_mut_slice();
+
+        for entry in entries.iter_mut() {
+            if entry.function == function && (index == None || index.unwrap() == entry.index) {
+                match reg {
+                    CpuidReg::EAX => {
+                        entry.eax = value;
+                    }
+                    CpuidReg::EBX => {
+                        entry.ebx = value;
+                    }
+                    CpuidReg::ECX => {
+                        entry.ecx = value;
+                    }
+                    CpuidReg::EDX => {
+                        entry.edx = value;
+                    }
+                }
+            }
+        }
+    }
+
+    pub fn patch_cpuid(cpuid: &mut CpuId, patches: Vec<CpuidPatch>) {
+        let entries = cpuid.as_mut_slice();
+
+        for entry in entries.iter_mut() {
+            for patch in patches.iter() {
+                if entry.function == patch.function && entry.index == patch.index {
+                    if let Some(flags_bit) = patch.flags_bit {
+                        entry.flags |= 1 << flags_bit;
+                    }
+                    if let Some(eax_bit) = patch.eax_bit {
+                        entry.eax |= 1 << eax_bit;
+                    }
+                    if let Some(ebx_bit) = patch.ebx_bit {
+                        entry.ebx |= 1 << ebx_bit;
+                    }
+                    if let Some(ecx_bit) = patch.ecx_bit {
+                        entry.ecx |= 1 << ecx_bit;
+                    }
+                    if let Some(edx_bit) = patch.edx_bit {
+                        entry.edx |= 1 << edx_bit;
+                    }
+                }
+            }
+        }
+    }
+}
+
+pub fn configure_vcpu(
+    fd: &VcpuFd,
+    id: u8,
+    kernel_entry_point: Option<EntryPoint>,
+    vm_memory: &GuestMemoryAtomic<GuestMemoryMmap>,
+    cpuid: CpuId,
+) -> super::Result<()> {
+    let mut cpuid = cpuid;
+    CpuidPatch::set_cpuid_reg(&mut cpuid, 0xb, None, CpuidReg::EDX, u32::from(id));
+    fd.set_cpuid2(&cpuid)
+        .map_err(Error::SetSupportedCpusFailed)?;
+
+    regs::setup_msrs(fd).map_err(Error::MSRSConfiguration)?;
+    if let Some(kernel_entry_point) = kernel_entry_point {
+        // Safe to unwrap because this method is called after the VM is configured
+        regs::setup_regs(
+            fd,
+            kernel_entry_point.entry_addr.raw_value(),
+            layout::BOOT_STACK_POINTER.raw_value(),
+            layout::ZERO_PAGE_START.raw_value(),
+            kernel_entry_point.protocol,
+        )
+        .map_err(Error::REGSConfiguration)?;
+        regs::setup_fpu(fd).map_err(Error::FPUConfiguration)?;
+        regs::setup_sregs(&vm_memory.memory(), fd, kernel_entry_point.protocol)
+            .map_err(Error::SREGSConfiguration)?;
+    }
+    interrupts::set_lint(fd).map_err(Error::LocalIntConfiguration)?;
+    Ok(())
+}
+
 /// Returns a Vec of the valid memory addresses.
 /// These should be used to configure the GuestMemory structure for the platform.
 /// For x86_64 all addresses are valid from the start of the kernel except a
@@ -151,20 +290,36 @@ pub fn configure_system(
     guest_mem: &GuestMemoryMmap,
     cmdline_addr: GuestAddress,
     cmdline_size: usize,
-    num_cpus: u8,
+    initramfs: &Option<InitramfsConfig>,
+    _num_cpus: u8,
     setup_hdr: Option<setup_header>,
     rsdp_addr: Option<GuestAddress>,
     boot_prot: BootProtocol,
 ) -> super::Result<()> {
     // Note that this puts the mptable at the last 1k of Linux's 640k base RAM
-    mptable::setup_mptable(guest_mem, num_cpus).map_err(Error::MpTableSetup)?;
+    #[cfg(not(feature = "acpi"))]
+    mptable::setup_mptable(guest_mem, _num_cpus).map_err(Error::MpTableSetup)?;
+
+    // Check that the RAM is not smaller than the RSDP start address
+    if let Some(rsdp_addr) = rsdp_addr {
+        if rsdp_addr.0 > guest_mem.last_addr().0 {
+            return Err(super::Error::RSDPPastRamEnd);
+        }
+    }
 
     match boot_prot {
         BootProtocol::PvhBoot => {
-            configure_pvh(guest_mem, cmdline_addr, rsdp_addr)?;
+            configure_pvh(guest_mem, cmdline_addr, initramfs, rsdp_addr)?;
         }
         BootProtocol::LinuxBoot => {
-            configure_64bit_boot(guest_mem, cmdline_addr, cmdline_size, setup_hdr, rsdp_addr)?;
+            configure_64bit_boot(
+                guest_mem,
+                cmdline_addr,
+                cmdline_size,
+                initramfs,
+                setup_hdr,
+                rsdp_addr,
+            )?;
         }
     }
 
@@ -174,6 +329,7 @@ pub fn configure_system(
 fn configure_pvh(
     guest_mem: &GuestMemoryMmap,
     cmdline_addr: GuestAddress,
+    initramfs: &Option<InitramfsConfig>,
     rsdp_addr: Option<GuestAddress>,
 ) -> super::Result<()> {
     const XEN_HVM_START_MAGIC_VALUE: u32 = 0x336ec578;
@@ -190,6 +346,24 @@ fn configure_pvh(
         start_info.0.rsdp_paddr = rsdp_addr.0;
     }
 
+    if let Some(initramfs_config) = initramfs {
+        // The initramfs has been written to guest memory already, here we just need to
+        // create the module structure that describes it.
+        let ramdisk_mod: ModlistEntryWrapper = ModlistEntryWrapper(hvm_modlist_entry {
+            paddr: initramfs_config.address.raw_value(),
+            size: initramfs_config.size as u64,
+            ..Default::default()
+        });
+
+        start_info.0.nr_modules += 1;
+        start_info.0.modlist_paddr = layout::MODLIST_START.raw_value();
+
+        // Write the modlist struct to guest memory.
+        guest_mem
+            .write_obj(ramdisk_mod, layout::MODLIST_START)
+            .map_err(super::Error::ModlistSetup)?;
+    }
+
     // Vector to hold the memory maps which needs to be written to guest memory
     // at MEMMAP_START after all of the mappings are recorded.
     let mut memmap: Vec<hvm_memmap_table_entry> = Vec::new();
@@ -293,6 +467,7 @@ fn configure_64bit_boot(
     guest_mem: &GuestMemoryMmap,
     cmdline_addr: GuestAddress,
     cmdline_size: usize,
+    initramfs: &Option<InitramfsConfig>,
     setup_hdr: Option<setup_header>,
     rsdp_addr: Option<GuestAddress>,
 ) -> super::Result<()> {
@@ -319,6 +494,11 @@ fn configure_64bit_boot(
     params.0.hdr.cmd_line_ptr = cmdline_addr.raw_value() as u32;
     params.0.hdr.cmdline_size = cmdline_size as u32;
 
+    if let Some(initramfs_config) = initramfs {
+        params.0.hdr.ramdisk_image = initramfs_config.address.raw_value() as u32;
+        params.0.hdr.ramdisk_size = initramfs_config.size as u32;
+    }
+
     add_e820_entry(&mut params.0, 0, layout::EBDA_START.raw_value(), E820_RAM)?;
 
     let mem_end = guest_mem.last_addr();
@@ -388,6 +568,66 @@ fn add_e820_entry(
     Ok(())
 }
 
+/// Returns the memory address where the initramfs could be loaded.
+pub fn initramfs_load_addr(
+    guest_mem: &GuestMemoryMmap,
+    initramfs_size: usize,
+) -> super::Result<u64> {
+    let first_region = guest_mem
+        .find_region(GuestAddress::new(0))
+        .ok_or(super::Error::InitramfsAddress)?;
+    // It's safe to cast to usize because the size of a region can't be greater than usize.
+    let lowmem_size = first_region.len() as usize;
+
+    if lowmem_size < initramfs_size {
+        return Err(super::Error::InitramfsAddress);
+    }
+
+    let aligned_addr: u64 = ((lowmem_size - initramfs_size) & !(crate::pagesize() - 1)) as u64;
+    Ok(aligned_addr)
+}
+
+pub fn get_host_cpu_phys_bits() -> u8 {
+    use std::arch::x86_64;
+    unsafe {
+        let leaf = x86_64::__cpuid(0x8000_0000);
+
+        // Detect and handle AMD SME (Secure Memory Encryption) properly.
+        // Some physical address bits may become reserved when the feature is enabled.
+        // See AMD64 Architecture Programmer's Manual Volume 2, Section 7.10.1
+        let reduced = if leaf.eax >= 0x8000_001f
+            && leaf.ebx == 0x6874_7541    // Vendor ID: AuthenticAMD
+            && leaf.ecx == 0x444d_4163
+            && leaf.edx == 0x6974_6e65
+            && x86_64::__cpuid(0x8000_001f).eax & 0x1 != 0
+        {
+            (x86_64::__cpuid(0x8000_001f).ebx >> 6) & 0x3f
+        } else {
+            0
+        };
+
+        if leaf.eax >= 0x8000_0008 {
+            let leaf = x86_64::__cpuid(0x8000_0008);
+            ((leaf.eax & 0xff) - reduced) as u8
+        } else {
+            36
+        }
+    }
+}
+
+pub fn check_required_kvm_extensions(kvm: &Kvm) -> super::Result<()> {
+    if !kvm.check_extension(Cap::SignalMsi) {
+        return Err(super::Error::CapabilityMissing(Cap::SignalMsi));
+    }
+    if !kvm.check_extension(Cap::TscDeadlineTimer) {
+        return Err(super::Error::CapabilityMissing(Cap::TscDeadlineTimer));
+    }
+    if !kvm.check_extension(Cap::SplitIrqchip) {
+        return Err(super::Error::CapabilityMissing(Cap::SplitIrqchip));
+    }
+    Ok(())
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -417,9 +657,10 @@ mod tests {
             &gm,
             GuestAddress(0),
             0,
+            &None,
             1,
             None,
-            None,
+            Some(layout::RSDP_POINTER),
             BootProtocol::LinuxBoot,
         );
         assert!(config_err.is_err());
@@ -437,6 +678,7 @@ mod tests {
             &gm,
             GuestAddress(0),
             0,
+            &None,
             no_vcpus,
             None,
             None,
@@ -448,6 +690,7 @@ mod tests {
             &gm,
             GuestAddress(0),
             0,
+            &None,
             no_vcpus,
             None,
             None,
@@ -468,6 +711,7 @@ mod tests {
             &gm,
             GuestAddress(0),
             0,
+            &None,
             no_vcpus,
             None,
             None,
@@ -479,6 +723,7 @@ mod tests {
             &gm,
             GuestAddress(0),
             0,
+            &None,
             no_vcpus,
             None,
             None,
@@ -499,6 +744,7 @@ mod tests {
             &gm,
             GuestAddress(0),
             0,
+            &None,
             no_vcpus,
             None,
             None,
@@ -510,6 +756,7 @@ mod tests {
             &gm,
             GuestAddress(0),
             0,
+            &None,
             no_vcpus,
             None,
             None,

arch/src/x86_64/regs.rs

@@ -14,7 +14,9 @@ use super::BootProtocol;
 use arch_gen::x86::msr_index;
 use kvm_bindings::{kvm_fpu, kvm_msr_entry, kvm_regs, kvm_sregs, Msrs};
 use kvm_ioctls::VcpuFd;
-use layout::{BOOT_GDT_START, BOOT_IDT_START, PDE_START, PDPTE_START, PML4_START, PVH_INFO_START};
+use layout::{
+    BOOT_GDT_START, BOOT_IDT_START, PDE_START, PDPTE_START, PML4_START, PML5_START, PVH_INFO_START,
+};
 use vm_memory::{Address, Bytes, GuestMemory, GuestMemoryError, GuestMemoryMmap};
 
 // MTRR constants
@@ -45,6 +47,8 @@ pub enum Error {
     WritePDEAddress(GuestMemoryError),
     /// Writing PML4 to RAM failed.
     WritePML4Address(GuestMemoryError),
+    /// Writing PML5 to RAM failed.
+    WritePML5Address(GuestMemoryError),
 }
 
 pub type Result<T> = result::Result<T, Error>;
@@ -70,7 +74,7 @@ pub fn setup_fpu(vcpu: &VcpuFd) -> Result<()> {
 ///
 /// * `vcpu` - Structure for the VCPU that holds the VCPU's fd.
 pub fn setup_msrs(vcpu: &VcpuFd) -> Result<()> {
-    vcpu.set_msrs(&create_msr_entries())
+    vcpu.set_msrs(&boot_msr_entries())
         .map_err(Error::SetModelSpecificRegisters)?;
 
     Ok(())
@@ -138,6 +142,7 @@ const EFER_LME: u64 = 0x100;
 const X86_CR0_PE: u64 = 0x1;
 const X86_CR0_PG: u64 = 0x80000000;
 const X86_CR4_PAE: u64 = 0x20;
+const X86_CR4_LA57: u64 = 0x1000;
 
 fn write_gdt_table(table: &[u64], guest_mem: &GuestMemoryMmap) -> Result<()> {
     let boot_gdt_addr = BOOT_GDT_START;
@@ -220,7 +225,18 @@ fn configure_segments_and_sregs(
 }
 
 fn setup_page_tables(mem: &GuestMemoryMmap, sregs: &mut kvm_sregs) -> Result<()> {
-    // Puts PML4 right after zero page but aligned to 4k.
+    // Puts PML5 or PML4 right after zero page but aligned to 4k.
+
+    if unsafe { std::arch::x86_64::__cpuid(7).ecx } & (1 << 16) != 0 {
+        // Entry covering VA [0..256TB)
+        mem.write_obj(PML4_START.raw_value() | 0x03, PML5_START)
+            .map_err(Error::WritePML5Address)?;
+
+        sregs.cr3 = PML5_START.raw_value();
+        sregs.cr4 |= X86_CR4_LA57;
+    } else {
+        sregs.cr3 = PML4_START.raw_value();
+    }
 
     // Entry covering VA [0..512GB)
     mem.write_obj(PDPTE_START.raw_value() | 0x03, PML4_START)
@@ -229,6 +245,7 @@ fn setup_page_tables(mem: &GuestMemoryMmap, sregs: &mut kvm_sregs) -> Result<()>
     // Entry covering VA [0..1GB)
     mem.write_obj(PDE_START.raw_value() | 0x03, PDPTE_START)
         .map_err(Error::WritePDPTEAddress)?;
+
     // 512 2MB entries together covering VA [0..1GB). Note we are assuming
     // CPU supports 2MB pages (/proc/cpuinfo has 'pse'). All modern CPUs do.
     for i in 0..512 {
@@ -236,74 +253,49 @@ fn setup_page_tables(mem: &GuestMemoryMmap, sregs: &mut kvm_sregs) -> Result<()>
             .map_err(Error::WritePDEAddress)?;
     }
 
-    sregs.cr3 = PML4_START.raw_value();
     sregs.cr4 |= X86_CR4_PAE;
     sregs.cr0 |= X86_CR0_PG;
+
     Ok(())
 }
 
-fn create_msr_entries() -> Msrs {
-    let mut entries = Vec::<kvm_msr_entry>::new();
+macro_rules! kvm_msr {
+    ($msr:expr) => {
+        kvm_msr_entry {
+            index: $msr,
+            data: 0x0,
+            ..Default::default()
+        }
+    };
+}
 
-    entries.push(kvm_msr_entry {
-        index: msr_index::MSR_IA32_SYSENTER_CS,
-        data: 0x0,
-        ..Default::default()
-    });
-    entries.push(kvm_msr_entry {
-        index: msr_index::MSR_IA32_SYSENTER_ESP,
-        data: 0x0,
-        ..Default::default()
-    });
-    entries.push(kvm_msr_entry {
-        index: msr_index::MSR_IA32_SYSENTER_EIP,
-        data: 0x0,
-        ..Default::default()
-    });
-    // x86_64 specific msrs, we only run on x86_64 not x86.
-    entries.push(kvm_msr_entry {
-        index: msr_index::MSR_STAR,
-        data: 0x0,
-        ..Default::default()
-    });
-    entries.push(kvm_msr_entry {
-        index: msr_index::MSR_CSTAR,
-        data: 0x0,
-        ..Default::default()
-    });
-    entries.push(kvm_msr_entry {
-        index: msr_index::MSR_KERNEL_GS_BASE,
-        data: 0x0,
-        ..Default::default()
-    });
-    entries.push(kvm_msr_entry {
-        index: msr_index::MSR_SYSCALL_MASK,
-        data: 0x0,
-        ..Default::default()
-    });
-    entries.push(kvm_msr_entry {
-        index: msr_index::MSR_LSTAR,
-        data: 0x0,
-        ..Default::default()
-    });
-    // end of x86_64 specific code
-    entries.push(kvm_msr_entry {
-        index: msr_index::MSR_IA32_TSC,
-        data: 0x0,
-        ..Default::default()
-    });
-    entries.push(kvm_msr_entry {
-        index: msr_index::MSR_IA32_MISC_ENABLE,
-        data: msr_index::MSR_IA32_MISC_ENABLE_FAST_STRING as u64,
-        ..Default::default()
-    });
-    entries.push(kvm_msr_entry {
-        index: msr_index::MSR_MTRRdefType,
-        data: MTRR_ENABLE | MTRR_MEM_TYPE_WB,
-        ..Default::default()
-    });
+macro_rules! kvm_msr_data {
+    ($msr:expr, $data:expr) => {
+        kvm_msr_entry {
+            index: $msr,
+            data: $data,
+            ..Default::default()
+        }
+    };
+}
 
-    Msrs::from_entries(&entries)
+pub fn boot_msr_entries() -> Msrs {
+    Msrs::from_entries(&[
+        kvm_msr!(msr_index::MSR_IA32_SYSENTER_CS),
+        kvm_msr!(msr_index::MSR_IA32_SYSENTER_ESP),
+        kvm_msr!(msr_index::MSR_IA32_SYSENTER_EIP),
+        kvm_msr!(msr_index::MSR_STAR),
+        kvm_msr!(msr_index::MSR_CSTAR),
+        kvm_msr!(msr_index::MSR_LSTAR),
+        kvm_msr!(msr_index::MSR_KERNEL_GS_BASE),
+        kvm_msr!(msr_index::MSR_SYSCALL_MASK),
+        kvm_msr!(msr_index::MSR_IA32_TSC),
+        kvm_msr_data!(
+            msr_index::MSR_IA32_MISC_ENABLE,
+            msr_index::MSR_IA32_MISC_ENABLE_FAST_STRING as u64
+        ),
+        kvm_msr_data!(msr_index::MSR_MTRRdefType, MTRR_ENABLE | MTRR_MEM_TYPE_WB),
+    ])
 }
 
 #[cfg(test)]
@@ -393,8 +385,11 @@ mod tests {
         let gm = create_guest_mem();
         setup_page_tables(&gm, &mut sregs).unwrap();
 
-        assert_eq!(0xa003, read_u64(&gm, PML4_START));
-        assert_eq!(0xb003, read_u64(&gm, PDPTE_START));
+        if unsafe { std::arch::x86_64::__cpuid(7).ecx } & (1 << 16) != 0 {
+            assert_eq!(0xa003, read_u64(&gm, PML5_START));
+        }
+        assert_eq!(0xb003, read_u64(&gm, PML4_START));
+        assert_eq!(0xc003, read_u64(&gm, PDPTE_START));
         for i in 0..512 {
             assert_eq!(
                 (i << 21) + 0x83u64,
@@ -402,7 +397,11 @@ mod tests {
             );
         }
 
-        assert_eq!(PML4_START.raw_value(), sregs.cr3);
+        if unsafe { std::arch::x86_64::__cpuid(7).ecx } & (1 << 16) != 0 {
+            assert_eq!(PML5_START.raw_value(), sregs.cr3);
+        } else {
+            assert_eq!(PML4_START.raw_value(), sregs.cr3);
+        }
         assert_eq!(X86_CR4_PAE, sregs.cr4);
         assert_eq!(X86_CR0_PG, sregs.cr0);
     }
@@ -451,7 +450,7 @@ mod tests {
         // Official entries that were setup when we did setup_msrs. We need to assert that the
         // tenth one (i.e the one with index msr_index::MSR_IA32_MISC_ENABLE has the data we
         // expect.
-        let entry_vec = create_msr_entries();
+        let entry_vec = boot_msr_entries();
         assert_eq!(entry_vec.as_slice()[9], msrs.as_slice()[0]);
     }
 

devices/Cargo.toml

@@ -4,14 +4,19 @@ version = "0.1.0"
 authors = ["The Chromium OS Authors"]
 
 [dependencies]
+anyhow = "1.0"
 bitflags = ">=1.2.1"
 byteorder = "1.3.4"
 epoll = ">=4.0.1"
-libc = "0.2.68"
+libc = "0.2.71"
 log = "0.4.8"
+serde = {version = ">=1.0.27", features = ["rc"] }
+serde_derive = ">=1.0.27"
+serde_json = ">=1.0.9"
 vm-device = { path = "../vm-device" }
 acpi_tables = { path = "../acpi_tables", optional = true }
-vm-memory = "0.1.0"
+vm-memory = "0.2.1"
+vm-migration = { path = "../vm-migration" }
 vmm-sys-util = ">=0.3.1"
 
 [dev-dependencies]
@@ -21,3 +26,4 @@ tempfile = "3.1.0"
 default = []
 acpi = ["acpi_tables"]
 cmos = []
+fwdebug = []

devices/src/gic.rs

@@ -0,0 +1,85 @@
+// Copyright 2020, ARM Limited.
+//
+// SPDX-License-Identifier: Apache-2.0 AND BSD-3-Clause
+
+use super::interrupt_controller::{Error, InterruptController};
+use std::result;
+use std::sync::Arc;
+use vm_device::interrupt::{
+    InterruptIndex, InterruptManager, InterruptSourceGroup, MsiIrqGroupConfig,
+};
+use vm_migration::{Migratable, MigratableError, Pausable, Snapshot, Snapshottable, Transportable};
+
+type Result<T> = result::Result<T, Error>;
+
+// Reserve 32 IRQs (GSI 32 ~ 64) for legacy device.
+// GsiAllocator should allocate beyond this: from 64 on
+pub const IRQ_LEGACY_COUNT: usize = 32;
+pub const IRQ_SPI_OFFSET: usize = 32;
+
+// This Gic struct implements InterruptController to provide interrupt delivery service.
+// The Gic source files in arch/ folder maintain the Aarch64 specific Gic device.
+// The 2 Gic instances could be merged together.
+// Leave this refactoring to future. Two options may be considered:
+//   1. Move Gic*.rs from arch/ folder here.
+//   2. Move this file and ioapic.rs to arch/, as they are architecture specific.
+pub struct Gic {
+    interrupt_source_group: Arc<Box<dyn InterruptSourceGroup>>,
+}
+
+impl Gic {
+    pub fn new(
+        _vcpu_count: u8,
+        interrupt_manager: Arc<dyn InterruptManager<GroupConfig = MsiIrqGroupConfig>>,
+    ) -> Result<Gic> {
+        let interrupt_source_group = interrupt_manager
+            .create_group(MsiIrqGroupConfig {
+                base: IRQ_SPI_OFFSET as InterruptIndex,
+                count: IRQ_LEGACY_COUNT as InterruptIndex,
+            })
+            .map_err(Error::CreateInterruptSourceGroup)?;
+
+        Ok(Gic {
+            interrupt_source_group,
+        })
+    }
+}
+
+impl InterruptController for Gic {
+    fn enable(&self) -> Result<()> {
+        &self
+            .interrupt_source_group
+            .enable()
+            .map_err(Error::EnableInterrupt)?;
+        Ok(())
+    }
+
+    // This should be called anytime an interrupt needs to be injected into the
+    // running guest.
+    fn service_irq(&mut self, irq: usize) -> Result<()> {
+        self.interrupt_source_group
+            .trigger(irq as InterruptIndex)
+            .map_err(Error::TriggerInterrupt)?;
+
+        Ok(())
+    }
+}
+
+const GIC_SNAPSHOT_ID: &str = "gic";
+impl Snapshottable for Gic {
+    fn id(&self) -> String {
+        GIC_SNAPSHOT_ID.to_string()
+    }
+
+    fn snapshot(&self) -> std::result::Result<Snapshot, MigratableError> {
+        unimplemented!();
+    }
+
+    fn restore(&mut self, _snapshot: Snapshot) -> std::result::Result<(), MigratableError> {
+        unimplemented!();
+    }
+}
+
+impl Pausable for Gic {}
+impl Transportable for Gic {}
+impl Migratable for Gic {}

devices/src/interrupt_controller.rs

@@ -0,0 +1,61 @@
+// Copyright 2020, ARM Limited.
+//
+// SPDX-License-Identifier: Apache-2.0 AND BSD-3-Clause
+
+use std::io;
+use std::result;
+
+#[derive(Debug)]
+pub enum Error {
+    /// Invalid destination mode.
+    InvalidDestinationMode,
+    /// Invalid trigger mode.
+    InvalidTriggerMode,
+    /// Invalid delivery mode.
+    InvalidDeliveryMode,
+    /// Failed creating the interrupt source group.
+    CreateInterruptSourceGroup(io::Error),
+    /// Failed triggering the interrupt.
+    TriggerInterrupt(io::Error),
+    /// Failed masking the interrupt.
+    MaskInterrupt(io::Error),
+    /// Failed unmasking the interrupt.
+    UnmaskInterrupt(io::Error),
+    /// Failed updating the interrupt.
+    UpdateInterrupt(io::Error),
+    /// Failed enabling the interrupt.
+    EnableInterrupt(io::Error),
+}
+
+type Result<T> = result::Result<T, Error>;
+
+pub struct MsiMessage {
+    // Message Address Register
+    //   31-20: Base address. Fixed value (0x0FEE)
+    //   19-12: Destination ID
+    //   11-4:  Reserved
+    //   3:     Redirection Hint indication
+    //   2:     Destination Mode
+    //   1-0:   Reserved
+    pub addr: u32,
+    // Message Data Register
+    //   32-16: Reserved
+    //   15:    Trigger Mode. 0 = Edge, 1 = Level
+    //   14:    Level. 0 = Deassert, 1 = Assert
+    //   13-11: Reserved
+    //   10-8:  Delivery Mode
+    //   7-0:   Vector
+    pub data: u32,
+}
+
+// Introduce trait InterruptController to uniform the interrupt
+// service provided for devices.
+// Device manager uses this trait without caring whether it is a
+// IOAPIC (X86) or GIC (Arm).
+pub trait InterruptController: Send {
+    fn service_irq(&mut self, irq: usize) -> Result<()>;
+    #[cfg(target_arch = "aarch64")]
+    fn enable(&self) -> Result<()>;
+    #[cfg(target_arch = "x86_64")]
+    fn end_of_interrupt(&mut self, vec: u8);
+}

devices/src/ioapic.rs

@@ -9,9 +9,10 @@
 // Implementation of an intel 82093AA Input/Output Advanced Programmable Interrupt Controller
 // See https://pdos.csail.mit.edu/6.828/2016/readings/ia32/ioapic.pdf for a specification.
 
+use super::interrupt_controller::{Error, InterruptController};
 use crate::BusDevice;
+use anyhow::anyhow;
 use byteorder::{ByteOrder, LittleEndian};
-use std::io;
 use std::result;
 use std::sync::Arc;
 use vm_device::interrupt::{
@@ -19,28 +20,14 @@ use vm_device::interrupt::{
     MsiIrqGroupConfig, MsiIrqSourceConfig,
 };
 use vm_memory::GuestAddress;
+use vm_migration::{
+    Migratable, MigratableError, Pausable, Snapshot, SnapshotDataSection, Snapshottable,
+    Transportable,
+};
 
-#[derive(Debug)]
-pub enum Error {
-    /// Invalid destination mode.
-    InvalidDestinationMode,
-    /// Invalid trigger mode.
-    InvalidTriggerMode,
-    /// Invalid delivery mode.
-    InvalidDeliveryMode,
-    /// Failed creating the interrupt source group.
-    CreateInterruptSourceGroup(io::Error),
-    /// Failed triggering the interrupt.
-    TriggerInterrupt(io::Error),
-    /// Failed masking the interrupt.
-    MaskInterrupt(io::Error),
-    /// Failed unmasking the interrupt.
-    UnmaskInterrupt(io::Error),
-    /// Failed updating the interrupt.
-    UpdateInterrupt(io::Error),
-    /// Failed enabling the interrupt.
-    EnableInterrupt(io::Error),
-}
+#[derive(Serialize, Deserialize)]
+#[serde(remote = "GuestAddress")]
+pub struct GuestAddressDef(pub u64);
 
 type Result<T> = result::Result<T, Error>;
 
@@ -99,25 +86,6 @@ fn set_remote_irr(entry: &mut RedirectionTableEntry, val: u8) {
     *entry |= u64::from(val & 0x1) << 14;
 }
 
-pub struct MsiMessage {
-    // Message Address Register
-    //   31-20: Base address. Fixed value (0x0FEE)
-    //   19-12: Destination ID
-    //   11-4:  Reserved
-    //   3:     Redirection Hint indication
-    //   2:     Destination Mode
-    //   1-0:   Reserved
-    pub addr: u32,
-    // Message Data Register
-    //   32-16: Reserved
-    //   15:    Trigger Mode. 0 = Edge, 1 = Level
-    //   14:    Level. 0 = Deassert, 1 = Assert
-    //   13-11: Reserved
-    //   10-8:  Delivery Mode
-    //   7-0:   Vector
-    pub data: u32,
-}
-
 pub const NUM_IOAPIC_PINS: usize = 24;
 const IOAPIC_VERSION_ID: u32 = 0x0017_0011;
 
@@ -166,13 +134,25 @@ fn decode_irq_from_selector(selector: u8) -> (usize, bool) {
 }
 
 pub struct Ioapic {
-    id: u32,
+    id: String,
+    id_reg: u32,
     reg_sel: u32,
     reg_entries: [RedirectionTableEntry; NUM_IOAPIC_PINS],
+    used_entries: [bool; NUM_IOAPIC_PINS],
     apic_address: GuestAddress,
     interrupt_source_group: Arc<Box<dyn InterruptSourceGroup>>,
 }
 
+#[derive(Serialize, Deserialize)]
+pub struct IoapicState {
+    id_reg: u32,
+    reg_sel: u32,
+    reg_entries: [RedirectionTableEntry; NUM_IOAPIC_PINS],
+    used_entries: [bool; NUM_IOAPIC_PINS],
+    #[serde(with = "GuestAddressDef")]
+    apic_address: GuestAddress,
+}
+
 impl BusDevice for Ioapic {
     fn read(&mut self, _base: u64, offset: u64, data: &mut [u8]) {
         assert!(data.len() == 4);
@@ -210,6 +190,7 @@ impl BusDevice for Ioapic {
 
 impl Ioapic {
     pub fn new(
+        id: String,
         apic_address: GuestAddress,
         interrupt_manager: Arc<dyn InterruptManager<GroupConfig = MsiIrqGroupConfig>>,
     ) -> Result<Ioapic> {
@@ -225,43 +206,87 @@ impl Ioapic {
             .map_err(Error::EnableInterrupt)?;
 
         Ok(Ioapic {
-            id: 0,
+            id,
+            id_reg: 0,
             reg_sel: 0,
             reg_entries: [0; NUM_IOAPIC_PINS],
+            used_entries: [false; NUM_IOAPIC_PINS],
             apic_address,
             interrupt_source_group,
         })
     }
 
-    // The ioapic must be informed about EOIs in order to deassert interrupts
-    // already sent.
-    pub fn end_of_interrupt(&mut self, vec: u8) {
-        for i in 0..NUM_IOAPIC_PINS {
-            let entry = &mut self.reg_entries[i];
-            // Clear Remote IRR bit
-            if vector(*entry) == vec && trigger_mode(*entry) == 1 {
-                set_remote_irr(entry, 0);
+    fn ioapic_write(&mut self, val: u32) {
+        debug!("IOAPIC_W reg 0x{:x}, val 0x{:x}", self.reg_sel, val);
+
+        match self.reg_sel as u8 {
+            IOAPIC_REG_ID => self.id_reg = (val >> 24) & 0xf,
+            IOWIN_OFF..=REG_MAX_OFFSET => {
+                let (index, is_high_bits) = decode_irq_from_selector(self.reg_sel as u8);
+                if is_high_bits {
+                    self.reg_entries[index] &= 0xffff_ffff;
+                    self.reg_entries[index] |= u64::from(val) << 32;
+                } else {
+                    // Ensure not to override read-only bits:
+                    // - Delivery Status (bit 12)
+                    // - Remote IRR (bit 14)
+                    self.reg_entries[index] &= 0xffff_ffff_0000_5000;
+                    self.reg_entries[index] |= u64::from(val) & 0xffff_afff;
+                }
+                // The entry must be updated through the interrupt source
+                // group.
+                if let Err(e) = self.update_entry(index) {
+                    error!("Failed updating IOAPIC entry: {:?}", e);
+                }
+                // Store the information this IRQ is now being used.
+                self.used_entries[index] = true;
+            }
+            _ => error!("IOAPIC: invalid write to register offset"),
+        }
+    }
+
+    fn ioapic_read(&self) -> u32 {
+        debug!("IOAPIC_R reg 0x{:x}", self.reg_sel);
+
+        match self.reg_sel as u8 {
+            IOAPIC_REG_VERSION => IOAPIC_VERSION_ID,
+            IOAPIC_REG_ID | IOAPIC_REG_ARBITRATION_ID => (self.id_reg & 0xf) << 24,
+            IOWIN_OFF..=REG_MAX_OFFSET => {
+                let (index, is_high_bits) = decode_irq_from_selector(self.reg_sel as u8);
+                if is_high_bits {
+                    (self.reg_entries[index] >> 32) as u32
+                } else {
+                    (self.reg_entries[index] & 0xffff_ffff) as u32
+                }
+            }
+            _ => {
+                error!("IOAPIC: invalid read from register offset");
+                0
             }
         }
     }
 
-    // This should be called anytime an interrupt needs to be injected into the
-    // running guest.
-    pub fn service_irq(&mut self, irq: usize) -> Result<()> {
-        let entry = &mut self.reg_entries[irq];
-
-        self.interrupt_source_group
-            .trigger(irq as InterruptIndex)
-            .map_err(Error::TriggerInterrupt)?;
-        debug!("Interrupt successfully delivered");
-
-        // If trigger mode is level sensitive, set the Remote IRR bit.
-        // It will be cleared when the EOI is received.
-        if trigger_mode(*entry) == 1 {
-            set_remote_irr(entry, 1);
+    fn state(&self) -> IoapicState {
+        IoapicState {
+            id_reg: self.id_reg,
+            reg_sel: self.reg_sel,
+            reg_entries: self.reg_entries,
+            used_entries: self.used_entries,
+            apic_address: self.apic_address,
+        }
+    }
+
+    fn set_state(&mut self, state: &IoapicState) -> Result<()> {
+        self.id_reg = state.id_reg;
+        self.reg_sel = state.reg_sel;
+        self.reg_entries = state.reg_entries;
+        self.used_entries = state.used_entries;
+        self.apic_address = state.apic_address;
+        for (irq, entry) in self.used_entries.iter().enumerate() {
+            if *entry {
+                self.update_entry(irq)?;
+            }
         }
-        // Clear the Delivery Status bit
-        set_delivery_status(entry, 0);
 
         Ok(())
     }
@@ -337,52 +362,84 @@ impl Ioapic {
 
         Ok(())
     }
+}
 
-    fn ioapic_write(&mut self, val: u32) {
-        debug!("IOAPIC_W reg 0x{:x}, val 0x{:x}", self.reg_sel, val);
-
-        match self.reg_sel as u8 {
-            IOAPIC_REG_ID => self.id = (val >> 24) & 0xf,
-            IOWIN_OFF..=REG_MAX_OFFSET => {
-                let (index, is_high_bits) = decode_irq_from_selector(self.reg_sel as u8);
-                if is_high_bits {
-                    self.reg_entries[index] &= 0xffff_ffff;
-                    self.reg_entries[index] |= u64::from(val) << 32;
-                } else {
-                    // Ensure not to override read-only bits:
-                    // - Delivery Status (bit 12)
-                    // - Remote IRR (bit 14)
-                    self.reg_entries[index] &= 0xffff_ffff_0000_5000;
-                    self.reg_entries[index] |= u64::from(val) & 0xffff_afff;
-                }
-                // The entry must be updated through the interrupt source
-                // group.
-                if let Err(e) = self.update_entry(index) {
-                    error!("Failed updating IOAPIC entry: {:?}", e);
-                }
+impl InterruptController for Ioapic {
+    // The ioapic must be informed about EOIs in order to deassert interrupts
+    // already sent.
+    fn end_of_interrupt(&mut self, vec: u8) {
+        for i in 0..NUM_IOAPIC_PINS {
+            let entry = &mut self.reg_entries[i];
+            // Clear Remote IRR bit
+            if vector(*entry) == vec && trigger_mode(*entry) == 1 {
+                set_remote_irr(entry, 0);
             }
-            _ => error!("IOAPIC: invalid write to register offset"),
         }
     }
 
-    fn ioapic_read(&self) -> u32 {
-        debug!("IOAPIC_R reg 0x{:x}", self.reg_sel);
+    // This should be called anytime an interrupt needs to be injected into the
+    // running guest.
+    fn service_irq(&mut self, irq: usize) -> Result<()> {
+        let entry = &mut self.reg_entries[irq];
 
-        match self.reg_sel as u8 {
-            IOAPIC_REG_VERSION => IOAPIC_VERSION_ID,
-            IOAPIC_REG_ID | IOAPIC_REG_ARBITRATION_ID => (self.id & 0xf) << 24,
-            IOWIN_OFF..=REG_MAX_OFFSET => {
-                let (index, is_high_bits) = decode_irq_from_selector(self.reg_sel as u8);
-                if is_high_bits {
-                    (self.reg_entries[index] >> 32) as u32
-                } else {
-                    (self.reg_entries[index] & 0xffff_ffff) as u32
-                }
-            }
-            _ => {
-                error!("IOAPIC: invalid read from register offset");
-                0
-            }
+        self.interrupt_source_group
+            .trigger(irq as InterruptIndex)
+            .map_err(Error::TriggerInterrupt)?;
+        debug!("Interrupt successfully delivered");
+
+        // If trigger mode is level sensitive, set the Remote IRR bit.
+        // It will be cleared when the EOI is received.
+        if trigger_mode(*entry) == 1 {
+            set_remote_irr(entry, 1);
         }
+        // Clear the Delivery Status bit
+        set_delivery_status(entry, 0);
+
+        Ok(())
     }
 }
+
+impl Snapshottable for Ioapic {
+    fn id(&self) -> String {
+        self.id.clone()
+    }
+
+    fn snapshot(&self) -> std::result::Result<Snapshot, MigratableError> {
+        let snapshot =
+            serde_json::to_vec(&self.state()).map_err(|e| MigratableError::Snapshot(e.into()))?;
+
+        let mut ioapic_snapshot = Snapshot::new(self.id.as_str());
+        ioapic_snapshot.add_data_section(SnapshotDataSection {
+            id: format!("{}-section", self.id),
+            snapshot,
+        });
+
+        Ok(ioapic_snapshot)
+    }
+
+    fn restore(&mut self, snapshot: Snapshot) -> std::result::Result<(), MigratableError> {
+        if let Some(ioapic_section) = snapshot.snapshot_data.get(&format!("{}-section", self.id)) {
+            let ioapic_state = match serde_json::from_slice(&ioapic_section.snapshot) {
+                Ok(state) => state,
+                Err(error) => {
+                    return Err(MigratableError::Restore(anyhow!(
+                        "Could not deserialize IOAPIC {}",
+                        error
+                    )))
+                }
+            };
+
+            return self.set_state(&ioapic_state).map_err(|e| {
+                MigratableError::Restore(anyhow!("Could not restore IOAPIC state {:?}", e))
+            });
+        }
+
+        Err(MigratableError::Restore(anyhow!(
+            "Could not find IOAPIC snapshot section"
+        )))
+    }
+}
+
+impl Pausable for Ioapic {}
+impl Transportable for Ioapic {}
+impl Migratable for Ioapic {}

devices/src/legacy/fwdebug.rs

@@ -0,0 +1,40 @@
+// Portions Copyright 2017 The Chromium OS Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE-BSD-3-Clause file.
+//
+// Copyright © 2020 Intel Corporation
+//
+// SPDX-License-Identifier: Apache-2.0 AND BSD-3-Clause
+//
+
+use BusDevice;
+
+/// Provides firmware debug output via I/O port controls
+#[derive(Default)]
+pub struct FwDebugDevice {}
+
+impl FwDebugDevice {
+    pub fn new() -> Self {
+        Self {}
+    }
+}
+
+/// FwDebugDevice sits on the I/O bus as 0x402 and receives ASCII characters
+impl BusDevice for FwDebugDevice {
+    /// Upon read return the magic value to indicate that there is a debug port
+    fn read(&mut self, _base: u64, _offset: u64, data: &mut [u8]) {
+        if data.len() == 1 {
+            data[0] = 0xe9
+        } else {
+            error!("Invalid read size on debug port: {}", data.len())
+        }
+    }
+
+    fn write(&mut self, _base: u64, _offset: u64, data: &[u8]) {
+        if data.len() == 1 {
+            print!("{}", data[0] as char);
+        } else {
+            error!("Invalid write size on debug port: {}", data.len())
+        }
+    }
+}

devices/src/legacy/mod.rs

@@ -7,10 +7,19 @@
 
 #[cfg(feature = "cmos")]
 mod cmos;
+#[cfg(feature = "fwdebug")]
+mod fwdebug;
 mod i8042;
+#[cfg(target_arch = "aarch64")]
+mod rtc_pl031;
 mod serial;
 
 #[cfg(feature = "cmos")]
 pub use self::cmos::Cmos;
+#[cfg(feature = "fwdebug")]
+pub use self::fwdebug::FwDebugDevice;
 pub use self::i8042::I8042Device;
 pub use self::serial::Serial;
+
+#[cfg(target_arch = "aarch64")]
+pub use self::rtc_pl031::RTC;

devices/src/legacy/rtc_pl031.rs

@@ -0,0 +1,626 @@
+// Copyright 2020 Arm Limited (or its affiliates). All rights reserved.
+// Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+//! ARM PL031 Real Time Clock
+//!
+//! This module implements a PL031 Real Time Clock (RTC) that provides to provides long time base counter.
+//! This is achieved by generating an interrupt signal after counting for a programmed number of cycles of
+//! a real-time clock input.
+//!
+use std::fmt;
+use std::sync::Arc;
+use std::time::Instant;
+use std::{io, result};
+
+use crate::BusDevice;
+use vm_device::interrupt::InterruptSourceGroup;
+
+// As you can see in https://static.docs.arm.com/ddi0224/c/real_time_clock_pl031_r1p3_technical_reference_manual_DDI0224C.pdf
+// at section 3.2 Summary of RTC registers, the total size occupied by this device is 0x000 -> 0xFFC + 4 = 0x1000.
+// From 0x0 to 0x1C we have following registers:
+const RTCDR: u64 = 0x0; // Data Register.
+const RTCMR: u64 = 0x4; // Match Register.
+const RTCLR: u64 = 0x8; // Load Regiser.
+const RTCCR: u64 = 0xc; // Control Register.
+const RTCIMSC: u64 = 0x10; // Interrupt Mask Set or Clear Register.
+const RTCRIS: u64 = 0x14; // Raw Interrupt Status.
+const RTCMIS: u64 = 0x18; // Masked Interrupt Status.
+const RTCICR: u64 = 0x1c; // Interrupt Clear Register.
+                          // From 0x020 to 0xFDC => reserved space.
+                          // From 0xFE0 to 0x1000 => Peripheral and PrimeCell Identification Registers which are Read Only registers.
+                          // AMBA standard devices have CIDs (Cell IDs) and PIDs (Peripheral IDs). The linux kernel will look for these in order to assert the identity
+                          // of these devices (i.e look at the `amba_device_try_add` function).
+                          // We are putting the expected values (look at 'Reset value' column from above mentioned document) in an array.
+const PL031_ID: [u8; 8] = [0x31, 0x10, 0x14, 0x00, 0x0d, 0xf0, 0x05, 0xb1];
+// We are only interested in the margins.
+const AMBA_ID_LOW: u64 = 0xFE0;
+const AMBA_ID_HIGH: u64 = 0x1000;
+/// Constant to convert seconds to nanoseconds.
+pub const NANOS_PER_SECOND: u64 = 1_000_000_000;
+
+#[allow(unused_macros)]
+macro_rules! generate_read_fn {
+    ($fn_name: ident, $data_type: ty, $byte_type: ty, $type_size: expr, $endian_type: ident) => {
+        #[allow(dead_code)]
+        pub fn $fn_name(input: &[$byte_type]) -> $data_type {
+            assert!($type_size == std::mem::size_of::<$data_type>());
+            let mut array = [0u8; $type_size];
+            for (byte, read) in array.iter_mut().zip(input.iter().cloned()) {
+                *byte = read as u8;
+            }
+            <$data_type>::$endian_type(array)
+        }
+    };
+}
+
+#[allow(unused_macros)]
+macro_rules! generate_write_fn {
+    ($fn_name: ident, $data_type: ty, $byte_type: ty, $endian_type: ident) => {
+        #[allow(dead_code)]
+        pub fn $fn_name(buf: &mut [$byte_type], n: $data_type) {
+            for (byte, read) in buf
+                .iter_mut()
+                .zip(<$data_type>::$endian_type(n).iter().cloned())
+            {
+                *byte = read as $byte_type;
+            }
+        }
+    };
+}
+
+generate_read_fn!(read_le_u16, u16, u8, 2, from_le_bytes);
+generate_read_fn!(read_le_u32, u32, u8, 4, from_le_bytes);
+generate_read_fn!(read_le_u64, u64, u8, 8, from_le_bytes);
+generate_read_fn!(read_le_i32, i32, i8, 4, from_le_bytes);
+
+generate_read_fn!(read_be_u16, u16, u8, 2, from_be_bytes);
+generate_read_fn!(read_be_u32, u32, u8, 4, from_be_bytes);
+
+generate_write_fn!(write_le_u16, u16, u8, to_le_bytes);
+generate_write_fn!(write_le_u32, u32, u8, to_le_bytes);
+generate_write_fn!(write_le_u64, u64, u8, to_le_bytes);
+generate_write_fn!(write_le_i32, i32, i8, to_le_bytes);
+
+generate_write_fn!(write_be_u16, u16, u8, to_be_bytes);
+generate_write_fn!(write_be_u32, u32, u8, to_be_bytes);
+
+#[derive(Debug)]
+pub enum Error {
+    BadWriteOffset(u64),
+    InterruptFailure(io::Error),
+}
+
+impl fmt::Display for Error {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        match self {
+            Error::BadWriteOffset(offset) => write!(f, "Bad Write Offset: {}", offset),
+            Error::InterruptFailure(e) => write!(f, "Failed to trigger interrupt: {}", e),
+        }
+    }
+}
+
+type Result<T> = result::Result<T, Error>;
+
+/// Wrapper over `libc::clockid_t` to specify Linux Kernel clock source.
+pub enum ClockType {
+    /// Equivalent to `libc::CLOCK_MONOTONIC`.
+    Monotonic,
+    /// Equivalent to `libc::CLOCK_REALTIME`.
+    #[allow(dead_code)]
+    Real,
+    /// Equivalent to `libc::CLOCK_PROCESS_CPUTIME_ID`.
+    ProcessCpu,
+    /// Equivalent to `libc::CLOCK_THREAD_CPUTIME_ID`.
+    #[allow(dead_code)]
+    ThreadCpu,
+}
+
+impl Into<libc::clockid_t> for ClockType {
+    fn into(self) -> libc::clockid_t {
+        match self {
+            ClockType::Monotonic => libc::CLOCK_MONOTONIC,
+            ClockType::Real => libc::CLOCK_REALTIME,
+            ClockType::ProcessCpu => libc::CLOCK_PROCESS_CPUTIME_ID,
+            ClockType::ThreadCpu => libc::CLOCK_THREAD_CPUTIME_ID,
+        }
+    }
+}
+
+/// Structure representing the date in local time with nanosecond precision.
+pub struct LocalTime {
+    /// Seconds in current minute.
+    sec: i32,
+    /// Minutes in current hour.
+    min: i32,
+    /// Hours in current day, 24H format.
+    hour: i32,
+    /// Days in current month.
+    mday: i32,
+    /// Months in current year.
+    mon: i32,
+    /// Years passed since 1900 BC.
+    year: i32,
+    /// Nanoseconds in current second.
+    nsec: i64,
+}
+
+impl LocalTime {
+    /// Returns the [LocalTime](struct.LocalTime.html) structure for the calling moment.
+    #[allow(dead_code)]
+    pub fn now() -> LocalTime {
+        let mut timespec = libc::timespec {
+            tv_sec: 0,
+            tv_nsec: 0,
+        };
+        let mut tm: libc::tm = libc::tm {
+            tm_sec: 0,
+            tm_min: 0,
+            tm_hour: 0,
+            tm_mday: 0,
+            tm_mon: 0,
+            tm_year: 0,
+            tm_wday: 0,
+            tm_yday: 0,
+            tm_isdst: 0,
+            tm_gmtoff: 0,
+            tm_zone: std::ptr::null(),
+        };
+
+        // Safe because the parameters are valid.
+        unsafe {
+            libc::clock_gettime(libc::CLOCK_REALTIME, &mut timespec);
+            libc::localtime_r(&timespec.tv_sec, &mut tm);
+        }
+
+        LocalTime {
+            sec: tm.tm_sec,
+            min: tm.tm_min,
+            hour: tm.tm_hour,
+            mday: tm.tm_mday,
+            mon: tm.tm_mon,
+            year: tm.tm_year,
+            nsec: timespec.tv_nsec,
+        }
+    }
+}
+
+impl fmt::Display for LocalTime {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(
+            f,
+            "{}-{:02}-{:02}T{:02}:{:02}:{:02}.{:09}",
+            self.year + 1900,
+            self.mon + 1,
+            self.mday,
+            self.hour,
+            self.min,
+            self.sec,
+            self.nsec
+        )
+    }
+}
+
+/// Holds a micro-second resolution timestamp with both the real time and cpu time.
+#[derive(Clone)]
+pub struct TimestampUs {
+    /// Real time in microseconds.
+    pub time_us: u64,
+    /// Cpu time in microseconds.
+    pub cputime_us: u64,
+}
+
+impl Default for TimestampUs {
+    fn default() -> TimestampUs {
+        TimestampUs {
+            time_us: get_time(ClockType::Monotonic) / 1000,
+            cputime_us: get_time(ClockType::ProcessCpu) / 1000,
+        }
+    }
+}
+
+/// Returns a timestamp in nanoseconds from a monotonic clock.
+///
+/// Uses `_rdstc` on `x86_64` and [`get_time`](fn.get_time.html) on other architectures.
+#[allow(dead_code)]
+pub fn timestamp_cycles() -> u64 {
+    #[cfg(target_arch = "x86_64")]
+    // Safe because there's nothing that can go wrong with this call.
+    unsafe {
+        std::arch::x86_64::_rdtsc() as u64
+    }
+    #[cfg(not(target_arch = "x86_64"))]
+    {
+        get_time(ClockType::Monotonic)
+    }
+}
+
+/// Returns a timestamp in nanoseconds based on the provided clock type.
+///
+/// # Arguments
+///
+/// * `clock_type` - Identifier of the Linux Kernel clock on which to act.
+pub fn get_time(clock_type: ClockType) -> u64 {
+    let mut time_struct = libc::timespec {
+        tv_sec: 0,
+        tv_nsec: 0,
+    };
+    // Safe because the parameters are valid.
+    unsafe { libc::clock_gettime(clock_type.into(), &mut time_struct) };
+    seconds_to_nanoseconds(time_struct.tv_sec).unwrap() as u64 + (time_struct.tv_nsec as u64)
+}
+
+/// Converts a timestamp in seconds to an equivalent one in nanoseconds.
+/// Returns `None` if the conversion overflows.
+///
+/// # Arguments
+///
+/// * `value` - Timestamp in seconds.
+pub fn seconds_to_nanoseconds(value: i64) -> Option<i64> {
+    value.checked_mul(NANOS_PER_SECOND as i64)
+}
+
+/// A RTC device following the PL031 specification..
+pub struct RTC {
+    previous_now: Instant,
+    tick_offset: i64,
+    // This is used for implementing the RTC alarm. However, in Firecracker we do not need it.
+    match_value: u32,
+    // Writes to this register load an update value into the RTC.
+    load: u32,
+    imsc: u32,
+    ris: u32,
+    interrupt: Arc<Box<dyn InterruptSourceGroup>>,
+}
+
+impl RTC {
+    /// Constructs an AMBA PL031 RTC device.
+    pub fn new(interrupt: Arc<Box<dyn InterruptSourceGroup>>) -> RTC {
+        RTC {
+            // This is used only for duration measuring purposes.
+            previous_now: Instant::now(),
+            tick_offset: get_time(ClockType::Real) as i64,
+            match_value: 0,
+            load: 0,
+            imsc: 0,
+            ris: 0,
+            interrupt,
+        }
+    }
+
+    fn trigger_interrupt(&mut self) -> Result<()> {
+        self.interrupt.trigger(0).map_err(Error::InterruptFailure)?;
+        Ok(())
+    }
+
+    fn get_time(&self) -> u32 {
+        let ts = (self.tick_offset as i128)
+            + (Instant::now().duration_since(self.previous_now).as_nanos() as i128);
+        (ts / NANOS_PER_SECOND as i128) as u32
+    }
+
+    fn handle_write(&mut self, offset: u64, val: u32) -> Result<()> {
+        match offset {
+            RTCMR => {
+                // The MR register is used for implementing the RTC alarm. A real time clock alarm is
+                // a feature that can be used to allow a computer to 'wake up' after shut down to execute
+                // tasks every day or on a certain day. It can sometimes be found in the 'Power Management'
+                // section of a motherboard's BIOS setup. This is functionality that extends beyond
+                // Firecracker intended use. However, we increment a metric just in case.
+                self.match_value = val;
+            }
+            RTCLR => {
+                self.load = val;
+                self.previous_now = Instant::now();
+                // If the unwrap fails, then the internal value of the clock has been corrupted and
+                // we want to terminate the execution of the process.
+                self.tick_offset = seconds_to_nanoseconds(i64::from(val)).unwrap();
+            }
+            RTCIMSC => {
+                self.imsc = val & 1;
+                self.trigger_interrupt()?;
+            }
+            RTCICR => {
+                // As per above mentioned doc, the interrupt is cleared by writing any data value to
+                // the Interrupt Clear Register.
+                self.ris = 0;
+                self.trigger_interrupt()?;
+            }
+            RTCCR => (), // ignore attempts to turn off the timer.
+            o => {
+                return Err(Error::BadWriteOffset(o));
+            }
+        }
+        Ok(())
+    }
+}
+
+impl BusDevice for RTC {
+    fn read(&mut self, _base: u64, offset: u64, data: &mut [u8]) {
+        let v;
+        let mut read_ok = true;
+
+        if offset < AMBA_ID_HIGH && offset >= AMBA_ID_LOW {
+            let index = ((offset - AMBA_ID_LOW) >> 2) as usize;
+            v = u32::from(PL031_ID[index]);
+        } else {
+            v = match offset {
+                RTCDR => self.get_time(),
+                RTCMR => {
+                    // Even though we are not implementing RTC alarm we return the last value
+                    self.match_value
+                }
+                RTCLR => self.load,
+                RTCCR => 1, // RTC is always enabled.
+                RTCIMSC => self.imsc,
+                RTCRIS => self.ris,
+                RTCMIS => self.ris & self.imsc,
+                _ => {
+                    read_ok = false;
+                    0
+                }
+            };
+        }
+        if read_ok && data.len() <= 4 {
+            write_le_u32(data, v);
+        } else {
+            warn!(
+                "Invalid RTC PL031 read: offset {}, data length {}",
+                offset,
+                data.len()
+            );
+        }
+    }
+
+    fn write(&mut self, _base: u64, offset: u64, data: &[u8]) {
+        if data.len() <= 4 {
+            let v = read_le_u32(&data[..]);
+            if let Err(e) = self.handle_write(offset, v) {
+                warn!("Failed to write to RTC PL031 device: {}", e);
+            }
+        } else {
+            warn!(
+                "Invalid RTC PL031 write: offset {}, data length {}",
+                offset,
+                data.len()
+            );
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use std::sync::Arc;
+    use vm_device::interrupt::{InterruptIndex, InterruptSourceConfig};
+    use vmm_sys_util::eventfd::EventFd;
+
+    const LEGACY_RTC_MAPPED_IO_START: u64 = 0x0901_0000;
+
+    #[test]
+    fn test_get_time() {
+        for _ in 0..1000 {
+            assert!(get_time(ClockType::Monotonic) <= get_time(ClockType::Monotonic));
+        }
+
+        for _ in 0..1000 {
+            assert!(get_time(ClockType::ProcessCpu) <= get_time(ClockType::ProcessCpu));
+        }
+
+        for _ in 0..1000 {
+            assert!(get_time(ClockType::ThreadCpu) <= get_time(ClockType::ThreadCpu));
+        }
+
+        assert_ne!(get_time(ClockType::Real), 0);
+    }
+
+    #[test]
+    fn test_local_time_display() {
+        let local_time = LocalTime {
+            sec: 30,
+            min: 15,
+            hour: 10,
+            mday: 4,
+            mon: 6,
+            year: 119,
+            nsec: 123_456_789,
+        };
+        assert_eq!(
+            String::from("2019-07-04T10:15:30.123456789"),
+            local_time.to_string()
+        );
+
+        let local_time = LocalTime {
+            sec: 5,
+            min: 5,
+            hour: 5,
+            mday: 23,
+            mon: 7,
+            year: 44,
+            nsec: 123,
+        };
+        assert_eq!(
+            String::from("1944-08-23T05:05:05.000000123"),
+            local_time.to_string()
+        );
+
+        let local_time = LocalTime::now();
+        assert!(local_time.mon >= 0 && local_time.mon <= 11);
+    }
+
+    #[test]
+    fn test_seconds_to_nanoseconds() {
+        assert_eq!(
+            seconds_to_nanoseconds(100).unwrap() as u64,
+            100 * NANOS_PER_SECOND
+        );
+
+        assert!(seconds_to_nanoseconds(9_223_372_037).is_none());
+    }
+
+    struct TestInterrupt {
+        event_fd: EventFd,
+    }
+
+    impl InterruptSourceGroup for TestInterrupt {
+        fn trigger(&self, _index: InterruptIndex) -> result::Result<(), std::io::Error> {
+            self.event_fd.write(1)
+        }
+
+        fn update(
+            &self,
+            _index: InterruptIndex,
+            _config: InterruptSourceConfig,
+        ) -> result::Result<(), std::io::Error> {
+            Ok(())
+        }
+
+        fn notifier(&self, _index: InterruptIndex) -> Option<&EventFd> {
+            Some(&self.event_fd)
+        }
+    }
+
+    impl TestInterrupt {
+        fn new(event_fd: EventFd) -> Self {
+            TestInterrupt { event_fd }
+        }
+    }
+
+    #[test]
+    fn test_rtc_read_write_and_event() {
+        let intr_evt = EventFd::new(libc::EFD_NONBLOCK).unwrap();
+
+        let mut rtc = RTC::new(Arc::new(Box::new(TestInterrupt::new(
+            intr_evt.try_clone().unwrap(),
+        ))));
+        let mut data = [0; 4];
+
+        // Read and write to the MR register.
+        write_le_u32(&mut data, 123);
+        rtc.write(LEGACY_RTC_MAPPED_IO_START, RTCMR, &mut data);
+        rtc.read(LEGACY_RTC_MAPPED_IO_START, RTCMR, &mut data);
+        let v = read_le_u32(&data[..]);
+        assert_eq!(v, 123);
+
+        // Read and write to the LR register.
+        let v = get_time(ClockType::Real);
+        write_le_u32(&mut data, (v / NANOS_PER_SECOND) as u32);
+        let previous_now_before = rtc.previous_now;
+        rtc.write(LEGACY_RTC_MAPPED_IO_START, RTCLR, &mut data);
+
+        assert!(rtc.previous_now > previous_now_before);
+
+        rtc.read(LEGACY_RTC_MAPPED_IO_START, RTCLR, &mut data);
+        let v_read = read_le_u32(&data[..]);
+        assert_eq!((v / NANOS_PER_SECOND) as u32, v_read);
+
+        // Read and write to IMSC register.
+        // Test with non zero value.
+        let non_zero = 1;
+        write_le_u32(&mut data, non_zero);
+        rtc.write(LEGACY_RTC_MAPPED_IO_START, RTCIMSC, &mut data);
+        // The interrupt line should be on.
+        assert!(rtc.interrupt.notifier(0).unwrap().read().unwrap() == 1);
+        rtc.read(LEGACY_RTC_MAPPED_IO_START, RTCIMSC, &mut data);
+        let v = read_le_u32(&data[..]);
+        assert_eq!(non_zero & 1, v);
+
+        // Now test with 0.
+        write_le_u32(&mut data, 0);
+        rtc.write(LEGACY_RTC_MAPPED_IO_START, RTCIMSC, &mut data);
+        rtc.read(LEGACY_RTC_MAPPED_IO_START, RTCIMSC, &mut data);
+        let v = read_le_u32(&data[..]);
+        assert_eq!(0, v);
+
+        // Read and write to the ICR register.
+        write_le_u32(&mut data, 1);
+        rtc.write(LEGACY_RTC_MAPPED_IO_START, RTCICR, &mut data);
+        // The interrupt line should be on.
+        assert!(rtc.interrupt.notifier(0).unwrap().read().unwrap() > 1);
+        let v_before = read_le_u32(&data[..]);
+
+        rtc.read(LEGACY_RTC_MAPPED_IO_START, RTCICR, &mut data);
+        let v = read_le_u32(&data[..]);
+        // ICR is a  write only register. Data received should stay equal to data sent.
+        assert_eq!(v, v_before);
+
+        // Attempts to turn off the RTC should not go through.
+        write_le_u32(&mut data, 0);
+        rtc.write(LEGACY_RTC_MAPPED_IO_START, RTCCR, &mut data);
+        rtc.read(LEGACY_RTC_MAPPED_IO_START, RTCCR, &mut data);
+        let v = read_le_u32(&data[..]);
+        assert_eq!(v, 1);
+
+        // Attempts to write beyond the writable space. Using here the space used to read
+        // the CID and PID from.
+        write_le_u32(&mut data, 0);
+        rtc.write(LEGACY_RTC_MAPPED_IO_START, AMBA_ID_LOW, &mut data);
+        // However, reading from the AMBA_ID_LOW should succeed upon read.
+
+        let mut data = [0; 4];
+        rtc.read(LEGACY_RTC_MAPPED_IO_START, AMBA_ID_LOW, &mut data);
+        let index = AMBA_ID_LOW + 3;
+        assert_eq!(data[0], PL031_ID[((index - AMBA_ID_LOW) >> 2) as usize]);
+    }
+
+    macro_rules! byte_order_test_read_write {
+        ($test_name: ident, $write_fn_name: ident, $read_fn_name: ident, $is_be: expr, $data_type: ty) => {
+            #[test]
+            fn $test_name() {
+                #[allow(overflowing_literals)]
+                let test_cases = [
+                    (
+                        0x0123_4567_89AB_CDEF as u64,
+                        [0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef],
+                    ),
+                    (
+                        0x0000_0000_0000_0000 as u64,
+                        [0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00],
+                    ),
+                    (
+                        0x1923_2345_ABF3_CCD4 as u64,
+                        [0x19, 0x23, 0x23, 0x45, 0xAB, 0xF3, 0xCC, 0xD4],
+                    ),
+                    (
+                        0x0FF0_0FF0_0FF0_0FF0 as u64,
+                        [0x0F, 0xF0, 0x0F, 0xF0, 0x0F, 0xF0, 0x0F, 0xF0],
+                    ),
+                    (
+                        0xFFFF_FFFF_FFFF_FFFF as u64,
+                        [0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF],
+                    ),
+                    (
+                        0x89AB_12D4_C2D2_09BB as u64,
+                        [0x89, 0xAB, 0x12, 0xD4, 0xC2, 0xD2, 0x09, 0xBB],
+                    ),
+                ];
+
+                let type_size = std::mem::size_of::<$data_type>();
+                for (test_val, v_arr) in &test_cases {
+                    let v = *test_val as $data_type;
+                    let cmp_iter: Box<dyn Iterator<Item = _>> = if $is_be {
+                        Box::new(v_arr[(8 - type_size)..].iter())
+                    } else {
+                        Box::new(v_arr.iter().rev())
+                    };
+                    // test write
+                    let mut write_arr = vec![Default::default(); type_size];
+                    $write_fn_name(&mut write_arr, v);
+                    for (cmp, cur) in cmp_iter.zip(write_arr.iter()) {
+                        assert_eq!(*cmp, *cur as u8)
+                    }
+                    // test read
+                    let read_val = $read_fn_name(&write_arr);
+                    assert_eq!(v, read_val);
+                }
+            }
+        };
+    }
+
+    byte_order_test_read_write!(test_le_u16, write_le_u16, read_le_u16, false, u16);
+    byte_order_test_read_write!(test_le_u32, write_le_u32, read_le_u32, false, u32);
+    byte_order_test_read_write!(test_le_u64, write_le_u64, read_le_u64, false, u64);
+    byte_order_test_read_write!(test_le_i32, write_le_i32, read_le_i32, false, i32);
+    byte_order_test_read_write!(test_be_u16, write_be_u16, read_be_u16, true, u16);
+    byte_order_test_read_write!(test_be_u32, write_be_u32, read_be_u32, true, u32);
+}

devices/src/legacy/serial.rs

@@ -6,10 +6,15 @@
 // found in the LICENSE-BSD-3-Clause file.
 
 use crate::BusDevice;
+use anyhow::anyhow;
 use std::collections::VecDeque;
 use std::sync::Arc;
 use std::{io, result};
 use vm_device::interrupt::InterruptSourceGroup;
+use vm_migration::{
+    Migratable, MigratableError, Pausable, Snapshot, SnapshotDataSection, Snapshottable,
+    Transportable,
+};
 use vmm_sys_util::errno::Result;
 
 const LOOP_SIZE: usize = 0x40;
@@ -55,6 +60,7 @@ const DEFAULT_BAUD_DIVISOR: u16 = 12; // 9600 bps
 /// This can optionally write the guest's output to a Write trait object. To send input to the
 /// guest, use `queue_input_bytes`.
 pub struct Serial {
+    id: String,
     interrupt_enable: u8,
     interrupt_identification: u8,
     interrupt: Arc<Box<dyn InterruptSourceGroup>>,
@@ -68,12 +74,27 @@ pub struct Serial {
     out: Option<Box<dyn io::Write + Send>>,
 }
 
+#[derive(Serialize, Deserialize)]
+pub struct SerialState {
+    interrupt_enable: u8,
+    interrupt_identification: u8,
+    line_control: u8,
+    line_status: u8,
+    modem_control: u8,
+    modem_status: u8,
+    scratch: u8,
+    baud_divisor: u16,
+    in_buffer: VecDeque<u8>,
+}
+
 impl Serial {
     pub fn new(
+        id: String,
         interrupt: Arc<Box<dyn InterruptSourceGroup>>,
         out: Option<Box<dyn io::Write + Send>>,
     ) -> Serial {
         Serial {
+            id,
             interrupt_enable: 0,
             interrupt_identification: DEFAULT_INTERRUPT_IDENTIFICATION,
             interrupt,
@@ -90,15 +111,16 @@ impl Serial {
 
     /// Constructs a Serial port ready for output.
     pub fn new_out(
+        id: String,
         interrupt: Arc<Box<dyn InterruptSourceGroup>>,
         out: Box<dyn io::Write + Send>,
     ) -> Serial {
-        Self::new(interrupt, Some(out))
+        Self::new(id, interrupt, Some(out))
     }
 
     /// Constructs a Serial port with no connected output.
-    pub fn new_sink(interrupt: Arc<Box<dyn InterruptSourceGroup>>) -> Serial {
-        Self::new(interrupt, None)
+    pub fn new_sink(id: String, interrupt: Arc<Box<dyn InterruptSourceGroup>>) -> Serial {
+        Self::new(id, interrupt, None)
     }
 
     /// Queues raw bytes for the guest to read and signals the interrupt if the line status would
@@ -194,6 +216,32 @@ impl Serial {
         }
         Ok(())
     }
+
+    fn state(&self) -> SerialState {
+        SerialState {
+            interrupt_enable: self.interrupt_enable,
+            interrupt_identification: self.interrupt_identification,
+            line_control: self.line_control,
+            line_status: self.line_status,
+            modem_control: self.modem_control,
+            modem_status: self.modem_status,
+            scratch: self.scratch,
+            baud_divisor: self.baud_divisor,
+            in_buffer: self.in_buffer.clone(),
+        }
+    }
+
+    fn set_state(&mut self, state: &SerialState) {
+        self.interrupt_enable = state.interrupt_enable;
+        self.interrupt_identification = state.interrupt_identification;
+        self.line_control = state.line_control;
+        self.line_status = state.line_status;
+        self.modem_control = state.modem_control;
+        self.modem_status = state.modem_status;
+        self.scratch = state.scratch;
+        self.baud_divisor = state.baud_divisor;
+        self.in_buffer = state.in_buffer.clone();
+    }
 }
 
 impl BusDevice for Serial {
@@ -236,6 +284,51 @@ impl BusDevice for Serial {
     }
 }
 
+impl Snapshottable for Serial {
+    fn id(&self) -> String {
+        self.id.clone()
+    }
+
+    fn snapshot(&self) -> std::result::Result<Snapshot, MigratableError> {
+        let snapshot =
+            serde_json::to_vec(&self.state()).map_err(|e| MigratableError::Snapshot(e.into()))?;
+
+        let mut serial_snapshot = Snapshot::new(self.id.as_str());
+        serial_snapshot.add_data_section(SnapshotDataSection {
+            id: format!("{}-section", self.id),
+            snapshot,
+        });
+
+        Ok(serial_snapshot)
+    }
+
+    fn restore(&mut self, snapshot: Snapshot) -> std::result::Result<(), MigratableError> {
+        if let Some(serial_section) = snapshot.snapshot_data.get(&format!("{}-section", self.id)) {
+            let serial_state = match serde_json::from_slice(&serial_section.snapshot) {
+                Ok(state) => state,
+                Err(error) => {
+                    return Err(MigratableError::Restore(anyhow!(
+                        "Could not deserialize SERIAL {}",
+                        error
+                    )))
+                }
+            };
+
+            self.set_state(&serial_state);
+
+            return Ok(());
+        }
+
+        Err(MigratableError::Restore(anyhow!(
+            "Could not find the serial snapshot section"
+        )))
+    }
+}
+
+impl Pausable for Serial {}
+impl Transportable for Serial {}
+impl Migratable for Serial {}
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -244,6 +337,8 @@ mod tests {
     use vm_device::interrupt::{InterruptIndex, InterruptSourceConfig};
     use vmm_sys_util::eventfd::EventFd;
 
+    const SERIAL_NAME: &str = "serial";
+
     struct TestInterrupt {
         event_fd: EventFd,
     }
@@ -294,6 +389,7 @@ mod tests {
         let intr_evt = EventFd::new(0).unwrap();
         let serial_out = SharedBuffer::new();
         let mut serial = Serial::new_out(
+            String::from(SERIAL_NAME),
             Arc::new(Box::new(TestInterrupt::new(intr_evt.try_clone().unwrap()))),
             Box::new(serial_out.clone()),
         );
@@ -313,6 +409,7 @@ mod tests {
         let intr_evt = EventFd::new(0).unwrap();
         let serial_out = SharedBuffer::new();
         let mut serial = Serial::new_out(
+            String::from(SERIAL_NAME),
             Arc::new(Box::new(TestInterrupt::new(intr_evt.try_clone().unwrap()))),
             Box::new(serial_out.clone()),
         );
@@ -350,9 +447,10 @@ mod tests {
     #[test]
     fn serial_thr() {
         let intr_evt = EventFd::new(0).unwrap();
-        let mut serial = Serial::new_sink(Arc::new(Box::new(TestInterrupt::new(
-            intr_evt.try_clone().unwrap(),
-        ))));
+        let mut serial = Serial::new_sink(
+            String::from(SERIAL_NAME),
+            Arc::new(Box::new(TestInterrupt::new(intr_evt.try_clone().unwrap()))),
+        );
 
         // write 1 to the interrupt event fd, so that read doesn't block in case the event fd
         // counter doesn't change (for 0 it blocks)
@@ -371,9 +469,10 @@ mod tests {
     #[test]
     fn serial_dlab() {
         let intr_evt = EventFd::new(0).unwrap();
-        let mut serial = Serial::new_sink(Arc::new(Box::new(TestInterrupt::new(
-            intr_evt.try_clone().unwrap(),
-        ))));
+        let mut serial = Serial::new_sink(
+            String::from(SERIAL_NAME),
+            Arc::new(Box::new(TestInterrupt::new(intr_evt.try_clone().unwrap()))),
+        );
 
         serial.write(0, LCR as u64, &[LCR_DLAB_BIT as u8]);
         serial.write(0, DLAB_LOW as u64, &[0x12 as u8]);
@@ -391,9 +490,10 @@ mod tests {
     #[test]
     fn serial_modem() {
         let intr_evt = EventFd::new(0).unwrap();
-        let mut serial = Serial::new_sink(Arc::new(Box::new(TestInterrupt::new(
-            intr_evt.try_clone().unwrap(),
-        ))));
+        let mut serial = Serial::new_sink(
+            String::from(SERIAL_NAME),
+            Arc::new(Box::new(TestInterrupt::new(intr_evt.try_clone().unwrap()))),
+        );
 
         serial.write(0, MCR as u64, &[MCR_LOOP_BIT as u8]);
         serial.write(0, DATA as u64, &['a' as u8]);
@@ -416,9 +516,10 @@ mod tests {
     #[test]
     fn serial_scratch() {
         let intr_evt = EventFd::new(0).unwrap();
-        let mut serial = Serial::new_sink(Arc::new(Box::new(TestInterrupt::new(
-            intr_evt.try_clone().unwrap(),
-        ))));
+        let mut serial = Serial::new_sink(
+            String::from(SERIAL_NAME),
+            Arc::new(Box::new(TestInterrupt::new(intr_evt.try_clone().unwrap()))),
+        );
 
         serial.write(0, SCR as u64, &[0x12 as u8]);
 

devices/src/lib.rs

@@ -6,6 +6,7 @@
 // found in the LICENSE-BSD-3-Clause file.
 
 //! Emulates virtual and hardware devices.
+extern crate anyhow;
 #[macro_use]
 extern crate bitflags;
 extern crate byteorder;
@@ -15,9 +16,14 @@ extern crate libc;
 extern crate log;
 #[cfg(feature = "acpi")]
 extern crate acpi_tables;
+extern crate serde;
 extern crate vm_device;
 extern crate vm_memory;
+extern crate vm_migration;
 extern crate vmm_sys_util;
+#[macro_use]
+extern crate serde_derive;
+extern crate serde_json;
 
 use std::fs::File;
 use std::io;
@@ -25,6 +31,10 @@ use std::io;
 #[cfg(feature = "acpi")]
 mod acpi;
 mod bus;
+#[cfg(target_arch = "aarch64")]
+pub mod gic;
+pub mod interrupt_controller;
+#[cfg(target_arch = "x86_64")]
 pub mod ioapic;
 pub mod legacy;
 

docs/api.md

@@ -81,8 +81,8 @@ Shut the VMM down                   | `/vmm.shutdown` | N/A          | N/A
 Action                             | Endpoint            | Request Body              | Response Body     | Prerequisites
 -----------------------------------|---------------------|---------------------------|-------------------|---------------------------
 Create the VM                      | `/vm.create`        | `/schemas/VmConfig`       | N/A               | The VM is not created yet
-Delete the VM                      | `/vm.delete`        | N/A                       | N/A               | The VM is created but not booted
-Boot the VM                        | `/vm.boot`          | N/A                       | N/A               | The VM is created
+Delete the VM                      | `/vm.delete`        | N/A                       | N/A               | N/A
+Boot the VM                        | `/vm.boot`          | N/A                       | N/A               | The VM is created but not booted
 Shut the VM down                   | `/vm.shutdown`      | N/A                       | N/A               | The VM is booted
 Reboot the VM                      | `/vm.reboot`        | N/A                       | N/A               | The VM is booted
 Pause the VM                       | `/vm.pause`         | N/A                       | N/A               | The VM is booted
@@ -92,6 +92,11 @@ Remove memory from the VM          | `/vm.resize`        | `/schemas/VmResize`
 Dump the VM information            | `/vm.info`          | N/A                       | `/schemas/VmInfo` | The VM is created
 Add VFIO PCI device to the VM      | `/vm.add-device`    | `/schemas/VmAddDevice`    | N/A               | The VM is booted
 Remove VFIO PCI device from the VM | `/vm.remove-device` | `/schemas/VmRemoveDevice` | N/A               | The VM is booted
+Add disk device to the VM          | `/vm.add-disk`      | `/schemas/DiskConfig`     | N/A               | The VM is booted
+Add fs device to the VM            | `/vm.add-fs`        | `/schemas/FsConfig`       | N/A               | The VM is booted
+Add pmem device to the VM          | `/vm.add-pmem`      | `/schemas/PmemConfig`     | N/A               | The VM is booted
+Add network device to the VM       | `/vm.add-net`       | `/schemas/NetConfig`      | N/A               | The VM is booted
+Add vsock device to the VM         | `/vm.add-vsock`     | `/schemas/VsockConfig`    | N/A               | The VM is booted
 
 ### REST API Examples
 
@@ -370,4 +375,3 @@ APIs work together, let's look at a complete VM creation flow, from the
    user. This is abstracted by the
    [micro_http](https://github.com/firecracker-microvm/firecracker/tree/master/src/micro_http)
    crate.
-   

docs/arm64.md

@@ -0,0 +1,64 @@
+# How to build and run Cloud-hypervisor on Arm64
+
+Cloud-hypervisor is partially enabled on AArch64 architecture.
+Although all features are not ready yet, you can begin to test Cloud-hypervisor on a Arm64 host by following this guide.
+
+## Prerequisites
+
+On Arm64 machines, Cloud-hypervisor depends on an external library `libfdt-dev` for generating Flatted Device Tree (FDT).
+
+The long-term plan is to replace `libfdt-dev` with some pure-Rust component to get rid of such dependency.
+
+```bash
+sudo apt-get update
+sudo apt-get install libfdt-dev
+```
+
+## Build
+
+Before building, a hack trick need to be performed to get rid of some build error in vmm component. See [this](https://github.com/cloud-hypervisor/kvm-bindings/pull/1) for more info about this temporary workaround.
+
+```bash
+sed -i 's/"with-serde",\ //g' vmm/Cargo.toml
+```
+
+The support of AArch64 is in very early stage, only Virtio devices with MMIO tranport is available.
+
+```bash
+cargo build --no-default-features --features "mmio"
+```
+
+## Image
+
+Download kernel binary and rootfs image from AWS.
+
+```bash
+wget https://s3.amazonaws.com/spec.ccfc.min/img/aarch64/ubuntu_with_ssh/fsfiles/xenial.rootfs.ext4 -O rootfs.img
+wget https://s3.amazonaws.com/spec.ccfc.min/img/aarch64/ubuntu_with_ssh/kernel/vmlinux.bin -O kernel.bin
+```
+
+## Containerized build
+
+If you want to build and test Cloud Hypervisor without having to install all the required dependencies, you can also turn to the development script: dev_cli.sh.
+
+To build the development container:
+
+```bash
+./scripts/dev_cli.sh build-container
+```
+
+To build Cloud-hypervisor in the container:
+
+```bash
+./scripts/dev_cli.sh build
+```
+
+## Run
+
+Assuming you have built Cloud-hypervisor with the development container, a VM can be started with command:
+
+```bash
+sudo target/debug/cloud-hypervisor --kernel kernel.bin --disk path=rootfs.ext4 --cmdline "keep_bootcon console=hvc0 reboot=k panic=1 pci=off root=/dev/vda rw" --cpus boot=4 --memory size=512M --seccomp false --serial file=serial.log --log-file log.log -vvv
+```
+
+If the build was done out of the container, replace the binary path with `build/cargo_target/aarch64-unknown-linux-gnu/debug/cloud-hypervisor`.

docs/fs.md

@@ -60,20 +60,20 @@ Assuming you have `clear-kvm.img` and `custom-vmlinux.bin` on your system, here
     --disk path=clear-kvm.img \
     --kernel custom-vmlinux.bin \
     --cmdline "console=ttyS0 reboot=k panic=1 nomodules root=/dev/vda3" \
-    --fs tag=myfs,sock=/tmp/virtiofs,num_queues=1,queue_size=512
+    --fs tag=myfs,socket=/tmp/virtiofs,num_queues=1,queue_size=512
 ```
 
 By default, DAX is enabled with a cache window of 8GiB. You can specify a custom size (let's say 4GiB for this example) for the cache by explicitly setting DAX and the cache size:
 
 ```bash
---fs tag=virtiofs,sock=/tmp/virtiofs,num_queues=1,queue_size=512,dax=on,cache_size=4G
+--fs tag=virtiofs,socket=/tmp/virtiofs,num_queues=1,queue_size=512,dax=on,cache_size=4G
 
 ```
 
 In case you don't want to use a shared window of cache to pass the shared files content, this means you will have to explicitly disable DAX with `dax=off`. Note that in this case, the `cache_size` parameter will be ignored.
 
 ```bash
---fs tag=virtiofs,sock=/tmp/virtiofs,num_queues=1,queue_size=512,dax=off
+--fs tag=virtiofs,socket=/tmp/virtiofs,num_queues=1,queue_size=512,dax=off
 
 ```
 

docs/vhost-user-blk-testing.md

@@ -0,0 +1,104 @@
+# How to test vhost-user-blk with SPDK
+
+The purpose of this document is to illustrate how to test vhost-user-blk in Cloud Hypervisor with SPDK as the backend.
+
+## Framework
+
+It's a simple test to validate the block read/write between VM and block backend.
+```
+             +----+----------+          +-------------+-----------+
+             |    |          |          |             |           |
+             |    |vhost-user|----------| vhost-user  |    dpdk   |
+             |    |blk device|          | port 1      |           |
+             |    |          |          |             |           |
+             |    +----------+          +-------------+-----------+
+             |               |          |                         |
+             |      vm       |          |           spdk          |
+             |               |          |                         |
+          +--+----------------------------------------------------+--+
+          |  |                  hugepages                         |  |
+          |  +----------------------------------------------------+  |
+          |                                                          |
+          |                       host                               |
+          |                                                          |
+          +----------------------------------------------------------+
+```
+## Prerequisites
+
+Prior to running the test, the following steps need to be performed.
+- Enable hugepages
+- Install SPDK
+
+Here are some good references for detailing them.
+- spdk
+	* https://spdk.io/doc/
+
+## Test environment
+
+The below test environment is based on ubuntu release(16.04.1 LTS), as for other system, please check related document.
+The test runs with multiple queue (MQ) support enabled, using 4 queues defined for both SPDK and the virtual machine.
+Here are the details on how the test can be run.
+
+### The hugepages settings in host linux
+Add "default_hugepagesz=1G hugepagesz=1G hugepages=2" into host linux cmdline.
+As for how to change Ubuntu linux cmdline in grub file, please ref below link:
+https://www.ostechnix.com/configure-grub-2-boot-loader-settings-ubuntu-16-04/
+reboot Ubuntu
+sudo mount -t hugetlbfs -o pagesize=1G none /dev/hugepages
+
+### Download the SPDK code
+git clone https://github.com/spdk/spdk
+cd spdk
+git submodule update --init
+
+### Create the build dep
+./scripts/pkgdep.sh
+
+### Build spdk
+./configure
+make
+
+### Set the SPDk environment
+sudo HUGEMEM=2048 scripts/setup.sh
+sudo ./app/vhost/vhost -S /var/tmp -s 1024 -m 0x3 &
+
+### Create 512M block device
+sudo scripts/rpc.py bdev_malloc_create 512 512 -b Malloc0
+sudo scripts/rpc.py vhost_create_blk_controller --cpumask 0x1 vhost.1 Malloc0
+
+_Launch the VM_
+
+VMs run in client mode. They connect to the socket created by the `dpdkvhostuser` in the SPDK backend.
+```bash
+# From the test terminal. We need to create one vhost-user-blk device for the --disk.
+./cloud-hypervisor \
+        --cpus boot=4 \
+        --memory size=1024M,file=/dev/hugepages \
+        --kernel linux/arch/x86/boot/compressed/vmlinux.bin \
+        --cmdline "console=ttyS0 reboot=k panic=1 nomodules i8042.noaux i8042.nomux i8042.nopnp i8042.dumbkbd root=/dev/vda3 iommu=off" \
+        --disk "path=images/clear-kvm.img" "num_queues=4,queue_size=128,vhost_user=true,socket=/var/tmp/vhost.1" \
+        --console off \
+        --serial tty \
+        --rng
+```
+
+```bash
+# How to test the vhost-user-blk device with SPDK backend
+login in guest
+
+# Use lsblk command to find out vhost-user-blk device
+lsblk
+NAME   MAJ:MIN RM  SIZE RO TYPE MOUNTPOINT
+vda    253:0    0  8.5G  0 disk
+├─vda1 253:1    0  511M  0 part
+├─vda2 253:2    0   32M  0 part [SWAP]
+└─vda3 253:3    0    8G  0 part /
+vdb    253:16   0  512M  0 disk
+
+The vhost-user-blk device is /dev/vdb
+
+# How to do simple read/write test
+dd if=/dev/vdb of=/dev/null bs=2M iflag=direct
+dd of=/dev/vdb if=/dev/zero bs=2M oflag=direct count=256
+
+If you want to do fio test, please install fio binary into guest. The detailed info is not listed here.

net_util/Cargo.toml

@@ -4,14 +4,14 @@ version = "0.1.0"
 authors = ["The Chromium OS Authors"]
 
 [dependencies]
-libc = "0.2.68"
+libc = "0.2.71"
 rand = "0.7.3"
-serde = "1.0.105"
+serde = "1.0.111"
 vmm-sys-util = ">=0.3.1"
 
 net_gen = { path = "../net_gen" }
 
 [dev-dependencies]
 lazy_static = "1.3.0"
-pnet = "0.25.0"
-serde_json = "1.0.48"
+pnet = "0.26.0"
+serde_json = "1.0.55"

net_util/src/mac.rs

@@ -9,6 +9,7 @@ use rand::Rng;
 use std::fmt;
 use std::io;
 use std::result::Result;
+use std::str::FromStr;
 
 use serde::de::{Deserialize, Deserializer, Error};
 use serde::ser::{Serialize, Serializer};
@@ -120,6 +121,18 @@ impl<'de> Deserialize<'de> for MacAddr {
     }
 }
 
+pub enum MacAddrParseError {
+    InvalidValue(String),
+}
+
+impl FromStr for MacAddr {
+    type Err = MacAddrParseError;
+
+    fn from_str(s: &str) -> std::result::Result<Self, Self::Err> {
+        MacAddr::parse_str(s).map_err(|_| MacAddrParseError::InvalidValue(s.to_owned()))
+    }
+}
+
 #[cfg(test)]
 mod tests {
     extern crate serde_json;

net_util/src/tap.rs

@@ -5,15 +5,14 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the THIRD-PARTY file.
 
+use super::{create_sockaddr, create_socket, Error as NetUtilError, MacAddr};
+use mac::MAC_ADDR_LEN;
+use net_gen;
 use std::fs::File;
 use std::io::{Error as IoError, Read, Result as IoResult, Write};
 use std::net;
 use std::os::raw::*;
 use std::os::unix::io::{AsRawFd, FromRawFd, RawFd};
-
-use super::{create_sockaddr, create_socket, Error as NetUtilError};
-use libc;
-use net_gen;
 use vmm_sys_util::ioctl::{ioctl_with_mut_ref, ioctl_with_ref, ioctl_with_val};
 
 #[derive(Debug)]
@@ -31,6 +30,8 @@ pub enum Error {
     /// Failed to create a socket.
     NetUtil(NetUtilError),
     InvalidIfname,
+    /// Error parsing MAC data
+    MacParsing(()),
 }
 
 pub type Result<T> = ::std::result::Result<T, Error>;
@@ -176,6 +177,72 @@ impl Tap {
         Ok(())
     }
 
+    /// Set mac addr for tap interface.
+    pub fn set_mac_addr(&self, addr: MacAddr) -> Result<()> {
+        // Checking if the mac address already matches the desired one
+        // is useful to avoid making the "set ioctl" in the case where
+        // the VMM is running without the privilege to do that.
+        // In practice this comes from a reboot after the configuration
+        // has been update with the kernel generated address.
+        if self.get_mac_addr()? == addr {
+            return Ok(());
+        }
+
+        let sock = create_socket().map_err(Error::NetUtil)?;
+
+        let mut ifreq = self.get_ifreq();
+
+        // ioctl is safe. Called with a valid sock fd, and we check the return.
+        #[allow(clippy::cast_lossless)]
+        let ret =
+            unsafe { ioctl_with_ref(&sock, net_gen::sockios::SIOCGIFHWADDR as c_ulong, &ifreq) };
+        if ret < 0 {
+            return Err(Error::IoctlError(IoError::last_os_error()));
+        }
+        // We only access one field of the ifru union, hence this is safe.
+        unsafe {
+            let ifru_hwaddr = ifreq.ifr_ifru.ifru_hwaddr.as_mut();
+            for (i, v) in addr.get_bytes().iter().enumerate() {
+                ifru_hwaddr.sa_data[i] = *v as c_char;
+            }
+        }
+
+        // ioctl is safe. Called with a valid sock fd, and we check the return.
+        #[allow(clippy::cast_lossless)]
+        let ret =
+            unsafe { ioctl_with_ref(&sock, net_gen::sockios::SIOCSIFHWADDR as c_ulong, &ifreq) };
+        if ret < 0 {
+            return Err(Error::IoctlError(IoError::last_os_error()));
+        }
+
+        Ok(())
+    }
+
+    /// Get mac addr for tap interface.
+    pub fn get_mac_addr(&self) -> Result<MacAddr> {
+        let sock = create_socket().map_err(Error::NetUtil)?;
+
+        let ifreq = self.get_ifreq();
+
+        // ioctl is safe. Called with a valid sock fd, and we check the return.
+        #[allow(clippy::cast_lossless)]
+        let ret =
+            unsafe { ioctl_with_ref(&sock, net_gen::sockios::SIOCGIFHWADDR as c_ulong, &ifreq) };
+        if ret < 0 {
+            return Err(Error::IoctlError(IoError::last_os_error()));
+        }
+
+        // We only access one field of the ifru union, hence this is safe.
+        let addr = unsafe {
+            let ifru_hwaddr = ifreq.ifr_ifru.ifru_hwaddr.as_ref();
+            MacAddr::from_bytes(
+                &*(&ifru_hwaddr.sa_data[0..MAC_ADDR_LEN] as *const _ as *const [u8]),
+            )
+            .map_err(Error::MacParsing)?
+        };
+        Ok(addr)
+    }
+
     /// Set the netmask for the subnet that the tap interface will exist on.
     pub fn set_netmask(&self, netmask: net::Ipv4Addr) -> Result<()> {
         let sock = create_socket().map_err(Error::NetUtil)?;
@@ -219,9 +286,26 @@ impl Tap {
 
         let mut ifreq = self.get_ifreq();
 
+        #[allow(clippy::cast_lossless)]
+        let ret =
+            unsafe { ioctl_with_ref(&sock, net_gen::sockios::SIOCGIFFLAGS as c_ulong, &ifreq) };
+        if ret < 0 {
+            return Err(Error::IoctlError(IoError::last_os_error()));
+        }
+
+        // If TAP device is already up don't try and enable it
+        let ifru_flags = unsafe { ifreq.ifr_ifru.ifru_flags.as_ref() };
+        if ifru_flags
+            & (net_gen::net_device_flags_IFF_UP | net_gen::net_device_flags_IFF_RUNNING) as i16
+            == (net_gen::net_device_flags_IFF_UP | net_gen::net_device_flags_IFF_RUNNING) as i16
+        {
+            return Ok(());
+        }
+
         // We only access one field of the ifru union, hence this is safe.
         unsafe {
             let ifru_flags = ifreq.ifr_ifru.ifru_flags.as_mut();
+
             *ifru_flags =
                 (net_gen::net_device_flags_IFF_UP | net_gen::net_device_flags_IFF_RUNNING) as i16;
         }
@@ -443,7 +527,7 @@ mod tests {
         let interface = interfaces.into_iter().find(interface_name_matches).unwrap();
 
         if let Ok(Ethernet(tx, rx)) = datalink::channel(&interface, Default::default()) {
-            (interface.mac_address(), tx, rx)
+            (interface.mac.unwrap(), tx, rx)
         } else {
             panic!("datalink channel error or unhandled channel type");
         }

pci/Cargo.toml

@@ -5,10 +5,23 @@ authors = ["Samuel Ortiz <sameo@linux.intel.com>"]
 edition = "2018"
 
 [dependencies]
-vm-allocator = { path = "../vm-allocator" }
+anyhow = "1.0"
 byteorder = "1.3.4"
 devices = { path = "../devices" }
-libc = "0.2.68"
+kvm-bindings = { git = "https://github.com/cloud-hypervisor/kvm-bindings", branch = "ch" }
+kvm-ioctls = { git = "https://github.com/cloud-hypervisor/kvm-ioctls", branch = "ch" }
+vfio-ioctls = { git = "https://github.com/cloud-hypervisor/vfio-ioctls", branch = "ch" }
+vmm-sys-util = ">=0.3.1"
+libc = "0.2.71"
 log = "0.4.8"
+serde = {version = ">=1.0.27", features = ["rc"] }
+serde_derive = ">=1.0.27"
+serde_json = ">=1.0.9"
+vm-allocator = { path = "../vm-allocator" }
 vm-device = { path = "../vm-device" }
-vm-memory = "0.1.0"
+vm-memory = "0.2.1"
+vm-migration = { path = "../vm-migration" }
+
+[dependencies.vfio-bindings]
+git = "https://github.com/rust-vmm/vfio-bindings"
+features = ["fam-wrappers"]

pci/src/bus.rs

@@ -8,7 +8,6 @@ use crate::configuration::{
 use crate::device::{DeviceRelocation, Error as PciDeviceError, PciDevice};
 use byteorder::{ByteOrder, LittleEndian};
 use devices::BusDevice;
-use std;
 use std::any::Any;
 use std::collections::HashMap;
 use std::ops::DerefMut;
@@ -34,6 +33,8 @@ pub enum PciRootError {
     NoPciDeviceSlotAvailable,
     /// Invalid PCI device identifier provided.
     InvalidPciDeviceSlot(usize),
+    /// Valid PCI device identifier but already used.
+    AlreadyInUsePciDeviceSlot(usize),
 }
 pub type Result<T> = std::result::Result<T, PciRootError>;
 
@@ -53,6 +54,7 @@ impl PciRoot {
                 config: PciConfiguration::new(
                     VENDOR_ID_INTEL,
                     DEVICE_ID_INTEL_VIRT_PCIE_HOST,
+                    0,
                     PciClassCode::BridgeDevice,
                     &PciBridgeSubclass::HostBridge,
                     None,
@@ -154,6 +156,19 @@ impl PciBus {
         Err(PciRootError::NoPciDeviceSlotAvailable)
     }
 
+    pub fn get_device_id(&mut self, id: usize) -> Result<()> {
+        if id < NUM_DEVICE_IDS {
+            if !self.device_ids[id] {
+                self.device_ids[id] = true;
+                Ok(())
+            } else {
+                Err(PciRootError::AlreadyInUsePciDeviceSlot(id))
+            }
+        } else {
+            Err(PciRootError::InvalidPciDeviceSlot(id))
+        }
+    }
+
     pub fn put_device_id(&mut self, id: usize) -> Result<()> {
         if id < NUM_DEVICE_IDS {
             self.device_ids[id] = false;
@@ -239,7 +254,10 @@ impl PciConfigIo {
                     device.deref_mut(),
                     params.region_type,
                 ) {
-                    error!("Failed moving device BAR: {}", e);
+                    error!(
+                        "Failed moving device BAR: {}: 0x{:x}->0x{:x}(0x{:x})",
+                        e, params.old_base, params.new_base, params.len
+                    );
                 }
             }
 
@@ -355,7 +373,10 @@ impl PciConfigMmio {
                     device.deref_mut(),
                     params.region_type,
                 ) {
-                    error!("Failed moving device BAR: {}", e);
+                    error!(
+                        "Failed moving device BAR: {}: 0x{:x}->0x{:x}(0x{:x})",
+                        e, params.old_base, params.new_base, params.len
+                    );
                 }
             }
 

pci/src/configuration.rs

@@ -2,12 +2,13 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE-BSD-3-Clause file.
 
-use std::sync::{Arc, Mutex};
-
 use crate::device::BarReprogrammingParams;
 use crate::{MsixConfig, PciInterruptPin};
+use anyhow::anyhow;
 use byteorder::{ByteOrder, LittleEndian};
 use std::fmt::{self, Display};
+use std::sync::{Arc, Mutex};
+use vm_migration::{MigratableError, Pausable, Snapshot, SnapshotDataSection, Snapshottable};
 
 // The number of 32bit registers in the config space, 4096 bytes.
 const NUM_CONFIGURATION_REGISTERS: usize = 1024;
@@ -244,6 +245,53 @@ pub trait PciCapability {
     fn id(&self) -> PciCapabilityID;
 }
 
+fn encode_32_bits_bar_size(bar_size: u32) -> Option<u32> {
+    if bar_size > 0 {
+        return Some(!(bar_size - 1));
+    }
+    None
+}
+
+fn decode_32_bits_bar_size(bar_size: u32) -> Option<u32> {
+    if bar_size > 0 {
+        return Some(!bar_size + 1);
+    }
+    None
+}
+
+fn encode_64_bits_bar_size(bar_size: u64) -> Option<(u32, u32)> {
+    if bar_size > 0 {
+        let result = !(bar_size - 1);
+        let result_hi = (result >> 32) as u32;
+        let result_lo = (result & 0xffff_ffff) as u32;
+        return Some((result_hi, result_lo));
+    }
+    None
+}
+
+fn decode_64_bits_bar_size(bar_size_hi: u32, bar_size_lo: u32) -> Option<u64> {
+    let bar_size: u64 = ((bar_size_hi as u64) << 32) | (bar_size_lo as u64);
+    if bar_size > 0 {
+        return Some(!bar_size + 1);
+    }
+    None
+}
+
+#[derive(Serialize, Deserialize)]
+struct PciConfigurationState {
+    registers: Vec<u32>,
+    writable_bits: Vec<u32>,
+    bar_addr: Vec<u32>,
+    bar_size: Vec<u32>,
+    bar_used: Vec<bool>,
+    bar_type: Vec<Option<PciBarRegionType>>,
+    rom_bar_addr: u32,
+    rom_bar_size: u32,
+    rom_bar_used: bool,
+    last_capability: Option<(usize, usize)>,
+    msix_cap_reg_idx: Option<usize>,
+}
+
 /// Contains the configuration space of a PCI node.
 /// See the [specification](https://en.wikipedia.org/wiki/PCI_configuration_space).
 /// The configuration space is accessed with DWORD reads and writes from the guest.
@@ -264,7 +312,7 @@ pub struct PciConfiguration {
 }
 
 /// See pci_regs.h in kernel
-#[derive(Copy, Clone, PartialEq)]
+#[derive(Copy, Clone, PartialEq, Serialize, Deserialize)]
 pub enum PciBarRegionType {
     Memory32BitRegion = 0,
     IORegion = 0x01,
@@ -297,6 +345,10 @@ pub enum Error {
     CapabilityEmpty,
     CapabilityLengthInvalid(usize),
     CapabilitySpaceFull(usize),
+    Decode32BarSize,
+    Decode64BarSize,
+    Encode32BarSize,
+    Encode64BarSize,
     RomBarAddressInvalid(u64, u64),
     RomBarInUse(usize),
     RomBarInvalid(usize),
@@ -324,6 +376,10 @@ impl Display for Error {
             CapabilityEmpty => write!(f, "empty capabilities are invalid"),
             CapabilityLengthInvalid(l) => write!(f, "Invalid capability length {}", l),
             CapabilitySpaceFull(s) => write!(f, "capability of size {} doesn't fit", s),
+            Decode32BarSize => write!(f, "failed to decode 32 bits BAR size"),
+            Decode64BarSize => write!(f, "failed to decode 64 bits BAR size"),
+            Encode32BarSize => write!(f, "failed to encode 32 bits BAR size"),
+            Encode64BarSize => write!(f, "failed to encode 64 bits BAR size"),
             RomBarAddressInvalid(a, s) => write!(f, "address {} size {} too big", a, s),
             RomBarInUse(b) => write!(f, "rom bar {} already used", b),
             RomBarInvalid(b) => write!(f, "rom bar {} invalid, max {}", b, NUM_BAR_REGS - 1),
@@ -337,6 +393,7 @@ impl PciConfiguration {
     pub fn new(
         vendor_id: u16,
         device_id: u16,
+        revision_id: u8,
         class_code: PciClassCode,
         subclass: &dyn PciSubclass,
         programming_interface: Option<&dyn PciProgrammingInterface>,
@@ -359,7 +416,8 @@ impl PciConfiguration {
         };
         registers[2] = u32::from(class_code.get_register_value()) << 24
             | u32::from(subclass.get_register_value()) << 16
-            | u32::from(pi) << 8;
+            | u32::from(pi) << 8
+            | u32::from(revision_id);
         writable_bits[3] = 0x0000_00ff; // Cacheline size (r/w)
         match header_type {
             PciHeaderType::Device => {
@@ -390,6 +448,37 @@ impl PciConfiguration {
         }
     }
 
+    fn state(&self) -> PciConfigurationState {
+        PciConfigurationState {
+            registers: self.registers.to_vec(),
+            writable_bits: self.writable_bits.to_vec(),
+            bar_addr: self.bar_addr.to_vec(),
+            bar_size: self.bar_size.to_vec(),
+            bar_used: self.bar_used.to_vec(),
+            bar_type: self.bar_type.to_vec(),
+            rom_bar_addr: self.rom_bar_addr,
+            rom_bar_size: self.rom_bar_size,
+            rom_bar_used: self.rom_bar_used,
+            last_capability: self.last_capability,
+            msix_cap_reg_idx: self.msix_cap_reg_idx,
+        }
+    }
+
+    fn set_state(&mut self, state: &PciConfigurationState) {
+        self.registers.clone_from_slice(state.registers.as_slice());
+        self.writable_bits
+            .clone_from_slice(state.writable_bits.as_slice());
+        self.bar_addr.clone_from_slice(state.bar_addr.as_slice());
+        self.bar_size.clone_from_slice(state.bar_size.as_slice());
+        self.bar_used.clone_from_slice(state.bar_used.as_slice());
+        self.bar_type.clone_from_slice(state.bar_type.as_slice());
+        self.rom_bar_addr = state.rom_bar_addr;
+        self.rom_bar_size = state.rom_bar_size;
+        self.rom_bar_used = state.rom_bar_used;
+        self.last_capability = state.last_capability;
+        self.msix_cap_reg_idx = state.msix_cap_reg_idx;
+    }
+
     /// Reads a 32bit register from `reg_idx` in the register map.
     pub fn read_reg(&self, reg_idx: usize) -> u32 {
         *(self.registers.get(reg_idx).unwrap_or(&0xffff_ffff))
@@ -403,14 +492,14 @@ impl PciConfiguration {
             // Handle very specific case where the BAR is being written with
             // all 1's to retrieve the BAR size during next BAR reading.
             if value == 0xffff_ffff {
-                mask = self.bar_size[reg_idx - 4];
+                mask &= self.bar_size[reg_idx - 4];
             }
         } else if reg_idx == ROM_BAR_REG {
             // Handle very specific case where the BAR is being written with
             // all 1's on bits 31-11 to retrieve the BAR size during next BAR
             // reading.
             if value & ROM_BAR_ADDR_MASK == ROM_BAR_ADDR_MASK {
-                mask = self.rom_bar_size;
+                mask &= self.rom_bar_size;
             }
         }
 
@@ -494,6 +583,11 @@ impl PciConfiguration {
                 if end_addr > u64::from(u32::max_value()) {
                     return Err(Error::BarAddressInvalid(config.addr, config.size));
                 }
+
+                // Encode the BAR size as expected by the software running in
+                // the guest.
+                self.bar_size[config.reg_idx] =
+                    encode_32_bits_bar_size(config.size as u32).ok_or(Error::Encode32BarSize)?;
             }
             PciBarRegionType::Memory64BitRegion => {
                 if config.reg_idx + 1 >= NUM_BAR_REGS {
@@ -508,10 +602,16 @@ impl PciConfiguration {
                     return Err(Error::BarInUse64(config.reg_idx));
                 }
 
+                // Encode the BAR size as expected by the software running in
+                // the guest.
+                let (bar_size_hi, bar_size_lo) =
+                    encode_64_bits_bar_size(config.size).ok_or(Error::Encode64BarSize)?;
+
                 self.registers[bar_idx + 1] = (config.addr >> 32) as u32;
                 self.writable_bits[bar_idx + 1] = 0xffff_ffff;
                 self.bar_addr[config.reg_idx + 1] = self.registers[bar_idx + 1];
-                self.bar_size[config.reg_idx + 1] = (config.size >> 32) as u32;
+                self.bar_size[config.reg_idx] = bar_size_lo;
+                self.bar_size[config.reg_idx + 1] = bar_size_hi;
                 self.bar_used[config.reg_idx + 1] = true;
             }
         }
@@ -527,7 +627,6 @@ impl PciConfiguration {
         self.registers[bar_idx] = ((config.addr as u32) & mask) | lower_bits;
         self.writable_bits[bar_idx] = mask;
         self.bar_addr[config.reg_idx] = self.registers[bar_idx];
-        self.bar_size[config.reg_idx] = config.size as u32;
         self.bar_used[config.reg_idx] = true;
         self.bar_type[config.reg_idx] = Some(config.region_type);
         Ok(config.reg_idx)
@@ -559,7 +658,8 @@ impl PciConfiguration {
         self.registers[config.reg_idx] = (config.addr as u32) | active;
         self.writable_bits[config.reg_idx] = ROM_BAR_ADDR_MASK;
         self.rom_bar_addr = self.registers[config.reg_idx];
-        self.rom_bar_size = config.size as u32;
+        self.rom_bar_size =
+            encode_32_bits_bar_size(config.size as u32).ok_or(Error::Encode32BarSize)?;
         self.rom_bar_used = true;
         Ok(config.reg_idx)
     }
@@ -697,7 +797,11 @@ impl PciConfiguration {
                             );
                             let old_base = u64::from(self.bar_addr[bar_idx] & mask);
                             let new_base = u64::from(value & mask);
-                            let len = u64::from(self.bar_size[bar_idx]);
+                            let len = u64::from(
+                                decode_32_bits_bar_size(self.bar_size[bar_idx])
+                                    .ok_or(Error::Decode32BarSize)
+                                    .unwrap(),
+                            );
                             let region_type = bar_type;
 
                             self.bar_addr[bar_idx] = value;
@@ -733,8 +837,10 @@ impl PciConfiguration {
                         | u64::from(self.bar_addr[bar_idx - 1] & self.writable_bits[reg_idx - 1]);
                     let new_base = u64::from(value & mask) << 32
                         | u64::from(self.registers[reg_idx - 1] & self.writable_bits[reg_idx - 1]);
-                    let len = u64::from(self.bar_size[bar_idx]) << 32
-                        | u64::from(self.bar_size[bar_idx - 1]);
+                    let len =
+                        decode_64_bits_bar_size(self.bar_size[bar_idx], self.bar_size[bar_idx - 1])
+                            .ok_or(Error::Decode64BarSize)
+                            .unwrap();
                     let region_type = PciBarRegionType::Memory64BitRegion;
 
                     self.bar_addr[bar_idx] = value;
@@ -760,7 +866,11 @@ impl PciConfiguration {
             );
             let old_base = u64::from(self.rom_bar_addr & mask);
             let new_base = u64::from(value & mask);
-            let len = u64::from(self.rom_bar_size);
+            let len = u64::from(
+                decode_32_bits_bar_size(self.rom_bar_size)
+                    .ok_or(Error::Decode32BarSize)
+                    .unwrap(),
+            );
             let region_type = PciBarRegionType::Memory32BitRegion;
 
             self.rom_bar_addr = value;
@@ -777,6 +887,54 @@ impl PciConfiguration {
     }
 }
 
+impl Pausable for PciConfiguration {}
+
+impl Snapshottable for PciConfiguration {
+    fn id(&self) -> String {
+        String::from("pci_configuration")
+    }
+
+    fn snapshot(&self) -> std::result::Result<Snapshot, MigratableError> {
+        let snapshot =
+            serde_json::to_vec(&self.state()).map_err(|e| MigratableError::Snapshot(e.into()))?;
+
+        let mut config_snapshot = Snapshot::new(self.id().as_str());
+        config_snapshot.add_data_section(SnapshotDataSection {
+            id: format!("{}-section", self.id()),
+            snapshot,
+        });
+
+        Ok(config_snapshot)
+    }
+
+    fn restore(&mut self, snapshot: Snapshot) -> std::result::Result<(), MigratableError> {
+        if let Some(config_section) = snapshot
+            .snapshot_data
+            .get(&format!("{}-section", self.id()))
+        {
+            let config_state = match serde_json::from_slice(&config_section.snapshot) {
+                Ok(state) => state,
+                Err(error) => {
+                    return Err(MigratableError::Restore(anyhow!(
+                        "Could not deserialize {}: {}",
+                        self.id(),
+                        error
+                    )))
+                }
+            };
+
+            self.set_state(&config_state);
+
+            return Ok(());
+        }
+
+        Err(MigratableError::Restore(anyhow!(
+            "Could not find {} snapshot section",
+            self.id()
+        )))
+    }
+}
+
 impl Default for PciBarConfiguration {
     fn default() -> Self {
         PciBarConfiguration {
@@ -862,6 +1020,7 @@ mod tests {
         let mut cfg = PciConfiguration::new(
             0x1234,
             0x5678,
+            0x1,
             PciClassCode::MultimediaController,
             &PciMultimediaSubclass::AudioController,
             None,
@@ -917,6 +1076,7 @@ mod tests {
         let cfg = PciConfiguration::new(
             0x1234,
             0x5678,
+            0x1,
             PciClassCode::MultimediaController,
             &PciMultimediaSubclass::AudioController,
             Some(&TestPI::Test),

pci/src/lib.rs

@@ -6,6 +6,10 @@
 #[macro_use]
 extern crate log;
 extern crate devices;
+extern crate serde;
+#[macro_use]
+extern crate serde_derive;
+extern crate serde_json;
 extern crate vm_memory;
 
 mod bus;
@@ -13,6 +17,7 @@ mod configuration;
 mod device;
 mod msi;
 mod msix;
+mod vfio;
 
 pub use self::bus::{PciBus, PciConfigIo, PciConfigMmio, PciRoot, PciRootError};
 pub use self::configuration::{
@@ -25,6 +30,7 @@ pub use self::device::{
 };
 pub use self::msi::{msi_num_enabled_vectors, MsiCap, MsiConfig};
 pub use self::msix::{MsixCap, MsixConfig, MsixTableEntry, MSIX_TABLE_ENTRY_SIZE};
+pub use self::vfio::{VfioPciDevice, VfioPciError};
 
 /// PCI has four interrupt pins A->D.
 #[derive(Copy, Clone)]

pci/src/msix.rs

@@ -6,14 +6,17 @@
 extern crate byteorder;
 extern crate vm_memory;
 
-use std::sync::Arc;
-
 use crate::{PciCapability, PciCapabilityID};
+use anyhow::anyhow;
 use byteorder::{ByteOrder, LittleEndian};
+use std::io;
+use std::result;
+use std::sync::Arc;
 use vm_device::interrupt::{
     InterruptIndex, InterruptSourceConfig, InterruptSourceGroup, MsiIrqSourceConfig,
 };
 use vm_memory::ByteValued;
+use vm_migration::{MigratableError, Pausable, Snapshot, SnapshotDataSection, Snapshottable};
 
 const MAX_MSIX_VECTORS_PER_DEVICE: u16 = 2048;
 const MSIX_TABLE_ENTRIES_MODULO: u64 = 16;
@@ -25,7 +28,15 @@ const FUNCTION_MASK_MASK: u16 = (1 << FUNCTION_MASK_BIT) as u16;
 const MSIX_ENABLE_MASK: u16 = (1 << MSIX_ENABLE_BIT) as u16;
 pub const MSIX_TABLE_ENTRY_SIZE: usize = 16;
 
-#[derive(Debug, Clone)]
+#[derive(Debug)]
+enum Error {
+    /// Failed enabling the interrupt route.
+    EnableInterruptRoute(io::Error),
+    /// Failed updating the interrupt route.
+    UpdateInterruptRoute(io::Error),
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct MsixTableEntry {
     pub msg_addr_lo: u32,
     pub msg_addr_hi: u32,
@@ -50,6 +61,14 @@ impl Default for MsixTableEntry {
     }
 }
 
+#[derive(Serialize, Deserialize)]
+struct MsixConfigState {
+    table_entries: Vec<MsixTableEntry>,
+    pba_entries: Vec<u64>,
+    masked: bool,
+    enabled: bool,
+}
+
 pub struct MsixConfig {
     pub table_entries: Vec<MsixTableEntry>,
     pub pba_entries: Vec<u64>,
@@ -80,6 +99,46 @@ impl MsixConfig {
         }
     }
 
+    fn state(&self) -> MsixConfigState {
+        MsixConfigState {
+            table_entries: self.table_entries.clone(),
+            pba_entries: self.pba_entries.clone(),
+            masked: self.masked,
+            enabled: self.enabled,
+        }
+    }
+
+    fn set_state(&mut self, state: &MsixConfigState) -> result::Result<(), Error> {
+        self.table_entries = state.table_entries.clone();
+        self.pba_entries = state.pba_entries.clone();
+        self.masked = state.masked;
+        self.enabled = state.enabled;
+
+        if self.enabled && !self.masked {
+            for (idx, table_entry) in self.table_entries.iter().enumerate() {
+                if table_entry.masked() {
+                    continue;
+                }
+
+                let config = MsiIrqSourceConfig {
+                    high_addr: table_entry.msg_addr_hi,
+                    low_addr: table_entry.msg_addr_lo,
+                    data: table_entry.msg_data,
+                };
+
+                self.interrupt_source_group
+                    .update(idx as InterruptIndex, InterruptSourceConfig::MsiIrq(config))
+                    .map_err(Error::UpdateInterruptRoute)?;
+
+                self.interrupt_source_group
+                    .enable()
+                    .map_err(Error::EnableInterruptRoute)?;
+            }
+        }
+
+        Ok(())
+    }
+
     pub fn masked(&self) -> bool {
         self.masked
     }
@@ -365,6 +424,52 @@ impl MsixConfig {
     }
 }
 
+impl Pausable for MsixConfig {}
+
+impl Snapshottable for MsixConfig {
+    fn id(&self) -> String {
+        String::from("msix_config")
+    }
+
+    fn snapshot(&self) -> std::result::Result<Snapshot, MigratableError> {
+        let snapshot =
+            serde_json::to_vec(&self.state()).map_err(|e| MigratableError::Snapshot(e.into()))?;
+
+        let mut msix_snapshot = Snapshot::new(self.id().as_str());
+        msix_snapshot.add_data_section(SnapshotDataSection {
+            id: format!("{}-section", self.id()),
+            snapshot,
+        });
+
+        Ok(msix_snapshot)
+    }
+
+    fn restore(&mut self, snapshot: Snapshot) -> std::result::Result<(), MigratableError> {
+        if let Some(msix_section) = snapshot
+            .snapshot_data
+            .get(&format!("{}-section", self.id()))
+        {
+            let msix_state = match serde_json::from_slice(&msix_section.snapshot) {
+                Ok(state) => state,
+                Err(error) => {
+                    return Err(MigratableError::Restore(anyhow!(
+                        "Could not deserialize MSI-X {}",
+                        error
+                    )))
+                }
+            };
+
+            return self.set_state(&msix_state).map_err(|e| {
+                MigratableError::Restore(anyhow!("Could not restore MSI-X state {:?}", e))
+            });
+        }
+
+        Err(MigratableError::Restore(anyhow!(
+            "Could not find MSI-X snapshot section"
+        )))
+    }
+}
+
 #[allow(dead_code)]
 #[repr(packed)]
 #[derive(Clone, Copy, Default)]

pci/src/vfio.rs

@@ -4,30 +4,33 @@
 //
 
 extern crate devices;
-extern crate pci;
 extern crate vm_allocator;
 
-use crate::vfio_device::VfioDevice;
-use byteorder::{ByteOrder, LittleEndian};
-use devices::BusDevice;
-use kvm_bindings::kvm_userspace_memory_region;
-use kvm_ioctls::*;
-use pci::{
+use crate::{
     msi_num_enabled_vectors, BarReprogrammingParams, MsiConfig, MsixCap, MsixConfig,
     PciBarConfiguration, PciBarRegionType, PciCapabilityID, PciClassCode, PciConfiguration,
     PciDevice, PciDeviceError, PciHeaderType, PciSubclass, MSIX_TABLE_ENTRY_SIZE,
 };
+use byteorder::{ByteOrder, LittleEndian};
+use devices::BusDevice;
+use kvm_bindings::kvm_userspace_memory_region;
+use kvm_ioctls::*;
 use std::any::Any;
+use std::ops::Deref;
 use std::os::unix::io::AsRawFd;
 use std::ptr::null_mut;
 use std::sync::Arc;
 use std::{fmt, io, result};
 use vfio_bindings::bindings::vfio::*;
+use vfio_ioctls::{VfioDevice, VfioError};
 use vm_allocator::SystemAllocator;
 use vm_device::interrupt::{
     InterruptIndex, InterruptManager, InterruptSourceGroup, MsiIrqGroupConfig,
 };
-use vm_memory::{Address, GuestAddress, GuestUsize};
+use vm_memory::{
+    Address, GuestAddress, GuestAddressSpace, GuestMemoryAtomic, GuestMemoryMmap, GuestRegionMmap,
+    GuestUsize,
+};
 use vmm_sys_util::eventfd::EventFd;
 
 #[derive(Debug)]
@@ -41,6 +44,7 @@ pub enum VfioPciError {
     SetGsiRouting(kvm_ioctls::Error),
     MsiNotConfigured,
     MsixNotConfigured,
+    UpdateMemory(VfioError),
     UpdateMsiEventFd,
     UpdateMsixEventFd,
 }
@@ -62,6 +66,7 @@ impl fmt::Display for VfioPciError {
             VfioPciError::SetGsiRouting(e) => write!(f, "failed to set GSI routes for KVM: {}", e),
             VfioPciError::MsiNotConfigured => write!(f, "MSI interrupt not yet configured"),
             VfioPciError::MsixNotConfigured => write!(f, "MSI-X interrupt not yet configured"),
+            VfioPciError::UpdateMemory(e) => write!(f, "failed to update memory: {}", e),
             VfioPciError::UpdateMsiEventFd => write!(f, "failed to update MSI eventfd"),
             VfioPciError::UpdateMsixEventFd => write!(f, "failed to update MSI-X eventfd"),
         }
@@ -282,6 +287,7 @@ pub struct VfioPciDevice {
     configuration: PciConfiguration,
     mmio_regions: Vec<MmioRegion>,
     interrupt: Interrupt,
+    mem: GuestMemoryAtomic<GuestMemoryMmap>,
 }
 
 impl VfioPciDevice {
@@ -290,11 +296,13 @@ impl VfioPciDevice {
         vm_fd: &Arc<VmFd>,
         device: VfioDevice,
         interrupt_manager: &Arc<dyn InterruptManager<GroupConfig = MsiIrqGroupConfig>>,
+        mem: GuestMemoryAtomic<GuestMemoryMmap>,
     ) -> Result<Self> {
         let device = Arc::new(device);
         device.reset();
 
         let configuration = PciConfiguration::new(
+            0,
             0,
             0,
             PciClassCode::Other,
@@ -318,6 +326,7 @@ impl VfioPciDevice {
                 msi: None,
                 msix: None,
             },
+            mem,
         };
 
         vfio_pci_device.parse_capabilities(interrupt_manager);
@@ -610,6 +619,12 @@ impl VfioPciDevice {
             }
         }
     }
+
+    pub fn update_memory(&self, new_region: &Arc<GuestRegionMmap>) -> Result<()> {
+        self.device
+            .extend_dma_map(new_region)
+            .map_err(VfioPciError::UpdateMemory)
+    }
 }
 
 impl Drop for VfioPciDevice {
@@ -628,7 +643,11 @@ impl Drop for VfioPciDevice {
             }
         }
 
-        if self.device.unset_dma_map().is_err() {
+        if self
+            .device
+            .setup_dma_map(self.mem.memory().deref())
+            .is_err()
+        {
             error!("failed to remove all guest memory regions from iommu table");
         }
     }
@@ -834,7 +853,11 @@ impl PciDevice for VfioPciDevice {
             }
         }
 
-        if self.device.setup_dma_map().is_err() {
+        if self
+            .device
+            .setup_dma_map(self.mem.memory().deref())
+            .is_err()
+        {
             error!("failed to add all guest memory regions into iommu table");
         }
 

qcow/Cargo.toml

@@ -10,7 +10,7 @@ path = "src/qcow.rs"
 
 [dependencies]
 byteorder = "1.3.4"
-libc = "0.2.68"
+libc = "0.2.71"
 log = "0.4.8"
 remain = "0.2.1"
 vmm-sys-util = ">=0.3.1"

qcow/src/refcount.rs

@@ -2,7 +2,6 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE-BSD-3-Clause file.
 
-use std;
 use std::fmt::{self, Display};
 use std::io;
 

release-notes.md

@@ -1,10 +1,29 @@
+- [v0.8.0](#v080)
+    - [Experimental Snapshot and Restore Support](#experimental-snapshot-and-restore-support)
+    - [Experimental ARM64 Support](#experimental-arm64-support)
+    - [Support for Using 5-level Paging in Guests](#support-for-using-5-level-paging-in-guests)
+    - [Virtio Device Interrupt Suppression for Network Devices](#virtio-device-interrupt-suppression-for-network-devices)
+    - [`vhost_user_fs` Improvements](#vhost_user_fs-improvements)
+    - [Notable Bug Fixes](#notable-bug-fixes)
+    - [Command Line and API Changes](#command-line-and-api-changes)
+    - [Contributors](#contributors)
+- [v0.7.0](#v070)
+    - [Block, Network, Persistent Memory (PMEM), VirtioFS and Vsock hotplug](#block-network-persistent-memory-pmem-virtiofs-and-vsock-hotplug)
+    - [Alternative `libc` Support](#alternative-libc-support)
+    - [Multithreaded Multi Queued `vhost-user` Backends](#multithreaded-multi-queued-vhost-user-backends)
+    - [Initial RamFS Support](#initial-ramfs-support)
+    - [Alternative Memory Hotplug: `virtio-mem`](#alternative-memory-hotplug-virtio-mem)
+    - [`Seccomp` Sandboxing](#seccomp-sandboxing)
+    - [Updated Distribution Support](#updated-distribution-support)
+    - [Command Line and API Changes](#command-line-and-api-changes-1)
+    - [Contributors](#contributors-1)
 - [v0.6.0](#v060)
     - [Directly Assigned Devices Hotplug](#directly-assigned-devices-hotplug)
     - [Shared Filesystem Improvements](#shared-filesystem-improvements)
     - [Block and Networking IO Self Offloading](#block-and-networking-io-self-offloading)
     - [Command Line Interface](#command-line-interface)
     - [PVH Boot](#pvh-boot)
-    - [Contributors](#contributors)
+    - [Contributors](#contributors-2)
 - [v0.5.1](#v051)
 - [v0.5.0](#v050)
     - [Virtual Machine Dynamic Resizing](#virtual-machine-dynamic-resizing)
@@ -12,7 +31,7 @@
     - [New Interrupt Management Framework](#new-interrupt-management-framework)
     - [Development Tools](#development-tools)
     - [Kata Containers Integration](#kata-containers-integration)
-    - [Contributors](#contributors)
+    - [Contributors](#contributors-3)
 - [v0.4.0](#v040)
     - [Dynamic virtual CPUs addition](#dynamic-virtual-cpus-addition)
     - [Programmatic firmware tables generation](#programmatic-firmware-tables-generation)
@@ -21,7 +40,7 @@
     - [Userspace IOAPIC by default](#userspace-ioapic-by-default)
     - [PCI BAR reprogramming](#pci-bar-reprogramming)
     - [New `cloud-hypervisor` organization](#new-cloud-hypervisor-organization)
-    - [Contributors](#contributors-1)
+    - [Contributors](#contributors-4)
 - [v0.3.0](#v030)
     - [Block device offloading](#block-device-offloading)
     - [Network device backend](#network-device-backend)
@@ -48,6 +67,195 @@
     - [Unit testing](#unit-testing)
     - [Integration tests parallelization](#integration-tests-parallelization)
 
+# v0.8.0
+
+This release has been tracked through the [0.8.0 project](https://github.com/cloud-hypervisor/cloud-hypervisor/projects/10).
+
+Highlights for `cloud-hypervisor` version 0.8.0 include:
+
+### Experimental Snapshot and Restore Support
+
+This release includes the first version of the snapshot and restore feature.
+This allows a VM to be paused and then subsequently snapshotted. At a later
+point that snapshot may be restored into a new running VM identical to the
+original VM at the point it was paused.
+
+This feature can be used for offline migration from one VM host to another, to
+allow the upgrading or rebooting of the host machine transparently to the guest
+or for templating the VM. This is an experimental feature and cannot be used on
+a VM using passthrough (VFIO) devices. Issues with SMP have also been observed
+(#1176).
+
+### Experimental ARM64 Support
+
+Included in this release is experimental support for running on ARM64.
+Currently only `virtio-mmio` devices and a serial port are supported. Full
+details can be found in the [ARM64 documentation](docs/arm64.md).
+
+### Support for Using 5-level Paging in Guests
+
+If the host supports it the guest is now enabled for 5-level paging (aka LA57).
+This works when booting the Linux kernel with a vmlinux, bzImage or firmware
+based boot. However booting an ELF kernel built with `CONFIG_PVH=y` does not
+work due to current limitations in the PVH boot process.
+
+### Virtio Device Interrupt Suppression for Network Devices
+
+With `virtio-net` and `vhost-user-net` devices the guest can suppress
+interrupts from the VMM by using the `VIRTIO_RING_F_EVENT_IDX` feature. This
+can lead to an improvement in performance by reducing the number of interrupts
+the guest must service.
+
+### `vhost_user_fs` Improvements
+
+The implementation in Cloud Hypervisor of the VirtioFS server now supports sandboxing itself with `seccomp`.
+
+
+### Notable Bug Fixes
+
+* VMs that have not yet been booted can now be deleted (#1110).
+* By creating the `tap` device ahead of creating the VM it is not required to
+  run the `cloud-hypervisor` binary with `CAP_NET_ADMIN` (#1273).
+* Block I/O via `virtio-block` or `vhost-user-block` now correctly adheres to
+  the specification and synchronizes to the underlying filesystem as required
+  based on guest feature negotiation. This avoids potential data loss (#399,
+  #1216).
+* When booting with a large number of vCPUs then the ACPI table would be
+  overwritten by the SMP `MPTABLE`. When compiled with the `acpi` feature the
+  `MPTABLE` will no longer be generated (#1132).
+* Shutting down VMs that have been paused is now supported (#816).
+* Created socket files are deleted on shutdown (#1083).
+* Trying to use passthrough devices (VFIO) will be rejected on `mmio` builds
+  (#751).
+
+### Command Line and API Changes
+
+This is non exhaustive list of HTTP API and command line changes:
+
+* All user visible socket parameters are now consistently called `socket`
+  rather than `sock` in some cases.
+* The `ch-remote` tool now shows any error message generated by the VMM
+* The `wce` parameter has been removed from `--disk` as the feature is always
+  offered for negotiation.
+* `--net` has gained a `host_mac` option that allows the setting of the MAC
+  address for the `tap` device on the host.
+
+### Contributors
+
+Many thanks to everyone who has contributed to our 0.8.0 release including some new faces.
+
+* Anatol Belski <ab@php.net>
+* Arron Wang <arron.wang@intel.com>
+* Bo Chen <chen.bo@intel.com>
+* Dr. David Alan Gilbert <dgilbert@redhat.com>
+* Henry Wang <Henry.Wang@arm.com>
+* Hui Zhu <teawater@antfin.com>
+* LiYa'nan <oliverliyn@gmail.com>
+* Michael Zhao <michael.zhao@arm.com>
+* Rob Bradford <robert.bradford@intel.com>
+* Samuel Ortiz <sameo@linux.intel.com>
+* Sebastien Boeuf <sebastien.boeuf@intel.com>
+* Sergio Lopez <slp@redhat.com>
+
+# v0.7.0
+
+This release has been tracked through the [0.7.0 project](https://github.com/cloud-hypervisor/cloud-hypervisor/projects/7).
+
+Highlights for `cloud-hypervisor` version 0.7.0 include:
+
+### Block, Network, Persistent Memory (PMEM), VirtioFS and Vsock hotplug
+
+Further to our effort to support modifying a running guest we now support
+hotplug and unplug of the following virtio backed devices: block, network,
+pmem, virtio-fs and vsock. This functionality is available on the (default) PCI
+based tranport and is exposed through the HTTP API. The `ch-remote` utility
+provides a CLI for adding or removing these device types after the VM has
+booted. User can use the `id` parameter on the devices to choose names for
+devices to ease their removal.
+
+### Alternative `libc` Support
+
+Cloud Hypervisor can now be compiled with the `musl` C library and this release
+contains a static binary compiled using that toolchain.
+
+### Multithreaded Multi Queued `vhost-user` Backends
+
+The `vhost-user` backends for network and block support that are shipped by
+Cloud Hypervisor have been enhanced to support multiple threads and queues to
+improve throughput. These backends are used automatically if `vhost_user=true`
+is passed when the devices are created.
+
+### Initial RamFS Support
+
+By passing the `--initramfs` command line option the user can specify a file to
+be loaded into the guest memory to be used as the kernel initial filesystem.
+This is usually used to allow the loading of drivers needed to be able to
+access the real root filesystem but it can also be used standalone for a very
+minimal image.
+
+### Alternative Memory Hotplug: `virtio-mem`
+
+As well as supporting ACPI based hotplug Cloud Hypervisor now supports using
+the `virtio-mem` hotplug alternative. This can be controlled by the
+`hotplug_method` parameter on the `--memory` command line option. It currently
+requires kernel patches to be able to support it.
+
+### `Seccomp` Sandboxing
+
+Cloud Hypervisor now has support for restricting the system calls that the
+process can use via the `seccomp` security API. This on by default and is
+controlled by the `--seccomp` command line option.
+
+### Updated Distribution Support
+
+With the release of Ubuntu 20.04 we have added that to the list of supported
+distributions and is part of our regular testing programme.
+
+### Command Line and API Changes
+
+This is non exhaustive list of HTTP API and command line changes
+
+* New `id` fields added for devices to allow them to be named to ease removal.
+  If no name is specified the VMM chooses one.
+* Use `--memory`'s `shared` and `hugepages` controls for determining backing
+  memory instead of providing a path.
+* The `--vsock` parameter only takes one device as the Linux kernel only
+  supports a single Vsock device. The REST API has removed the vector for this
+  option and replaced it with a single optional field.
+* There is enhanced validation of the command line and API provided
+  configurations to ensure that the provided options are compatible e.g. that
+  shared memory is in use if any attempt is made to used a `vhost-user` backed
+  device.
+* `ch-remote` has added `add-disk`, `add-fs`, `add-net`, `add-pmem` and
+  `add-vsock` subcommands. For removal `remove-device` is used. The REST API
+  has appropriate new HTTP endpoints too.
+* Specifying a `size` with `--pmem` is no longer required and instead the size
+  will be obtained from the file. A `discard_writes` option has also been added
+  to provide the equivalent of a read-only file.
+* The parameters to `--block-backend` have been changed to more closely align
+  with those used by `--disk`.
+
+### Contributors
+
+Many thanks to everyone who has contributed to our 0.7.0 release including some new faces.
+
+* Alejandro Jimenez <alejandro.j.jimenez@oracle.com>
+* Bo Chen <chen.bo@intel.com>
+* Cathy Zhang <cathy.zhang@intel.com>
+* Damjan Georgievski <gdamjan@gmail.com>
+* Dean Sheather <dean@coder.com>
+* Eryu Guan <eguan@linux.alibaba.com>
+* Hui Zhu <teawater@antfin.com>
+* Jose Carlos Venegas Munoz <jose.carlos.venegas.munoz@intel.com>
+* Martin Xu <martin.xu@intel.com>
+* Muminul Islam <muislam@microsoft.com>
+* Rob Bradford <robert.bradford@intel.com>
+* Samuel Ortiz <sameo@linux.intel.com>
+* Sebastien Boeuf <sebastien.boeuf@intel.com>
+* Sergio Lopez <slp@redhat.com>
+* Yang Zhong <yang.zhong@intel.com>
+* Yi Sun <yi.y.sun@linux.intel.com>
+
 # v0.6.0
 
 This release has been tracked through the [0.6.0 project](https://github.com/cloud-hypervisor/cloud-hypervisor/projects/7).

resources/Dockerfile

@@ -1,6 +1,7 @@
 FROM ubuntu:18.04 as dev
 
-ARG RUST_TOOLCHAIN="1.42.0"
+ARG TARGETARCH="x86_64"
+ARG RUST_TOOLCHAIN="1.43.0"
 ARG CLH_SRC_DIR="/cloud-hypervisor"
 ARG CLH_BUILD_DIR="$CLH_SRC_DIR/build"
 ARG CARGO_REGISTRY_DIR="$CLH_BUILD_DIR/cargo_registry"
@@ -11,9 +12,9 @@ ENV RUSTUP_HOME=$CARGO_HOME
 ENV PATH="$PATH:$CARGO_HOME/bin"
 
 # Install all CI dependencies
-RUN apt-get update
-RUN apt-get -yq upgrade
-RUN DEBIAN_FRONTEND=noninteractive apt-get install -yq \
+RUN apt-get update \
+	&& apt-get -yq upgrade \
+	&& DEBIAN_FRONTEND=noninteractive apt-get install -yq \
 	build-essential \
 	bc \
 	docker.io \
@@ -21,6 +22,7 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get install -yq \
 	wget \
 	sudo \
 	mtools \
+	musl-tools \
 	libssl-dev \
 	pkg-config \
 	flex \
@@ -34,13 +36,31 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get install -yq \
 	libcap-ng-dev \
 	socat \
 	dosfstools \
+	cpio \
+	bsdtar \
+	libfdt-dev \
     && apt-get clean \
     && rm -rf /var/lib/apt/lists/*
 
+RUN if [ "$TARGETARCH" = "x86_64" ]; then \
+	apt-get update \
+	&& apt-get -yq upgrade \
+	&& DEBIAN_FRONTEND=noninteractive apt-get install -yq gcc-multilib \
+	&& apt-get clean \
+    && rm -rf /var/lib/apt/lists/*;	fi
+
+# Fix the libssl-dev install
+RUN cp /usr/include/"$TARGETARCH"-linux-gnu/openssl/opensslconf.h /usr/include/openssl/
+ENV OPENSSL_DIR=/usr/lib/"$TARGETARCH"-linux-gnu/
+ENV OPENSSL_LIB_DIR=/usr/lib/"$TARGETARCH"-linux-gnu/
+ENV OPENSSL_INCLUDE_DIR=/usr/include/
+
 # Install the rust toolchain
 RUN nohup curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain "$RUST_TOOLCHAIN" \
-    && rustup component add rustfmt \
-    && rustup component add clippy \
+    && rustup target add $TARGETARCH-unknown-linux-musl --toolchain "$RUST_TOOLCHAIN" \
+    && if [ "$TARGETARCH" = "x86_64" ]; then rustup toolchain add $RUST_TOOLCHAIN-x86_64-unknown-linux-musl; fi \
+    && if [ "$TARGETARCH" = "x86_64" ]; then rustup component add rustfmt; fi \
+    && if [ "$TARGETARCH" = "x86_64" ]; then rustup component add clippy; fi \
     && cargo install cargo-audit \
     && rm -rf "$CARGO_HOME/registry" \
     && ln -s "$CARGO_REGISTRY_DIR" "$CARGO_HOME/registry" \

resources/linux-config

@@ -178,7 +178,14 @@ CONFIG_NET_NS=y
 CONFIG_SCHED_AUTOGROUP=y
 # CONFIG_SYSFS_DEPRECATED is not set
 CONFIG_RELAY=y
-# CONFIG_BLK_DEV_INITRD is not set
+CONFIG_BLK_DEV_INITRD=y
+CONFIG_INITRAMFS_SOURCE=""
+CONFIG_RD_GZIP=y
+CONFIG_RD_BZIP2=y
+CONFIG_RD_LZMA=y
+CONFIG_RD_XZ=y
+CONFIG_RD_LZO=y
+CONFIG_RD_LZ4=y
 # CONFIG_BOOT_CONFIG is not set
 CONFIG_CC_OPTIMIZE_FOR_PERFORMANCE=y
 # CONFIG_CC_OPTIMIZE_FOR_SIZE is not set
@@ -279,7 +286,7 @@ CONFIG_X86_64_SMP=y
 CONFIG_ARCH_SUPPORTS_UPROBES=y
 CONFIG_FIX_EARLYCON_MEM=y
 CONFIG_DYNAMIC_PHYSICAL_MASK=y
-CONFIG_PGTABLE_LEVELS=4
+CONFIG_PGTABLE_LEVELS=5
 CONFIG_CC_HAS_SANE_STACKPROTECTOR=y
 
 #
@@ -364,7 +371,7 @@ CONFIG_X86_VSYSCALL_EMULATION=y
 # CONFIG_MICROCODE is not set
 CONFIG_X86_MSR=y
 CONFIG_X86_CPUID=y
-# CONFIG_X86_5LEVEL is not set
+CONFIG_X86_5LEVEL=y
 CONFIG_X86_DIRECT_GBPAGES=y
 # CONFIG_X86_CPA_STATISTICS is not set
 CONFIG_AMD_MEM_ENCRYPT=y
@@ -485,6 +492,7 @@ CONFIG_ACPI_HOTPLUG_CPU=y
 # CONFIG_ACPI_PROCESSOR_AGGREGATOR is not set
 CONFIG_ACPI_THERMAL=y
 CONFIG_ARCH_HAS_ACPI_TABLE_UPGRADE=y
+CONFIG_ACPI_TABLE_UPGRADE=y
 # CONFIG_ACPI_DEBUG is not set
 # CONFIG_ACPI_PCI_SLOT is not set
 CONFIG_ACPI_CONTAINER=y
@@ -2418,7 +2426,22 @@ CONFIG_ZLIB_INFLATE=y
 CONFIG_ZLIB_DEFLATE=y
 CONFIG_LZO_COMPRESS=y
 CONFIG_LZO_DECOMPRESS=y
-# CONFIG_XZ_DEC is not set
+CONFIG_LZ4_DECOMPRESS=y
+CONFIG_XZ_DEC=y
+CONFIG_XZ_DEC_X86=y
+CONFIG_XZ_DEC_POWERPC=y
+CONFIG_XZ_DEC_IA64=y
+CONFIG_XZ_DEC_ARM=y
+CONFIG_XZ_DEC_ARMTHUMB=y
+CONFIG_XZ_DEC_SPARC=y
+CONFIG_XZ_DEC_BCJ=y
+# CONFIG_XZ_DEC_TEST is not set
+CONFIG_DECOMPRESS_GZIP=y
+CONFIG_DECOMPRESS_BZIP2=y
+CONFIG_DECOMPRESS_LZMA=y
+CONFIG_DECOMPRESS_XZ=y
+CONFIG_DECOMPRESS_LZO=y
+CONFIG_DECOMPRESS_LZ4=y
 CONFIG_INTERVAL_TREE=y
 CONFIG_XARRAY_MULTI=y
 CONFIG_ASSOCIATIVE_ARRAY=y

scripts/dev_cli.sh

@@ -7,7 +7,7 @@
 CLI_NAME="Cloud Hypervisor"
 
 CTR_IMAGE_TAG="cloudhypervisor/dev"
-CTR_IMAGE_VERSION="v1"
+CTR_IMAGE_VERSION="v3"
 CTR_IMAGE="${CTR_IMAGE_TAG}:${CTR_IMAGE_VERSION}"
 
 DOCKER_RUNTIME="docker"
@@ -113,6 +113,13 @@ ensure_build_dir() {
     done
 }
 
+# Make sure we're using the latest dev container, by just pulling it.
+ensure_latest_ctr() {
+    $DOCKER_RUNTIME pull "$CTR_IMAGE"
+
+    ok_or_die "Error pulling container image. Aborting."
+}
+
 # Fix main directory permissions after a container ran as root.
 # Since the container ran as root, any files it creates will be owned by root.
 # This fixes that by recursively changing the ownership of /cloud-hypervisor to the
@@ -139,16 +146,18 @@ cmd_help() {
     echo ""
     echo "Available commands:"
     echo ""
-    echo "    build [--debug|--release] [-- [<cargo args>]]"
+    echo "    build [--debug|--release] [--libc musl|gnu] [-- [<cargo args>]]"
     echo "        Build the Cloud Hypervisor binaries."
     echo "        --debug               Build the debug binaries. This is the default."
     echo "        --release             Build the release binaries."
+    echo "        --libc                Select the C library Cloud Hypervisor will be built against. Default is gnu"
     echo ""
-    echo "    tests [--unit|--cargo|--all] [-- [<cargo test args>]]"
+    echo "    tests [--unit|--cargo|--all] [--libc musl|gnu] [-- [<cargo test args>]]"
     echo "        Run the Cloud Hypervisor tests."
     echo "        --unit               Run the unit tests."
     echo "        --cargo              Run the cargo tests."
     echo "        --integration        Run the integration tests."
+    echo "        --libc               Select the C library Cloud Hypervisor will be built against. Default is gnu"
     echo "        --all                Run all tests."
     echo ""
     echo "    build-container [--type]"
@@ -165,12 +174,19 @@ cmd_help() {
 
 cmd_build() {
     build="debug"
+    libc="gnu"
 
     while [ $# -gt 0 ]; do
 	case "$1" in
             "-h"|"--help")  { cmd_help; exit 1;     } ;;
             "--debug")      { build="debug";      } ;;
             "--release")    { build="release";    } ;;
+            "--libc")
+                shift
+                [[ "$1" =~ ^(musl|gnu)$ ]] || \
+                    die "Invalid libc: $1. Valid options are \"musl\" and \"gnu\"."
+                libc="$1"
+                ;;
             "--")           { shift; break;         } ;;
             *)
 		die "Unknown build argument: $1. Please use --help for help."
@@ -179,8 +195,23 @@ cmd_build() {
 	shift
     done
 
+    target="$(uname -m)-unknown-linux-${libc}"
+
     cargo_args=("$@")
     [ $build = "release" ] && cargo_args+=("--release")
+    cargo_args+=(--target "$target")
+    [ $(uname -m) = "aarch64" ] && cargo_args+=("--no-default-features")
+    [ $(uname -m) = "aarch64" ] && cargo_args+=(--features "mmio")
+
+    rustflags=""
+    if [ $(uname -m) = "aarch64" ] && [ $libc = "musl" ] ; then
+        rustflags="-C link-arg=-lgcc"
+    fi
+
+    # A workaround on Arm64 to avoid build errors in kvm-bindings
+    if [ $(uname -m) = "aarch64" ]; then
+        sed -i 's/"with-serde",\ //g' "$CLH_ROOT_DIR"/vmm/Cargo.toml
+    fi
 
     $DOCKER_RUNTIME run \
 	   --user "$(id -u):$(id -g)" \
@@ -188,10 +219,11 @@ cmd_build() {
 	   --rm \
 	   --volume /dev:/dev \
 	   --volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR" \
+	   --env RUSTFLAGS="$rustflags" \
 	   "$CTR_IMAGE" \
 	   cargo build \
 	         --target-dir "$CTR_CLH_CARGO_TARGET" \
-	         "${cargo_args[@]}" && say "Binaries placed under $CLH_CARGO_TARGET/$build"
+	         "${cargo_args[@]}" && say "Binaries placed under $CLH_CARGO_TARGET/$target/$build"
 }
 
 cmd_clean() {
@@ -212,15 +244,22 @@ cmd_tests() {
     unit=false
     cargo=false
     integration=false
+    libc="gnu"
 
     while [ $# -gt 0 ]; do
 	case "$1" in
-            "-h"|"--help")    { cmd_help; exit 1;     } ;;
-            "--unit")         { unit=true;      } ;;
-            "--cargo")        { cargo=true;    } ;;
-	    "--integration")  { integration=true;    } ;;
-	    "--all")          { cargo=true; unit=true; integration=true;  } ;;
-            "--")             { shift; break;         } ;;
+            "-h"|"--help")           { cmd_help; exit 1;     } ;;
+            "--unit")                { unit=true;      } ;;
+            "--cargo")               { cargo=true;    } ;;
+            "--integration")         { integration=true;    } ;;
+            "--libc")
+                shift
+                [[ "$1" =~ ^(musl|gnu)$ ]] || \
+                    die "Invalid libc: $1. Valid options are \"musl\" and \"gnu\"."
+                libc="$1"
+                ;;
+	    "--all")                 { cargo=true; unit=true; integration=true;  } ;;
+            "--")                    { shift; break;         } ;;
             *)
 		die "Unknown tests argument: $1. Please use --help for help."
 		;;
@@ -228,8 +267,16 @@ cmd_tests() {
 	shift
     done
 
+    target="$(uname -m)-unknown-linux-${libc}"
+    cflags=""
+    target_cc=""
+    if [[ "$target" == "x86_64-unknown-linux-musl" ]]; then
+	target_cc="musl-gcc"
+	cflags="-I /usr/include/x86_64-linux-musl/ -idirafter /usr/include/"
+    fi
+
     if [ "$unit" = true ] ;  then
-	say "Running unit tests..."
+	say "Running unit tests for $target..."
 	$DOCKER_RUNTIME run \
 	       --workdir "$CTR_CLH_ROOT_DIR" \
 	       --rm \
@@ -237,6 +284,9 @@ cmd_tests() {
 	       --device /dev/net/tun \
 	       --cap-add net_admin \
 	       --volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR" \
+	       --env BUILD_TARGET="$target" \
+	       --env CFLAGS="$cflags" \
+	       --env TARGET_CC="$target_cc" \
 	       "$CTR_IMAGE" \
 	       ./scripts/run_unit_tests.sh "$@" || fix_dir_perms $? || exit $?
     fi
@@ -252,7 +302,7 @@ cmd_tests() {
     fi
 
     if [ "$integration" = true ] ;  then
-	say "Running integration tests..."
+	say "Running integration tests for $target..."
 	$DOCKER_RUNTIME run \
 	       --workdir "$CTR_CLH_ROOT_DIR" \
 	       --rm \
@@ -265,6 +315,7 @@ cmd_tests() {
 	       --volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR" \
 	       --volume "$CLH_INTEGRATION_WORKLOADS:$CTR_CLH_INTEGRATION_WORKLOADS" \
 	       --env USER="root" \
+	       --env CH_LIBC="${libc}" \
 	       "$CTR_IMAGE" \
 	       ./scripts/run_integration_tests.sh "$@" || fix_dir_perms $? || exit $?
     fi
@@ -296,6 +347,7 @@ cmd_build-container() {
 	   --target $container_type \
 	   -t $CTR_IMAGE \
 	   -f $BUILD_DIR/Dockerfile \
+	   --build-arg TARGETARCH="$(uname -m)" \
 	   $BUILD_DIR
 }
 
@@ -325,5 +377,13 @@ cmd=cmd_$1
 shift
 
 ensure_build_dir
+if [ $(uname -m) = "x86_64" ]; then
+    ensure_latest_ctr
+fi
+
+# Before a public image for AArch64 ready, we build the container if needed.
+if [ $(uname -m) = "aarch64" ]; then
+    cmd_build-container
+fi
 
 $cmd "$@"

scripts/run_integration_tests.sh

@@ -3,6 +3,8 @@ set -x
 
 source $HOME/.cargo/env
 
+export BUILD_TARGET=${BUILD_TARGET-x86_64-unknown-linux-gnu}
+
 WORKLOADS_DIR="$HOME/workloads"
 mkdir -p "$WORKLOADS_DIR"
 
@@ -51,20 +53,46 @@ if [ ! -f "$BIONIC_OS_RAW_IMAGE" ]; then
 fi
 
 
-EOAN_OS_IMAGE_NAME="eoan-server-cloudimg-amd64.img"
-EOAN_OS_IMAGE_URL="https://cloudhypervisorstorage.blob.core.windows.net/images/$EOAN_OS_IMAGE_NAME"
-EOAN_OS_IMAGE="$WORKLOADS_DIR/$EOAN_OS_IMAGE_NAME"
-if [ ! -f "$EOAN_OS_IMAGE" ]; then
+FOCAL_OS_IMAGE_NAME="focal-server-cloudimg-amd64.img"
+FOCAL_OS_IMAGE_URL="https://cloudhypervisorstorage.blob.core.windows.net/images/$FOCAL_OS_IMAGE_NAME"
+FOCAL_OS_IMAGE="$WORKLOADS_DIR/$FOCAL_OS_IMAGE_NAME"
+if [ ! -f "$FOCAL_OS_IMAGE" ]; then
     pushd $WORKLOADS_DIR
-    time wget --quiet $EOAN_OS_IMAGE_URL || exit 1
+    time wget --quiet $FOCAL_OS_IMAGE_URL || exit 1
     popd
 fi
 
-EOAN_OS_RAW_IMAGE_NAME="eoan-server-cloudimg-amd64-raw.img"
-EOAN_OS_RAW_IMAGE="$WORKLOADS_DIR/$EOAN_OS_RAW_IMAGE_NAME"
-if [ ! -f "$EOAN_OS_RAW_IMAGE" ]; then
+FOCAL_OS_RAW_IMAGE_NAME="focal-server-cloudimg-amd64-raw.img"
+FOCAL_OS_RAW_IMAGE="$WORKLOADS_DIR/$FOCAL_OS_RAW_IMAGE_NAME"
+if [ ! -f "$FOCAL_OS_RAW_IMAGE" ]; then
     pushd $WORKLOADS_DIR
-    time qemu-img convert -p -f qcow2 -O raw $EOAN_OS_IMAGE_NAME $EOAN_OS_RAW_IMAGE_NAME || exit 1
+    time qemu-img convert -p -f qcow2 -O raw $FOCAL_OS_IMAGE_NAME $FOCAL_OS_RAW_IMAGE_NAME || exit 1
+    popd
+fi
+
+ALPINE_MINIROOTFS_URL="http://dl-cdn.alpinelinux.org/alpine/v3.11/releases/x86_64/alpine-minirootfs-3.11.3-x86_64.tar.gz"
+ALPINE_MINIROOTFS_TARBALL="$WORKLOADS_DIR/alpine-minirootfs-x86_64.tar.gz"
+if [ ! -f "$ALPINE_MINIROOTFS_TARBALL" ]; then
+    pushd $WORKLOADS_DIR
+    time wget --quiet $ALPINE_MINIROOTFS_URL -O $ALPINE_MINIROOTFS_TARBALL || exit 1
+    popd
+fi
+
+ALPINE_INITRAMFS_IMAGE="$WORKLOADS_DIR/alpine_initramfs.img"
+if [ ! -f "$ALPINE_INITRAMFS_IMAGE" ]; then
+    pushd $WORKLOADS_DIR
+    mkdir alpine-minirootfs
+    tar xf "$ALPINE_MINIROOTFS_TARBALL" -C alpine-minirootfs
+    cat > alpine-minirootfs/init <<-EOF
+		#! /bin/sh
+		mount -t devtmpfs dev /dev
+		echo \$TEST_STRING > /dev/console
+		poweroff -f
+	EOF
+    chmod +x alpine-minirootfs/init
+    cd alpine-minirootfs
+    find . -print0 |
+        cpio --null --create --verbose --owner root:root --format=newc > "$ALPINE_INITRAMFS_IMAGE"
     popd
 fi
 
@@ -182,13 +210,28 @@ sudo ip tuntap add name vunet-tap0 mode tap
 # Create tap interface with multipe queues support for vhost_user_net test.
 sudo ip tuntap add name vunet-tap1 mode tap multi_queue
 
-cargo build --release
-sudo setcap cap_net_admin+ep target/release/cloud-hypervisor
-sudo setcap cap_net_admin+ep target/release/vhost_user_net
+BUILD_TARGET="$(uname -m)-unknown-linux-${CH_LIBC}"
+CFLAGS=""
+TARGET_CC=""
+if [[ "${BUILD_TARGET}" == "x86_64-unknown-linux-musl" ]]; then
+TARGET_CC="musl-gcc"
+CFLAGS="-I /usr/include/x86_64-linux-musl/ -idirafter /usr/include/"
+fi
+
+cargo build --release --target $BUILD_TARGET
+strip target/$BUILD_TARGET/release/cloud-hypervisor
+strip target/$BUILD_TARGET/release/vhost_user_net
+strip target/$BUILD_TARGET/release/ch-remote
+
+# Copy for non-privileged net test
+cp target/$BUILD_TARGET/release/cloud-hypervisor target/$BUILD_TARGET/release/cloud-hypervisor-unprivileged
+
+sudo setcap cap_net_admin+ep target/$BUILD_TARGET/release/cloud-hypervisor
+sudo setcap cap_net_admin+ep target/$BUILD_TARGET/release/vhost_user_net
 
 # We always copy a fresh version of our binary for our L2 guest.
-cp target/release/cloud-hypervisor $VFIO_DIR
-cp target/release/ch-remote $VFIO_DIR
+cp target/$BUILD_TARGET/release/cloud-hypervisor $VFIO_DIR
+cp target/$BUILD_TARGET/release/ch-remote $VFIO_DIR
 
 # Enable KSM with some reasonable parameters so that it won't take too long
 # for the memory to be merged between two processes.
@@ -213,8 +256,16 @@ RES=$?
 
 if [ $RES -eq 0 ]; then
     # virtio-mmio based testing
-    cargo build --release --no-default-features --features "mmio"
-    sudo setcap cap_net_admin+ep target/release/cloud-hypervisor
+    cargo build --release --target $BUILD_TARGET --no-default-features --features "mmio"
+    strip target/$BUILD_TARGET/release/cloud-hypervisor
+    strip target/$BUILD_TARGET/release/vhost_user_net
+    strip target/$BUILD_TARGET/release/ch-remote
+
+    sudo setcap cap_net_admin+ep target/$BUILD_TARGET/release/cloud-hypervisor
+
+    # Ensure test binary has the same caps as the cloud-hypervisor one
+    time cargo test --no-run --features "integration_tests,mmio" -- --nocapture || exit 1
+    ls target/debug/deps/cloud_hypervisor-* | xargs -n 1 sudo setcap cap_net_admin+ep
 
     newgrp kvm << EOF
 export RUST_BACKTRACE=1

scripts/run_unit_tests.sh

@@ -2,13 +2,15 @@
 
 source $HOME/.cargo/env
 
-cargo test --workspace --no-run
-pushd target/debug
+BUILD_TARGET=${BUILD_TARGET-x86_64-unknown-linux-gnu}
+
+cargo test --target $BUILD_TARGET --workspace --no-run
+pushd target/$BUILD_TARGET/debug
 ls  | grep net_util | grep -v "\.d" | xargs -n 1 sudo setcap cap_net_admin,cap_net_raw+ep
 popd
 
 sudo adduser $USER kvm
 newgrp kvm << EOF || exit 1
   export RUST_BACKTRACE=1
-  cargo test --workspace "$@" || exit 1;
+  cargo test --target $BUILD_TARGET --workspace "$@" || exit 1;
 EOF

scripts/sha1sums

@@ -2,5 +2,6 @@ cf7cfa783082fc4d6b4d1c0a53e4402648c14b82 clear-31311-cloudguest.img
 142a410546b592ff9536b46bb410faf8ac11edee clear-31311-cloudguest-raw.img
 27f3b17962ace69b51f0ddc2012095e3109e6ed8 bionic-server-cloudimg-amd64.img
 8db9cc58b01452ce2d06c313177e6e74d8582d93 bionic-server-cloudimg-amd64-raw.img
-4a452cdcf781f95d31a1668ecb92a937c176709a eoan-server-cloudimg-amd64.img
-91fdfb21df8920fd55915edf7669282cda2505f6 eoan-server-cloudimg-amd64-raw.img
+d4a44acc6014d5f83dea1c625c43d677a95fa75f alpine-minirootfs-x86_64.tar.gz
+3cab64475048383e70a38550734e98961cfe0fd6 focal-server-cloudimg-amd64-raw.img
+c74a95665b189f93cb16a2c1770820c6a6a43fb0 focal-server-cloudimg-amd64.img

src/bin/ch-remote.rs

@@ -9,6 +9,7 @@ extern crate serde_json;
 extern crate vmm;
 
 use clap::{App, AppSettings, Arg, ArgMatches, SubCommand};
+use std::fmt;
 use std::io::{Read, Write};
 use std::os::unix::net::UnixStream;
 use std::process;
@@ -19,10 +20,44 @@ enum Error {
     StatusCodeParsing(std::num::ParseIntError),
     MissingProtocol,
     ContentLengthParsing(std::num::ParseIntError),
-    ServerResponse(StatusCode),
+    ServerResponse(StatusCode, Option<String>),
     InvalidCPUCount(std::num::ParseIntError),
     InvalidMemorySize(std::num::ParseIntError),
     AddDeviceConfig(vmm::config::Error),
+    AddDiskConfig(vmm::config::Error),
+    AddFsConfig(vmm::config::Error),
+    AddPmemConfig(vmm::config::Error),
+    AddNetConfig(vmm::config::Error),
+    AddVsockConfig(vmm::config::Error),
+    Restore(vmm::config::Error),
+}
+
+impl fmt::Display for Error {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        use Error::*;
+        match self {
+            Socket(e) => write!(f, "Error writing to HTTP socket: {}", e),
+            StatusCodeParsing(e) => write!(f, "Error parsing HTTP status code: {}", e),
+            MissingProtocol => write!(f, "HTTP output is missing protocol statement"),
+            ContentLengthParsing(e) => write!(f, "Error parsing HTTP Content-Length field: {}", e),
+            ServerResponse(s, o) => {
+                if let Some(o) = o {
+                    write!(f, "Server responded with an error: {:?}: {}", s, o)
+                } else {
+                    write!(f, "Server responded with an error: {:?}", s)
+                }
+            }
+            InvalidCPUCount(e) => write!(f, "Error parsing CPU count: {}", e),
+            InvalidMemorySize(e) => write!(f, "Error parsing memory size: {}", e),
+            AddDeviceConfig(e) => write!(f, "Error parsing device syntax: {}", e),
+            AddDiskConfig(e) => write!(f, "Error parsing disk syntax: {}", e),
+            AddFsConfig(e) => write!(f, "Error parsing filesystem syntax: {}", e),
+            AddPmemConfig(e) => write!(f, "Error parsing persistent memory syntax: {}", e),
+            AddNetConfig(e) => write!(f, "Error parsing network syntax: {}", e),
+            AddVsockConfig(e) => write!(f, "Error parsing vsock syntax: {}", e),
+            Restore(e) => write!(f, "Error parsing restore syntax: {}", e),
+        }
+    }
 }
 
 #[derive(Clone, Copy, Debug)]
@@ -57,10 +92,10 @@ impl StatusCode {
         ))
     }
 
-    fn check(self) -> Result<(), Error> {
+    fn is_server_error(self) -> bool {
         match self {
-            StatusCode::OK | StatusCode::Continue | StatusCode::NoContent => Ok(()),
-            _ => Err(Error::ServerResponse(self)),
+            StatusCode::OK | StatusCode::Continue | StatusCode::NoContent => false,
+            _ => true,
         }
     }
 }
@@ -117,10 +152,14 @@ fn parse_http_response(socket: &mut UnixStream) -> Result<Option<String>, Error>
             }
         }
     }
+    let body_string = content_length.and(Some(String::from(&res[body_offset.unwrap()..])));
+    let status_code = get_status_code(&res)?;
 
-    get_status_code(&res)?.check()?;
-
-    Ok(content_length.and(Some(String::from(&res[body_offset.unwrap()..]))))
+    if status_code.is_server_error() {
+        Err(Error::ServerResponse(status_code, body_string))
+    } else {
+        Ok(body_string)
+    }
 }
 
 fn simple_api_command(
@@ -213,6 +252,85 @@ fn remove_device_api_command(socket: &mut UnixStream, id: &str) -> Result<(), Er
     )
 }
 
+fn add_disk_api_command(socket: &mut UnixStream, config: &str) -> Result<(), Error> {
+    let disk_config = vmm::config::DiskConfig::parse(config).map_err(Error::AddDiskConfig)?;
+
+    simple_api_command(
+        socket,
+        "PUT",
+        "add-disk",
+        Some(&serde_json::to_string(&disk_config).unwrap()),
+    )
+}
+
+fn add_fs_api_command(socket: &mut UnixStream, config: &str) -> Result<(), Error> {
+    let fs_config = vmm::config::FsConfig::parse(config).map_err(Error::AddFsConfig)?;
+
+    simple_api_command(
+        socket,
+        "PUT",
+        "add-fs",
+        Some(&serde_json::to_string(&fs_config).unwrap()),
+    )
+}
+
+fn add_pmem_api_command(socket: &mut UnixStream, config: &str) -> Result<(), Error> {
+    let pmem_config = vmm::config::PmemConfig::parse(config).map_err(Error::AddPmemConfig)?;
+
+    simple_api_command(
+        socket,
+        "PUT",
+        "add-pmem",
+        Some(&serde_json::to_string(&pmem_config).unwrap()),
+    )
+}
+
+fn add_net_api_command(socket: &mut UnixStream, config: &str) -> Result<(), Error> {
+    let net_config = vmm::config::NetConfig::parse(config).map_err(Error::AddNetConfig)?;
+
+    simple_api_command(
+        socket,
+        "PUT",
+        "add-net",
+        Some(&serde_json::to_string(&net_config).unwrap()),
+    )
+}
+
+fn add_vsock_api_command(socket: &mut UnixStream, config: &str) -> Result<(), Error> {
+    let vsock_config = vmm::config::VsockConfig::parse(config).map_err(Error::AddVsockConfig)?;
+
+    simple_api_command(
+        socket,
+        "PUT",
+        "add-vsock",
+        Some(&serde_json::to_string(&vsock_config).unwrap()),
+    )
+}
+
+fn snapshot_api_command(socket: &mut UnixStream, url: &str) -> Result<(), Error> {
+    let snapshot_config = vmm::api::VmSnapshotConfig {
+        destination_url: String::from(url),
+    };
+
+    simple_api_command(
+        socket,
+        "PUT",
+        "snapshot",
+        Some(&serde_json::to_string(&snapshot_config).unwrap()),
+    )
+}
+
+fn restore_api_command(socket: &mut UnixStream, config: &str) -> Result<(), Error> {
+    let restore_config = vmm::config::RestoreConfig::parse(config).map_err(Error::Restore)?;
+
+    simple_api_command(
+        socket,
+        "PUT",
+        "restore",
+        Some(&serde_json::to_string(&restore_config).unwrap()),
+    )
+}
+
 fn do_command(matches: &ArgMatches) -> Result<(), Error> {
     let mut socket =
         UnixStream::connect(matches.value_of("api-socket").unwrap()).map_err(Error::Socket)?;
@@ -246,6 +364,62 @@ fn do_command(matches: &ArgMatches) -> Result<(), Error> {
                 .value_of("id")
                 .unwrap(),
         ),
+        Some("add-disk") => add_disk_api_command(
+            &mut socket,
+            matches
+                .subcommand_matches("add-disk")
+                .unwrap()
+                .value_of("disk_config")
+                .unwrap(),
+        ),
+        Some("add-fs") => add_fs_api_command(
+            &mut socket,
+            matches
+                .subcommand_matches("add-fs")
+                .unwrap()
+                .value_of("fs_config")
+                .unwrap(),
+        ),
+        Some("add-pmem") => add_pmem_api_command(
+            &mut socket,
+            matches
+                .subcommand_matches("add-pmem")
+                .unwrap()
+                .value_of("pmem_config")
+                .unwrap(),
+        ),
+        Some("add-net") => add_net_api_command(
+            &mut socket,
+            matches
+                .subcommand_matches("add-net")
+                .unwrap()
+                .value_of("net_config")
+                .unwrap(),
+        ),
+        Some("add-vsock") => add_vsock_api_command(
+            &mut socket,
+            matches
+                .subcommand_matches("add-vsock")
+                .unwrap()
+                .value_of("vsock_config")
+                .unwrap(),
+        ),
+        Some("snapshot") => snapshot_api_command(
+            &mut socket,
+            matches
+                .subcommand_matches("snapshot")
+                .unwrap()
+                .value_of("snapshot_config")
+                .unwrap(),
+        ),
+        Some("restore") => restore_api_command(
+            &mut socket,
+            matches
+                .subcommand_matches("restore")
+                .unwrap()
+                .value_of("restore_config")
+                .unwrap(),
+        ),
         Some(c) => simple_api_command(&mut socket, "PUT", c, None),
         None => unreachable!(),
     }
@@ -261,7 +435,7 @@ fn main() {
                 .long("api-socket")
                 .help("HTTP API socket path (UNIX domain socket).")
                 .takes_value(true)
-                .min_values(1)
+                .number_of_values(1)
                 .required(true),
         )
         .subcommand(
@@ -273,6 +447,51 @@ fn main() {
                         .help(vmm::config::DeviceConfig::SYNTAX),
                 ),
         )
+        .subcommand(
+            SubCommand::with_name("add-disk")
+                .about("Add block device")
+                .arg(
+                    Arg::with_name("disk_config")
+                        .index(1)
+                        .help(vmm::config::DiskConfig::SYNTAX),
+                ),
+        )
+        .subcommand(
+            SubCommand::with_name("add-fs")
+                .about("Add virtio-fs backed fs device")
+                .arg(
+                    Arg::with_name("fs_config")
+                        .index(1)
+                        .help(vmm::config::FsConfig::SYNTAX),
+                ),
+        )
+        .subcommand(
+            SubCommand::with_name("add-pmem")
+                .about("Add persistent memory device")
+                .arg(
+                    Arg::with_name("pmem_config")
+                        .index(1)
+                        .help(vmm::config::PmemConfig::SYNTAX),
+                ),
+        )
+        .subcommand(
+            SubCommand::with_name("add-net")
+                .about("Add network device")
+                .arg(
+                    Arg::with_name("net_config")
+                        .index(1)
+                        .help(vmm::config::NetConfig::SYNTAX),
+                ),
+        )
+        .subcommand(
+            SubCommand::with_name("add-vsock")
+                .about("Add vsock device")
+                .arg(
+                    Arg::with_name("vsock_config")
+                        .index(1)
+                        .help(vmm::config::VsockConfig::SYNTAX),
+                ),
+        )
         .subcommand(
             SubCommand::with_name("remove-device")
                 .about("Remove VFIO device")
@@ -300,12 +519,30 @@ fn main() {
                 ),
         )
         .subcommand(SubCommand::with_name("resume").about("Resume the VM"))
-        .subcommand(SubCommand::with_name("shutdown").about("Shutdown the VM"));
+        .subcommand(SubCommand::with_name("shutdown").about("Shutdown the VM"))
+        .subcommand(
+            SubCommand::with_name("snapshot")
+                .about("Create a snapshot from VM")
+                .arg(
+                    Arg::with_name("snapshot_config")
+                        .index(1)
+                        .help("<destination_url>"),
+                ),
+        )
+        .subcommand(
+            SubCommand::with_name("restore")
+                .about("Restore VM from a snapshot")
+                .arg(
+                    Arg::with_name("restore_config")
+                        .index(1)
+                        .help(vmm::config::RestoreConfig::SYNTAX),
+                ),
+        );
 
     let matches = app.get_matches();
 
     if let Err(e) = do_command(&matches) {
-        eprintln!("Error running command: {:?}", e);
+        eprintln!("Error running command: {}", e);
         process::exit(1)
     };
 }

src/bin/vhost_user_blk.rs

@@ -23,11 +23,7 @@ fn main() {
         .arg(
             Arg::with_name("block-backend")
                 .long("block-backend")
-                .help(
-                    "vhost-user-block backend parameters \
-                     \"image=<image_path>,sock=<socket_path>,num_queues=<number_of_queues>,\
-                     readonly=true|false,direct=true|false,poll_queue=true|false\"",
-                )
+                .help(vhost_user_block::SYNTAX)
                 .takes_value(true)
                 .min_values(1),
         )

src/bin/vhost_user_fs.rs

@@ -10,21 +10,23 @@ extern crate vhost_user_backend;
 extern crate vm_virtio;
 
 use clap::{App, Arg};
-use epoll;
 use futures::executor::{ThreadPool, ThreadPoolBuilder};
 use libc::EFD_NONBLOCK;
 use log::*;
+use seccomp::SeccompAction;
 use std::num::Wrapping;
-use std::sync::{Arc, RwLock};
+use std::sync::{Arc, Mutex, RwLock};
 use std::{convert, error, fmt, io, process};
 
 use vhost_rs::vhost_user::message::*;
-use vhost_rs::vhost_user::SlaveFsCacheReq;
+use vhost_rs::vhost_user::{Listener, SlaveFsCacheReq};
 use vhost_user_backend::{VhostUserBackend, VhostUserDaemon, Vring};
 use vhost_user_fs::descriptor_utils::Error as VufDescriptorError;
 use vhost_user_fs::descriptor_utils::{Reader, Writer};
 use vhost_user_fs::filesystem::FileSystem;
 use vhost_user_fs::passthrough::{self, PassthroughFs};
+use vhost_user_fs::sandbox::Sandbox;
+use vhost_user_fs::seccomp::enable_seccomp;
 use vhost_user_fs::server::Server;
 use vhost_user_fs::Error as VhostUserFsError;
 use virtio_bindings::bindings::virtio_net::*;
@@ -83,7 +85,7 @@ impl convert::From<Error> for io::Error {
     }
 }
 
-struct VhostUserFsBackend<F: FileSystem + Send + Sync + 'static> {
+struct VhostUserFsThread<F: FileSystem + Send + Sync + 'static> {
     mem: Option<GuestMemoryAtomic<GuestMemoryMmap>>,
     kill_evt: EventFd,
     server: Arc<Server<F>>,
@@ -93,9 +95,9 @@ struct VhostUserFsBackend<F: FileSystem + Send + Sync + 'static> {
     pool: ThreadPool,
 }
 
-impl<F: FileSystem + Send + Sync + 'static> Clone for VhostUserFsBackend<F> {
+impl<F: FileSystem + Send + Sync + 'static> Clone for VhostUserFsThread<F> {
     fn clone(&self) -> Self {
-        VhostUserFsBackend {
+        VhostUserFsThread {
             mem: self.mem.clone(),
             kill_evt: self.kill_evt.try_clone().unwrap(),
             server: self.server.clone(),
@@ -106,9 +108,9 @@ impl<F: FileSystem + Send + Sync + 'static> Clone for VhostUserFsBackend<F> {
     }
 }
 
-impl<F: FileSystem + Send + Sync + 'static> VhostUserFsBackend<F> {
+impl<F: FileSystem + Send + Sync + 'static> VhostUserFsThread<F> {
     fn new(fs: F, thread_pool_size: usize) -> Result<Self> {
-        Ok(VhostUserFsBackend {
+        Ok(VhostUserFsThread {
             mem: None,
             kill_evt: EventFd::new(EFD_NONBLOCK).map_err(Error::CreateKillEventFd)?,
             server: Arc::new(Server::new(fs)),
@@ -160,8 +162,9 @@ impl<F: FileSystem + Send + Sync + 'static> VhostUserFsBackend<F> {
                 let mut vring = vring_lock.write().unwrap();
 
                 if event_idx {
-                    if let Some(used_idx) = vring.mut_queue().add_used(&mem, head_index, 0) {
-                        if vring.needs_notification(&mem, Wrapping(used_idx)) {
+                    let queue = vring.mut_queue();
+                    if let Some(used_idx) = queue.add_used(&mem, head_index, 0) {
+                        if queue.needs_notification(&mem, Wrapping(used_idx)) {
                             vring.signal_used_queue().unwrap();
                         }
                     }
@@ -176,6 +179,17 @@ impl<F: FileSystem + Send + Sync + 'static> VhostUserFsBackend<F> {
     }
 }
 
+struct VhostUserFsBackend<F: FileSystem + Send + Sync + 'static> {
+    thread: Mutex<VhostUserFsThread<F>>,
+}
+
+impl<F: FileSystem + Send + Sync + 'static> VhostUserFsBackend<F> {
+    fn new(fs: F, thread_pool_size: usize) -> Result<Self> {
+        let thread = Mutex::new(VhostUserFsThread::new(fs, thread_pool_size)?);
+        Ok(VhostUserFsBackend { thread })
+    }
+}
+
 impl<F: FileSystem + Send + Sync + 'static> VhostUserBackend for VhostUserFsBackend<F> {
     fn num_queues(&self) -> usize {
         NUM_QUEUES
@@ -197,25 +211,27 @@ impl<F: FileSystem + Send + Sync + 'static> VhostUserBackend for VhostUserFsBack
     }
 
     fn set_event_idx(&mut self, enabled: bool) {
-        self.event_idx = enabled;
+        self.thread.lock().unwrap().event_idx = enabled;
     }
 
     fn update_memory(&mut self, mem: GuestMemoryMmap) -> VhostUserBackendResult<()> {
-        self.mem = Some(GuestMemoryAtomic::new(mem));
+        self.thread.lock().unwrap().mem = Some(GuestMemoryAtomic::new(mem));
         Ok(())
     }
 
     fn handle_event(
-        &mut self,
+        &self,
         device_event: u16,
         evset: epoll::Events,
         vrings: &[Arc<RwLock<Vring>>],
+        _thread_id: usize,
     ) -> VhostUserBackendResult<bool> {
         if evset != epoll::Events::EPOLLIN {
             return Err(Error::HandleEventNotEpollIn.into());
         }
 
-        let mem = match &self.mem {
+        let mut thread = self.thread.lock().unwrap();
+        let mem = match &thread.mem {
             Some(m) => m.memory(),
             None => return Err(Error::NoMemoryConfigured.into()),
         };
@@ -232,7 +248,7 @@ impl<F: FileSystem + Send + Sync + 'static> VhostUserBackend for VhostUserFsBack
             _ => return Err(Error::HandleEventUnknownEvent.into()),
         };
 
-        if self.event_idx {
+        if thread.event_idx {
             // vm-virtio's Queue implementation only checks avail_index
             // once, so to properly support EVENT_IDX we need to keep
             // calling process_queue() until it stops finding new
@@ -242,24 +258,27 @@ impl<F: FileSystem + Send + Sync + 'static> VhostUserBackend for VhostUserFsBack
                     let mut vring = vring_lock.write().unwrap();
                     vring.mut_queue().update_avail_event(&mem);
                 }
-                if !self.process_queue(vring_lock.clone())? {
+                if !thread.process_queue(vring_lock.clone())? {
                     break;
                 }
             }
         } else {
             // Without EVENT_IDX, a single call is enough.
-            self.process_queue(vring_lock)?;
+            thread.process_queue(vring_lock)?;
         }
 
         Ok(false)
     }
 
-    fn exit_event(&self) -> Option<(EventFd, Option<u16>)> {
-        Some((self.kill_evt.try_clone().unwrap(), Some(KILL_EVENT)))
+    fn exit_event(&self, _thread_index: usize) -> Option<(EventFd, Option<u16>)> {
+        Some((
+            self.thread.lock().unwrap().kill_evt.try_clone().unwrap(),
+            Some(KILL_EVENT),
+        ))
     }
 
     fn set_slave_req_fd(&mut self, vu_req: SlaveFsCacheReq) {
-        self.vu_req = Some(vu_req);
+        self.thread.lock().unwrap().vu_req = Some(vu_req);
     }
 }
 
@@ -278,6 +297,13 @@ fn main() {
         .arg(
             Arg::with_name("sock")
                 .long("sock")
+                .help("vhost-user socket path (deprecated)")
+                .takes_value(true)
+                .min_values(1),
+        )
+        .arg(
+            Arg::with_name("socket")
+                .long("socket")
                 .help("vhost-user socket path")
                 .takes_value(true)
                 .min_values(1),
@@ -294,42 +320,86 @@ fn main() {
                 .long("disable-xattr")
                 .help("Disable support for extended attributes"),
         )
+        .arg(
+            Arg::with_name("disable-sandbox")
+                .long("disable-sandbox")
+                .help("Don't set up a sandbox for the daemon"),
+        )
+        .arg(
+            Arg::with_name("seccomp")
+                .long("seccomp")
+                .help("Disable/debug seccomp security")
+                .possible_values(&["kill", "log", "trap", "none"])
+                .default_value("kill"),
+        )
         .get_matches();
 
     // Retrieve arguments
     let shared_dir = cmd_arguments
         .value_of("shared-dir")
         .expect("Failed to retrieve shared directory path");
-    let sock = cmd_arguments
-        .value_of("sock")
-        .expect("Failed to retrieve vhost-user socket path");
+    let socket = match cmd_arguments.value_of("socket") {
+        Some(path) => path,
+        None => {
+            println!("warning: use of deprecated parameter '--sock': Please use the '--socket' option instead.");
+            cmd_arguments
+                .value_of("sock")
+                .expect("Failed to retrieve vhost-user socket path")
+        }
+    };
+
     let thread_pool_size: usize = match cmd_arguments.value_of("thread-pool-size") {
         Some(size) => size.parse().expect("Invalid argument for thread-pool-size"),
         None => THREAD_POOL_SIZE,
     };
     let xattr: bool = !cmd_arguments.is_present("disable-xattr");
-
-    // Convert into appropriate types
-    let sock = String::from(sock);
-
-    let fs_cfg = passthrough::Config {
-        root_dir: shared_dir.to_string(),
-        xattr,
-        ..Default::default()
+    let create_sandbox: bool = !cmd_arguments.is_present("disable-sandbox");
+    let seccomp_mode: SeccompAction = match cmd_arguments.value_of("seccomp").unwrap() {
+        "none" => SeccompAction::Allow, // i.e. no seccomp
+        "kill" => SeccompAction::Kill,
+        "log" => SeccompAction::Log,
+        "trap" => SeccompAction::Trap,
+        _ => unreachable!(), // We told Arg possible_values
     };
+
+    let listener = Listener::new(socket, true).unwrap();
+
+    let fs_cfg = if create_sandbox {
+        let mut sandbox = Sandbox::new(shared_dir.to_string());
+        match sandbox.enter().unwrap() {
+            Some(child_pid) => {
+                unsafe { libc::waitpid(child_pid, std::ptr::null_mut(), 0) };
+                return;
+            }
+            None => passthrough::Config {
+                root_dir: "/".to_string(),
+                xattr,
+                proc_sfd_rawfd: sandbox.get_proc_self_fd(),
+                ..Default::default()
+            },
+        }
+    } else {
+        passthrough::Config {
+            root_dir: shared_dir.to_string(),
+            xattr,
+            ..Default::default()
+        }
+    };
+
+    // Must happen before we start the thread pool
+    if seccomp_mode != SeccompAction::Allow {
+        enable_seccomp(seccomp_mode).unwrap();
+    };
+
     let fs = PassthroughFs::new(fs_cfg).unwrap();
     let fs_backend = Arc::new(RwLock::new(
         VhostUserFsBackend::new(fs, thread_pool_size).unwrap(),
     ));
 
-    let mut daemon = VhostUserDaemon::new(
-        String::from("vhost-user-fs-backend"),
-        sock,
-        fs_backend.clone(),
-    )
-    .unwrap();
+    let mut daemon =
+        VhostUserDaemon::new(String::from("vhost-user-fs-backend"), fs_backend.clone()).unwrap();
 
-    if let Err(e) = daemon.start() {
+    if let Err(e) = daemon.start(listener) {
         error!("Failed to start daemon: {:?}", e);
         process::exit(1);
     }
@@ -338,7 +408,15 @@ fn main() {
         error!("Waiting for daemon failed: {:?}", e);
     }
 
-    let kill_evt = &fs_backend.read().unwrap().kill_evt;
+    let kill_evt = fs_backend
+        .read()
+        .unwrap()
+        .thread
+        .lock()
+        .unwrap()
+        .kill_evt
+        .try_clone()
+        .unwrap();
     if let Err(e) = kill_evt.write(1) {
         error!("Error shutting down worker thread: {:?}", e)
     }

src/bin/vhost_user_net.rs

@@ -21,11 +21,7 @@ fn main() {
         .arg(
             Arg::with_name("net-backend")
                 .long("net-backend")
-                .help(
-                    "vhost-user-net backend parameters \
-                     \"ip=<ip_addr>,mask=<net_mask>,sock=<socket_path>,\
-                     num_queues=<number_of_queues>,queue_size=<size_of_each_queue>,tap=<if_name>\"",
-                )
+                .help(vhost_user_net::SYNTAX)
                 .takes_value(true)
                 .min_values(1),
         )

test_data/cloud-init/clear/openstack/latest/user_data

@@ -74,7 +74,7 @@ write_files:
         bash -c "echo 1af4 1041 > /sys/bus/pci/drivers/vfio-pci/new_id"
         bash -c "echo 0000:00:06.0 > /sys/bus/pci/devices/0000\:00\:06.0/driver/unbind"
         bash -c "echo 1af4 1041 > /sys/bus/pci/drivers/vfio-pci/new_id"
-        # 512M ram requires 256 pages
-        echo 256 | sudo tee /proc/sys/vm/nr_hugepages
+        # 1G ram requires 512 pages
+        echo 512 | sudo tee /proc/sys/vm/nr_hugepages
         sudo chmod a+rwX /dev/hugepages
-        /mnt/cloud-hypervisor --kernel /mnt/vmlinux --cmdline "console=hvc0 reboot=k panic=1 nomodules i8042.noaux i8042.nomux i8042.nopnp i8042.dumbkbd root=PARTUUID=6fb4d1a8-6c8c-4dd7-9f7c-1fe0b9f2574c VFIOTAG" --disk path=/mnt/clear-31311-cloudguest.img path=/mnt/cloudinit.img --cpus boot=1 --memory size=512M,file=/dev/hugepages --device path=/sys/bus/pci/devices/0000:00:05.0/ path=/sys/bus/pci/devices/0000:00:06.0/ --api-socket /tmp/ch_api.sock
+        /mnt/cloud-hypervisor --kernel /mnt/vmlinux --cmdline "console=hvc0 reboot=k panic=1 nomodules i8042.noaux i8042.nomux i8042.nopnp i8042.dumbkbd root=PARTUUID=6fb4d1a8-6c8c-4dd7-9f7c-1fe0b9f2574c VFIOTAG" --disk path=/mnt/clear-31311-cloudguest.img path=/mnt/cloudinit.img --cpus boot=1 --memory size=512M,hotplug_size=1G,file=/dev/hugepages --device path=/sys/bus/pci/devices/0000:00:05.0/ path=/sys/bus/pci/devices/0000:00:06.0/ --api-socket /tmp/ch_api.sock

vfio/Cargo.toml

@@ -1,22 +0,0 @@
-[package]
-name = "vfio"
-version = "0.0.1"
-authors = ["The Cloud Hypervisor Authors"]
-
-[dependencies]
-arc-swap = ">=0.4.4"
-byteorder = "1.3.4"
-devices = { path = "../devices" }
-kvm-bindings = "0.2.0"
-kvm-ioctls = "0.5.0"
-libc = "0.2.68"
-log = "0.4.8"
-pci = { path = "../pci" }
-vm-allocator = { path = "../vm-allocator" }
-vm-device = { path = "../vm-device" }
-vm-memory = { version = "0.1.0", features = ["backend-mmap"] }
-vmm-sys-util = ">=0.3.1"
-
-[dependencies.vfio-bindings]
-git = "https://github.com/rust-vmm/vfio-bindings"
-features = ["fam-wrappers"]

vfio/src/lib.rs

@@ -1,61 +0,0 @@
-// Copyright © 2019 Intel Corporation
-//
-// SPDX-License-Identifier: Apache-2.0 OR BSD-3-Clause
-//
-
-//#![deny(missing_docs)]
-//! Virtual Function I/O (VFIO) API
-extern crate arc_swap;
-extern crate byteorder;
-extern crate devices;
-extern crate kvm_bindings;
-extern crate kvm_ioctls;
-#[macro_use]
-extern crate log;
-extern crate pci;
-extern crate vfio_bindings;
-extern crate vm_allocator;
-extern crate vm_device;
-extern crate vm_memory;
-#[macro_use]
-extern crate vmm_sys_util;
-
-mod vfio_device;
-mod vfio_ioctls;
-mod vfio_pci;
-
-use std::mem::size_of;
-
-pub use vfio_device::{VfioContainer, VfioDevice, VfioDmaMapping, VfioError};
-pub use vfio_pci::{VfioPciDevice, VfioPciError};
-
-// Returns a `Vec<T>` with a size in bytes at least as large as `size_in_bytes`.
-fn vec_with_size_in_bytes<T: Default>(size_in_bytes: usize) -> Vec<T> {
-    let rounded_size = (size_in_bytes + size_of::<T>() - 1) / size_of::<T>();
-    let mut v = Vec::with_capacity(rounded_size);
-    for _ in 0..rounded_size {
-        v.push(T::default())
-    }
-    v
-}
-
-// The kvm API has many structs that resemble the following `Foo` structure:
-//
-// ```
-// #[repr(C)]
-// struct Foo {
-//    some_data: u32
-//    entries: __IncompleteArrayField<__u32>,
-// }
-// ```
-//
-// In order to allocate such a structure, `size_of::<Foo>()` would be too small because it would not
-// include any space for `entries`. To make the allocation large enough while still being aligned
-// for `Foo`, a `Vec<Foo>` is created. Only the first element of `Vec<Foo>` would actually be used
-// as a `Foo`. The remaining memory in the `Vec<Foo>` is for `entries`, which must be contiguous
-// with `Foo`. This function is used to make the `Vec<Foo>` with enough space for `count` entries.
-pub fn vec_with_array_field<T: Default, F>(count: usize) -> Vec<T> {
-    let element_space = count * size_of::<F>();
-    let vec_size_bytes = size_of::<T>() + element_space;
-    vec_with_size_in_bytes(vec_size_bytes)
-}

vfio/src/vfio_device.rs

@@ -1,902 +0,0 @@
-// Copyright © 2019 Intel Corporation
-//
-// SPDX-License-Identifier: Apache-2.0 OR BSD-3-Clause
-//
-use crate::vec_with_array_field;
-use byteorder::{ByteOrder, LittleEndian};
-use kvm_ioctls::*;
-use std::collections::HashMap;
-use std::convert::TryInto;
-use std::ffi::CString;
-use std::fmt;
-use std::fs::{File, OpenOptions};
-use std::io;
-use std::mem;
-use std::num;
-use std::os::unix::io::{AsRawFd, FromRawFd, RawFd};
-use std::os::unix::prelude::FileExt;
-use std::path::{Path, PathBuf};
-use std::result;
-use std::sync::Arc;
-use std::u32;
-use vfio_bindings::bindings::vfio::*;
-use vfio_bindings::bindings::IrqSet;
-use vfio_ioctls::*;
-use vm_device::{get_host_address_range, ExternalDmaMapping};
-use vm_memory::{
-    Address, GuestAddress, GuestAddressSpace, GuestMemory, GuestMemoryAtomic, GuestMemoryMmap,
-    GuestMemoryRegion,
-};
-use vmm_sys_util::eventfd::EventFd;
-use vmm_sys_util::fam::FamStruct;
-use vmm_sys_util::ioctl::*;
-
-#[derive(Debug)]
-pub enum VfioError {
-    OpenContainer(io::Error),
-    OpenGroup(io::Error),
-    GetGroupStatus,
-    GroupViable,
-    VfioApiVersion,
-    VfioExtension,
-    VfioInvalidType,
-    VfioType1V2,
-    GroupSetContainer,
-    UnsetContainer,
-    ContainerSetIOMMU,
-    GroupGetDeviceFD,
-    KvmSetDeviceAttr(kvm_ioctls::Error),
-    VfioDeviceGetInfo,
-    VfioDeviceGetRegionInfo,
-    InvalidPath,
-    IommuDmaMap,
-    IommuDmaUnmap,
-    VfioDeviceGetIrqInfo,
-    VfioDeviceSetIrq,
-    VfioNoIommuGroup(String),
-    ReadLink(io::Error),
-    ParseInt(num::ParseIntError),
-}
-pub type Result<T> = std::result::Result<T, VfioError>;
-
-impl fmt::Display for VfioError {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        match self {
-            VfioError::OpenContainer(e) => {
-                write!(f, "failed to open /dev/vfio/vfio container: {}", e)
-            }
-            VfioError::OpenGroup(e) => {
-                write!(f, "failed to open /dev/vfio/$group_num group: {}", e)
-            }
-            VfioError::GetGroupStatus => write!(f, "failed to get Group Status"),
-            VfioError::GroupViable => write!(f, "group is inviable"),
-            VfioError::VfioApiVersion => write!(
-                f,
-                "vfio API version doesn't match with VFIO_API_VERSION defined in vfio-bindings"
-            ),
-            VfioError::VfioExtension => write!(f, "failed to check VFIO extension"),
-            VfioError::VfioInvalidType => write!(f, "invalid VFIO type"),
-            VfioError::VfioType1V2 => {
-                write!(f, "container dones't support VfioType1V2 IOMMU driver type")
-            }
-            VfioError::GroupSetContainer => {
-                write!(f, "failed to add vfio group into vfio container")
-            }
-            VfioError::UnsetContainer => write!(f, "failed to unset vfio container"),
-            VfioError::ContainerSetIOMMU => write!(
-                f,
-                "failed to set container's IOMMU driver type as VfioType1V2"
-            ),
-            VfioError::GroupGetDeviceFD => write!(f, "failed to get vfio device fd"),
-            VfioError::KvmSetDeviceAttr(e) => {
-                write!(f, "failed to set KVM vfio device's attribute: {}", e)
-            }
-            VfioError::VfioDeviceGetInfo => {
-                write!(f, "failed to get vfio device's info or info doesn't match")
-            }
-            VfioError::VfioDeviceGetRegionInfo => {
-                write!(f, "failed to get vfio device's region info")
-            }
-            VfioError::InvalidPath => write!(f, "invalid file path"),
-            VfioError::IommuDmaMap => write!(f, "failed to add guest memory map into iommu table"),
-            VfioError::IommuDmaUnmap => {
-                write!(f, "failed to remove guest memory map from iommu table")
-            }
-            VfioError::VfioDeviceGetIrqInfo => write!(f, "failed to get vfio device irq info"),
-            VfioError::VfioDeviceSetIrq => write!(f, "failed to set vfio deviece irq"),
-            VfioError::ReadLink(e) => write!(f, "failed to read link from path: {}", e),
-            VfioError::VfioNoIommuGroup(e) => write!(f, "Failed to find iommu_group for: {}", e),
-            VfioError::ParseInt(e) => write!(f, "failed to parse integer: {}", e),
-        }
-    }
-}
-
-#[repr(C)]
-#[derive(Debug, Default)]
-struct vfio_region_info_with_cap {
-    region_info: vfio_region_info,
-    cap_info: __IncompleteArrayField<u8>,
-}
-
-pub struct VfioContainer {
-    container: File,
-}
-
-impl VfioContainer {
-    fn new() -> Result<Self> {
-        let container = OpenOptions::new()
-            .read(true)
-            .write(true)
-            .open("/dev/vfio/vfio")
-            .map_err(VfioError::OpenContainer)?;
-
-        Ok(VfioContainer { container })
-    }
-
-    fn get_api_version(&self) -> i32 {
-        // Safe as file is vfio container fd and ioctl is defined by kernel.
-        unsafe { ioctl(self, VFIO_GET_API_VERSION()) }
-    }
-
-    fn check_extension(&self, val: u32) -> Result<()> {
-        if val != VFIO_TYPE1_IOMMU && val != VFIO_TYPE1v2_IOMMU {
-            return Err(VfioError::VfioInvalidType);
-        }
-
-        // Safe as file is vfio container and make sure val is valid.
-        let ret = unsafe { ioctl_with_val(self, VFIO_CHECK_EXTENSION(), val.into()) };
-        if ret != 1 {
-            return Err(VfioError::VfioExtension);
-        }
-
-        Ok(())
-    }
-
-    fn set_iommu(&self, val: u32) -> Result<()> {
-        if val != VFIO_TYPE1_IOMMU && val != VFIO_TYPE1v2_IOMMU {
-            return Err(VfioError::VfioInvalidType);
-        }
-
-        // Safe as file is vfio container and make sure val is valid.
-        let ret = unsafe { ioctl_with_val(self, VFIO_SET_IOMMU(), val.into()) };
-        if ret < 0 {
-            return Err(VfioError::ContainerSetIOMMU);
-        }
-
-        Ok(())
-    }
-
-    pub fn vfio_dma_map(&self, iova: u64, size: u64, user_addr: u64) -> Result<()> {
-        let dma_map = vfio_iommu_type1_dma_map {
-            argsz: mem::size_of::<vfio_iommu_type1_dma_map>() as u32,
-            flags: VFIO_DMA_MAP_FLAG_READ | VFIO_DMA_MAP_FLAG_WRITE,
-            vaddr: user_addr,
-            iova,
-            size,
-        };
-
-        // Safe as file is vfio container, dma_map is constructed by us, and
-        // we check the return value
-        let ret = unsafe { ioctl_with_ref(self, VFIO_IOMMU_MAP_DMA(), &dma_map) };
-        if ret != 0 {
-            return Err(VfioError::IommuDmaMap);
-        }
-
-        Ok(())
-    }
-
-    pub fn vfio_dma_unmap(&self, iova: u64, size: u64) -> Result<()> {
-        let mut dma_unmap = vfio_iommu_type1_dma_unmap {
-            argsz: mem::size_of::<vfio_iommu_type1_dma_unmap>() as u32,
-            flags: 0,
-            iova,
-            size,
-        };
-
-        // Safe as file is vfio container, dma_unmap is constructed by us, and
-        // we check the return value
-        let ret = unsafe { ioctl_with_mut_ref(self, VFIO_IOMMU_UNMAP_DMA(), &mut dma_unmap) };
-        if ret != 0 || dma_unmap.size != size {
-            return Err(VfioError::IommuDmaUnmap);
-        }
-
-        Ok(())
-    }
-}
-
-impl AsRawFd for VfioContainer {
-    fn as_raw_fd(&self) -> RawFd {
-        self.container.as_raw_fd()
-    }
-}
-
-struct VfioGroup {
-    group: File,
-    device: Arc<DeviceFd>,
-    container: Arc<VfioContainer>,
-}
-
-impl VfioGroup {
-    fn new(id: u32, device: Arc<DeviceFd>) -> Result<Self> {
-        let group_path = Path::new("/dev/vfio").join(id.to_string());
-        let group = OpenOptions::new()
-            .read(true)
-            .write(true)
-            .open(&group_path)
-            .map_err(VfioError::OpenGroup)?;
-
-        let mut group_status = vfio_group_status {
-            argsz: mem::size_of::<vfio_group_status>() as u32,
-            flags: 0,
-        };
-        // Safe as we are the owner of group and group_status which are valid value.
-        let mut ret =
-            unsafe { ioctl_with_mut_ref(&group, VFIO_GROUP_GET_STATUS(), &mut group_status) };
-        if ret < 0 {
-            return Err(VfioError::GetGroupStatus);
-        }
-
-        if group_status.flags != VFIO_GROUP_FLAGS_VIABLE {
-            return Err(VfioError::GroupViable);
-        }
-
-        let container = Arc::new(VfioContainer::new()?);
-        if container.get_api_version() as u32 != VFIO_API_VERSION {
-            return Err(VfioError::VfioApiVersion);
-        }
-
-        container.check_extension(VFIO_TYPE1v2_IOMMU)?;
-
-        // Safe as we are the owner of group and container_raw_fd which are valid value,
-        // and we verify the ret value
-        let container_raw_fd = container.as_raw_fd();
-        ret = unsafe { ioctl_with_ref(&group, VFIO_GROUP_SET_CONTAINER(), &container_raw_fd) };
-        if ret < 0 {
-            return Err(VfioError::GroupSetContainer);
-        }
-
-        container.set_iommu(VFIO_TYPE1v2_IOMMU)?;
-
-        Self::kvm_device_add_group(&device, &group)?;
-
-        Ok(VfioGroup {
-            group,
-            device,
-            container,
-        })
-    }
-
-    fn kvm_device_add_group(device_fd: &Arc<DeviceFd>, group: &File) -> Result<()> {
-        let group_fd = group.as_raw_fd();
-        let group_fd_ptr = &group_fd as *const i32;
-        let dev_attr = kvm_bindings::kvm_device_attr {
-            flags: 0,
-            group: kvm_bindings::KVM_DEV_VFIO_GROUP,
-            attr: u64::from(kvm_bindings::KVM_DEV_VFIO_GROUP_ADD),
-            addr: group_fd_ptr as u64,
-        };
-
-        device_fd
-            .set_device_attr(&dev_attr)
-            .map_err(VfioError::KvmSetDeviceAttr)
-    }
-
-    fn kvm_device_del_group(&self) -> std::result::Result<(), kvm_ioctls::Error> {
-        let group_fd = self.as_raw_fd();
-        let group_fd_ptr = &group_fd as *const i32;
-        let dev_attr = kvm_bindings::kvm_device_attr {
-            flags: 0,
-            group: kvm_bindings::KVM_DEV_VFIO_GROUP,
-            attr: u64::from(kvm_bindings::KVM_DEV_VFIO_GROUP_DEL),
-            addr: group_fd_ptr as u64,
-        };
-
-        self.device.set_device_attr(&dev_attr)
-    }
-
-    fn unset_container(&self) -> std::result::Result<(), io::Error> {
-        let container_raw_fd = self.container.as_raw_fd();
-
-        // Safe as we are the owner of self and container_raw_fd which are valid value.
-        let ret = unsafe { ioctl_with_ref(self, VFIO_GROUP_UNSET_CONTAINER(), &container_raw_fd) };
-        if ret < 0 {
-            return Err(io::Error::last_os_error());
-        }
-
-        Ok(())
-    }
-
-    fn get_device(&self, name: &Path) -> Result<VfioDeviceInfo> {
-        let uuid_osstr = name.file_name().ok_or(VfioError::InvalidPath)?;
-        let uuid_str = uuid_osstr.to_str().ok_or(VfioError::InvalidPath)?;
-        let path: CString = CString::new(uuid_str.as_bytes()).expect("CString::new() failed");
-        let path_ptr = path.as_ptr();
-
-        // Safe as we are the owner of self and path_ptr which are valid value.
-        let fd = unsafe { ioctl_with_ptr(self, VFIO_GROUP_GET_DEVICE_FD(), path_ptr) };
-        if fd < 0 {
-            return Err(VfioError::GroupGetDeviceFD);
-        }
-
-        // Safe as fd is valid FD
-        let device = unsafe { File::from_raw_fd(fd) };
-
-        let mut dev_info = vfio_device_info {
-            argsz: mem::size_of::<vfio_device_info>() as u32,
-            flags: 0,
-            num_regions: 0,
-            num_irqs: 0,
-        };
-        // Safe as we are the owner of dev and dev_info which are valid value,
-        // and we verify the return value.
-        let ret = unsafe { ioctl_with_mut_ref(&device, VFIO_DEVICE_GET_INFO(), &mut dev_info) };
-        if ret < 0
-            || (dev_info.flags & VFIO_DEVICE_FLAGS_PCI) == 0
-            || dev_info.num_regions < VFIO_PCI_CONFIG_REGION_INDEX + 1
-            || dev_info.num_irqs < VFIO_PCI_MSIX_IRQ_INDEX + 1
-        {
-            return Err(VfioError::VfioDeviceGetInfo);
-        }
-
-        Ok(VfioDeviceInfo {
-            device,
-            flags: dev_info.flags,
-            num_regions: dev_info.num_regions,
-            num_irqs: dev_info.num_irqs,
-        })
-    }
-}
-
-impl AsRawFd for VfioGroup {
-    fn as_raw_fd(&self) -> RawFd {
-        self.group.as_raw_fd()
-    }
-}
-
-impl Drop for VfioGroup {
-    fn drop(&mut self) {
-        match self.kvm_device_del_group() {
-            Ok(_) => {}
-            Err(e) => {
-                error!("Could not delete VFIO group: {:?}", e);
-            }
-        }
-
-        match self.unset_container() {
-            Ok(_) => {}
-            Err(e) => {
-                error!("Could not unset container: {:?}", e);
-            }
-        }
-    }
-}
-
-struct VfioRegion {
-    flags: u32,
-    size: u64,
-    offset: u64,
-    mmap: (u64, u64),
-}
-
-struct VfioIrq {
-    flags: u32,
-    index: u32,
-    count: u32,
-}
-
-struct VfioDeviceInfo {
-    device: File,
-    flags: u32,
-    num_regions: u32,
-    num_irqs: u32,
-}
-
-impl VfioDeviceInfo {
-    fn get_irqs(&self) -> Result<HashMap<u32, VfioIrq>> {
-        let mut irqs: HashMap<u32, VfioIrq> = HashMap::new();
-
-        for index in 0..self.num_irqs {
-            let mut irq_info = vfio_irq_info {
-                argsz: mem::size_of::<vfio_irq_info>() as u32,
-                flags: 0,
-                index,
-                count: 0,
-            };
-
-            let ret = unsafe {
-                ioctl_with_mut_ref(&self.device, VFIO_DEVICE_GET_IRQ_INFO(), &mut irq_info)
-            };
-            if ret < 0 {
-                warn!("Could not get VFIO IRQ info for index {:}", index);
-                continue;
-            }
-
-            let irq = VfioIrq {
-                flags: irq_info.flags,
-                index,
-                count: irq_info.count,
-            };
-
-            debug!("IRQ #{}", index);
-            debug!("\tflag 0x{:x}", irq.flags);
-            debug!("\tindex {}", irq.index);
-            debug!("\tcount {}", irq.count);
-
-            irqs.insert(index, irq);
-        }
-
-        Ok(irqs)
-    }
-
-    fn get_regions(&self) -> Result<Vec<VfioRegion>> {
-        let mut regions: Vec<VfioRegion> = Vec::new();
-
-        for i in VFIO_PCI_BAR0_REGION_INDEX..self.num_regions {
-            let argsz: u32 = mem::size_of::<vfio_region_info>() as u32;
-
-            let mut reg_info = vfio_region_info {
-                argsz,
-                flags: 0,
-                index: i,
-                cap_offset: 0,
-                size: 0,
-                offset: 0,
-            };
-            // Safe as we are the owner of dev and reg_info which are valid value,
-            // and we verify the return value.
-            let mut ret = unsafe {
-                ioctl_with_mut_ref(&self.device, VFIO_DEVICE_GET_REGION_INFO(), &mut reg_info)
-            };
-            if ret < 0 {
-                warn!("Could not get region #{} info", i);
-                continue;
-            }
-
-            let mut mmap_size: u64 = reg_info.size;
-            let mut mmap_offset: u64 = 0;
-            if reg_info.flags & VFIO_REGION_INFO_FLAG_CAPS != 0 && reg_info.argsz > argsz {
-                let cap_len: usize = (reg_info.argsz - argsz) as usize;
-                let mut region_with_cap =
-                    vec_with_array_field::<vfio_region_info_with_cap, u8>(cap_len);
-                region_with_cap[0].region_info.argsz = reg_info.argsz;
-                region_with_cap[0].region_info.flags = 0;
-                region_with_cap[0].region_info.index = i;
-                region_with_cap[0].region_info.cap_offset = 0;
-                region_with_cap[0].region_info.size = 0;
-                region_with_cap[0].region_info.offset = 0;
-                // Safe as we are the owner of dev and region_info which are valid value,
-                // and we verify the return value.
-                ret = unsafe {
-                    ioctl_with_mut_ref(
-                        &self.device,
-                        VFIO_DEVICE_GET_REGION_INFO(),
-                        &mut (region_with_cap[0].region_info),
-                    )
-                };
-                if ret < 0 {
-                    warn!("Could not get region #{} info", i);
-                    continue;
-                }
-                // region_with_cap[0].cap_info may contain vfio_region_info_cap_sparse_mmap
-                // struct or vfio_region_info_cap_type struct. Both of them begin with
-                // vfio_info_cap_header.
-                // so safe to convert cap_info into vfio_info_cap_header pointer first, and
-                // safe to access its elments through this poiner.
-                #[allow(clippy::cast_ptr_alignment)]
-                let cap_header =
-                    unsafe { region_with_cap[0].cap_info.as_ptr() as *const vfio_info_cap_header };
-                if unsafe { u32::from((*cap_header).id) } == VFIO_REGION_INFO_CAP_SPARSE_MMAP {
-                    // cap_info is vfio_region_sparse_mmap here
-                    // so safe to convert cap_info into vfio_info_region_sparse_mmap pointer, and
-                    // safe to access its elements through this pointer.
-                    #[allow(clippy::cast_ptr_alignment)]
-                    let sparse_mmap = unsafe {
-                        region_with_cap[0].cap_info.as_ptr()
-                            as *const vfio_region_info_cap_sparse_mmap
-                    };
-                    let mmap_area = unsafe {
-                        (*sparse_mmap).areas.as_ptr() as *const vfio_region_sparse_mmap_area
-                    };
-                    mmap_size = unsafe { (*mmap_area).size };
-                    mmap_offset = unsafe { (*mmap_area).offset };
-                }
-            }
-
-            let region = VfioRegion {
-                flags: reg_info.flags,
-                size: reg_info.size,
-                offset: reg_info.offset,
-                mmap: (mmap_offset, mmap_size),
-            };
-
-            debug!("Region #{}", i);
-            debug!("\tflag 0x{:x}", region.flags);
-            debug!("\tsize 0x{:x}", region.size);
-            debug!("\toffset 0x{:x}", region.offset);
-
-            regions.push(region);
-        }
-
-        Ok(regions)
-    }
-}
-
-/// This structure implements the ExternalDmaMapping trait. It is meant to
-/// be used when the caller tries to provide a way to update the mappings
-/// associated with a specific VFIO container.
-pub struct VfioDmaMapping {
-    container: Arc<VfioContainer>,
-    memory: GuestMemoryAtomic<GuestMemoryMmap>,
-}
-
-impl VfioDmaMapping {
-    pub fn new(container: Arc<VfioContainer>, memory: GuestMemoryAtomic<GuestMemoryMmap>) -> Self {
-        VfioDmaMapping { container, memory }
-    }
-}
-
-impl ExternalDmaMapping for VfioDmaMapping {
-    fn map(&self, iova: u64, gpa: u64, size: u64) -> result::Result<(), io::Error> {
-        let user_addr = if let Some(addr) = get_host_address_range(
-            &self.memory.memory(),
-            GuestAddress(gpa),
-            size.try_into().unwrap(),
-        ) {
-            addr as u64
-        } else {
-            return Err(io::Error::new(
-                io::ErrorKind::Other,
-                format!(
-                    "failed to convert guest address 0x{:x} into \
-                     host user virtual address",
-                    gpa
-                ),
-            ));
-        };
-
-        self.container
-            .vfio_dma_map(iova, size, user_addr)
-            .map_err(|e| {
-                io::Error::new(
-                    io::ErrorKind::Other,
-                    format!(
-                        "failed to map memory for VFIO container, \
-                         iova 0x{:x}, gpa 0x{:x}, size 0x{:x}: {:?}",
-                        iova, gpa, size, e
-                    ),
-                )
-            })
-    }
-
-    fn unmap(&self, iova: u64, size: u64) -> result::Result<(), io::Error> {
-        self.container.vfio_dma_unmap(iova, size).map_err(|e| {
-            io::Error::new(
-                io::ErrorKind::Other,
-                format!(
-                    "failed to unmap memory for VFIO container, \
-                     iova 0x{:x}, size 0x{:x}: {:?}",
-                    iova, size, e
-                ),
-            )
-        })
-    }
-}
-
-/// Vfio device for exposing regions which could be read/write to kernel vfio device.
-pub struct VfioDevice {
-    device: File,
-    flags: u32,
-    group: VfioGroup,
-    regions: Vec<VfioRegion>,
-    irqs: HashMap<u32, VfioIrq>,
-    mem: GuestMemoryAtomic<GuestMemoryMmap>,
-    iommu_attached: bool,
-}
-
-impl VfioDevice {
-    /// Create a new vfio device, then guest read/write on this device could be
-    /// transfered into kernel vfio.
-    /// sysfspath specify the vfio device path in sys file system.
-    pub fn new(
-        sysfspath: &Path,
-        device_fd: Arc<DeviceFd>,
-        mem: GuestMemoryAtomic<GuestMemoryMmap>,
-        iommu_attached: bool,
-    ) -> Result<Self> {
-        let uuid_path: PathBuf = [sysfspath, Path::new("iommu_group")].iter().collect();
-        if !uuid_path.exists() {
-            return Err(VfioError::VfioNoIommuGroup(
-                uuid_path.to_str().unwrap().to_string(),
-            ));
-        }
-        let group_path = uuid_path.read_link().map_err(VfioError::ReadLink)?;
-        let group_osstr = group_path.file_name().ok_or(VfioError::InvalidPath)?;
-        let group_str = group_osstr.to_str().ok_or(VfioError::InvalidPath)?;
-        let group_id = group_str.parse::<u32>().map_err(VfioError::ParseInt)?;
-
-        let group = VfioGroup::new(group_id, device_fd)?;
-        let device_info = group.get_device(sysfspath)?;
-        let regions = device_info.get_regions()?;
-        let irqs = device_info.get_irqs()?;
-
-        Ok(VfioDevice {
-            device: device_info.device,
-            flags: device_info.flags,
-            group,
-            regions,
-            irqs,
-            mem,
-            iommu_attached,
-        })
-    }
-
-    /// VFIO device reset.
-    /// Only if the device supports being reset.
-    pub fn reset(&self) {
-        if self.flags & VFIO_DEVICE_FLAGS_RESET != 0 {
-            unsafe { ioctl(self, VFIO_DEVICE_RESET()) };
-        }
-    }
-
-    /// Enables a VFIO device IRQs.
-    /// This maps a vector of EventFds to all VFIO managed interrupts. In other words, this
-    /// tells VFIO which EventFd to write into whenever one of the device interrupt vector
-    /// is triggered.
-    ///
-    /// # Arguments
-    ///
-    /// * `irq_index` - The type (INTX, MSI or MSI-X) of interrupts to enable.
-    /// * `event_fds` - The EventFds vector that matches all the supported VFIO interrupts.
-    pub fn enable_irq(&self, irq_index: u32, event_fds: Vec<&EventFd>) -> Result<()> {
-        let irq = self
-            .irqs
-            .get(&irq_index)
-            .ok_or(VfioError::VfioDeviceSetIrq)?;
-        if irq.count == 0 {
-            return Err(VfioError::VfioDeviceSetIrq);
-        }
-
-        let mut irq_set_wrapper = IrqSet::new(event_fds.len() * mem::size_of::<u32>());
-        let mut irq_set = irq_set_wrapper.as_mut_fam_struct();
-        let fds = irq_set.as_mut_slice();
-
-        for (index, event_fd) in event_fds.iter().enumerate() {
-            let fds_offset = index * mem::size_of::<u32>();
-            let fd = &mut fds[fds_offset..fds_offset + mem::size_of::<u32>()];
-            LittleEndian::write_u32(fd, event_fd.as_raw_fd() as u32);
-        }
-
-        irq_set.argsz = mem::size_of::<vfio_irq_set>() as u32
-            + (event_fds.len() * mem::size_of::<u32>()) as u32;
-        irq_set.flags = VFIO_IRQ_SET_DATA_EVENTFD | VFIO_IRQ_SET_ACTION_TRIGGER;
-        irq_set.index = irq_index;
-        irq_set.start = 0;
-        irq_set.count = irq.count;
-
-        // Safe as we are the owner of self and irq_set which are valid value
-        let ret = unsafe { ioctl_with_ref(self, VFIO_DEVICE_SET_IRQS(), irq_set) };
-        if ret < 0 {
-            return Err(VfioError::VfioDeviceSetIrq);
-        }
-
-        Ok(())
-    }
-
-    /// Disables a VFIO device IRQs
-    ///
-    /// # Arguments
-    ///
-    /// * `irq_index` - The type (INTX, MSI or MSI-X) of interrupts to disable.
-    pub fn disable_irq(&self, irq_index: u32) -> Result<()> {
-        let irq = self
-            .irqs
-            .get(&irq_index)
-            .ok_or(VfioError::VfioDeviceSetIrq)?;
-        if irq.count == 0 {
-            return Err(VfioError::VfioDeviceSetIrq);
-        }
-
-        let mut irq_set = vec_with_array_field::<vfio_irq_set, u32>(0);
-        irq_set[0].argsz = mem::size_of::<vfio_irq_set>() as u32;
-        irq_set[0].flags = VFIO_IRQ_SET_ACTION_TRIGGER | VFIO_IRQ_SET_DATA_NONE;
-        irq_set[0].index = irq_index;
-        irq_set[0].start = 0;
-        irq_set[0].count = 0;
-
-        // Safe as we are the owner of self and irq_set which are valid value
-        let ret = unsafe { ioctl_with_ref(self, VFIO_DEVICE_SET_IRQS(), &irq_set[0]) };
-        if ret < 0 {
-            return Err(VfioError::VfioDeviceSetIrq);
-        }
-
-        Ok(())
-    }
-
-    /// Wrapper to enable MSI IRQs.
-    pub fn enable_msi(&self, fds: Vec<&EventFd>) -> Result<()> {
-        self.enable_irq(VFIO_PCI_MSI_IRQ_INDEX, fds)
-    }
-
-    /// Wrapper to disable MSI IRQs.
-    pub fn disable_msi(&self) -> Result<()> {
-        self.disable_irq(VFIO_PCI_MSI_IRQ_INDEX)
-    }
-
-    /// Wrapper to enable MSI-X IRQs.
-    pub fn enable_msix(&self, fds: Vec<&EventFd>) -> Result<()> {
-        self.enable_irq(VFIO_PCI_MSIX_IRQ_INDEX, fds)
-    }
-
-    /// Wrapper to disable MSI-X IRQs.
-    pub fn disable_msix(&self) -> Result<()> {
-        self.disable_irq(VFIO_PCI_MSIX_IRQ_INDEX)
-    }
-
-    /// get a region's flag
-    pub fn get_region_flags(&self, index: u32) -> u32 {
-        match self.regions.get(index as usize) {
-            Some(v) => v.flags,
-            None => 0,
-        }
-    }
-
-    /// get a region's offset
-    pub fn get_region_offset(&self, index: u32) -> u64 {
-        match self.regions.get(index as usize) {
-            Some(v) => v.offset,
-            None => 0,
-        }
-    }
-
-    /// get a region's mmap info
-    pub fn get_region_mmap(&self, index: u32) -> (u64, u64) {
-        match self.regions.get(index as usize) {
-            Some(v) => v.mmap,
-            None => {
-                warn!("get_region_mmap with invalid index: {}", index);
-                (0, 0)
-            }
-        }
-    }
-
-    /// get a region's size
-    pub fn get_region_size(&self, index: u32) -> u64 {
-        match self.regions.get(index as usize) {
-            Some(v) => v.size,
-            None => {
-                warn!("get_region_size with invalid index: {}", index);
-                0
-            }
-        }
-    }
-
-    /// Read region's data from VFIO device into buf
-    /// index: region num
-    /// buf: data destination and buf length is read size
-    /// addr: offset in the region
-    pub fn region_read(&self, index: u32, buf: &mut [u8], addr: u64) {
-        let region: &VfioRegion;
-        match self.regions.get(index as usize) {
-            Some(v) => region = v,
-            None => {
-                warn!("region read with invalid index: {}", index);
-                return;
-            }
-        }
-
-        let size = buf.len() as u64;
-        if size > region.size || addr + size > region.size {
-            warn!(
-                "region read with invalid parameter, add: {}, size: {}",
-                addr, size
-            );
-            return;
-        }
-
-        if let Err(e) = self.device.read_exact_at(buf, region.offset + addr) {
-            warn!(
-                "Failed to read region in index: {}, addr: {}, error: {}",
-                index, addr, e
-            );
-        }
-    }
-
-    /// write the data from buf into a vfio device region
-    /// index: region num
-    /// buf: data src and buf length is write size
-    /// addr: offset in the region
-    pub fn region_write(&self, index: u32, buf: &[u8], addr: u64) {
-        let stub: &VfioRegion;
-        match self.regions.get(index as usize) {
-            Some(v) => stub = v,
-            None => {
-                warn!("region write with invalid index: {}", index);
-                return;
-            }
-        }
-
-        let size = buf.len() as u64;
-        if size > stub.size
-            || addr + size > stub.size
-            || (stub.flags & VFIO_REGION_INFO_FLAG_WRITE) == 0
-        {
-            warn!(
-                "region write with invalid parameter, add: {}, size: {}",
-                addr, size
-            );
-            return;
-        }
-
-        if let Err(e) = self.device.write_all_at(buf, stub.offset + addr) {
-            warn!(
-                "Failed to write region in index: {}, addr: {}, error: {}",
-                index, addr, e
-            );
-        }
-    }
-
-    pub fn get_container(&self) -> Arc<VfioContainer> {
-        self.group.container.clone()
-    }
-
-    fn vfio_dma_map(&self, iova: u64, size: u64, user_addr: u64) -> Result<()> {
-        self.group.container.vfio_dma_map(iova, size, user_addr)
-    }
-
-    fn vfio_dma_unmap(&self, iova: u64, size: u64) -> Result<()> {
-        self.group.container.vfio_dma_unmap(iova, size)
-    }
-
-    /// Add all guest memory regions into vfio container's iommu table,
-    /// then vfio kernel driver could access guest memory from gfn
-    pub fn setup_dma_map(&self) -> Result<()> {
-        if !self.iommu_attached {
-            self.mem.memory().with_regions(|_index, region| {
-                self.vfio_dma_map(
-                    region.start_addr().raw_value(),
-                    region.len() as u64,
-                    region.as_ptr() as u64,
-                )
-            })?;
-        }
-        Ok(())
-    }
-
-    /// remove all guest memory regions from vfio containers iommu table
-    /// then vfio kernel driver couldn't access this guest memory
-    pub fn unset_dma_map(&self) -> Result<()> {
-        if !self.iommu_attached {
-            self.mem.memory().with_regions(|_index, region| {
-                self.vfio_dma_unmap(region.start_addr().raw_value(), region.len() as u64)
-            })?;
-        }
-        Ok(())
-    }
-
-    /// Return the maximum numner of interrupts a VFIO device can request.
-    /// This is used for pre-allocating the VFIO PCI routes.
-    pub fn max_interrupts(&self) -> u32 {
-        let mut max_interrupts = 0;
-        let irq_indexes = vec![
-            VFIO_PCI_INTX_IRQ_INDEX,
-            VFIO_PCI_MSI_IRQ_INDEX,
-            VFIO_PCI_MSIX_IRQ_INDEX,
-        ];
-
-        for index in irq_indexes {
-            if let Some(irq_info) = self.irqs.get(&index) {
-                if irq_info.count > max_interrupts {
-                    max_interrupts = irq_info.count;
-                }
-            }
-        }
-
-        max_interrupts
-    }
-}
-
-impl AsRawFd for VfioDevice {
-    fn as_raw_fd(&self) -> RawFd {
-        self.device.as_raw_fd()
-    }
-}

vfio/src/vfio_ioctls.rs

@@ -1,36 +0,0 @@
-// Copyright © 2019 Intel Corporation
-//
-// SPDX-License-Identifier: Apache-2.0 OR BSD-3-Clause
-//
-#![allow(non_upper_case_globals)]
-#![allow(non_camel_case_types)]
-#![allow(non_snake_case)]
-
-use vfio_bindings::bindings::vfio::*;
-
-ioctl_io_nr!(VFIO_GET_API_VERSION, VFIO_TYPE, VFIO_BASE);
-ioctl_io_nr!(VFIO_CHECK_EXTENSION, VFIO_TYPE, VFIO_BASE + 1);
-ioctl_io_nr!(VFIO_SET_IOMMU, VFIO_TYPE, VFIO_BASE + 2);
-ioctl_io_nr!(VFIO_GROUP_GET_STATUS, VFIO_TYPE, VFIO_BASE + 3);
-ioctl_io_nr!(VFIO_GROUP_SET_CONTAINER, VFIO_TYPE, VFIO_BASE + 4);
-ioctl_io_nr!(VFIO_GROUP_UNSET_CONTAINER, VFIO_TYPE, VFIO_BASE + 5);
-ioctl_io_nr!(VFIO_GROUP_GET_DEVICE_FD, VFIO_TYPE, VFIO_BASE + 6);
-ioctl_io_nr!(VFIO_DEVICE_GET_INFO, VFIO_TYPE, VFIO_BASE + 7);
-ioctl_io_nr!(VFIO_DEVICE_GET_REGION_INFO, VFIO_TYPE, VFIO_BASE + 8);
-ioctl_io_nr!(VFIO_DEVICE_GET_IRQ_INFO, VFIO_TYPE, VFIO_BASE + 9);
-ioctl_io_nr!(VFIO_DEVICE_SET_IRQS, VFIO_TYPE, VFIO_BASE + 10);
-ioctl_io_nr!(VFIO_DEVICE_RESET, VFIO_TYPE, VFIO_BASE + 11);
-ioctl_io_nr!(
-    VFIO_DEVICE_GET_PCI_HOT_RESET_INFO,
-    VFIO_TYPE,
-    VFIO_BASE + 12
-);
-ioctl_io_nr!(VFIO_DEVICE_PCI_HOT_RESET, VFIO_TYPE, VFIO_BASE + 13);
-ioctl_io_nr!(VFIO_DEVICE_QUERY_GFX_PLANE, VFIO_TYPE, VFIO_BASE + 14);
-ioctl_io_nr!(VFIO_DEVICE_GET_GFX_DMABUF, VFIO_TYPE, VFIO_BASE + 15);
-ioctl_io_nr!(VFIO_DEVICE_IOEVENTFD, VFIO_TYPE, VFIO_BASE + 16);
-ioctl_io_nr!(VFIO_IOMMU_GET_INFO, VFIO_TYPE, VFIO_BASE + 12);
-ioctl_io_nr!(VFIO_IOMMU_MAP_DMA, VFIO_TYPE, VFIO_BASE + 13);
-ioctl_io_nr!(VFIO_IOMMU_UNMAP_DMA, VFIO_TYPE, VFIO_BASE + 14);
-ioctl_io_nr!(VFIO_IOMMU_ENABLE, VFIO_TYPE, VFIO_BASE + 15);
-ioctl_io_nr!(VFIO_IOMMU_DISABLE, VFIO_TYPE, VFIO_BASE + 16);

vhost_user_backend/Cargo.toml

@@ -11,10 +11,10 @@ mmio_support = ["vm-virtio/mmio_support"]
 
 [dependencies]
 epoll = ">=4.0.1"
-libc = "0.2.68"
+libc = "0.2.71"
 log = "0.4.8"
 virtio-bindings = "0.1.0"
-vm-memory = "0.1.0"
+vm-memory = "0.2.1"
 vm-virtio = { path = "../vm-virtio" }
 vmm-sys-util = ">=0.3.1"
 vhost_rs = { git = "https://github.com/cloud-hypervisor/vhost", branch = "dragonball", package = "vhost", features = ["vhost-user-slave"] }

vhost_user_backend/src/lib.rs

@@ -20,7 +20,7 @@ use vhost_rs::vhost_user::message::{
     VhostUserVirtioFeatures, VhostUserVringAddrFlags, VhostUserVringState,
 };
 use vhost_rs::vhost_user::{
-    Error as VhostUserError, Result as VhostUserResult, SlaveFsCacheReq, SlaveListener,
+    Error as VhostUserError, Listener, Result as VhostUserResult, SlaveFsCacheReq, SlaveListener,
     VhostUserSlaveReqHandler,
 };
 use virtio_bindings::bindings::virtio_ring::VIRTIO_RING_F_EVENT_IDX;
@@ -64,9 +64,12 @@ pub trait VhostUserBackend: Send + Sync + 'static {
     /// Depth of each queue.
     fn max_queue_size(&self) -> usize;
 
-    /// Virtio features.
+    /// Available virtio features.
     fn features(&self) -> u64;
 
+    /// Acked virtio features.
+    fn acked_features(&mut self, _features: u64) {}
+
     /// Virtio protocol features.
     fn protocol_features(&self) -> VhostUserProtocolFeatures;
 
@@ -81,10 +84,11 @@ pub trait VhostUserBackend: Send + Sync + 'static {
     /// virtqueues on its own, but does not know what to do with events
     /// happening on custom listeners.
     fn handle_event(
-        &mut self,
+        &self,
         device_event: u16,
         evset: epoll::Events,
         vrings: &[Arc<RwLock<Vring>>],
+        thread_id: usize,
     ) -> result::Result<bool, io::Error>;
 
     /// Get virtio device configuration.
@@ -105,7 +109,7 @@ pub trait VhostUserBackend: Send + Sync + 'static {
     /// When this EventFd is written to the worker thread will exit. An optional id may
     /// also be provided, if it not provided then the exit event will be first event id
     /// after the last queue
-    fn exit_event(&self) -> Option<(EventFd, Option<u16>)> {
+    fn exit_event(&self, _thread_index: usize) -> Option<(EventFd, Option<u16>)> {
         None
     }
 
@@ -113,13 +117,16 @@ pub trait VhostUserBackend: Send + Sync + 'static {
     /// A default implementation is provided as we cannot expect all backends
     /// to implement this function.
     fn set_slave_req_fd(&mut self, _vu_req: SlaveFsCacheReq) {}
+
+    fn queues_per_thread(&self) -> Vec<u64> {
+        vec![0xffff_ffff]
+    }
 }
 
 /// This structure is the public API the backend is allowed to interact with
 /// in order to run a fully functional vhost-user daemon.
 pub struct VhostUserDaemon<S: VhostUserBackend> {
     name: String,
-    sock_path: String,
     handler: Arc<Mutex<VhostUserHandler<S>>>,
     main_thread: Option<thread::JoinHandle<Result<()>>>,
 }
@@ -131,14 +138,13 @@ impl<S: VhostUserBackend> VhostUserDaemon<S> {
     /// listening onto registered event. Those events can be vring events or
     /// custom events from the backend, but they get to be registered later
     /// during the sequence.
-    pub fn new(name: String, sock_path: String, backend: Arc<RwLock<S>>) -> Result<Self> {
+    pub fn new(name: String, backend: Arc<RwLock<S>>) -> Result<Self> {
         let handler = Arc::new(Mutex::new(
             VhostUserHandler::new(backend).map_err(Error::NewVhostUserHandler)?,
         ));
 
         Ok(VhostUserDaemon {
             name,
-            sock_path,
             handler,
             main_thread: None,
         })
@@ -148,10 +154,9 @@ impl<S: VhostUserBackend> VhostUserDaemon<S> {
     /// all requests coming through this socket. This runs in an infinite loop
     /// that should be terminating once the other end of the socket (the VMM)
     /// disconnects.
-    pub fn start(&mut self) -> Result<()> {
-        let mut slave_listener =
-            SlaveListener::new(self.sock_path.as_str(), true, self.handler.clone())
-                .map_err(Error::CreateSlaveListener)?;
+    pub fn start(&mut self, listener: Listener) -> Result<()> {
+        let mut slave_listener = SlaveListener::new(listener, self.handler.clone())
+            .map_err(Error::CreateSlaveListener)?;
         let mut slave_handler = slave_listener
             .accept()
             .map_err(Error::CreateSlaveReqHandler)?
@@ -187,8 +192,8 @@ impl<S: VhostUserBackend> VhostUserDaemon<S> {
     /// Retrieve the vring worker. This is necessary to perform further
     /// actions like registering and unregistering some extra event file
     /// descriptors.
-    pub fn get_vring_worker(&self) -> Arc<VringWorker> {
-        self.handler.lock().unwrap().get_vring_worker()
+    pub fn get_vring_workers(&self) -> Vec<Arc<VringWorker>> {
+        self.handler.lock().unwrap().get_vring_workers()
     }
 }
 
@@ -208,8 +213,6 @@ pub struct Vring {
     call: Option<EventFd>,
     err: Option<EventFd>,
     enabled: bool,
-    event_idx: bool,
-    signalled_used: Option<Wrapping<u16>>,
 }
 
 impl Vring {
@@ -220,8 +223,6 @@ impl Vring {
             call: None,
             err: None,
             enabled: false,
-            event_idx: false,
-            signalled_used: None,
         }
     }
 
@@ -229,31 +230,6 @@ impl Vring {
         &mut self.queue
     }
 
-    pub fn set_event_idx(&mut self, enabled: bool) {
-        /* Also reset the last signalled event */
-        self.signalled_used = None;
-        self.event_idx = enabled;
-    }
-
-    pub fn needs_notification(&mut self, mem: &GuestMemoryMmap, used_idx: Wrapping<u16>) -> bool {
-        if !self.event_idx {
-            return true;
-        }
-
-        let mut notify = true;
-
-        if let Some(old_idx) = self.signalled_used {
-            if let Some(used_event) = self.mut_queue().get_used_event(&mem) {
-                if (used_idx - used_event - Wrapping(1u16)) >= (used_idx - old_idx) {
-                    notify = false;
-                }
-            }
-        }
-
-        self.signalled_used = Some(used_idx);
-        notify
-    }
-
     pub fn signal_used_queue(&mut self) -> result::Result<(), io::Error> {
         if let Some(call) = self.call.as_ref() {
             call.write(1)
@@ -283,6 +259,7 @@ struct VringEpollHandler<S: VhostUserBackend> {
     backend: Arc<RwLock<S>>,
     vrings: Vec<Arc<RwLock<Vring>>>,
     exit_event_id: Option<u16>,
+    thread_id: usize,
 }
 
 impl<S: VhostUserBackend> VringEpollHandler<S> {
@@ -310,9 +287,9 @@ impl<S: VhostUserBackend> VringEpollHandler<S> {
         }
 
         self.backend
-            .write()
+            .read()
             .unwrap()
-            .handle_event(device_event, evset, &self.vrings)
+            .handle_event(device_event, evset, &self.vrings, self.thread_id)
             .map_err(VringEpollHandlerError::HandleEventBackendHandling)
     }
 }
@@ -330,7 +307,13 @@ enum VringWorkerError {
 type VringWorkerResult<T> = std::result::Result<T, VringWorkerError>;
 
 pub struct VringWorker {
-    epoll_fd: RawFd,
+    epoll_file: File,
+}
+
+impl AsRawFd for VringWorker {
+    fn as_raw_fd(&self) -> RawFd {
+        self.epoll_file.as_raw_fd()
+    }
 }
 
 impl VringWorker {
@@ -339,7 +322,7 @@ impl VringWorker {
         let mut events = vec![epoll::Event::new(epoll::Events::empty(), 0); EPOLL_EVENTS_LEN];
 
         'epoll: loop {
-            let num_events = match epoll::wait(self.epoll_fd, -1, &mut events[..]) {
+            let num_events = match epoll::wait(self.epoll_file.as_raw_fd(), -1, &mut events[..]) {
                 Ok(res) => res,
                 Err(e) => {
                     if e.kind() == io::ErrorKind::Interrupted {
@@ -393,7 +376,7 @@ impl VringWorker {
         data: u64,
     ) -> result::Result<(), io::Error> {
         epoll::ctl(
-            self.epoll_fd,
+            self.epoll_file.as_raw_fd(),
             epoll::ControlOptions::EPOLL_CTL_ADD,
             fd,
             epoll::Event::new(ev_type, data),
@@ -410,7 +393,7 @@ impl VringWorker {
         data: u64,
     ) -> result::Result<(), io::Error> {
         epoll::ctl(
-            self.epoll_fd,
+            self.epoll_file.as_raw_fd(),
             epoll::ControlOptions::EPOLL_CTL_DEL,
             fd,
             epoll::Event::new(ev_type, data),
@@ -453,22 +436,24 @@ type VhostUserHandlerResult<T> = std::result::Result<T, VhostUserHandlerError>;
 
 struct VhostUserHandler<S: VhostUserBackend> {
     backend: Arc<RwLock<S>>,
-    worker: Arc<VringWorker>,
+    workers: Vec<Arc<VringWorker>>,
     owned: bool,
     features_acked: bool,
     acked_features: u64,
     acked_protocol_features: u64,
     num_queues: usize,
     max_queue_size: usize,
+    queues_per_thread: Vec<u64>,
     memory: Option<Memory>,
     vrings: Vec<Arc<RwLock<Vring>>>,
-    worker_thread: Option<thread::JoinHandle<VringWorkerResult<()>>>,
+    worker_threads: Vec<thread::JoinHandle<VringWorkerResult<()>>>,
 }
 
 impl<S: VhostUserBackend> VhostUserHandler<S> {
     fn new(backend: Arc<RwLock<S>>) -> VhostUserHandlerResult<Self> {
         let num_queues = backend.read().unwrap().num_queues();
         let max_queue_size = backend.read().unwrap().max_queue_size();
+        let queues_per_thread = backend.read().unwrap().queues_per_thread();
 
         let mut vrings: Vec<Arc<RwLock<Vring>>> = Vec::new();
         for _ in 0..num_queues {
@@ -476,14 +461,20 @@ impl<S: VhostUserBackend> VhostUserHandler<S> {
             vrings.push(vring);
         }
 
-        // Create the epoll file descriptor
-        let epoll_fd = epoll::create(true).map_err(VhostUserHandlerError::EpollCreateFd)?;
+        let mut workers = Vec::new();
+        let mut worker_threads = Vec::new();
+        for (thread_id, queues_mask) in queues_per_thread.iter().enumerate() {
+            // Create the epoll file descriptor
+            let epoll_fd = epoll::create(true).map_err(VhostUserHandlerError::EpollCreateFd)?;
+            // Use 'File' to enforce closing on 'epoll_fd'
+            let epoll_file = unsafe { File::from_raw_fd(epoll_fd) };
 
-        let vring_worker = Arc::new(VringWorker { epoll_fd });
-        let worker = vring_worker.clone();
+            let vring_worker = Arc::new(VringWorker { epoll_file });
+            let worker = vring_worker.clone();
 
-        let exit_event_id =
-            if let Some((exit_event_fd, exit_event_id)) = backend.read().unwrap().exit_event() {
+            let exit_event_id = if let Some((exit_event_fd, exit_event_id)) =
+                backend.read().unwrap().exit_event(thread_id)
+            {
                 let exit_event_id = exit_event_id.unwrap_or(num_queues as u16);
                 worker
                     .register_listener(
@@ -497,36 +488,47 @@ impl<S: VhostUserBackend> VhostUserHandler<S> {
                 None
             };
 
-        let vring_handler = VringEpollHandler {
-            backend: backend.clone(),
-            vrings: vrings.clone(),
-            exit_event_id,
-        };
+            let mut thread_vrings: Vec<Arc<RwLock<Vring>>> = Vec::new();
+            for (index, vring) in vrings.iter().enumerate() {
+                if (queues_mask >> index) & 1u64 == 1u64 {
+                    thread_vrings.push(vring.clone());
+                }
+            }
 
-        let worker_thread = Some(
-            thread::Builder::new()
+            let vring_handler = VringEpollHandler {
+                backend: backend.clone(),
+                vrings: thread_vrings,
+                exit_event_id,
+                thread_id,
+            };
+
+            let worker_thread = thread::Builder::new()
                 .name("vring_worker".to_string())
                 .spawn(move || vring_worker.run(vring_handler))
-                .map_err(VhostUserHandlerError::SpawnVringWorker)?,
-        );
+                .map_err(VhostUserHandlerError::SpawnVringWorker)?;
+
+            workers.push(worker);
+            worker_threads.push(worker_thread);
+        }
 
         Ok(VhostUserHandler {
             backend,
-            worker,
+            workers,
             owned: false,
             features_acked: false,
             acked_features: 0,
             acked_protocol_features: 0,
             num_queues,
             max_queue_size,
+            queues_per_thread,
             memory: None,
             vrings,
-            worker_thread,
+            worker_threads,
         })
     }
 
-    fn get_vring_worker(&self) -> Arc<VringWorker> {
-        self.worker.clone()
+    fn get_vring_workers(&self) -> Vec<Arc<VringWorker>> {
+        self.workers.clone()
     }
 
     fn vmm_va_to_gpa(&self, vmm_va: u64) -> VhostUserHandlerResult<u64> {
@@ -584,6 +586,11 @@ impl<S: VhostUserBackend> VhostUserSlaveReqHandler for VhostUserHandler<S> {
             vring.write().unwrap().enabled = vring_enabled;
         }
 
+        self.backend
+            .write()
+            .unwrap()
+            .acked_features(self.acked_features);
+
         Ok(())
     }
 
@@ -702,6 +709,7 @@ impl<S: VhostUserBackend> VhostUserSlaveReqHandler for VhostUserHandler<S> {
         self.vrings[index as usize]
             .write()
             .unwrap()
+            .mut_queue()
             .set_event_idx(event_idx);
         self.backend.write().unwrap().set_event_idx(event_idx);
         Ok(())
@@ -718,9 +726,20 @@ impl<S: VhostUserBackend> VhostUserSlaveReqHandler for VhostUserHandler<S> {
         // VHOST_USER_GET_VRING_BASE.
         self.vrings[index as usize].write().unwrap().queue.ready = false;
         if let Some(fd) = self.vrings[index as usize].read().unwrap().kick.as_ref() {
-            self.worker
-                .unregister_listener(fd.as_raw_fd(), epoll::Events::EPOLLIN, u64::from(index))
-                .map_err(VhostUserError::ReqHandlerError)?;
+            for (thread_index, queues_mask) in self.queues_per_thread.iter().enumerate() {
+                let shifted_queues_mask = queues_mask >> index;
+                if shifted_queues_mask & 1u64 == 1u64 {
+                    let evt_idx = queues_mask.count_ones() - shifted_queues_mask.count_ones();
+                    self.workers[thread_index]
+                        .unregister_listener(
+                            fd.as_raw_fd(),
+                            epoll::Events::EPOLLIN,
+                            u64::from(evt_idx),
+                        )
+                        .map_err(VhostUserError::ReqHandlerError)?;
+                    break;
+                }
+            }
         }
 
         let next_avail = self.vrings[index as usize]
@@ -752,9 +771,20 @@ impl<S: VhostUserBackend> VhostUserSlaveReqHandler for VhostUserHandler<S> {
         // VHOST_USER_GET_VRING_BASE.
         self.vrings[index as usize].write().unwrap().queue.ready = true;
         if let Some(fd) = self.vrings[index as usize].read().unwrap().kick.as_ref() {
-            self.worker
-                .register_listener(fd.as_raw_fd(), epoll::Events::EPOLLIN, u64::from(index))
-                .map_err(VhostUserError::ReqHandlerError)?;
+            for (thread_index, queues_mask) in self.queues_per_thread.iter().enumerate() {
+                let shifted_queues_mask = queues_mask >> index;
+                if shifted_queues_mask & 1u64 == 1u64 {
+                    let evt_idx = queues_mask.count_ones() - shifted_queues_mask.count_ones();
+                    self.workers[thread_index]
+                        .register_listener(
+                            fd.as_raw_fd(),
+                            epoll::Events::EPOLLIN,
+                            u64::from(evt_idx),
+                        )
+                        .map_err(VhostUserError::ReqHandlerError)?;
+                    break;
+                }
+            }
         }
 
         Ok(())
@@ -837,7 +867,7 @@ impl<S: VhostUserBackend> VhostUserSlaveReqHandler for VhostUserHandler<S> {
 
 impl<S: VhostUserBackend> Drop for VhostUserHandler<S> {
     fn drop(&mut self) {
-        if let Some(thread) = self.worker_thread.take() {
+        for thread in self.worker_threads.drain(..) {
             if let Err(e) = thread.join() {
                 error!("Error in vring worker: {:?}", e);
             }

vhost_user_block/Cargo.toml

@@ -7,12 +7,13 @@ edition = "2018"
 [dependencies]
 bitflags = "1.1.0"
 epoll = ">=4.0.1"
-libc = "0.2.68"
+libc = "0.2.71"
 log = "0.4.8"
 qcow = { path = "../qcow" }
 vhost_user_backend = { path = "../vhost_user_backend" }
 vhost_rs = { git = "https://github.com/cloud-hypervisor/vhost", branch = "dragonball", package = "vhost", features = ["vhost-user-slave"] }
 virtio-bindings = "0.1.0"
-vm-memory = "0.1.0"
+vm-memory = "0.2.1"
 vm-virtio = { path = "../vm-virtio" }
+vmm = { path = "../vmm" }
 vmm-sys-util = ">=0.3.1"

vhost_user_block/src/lib.rs

@@ -13,7 +13,6 @@ extern crate vhost_rs;
 extern crate vhost_user_backend;
 extern crate vm_virtio;
 
-use epoll;
 use libc::EFD_NONBLOCK;
 use log::*;
 use qcow::{self, ImageType, QcowFile};
@@ -21,25 +20,29 @@ use std::fs::File;
 use std::fs::OpenOptions;
 use std::io::Read;
 use std::io::{Seek, SeekFrom, Write};
-use std::mem;
 use std::num::Wrapping;
+use std::ops::DerefMut;
 use std::os::unix::fs::OpenOptionsExt;
 use std::path::PathBuf;
 use std::process;
-use std::slice;
-use std::sync::{Arc, RwLock};
+use std::result;
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::{Arc, Mutex, RwLock};
 use std::time::Instant;
 use std::vec::Vec;
 use std::{convert, error, fmt, io};
 use vhost_rs::vhost_user::message::*;
-use vhost_user_backend::{VhostUserBackend, VhostUserDaemon, Vring, VringWorker};
+use vhost_rs::vhost_user::Listener;
+use vhost_user_backend::{VhostUserBackend, VhostUserDaemon, Vring};
 use virtio_bindings::bindings::virtio_blk::*;
 use virtio_bindings::bindings::virtio_ring::VIRTIO_RING_F_EVENT_IDX;
-use vm_memory::{Bytes, GuestMemoryError, GuestMemoryMmap};
+use vm_memory::ByteValued;
+use vm_memory::{Bytes, GuestMemoryMmap};
 use vm_virtio::block::{build_disk_image_id, Request};
+use vm_virtio::VirtioBlockConfig;
+use vmm::config::{OptionParser, OptionParserError, Toggle};
 use vmm_sys_util::eventfd::EventFd;
 
-const QUEUE_SIZE: usize = 1024;
 const SECTOR_SHIFT: u8 = 9;
 const SECTOR_SIZE: u64 = (0x01 as u64) << SECTOR_SHIFT;
 const BLK_SIZE: u32 = 512;
@@ -51,37 +54,30 @@ const POLL_QUEUE_US: u128 = 50;
 trait DiskFile: Read + Seek + Write + Send + Sync {}
 impl<D: Read + Seek + Write + Send + Sync> DiskFile for D {}
 
-pub type Result<T> = std::result::Result<T, Error>;
-pub type VhostUserBackendResult<T> = std::result::Result<T, std::io::Error>;
+type Result<T> = std::result::Result<T, Error>;
+type VhostUserBackendResult<T> = std::result::Result<T, std::io::Error>;
 
 #[derive(Debug)]
-pub enum Error {
-    /// Failed to detect image type.
-    DetectImageType,
-    /// Bad memory address.
-    GuestMemory(GuestMemoryError),
-    /// Can't open image file.
-    OpenImage,
-    /// Failed to parse direct parameter.
-    ParseDirectParam,
-    /// Failed to parse image parameter.
-    ParseImageParam,
-    /// Failed to parse sock parameter.
-    ParseSockParam,
-    /// Failed to parse readonly parameter.
-    ParseReadOnlyParam,
-    /// Failed parsing fs number of queues parameter.
-    ParseBlkNumQueuesParam(std::num::ParseIntError),
-    /// Failed to parse the poll_queue parameter.
-    ParsePollQueueParam,
-    /// Failed to handle event other than input event.
-    HandleEventNotEpollIn,
+enum Error {
     /// Failed to create kill eventfd
     CreateKillEventFd(io::Error),
+    /// Failed to parse configuration string
+    FailedConfigParse(OptionParserError),
+    /// Failed to handle event other than input event.
+    HandleEventNotEpollIn,
     /// Failed to handle unknown event.
     HandleEventUnknownEvent,
+    /// No path provided
+    PathParameterMissing,
+    /// No socket provided
+    SocketParameterMissing,
 }
 
+pub const SYNTAX: &str = "vhost-user-block backend parameters \
+ \"path=<image_path>,socket=<socket_path>,num_queues=<number_of_queues>,\
+ queue_size=<size_of_each_queue>,readonly=true|false,direct=true|false,\
+ poll_queue=true|false\"";
+
 impl fmt::Display for Error {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         write!(f, "vhost_user_block_error: {:?}", self)
@@ -96,70 +92,35 @@ impl convert::From<Error> for io::Error {
     }
 }
 
-pub struct VhostUserBlkBackend {
+struct VhostUserBlkThread {
     mem: Option<GuestMemoryMmap>,
-    vring_worker: Option<Arc<VringWorker>>,
-    disk_image: Box<dyn DiskFile>,
+    disk_image: Arc<Mutex<dyn DiskFile>>,
     disk_image_id: Vec<u8>,
     disk_nsectors: u64,
-    config: virtio_blk_config,
-    rdonly: bool,
-    poll_queue: bool,
     event_idx: bool,
     kill_evt: EventFd,
+    writeback: Arc<AtomicBool>,
 }
 
-impl VhostUserBlkBackend {
-    pub fn new(
-        image_path: String,
-        num_queues: usize,
-        rdonly: bool,
-        direct: bool,
-        poll_queue: bool,
+impl VhostUserBlkThread {
+    fn new(
+        disk_image: Arc<Mutex<dyn DiskFile>>,
+        disk_image_id: Vec<u8>,
+        disk_nsectors: u64,
+        writeback: Arc<AtomicBool>,
     ) -> Result<Self> {
-        let mut options = OpenOptions::new();
-        options.read(true);
-        options.write(!rdonly);
-        if direct {
-            options.custom_flags(libc::O_DIRECT);
-        }
-        let image: File = options.open(&image_path).unwrap();
-        let mut raw_img: vm_virtio::RawFile = vm_virtio::RawFile::new(image, direct);
-
-        let image_id = build_disk_image_id(&PathBuf::from(&image_path));
-        let image_type = qcow::detect_image_type(&mut raw_img).unwrap();
-        let mut image = match image_type {
-            ImageType::Raw => Box::new(raw_img) as Box<dyn DiskFile>,
-            ImageType::Qcow2 => Box::new(QcowFile::from(raw_img).unwrap()) as Box<dyn DiskFile>,
-        };
-
-        let nsectors = (image.seek(SeekFrom::End(0)).unwrap() as u64) / SECTOR_SIZE;
-        let mut config = virtio_blk_config::default();
-
-        config.capacity = nsectors;
-        config.blk_size = BLK_SIZE;
-        config.size_max = 65535;
-        config.seg_max = 128 - 2;
-        config.min_io_size = 1;
-        config.opt_io_size = 1;
-        config.num_queues = num_queues as u16;
-        config.wce = 1;
-
-        Ok(VhostUserBlkBackend {
+        Ok(VhostUserBlkThread {
             mem: None,
-            vring_worker: None,
-            disk_image: image,
-            disk_image_id: image_id,
-            disk_nsectors: nsectors,
-            config,
-            rdonly,
-            poll_queue,
+            disk_image,
+            disk_image_id,
+            disk_nsectors,
             event_idx: false,
             kill_evt: EventFd::new(EFD_NONBLOCK).map_err(Error::CreateKillEventFd)?,
+            writeback,
         })
     }
 
-    pub fn process_queue(&mut self, vring: &mut Vring) -> bool {
+    fn process_queue(&mut self, vring: &mut Vring) -> bool {
         let mut used_any = false;
         let mem = match self.mem.as_ref() {
             Some(m) => m,
@@ -170,10 +131,11 @@ impl VhostUserBlkBackend {
             debug!("got an element in the queue");
             let len;
             match Request::parse(&head, mem) {
-                Ok(request) => {
+                Ok(mut request) => {
                     debug!("element is a valid request");
+                    request.set_writeback(self.writeback.load(Ordering::SeqCst));
                     let status = match request.execute(
-                        &mut self.disk_image,
+                        &mut self.disk_image.lock().unwrap().deref_mut(),
                         self.disk_nsectors,
                         mem,
                         &self.disk_image_id,
@@ -196,8 +158,9 @@ impl VhostUserBlkBackend {
             }
 
             if self.event_idx {
-                if let Some(used_idx) = vring.mut_queue().add_used(mem, head.index, len) {
-                    if vring.needs_notification(&mem, Wrapping(used_idx)) {
+                let queue = vring.mut_queue();
+                if let Some(used_idx) = queue.add_used(mem, head.index, len) {
+                    if queue.needs_notification(&mem, Wrapping(used_idx)) {
                         debug!("signalling queue");
                         vring.signal_used_queue().unwrap();
                     } else {
@@ -215,9 +178,103 @@ impl VhostUserBlkBackend {
 
         used_any
     }
+}
 
-    pub fn set_vring_worker(&mut self, vring_worker: Option<Arc<VringWorker>>) {
-        self.vring_worker = vring_worker;
+struct VhostUserBlkBackend {
+    threads: Vec<Mutex<VhostUserBlkThread>>,
+    config: VirtioBlockConfig,
+    rdonly: bool,
+    poll_queue: bool,
+    queues_per_thread: Vec<u64>,
+    queue_size: usize,
+    acked_features: u64,
+    writeback: Arc<AtomicBool>,
+}
+
+impl VhostUserBlkBackend {
+    fn new(
+        image_path: String,
+        num_queues: usize,
+        rdonly: bool,
+        direct: bool,
+        poll_queue: bool,
+        queue_size: usize,
+    ) -> Result<Self> {
+        let mut options = OpenOptions::new();
+        options.read(true);
+        options.write(!rdonly);
+        if direct {
+            options.custom_flags(libc::O_DIRECT);
+        }
+        let image: File = options.open(&image_path).unwrap();
+        let mut raw_img: vm_virtio::RawFile = vm_virtio::RawFile::new(image, direct);
+
+        let image_id = build_disk_image_id(&PathBuf::from(&image_path));
+        let image_type = qcow::detect_image_type(&mut raw_img).unwrap();
+        let image = match image_type {
+            ImageType::Raw => Arc::new(Mutex::new(raw_img)) as Arc<Mutex<dyn DiskFile>>,
+            ImageType::Qcow2 => {
+                Arc::new(Mutex::new(QcowFile::from(raw_img).unwrap())) as Arc<Mutex<dyn DiskFile>>
+            }
+        };
+
+        let nsectors = (image.lock().unwrap().seek(SeekFrom::End(0)).unwrap() as u64) / SECTOR_SIZE;
+        let mut config = VirtioBlockConfig::default();
+
+        config.capacity = nsectors;
+        config.blk_size = BLK_SIZE;
+        config.size_max = 65535;
+        config.seg_max = 128 - 2;
+        config.min_io_size = 1;
+        config.opt_io_size = 1;
+        config.num_queues = num_queues as u16;
+        config.writeback = 1;
+
+        let mut queues_per_thread = Vec::new();
+        let mut threads = Vec::new();
+        let writeback = Arc::new(AtomicBool::new(true));
+        for i in 0..num_queues {
+            let thread = Mutex::new(VhostUserBlkThread::new(
+                image.clone(),
+                image_id.clone(),
+                nsectors,
+                writeback.clone(),
+            )?);
+            threads.push(thread);
+            queues_per_thread.push(0b1 << i);
+        }
+
+        Ok(VhostUserBlkBackend {
+            threads,
+            config,
+            rdonly,
+            poll_queue,
+            queues_per_thread,
+            queue_size,
+            acked_features: 0,
+            writeback,
+        })
+    }
+
+    fn update_writeback(&mut self) {
+        // Use writeback from config if VIRTIO_BLK_F_CONFIG_WCE
+        let writeback =
+            if self.acked_features & 1 << VIRTIO_BLK_F_CONFIG_WCE == 1 << VIRTIO_BLK_F_CONFIG_WCE {
+                self.config.writeback == 1
+            } else {
+                // Else check if VIRTIO_BLK_F_FLUSH negotiated
+                self.acked_features & 1 << VIRTIO_BLK_F_FLUSH == 1 << VIRTIO_BLK_F_FLUSH
+            };
+
+        info!(
+            "Changing cache mode to {}",
+            if writeback {
+                "writeback"
+            } else {
+                "writethrough"
+            }
+        );
+        self.writeback.store(writeback, Ordering::SeqCst);
     }
 }
 
@@ -227,7 +284,7 @@ impl VhostUserBackend for VhostUserBlkBackend {
     }
 
     fn max_queue_size(&self) -> usize {
-        QUEUE_SIZE
+        self.queue_size as usize
     }
 
     fn features(&self) -> u64 {
@@ -235,6 +292,7 @@ impl VhostUserBackend for VhostUserBlkBackend {
             | 1 << VIRTIO_BLK_F_CONFIG_WCE
             | 1 << VIRTIO_RING_F_EVENT_IDX
             | 1 << VIRTIO_F_VERSION_1
+            | 1 << VIRTIO_BLK_F_FLUSH
             | VhostUserVirtioFeatures::PROTOCOL_FEATURES.bits();
 
         if self.rdonly {
@@ -243,24 +301,34 @@ impl VhostUserBackend for VhostUserBlkBackend {
         avail_features
     }
 
+    fn acked_features(&mut self, features: u64) {
+        self.acked_features = features;
+        self.update_writeback();
+    }
+
     fn protocol_features(&self) -> VhostUserProtocolFeatures {
         VhostUserProtocolFeatures::CONFIG
     }
 
     fn set_event_idx(&mut self, enabled: bool) {
-        self.event_idx = enabled;
+        for thread in self.threads.iter() {
+            thread.lock().unwrap().event_idx = enabled;
+        }
     }
 
     fn update_memory(&mut self, mem: GuestMemoryMmap) -> VhostUserBackendResult<()> {
-        self.mem = Some(mem);
+        for thread in self.threads.iter() {
+            thread.lock().unwrap().mem = Some(mem.clone());
+        }
         Ok(())
     }
 
     fn handle_event(
-        &mut self,
+        &self,
         device_event: u16,
         evset: epoll::Events,
         vrings: &[Arc<RwLock<Vring>>],
+        thread_id: usize,
     ) -> VhostUserBackendResult<bool> {
         if evset != epoll::Events::EPOLLIN {
             return Err(Error::HandleEventNotEpollIn.into());
@@ -268,16 +336,17 @@ impl VhostUserBackend for VhostUserBlkBackend {
 
         debug!("event received: {:?}", device_event);
 
+        let mut thread = self.threads[thread_id].lock().unwrap();
         match device_event {
-            q if device_event < self.config.num_queues => {
-                let mut vring = vrings[q as usize].write().unwrap();
+            0 => {
+                let mut vring = vrings[0].write().unwrap();
 
                 if self.poll_queue {
                     // Actively poll the queue until POLL_QUEUE_US has passed
                     // without seeing a new request.
                     let mut now = Instant::now();
                     loop {
-                        if self.process_queue(&mut vring) {
+                        if thread.process_queue(&mut vring) {
                             now = Instant::now();
                         } else if now.elapsed().as_micros() > POLL_QUEUE_US {
                             break;
@@ -285,7 +354,7 @@ impl VhostUserBackend for VhostUserBlkBackend {
                     }
                 }
 
-                if self.event_idx {
+                if thread.event_idx {
                     // vm-virtio's Queue implementation only checks avail_index
                     // once, so to properly support EVENT_IDX we need to keep
                     // calling process_queue() until it stops finding new
@@ -293,14 +362,14 @@ impl VhostUserBackend for VhostUserBlkBackend {
                     loop {
                         vring
                             .mut_queue()
-                            .update_avail_event(self.mem.as_ref().unwrap());
-                        if !self.process_queue(&mut vring) {
+                            .update_avail_event(thread.mem.as_ref().unwrap());
+                        if !thread.process_queue(&mut vring) {
                             break;
                         }
                     }
                 } else {
                     // Without EVENT_IDX, a single call is enough.
-                    self.process_queue(&mut vring);
+                    thread.process_queue(&mut vring);
                 }
 
                 Ok(false)
@@ -310,86 +379,98 @@ impl VhostUserBackend for VhostUserBlkBackend {
     }
 
     fn get_config(&self, _offset: u32, _size: u32) -> Vec<u8> {
-        // self.config is a statically allocated virtio_blk_config
-        let buf = unsafe {
-            slice::from_raw_parts(
-                &self.config as *const virtio_blk_config as *const _,
-                mem::size_of::<virtio_blk_config>(),
-            )
-        };
-
-        buf.to_vec()
+        self.config.as_slice().to_vec()
     }
 
-    fn exit_event(&self) -> Option<(EventFd, Option<u16>)> {
-        Some((self.kill_evt.try_clone().unwrap(), None))
+    fn set_config(&mut self, offset: u32, data: &[u8]) -> result::Result<(), io::Error> {
+        let config_slice = self.config.as_mut_slice();
+        let data_len = data.len() as u32;
+        let config_len = config_slice.len() as u32;
+        if offset + data_len > config_len {
+            error!("Failed to write config space");
+            return Err(io::Error::from_raw_os_error(libc::EINVAL));
+        }
+        let (_, right) = config_slice.split_at_mut(offset as usize);
+        right.copy_from_slice(&data[..]);
+        self.update_writeback();
+        Ok(())
+    }
+
+    fn exit_event(&self, thread_index: usize) -> Option<(EventFd, Option<u16>)> {
+        // The exit event is placed after the queue, which is event index 1.
+        Some((
+            self.threads[thread_index]
+                .lock()
+                .unwrap()
+                .kill_evt
+                .try_clone()
+                .unwrap(),
+            Some(1),
+        ))
+    }
+
+    fn queues_per_thread(&self) -> Vec<u64> {
+        self.queues_per_thread.clone()
     }
 }
 
-pub struct VhostUserBlkBackendConfig<'a> {
-    pub image: &'a str,
-    pub sock: &'a str,
-    pub num_queues: usize,
-    pub readonly: bool,
-    pub direct: bool,
-    pub poll_queue: bool,
+struct VhostUserBlkBackendConfig {
+    path: String,
+    socket: String,
+    num_queues: usize,
+    queue_size: usize,
+    readonly: bool,
+    direct: bool,
+    poll_queue: bool,
 }
 
-impl<'a> VhostUserBlkBackendConfig<'a> {
-    pub fn parse(backend: &'a str) -> Result<Self> {
-        let params_list: Vec<&str> = backend.split(',').collect();
+impl VhostUserBlkBackendConfig {
+    fn parse(backend: &str) -> Result<Self> {
+        let mut parser = OptionParser::new();
+        parser
+            .add("path")
+            .add("readonly")
+            .add("direct")
+            .add("num_queues")
+            .add("queue_size")
+            .add("socket")
+            .add("poll_queue");
+        parser.parse(backend).map_err(Error::FailedConfigParse)?;
 
-        let mut image: &str = "";
-        let mut sock: &str = "";
-        let mut num_queues_str: &str = "";
-        let mut readonly: bool = false;
-        let mut direct: bool = false;
-        let mut poll_queue: bool = true;
+        let path = parser.get("path").ok_or(Error::PathParameterMissing)?;
+        let readonly = parser
+            .convert::<Toggle>("readonly")
+            .map_err(Error::FailedConfigParse)?
+            .unwrap_or(Toggle(false))
+            .0;
+        let direct = parser
+            .convert::<Toggle>("direct")
+            .map_err(Error::FailedConfigParse)?
+            .unwrap_or(Toggle(false))
+            .0;
+        let num_queues = parser
+            .convert("num_queues")
+            .map_err(Error::FailedConfigParse)?
+            .unwrap_or(1);
+        let socket = parser.get("socket").ok_or(Error::SocketParameterMissing)?;
+        let poll_queue = parser
+            .convert::<Toggle>("poll_queue")
+            .map_err(Error::FailedConfigParse)?
+            .unwrap_or_else(|| Toggle(true))
+            .0;
+        let queue_size = parser
+            .convert("queue_size")
+            .map_err(Error::FailedConfigParse)?
+            .unwrap_or(1024);
 
-        for param in params_list.iter() {
-            if param.starts_with("image=") {
-                image = &param[6..];
-            } else if param.starts_with("sock=") {
-                sock = &param[5..];
-            } else if param.starts_with("num_queues=") {
-                num_queues_str = &param[11..];
-            } else if param.starts_with("readonly=") {
-                readonly = match param[9..].parse::<bool>() {
-                    Ok(b) => b,
-                    Err(_) => return Err(Error::ParseReadOnlyParam),
-                }
-            } else if param.starts_with("direct=") {
-                direct = match param[7..].parse::<bool>() {
-                    Ok(b) => b,
-                    Err(_) => return Err(Error::ParseDirectParam),
-                }
-            } else if param.starts_with("poll_queue=") {
-                poll_queue = match param[11..].parse::<bool>() {
-                    Ok(b) => b,
-                    Err(_) => return Err(Error::ParsePollQueueParam),
-                }
-            }
-        }
-
-        let mut num_queues: usize = 1;
-        if image.is_empty() {
-            return Err(Error::ParseImageParam);
-        }
-        if sock.is_empty() {
-            return Err(Error::ParseSockParam);
-        }
-        if !num_queues_str.is_empty() {
-            num_queues = num_queues_str
-                .parse()
-                .map_err(Error::ParseBlkNumQueuesParam)?;
-        }
         Ok(VhostUserBlkBackendConfig {
-            image,
-            sock,
+            path,
+            socket,
             num_queues,
             readonly,
             direct,
             poll_queue,
+            queue_size,
         })
     }
 }
@@ -405,33 +486,26 @@ pub fn start_block_backend(backend_command: &str) {
 
     let blk_backend = Arc::new(RwLock::new(
         VhostUserBlkBackend::new(
-            backend_config.image.to_string(),
+            backend_config.path,
             backend_config.num_queues,
             backend_config.readonly,
             backend_config.direct,
             backend_config.poll_queue,
+            backend_config.queue_size,
         )
         .unwrap(),
     ));
 
     debug!("blk_backend is created!\n");
 
+    let listener = Listener::new(&backend_config.socket, true).unwrap();
+
     let name = "vhost-user-blk-backend";
-    let mut blk_daemon = VhostUserDaemon::new(
-        name.to_string(),
-        backend_config.sock.to_string(),
-        blk_backend.clone(),
-    )
-    .unwrap();
+    let mut blk_daemon = VhostUserDaemon::new(name.to_string(), blk_backend.clone()).unwrap();
+
     debug!("blk_daemon is created!\n");
 
-    let vring_worker = blk_daemon.get_vring_worker();
-    blk_backend
-        .write()
-        .unwrap()
-        .set_vring_worker(Some(vring_worker));
-
-    if let Err(e) = blk_daemon.start() {
+    if let Err(e) = blk_daemon.start(listener) {
         error!(
             "Failed to start daemon for vhost-user-block with error: {:?}\n",
             e
@@ -443,8 +517,9 @@ pub fn start_block_backend(backend_command: &str) {
         error!("Error from the main thread: {:?}", e);
     }
 
-    let kill_evt = &blk_backend.write().unwrap().kill_evt;
-    if let Err(e) = kill_evt.write(1) {
-        error!("Error shutting down worker thread: {:?}", e)
+    for thread in blk_backend.read().unwrap().threads.iter() {
+        if let Err(e) = thread.lock().unwrap().kill_evt.write(1) {
+            error!("Error shutting down worker thread: {:?}", e)
+        }
     }
 }

vhost_user_fs/Cargo.toml

@@ -6,8 +6,11 @@ edition = "2018"
 
 [dependencies]
 bitflags = "1.1.0"
-libc = "0.2.68"
+libc = "0.2.71"
 log = "0.4.8"
-vm-memory = "0.1.0"
+# Match the version in vmm
+seccomp = { git = "https://github.com/firecracker-microvm/firecracker", tag = "v0.21.1" }
+tempdir= "0.3.7"
+vm-memory = "0.2.1"
 vm-virtio = { path = "../vm-virtio" }
 vhost_rs = { git = "https://github.com/cloud-hypervisor/vhost", branch = "dragonball", package = "vhost", features = ["vhost-user-slave"] }

vhost_user_fs/src/descriptor_utils.rs

@@ -563,7 +563,7 @@ mod tests {
         use DescriptorType::*;
 
         let memory_start_addr = GuestAddress(0x0);
-        let memory = GuestMemoryMmap::from_ranges(&vec![(memory_start_addr, 0x10000)]).unwrap();
+        let memory = GuestMemoryMmap::from_ranges(&[(memory_start_addr, 0x10000)]).unwrap();
 
         let chain = create_descriptor_chain(
             &memory,
@@ -583,15 +583,15 @@ mod tests {
         assert_eq!(reader.bytes_read(), 0);
 
         let mut buffer = [0 as u8; 64];
-        if let Err(_) = reader.read_exact(&mut buffer) {
-            panic!("read_exact should not fail here");
+        if let Err(e) = reader.read_exact(&mut buffer) {
+            panic!("read_exact should not fail here: {:?}", e);
         }
 
         assert_eq!(reader.available_bytes(), 42);
         assert_eq!(reader.bytes_read(), 64);
 
         match reader.read(&mut buffer) {
-            Err(_) => panic!("read should not fail here"),
+            Err(e) => panic!("read should not fail here: {:?}", e),
             Ok(length) => assert_eq!(length, 42),
         }
 
@@ -604,7 +604,7 @@ mod tests {
         use DescriptorType::*;
 
         let memory_start_addr = GuestAddress(0x0);
-        let memory = GuestMemoryMmap::from_ranges(&vec![(memory_start_addr, 0x10000)]).unwrap();
+        let memory = GuestMemoryMmap::from_ranges(&[(memory_start_addr, 0x10000)]).unwrap();
 
         let chain = create_descriptor_chain(
             &memory,
@@ -623,16 +623,16 @@ mod tests {
         assert_eq!(writer.available_bytes(), 106);
         assert_eq!(writer.bytes_written(), 0);
 
-        let mut buffer = [0 as u8; 64];
-        if let Err(_) = writer.write_all(&mut buffer) {
-            panic!("write_all should not fail here");
+        let buffer = [0 as u8; 64];
+        if let Err(e) = writer.write_all(&buffer) {
+            panic!("write_all should not fail here: {:?}", e);
         }
 
         assert_eq!(writer.available_bytes(), 42);
         assert_eq!(writer.bytes_written(), 64);
 
-        match writer.write(&mut buffer) {
-            Err(_) => panic!("write should not fail here"),
+        match writer.write(&buffer) {
+            Err(e) => panic!("write should not fail here {:?}", e),
             Ok(length) => assert_eq!(length, 42),
         }
 
@@ -645,7 +645,7 @@ mod tests {
         use DescriptorType::*;
 
         let memory_start_addr = GuestAddress(0x0);
-        let memory = GuestMemoryMmap::from_ranges(&vec![(memory_start_addr, 0x10000)]).unwrap();
+        let memory = GuestMemoryMmap::from_ranges(&[(memory_start_addr, 0x10000)]).unwrap();
 
         let chain = create_descriptor_chain(
             &memory,
@@ -670,7 +670,7 @@ mod tests {
         use DescriptorType::*;
 
         let memory_start_addr = GuestAddress(0x0);
-        let memory = GuestMemoryMmap::from_ranges(&vec![(memory_start_addr, 0x10000)]).unwrap();
+        let memory = GuestMemoryMmap::from_ranges(&[(memory_start_addr, 0x10000)]).unwrap();
 
         let chain = create_descriptor_chain(
             &memory,
@@ -695,7 +695,7 @@ mod tests {
         use DescriptorType::*;
 
         let memory_start_addr = GuestAddress(0x0);
-        let memory = GuestMemoryMmap::from_ranges(&vec![(memory_start_addr, 0x10000)]).unwrap();
+        let memory = GuestMemoryMmap::from_ranges(&[(memory_start_addr, 0x10000)]).unwrap();
 
         let chain = create_descriptor_chain(
             &memory,
@@ -743,9 +743,9 @@ mod tests {
         use DescriptorType::*;
 
         let memory_start_addr = GuestAddress(0x0);
-        let memory = GuestMemoryMmap::from_ranges(&vec![(memory_start_addr, 0x10000)]).unwrap();
+        let memory = GuestMemoryMmap::from_ranges(&[(memory_start_addr, 0x10000)]).unwrap();
 
-        let secret: Le32 = 0x12345678.into();
+        let secret: Le32 = 0x1234_5678.into();
 
         // Create a descriptor chain with memory regions that are properly separated.
         let chain_writer = create_descriptor_chain(
@@ -757,8 +757,8 @@ mod tests {
         )
         .expect("create_descriptor_chain failed");
         let mut writer = Writer::new(&memory, chain_writer).expect("failed to create Writer");
-        if let Err(_) = writer.write_obj(secret) {
-            panic!("write_obj should not fail here");
+        if let Err(e) = writer.write_obj(secret) {
+            panic!("write_obj should not fail here: {:?}", e);
         }
 
         // Now create new descriptor chain pointing to the same memory and try to read it.
@@ -772,7 +772,7 @@ mod tests {
         .expect("create_descriptor_chain failed");
         let mut reader = Reader::new(&memory, chain_reader).expect("failed to create Reader");
         match reader.read_obj::<Le32>() {
-            Err(_) => panic!("read_obj should not fail here"),
+            Err(e) => panic!("read_obj should not fail here: {:?}", e),
             Ok(read_secret) => assert_eq!(read_secret, secret),
         }
     }
@@ -782,7 +782,7 @@ mod tests {
         use DescriptorType::*;
 
         let memory_start_addr = GuestAddress(0x0);
-        let memory = GuestMemoryMmap::from_ranges(&vec![(memory_start_addr, 0x10000)]).unwrap();
+        let memory = GuestMemoryMmap::from_ranges(&[(memory_start_addr, 0x10000)]).unwrap();
 
         let chain = create_descriptor_chain(
             &memory,
@@ -795,8 +795,7 @@ mod tests {
 
         let mut reader = Reader::new(&memory, chain).expect("failed to create Reader");
 
-        let mut buf = Vec::with_capacity(1024);
-        buf.resize(1024, 0);
+        let mut buf = vec![0; 1024];
 
         assert_eq!(
             reader
@@ -812,7 +811,7 @@ mod tests {
         use DescriptorType::*;
 
         let memory_start_addr = GuestAddress(0x0);
-        let memory = GuestMemoryMmap::from_ranges(&vec![(memory_start_addr, 0x10000)]).unwrap();
+        let memory = GuestMemoryMmap::from_ranges(&[(memory_start_addr, 0x10000)]).unwrap();
 
         let chain = create_descriptor_chain(
             &memory,
@@ -841,7 +840,7 @@ mod tests {
         use DescriptorType::*;
 
         let memory_start_addr = GuestAddress(0x0);
-        let memory = GuestMemoryMmap::from_ranges(&vec![(memory_start_addr, 0x10000)]).unwrap();
+        let memory = GuestMemoryMmap::from_ranges(&[(memory_start_addr, 0x10000)]).unwrap();
 
         let chain = create_descriptor_chain(
             &memory,
@@ -870,7 +869,7 @@ mod tests {
         use DescriptorType::*;
 
         let memory_start_addr = GuestAddress(0x0);
-        let memory = GuestMemoryMmap::from_ranges(&vec![(memory_start_addr, 0x10000)]).unwrap();
+        let memory = GuestMemoryMmap::from_ranges(&[(memory_start_addr, 0x10000)]).unwrap();
 
         let chain = create_descriptor_chain(
             &memory,
@@ -899,7 +898,7 @@ mod tests {
         use DescriptorType::*;
 
         let memory_start_addr = GuestAddress(0x0);
-        let memory = GuestMemoryMmap::from_ranges(&vec![(memory_start_addr, 0x10000)]).unwrap();
+        let memory = GuestMemoryMmap::from_ranges(&[(memory_start_addr, 0x10000)]).unwrap();
 
         let chain = create_descriptor_chain(
             &memory,
@@ -928,7 +927,7 @@ mod tests {
         use DescriptorType::*;
 
         let memory_start_addr = GuestAddress(0x0);
-        let memory = GuestMemoryMmap::from_ranges(&vec![(memory_start_addr, 0x10000)]).unwrap();
+        let memory = GuestMemoryMmap::from_ranges(&[(memory_start_addr, 0x10000)]).unwrap();
 
         let chain = create_descriptor_chain(
             &memory,
@@ -947,7 +946,7 @@ mod tests {
         .expect("create_descriptor_chain failed");
         let mut reader = Reader::new(&memory, chain).expect("failed to create Reader");
 
-        if let Ok(_) = reader.split_at(256) {
+        if reader.split_at(256).is_ok() {
             panic!("successfully split Reader with out of bounds offset");
         }
     }
@@ -957,7 +956,7 @@ mod tests {
         use DescriptorType::*;
 
         let memory_start_addr = GuestAddress(0x0);
-        let memory = GuestMemoryMmap::from_ranges(&vec![(memory_start_addr, 0x10000)]).unwrap();
+        let memory = GuestMemoryMmap::from_ranges(&[(memory_start_addr, 0x10000)]).unwrap();
 
         let chain = create_descriptor_chain(
             &memory,
@@ -981,7 +980,7 @@ mod tests {
         use DescriptorType::*;
 
         let memory_start_addr = GuestAddress(0x0);
-        let memory = GuestMemoryMmap::from_ranges(&vec![(memory_start_addr, 0x10000)]).unwrap();
+        let memory = GuestMemoryMmap::from_ranges(&[(memory_start_addr, 0x10000)]).unwrap();
 
         let chain = create_descriptor_chain(
             &memory,

vhost_user_fs/src/filesystem.rs

@@ -9,8 +9,6 @@ use std::io;
 use std::mem;
 use std::time::Duration;
 
-use libc;
-
 use crate::fuse;
 
 use super::fs_cache_req_handler::FsCacheReqHandler;
@@ -682,11 +680,11 @@ pub trait FileSystem {
     /// implementation did not return a `Handle` from `open` then the contents of `handle` are
     /// undefined.
     ///
-    /// If the `FsOptions::HANDLE_KILLPRIV` feature is not enabled then then the file system is
-    /// expected to clear the setuid and setgid bits.
-    ///
     /// If `delayed_write` is true then it indicates that this is a write for buffered data.
     ///
+    /// If `kill_priv` is true then it indicates that the file system is expected to clear the
+    /// setuid and setgid bits.
+    ///
     /// This method should return exactly the number of bytes requested by the kernel, except in the
     /// case of error. An exception to this rule is if the file was opened with the "direct I/O"
     /// option (`libc::O_DIRECT`), in which case the kernel will forward the return code from this
@@ -702,6 +700,7 @@ pub trait FileSystem {
         offset: u64,
         lock_owner: Option<u64>,
         delayed_write: bool,
+        kill_priv: bool,
         flags: u32,
     ) -> io::Result<usize> {
         Err(io::Error::from_raw_os_error(libc::ENOSYS))
@@ -1096,6 +1095,22 @@ pub trait FileSystem {
         Err(io::Error::from_raw_os_error(libc::ENOSYS))
     }
 
+    #[allow(clippy::too_many_arguments)]
+    fn copyfilerange(
+        &self,
+        ctx: Context,
+        inode_in: Self::Inode,
+        handle_in: Self::Handle,
+        offset_in: u64,
+        inode_out: Self::Inode,
+        handle_out: Self::Handle,
+        offset_out: u64,
+        len: u64,
+        flags: u64,
+    ) -> io::Result<usize> {
+        Err(io::Error::from_raw_os_error(libc::ENOSYS))
+    }
+
     /// TODO: support this
     fn getlk(&self) -> io::Result<()> {
         Err(io::Error::from_raw_os_error(libc::ENOSYS))

vhost_user_fs/src/fs_cache_req_handler.rs

@@ -42,7 +42,8 @@ impl FsCacheReqHandler for SlaveFsCacheReq {
             VhostUserFSSlaveMsgFlags::MAP_R
         };
 
-        self.fs_slave_map(&msg, fd)
+        self.fs_slave_map(&msg, fd)?;
+        Ok(())
     }
 
     fn unmap(&mut self, requests: Vec<fuse::RemovemappingOne>) -> io::Result<()> {

vhost_user_fs/src/fuse.rs

@@ -5,7 +5,6 @@
 use std::mem;
 
 use bitflags::bitflags;
-use libc;
 use vm_memory::ByteValued;
 
 /// Version number of this interface.
@@ -55,6 +54,9 @@ const FOPEN_KEEP_CACHE: u32 = 2;
 /// The file is not seekable.
 const FOPEN_NONSEEKABLE: u32 = 4;
 
+/// Allow caching this directory.
+const FOPEN_CACHE_DIR: u32 = 8;
+
 bitflags! {
     /// Options controlling the behavior of files opened by the server in response
     /// to an open or create request.
@@ -62,6 +64,7 @@ bitflags! {
         const DIRECT_IO = FOPEN_DIRECT_IO;
         const KEEP_CACHE = FOPEN_KEEP_CACHE;
         const NONSEEKABLE = FOPEN_NONSEEKABLE;
+        const CACHE_DIR = FOPEN_CACHE_DIR;
     }
 }
 
@@ -130,6 +133,21 @@ const HANDLE_KILLPRIV: u32 = 524_288;
 /// FileSystem supports posix acls.
 const POSIX_ACL: u32 = 1_048_576;
 
+/// Reading the device after abort returns ECONNABORTED.
+const ABORT_ERROR: u32 = 2_097_152;
+
+/// Init_out.max_pages contains the max number of req pages.
+const MAX_PAGES: u32 = 4_194_304;
+
+/// Cache READLINK responses
+const CACHE_SYMLINKS: u32 = 8_388_608;
+
+/// Kernel supports zero-message opendir
+const NO_OPENDIR_SUPPORT: u32 = 16_777_216;
+
+/// Only invalidate cached pages on explicit request
+const EXPLICIT_INVAL_DATA: u32 = 33_554_432;
+
 bitflags! {
     /// A bitfield passed in as a parameter to and returned from the `init` method of the
     /// `FileSystem` trait.
@@ -296,6 +314,43 @@ bitflags! {
         ///
         /// This feature is disabled by default.
         const POSIX_ACL = POSIX_ACL;
+
+        /// Indicates that if the connection is gone because of sysfs abort, reading from the device
+        /// will return -ECONNABORTED.
+        ///
+        /// This feature is not currently supported.
+        const ABORT_ERROR = ABORT_ERROR;
+
+        /// Indicates support for negotiating the maximum number of pages supported.
+        ///
+        /// If this feature is enabled, we can tell the kernel the maximum number of pages that we
+        /// support to transfer in a single request.
+        ///
+        /// This feature is enabled by default if supported by the kernel.
+        const MAX_PAGES = MAX_PAGES;
+
+        /// Indicates that the kernel supports caching READLINK responses.
+        ///
+        /// This feature is not currently supported.
+        const CACHE_SYMLINKS = CACHE_SYMLINKS;
+
+        /// Indicates support for zero-message opens. If this flag is set in the `capable` parameter
+        /// of the `init` trait method, then the file system may return `ENOSYS` from the opendir() handler
+        /// to indicate success. Further attempts to open directories will be handled in the kernel. (If
+        /// this flag is not set, returning ENOSYS will be treated as an error and signaled to the
+        /// caller).
+        ///
+        /// Setting (or not setting) the field in the `FsOptions` returned from the `init` method
+        /// has no effect.
+        const ZERO_MESSAGE_OPENDIR = NO_OPENDIR_SUPPORT;
+
+        /// Indicates support for explicit data invalidation. If this feature is enabled, the
+        /// server is fully responsible for data cache invalidation, and the kernel won't
+        /// invalidate files data cache on size change and only truncate that cache to new size
+        /// in case the size decreased.
+        ///
+        /// This feature is not currently supported.
+        const EXPLICIT_INVAL_DATA = EXPLICIT_INVAL_DATA;
     }
 }
 
@@ -317,6 +372,9 @@ pub const WRITE_CACHE: u32 = 1;
 /// `lock_owner` field is valid.
 pub const WRITE_LOCKOWNER: u32 = 2;
 
+/// Kill suid and sgid bits
+pub const WRITE_KILL_PRIV: u32 = 4;
+
 // Read flags.
 pub const READ_LOCKOWNER: u32 = 2;
 
@@ -337,6 +395,9 @@ const IOCTL_32BIT: u32 = 8;
 /// Is a directory
 const IOCTL_DIR: u32 = 16;
 
+/// x32 compat ioctl on 64bit machine (64bit time_t)
+const IOCTL_COMPAT_X32: u32 = 32;
+
 /// Maximum of in_iovecs + out_iovecs
 const IOCTL_MAX_IOV: u32 = 256;
 
@@ -357,6 +418,9 @@ bitflags! {
         /// Is a directory
         const IOCTL_DIR = IOCTL_DIR;
 
+        /// x32 compat ioctl on 64bit machine (64bit time_t)
+        const IOCTL_COMPAT_X32 = IOCTL_COMPAT_X32;
+
         /// Maximum of in_iovecs + out_iovecs
         const IOCTL_MAX_IOV = IOCTL_MAX_IOV;
     }
@@ -835,7 +899,9 @@ pub struct InitOut {
     pub congestion_threshold: u16,
     pub max_write: u32,
     pub time_gran: u32,
-    pub unused: [u32; 9],
+    pub max_pages: u16,
+    pub padding: u16,
+    pub unused: [u32; 8],
 }
 unsafe impl ByteValued for InitOut {}
 
@@ -1049,6 +1115,19 @@ pub struct LseekOut {
 }
 unsafe impl ByteValued for LseekOut {}
 
+#[repr(C)]
+#[derive(Debug, Default, Copy, Clone)]
+pub struct CopyfilerangeIn {
+    pub fh_in: u64,
+    pub off_in: u64,
+    pub nodeid_out: u64,
+    pub fh_out: u64,
+    pub off_out: u64,
+    pub len: u64,
+    pub flags: u64,
+}
+unsafe impl ByteValued for CopyfilerangeIn {}
+
 bitflags! {
     pub struct SetupmappingFlags: u64 {
     const WRITE = 0x1;

vhost_user_fs/src/lib.rs

@@ -12,6 +12,8 @@ pub mod fs_cache_req_handler;
 pub mod fuse;
 pub mod multikey;
 pub mod passthrough;
+pub mod sandbox;
+pub mod seccomp;
 pub mod server;
 
 use std::ffi::FromBytesWithNulError;

vhost_user_fs/src/passthrough.rs

@@ -2,6 +2,13 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
+use super::fs_cache_req_handler::FsCacheReqHandler;
+use crate::filesystem::{
+    Context, DirEntry, Entry, FileSystem, FsOptions, GetxattrReply, ListxattrReply, OpenOptions,
+    SetattrValid, ZeroCopyReader, ZeroCopyWriter,
+};
+use crate::fuse;
+use crate::multikey::MultikeyBTreeMap;
 use std::collections::btree_map;
 use std::collections::BTreeMap;
 use std::ffi::{CStr, CString};
@@ -13,22 +20,12 @@ use std::str::FromStr;
 use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
 use std::sync::{Arc, RwLock};
 use std::time::Duration;
-
-use libc;
 use vm_memory::ByteValued;
 
-use super::fs_cache_req_handler::FsCacheReqHandler;
-use crate::filesystem::{
-    Context, DirEntry, Entry, FileSystem, FsOptions, GetxattrReply, ListxattrReply, OpenOptions,
-    SetattrValid, ZeroCopyReader, ZeroCopyWriter,
-};
-use crate::fuse;
-use crate::multikey::MultikeyBTreeMap;
-
 const CURRENT_DIR_CSTR: &[u8] = b".\0";
 const PARENT_DIR_CSTR: &[u8] = b"..\0";
 const EMPTY_CSTR: &[u8] = b"\0";
-const PROC_CSTR: &[u8] = b"/proc\0";
+const PROC_CSTR: &[u8] = b"/proc/self/fd\0";
 
 type Inode = u64;
 type Handle = u64;
@@ -238,6 +235,13 @@ pub struct Config {
     ///
     /// The default value for this options is `false`.
     pub xattr: bool,
+
+    /// Optional file descriptor for /proc/self/fd. Callers can obtain a file descriptor and pass it
+    /// here, so there's no need to open it in PassthroughFs::new(). This is specially useful for
+    /// sandboxing.
+    ///
+    /// The default is `None`.
+    pub proc_sfd_rawfd: Option<RawFd>,
 }
 
 impl Default for Config {
@@ -249,6 +253,7 @@ impl Default for Config {
             writeback: false,
             root_dir: String::from("/"),
             xattr: false,
+            proc_sfd_rawfd: None,
         }
     }
 }
@@ -271,11 +276,11 @@ pub struct PassthroughFs {
     handles: RwLock<BTreeMap<Handle, Arc<HandleData>>>,
     next_handle: AtomicU64,
 
-    // File descriptor pointing to the `/proc` directory. This is used to convert an fd from
+    // File descriptor pointing to the `/proc/self/fd` directory. This is used to convert an fd from
     // `inodes` into one that can go into `handles`. This is accomplished by reading the
-    // `self/fd/{}` symlink. We keep an open fd here in case the file system tree that we are meant
-    // to be serving doesn't have access to `/proc`.
-    proc: File,
+    // `/proc/self/fd/{}` symlink. We keep an open fd here in case the file system tree that we are
+    // meant to be serving doesn't have access to `/proc/self/fd`.
+    proc_self_fd: File,
 
     // Whether writeback caching is enabled for this directory. This will only be true when
     // `cfg.writeback` is true and `init` was called with `FsOptions::WRITEBACK_CACHE`.
@@ -286,23 +291,29 @@ pub struct PassthroughFs {
 
 impl PassthroughFs {
     pub fn new(cfg: Config) -> io::Result<PassthroughFs> {
-        // Safe because this is a constant value and a valid C string.
-        let proc_cstr = unsafe { CStr::from_bytes_with_nul_unchecked(PROC_CSTR) };
+        let fd = if let Some(fd) = cfg.proc_sfd_rawfd {
+            fd
+        } else {
+            // Safe because this is a constant value and a valid C string.
+            let proc_cstr = unsafe { CStr::from_bytes_with_nul_unchecked(PROC_CSTR) };
 
-        // Safe because this doesn't modify any memory and we check the return value.
-        let fd = unsafe {
-            libc::openat(
-                libc::AT_FDCWD,
-                proc_cstr.as_ptr(),
-                libc::O_PATH | libc::O_NOFOLLOW | libc::O_CLOEXEC,
-            )
+            // Safe because this doesn't modify any memory and we check the return value.
+            let fd = unsafe {
+                libc::openat(
+                    libc::AT_FDCWD,
+                    proc_cstr.as_ptr(),
+                    libc::O_PATH | libc::O_NOFOLLOW | libc::O_CLOEXEC,
+                )
+            };
+            if fd < 0 {
+                return Err(io::Error::last_os_error());
+            }
+
+            fd
         };
-        if fd < 0 {
-            return Err(io::Error::last_os_error());
-        }
 
-        // Safe because we just opened this fd.
-        let proc = unsafe { File::from_raw_fd(fd) };
+        // Safe because we just opened this fd or it was provided by our caller.
+        let proc_self_fd = unsafe { File::from_raw_fd(fd) };
 
         Ok(PassthroughFs {
             inodes: RwLock::new(MultikeyBTreeMap::new()),
@@ -311,7 +322,7 @@ impl PassthroughFs {
             handles: RwLock::new(BTreeMap::new()),
             next_handle: AtomicU64::new(0),
 
-            proc,
+            proc_self_fd,
 
             writeback: AtomicBool::new(false),
             cfg,
@@ -319,7 +330,7 @@ impl PassthroughFs {
     }
 
     pub fn keep_fds(&self) -> Vec<RawFd> {
-        vec![self.proc.as_raw_fd()]
+        vec![self.proc_self_fd.as_raw_fd()]
     }
 
     fn open_inode(&self, inode: Inode, mut flags: i32) -> io::Result<File> {
@@ -331,7 +342,7 @@ impl PassthroughFs {
             .map(Arc::clone)
             .ok_or_else(ebadf)?;
 
-        let pathname = CString::new(format!("self/fd/{}", data.file.as_raw_fd()))
+        let pathname = CString::new(format!("{}", data.file.as_raw_fd()))
             .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?;
 
         // When writeback caching is enabled, the kernel may send read requests even if the
@@ -359,7 +370,7 @@ impl PassthroughFs {
         // to follow the `/proc/self/fd` symlink to get the file.
         let fd = unsafe {
             libc::openat(
-                self.proc.as_raw_fd(),
+                self.proc_self_fd.as_raw_fd(),
                 pathname.as_ptr(),
                 (flags | libc::O_CLOEXEC) & (!libc::O_NOFOLLOW),
             )
@@ -557,7 +568,13 @@ impl PassthroughFs {
                 OpenOptions::DIRECT_IO,
                 flags & (libc::O_DIRECTORY as u32) == 0,
             ),
-            CachePolicy::Always => opts |= OpenOptions::KEEP_CACHE,
+            CachePolicy::Always => {
+                if flags & (libc::O_DIRECTORY as u32) == 0 {
+                    opts |= OpenOptions::KEEP_CACHE;
+                } else {
+                    opts |= OpenOptions::CACHE_DIR;
+                }
+            }
             _ => {}
         };
 
@@ -991,11 +1008,15 @@ impl FileSystem for PassthroughFs {
         offset: u64,
         _lock_owner: Option<u64>,
         _delayed_write: bool,
+        kill_priv: bool,
         _flags: u32,
     ) -> io::Result<usize> {
-        // We need to change credentials during a write so that the kernel will remove setuid or
-        // setgid bits from the file if it was written to by someone other than the owner.
-        let (_uid, _gid) = set_creds(ctx.uid, ctx.gid)?;
+        if kill_priv {
+            // We need to change credentials during a write so that the kernel will remove setuid
+            // or setgid bits from the file if it was written to by someone other than the owner.
+            let (_uid, _gid) = set_creds(ctx.uid, ctx.gid)?;
+        }
+
         let data = self
             .handles
             .read()
@@ -1055,7 +1076,7 @@ impl FileSystem for PassthroughFs {
             let fd = hd.file.write().unwrap().as_raw_fd();
             Data::Handle(hd, fd)
         } else {
-            let pathname = CString::new(format!("self/fd/{}", inode_data.file.as_raw_fd()))
+            let pathname = CString::new(format!("{}", inode_data.file.as_raw_fd()))
                 .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?;
             Data::ProcPath(pathname)
         };
@@ -1066,7 +1087,7 @@ impl FileSystem for PassthroughFs {
                 match data {
                     Data::Handle(_, fd) => libc::fchmod(fd, attr.st_mode),
                     Data::ProcPath(ref p) => {
-                        libc::fchmodat(self.proc.as_raw_fd(), p.as_ptr(), attr.st_mode, 0)
+                        libc::fchmodat(self.proc_self_fd.as_raw_fd(), p.as_ptr(), attr.st_mode, 0)
                     }
                 }
             };
@@ -1152,7 +1173,7 @@ impl FileSystem for PassthroughFs {
             let res = match data {
                 Data::Handle(_, fd) => unsafe { libc::futimens(fd, tvs.as_ptr()) },
                 Data::ProcPath(ref p) => unsafe {
-                    libc::utimensat(self.proc.as_raw_fd(), p.as_ptr(), tvs.as_ptr(), 0)
+                    libc::utimensat(self.proc_self_fd.as_raw_fd(), p.as_ptr(), tvs.as_ptr(), 0)
                 },
             };
             if res < 0 {
@@ -1644,4 +1665,60 @@ impl FileSystem for PassthroughFs {
             Ok(res as u64)
         }
     }
+
+    fn copyfilerange(
+        &self,
+        _ctx: Context,
+        inode_in: Inode,
+        handle_in: Handle,
+        offset_in: u64,
+        inode_out: Inode,
+        handle_out: Handle,
+        offset_out: u64,
+        len: u64,
+        flags: u64,
+    ) -> io::Result<usize> {
+        let data_in = self
+            .handles
+            .read()
+            .unwrap()
+            .get(&handle_in)
+            .filter(|hd| hd.inode == inode_in)
+            .map(Arc::clone)
+            .ok_or_else(ebadf)?;
+
+        // Take just a read lock as we're not going to alter the file descriptor offset.
+        let fd_in = data_in.file.read().unwrap().as_raw_fd();
+
+        let data_out = self
+            .handles
+            .read()
+            .unwrap()
+            .get(&handle_out)
+            .filter(|hd| hd.inode == inode_out)
+            .map(Arc::clone)
+            .ok_or_else(ebadf)?;
+
+        // Take just a read lock as we're not going to alter the file descriptor offset.
+        let fd_out = data_out.file.read().unwrap().as_raw_fd();
+
+        // Safe because this will only modify `offset_in` and `offset_out` and we check
+        // the return value.
+        let res = unsafe {
+            libc::syscall(
+                libc::SYS_copy_file_range,
+                fd_in,
+                &mut (offset_in as i64) as &mut _ as *mut _,
+                fd_out,
+                &mut (offset_out as i64) as &mut _ as *mut _,
+                len,
+                flags,
+            )
+        };
+        if res < 0 {
+            Err(io::Error::last_os_error())
+        } else {
+            Ok(res as usize)
+        }
+    }
 }

vhost_user_fs/src/sandbox.rs

@@ -0,0 +1,292 @@
+// Copyright 2020 Red Hat, Inc. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+use std::ffi::CString;
+use std::os::unix::io::RawFd;
+use std::{fmt, io};
+
+use tempdir::TempDir;
+
+#[derive(Debug)]
+pub enum Error {
+    /// Failed to bind mount `/proc/self/fd` into a temporary directory.
+    BindMountProcSelfFd(io::Error),
+    /// Failed to bind mount shared directory.
+    BindMountSharedDir(io::Error),
+    /// Failed to change to the old root directory.
+    ChdirOldRoot(io::Error),
+    /// Failed to change to the new root directory.
+    ChdirNewRoot(io::Error),
+    /// Failed to clean the properties of the mount point.
+    CleanMount(io::Error),
+    /// Failed to create a temporary directory.
+    CreateTempDir(io::Error),
+    /// Call to libc::fork returned an error.
+    Fork(io::Error),
+    /// Error bind-mounting a directory.
+    MountBind(io::Error),
+    /// Failed to mount old root.
+    MountOldRoot(io::Error),
+    /// Error mounting proc.
+    MountProc(io::Error),
+    /// Failed to mount new root.
+    MountNewRoot(io::Error),
+    /// Error mounting target directory.
+    MountTarget(io::Error),
+    /// Failed to open new root.
+    OpenNewRoot(io::Error),
+    /// Failed to open old root.
+    OpenOldRoot(io::Error),
+    /// Failed to open `/proc/self/fd`.
+    OpenProcSelfFd(io::Error),
+    /// Error switching root directory.
+    PivotRoot(io::Error),
+    /// Failed to remove temporary directory.
+    RmdirTempDir(io::Error),
+    /// Failed to lazily unmount old root.
+    UmountOldRoot(io::Error),
+    /// Failed to lazily unmount temporary directory.
+    UmountTempDir(io::Error),
+    /// Call to libc::unshare returned an error.
+    Unshare(io::Error),
+}
+
+impl fmt::Display for Error {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "vhost_user_fs_sandbox_error: {:?}", self)
+    }
+}
+
+/// A helper for creating a sandbox for isolating the service.
+pub struct Sandbox {
+    /// The directory that is going to be shared with the VM. The sandbox will be constructed on top
+    /// of this directory.
+    shared_dir: String,
+    /// A file descriptor for `/proc/self/fd` obtained from the sandboxed context.
+    proc_self_fd: Option<RawFd>,
+}
+
+impl Sandbox {
+    pub fn new(shared_dir: String) -> Self {
+        Sandbox {
+            shared_dir,
+            proc_self_fd: None,
+        }
+    }
+
+    // Make `self.shared_dir` our root directory, and get an isolated file descriptor for
+    // `/proc/self/fd`.
+    //
+    // This is based on virtiofsd's setup_namespaces() and setup_mounts(), and it's very similar to
+    // the strategy used in containers. Consists on a careful sequence of mounts and bind-mounts to
+    // ensure it's not possible to escape the sandbox through `self.shared_dir` nor the file
+    // descriptor obtained for `/proc/self/fd`.
+    //
+    // It's ugly, but it's the only way until Linux implements a proper containerization API.
+    fn setup_mounts(&mut self) -> Result<(), Error> {
+        // Ensure our mount changes don't affect the parent mount namespace.
+        let c_root_dir = CString::new("/").unwrap();
+        let ret = unsafe {
+            libc::mount(
+                std::ptr::null(),
+                c_root_dir.as_ptr(),
+                std::ptr::null(),
+                libc::MS_SLAVE | libc::MS_REC,
+                std::ptr::null(),
+            )
+        };
+        if ret != 0 {
+            return Err(Error::CleanMount(std::io::Error::last_os_error()));
+        }
+
+        // Mount `/proc` in this context.
+        let c_proc_dir = CString::new("/proc").unwrap();
+        let c_proc_fs = CString::new("proc").unwrap();
+        let ret = unsafe {
+            libc::mount(
+                c_proc_fs.as_ptr(),
+                c_proc_dir.as_ptr(),
+                c_proc_fs.as_ptr(),
+                libc::MS_NODEV | libc::MS_NOEXEC | libc::MS_NOSUID | libc::MS_RELATIME,
+                std::ptr::null(),
+            )
+        };
+        if ret != 0 {
+            return Err(Error::MountProc(std::io::Error::last_os_error()));
+        }
+
+        // Bind-mount `/proc/self/fd` info a temporary directory, preventing access to ancestor
+        // directories.
+        let c_proc_self_fd = CString::new("/proc/self/fd").unwrap();
+        let tmp_dir = TempDir::new("vhostuserfs-")
+            .map_err(|_| Error::CreateTempDir(std::io::Error::last_os_error()))?;
+        let c_tmp_dir = CString::new(tmp_dir.into_path().to_str().unwrap()).unwrap();
+        let ret = unsafe {
+            libc::mount(
+                c_proc_self_fd.as_ptr(),
+                c_tmp_dir.as_ptr(),
+                std::ptr::null(),
+                libc::MS_BIND,
+                std::ptr::null(),
+            )
+        };
+        if ret < 0 {
+            return Err(Error::BindMountProcSelfFd(std::io::Error::last_os_error()));
+        }
+
+        // Obtain a file descriptor for `/proc/self/fd` through the bind-mounted temporary directory.
+        let proc_self_fd = unsafe { libc::open(c_tmp_dir.as_ptr(), libc::O_PATH) };
+        if proc_self_fd < 0 {
+            return Err(Error::OpenProcSelfFd(std::io::Error::last_os_error()));
+        }
+        self.proc_self_fd = Some(proc_self_fd);
+
+        // Now that we have a file descriptor for `/proc/self/fd`, we no longer need the bind-mount.
+        // Unmount it and remove the temporary directory.
+        let ret = unsafe { libc::umount2(c_tmp_dir.as_ptr(), libc::MNT_DETACH) };
+        if ret < 0 {
+            return Err(Error::UmountTempDir(std::io::Error::last_os_error()));
+        }
+        let ret = unsafe { libc::rmdir(c_tmp_dir.as_ptr()) };
+        if ret < 0 {
+            return Err(Error::RmdirTempDir(std::io::Error::last_os_error()));
+        }
+
+        // Bind-mount `self.shared_dir` on itself so we can use as new root on `pivot_root` syscall.
+        let c_shared_dir = CString::new(self.shared_dir.clone()).unwrap();
+        let ret = unsafe {
+            libc::mount(
+                c_shared_dir.as_ptr(),
+                c_shared_dir.as_ptr(),
+                std::ptr::null(),
+                libc::MS_BIND | libc::MS_REC,
+                std::ptr::null(),
+            )
+        };
+        if ret < 0 {
+            return Err(Error::BindMountSharedDir(std::io::Error::last_os_error()));
+        }
+
+        // Get a file descriptor to our old root so we can reference it after switching root.
+        let oldroot_fd = unsafe {
+            libc::open(
+                c_root_dir.as_ptr(),
+                libc::O_DIRECTORY | libc::O_RDONLY | libc::O_CLOEXEC,
+            )
+        };
+        if oldroot_fd < 0 {
+            return Err(Error::OpenOldRoot(std::io::Error::last_os_error()));
+        }
+
+        // Get a file descriptor to the new root so we can reference it after switching root.
+        let newroot_fd = unsafe {
+            libc::open(
+                c_shared_dir.as_ptr(),
+                libc::O_DIRECTORY | libc::O_RDONLY | libc::O_CLOEXEC,
+            )
+        };
+        if newroot_fd < 0 {
+            return Err(Error::OpenNewRoot(std::io::Error::last_os_error()));
+        }
+
+        // Change to new root directory to prepare for `pivot_root` syscall.
+        let ret = unsafe { libc::fchdir(newroot_fd) };
+        if ret < 0 {
+            return Err(Error::ChdirNewRoot(std::io::Error::last_os_error()));
+        }
+
+        // Call to `pivot_root` using `.` as both new and old root.
+        let c_current_dir = CString::new(".").unwrap();
+        let ret = unsafe {
+            libc::syscall(
+                libc::SYS_pivot_root,
+                c_current_dir.as_ptr(),
+                c_current_dir.as_ptr(),
+            )
+        };
+        if ret < 0 {
+            return Err(Error::PivotRoot(std::io::Error::last_os_error()));
+        }
+
+        // Change to old root directory to prepare for cleaning up and unmounting it.
+        let ret = unsafe { libc::fchdir(oldroot_fd) };
+        if ret < 0 {
+            return Err(Error::ChdirOldRoot(std::io::Error::last_os_error()));
+        }
+
+        // Clean up old root to avoid mount namespace propagation.
+        let c_empty = CString::new("").unwrap();
+        let ret = unsafe {
+            libc::mount(
+                c_empty.as_ptr(),
+                c_current_dir.as_ptr(),
+                c_empty.as_ptr(),
+                libc::MS_SLAVE | libc::MS_REC,
+                std::ptr::null(),
+            )
+        };
+        if ret != 0 {
+            return Err(Error::CleanMount(std::io::Error::last_os_error()));
+        }
+
+        // Lazily unmount old root.
+        let ret = unsafe { libc::umount2(c_current_dir.as_ptr(), libc::MNT_DETACH) };
+        if ret < 0 {
+            return Err(Error::UmountOldRoot(std::io::Error::last_os_error()));
+        }
+
+        // Change to new root.
+        let ret = unsafe { libc::fchdir(newroot_fd) };
+        if ret < 0 {
+            return Err(Error::ChdirNewRoot(std::io::Error::last_os_error()));
+        }
+
+        // We no longer need these file descriptors, so close them.
+        unsafe { libc::close(newroot_fd) };
+        unsafe { libc::close(oldroot_fd) };
+
+        Ok(())
+    }
+
+    /// Set up sandbox, fork and jump into it.
+    ///
+    /// On success, the returned value will be the PID of the child for the parent and `None` for
+    /// the child itself, with the latter running isolated in `self.shared_dir`.
+    pub fn enter(&mut self) -> Result<Option<i32>, Error> {
+        let uid = unsafe { libc::geteuid() };
+
+        let flags = if uid == 0 {
+            libc::CLONE_NEWPID | libc::CLONE_NEWNS | libc::CLONE_NEWNET
+        } else {
+            // If running as an unprivileged user, rely on user_namespaces(7) for isolation. The
+            // main limitation of this strategy is that only the current uid/gid are mapped into
+            // the new namespace, so most operations on permissions will fail.
+            libc::CLONE_NEWPID | libc::CLONE_NEWNS | libc::CLONE_NEWNET | libc::CLONE_NEWUSER
+        };
+
+        let ret = unsafe { libc::unshare(flags) };
+        if ret != 0 {
+            return Err(Error::Unshare(std::io::Error::last_os_error()));
+        }
+
+        let child = unsafe { libc::fork() };
+        match child {
+            0 => {
+                // This is the child. Request to receive SIGTERM on parent's death.
+                unsafe { libc::prctl(libc::PR_SET_PDEATHSIG, libc::SIGTERM) };
+                self.setup_mounts()?;
+                Ok(None)
+            }
+            x if x > 0 => {
+                // This is the parent.
+                Ok(Some(child))
+            }
+            _ => Err(Error::Fork(std::io::Error::last_os_error())),
+        }
+    }
+
+    pub fn get_proc_self_fd(&self) -> Option<RawFd> {
+        self.proc_self_fd
+    }
+}

vhost_user_fs/src/seccomp.rs

@@ -0,0 +1,141 @@
+// Copyright 2020 Red Hat, Inc. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+use seccomp::{allow_syscall, BpfProgram, SeccompAction, SeccompFilter};
+use std::convert::TryInto;
+use std::{convert, fmt};
+
+#[derive(Debug)]
+pub enum Error {
+    /// Cannot create seccomp filter
+    CreateSeccompFilter(seccomp::SeccompError),
+
+    /// Cannot apply seccomp filter
+    ApplySeccompFilter(seccomp::Error),
+}
+
+impl convert::From<seccomp::Error> for Error {
+    fn from(e: seccomp::Error) -> Self {
+        Error::ApplySeccompFilter(e)
+    }
+}
+
+impl fmt::Display for Error {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "vhost_user_fs_seccomp_error: {:?}", self)
+    }
+}
+
+fn vuf_filter(action: SeccompAction) -> Result<SeccompFilter, Error> {
+    Ok(SeccompFilter::new(
+        vec![
+            allow_syscall(libc::SYS_accept4),
+            allow_syscall(libc::SYS_brk),
+            allow_syscall(libc::SYS_capget), // For CAP_FSETID
+            allow_syscall(libc::SYS_capset),
+            allow_syscall(libc::SYS_clock_gettime),
+            allow_syscall(libc::SYS_clone),
+            allow_syscall(libc::SYS_close),
+            allow_syscall(libc::SYS_copy_file_range),
+            allow_syscall(libc::SYS_dup),
+            #[cfg(target_arch = "x86_64")]
+            allow_syscall(libc::SYS_epoll_create),
+            allow_syscall(libc::SYS_epoll_create1),
+            allow_syscall(libc::SYS_epoll_ctl),
+            allow_syscall(libc::SYS_epoll_pwait),
+            #[cfg(target_arch = "x86_64")]
+            allow_syscall(libc::SYS_epoll_wait),
+            allow_syscall(libc::SYS_eventfd2),
+            allow_syscall(libc::SYS_exit),
+            allow_syscall(libc::SYS_exit_group),
+            allow_syscall(libc::SYS_fallocate),
+            allow_syscall(libc::SYS_fchdir),
+            allow_syscall(libc::SYS_fchmodat),
+            allow_syscall(libc::SYS_fchownat),
+            allow_syscall(libc::SYS_fcntl),
+            allow_syscall(libc::SYS_fdatasync),
+            allow_syscall(libc::SYS_fgetxattr),
+            allow_syscall(libc::SYS_flistxattr),
+            allow_syscall(libc::SYS_flock),
+            allow_syscall(libc::SYS_fremovexattr),
+            allow_syscall(libc::SYS_fsetxattr),
+            allow_syscall(libc::SYS_fstat),
+            #[cfg(target_arch = "x86_64")]
+            allow_syscall(libc::SYS_fstatfs),
+            allow_syscall(libc::SYS_fsync),
+            #[cfg(target_arch = "x86_64")]
+            allow_syscall(libc::SYS_ftruncate),
+            allow_syscall(libc::SYS_futex),
+            #[cfg(target_arch = "x86_64")]
+            allow_syscall(libc::SYS_getdents),
+            allow_syscall(libc::SYS_getdents64),
+            allow_syscall(libc::SYS_getegid),
+            allow_syscall(libc::SYS_geteuid),
+            allow_syscall(libc::SYS_getpid),
+            allow_syscall(libc::SYS_gettid),
+            allow_syscall(libc::SYS_gettimeofday),
+            allow_syscall(libc::SYS_getxattr),
+            allow_syscall(libc::SYS_linkat),
+            allow_syscall(libc::SYS_listxattr),
+            allow_syscall(libc::SYS_lseek),
+            allow_syscall(libc::SYS_madvise),
+            allow_syscall(libc::SYS_mkdirat),
+            allow_syscall(libc::SYS_mknodat),
+            allow_syscall(libc::SYS_mmap),
+            allow_syscall(libc::SYS_mprotect),
+            allow_syscall(libc::SYS_mremap),
+            allow_syscall(libc::SYS_munmap),
+            allow_syscall(libc::SYS_newfstatat),
+            #[cfg(target_arch = "x86_64")]
+            allow_syscall(libc::SYS_open),
+            allow_syscall(libc::SYS_openat),
+            allow_syscall(libc::SYS_prctl), // TODO restrict to just PR_SET_NAME?
+            allow_syscall(libc::SYS_preadv),
+            allow_syscall(libc::SYS_pread64),
+            allow_syscall(libc::SYS_pwritev),
+            allow_syscall(libc::SYS_pwrite64),
+            allow_syscall(libc::SYS_read),
+            allow_syscall(libc::SYS_readlinkat),
+            allow_syscall(libc::SYS_recvmsg),
+            allow_syscall(libc::SYS_renameat),
+            allow_syscall(libc::SYS_renameat2),
+            allow_syscall(libc::SYS_removexattr),
+            allow_syscall(libc::SYS_rt_sigaction),
+            allow_syscall(libc::SYS_rt_sigprocmask),
+            allow_syscall(libc::SYS_rt_sigreturn),
+            allow_syscall(libc::SYS_sched_getaffinity), // used by thread_pool
+            allow_syscall(libc::SYS_sendmsg),
+            allow_syscall(libc::SYS_setresgid),
+            allow_syscall(libc::SYS_setresuid),
+            //allow_syscall(libc::SYS_setresgid32),  Needed on some platforms,
+            //allow_syscall(libc::SYS_setresuid32),  Needed on some platforms
+            allow_syscall(libc::SYS_set_robust_list),
+            allow_syscall(libc::SYS_setxattr),
+            allow_syscall(libc::SYS_sigaltstack),
+            allow_syscall(libc::SYS_statx),
+            allow_syscall(libc::SYS_symlinkat),
+            #[cfg(target_arch = "x86_64")]
+            allow_syscall(libc::SYS_time), // Rarely needed, except on static builds
+            allow_syscall(libc::SYS_tgkill),
+            allow_syscall(libc::SYS_umask),
+            #[cfg(target_arch = "x86_64")]
+            allow_syscall(libc::SYS_unlink),
+            allow_syscall(libc::SYS_unlinkat),
+            allow_syscall(libc::SYS_unshare),
+            allow_syscall(libc::SYS_utimensat),
+            allow_syscall(libc::SYS_write),
+            allow_syscall(libc::SYS_writev),
+        ]
+        .into_iter()
+        .collect(),
+        action,
+    )?)
+}
+
+pub fn enable_seccomp(action: SeccompAction) -> Result<(), Error> {
+    let scfilter = vuf_filter(action)?;
+    let bpfprog: BpfProgram = scfilter.try_into()?;
+    SeccompFilter::apply(bpfprog.try_into().unwrap()).unwrap();
+    Ok(())
+}

vhost_user_fs/src/server.rs

@@ -2,14 +2,6 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
-use std::ffi::CStr;
-use std::fs::File;
-use std::io::{self, Read, Write};
-use std::mem::size_of;
-
-use libc;
-use vm_memory::ByteValued;
-
 use super::fs_cache_req_handler::FsCacheReqHandler;
 use crate::descriptor_utils::{Reader, Writer};
 use crate::filesystem::{
@@ -18,8 +10,14 @@ use crate::filesystem::{
 };
 use crate::fuse::*;
 use crate::{Error, Result};
+use std::convert::TryInto;
+use std::ffi::CStr;
+use std::fs::File;
+use std::io::{self, Read, Write};
+use std::mem::size_of;
+use vm_memory::ByteValued;
 
-const MAX_BUFFER_SIZE: u32 = (1 << 20);
+const MAX_BUFFER_SIZE: u32 = 1 << 20;
 const DIRENT_PADDING: [u8; 8] = [0; 8];
 
 struct ZCReader<'a>(Reader<'a>);
@@ -124,6 +122,7 @@ impl<F: FileSystem + Sync> Server<F> {
             x if x == Opcode::Readdirplus as u32 => self.readdirplus(in_header, r, w),
             x if x == Opcode::Rename2 as u32 => self.rename2(in_header, r, w),
             x if x == Opcode::Lseek as u32 => self.lseek(in_header, r, w),
+            x if x == Opcode::CopyFileRange as u32 => self.copyfilerange(in_header, r, w),
             x if x == Opcode::SetupMapping as u32 => self.setupmapping(in_header, r, w, vu_req),
             x if x == Opcode::RemoveMapping as u32 => self.removemapping(in_header, r, w, vu_req),
             _ => reply_error(
@@ -634,6 +633,7 @@ impl<F: FileSystem + Sync> Server<F> {
         };
 
         let delayed_write = write_flags & WRITE_CACHE != 0;
+        let kill_priv = write_flags & WRITE_KILL_PRIV != 0;
 
         let data_reader = ZCReader(r);
 
@@ -646,6 +646,7 @@ impl<F: FileSystem + Sync> Server<F> {
             offset,
             owner,
             delayed_write,
+            kill_priv,
             flags,
         ) {
             Ok(count) => {
@@ -902,10 +903,14 @@ impl<F: FileSystem + Sync> Server<F> {
             | FsOptions::HANDLE_KILLPRIV
             | FsOptions::ASYNC_DIO
             | FsOptions::HAS_IOCTL_DIR
-            | FsOptions::ATOMIC_O_TRUNC;
+            | FsOptions::ATOMIC_O_TRUNC
+            | FsOptions::MAX_PAGES;
 
         let capable = FsOptions::from_bits_truncate(flags);
 
+        let page_size: u32 = unsafe { libc::sysconf(libc::_SC_PAGESIZE).try_into().unwrap() };
+        let max_pages = ((MAX_BUFFER_SIZE - 1) / page_size) + 1;
+
         match self.fs.init(capable) {
             Ok(want) => {
                 let enabled = capable & (want | supported);
@@ -919,6 +924,7 @@ impl<F: FileSystem + Sync> Server<F> {
                     congestion_threshold: (::std::u16::MAX / 4) * 3,
                     max_write: MAX_BUFFER_SIZE,
                     time_gran: 1, // nanoseconds
+                    max_pages: max_pages.try_into().unwrap(),
                     ..Default::default()
                 };
 
@@ -1261,6 +1267,41 @@ impl<F: FileSystem + Sync> Server<F> {
             Err(e) => reply_error(e, in_header.unique, w),
         }
     }
+
+    fn copyfilerange(&self, in_header: InHeader, mut r: Reader, w: Writer) -> Result<usize> {
+        let CopyfilerangeIn {
+            fh_in,
+            off_in,
+            nodeid_out,
+            fh_out,
+            off_out,
+            len,
+            flags,
+            ..
+        } = r.read_obj().map_err(Error::DecodeMessage)?;
+
+        match self.fs.copyfilerange(
+            Context::from(in_header),
+            in_header.nodeid.into(),
+            fh_in.into(),
+            off_in,
+            nodeid_out.into(),
+            fh_out.into(),
+            off_out,
+            len,
+            flags,
+        ) {
+            Ok(count) => {
+                let out = WriteOut {
+                    size: count as u32,
+                    ..Default::default()
+                };
+
+                reply_ok(Some(out), None, in_header.unique, w)
+            }
+            Err(e) => reply_error(e, in_header.unique, w),
+        }
+    }
 }
 
 fn reply_ok<T: ByteValued>(

vhost_user_net/Cargo.toml

@@ -7,12 +7,13 @@ edition = "2018"
 [dependencies]
 bitflags = "1.1.0"
 epoll = ">=4.0.1"
-libc = "0.2.68"
+libc = "0.2.71"
 log = "0.4.8"
 net_util = { path = "../net_util" }
 vhost_user_backend = { path = "../vhost_user_backend" }
 vhost_rs = { git = "https://github.com/cloud-hypervisor/vhost", branch = "dragonball", package = "vhost", features = ["vhost-user-slave"] }
 virtio-bindings = "0.1.0"
-vm-memory = "0.1.0"
+vm-memory = "0.2.1"
 vm-virtio = { path = "../vm-virtio" }
+vmm = { path = "../vmm" }
 vmm-sys-util = ">=0.3.1"

vhost_user_net/src/lib.rs

@@ -11,27 +11,27 @@ extern crate net_util;
 extern crate vhost_rs;
 extern crate vhost_user_backend;
 extern crate vm_virtio;
+extern crate vmm;
 
-use epoll;
-use libc::{self, EAGAIN, EFD_NONBLOCK};
+use libc::{self, EFD_NONBLOCK};
 use log::*;
-use net_util::Tap;
-use std::convert::TryFrom;
+use net_util::{MacAddr, Tap};
 use std::fmt;
-use std::io::Read;
 use std::io::{self};
 use std::net::Ipv4Addr;
 use std::os::unix::io::AsRawFd;
 use std::process;
-use std::sync::{Arc, RwLock};
+use std::sync::{Arc, Mutex, RwLock};
 use std::vec::Vec;
 use vhost_rs::vhost_user::message::*;
-use vhost_rs::vhost_user::Error as VhostUserError;
+use vhost_rs::vhost_user::{Error as VhostUserError, Listener};
 use vhost_user_backend::{VhostUserBackend, VhostUserDaemon, Vring, VringWorker};
 use virtio_bindings::bindings::virtio_net::*;
-use vm_memory::GuestMemoryMmap;
+use virtio_bindings::bindings::virtio_ring::VIRTIO_RING_F_EVENT_IDX;
+use vm_memory::{GuestMemoryAtomic, GuestMemoryMmap};
 use vm_virtio::net_util::{open_tap, RxVirtio, TxVirtio};
-use vm_virtio::Queue;
+use vm_virtio::NetQueuePair;
+use vmm::config::{OptionParser, OptionParserError};
 use vmm_sys_util::eventfd::EventFd;
 
 pub type VhostUserResult<T> = std::result::Result<T, VhostUserError>;
@@ -52,8 +52,10 @@ pub enum Error {
     EpollCreateFd,
     /// Failed to read Tap.
     FailedReadTap,
+    /// Failed to parse configuration string
+    FailedConfigParse(OptionParserError),
     /// Failed to signal used queue.
-    FailedSignalingUsedQueue,
+    FailedSignalingUsedQueue(io::Error),
     /// Failed to handle event other than input event.
     HandleEventNotEpollIn,
     /// Failed to handle unknown event.
@@ -64,20 +66,18 @@ pub enum Error {
     NoVringCallFdNotify,
     /// No memory configured.
     NoMemoryConfigured,
-    /// Failed to parse sock parameter.
-    ParseSockParam,
-    /// Failed to parse ip parameter.
-    ParseIpParam(std::net::AddrParseError),
-    /// Failed to parse mask parameter.
-    ParseMaskParam(std::net::AddrParseError),
-    /// Failed to parse queue number.
-    ParseQueueNumParam(std::num::ParseIntError),
-    /// Failed to parse queue size.
-    ParseQueueSizeParam(std::num::ParseIntError),
     /// Open tap device failed.
     OpenTap(vm_virtio::net_util::Error),
+    /// No socket provided
+    SocketParameterMissing,
+    /// Underlying QueuePair error
+    NetQueuePair(vm_virtio::Error),
 }
 
+pub const SYNTAX: &str = "vhost-user-net backend parameters \
+\"ip=<ip_addr>,mask=<net_mask>,socket=<socket_path>,\
+num_queues=<number_of_queues>,queue_size=<size_of_each_queue>,tap=<if_name>\"";
+
 impl fmt::Display for Error {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         write!(f, "vhost_user_net_error: {:?}", self)
@@ -92,163 +92,74 @@ impl std::convert::From<Error> for std::io::Error {
     }
 }
 
-pub struct VhostUserNetBackend {
-    mem: Option<GuestMemoryMmap>,
+struct VhostUserNetThread {
+    net: NetQueuePair,
     vring_worker: Option<Arc<VringWorker>>,
     kill_evt: EventFd,
-    taps: Vec<(Tap, usize)>,
-    rxs: Vec<RxVirtio>,
-    txs: Vec<TxVirtio>,
-    rx_tap_listenings: Vec<bool>,
-    num_queues: usize,
-    queue_size: u16,
 }
 
-impl VhostUserNetBackend {
+impl VhostUserNetThread {
     /// Create a new virtio network device with the given TAP interface.
-    pub fn new_with_tap(taps: Vec<Tap>, num_queues: usize, queue_size: u16) -> Result<Self> {
-        let mut taps_v: Vec<(Tap, usize)> = Vec::new();
-        for (i, tap) in taps.iter().enumerate() {
-            taps_v.push((tap.clone(), num_queues + i));
-        }
-
-        let mut rxs: Vec<RxVirtio> = Vec::new();
-        let mut txs: Vec<TxVirtio> = Vec::new();
-        let mut rx_tap_listenings: Vec<bool> = Vec::new();
-
-        for _ in 0..taps.len() {
-            let rx = RxVirtio::new();
-            rxs.push(rx);
-            let tx = TxVirtio::new();
-            txs.push(tx);
-            rx_tap_listenings.push(false);
-        }
-
-        Ok(VhostUserNetBackend {
-            mem: None,
+    fn new(tap: Tap) -> Result<Self> {
+        Ok(VhostUserNetThread {
             vring_worker: None,
             kill_evt: EventFd::new(EFD_NONBLOCK).map_err(Error::CreateKillEventFd)?,
-            taps: taps_v,
-            rxs,
-            txs,
-            rx_tap_listenings,
-            num_queues,
-            queue_size,
+            net: NetQueuePair {
+                mem: None,
+                tap,
+                rx: RxVirtio::new(),
+                tx: TxVirtio::new(),
+                rx_tap_listening: false,
+                epoll_fd: None,
+            },
         })
     }
 
-    /// Create a new virtio network device with the given IP address and
-    /// netmask.
-    pub fn new(
+    pub fn set_vring_worker(&mut self, vring_worker: Option<Arc<VringWorker>>) {
+        self.net.epoll_fd = Some(vring_worker.as_ref().unwrap().as_raw_fd());
+        self.vring_worker = vring_worker;
+    }
+}
+
+pub struct VhostUserNetBackend {
+    threads: Vec<Mutex<VhostUserNetThread>>,
+    num_queues: usize,
+    queue_size: u16,
+    queues_per_thread: Vec<u64>,
+}
+
+impl VhostUserNetBackend {
+    fn new(
         ip_addr: Ipv4Addr,
+        host_mac: MacAddr,
         netmask: Ipv4Addr,
         num_queues: usize,
         queue_size: u16,
         ifname: Option<&str>,
     ) -> Result<Self> {
-        let taps = open_tap(ifname, Some(ip_addr), Some(netmask), num_queues / 2)
-            .map_err(Error::OpenTap)?;
+        let mut taps = open_tap(
+            ifname,
+            Some(ip_addr),
+            Some(netmask),
+            &mut Some(host_mac),
+            num_queues / 2,
+        )
+        .map_err(Error::OpenTap)?;
 
-        Self::new_with_tap(taps, num_queues, queue_size)
-    }
-
-    // Copies a single frame from `self.rx.frame_buf` into the guest. Returns true
-    // if a buffer was used, and false if the frame must be deferred until a buffer
-    // is made available by the driver.
-    fn rx_single_frame(&mut self, mut queue: &mut Queue, index: usize) -> Result<bool> {
-        let mem = self.mem.as_ref().ok_or(Error::NoMemoryConfigured)?;
-
-        let next_desc = queue.iter(&mem).next();
-
-        if next_desc.is_none() {
-            // Queue has no available descriptors
-            if self.rx_tap_listenings[index] {
-                self.vring_worker
-                    .as_ref()
-                    .unwrap()
-                    .unregister_listener(
-                        self.taps[index].0.as_raw_fd(),
-                        epoll::Events::EPOLLIN,
-                        u64::try_from(self.taps[index].1).unwrap(),
-                    )
-                    .unwrap();
-                self.rx_tap_listenings[index] = false;
-            }
-            return Ok(false);
+        let mut queues_per_thread = Vec::new();
+        let mut threads = Vec::new();
+        for (i, tap) in taps.drain(..).enumerate() {
+            let thread = Mutex::new(VhostUserNetThread::new(tap)?);
+            threads.push(thread);
+            queues_per_thread.push(0b11 << (i * 2));
         }
 
-        let write_complete = self.rxs[index].process_desc_chain(&mem, next_desc, &mut queue);
-
-        Ok(write_complete)
-    }
-
-    fn process_rx(&mut self, vring: &mut Vring, index: usize) -> Result<()> {
-        // Read as many frames as possible.
-        loop {
-            match self.read_tap(index) {
-                Ok(count) => {
-                    self.rxs[index].bytes_read = count;
-                    if !self.rx_single_frame(&mut vring.mut_queue(), index)? {
-                        self.rxs[index].deferred_frame = true;
-                        break;
-                    }
-                }
-                Err(e) => {
-                    // The tap device is non-blocking, so any error aside from EAGAIN is
-                    // unexpected.
-                    match e.raw_os_error() {
-                        Some(err) if err == EAGAIN => (),
-                        _ => {
-                            error!("Failed to read tap: {:?}", e);
-                            return Err(Error::FailedReadTap);
-                        }
-                    };
-                    break;
-                }
-            }
-        }
-        if self.rxs[index].deferred_irqs {
-            self.rxs[index].deferred_irqs = false;
-            vring.signal_used_queue().unwrap();
-            Ok(())
-        } else {
-            Ok(())
-        }
-    }
-
-    fn resume_rx(&mut self, vring: &mut Vring, index: usize) -> Result<()> {
-        if self.rxs[index].deferred_frame {
-            if self.rx_single_frame(&mut vring.mut_queue(), index)? {
-                self.rxs[index].deferred_frame = false;
-                // process_rx() was interrupted possibly before consuming all
-                // packets in the tap; try continuing now.
-                self.process_rx(vring, index)
-            } else if self.rxs[index].deferred_irqs {
-                self.rxs[index].deferred_irqs = false;
-                vring.signal_used_queue().unwrap();
-                Ok(())
-            } else {
-                Ok(())
-            }
-        } else {
-            Ok(())
-        }
-    }
-
-    fn process_tx(&mut self, mut queue: &mut Queue, index: usize) -> Result<()> {
-        let mem = self.mem.as_ref().ok_or(Error::NoMemoryConfigured)?;
-
-        self.txs[index].process_desc_chain(&mem, &mut self.taps[index].0, &mut queue);
-
-        Ok(())
-    }
-
-    fn read_tap(&mut self, index: usize) -> io::Result<usize> {
-        self.taps[index].0.read(&mut self.rxs[index].frame_buf)
-    }
-
-    pub fn set_vring_worker(&mut self, vring_worker: Option<Arc<VringWorker>>) {
-        self.vring_worker = vring_worker;
+        Ok(VhostUserNetBackend {
+            threads,
+            num_queues,
+            queue_size,
+            queues_per_thread,
+        })
     }
 }
 
@@ -269,6 +180,7 @@ impl VhostUserBackend for VhostUserNetBackend {
             | 1 << VIRTIO_NET_F_HOST_TSO4
             | 1 << VIRTIO_NET_F_HOST_UFO
             | 1 << VIRTIO_F_VERSION_1
+            | 1 << VIRTIO_RING_F_EVENT_IDX
             | VhostUserVirtioFeatures::PROTOCOL_FEATURES.bits()
     }
 
@@ -279,59 +191,59 @@ impl VhostUserBackend for VhostUserNetBackend {
     fn set_event_idx(&mut self, _enabled: bool) {}
 
     fn update_memory(&mut self, mem: GuestMemoryMmap) -> VhostUserBackendResult<()> {
-        self.mem = Some(mem);
+        for thread in self.threads.iter() {
+            thread.lock().unwrap().net.mem = Some(GuestMemoryAtomic::new(mem.clone()));
+        }
         Ok(())
     }
 
     fn handle_event(
-        &mut self,
+        &self,
         device_event: u16,
         evset: epoll::Events,
         vrings: &[Arc<RwLock<Vring>>],
+        thread_id: usize,
     ) -> VhostUserBackendResult<bool> {
         if evset != epoll::Events::EPOLLIN {
             return Err(Error::HandleEventNotEpollIn.into());
         }
 
-        let tap_start_index = self.num_queues as u16;
-        let tap_end_index = (self.num_queues + self.num_queues / 2 - 1) as u16;
-
+        let mut thread = self.threads[thread_id].lock().unwrap();
         match device_event {
-            x if ((x < self.num_queues as u16) && (x % 2 == 0)) => {
-                let index = (x / 2) as usize;
-                let mut vring = vrings[x as usize].write().unwrap();
-                self.resume_rx(&mut vring, index)?;
-
-                if !self.rx_tap_listenings[index] {
-                    self.vring_worker.as_ref().unwrap().register_listener(
-                        self.taps[index].0.as_raw_fd(),
-                        epoll::Events::EPOLLIN,
-                        u64::try_from(self.taps[index].1).unwrap(),
-                    )?;
-                    self.rx_tap_listenings[index] = true;
+            0 => {
+                let mut vring = vrings[0].write().unwrap();
+                if thread
+                    .net
+                    .resume_rx(&mut vring.mut_queue())
+                    .map_err(Error::NetQueuePair)?
+                {
+                    vring
+                        .signal_used_queue()
+                        .map_err(Error::FailedSignalingUsedQueue)?
                 }
             }
-            x if ((x < self.num_queues as u16) && (x % 2 != 0)) => {
-                let index = ((x - 1) / 2) as usize;
-                let mut vring = vrings[x as usize].write().unwrap();
-                self.process_tx(&mut vring.mut_queue(), index)?;
-            }
-            x if x >= tap_start_index && x <= tap_end_index => {
-                let index = x as usize - self.num_queues;
-                let mut vring = vrings[2 * index].write().unwrap();
-                if self.rxs[index].deferred_frame
-                // Process a deferred frame first if available. Don't read from tap again
-                // until we manage to receive this deferred frame.
+            1 => {
+                let mut vring = vrings[1].write().unwrap();
+                if thread
+                    .net
+                    .process_tx(&mut vring.mut_queue())
+                    .map_err(Error::NetQueuePair)?
                 {
-                    if self.rx_single_frame(&mut vring.mut_queue(), index)? {
-                        self.rxs[index].deferred_frame = false;
-                        self.process_rx(&mut vring, index)?;
-                    } else if self.rxs[index].deferred_irqs {
-                        self.rxs[index].deferred_irqs = false;
-                        vring.signal_used_queue()?;
-                    }
-                } else {
-                    self.process_rx(&mut vring, index)?;
+                    vring
+                        .signal_used_queue()
+                        .map_err(Error::FailedSignalingUsedQueue)?
+                }
+            }
+            2 => {
+                let mut vring = vrings[0].write().unwrap();
+                if thread
+                    .net
+                    .process_rx_tap(&mut vring.mut_queue())
+                    .map_err(Error::NetQueuePair)?
+                {
+                    vring
+                        .signal_used_queue()
+                        .map_err(Error::FailedSignalingUsedQueue)?
                 }
             }
             _ => return Err(Error::HandleEventUnknownEvent.into()),
@@ -340,78 +252,78 @@ impl VhostUserBackend for VhostUserNetBackend {
         Ok(false)
     }
 
-    fn exit_event(&self) -> Option<(EventFd, Option<u16>)> {
-        let tap_end_index = (self.num_queues + self.num_queues / 2 - 1) as u16;
-        let kill_index = tap_end_index + 1;
-        Some((self.kill_evt.try_clone().unwrap(), Some(kill_index)))
+    fn exit_event(&self, thread_index: usize) -> Option<(EventFd, Option<u16>)> {
+        // The exit event is placed after the queues and the tap event, which
+        // is event index 3.
+        Some((
+            self.threads[thread_index]
+                .lock()
+                .unwrap()
+                .kill_evt
+                .try_clone()
+                .unwrap(),
+            Some(3),
+        ))
+    }
+
+    fn queues_per_thread(&self) -> Vec<u64> {
+        self.queues_per_thread.clone()
     }
 }
 
-pub struct VhostUserNetBackendConfig<'a> {
+pub struct VhostUserNetBackendConfig {
     pub ip: Ipv4Addr,
+    pub host_mac: MacAddr,
     pub mask: Ipv4Addr,
-    pub sock: &'a str,
+    pub socket: String,
     pub num_queues: usize,
     pub queue_size: u16,
-    pub tap: Option<&'a str>,
+    pub tap: Option<String>,
 }
 
-impl<'a> VhostUserNetBackendConfig<'a> {
-    pub fn parse(backend: &'a str) -> Result<Self> {
-        let params_list: Vec<&str> = backend.split(',').collect();
+impl VhostUserNetBackendConfig {
+    pub fn parse(backend: &str) -> Result<Self> {
+        let mut parser = OptionParser::new();
 
-        let mut ip_str: &str = "";
-        let mut mask_str: &str = "";
-        let mut sock: &str = "";
-        let mut num_queues_str: &str = "";
-        let mut queue_size_str: &str = "";
-        let mut tap_str: &str = "";
+        parser
+            .add("tap")
+            .add("ip")
+            .add("host_mac")
+            .add("mask")
+            .add("queue_size")
+            .add("num_queues")
+            .add("socket");
 
-        for param in params_list.iter() {
-            if param.starts_with("ip=") {
-                ip_str = &param[3..];
-            } else if param.starts_with("mask=") {
-                mask_str = &param[5..];
-            } else if param.starts_with("sock=") {
-                sock = &param[5..];
-            } else if param.starts_with("num_queues=") {
-                num_queues_str = &param[11..];
-            } else if param.starts_with("queue_size=") {
-                queue_size_str = &param[11..];
-            } else if param.starts_with("tap=") {
-                tap_str = &param[4..];
-            }
-        }
+        parser.parse(backend).map_err(Error::FailedConfigParse)?;
 
-        let mut ip: Ipv4Addr = Ipv4Addr::new(192, 168, 100, 1);
-        let mut mask: Ipv4Addr = Ipv4Addr::new(255, 255, 255, 0);
-        let mut num_queues: usize = 2;
-        let mut queue_size: u16 = 256;
-        let mut tap: Option<&str> = None;
-
-        if sock.is_empty() {
-            return Err(Error::ParseSockParam);
-        }
-        if !ip_str.is_empty() {
-            ip = ip_str.parse().map_err(Error::ParseIpParam)?;
-        }
-        if !mask_str.is_empty() {
-            mask = mask_str.parse().map_err(Error::ParseMaskParam)?;
-        }
-        if !num_queues_str.is_empty() {
-            num_queues = num_queues_str.parse().map_err(Error::ParseQueueNumParam)?;
-        }
-        if !queue_size_str.is_empty() {
-            queue_size = queue_size_str.parse().map_err(Error::ParseQueueSizeParam)?;
-        }
-        if !tap_str.is_empty() {
-            tap = Some(tap_str);
-        }
+        let tap = parser.get("tap");
+        let ip = parser
+            .convert("ip")
+            .map_err(Error::FailedConfigParse)?
+            .unwrap_or_else(|| Ipv4Addr::new(192, 168, 100, 1));
+        let host_mac = parser
+            .convert("host_mac")
+            .map_err(Error::FailedConfigParse)?
+            .unwrap_or_else(MacAddr::local_random);
+        let mask = parser
+            .convert("mask")
+            .map_err(Error::FailedConfigParse)?
+            .unwrap_or_else(|| Ipv4Addr::new(255, 255, 255, 0));
+        let queue_size = parser
+            .convert("queue_size")
+            .map_err(Error::FailedConfigParse)?
+            .unwrap_or(256);
+        let num_queues = parser
+            .convert("num_queues")
+            .map_err(Error::FailedConfigParse)?
+            .unwrap_or(2);
+        let socket = parser.get("socket").ok_or(Error::SocketParameterMissing)?;
 
         Ok(VhostUserNetBackendConfig {
             ip,
+            host_mac,
             mask,
-            sock,
+            socket,
             num_queues,
             queue_size,
             tap,
@@ -423,38 +335,50 @@ pub fn start_net_backend(backend_command: &str) {
     let backend_config = match VhostUserNetBackendConfig::parse(backend_command) {
         Ok(config) => config,
         Err(e) => {
-            println!("Failed parsing parameters {:?}", e);
+            eprintln!("Failed parsing parameters {:?}", e);
             process::exit(1);
         }
     };
 
+    let tap = if let Some(tap) = backend_config.tap.as_ref() {
+        Some(tap.as_str())
+    } else {
+        None
+    };
+
     let net_backend = Arc::new(RwLock::new(
         VhostUserNetBackend::new(
             backend_config.ip,
+            backend_config.host_mac,
             backend_config.mask,
             backend_config.num_queues,
             backend_config.queue_size,
-            backend_config.tap,
+            tap,
         )
         .unwrap(),
     ));
 
-    let mut net_daemon = VhostUserDaemon::new(
-        "vhost-user-net-backend".to_string(),
-        backend_config.sock.to_string(),
-        net_backend.clone(),
-    )
-    .unwrap();
+    let listener = Listener::new(&backend_config.socket, true).unwrap();
 
-    let vring_worker = net_daemon.get_vring_worker();
+    let mut net_daemon =
+        VhostUserDaemon::new("vhost-user-net-backend".to_string(), net_backend.clone()).unwrap();
 
-    net_backend
-        .write()
-        .unwrap()
-        .set_vring_worker(Some(vring_worker));
+    let mut vring_workers = net_daemon.get_vring_workers();
 
-    if let Err(e) = net_daemon.start() {
-        println!(
+    if vring_workers.len() != net_backend.read().unwrap().threads.len() {
+        error!("Number of vring workers must be identical to the number of backend threads");
+        process::exit(1);
+    }
+
+    for thread in net_backend.read().unwrap().threads.iter() {
+        thread
+            .lock()
+            .unwrap()
+            .set_vring_worker(Some(vring_workers.remove(0)));
+    }
+
+    if let Err(e) = net_daemon.start(listener) {
+        error!(
             "failed to start daemon for vhost-user-net with error: {:?}",
             e
         );
@@ -465,8 +389,9 @@ pub fn start_net_backend(backend_command: &str) {
         error!("Error from the main thread: {:?}", e);
     }
 
-    let kill_evt = &net_backend.write().unwrap().kill_evt;
-    if let Err(e) = kill_evt.write(1) {
-        error!("Error shutting down worker thread: {:?}", e)
+    for thread in net_backend.read().unwrap().threads.iter() {
+        if let Err(e) = thread.lock().unwrap().kill_evt.write(1) {
+            error!("Error shutting down worker thread: {:?}", e)
+        }
     }
 }

vm-allocator/Cargo.toml

@@ -5,5 +5,6 @@ authors = ["The Chromium OS Authors"]
 edition = "2018"
 
 [dependencies]
-libc = "0.2.68"
-vm-memory = "0.1.0"
+libc = "0.2.71"
+vm-memory = "0.2.1"
+arch = { path = "../arch" }

vm-allocator/src/gsi.rs

@@ -2,6 +2,9 @@
 //
 // SPDX-License-Identifier: Apache-2.0 OR BSD-3-Clause
 
+#[cfg(target_arch = "aarch64")]
+use arch;
+#[cfg(target_arch = "x86_64")]
 use std::collections::btree_map::BTreeMap;
 use std::result;
 
@@ -13,12 +16,14 @@ pub enum Error {
 pub type Result<T> = result::Result<T, Error>;
 
 /// GsiApic
+#[cfg(target_arch = "x86_64")]
 #[derive(Copy, Clone)]
 pub struct GsiApic {
     base: u32,
     irqs: u32,
 }
 
+#[cfg(target_arch = "x86_64")]
 impl GsiApic {
     /// New GSI APIC
     pub fn new(base: u32, irqs: u32) -> Self {
@@ -28,12 +33,14 @@ impl GsiApic {
 
 /// GsiAllocator
 pub struct GsiAllocator {
+    #[cfg(target_arch = "x86_64")]
     apics: BTreeMap<u32, u32>,
     next_irq: u32,
     next_gsi: u32,
 }
 
 impl GsiAllocator {
+    #[cfg(target_arch = "x86_64")]
     /// New GSI allocator
     pub fn new(apics: Vec<GsiApic>) -> Self {
         let mut allocator = GsiAllocator {
@@ -57,13 +64,23 @@ impl GsiAllocator {
         allocator
     }
 
-    /// Allocate a GSI
-    pub fn allocate_gsi(&mut self) -> Result<u32> {
-        self.next_gsi = self.next_gsi.checked_add(1).ok_or(Error::Overflow)?;
-
-        Ok(self.next_gsi - 1)
+    #[cfg(target_arch = "aarch64")]
+    /// New GSI allocator
+    pub fn new() -> Self {
+        GsiAllocator {
+            next_irq: arch::IRQ_BASE,
+            next_gsi: arch::IRQ_BASE,
+        }
     }
 
+    /// Allocate a GSI
+    pub fn allocate_gsi(&mut self) -> Result<u32> {
+        let gsi = self.next_gsi;
+        self.next_gsi = self.next_gsi.checked_add(1).ok_or(Error::Overflow)?;
+        Ok(gsi)
+    }
+
+    #[cfg(target_arch = "x86_64")]
     /// Allocate an IRQ
     pub fn allocate_irq(&mut self) -> Result<u32> {
         let mut irq: u32 = 0;
@@ -81,4 +98,12 @@ impl GsiAllocator {
 
         Ok(irq)
     }
+
+    #[cfg(target_arch = "aarch64")]
+    /// Allocate an IRQ
+    pub fn allocate_irq(&mut self) -> Result<u32> {
+        let irq = self.next_irq;
+        self.next_irq = self.next_irq.checked_add(1).ok_or(Error::Overflow)?;
+        Ok(irq)
+    }
 }

vm-allocator/src/lib.rs

@@ -18,5 +18,7 @@ mod gsi;
 mod system;
 
 pub use crate::address::AddressAllocator;
-pub use crate::gsi::{GsiAllocator, GsiApic};
+pub use crate::gsi::GsiAllocator;
+#[cfg(target_arch = "x86_64")]
+pub use crate::gsi::GsiApic;
 pub use crate::system::SystemAllocator;

vm-allocator/src/system.rs

@@ -10,7 +10,9 @@
 use vm_memory::{GuestAddress, GuestUsize};
 
 use crate::address::AddressAllocator;
-use crate::gsi::{GsiAllocator, GsiApic};
+use crate::gsi::GsiAllocator;
+#[cfg(target_arch = "x86_64")]
+use crate::gsi::GsiApic;
 
 use libc::{sysconf, _SC_PAGESIZE};
 
@@ -39,6 +41,7 @@ fn pagesize() -> usize {
 ///
 /// ```
 pub struct SystemAllocator {
+    #[cfg(target_arch = "x86_64")]
     io_address_space: AddressAllocator,
     mmio_address_space: AddressAllocator,
     mmio_hole_address_space: AddressAllocator,
@@ -49,25 +52,32 @@ impl SystemAllocator {
     /// Creates a new `SystemAllocator` for managing addresses and irq numvers.
     /// Can return `None` if `base` + `size` overflows a u64
     ///
-    /// * `io_base` - The starting address of IO memory.
-    /// * `io_size` - The size of IO memory.
+    /// * `io_base` - (X86) The starting address of IO memory.
+    /// * `io_size` - (X86) The size of IO memory.
     /// * `mmio_base` - The starting address of MMIO memory.
     /// * `mmio_size` - The size of MMIO memory.
-    /// * `first_irq` - The first irq number to give out.
+    /// * `mmio_hole_base` - The starting address of MMIO memory in 32-bit address space.
+    /// * `mmio_hole_size` - The size of MMIO memory in 32-bit address space.
+    /// * `apics` - (X86) Vector of APIC's.
+    ///
     pub fn new(
-        io_base: GuestAddress,
-        io_size: GuestUsize,
+        #[cfg(target_arch = "x86_64")] io_base: GuestAddress,
+        #[cfg(target_arch = "x86_64")] io_size: GuestUsize,
         mmio_base: GuestAddress,
         mmio_size: GuestUsize,
         mmio_hole_base: GuestAddress,
         mmio_hole_size: GuestUsize,
-        apics: Vec<GsiApic>,
+        #[cfg(target_arch = "x86_64")] apics: Vec<GsiApic>,
     ) -> Option<Self> {
         Some(SystemAllocator {
+            #[cfg(target_arch = "x86_64")]
             io_address_space: AddressAllocator::new(io_base, io_size)?,
             mmio_address_space: AddressAllocator::new(mmio_base, mmio_size)?,
             mmio_hole_address_space: AddressAllocator::new(mmio_hole_base, mmio_hole_size)?,
+            #[cfg(target_arch = "x86_64")]
             gsi_allocator: GsiAllocator::new(apics),
+            #[cfg(target_arch = "aarch64")]
+            gsi_allocator: GsiAllocator::new(),
         })
     }
 
@@ -81,6 +91,7 @@ impl SystemAllocator {
         self.gsi_allocator.allocate_gsi().ok()
     }
 
+    #[cfg(target_arch = "x86_64")]
     /// Reserves a section of `size` bytes of IO address space.
     pub fn allocate_io_addresses(
         &mut self,
@@ -120,6 +131,7 @@ impl SystemAllocator {
         )
     }
 
+    #[cfg(target_arch = "x86_64")]
     /// Free an IO address range.
     /// We can only free a range if it matches exactly an already allocated range.
     pub fn free_io_addresses(&mut self, address: GuestAddress, size: GuestUsize) {

vm-device/Cargo.toml

@@ -10,6 +10,6 @@ thiserror = "1.0"
 serde = {version = ">=1.0.27", features = ["rc"] }
 serde_derive = ">=1.0.27"
 serde_json = ">=1.0.9"
-vm-memory = { version = "0.1.0", features = ["backend-mmap"] }
+vm-memory = { version = "0.2.1", features = ["backend-mmap"] }
 vmm-sys-util = ">=0.3.1"
 

vm-device/src/lib.rs

@@ -1,116 +1,38 @@
-extern crate serde;
-extern crate thiserror;
+#[macro_use]
+extern crate serde_derive;
 extern crate vm_memory;
 
 pub mod interrupt;
 
-use vm_memory::{
-    Address, GuestAddress, GuestMemory, GuestMemoryMmap, GuestMemoryRegion, GuestRegionMmap,
-    MemoryRegionAddress,
-};
-
-use thiserror::Error;
-
-/// Trait meant for triggering the DMA mapping update related to an external
-/// device not managed fully through virtio. It is dedicated to virtio-iommu
-/// in order to trigger the map update anytime the mapping is updated from the
-/// guest.
-pub trait ExternalDmaMapping: Send + Sync {
-    /// Map a memory range
-    fn map(&self, iova: u64, gpa: u64, size: u64) -> std::result::Result<(), std::io::Error>;
-
-    /// Unmap a memory range
-    fn unmap(&self, iova: u64, size: u64) -> std::result::Result<(), std::io::Error>;
+/// Type of Message Singaled Interrupt
+#[derive(Copy, Clone, Debug, PartialEq, Serialize, Deserialize)]
+pub enum MsiIrqType {
+    /// PCI MSI IRQ numbers.
+    PciMsi,
+    /// PCI MSIx IRQ numbers.
+    PciMsix,
+    /// Generic MSI IRQ numbers.
+    GenericMsi,
 }
 
-#[derive(Error, Debug)]
-pub enum MigratableError {
-    #[error("Failed to pause migratable component: {0}")]
-    Pause(#[source] anyhow::Error),
-
-    #[error("Failed to resume migratable component: {0}")]
-    Resume(#[source] anyhow::Error),
-}
-
-/// A Pausable component can be paused and resumed.
-pub trait Pausable {
-    /// Pause the component.
-    fn pause(&mut self) -> std::result::Result<(), MigratableError>;
-
-    /// Resume the component.
-    fn resume(&mut self) -> std::result::Result<(), MigratableError>;
-}
-
-/// A snapshotable component can be snapshoted.
-pub trait Snapshotable {}
-
-/// Trait to be implemented by any component (device, CPU, RAM, etc) that
-/// can be migrated.
-/// All migratable components are paused before being snapshotted, and then
-/// eventually resumed. Thus any Migratable component must be both Pausable
-/// and Snapshotable.
-pub trait Migratable: Send + Pausable + Snapshotable {}
-
-fn get_region_host_address_range(
-    region: &GuestRegionMmap,
-    addr: MemoryRegionAddress,
-    size: usize,
-) -> Option<*mut u8> {
-    region.check_address(addr).and_then(|addr| {
-        region
-            .checked_offset(addr, size)
-            .map(|_| region.as_ptr().wrapping_offset(addr.raw_value() as isize))
-    })
-}
-
-/// Convert an absolute address into an address space (GuestMemory)
-/// to a host pointer and verify that the provided size define a valid
-/// range within a single memory region.
-/// Return None if it is out of bounds or if addr+size overlaps a single region.
-///
-/// This is a temporary vm-memory wrapper.
-pub fn get_host_address_range(
-    mem: &GuestMemoryMmap,
-    addr: GuestAddress,
-    size: usize,
-) -> Option<*mut u8> {
-    mem.to_region_addr(addr)
-        .and_then(|(r, addr)| get_region_host_address_range(r, addr, size))
-}
-
-#[cfg(test)]
-mod tests {
-
-    use super::*;
-    use vm_memory::{GuestAddress, GuestMemoryMmap};
-
-    #[test]
-    fn test_get_host_address_range() {
-        let start_addr1 = GuestAddress(0x0);
-        let start_addr2 = GuestAddress(0x1000);
-        let guest_mem =
-            GuestMemoryMmap::from_ranges(&[(start_addr1, 0x400), (start_addr2, 0x400)]).unwrap();
-
-        assert!(get_host_address_range(&guest_mem, GuestAddress(0x600), 0x100).is_none());
-
-        // Overlapping range
-        assert!(get_host_address_range(&guest_mem, GuestAddress(0x1000), 0x500).is_none());
-
-        // Overlapping range
-        assert!(get_host_address_range(&guest_mem, GuestAddress(0x1200), 0x500).is_none());
-
-        let ptr = get_host_address_range(&guest_mem, GuestAddress(0x1000), 0x100).unwrap();
-
-        let ptr0 = get_host_address_range(&guest_mem, GuestAddress(0x1100), 0x100).unwrap();
-
-        let ptr1 = guest_mem.get_host_address(GuestAddress(0x1200)).unwrap();
-        assert_eq!(
-            ptr,
-            guest_mem
-                .find_region(GuestAddress(0x1100))
-                .unwrap()
-                .as_ptr()
-        );
-        assert_eq!(unsafe { ptr0.offset(0x100) }, ptr1);
-    }
+/// Enumeration for device resources.
+#[allow(missing_docs)]
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub enum Resource {
+    /// IO Port address range.
+    PioAddressRange { base: u16, size: u16 },
+    /// Memory Mapped IO address range.
+    MmioAddressRange { base: u64, size: u64 },
+    /// Legacy IRQ number.
+    LegacyIrq(u32),
+    /// Message Signaled Interrupt
+    MsiIrq {
+        ty: MsiIrqType,
+        base: u32,
+        size: u32,
+    },
+    /// Network Interface Card MAC address.
+    MacAddress(String),
+    /// KVM memslot index.
+    KvmMemSlot(u32),
 }

vm-migration/Cargo.toml

@@ -0,0 +1,12 @@
+[package]
+name = "vm-migration"
+version = "0.1.0"
+authors = ["The Cloud Hypervisor Authors"]
+edition = "2018"
+
+[dependencies]
+anyhow = "1.0"
+thiserror = "1.0"
+serde = {version = ">=1.0.27", features = ["rc"] }
+serde_derive = ">=1.0.27"
+serde_json = ">=1.0.9"

vm-migration/src/lib.rs

@@ -0,0 +1,162 @@
+// Copyright © 2019 Intel Corporation
+//
+// SPDX-License-Identifier: Apache-2.0 AND BSD-3-Clause
+//
+
+extern crate serde;
+extern crate thiserror;
+#[macro_use]
+extern crate serde_derive;
+
+use thiserror::Error;
+
+#[derive(Error, Debug)]
+pub enum MigratableError {
+    #[error("Failed to pause migratable component: {0}")]
+    Pause(#[source] anyhow::Error),
+
+    #[error("Failed to resume migratable component: {0}")]
+    Resume(#[source] anyhow::Error),
+
+    #[error("Failed to snapshot migratable component: {0}")]
+    Snapshot(#[source] anyhow::Error),
+
+    #[error("Failed to restore migratable component: {0}")]
+    Restore(#[source] anyhow::Error),
+
+    #[error("Failed to send migratable component snapshot: {0}")]
+    MigrateSend(#[source] anyhow::Error),
+
+    #[error("Failed to receive migratable component snapshot: {0}")]
+    MigrateReceive(#[source] anyhow::Error),
+}
+
+/// A Pausable component can be paused and resumed.
+pub trait Pausable {
+    /// Pause the component.
+    fn pause(&mut self) -> std::result::Result<(), MigratableError> {
+        Ok(())
+    }
+
+    /// Resume the component.
+    fn resume(&mut self) -> std::result::Result<(), MigratableError> {
+        Ok(())
+    }
+}
+
+/// A Snapshottable component snapshot section.
+/// Migratable component can split their migration snapshot into
+/// separate sections.
+/// Splitting a component migration data into different sections
+/// allows for easier and forward compatible extensions.
+#[derive(Clone, Default, Deserialize, Serialize)]
+pub struct SnapshotDataSection {
+    /// The section id.
+    pub id: String,
+
+    /// The section serialized snapshot.
+    pub snapshot: Vec<u8>,
+}
+
+/// A Snapshottable component's snapshot is a tree of snapshots, where leafs
+/// contain the snapshot data. Nodes of this tree track all their children
+/// through the snapshots field, which is basically their sub-components.
+/// Leaves will typically have an empty snapshots map, while nodes usually
+/// carry an empty snapshot_data.
+///
+/// For example, a device manager snapshot is the composition of all its
+/// devices snapshots. The device manager Snapshot would have no snapshot_data
+/// but one Snapshot child per tracked device. Then each device's Snapshot
+/// would carry an empty snapshots map but a map of SnapshotDataSection, i.e.
+/// the actual device snapshot data.
+#[derive(Clone, Default, Deserialize, Serialize)]
+pub struct Snapshot {
+    /// The Snapshottable component id.
+    pub id: String,
+
+    /// The Snapshottable component snapshots.
+    pub snapshots: std::collections::HashMap<String, Box<Snapshot>>,
+
+    /// The Snapshottable component's snapshot data.
+    /// A map of snapshot sections, indexed by the section ids.
+    pub snapshot_data: std::collections::HashMap<String, SnapshotDataSection>,
+}
+
+impl Snapshot {
+    /// Create an empty Snapshot.
+    pub fn new(id: &str) -> Self {
+        Snapshot {
+            id: id.to_string(),
+            ..Default::default()
+        }
+    }
+
+    /// Add a sub-component's Snapshot to the Snapshot.
+    pub fn add_snapshot(&mut self, snapshot: Snapshot) {
+        self.snapshots
+            .insert(snapshot.id.clone(), Box::new(snapshot));
+    }
+
+    /// Add a SnapshotDatasection to the component snapshot data.
+    pub fn add_data_section(&mut self, section: SnapshotDataSection) {
+        self.snapshot_data.insert(section.id.clone(), section);
+    }
+}
+
+/// A snapshottable component can be snapshotted.
+pub trait Snapshottable: Pausable {
+    /// The snapshottable component id.
+    fn id(&self) -> String {
+        String::new()
+    }
+
+    /// Take a component snapshot.
+    fn snapshot(&self) -> std::result::Result<Snapshot, MigratableError> {
+        Ok(Snapshot::new(""))
+    }
+
+    /// Restore a component from its snapshot.
+    fn restore(&mut self, _snapshot: Snapshot) -> std::result::Result<(), MigratableError> {
+        Ok(())
+    }
+}
+
+/// A transportable component can be sent or receive to a specific URL.
+///
+/// This trait is meant to be used for component that have custom
+/// transport handlers.
+pub trait Transportable: Pausable + Snapshottable {
+    /// Send a component snapshot.
+    ///
+    /// # Arguments
+    ///
+    /// * `snapshot` - The migratable component snapshot to send.
+    /// * `destination_url` - The destination URL to send the snapshot to. This
+    ///                       could be an HTTP endpoint, a TCP address or a local file.
+    fn send(
+        &self,
+        _snapshot: &Snapshot,
+        _destination_url: &str,
+    ) -> std::result::Result<(), MigratableError> {
+        Ok(())
+    }
+
+    /// Receive a component snapshot.
+    ///
+    /// # Arguments
+    ///
+    /// * `source_url` - The source URL to fetch the snapshot from. This could be an HTTP
+    ///                  endpoint, a TCP address or a local file.
+    fn recv(&self, _source_url: &str) -> std::result::Result<Snapshot, MigratableError> {
+        Ok(Snapshot::new(""))
+    }
+}
+
+/// Trait to be implemented by any component (device, CPU, RAM, etc) that
+/// can be migrated.
+/// All migratable components are paused before being snapshotted, and then
+/// eventually resumed. Thus any Migratable component must be both Pausable
+/// and Snapshottable.
+/// Moreover a migratable component can be transported to a remote or local
+/// destination and thus must be Transportable.
+pub trait Migratable: Send + Pausable + Snapshottable + Transportable {}

vm-virtio/Cargo.toml

@@ -10,19 +10,25 @@ pci_support = ["pci"]
 mmio_support = []
 
 [dependencies]
+anyhow = "1.0"
 arc-swap = ">=0.4.4"
 byteorder = "1.3.4"
 devices = { path = "../devices" }
 epoll = ">=4.0.1"
-libc = "0.2.68"
+libc = "0.2.71"
 log = "0.4.8"
 net_gen = { path = "../net_gen" }
 net_util = { path = "../net_util" }
 pci = { path = "../pci", optional = true }
+serde = ">=1.0.27"
+serde_derive = ">=1.0.27"
+serde_json = ">=1.0.9"
 tempfile = "3.1.0"
-virtio-bindings = { git = "https://github.com/rust-vmm/virtio-bindings", version = "0.1", features = ["virtio-v5_0_0"]}
+virtio-bindings = { version = "0.1", features = ["virtio-v5_0_0"]}
+vfio-ioctls = { git = "https://github.com/cloud-hypervisor/vfio-ioctls", branch = "ch" }
 vm-allocator = { path = "../vm-allocator" }
 vm-device = { path = "../vm-device" }
-vm-memory = { version = "0.1.0", features = ["backend-mmap", "backend-atomic"] }
+vm-memory = { version = "0.2.1", features = ["backend-mmap", "backend-atomic"] }
+vm-migration = { path = "../vm-migration" }
 vmm-sys-util = ">=0.3.1"
 vhost_rs = { git = "https://github.com/cloud-hypervisor/vhost", branch = "dragonball", package = "vhost", features = ["vhost-user-master", "vhost-user-slave"] }

vm-virtio/src/block.rs

@@ -14,8 +14,9 @@ use super::{
     VirtioDeviceType, VirtioInterruptType,
 };
 use crate::VirtioInterrupt;
-use epoll;
+use anyhow::anyhow;
 use libc::{c_void, EFD_NONBLOCK};
+use serde::ser::{Serialize, SerializeStruct, Serializer};
 use std::alloc::{alloc_zeroed, dealloc, Layout};
 use std::cmp;
 use std::convert::TryInto;
@@ -23,7 +24,7 @@ use std::fs::{File, Metadata};
 use std::io::{self, Read, Seek, SeekFrom, Write};
 use std::ops::DerefMut;
 use std::os::linux::fs::MetadataExt;
-use std::os::unix::io::{AsRawFd, RawFd};
+use std::os::unix::io::{AsRawFd, FromRawFd, RawFd};
 use std::path::PathBuf;
 use std::result;
 use std::slice;
@@ -31,11 +32,15 @@ use std::sync::atomic::{AtomicBool, Ordering};
 use std::sync::{Arc, Mutex};
 use std::thread;
 use virtio_bindings::bindings::virtio_blk::*;
-use vm_device::{Migratable, MigratableError, Pausable, Snapshotable};
+use virtio_bindings::bindings::virtio_ring::VIRTIO_RING_F_EVENT_IDX;
 use vm_memory::{
     ByteValued, Bytes, GuestAddress, GuestAddressSpace, GuestMemory, GuestMemoryAtomic,
     GuestMemoryError, GuestMemoryMmap,
 };
+use vm_migration::{
+    Migratable, MigratableError, Pausable, Snapshot, SnapshotDataSection, Snapshottable,
+    Transportable,
+};
 use vmm_sys_util::{eventfd::EventFd, seek_hole::SeekHole, write_zeroes::PunchHole};
 
 const SECTOR_SHIFT: u8 = 9;
@@ -344,7 +349,7 @@ impl Write for RawFile {
     }
 
     fn flush(&mut self) -> std::io::Result<()> {
-        self.file.flush()
+        self.file.sync_all()
     }
 }
 
@@ -473,6 +478,7 @@ pub struct Request {
     data_addr: GuestAddress,
     data_len: u32,
     pub status_addr: GuestAddress,
+    writeback: bool,
 }
 
 impl Request {
@@ -491,6 +497,7 @@ impl Request {
             data_addr: GuestAddress(0),
             data_len: 0,
             status_addr: GuestAddress(0),
+            writeback: true,
         };
 
         let data_desc;
@@ -570,13 +577,11 @@ impl Request {
             RequestType::Out => {
                 mem.write_all_to(self.data_addr, disk, self.data_len as usize)
                     .map_err(ExecuteError::Write)?;
-            }
-            RequestType::Flush => match disk.flush() {
-                Ok(_) => {
-                    return Ok(0);
+                if !self.writeback {
+                    disk.flush().map_err(ExecuteError::Flush)?;
                 }
-                Err(e) => return Err(ExecuteError::Flush(e)),
-            },
+            }
+            RequestType::Flush => disk.flush().map_err(ExecuteError::Flush)?,
             RequestType::GetDeviceID => {
                 if (self.data_len as usize) < disk_id.len() {
                     return Err(ExecuteError::BadRequest(Error::InvalidOffset));
@@ -588,6 +593,10 @@ impl Request {
         };
         Ok(0)
     }
+
+    pub fn set_writeback(&mut self, writeback: bool) {
+        self.writeback = writeback
+    }
 }
 
 struct BlockEpollHandler<T: DiskFile> {
@@ -599,6 +608,8 @@ struct BlockEpollHandler<T: DiskFile> {
     disk_image_id: Vec<u8>,
     kill_evt: EventFd,
     pause_evt: EventFd,
+    event_idx: bool,
+    writeback: Arc<AtomicBool>,
 }
 
 impl<T: DiskFile> BlockEpollHandler<T> {
@@ -611,7 +622,9 @@ impl<T: DiskFile> BlockEpollHandler<T> {
         for avail_desc in queue.iter(&mem) {
             let len;
             match Request::parse(&avail_desc, &mem) {
-                Ok(request) => {
+                Ok(mut request) => {
+                    request.set_writeback(self.writeback.load(Ordering::SeqCst));
+
                     let mut disk_image_locked = self.disk_image.lock().unwrap();
                     let mut disk_image = disk_image_locked.deref_mut();
                     let status = match request.execute(
@@ -680,24 +693,26 @@ impl<T: DiskFile> BlockEpollHandler<T> {
     ) -> result::Result<(), DeviceError> {
         // Create the epoll file descriptor
         let epoll_fd = epoll::create(true).map_err(DeviceError::EpollCreateFd)?;
+        // Use 'File' to enforce closing on 'epoll_fd'
+        let epoll_file = unsafe { File::from_raw_fd(epoll_fd) };
 
         // Add events
         epoll::ctl(
-            epoll_fd,
+            epoll_file.as_raw_fd(),
             epoll::ControlOptions::EPOLL_CTL_ADD,
             queue_evt.as_raw_fd(),
             epoll::Event::new(epoll::Events::EPOLLIN, u64::from(QUEUE_AVAIL_EVENT)),
         )
         .map_err(DeviceError::EpollCtl)?;
         epoll::ctl(
-            epoll_fd,
+            epoll_file.as_raw_fd(),
             epoll::ControlOptions::EPOLL_CTL_ADD,
             self.kill_evt.as_raw_fd(),
             epoll::Event::new(epoll::Events::EPOLLIN, u64::from(KILL_EVENT)),
         )
         .map_err(DeviceError::EpollCtl)?;
         epoll::ctl(
-            epoll_fd,
+            epoll_file.as_raw_fd(),
             epoll::ControlOptions::EPOLL_CTL_ADD,
             self.pause_evt.as_raw_fd(),
             epoll::Event::new(epoll::Events::EPOLLIN, u64::from(PAUSE_EVENT)),
@@ -708,7 +723,7 @@ impl<T: DiskFile> BlockEpollHandler<T> {
         let mut events = vec![epoll::Event::new(epoll::Events::empty(), 0); EPOLL_EVENTS_LEN];
 
         'epoll: loop {
-            let num_events = match epoll::wait(epoll_fd, -1, &mut events[..]) {
+            let num_events = match epoll::wait(epoll_file.as_raw_fd(), -1, &mut events[..]) {
                 Ok(res) => res,
                 Err(e) => {
                     if e.kind() == io::ErrorKind::Interrupted {
@@ -733,6 +748,28 @@ impl<T: DiskFile> BlockEpollHandler<T> {
                         if let Err(e) = queue_evt.read() {
                             error!("Failed to get queue event: {:?}", e);
                             break 'epoll;
+                        } else if self.event_idx {
+                            // vm-virtio's Queue implementation only checks avail_index
+                            // once, so to properly support EVENT_IDX we need to keep
+                            // calling process_queue() until it stops finding new
+                            // requests on the queue.
+                            loop {
+                                if self.process_queue() {
+                                    self.queue.update_avail_event(&self.mem.memory());
+
+                                    if self.queue.needs_notification(
+                                        &self.mem.memory(),
+                                        self.queue.next_used,
+                                    ) {
+                                        if let Err(e) = self.signal_used_queue() {
+                                            error!("Failed to signal used queue: {:?}", e);
+                                            break 'epoll;
+                                        }
+                                    }
+                                } else {
+                                    break;
+                                }
+                            }
                         } else if self.process_queue() {
                             if let Err(e) = self.signal_used_queue() {
                                 error!("Failed to signal used queue: {:?}", e);
@@ -766,7 +803,7 @@ impl<T: DiskFile> BlockEpollHandler<T> {
     }
 }
 
-#[derive(Copy, Clone, Debug, Default)]
+#[derive(Copy, Clone, Debug, Default, Deserialize)]
 #[repr(C, packed)]
 pub struct VirtioBlockGeometry {
     pub cylinders: u16,
@@ -774,9 +811,31 @@ pub struct VirtioBlockGeometry {
     pub sectors: u8,
 }
 
+// We must explicitly implement Serialize since the structure is packed and
+// it's unsafe to borrow from a packed structure. And by default, if we derive
+// Serialize from serde, it will borrow the values from the structure.
+// That's why this implementation copies each field separately before it
+// serializes the entire structure field by field.
+impl Serialize for VirtioBlockGeometry {
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        let cylinders = self.cylinders;
+        let heads = self.heads;
+        let sectors = self.sectors;
+
+        let mut virtio_block_geometry = serializer.serialize_struct("VirtioBlockGeometry", 4)?;
+        virtio_block_geometry.serialize_field("cylinders", &cylinders)?;
+        virtio_block_geometry.serialize_field("heads", &heads)?;
+        virtio_block_geometry.serialize_field("sectors", &sectors)?;
+        virtio_block_geometry.end()
+    }
+}
+
 unsafe impl ByteValued for VirtioBlockGeometry {}
 
-#[derive(Copy, Clone, Debug, Default)]
+#[derive(Copy, Clone, Debug, Default, Deserialize)]
 #[repr(C, packed)]
 pub struct VirtioBlockConfig {
     pub capacity: u64,
@@ -788,7 +847,7 @@ pub struct VirtioBlockConfig {
     pub alignment_offset: u8,
     pub min_io_size: u16,
     pub opt_io_size: u32,
-    pub wce: u8,
+    pub writeback: u8,
     unused: u8,
     pub num_queues: u16,
     pub max_discard_sectors: u32,
@@ -800,10 +859,67 @@ pub struct VirtioBlockConfig {
     unused1: [u8; 3],
 }
 
+// We must explicitly implement Serialize since the structure is packed and
+// it's unsafe to borrow from a packed structure. And by default, if we derive
+// Serialize from serde, it will borrow the values from the structure.
+// That's why this implementation copies each field separately before it
+// serializes the entire structure field by field.
+impl Serialize for VirtioBlockConfig {
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        let capacity = self.capacity;
+        let size_max = self.size_max;
+        let seg_max = self.seg_max;
+        let geometry = self.geometry;
+        let blk_size = self.blk_size;
+        let physical_block_exp = self.physical_block_exp;
+        let alignment_offset = self.alignment_offset;
+        let min_io_size = self.min_io_size;
+        let opt_io_size = self.opt_io_size;
+        let writeback = self.writeback;
+        let unused = self.unused;
+        let num_queues = self.num_queues;
+        let max_discard_sectors = self.max_discard_sectors;
+        let max_discard_seg = self.max_discard_seg;
+        let discard_sector_alignment = self.discard_sector_alignment;
+        let max_write_zeroes_sectors = self.max_write_zeroes_sectors;
+        let max_write_zeroes_seg = self.max_write_zeroes_seg;
+        let write_zeroes_may_unmap = self.write_zeroes_may_unmap;
+        let unused1 = self.unused1;
+
+        let mut virtio_block_config = serializer.serialize_struct("VirtioBlockConfig", 60)?;
+        virtio_block_config.serialize_field("capacity", &capacity)?;
+        virtio_block_config.serialize_field("size_max", &size_max)?;
+        virtio_block_config.serialize_field("seg_max", &seg_max)?;
+        virtio_block_config.serialize_field("geometry", &geometry)?;
+        virtio_block_config.serialize_field("blk_size", &blk_size)?;
+        virtio_block_config.serialize_field("physical_block_exp", &physical_block_exp)?;
+        virtio_block_config.serialize_field("alignment_offset", &alignment_offset)?;
+        virtio_block_config.serialize_field("min_io_size", &min_io_size)?;
+        virtio_block_config.serialize_field("opt_io_size", &opt_io_size)?;
+        virtio_block_config.serialize_field("writeback", &writeback)?;
+        virtio_block_config.serialize_field("unused", &unused)?;
+        virtio_block_config.serialize_field("num_queues", &num_queues)?;
+        virtio_block_config.serialize_field("max_discard_sectors", &max_discard_sectors)?;
+        virtio_block_config.serialize_field("max_discard_seg", &max_discard_seg)?;
+        virtio_block_config
+            .serialize_field("discard_sector_alignment", &discard_sector_alignment)?;
+        virtio_block_config
+            .serialize_field("max_write_zeroes_sectors", &max_write_zeroes_sectors)?;
+        virtio_block_config.serialize_field("max_write_zeroes_seg", &max_write_zeroes_seg)?;
+        virtio_block_config.serialize_field("write_zeroes_may_unmap", &write_zeroes_may_unmap)?;
+        virtio_block_config.serialize_field("unused1", &unused1)?;
+        virtio_block_config.end()
+    }
+}
+
 unsafe impl ByteValued for VirtioBlockConfig {}
 
 /// Virtio device for exposing block level read/write operations on a host file.
 pub struct Block<T: DiskFile> {
+    id: String,
     kill_evt: Option<EventFd>,
     disk_image: Arc<Mutex<T>>,
     disk_path: PathBuf,
@@ -817,6 +933,16 @@ pub struct Block<T: DiskFile> {
     pause_evt: Option<EventFd>,
     paused: Arc<AtomicBool>,
     queue_size: Vec<u16>,
+    writeback: Arc<AtomicBool>,
+}
+
+#[derive(Serialize, Deserialize)]
+pub struct BlockState {
+    pub disk_path: PathBuf,
+    pub disk_nsectors: u64,
+    pub avail_features: u64,
+    pub acked_features: u64,
+    pub config: VirtioBlockConfig,
 }
 
 impl<T: DiskFile> Block<T> {
@@ -824,6 +950,7 @@ impl<T: DiskFile> Block<T> {
     ///
     /// The given file must be seekable and sizable.
     pub fn new(
+        id: String,
         mut disk_image: T,
         disk_path: PathBuf,
         is_disk_read_only: bool,
@@ -840,7 +967,10 @@ impl<T: DiskFile> Block<T> {
             );
         }
 
-        let mut avail_features = (1u64 << VIRTIO_F_VERSION_1) | (1u64 << VIRTIO_BLK_F_FLUSH);
+        let mut avail_features = (1u64 << VIRTIO_F_VERSION_1)
+            | (1u64 << VIRTIO_BLK_F_FLUSH)
+            | (1u64 << VIRTIO_RING_F_EVENT_IDX)
+            | (1u64 << VIRTIO_BLK_F_CONFIG_WCE);
 
         if iommu {
             avail_features |= 1u64 << VIRTIO_F_IOMMU_PLATFORM;
@@ -853,6 +983,7 @@ impl<T: DiskFile> Block<T> {
         let disk_nsectors = disk_size / SECTOR_SIZE;
         let mut config = VirtioBlockConfig {
             capacity: disk_nsectors,
+            writeback: 1,
             ..Default::default()
         };
 
@@ -862,6 +993,7 @@ impl<T: DiskFile> Block<T> {
         }
 
         Ok(Block {
+            id,
             kill_evt: None,
             disk_image: Arc::new(Mutex::new(disk_image)),
             disk_path,
@@ -875,8 +1007,50 @@ impl<T: DiskFile> Block<T> {
             pause_evt: None,
             paused: Arc::new(AtomicBool::new(false)),
             queue_size: vec![queue_size; num_queues],
+            writeback: Arc::new(AtomicBool::new(true)),
         })
     }
+
+    fn state(&self) -> BlockState {
+        BlockState {
+            disk_path: self.disk_path.clone(),
+            disk_nsectors: self.disk_nsectors,
+            avail_features: self.avail_features,
+            acked_features: self.acked_features,
+            config: self.config,
+        }
+    }
+
+    fn set_state(&mut self, state: &BlockState) -> io::Result<()> {
+        self.disk_path = state.disk_path.clone();
+        self.disk_nsectors = state.disk_nsectors;
+        self.avail_features = state.avail_features;
+        self.acked_features = state.acked_features;
+        self.config = state.config;
+
+        Ok(())
+    }
+
+    fn update_writeback(&mut self) {
+        // Use writeback from config if VIRTIO_BLK_F_CONFIG_WCE
+        let writeback =
+            if self.acked_features & 1 << VIRTIO_BLK_F_CONFIG_WCE == 1 << VIRTIO_BLK_F_CONFIG_WCE {
+                self.config.writeback == 1
+            } else {
+                // Else check if VIRTIO_BLK_F_FLUSH negotiated
+                self.acked_features & 1 << VIRTIO_BLK_F_FLUSH == 1 << VIRTIO_BLK_F_FLUSH
+            };
+
+        info!(
+            "Changing cache mode to {}",
+            if writeback {
+                "writeback"
+            } else {
+                "writethrough"
+            }
+        );
+        self.writeback.store(writeback, Ordering::SeqCst);
+    }
 }
 
 impl<T: DiskFile> Drop for Block<T> {
@@ -938,6 +1112,7 @@ impl<T: 'static + DiskFile + Send> VirtioDevice for Block<T> {
         }
         let (_, right) = config_slice.split_at_mut(offset as usize);
         right.copy_from_slice(&data[..]);
+        self.update_writeback();
     }
 
     fn activate(
@@ -997,6 +1172,10 @@ impl<T: 'static + DiskFile + Send> VirtioDevice for Block<T> {
         }
         self.queue_evts = Some(tmp_queue_evts);
 
+        let event_idx = self.acked_features & 1u64 << VIRTIO_RING_F_EVENT_IDX
+            == 1u64 << VIRTIO_RING_F_EVENT_IDX;
+        self.update_writeback();
+
         let mut epoll_threads = Vec::new();
         for _ in 0..self.queue_size.len() {
             let mut handler = BlockEpollHandler {
@@ -1008,8 +1187,12 @@ impl<T: 'static + DiskFile + Send> VirtioDevice for Block<T> {
                 disk_image_id: disk_image_id.clone(),
                 kill_evt: kill_evt.try_clone().unwrap(),
                 pause_evt: pause_evt.try_clone().unwrap(),
+                event_idx,
+                writeback: self.writeback.clone(),
             };
 
+            handler.queue.set_event_idx(event_idx);
+
             let queue_evt = queue_evts.remove(0);
             let paused = self.paused.clone();
             thread::Builder::new()
@@ -1051,5 +1234,45 @@ impl<T: 'static + DiskFile + Send> VirtioDevice for Block<T> {
 }
 
 virtio_pausable!(Block, T: 'static + DiskFile + Send);
-impl<T: 'static + DiskFile + Send> Snapshotable for Block<T> {}
+impl<T: 'static + DiskFile + Send> Snapshottable for Block<T> {
+    fn id(&self) -> String {
+        self.id.clone()
+    }
+
+    fn snapshot(&self) -> std::result::Result<Snapshot, MigratableError> {
+        let snapshot =
+            serde_json::to_vec(&self.state()).map_err(|e| MigratableError::Snapshot(e.into()))?;
+
+        let mut block_snapshot = Snapshot::new(self.id.as_str());
+        block_snapshot.add_data_section(SnapshotDataSection {
+            id: format!("{}-section", self.id),
+            snapshot,
+        });
+
+        Ok(block_snapshot)
+    }
+
+    fn restore(&mut self, snapshot: Snapshot) -> std::result::Result<(), MigratableError> {
+        if let Some(block_section) = snapshot.snapshot_data.get(&format!("{}-section", self.id)) {
+            let block_state = match serde_json::from_slice(&block_section.snapshot) {
+                Ok(state) => state,
+                Err(error) => {
+                    return Err(MigratableError::Restore(anyhow!(
+                        "Could not deserialize BLOCK {}",
+                        error
+                    )))
+                }
+            };
+
+            return self.set_state(&block_state).map_err(|e| {
+                MigratableError::Restore(anyhow!("Could not restore BLOCK state {:?}", e))
+            });
+        }
+
+        Err(MigratableError::Restore(anyhow!(
+            "Could not find BLOCK snapshot section"
+        )))
+    }
+}
+impl<T: 'static + DiskFile + Send> Transportable for Block<T> {}
 impl<T: 'static + DiskFile + Send> Migratable for Block<T> {}

vm-virtio/src/console.rs

@@ -7,21 +7,25 @@ use super::{
     VirtioInterruptType, VIRTIO_F_IOMMU_PLATFORM, VIRTIO_F_VERSION_1,
 };
 use crate::VirtioInterrupt;
-use epoll;
+use anyhow::anyhow;
 use libc::EFD_NONBLOCK;
-use std;
+use serde::ser::{Serialize, SerializeStruct, Serializer};
 use std::cmp;
 use std::collections::VecDeque;
+use std::fs::File;
 use std::io;
 use std::io::Write;
 use std::ops::DerefMut;
-use std::os::unix::io::AsRawFd;
+use std::os::unix::io::{AsRawFd, FromRawFd};
 use std::result;
 use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
 use std::sync::{Arc, Mutex};
 use std::thread;
-use vm_device::{Migratable, MigratableError, Pausable, Snapshotable};
 use vm_memory::{ByteValued, Bytes, GuestAddressSpace, GuestMemoryAtomic, GuestMemoryMmap};
+use vm_migration::{
+    Migratable, MigratableError, Pausable, Snapshot, SnapshotDataSection, Snapshottable,
+    Transportable,
+};
 use vmm_sys_util::eventfd::EventFd;
 
 const QUEUE_SIZE: u16 = 256;
@@ -43,8 +47,8 @@ const PAUSE_EVENT: DeviceEventT = 5;
 //Console size feature bit
 const VIRTIO_CONSOLE_F_SIZE: u64 = 0;
 
-#[derive(Copy, Clone, Debug, Default)]
-#[repr(C)]
+#[derive(Copy, Clone, Debug, Default, Deserialize)]
+#[repr(C, packed)]
 pub struct VirtioConsoleConfig {
     cols: u16,
     rows: u16,
@@ -52,6 +56,30 @@ pub struct VirtioConsoleConfig {
     emerg_wr: u32,
 }
 
+// We must explicitly implement Serialize since the structure is packed and
+// it's unsafe to borrow from a packed structure. And by default, if we derive
+// Serialize from serde, it will borrow the values from the structure.
+// That's why this implementation copies each field separately before it
+// serializes the entire structure field by field.
+impl Serialize for VirtioConsoleConfig {
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        let cols = self.cols;
+        let rows = self.rows;
+        let max_nr_ports = self.max_nr_ports;
+        let emerg_wr = self.emerg_wr;
+
+        let mut virtio_console_config = serializer.serialize_struct("VirtioConsoleConfig", 12)?;
+        virtio_console_config.serialize_field("cols", &cols)?;
+        virtio_console_config.serialize_field("rows", &rows)?;
+        virtio_console_config.serialize_field("max_nr_ports", &max_nr_ports)?;
+        virtio_console_config.serialize_field("emerg_wr", &emerg_wr)?;
+        virtio_console_config.end()
+    }
+}
+
 // Safe because it only has data and has no implicit padding.
 unsafe impl ByteValued for VirtioConsoleConfig {}
 
@@ -157,31 +185,33 @@ impl ConsoleEpollHandler {
     fn run(&mut self, paused: Arc<AtomicBool>) -> result::Result<(), DeviceError> {
         // Create the epoll file descriptor
         let epoll_fd = epoll::create(true).map_err(DeviceError::EpollCreateFd)?;
+        // Use 'File' to enforce closing on 'epoll_fd'
+        let epoll_file = unsafe { File::from_raw_fd(epoll_fd) };
 
         // Add events
         epoll::ctl(
-            epoll_fd,
+            epoll_file.as_raw_fd(),
             epoll::ControlOptions::EPOLL_CTL_ADD,
             self.input_queue_evt.as_raw_fd(),
             epoll::Event::new(epoll::Events::EPOLLIN, u64::from(INPUT_QUEUE_EVENT)),
         )
         .map_err(DeviceError::EpollCtl)?;
         epoll::ctl(
-            epoll_fd,
+            epoll_file.as_raw_fd(),
             epoll::ControlOptions::EPOLL_CTL_ADD,
             self.output_queue_evt.as_raw_fd(),
             epoll::Event::new(epoll::Events::EPOLLIN, u64::from(OUTPUT_QUEUE_EVENT)),
         )
         .map_err(DeviceError::EpollCtl)?;
         epoll::ctl(
-            epoll_fd,
+            epoll_file.as_raw_fd(),
             epoll::ControlOptions::EPOLL_CTL_ADD,
             self.input_evt.as_raw_fd(),
             epoll::Event::new(epoll::Events::EPOLLIN, u64::from(INPUT_EVENT)),
         )
         .map_err(DeviceError::EpollCtl)?;
         epoll::ctl(
-            epoll_fd,
+            epoll_file.as_raw_fd(),
             epoll::ControlOptions::EPOLL_CTL_ADD,
             self.config_evt.as_raw_fd(),
             epoll::Event::new(epoll::Events::EPOLLIN, u64::from(CONFIG_EVENT)),
@@ -189,14 +219,14 @@ impl ConsoleEpollHandler {
         .map_err(DeviceError::EpollCtl)?;
 
         epoll::ctl(
-            epoll_fd,
+            epoll_file.as_raw_fd(),
             epoll::ControlOptions::EPOLL_CTL_ADD,
             self.kill_evt.as_raw_fd(),
             epoll::Event::new(epoll::Events::EPOLLIN, u64::from(KILL_EVENT)),
         )
         .map_err(DeviceError::EpollCtl)?;
         epoll::ctl(
-            epoll_fd,
+            epoll_file.as_raw_fd(),
             epoll::ControlOptions::EPOLL_CTL_ADD,
             self.pause_evt.as_raw_fd(),
             epoll::Event::new(epoll::Events::EPOLLIN, u64::from(PAUSE_EVENT)),
@@ -207,7 +237,7 @@ impl ConsoleEpollHandler {
         let mut events = vec![epoll::Event::new(epoll::Events::empty(), 0); EPOLL_EVENTS_LEN];
 
         'epoll: loop {
-            let num_events = match epoll::wait(epoll_fd, -1, &mut events[..]) {
+            let num_events = match epoll::wait(epoll_file.as_raw_fd(), -1, &mut events[..]) {
                 Ok(res) => res,
                 Err(e) => {
                     if e.kind() == io::ErrorKind::Interrupted {
@@ -343,6 +373,7 @@ impl VirtioConsoleConfig {
 
 /// Virtio device for exposing console to the guest OS through virtio.
 pub struct Console {
+    id: String,
     kill_evt: Option<EventFd>,
     pause_evt: Option<EventFd>,
     avail_features: u64,
@@ -356,9 +387,18 @@ pub struct Console {
     paused: Arc<AtomicBool>,
 }
 
+#[derive(Serialize, Deserialize)]
+pub struct ConsoleState {
+    avail_features: u64,
+    acked_features: u64,
+    config: VirtioConsoleConfig,
+    in_buffer: VecDeque<u8>,
+}
+
 impl Console {
     /// Create a new virtio console device that gets random data from /dev/urandom.
     pub fn new(
+        id: String,
         out: Box<dyn io::Write + Send + Sync + 'static>,
         cols: u16,
         rows: u16,
@@ -383,6 +423,7 @@ impl Console {
 
         Ok((
             Console {
+                id,
                 kill_evt: None,
                 pause_evt: None,
                 avail_features,
@@ -398,6 +439,24 @@ impl Console {
             console_input,
         ))
     }
+
+    fn state(&self) -> ConsoleState {
+        ConsoleState {
+            avail_features: self.avail_features,
+            acked_features: self.acked_features,
+            config: *(self.config.lock().unwrap()),
+            in_buffer: self.input.in_buffer.lock().unwrap().clone(),
+        }
+    }
+
+    fn set_state(&mut self, state: &ConsoleState) -> io::Result<()> {
+        self.avail_features = state.avail_features;
+        self.acked_features = state.acked_features;
+        *(self.config.lock().unwrap()) = state.config;
+        *(self.input.in_buffer.lock().unwrap()) = state.in_buffer.clone();
+
+        Ok(())
+    }
 }
 
 impl Drop for Console {
@@ -563,5 +622,45 @@ impl VirtioDevice for Console {
 }
 
 virtio_pausable!(Console);
-impl Snapshotable for Console {}
+impl Snapshottable for Console {
+    fn id(&self) -> String {
+        self.id.clone()
+    }
+
+    fn snapshot(&self) -> std::result::Result<Snapshot, MigratableError> {
+        let snapshot =
+            serde_json::to_vec(&self.state()).map_err(|e| MigratableError::Snapshot(e.into()))?;
+
+        let mut console_snapshot = Snapshot::new(self.id.as_str());
+        console_snapshot.add_data_section(SnapshotDataSection {
+            id: format!("{}-section", self.id),
+            snapshot,
+        });
+
+        Ok(console_snapshot)
+    }
+
+    fn restore(&mut self, snapshot: Snapshot) -> std::result::Result<(), MigratableError> {
+        if let Some(console_section) = snapshot.snapshot_data.get(&format!("{}-section", self.id)) {
+            let console_state = match serde_json::from_slice(&console_section.snapshot) {
+                Ok(state) => state,
+                Err(error) => {
+                    return Err(MigratableError::Restore(anyhow!(
+                        "Could not deserialize CONSOLE {}",
+                        error
+                    )))
+                }
+            };
+
+            return self.set_state(&console_state).map_err(|e| {
+                MigratableError::Restore(anyhow!("Could not restore CONSOLE state {:?}", e))
+            });
+        }
+
+        Err(MigratableError::Restore(anyhow!(
+            "Could not find CONSOLE snapshot section"
+        )))
+    }
+}
+impl Transportable for Console {}
 impl Migratable for Console {}

vm-virtio/src/device.rs

@@ -6,7 +6,7 @@
 //
 // SPDX-License-Identifier: Apache-2.0 AND BSD-3-Clause
 
-use super::*;
+use crate::{ActivateResult, Error, Queue};
 use std::sync::Arc;
 use vm_memory::{GuestAddress, GuestMemoryAtomic, GuestMemoryMmap, GuestUsize};
 use vmm_sys_util::eventfd::EventFd;
@@ -34,6 +34,15 @@ pub trait VirtioInterrupt: Send + Sync {
 pub type VirtioIommuRemapping =
     Box<dyn Fn(u64) -> std::result::Result<u64, std::io::Error> + Send + Sync>;
 
+#[derive(Clone)]
+pub struct UserspaceMapping {
+    pub host_addr: u64,
+    pub mem_slot: u32,
+    pub addr: GuestAddress,
+    pub len: GuestUsize,
+    pub mergeable: bool,
+}
+
 #[derive(Clone)]
 pub struct VirtioSharedMemory {
     pub offset: u64,
@@ -42,6 +51,8 @@ pub struct VirtioSharedMemory {
 
 #[derive(Clone)]
 pub struct VirtioSharedMemoryList {
+    pub host_addr: u64,
+    pub mem_slot: u32,
     pub addr: GuestAddress,
     pub len: GuestUsize,
     pub region_list: Vec<VirtioSharedMemory>,
@@ -97,6 +108,14 @@ pub trait VirtioDevice: Send {
         None
     }
 
+    /// Updates the list of shared memory regions required by the device.
+    fn set_shm_regions(
+        &mut self,
+        _shm_regions: VirtioSharedMemoryList,
+    ) -> std::result::Result<(), Error> {
+        std::unimplemented!()
+    }
+
     fn iommu_translate(&self, addr: u64) -> u64 {
         addr
     }
@@ -106,6 +125,15 @@ pub trait VirtioDevice: Send {
     /// every device as part of shutting down the VM. Acting on the device
     /// after a shutdown() can lead to unpredictable results.
     fn shutdown(&mut self) {}
+
+    fn update_memory(&mut self, _mem: &GuestMemoryMmap) -> std::result::Result<(), Error> {
+        Ok(())
+    }
+
+    /// Returns the list of userspace mappings associated with this device.
+    fn userspace_mappings(&self) -> Vec<UserspaceMapping> {
+        Vec::new()
+    }
 }
 
 /// Trait providing address translation the same way a physical DMA remapping

vm-virtio/src/iommu.rs

@@ -8,24 +8,29 @@ use super::{
     VirtioDeviceType, VIRTIO_F_VERSION_1,
 };
 use crate::{DmaRemapping, VirtioInterrupt, VirtioInterruptType};
-use epoll;
+use anyhow::anyhow;
 use libc::EFD_NONBLOCK;
 use std::cmp;
 use std::collections::BTreeMap;
 use std::fmt::{self, Display};
+use std::fs::File;
 use std::io::{self, Write};
 use std::mem::size_of;
 use std::ops::Bound::Included;
-use std::os::unix::io::AsRawFd;
+use std::os::unix::io::{AsRawFd, FromRawFd};
 use std::result;
 use std::sync::atomic::{AtomicBool, Ordering};
 use std::sync::{Arc, RwLock};
 use std::thread;
-use vm_device::{ExternalDmaMapping, Migratable, MigratableError, Pausable, Snapshotable};
+use vfio_ioctls::ExternalDmaMapping;
 use vm_memory::{
     Address, ByteValued, Bytes, GuestAddress, GuestAddressSpace, GuestMemoryAtomic,
     GuestMemoryError, GuestMemoryMmap,
 };
+use vm_migration::{
+    Migratable, MigratableError, Pausable, Snapshot, SnapshotDataSection, Snapshottable,
+    Transportable,
+};
 use vmm_sys_util::eventfd::EventFd;
 
 /// Queues sizes
@@ -646,31 +651,33 @@ impl IommuEpollHandler {
     fn run(&mut self, paused: Arc<AtomicBool>) -> result::Result<(), DeviceError> {
         // Create the epoll file descriptor
         let epoll_fd = epoll::create(true).map_err(DeviceError::EpollCreateFd)?;
+        // Use 'File' to enforce closing on 'epoll_fd'
+        let epoll_file = unsafe { File::from_raw_fd(epoll_fd) };
 
         // Add events
         epoll::ctl(
-            epoll_fd,
+            epoll_file.as_raw_fd(),
             epoll::ControlOptions::EPOLL_CTL_ADD,
             self.queue_evts[0].as_raw_fd(),
             epoll::Event::new(epoll::Events::EPOLLIN, u64::from(REQUEST_Q_EVENT)),
         )
         .map_err(DeviceError::EpollCtl)?;
         epoll::ctl(
-            epoll_fd,
+            epoll_file.as_raw_fd(),
             epoll::ControlOptions::EPOLL_CTL_ADD,
             self.queue_evts[1].as_raw_fd(),
             epoll::Event::new(epoll::Events::EPOLLIN, u64::from(EVENT_Q_EVENT)),
         )
         .map_err(DeviceError::EpollCtl)?;
         epoll::ctl(
-            epoll_fd,
+            epoll_file.as_raw_fd(),
             epoll::ControlOptions::EPOLL_CTL_ADD,
             self.kill_evt.as_raw_fd(),
             epoll::Event::new(epoll::Events::EPOLLIN, u64::from(KILL_EVENT)),
         )
         .map_err(DeviceError::EpollCtl)?;
         epoll::ctl(
-            epoll_fd,
+            epoll_file.as_raw_fd(),
             epoll::ControlOptions::EPOLL_CTL_ADD,
             self.pause_evt.as_raw_fd(),
             epoll::Event::new(epoll::Events::EPOLLIN, u64::from(PAUSE_EVENT)),
@@ -681,7 +688,7 @@ impl IommuEpollHandler {
         let mut events = vec![epoll::Event::new(epoll::Events::empty(), 0); EPOLL_EVENTS_LEN];
 
         'epoll: loop {
-            let num_events = match epoll::wait(epoll_fd, -1, &mut events[..]) {
+            let num_events = match epoll::wait(epoll_file.as_raw_fd(), -1, &mut events[..]) {
                 Ok(res) => res,
                 Err(e) => {
                     if e.kind() == io::ErrorKind::Interrupted {
@@ -753,7 +760,7 @@ impl IommuEpollHandler {
     }
 }
 
-#[derive(Clone, Copy)]
+#[derive(Clone, Copy, Serialize, Deserialize)]
 struct Mapping {
     gpa: u64,
     size: u64,
@@ -792,6 +799,7 @@ impl DmaRemapping for IommuMapping {
 }
 
 pub struct Iommu {
+    id: String,
     kill_evt: Option<EventFd>,
     pause_evt: Option<EventFd>,
     avail_features: u64,
@@ -806,8 +814,16 @@ pub struct Iommu {
     paused: Arc<AtomicBool>,
 }
 
+#[derive(Serialize, Deserialize)]
+struct IommuState {
+    avail_features: u64,
+    acked_features: u64,
+    endpoints: BTreeMap<u32, u32>,
+    mappings: BTreeMap<u32, BTreeMap<u64, Mapping>>,
+}
+
 impl Iommu {
-    pub fn new() -> io::Result<(Self, Arc<IommuMapping>)> {
+    pub fn new(id: String) -> io::Result<(Self, Arc<IommuMapping>)> {
         let config = VirtioIommuConfig {
             page_size_mask: VIRTIO_IOMMU_PAGE_SIZE_MASK,
             probe_size: PROBE_PROP_SIZE,
@@ -821,6 +837,7 @@ impl Iommu {
 
         Ok((
             Iommu {
+                id,
                 kill_evt: None,
                 pause_evt: None,
                 avail_features: 1u64 << VIRTIO_F_VERSION_1
@@ -840,6 +857,24 @@ impl Iommu {
         ))
     }
 
+    fn state(&self) -> IommuState {
+        IommuState {
+            avail_features: self.avail_features,
+            acked_features: self.acked_features,
+            endpoints: self.mapping.endpoints.read().unwrap().clone(),
+            mappings: self.mapping.mappings.read().unwrap().clone(),
+        }
+    }
+
+    fn set_state(&mut self, state: &IommuState) -> io::Result<()> {
+        self.avail_features = state.avail_features;
+        self.acked_features = state.acked_features;
+        *(self.mapping.endpoints.write().unwrap()) = state.endpoints.clone();
+        *(self.mapping.mappings.write().unwrap()) = state.mappings.clone();
+
+        Ok(())
+    }
+
     // This function lets the caller specify a list of devices attached to the
     // virtual IOMMU. This list is translated into a virtio-iommu configuration
     // topology, so that it can be understood by the guest driver.
@@ -1038,5 +1073,45 @@ impl VirtioDevice for Iommu {
 }
 
 virtio_pausable!(Iommu);
-impl Snapshotable for Iommu {}
+impl Snapshottable for Iommu {
+    fn id(&self) -> String {
+        self.id.clone()
+    }
+
+    fn snapshot(&self) -> std::result::Result<Snapshot, MigratableError> {
+        let snapshot =
+            serde_json::to_vec(&self.state()).map_err(|e| MigratableError::Snapshot(e.into()))?;
+
+        let mut iommu_snapshot = Snapshot::new(self.id.as_str());
+        iommu_snapshot.add_data_section(SnapshotDataSection {
+            id: format!("{}-section", self.id),
+            snapshot,
+        });
+
+        Ok(iommu_snapshot)
+    }
+
+    fn restore(&mut self, snapshot: Snapshot) -> std::result::Result<(), MigratableError> {
+        if let Some(iommu_section) = snapshot.snapshot_data.get(&format!("{}-section", self.id)) {
+            let iommu_state = match serde_json::from_slice(&iommu_section.snapshot) {
+                Ok(state) => state,
+                Err(error) => {
+                    return Err(MigratableError::Restore(anyhow!(
+                        "Could not deserialize IOMMU {}",
+                        error
+                    )))
+                }
+            };
+
+            return self.set_state(&iommu_state).map_err(|e| {
+                MigratableError::Restore(anyhow!("Could not restore IOMMU state {:?}", e))
+            });
+        }
+
+        Err(MigratableError::Restore(anyhow!(
+            "Could not find IOMMU snapshot section"
+        )))
+    }
+}
+impl Transportable for Iommu {}
 impl Migratable for Iommu {}

vm-virtio/src/lib.rs

@@ -16,6 +16,10 @@ extern crate epoll;
 extern crate log;
 #[cfg(feature = "pci_support")]
 extern crate pci;
+extern crate serde;
+#[macro_use]
+extern crate serde_derive;
+extern crate serde_json;
 extern crate vhost_rs;
 extern crate virtio_bindings;
 extern crate vm_device;
@@ -29,6 +33,7 @@ mod device;
 pub mod block;
 mod console;
 mod iommu;
+pub mod mem;
 pub mod net;
 pub mod net_util;
 mod pmem;
@@ -43,6 +48,7 @@ pub use self::block::*;
 pub use self::console::*;
 pub use self::device::*;
 pub use self::iommu::*;
+pub use self::mem::*;
 pub use self::net::*;
 pub use self::net_util::*;
 pub use self::pmem::*;
@@ -62,11 +68,11 @@ const VIRTIO_F_IOMMU_PLATFORM: u32 = 33;
 const VIRTIO_F_IN_ORDER: u32 = 35;
 
 // Types taken from linux/virtio_ids.h
-#[derive(Copy, Clone)]
+#[derive(Copy, Clone, Debug)]
 #[allow(dead_code)]
 #[allow(non_camel_case_types)]
 #[repr(C)]
-enum VirtioDeviceType {
+pub enum VirtioDeviceType {
     TYPE_NET = 1,
     TYPE_BLOCK = 2,
     TYPE_CONSOLE = 3,
@@ -77,6 +83,7 @@ enum VirtioDeviceType {
     TYPE_INPUT = 18,
     TYPE_VSOCK = 19,
     TYPE_IOMMU = 23,
+    TYPE_MEM = 24,
     TYPE_FS = 26,
     TYPE_PMEM = 27,
     TYPE_UNKNOWN = 0xFF,
@@ -95,6 +102,7 @@ impl From<u32> for VirtioDeviceType {
             18 => VirtioDeviceType::TYPE_INPUT,
             19 => VirtioDeviceType::TYPE_VSOCK,
             23 => VirtioDeviceType::TYPE_IOMMU,
+            24 => VirtioDeviceType::TYPE_MEM,
             26 => VirtioDeviceType::TYPE_FS,
             27 => VirtioDeviceType::TYPE_PMEM,
             _ => VirtioDeviceType::TYPE_UNKNOWN,
@@ -118,6 +126,7 @@ impl fmt::Display for VirtioDeviceType {
             VirtioDeviceType::TYPE_INPUT => "input",
             VirtioDeviceType::TYPE_VSOCK => "vsock",
             VirtioDeviceType::TYPE_IOMMU => "iommu",
+            VirtioDeviceType::TYPE_MEM => "mem",
             VirtioDeviceType::TYPE_FS => "fs",
             VirtioDeviceType::TYPE_PMEM => "pmem",
             VirtioDeviceType::TYPE_UNKNOWN => "UNKNOWN",
@@ -171,8 +180,15 @@ pub enum Error {
         event: DeviceEventT,
     },
     IoError(io::Error),
+    RegisterListener(io::Error),
+    UnregisterListener(io::Error),
     EpollCreateFd(io::Error),
     EpollCtl(io::Error),
     EpollWait(io::Error),
     FailedSignalingDriver(io::Error),
+    VhostUserUpdateMemory(vhost_user::Error),
+    EventfdError(io::Error),
+    SetShmRegionsNotSupported,
+    EpollHander(String),
+    NoMemoryConfigured,
 }

vm-virtio/src/mem.rs

@@ -0,0 +1,983 @@
+// Copyright (c) 2020 Ant Financial
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use super::Error as DeviceError;
+use super::{
+    ActivateError, ActivateResult, DescriptorChain, DeviceEventT, Queue, VirtioDevice,
+    VirtioDeviceType, VIRTIO_F_VERSION_1,
+};
+use crate::{VirtioInterrupt, VirtioInterruptType};
+use libc::EFD_NONBLOCK;
+use std::cmp;
+use std::fs::File;
+use std::io::{self, Write};
+use std::mem::size_of;
+use std::os::unix::io::{AsRawFd, FromRawFd, RawFd};
+use std::result;
+use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
+use std::sync::mpsc;
+use std::sync::{Arc, Mutex};
+use std::thread;
+use vm_memory::{
+    Address, ByteValued, Bytes, GuestAddress, GuestAddressSpace, GuestMemoryAtomic,
+    GuestMemoryError, GuestMemoryMmap, GuestMemoryRegion, GuestRegionMmap,
+};
+use vm_migration::{Migratable, MigratableError, Pausable, Snapshottable, Transportable};
+use vmm_sys_util::eventfd::EventFd;
+
+const QUEUE_SIZE: u16 = 128;
+const NUM_QUEUES: usize = 1;
+const QUEUE_SIZES: &[u16] = &[QUEUE_SIZE];
+
+// Use 2 MiB alignment so transparent hugepages can be used by KVM.
+pub const VIRTIO_MEM_DEFAULT_BLOCK_SIZE: u64 = 512 * 4096;
+const VIRTIO_MEM_USABLE_EXTENT: u64 = 256 * 1024 * 1024;
+
+// Request processed successfully, applicable for
+// - VIRTIO_MEM_REQ_PLUG
+// - VIRTIO_MEM_REQ_UNPLUG
+// - VIRTIO_MEM_REQ_UNPLUG_ALL
+// - VIRTIO_MEM_REQ_STATE
+const VIRTIO_MEM_RESP_ACK: u16 = 0;
+
+// Request denied - e.g. trying to plug more than requested, applicable for
+// - VIRTIO_MEM_REQ_PLUG
+const VIRTIO_MEM_RESP_NACK: u16 = 1;
+
+// Request cannot be processed right now, try again later, applicable for
+// - VIRTIO_MEM_REQ_PLUG
+// - VIRTIO_MEM_REQ_UNPLUG
+// - VIRTIO_MEM_REQ_UNPLUG_ALL
+// VIRTIO_MEM_RESP_BUSY: u16 = 2;
+
+// Error in request (e.g. addresses/alignemnt), applicable for
+// - VIRTIO_MEM_REQ_PLUG
+// - VIRTIO_MEM_REQ_UNPLUG
+// - VIRTIO_MEM_REQ_STATE
+const VIRTIO_MEM_RESP_ERROR: u16 = 3;
+
+// State of memory blocks is "plugged"
+const VIRTIO_MEM_STATE_PLUGGED: u16 = 0;
+// State of memory blocks is "unplugged"
+const VIRTIO_MEM_STATE_UNPLUGGED: u16 = 1;
+// State of memory blocks is "mixed"
+const VIRTIO_MEM_STATE_MIXED: u16 = 2;
+
+// request to plug memory blocks
+const VIRTIO_MEM_REQ_PLUG: u16 = 0;
+// request to unplug memory blocks
+const VIRTIO_MEM_REQ_UNPLUG: u16 = 1;
+// request to unplug all blocks and shrink the usable size
+const VIRTIO_MEM_REQ_UNPLUG_ALL: u16 = 2;
+// request information about the plugged state of memory blocks
+const VIRTIO_MEM_REQ_STATE: u16 = 3;
+
+// Get resize event.
+const RESIZE_EVENT: DeviceEventT = 0;
+// New descriptors are pending on the virtio queue.
+const QUEUE_AVAIL_EVENT: DeviceEventT = 1;
+// The device has been dropped.
+const KILL_EVENT: DeviceEventT = 2;
+// The device should be paused.
+const PAUSE_EVENT: DeviceEventT = 3;
+
+#[derive(Debug)]
+pub enum Error {
+    // Guest gave us bad memory addresses.
+    GuestMemory(GuestMemoryError),
+    // Guest gave us a write only descriptor that protocol says to read from.
+    UnexpectedWriteOnlyDescriptor,
+    // Guest gave us a read only descriptor that protocol says to write to.
+    UnexpectedReadOnlyDescriptor,
+    // Guest gave us too few descriptors in a descriptor chain.
+    DescriptorChainTooShort,
+    // Guest gave us a buffer that was too short to use.
+    BufferLengthTooSmall,
+    // Guest sent us invalid request.
+    InvalidRequest,
+    // Failed to EventFd write.
+    EventFdWriteFail(std::io::Error),
+    // Failed to EventFd try_clone.
+    EventFdTryCloneFail(std::io::Error),
+    // Failed to MpscRecv.
+    MpscRecvFail(mpsc::RecvError),
+    // Resize invalid argument
+    ResizeInval(String),
+    // Fail to resize trigger
+    ResizeTriggerFail(DeviceError),
+}
+
+// Got from qemu/include/standard-headers/linux/virtio_mem.h
+// rust union doesn't support std::default::Default that
+// need by mem.read_obj.
+// Then move virtio_mem_req_plug, virtio_mem_req_unplug and
+// virtio_mem_req_state to virtio_mem_req.
+#[repr(C)]
+#[derive(Copy, Clone, Debug, Default)]
+struct VirtioMemReq {
+    req_type: u16,
+    padding: [u16; 3],
+    addr: u64,
+    nb_blocks: u16,
+}
+
+// Safe because it only has data and has no implicit padding.
+unsafe impl ByteValued for VirtioMemReq {}
+
+// Got from qemu/include/standard-headers/linux/virtio_mem.h
+#[repr(C)]
+#[derive(Copy, Clone, Debug, Default)]
+struct VirtioMemRespState {
+    state: u16,
+}
+
+#[repr(C)]
+#[derive(Copy, Clone, Debug, Default)]
+struct VirtioMemResp {
+    resp_type: u16,
+    padding: [u16; 3],
+
+    state: VirtioMemRespState,
+}
+
+// Safe because it only has data and has no implicit padding.
+unsafe impl ByteValued for VirtioMemResp {}
+
+// Got from qemu/include/standard-headers/linux/virtio_mem.h
+#[repr(C, packed)]
+#[derive(Copy, Clone, Debug, Default)]
+struct VirtioMemConfig {
+    // Block size and alignment. Cannot change.
+    block_size: u32,
+    // Valid with VIRTIO_MEM_F_ACPI_PXM. Cannot change.
+    node_id: u16,
+    padding: u16,
+    // Start address of the memory region. Cannot change.
+    addr: u64,
+    // Region size (maximum). Cannot change.
+    region_size: u64,
+    // Currently usable region size. Can grow up to region_size. Can
+    // shrink due to VIRTIO_MEM_REQ_UNPLUG_ALL (in which case no config
+    // update will be sent).
+    usable_region_size: u64,
+    // Currently used size. Changes due to plug/unplug requests, but no
+    // config updates will be sent.
+    plugged_size: u64,
+    // Requested size. New plug requests cannot exceed it. Can change.
+    requested_size: u64,
+}
+
+// Safe because it only has data and has no implicit padding.
+unsafe impl ByteValued for VirtioMemConfig {}
+
+struct Request {
+    req: VirtioMemReq,
+    status_addr: GuestAddress,
+}
+
+impl Request {
+    fn parse(
+        avail_desc: &DescriptorChain,
+        mem: &GuestMemoryMmap,
+    ) -> result::Result<Request, Error> {
+        // The head contains the request type which MUST be readable.
+        if avail_desc.is_write_only() {
+            return Err(Error::UnexpectedWriteOnlyDescriptor);
+        }
+        if avail_desc.len as usize != size_of::<VirtioMemReq>() {
+            return Err(Error::InvalidRequest);
+        }
+        let req: VirtioMemReq = mem.read_obj(avail_desc.addr).map_err(Error::GuestMemory)?;
+
+        let status_desc = avail_desc
+            .next_descriptor()
+            .ok_or(Error::DescriptorChainTooShort)?;
+
+        // The status MUST always be writable
+        if !status_desc.is_write_only() {
+            return Err(Error::UnexpectedReadOnlyDescriptor);
+        }
+
+        if (status_desc.len as usize) < size_of::<VirtioMemResp>() {
+            return Err(Error::BufferLengthTooSmall);
+        }
+
+        Ok(Request {
+            req,
+            status_addr: status_desc.addr,
+        })
+    }
+}
+
+pub struct Resize {
+    size: Arc<AtomicU64>,
+    tx: mpsc::Sender<Result<(), Error>>,
+    rx: Option<mpsc::Receiver<Result<(), Error>>>,
+    evt: EventFd,
+}
+
+impl Resize {
+    pub fn new() -> io::Result<Self> {
+        let (tx, rx) = mpsc::channel();
+
+        Ok(Resize {
+            size: Arc::new(AtomicU64::new(0)),
+            tx,
+            rx: Some(rx),
+            evt: EventFd::new(EFD_NONBLOCK)?,
+        })
+    }
+
+    pub fn try_clone(&self) -> Result<Self, Error> {
+        Ok(Resize {
+            size: self.size.clone(),
+            tx: self.tx.clone(),
+            rx: None,
+            evt: self.evt.try_clone().map_err(Error::EventFdTryCloneFail)?,
+        })
+    }
+
+    pub fn work(&self, size: u64) -> Result<(), Error> {
+        if let Some(rx) = &self.rx {
+            self.size.store(size, Ordering::SeqCst);
+            self.evt.write(1).map_err(Error::EventFdWriteFail)?;
+            rx.recv().map_err(Error::MpscRecvFail)?
+        } else {
+            panic!("work should not work with cloned resize")
+        }
+    }
+
+    fn get_size(&self) -> u64 {
+        self.size.load(Ordering::SeqCst)
+    }
+
+    fn send(&self, r: Result<(), Error>) -> Result<(), mpsc::SendError<Result<(), Error>>> {
+        self.tx.send(r)
+    }
+}
+
+struct MemEpollHandler {
+    host_addr: u64,
+    host_fd: Option<RawFd>,
+    mem_state: Vec<bool>,
+    config: Arc<Mutex<VirtioMemConfig>>,
+    resize: Resize,
+    queue: Queue,
+    mem: GuestMemoryAtomic<GuestMemoryMmap>,
+    interrupt_cb: Arc<dyn VirtioInterrupt>,
+    queue_evt: EventFd,
+    kill_evt: EventFd,
+    pause_evt: EventFd,
+}
+
+struct StateChangeRequest<'a> {
+    config: VirtioMemConfig,
+    addr: u64,
+    size: u64,
+    nb_blocks: u16,
+    mem_state: &'a mut Vec<bool>,
+    host_addr: u64,
+    host_fd: Option<RawFd>,
+    plug: bool,
+}
+
+impl MemEpollHandler {
+    fn virtio_mem_valid_range(config: &VirtioMemConfig, addr: u64, size: u64) -> bool {
+        // address properly aligned?
+        if addr % config.block_size as u64 != 0 {
+            return false;
+        }
+
+        // reasonable size
+        if addr + size <= addr || size == 0 {
+            return false;
+        }
+
+        // start address in usable range?
+        if addr < config.addr || addr >= config.addr + config.usable_region_size {
+            return false;
+        }
+
+        // end address in usable range?
+        if addr + size > config.addr + config.usable_region_size {
+            return false;
+        }
+
+        true
+    }
+
+    fn virtio_mem_check_bitmap(
+        bit_index: usize,
+        nb_blocks: u16,
+        mem_state: &[bool],
+        plug: bool,
+    ) -> bool {
+        for state in mem_state.iter().skip(bit_index).take(nb_blocks as usize) {
+            if *state != plug {
+                return false;
+            }
+        }
+        true
+    }
+
+    fn virtio_mem_set_bitmap(
+        bit_index: usize,
+        nb_blocks: u16,
+        mem_state: &mut Vec<bool>,
+        plug: bool,
+    ) {
+        for state in mem_state
+            .iter_mut()
+            .skip(bit_index)
+            .take(nb_blocks as usize)
+        {
+            *state = plug;
+        }
+    }
+
+    fn virtio_mem_state_change_request(r: StateChangeRequest) -> u16 {
+        if r.plug && (r.config.plugged_size + r.size > r.config.requested_size) {
+            return VIRTIO_MEM_RESP_NACK;
+        }
+        if !MemEpollHandler::virtio_mem_valid_range(&r.config, r.addr, r.size) {
+            return VIRTIO_MEM_RESP_ERROR;
+        }
+
+        let offset = r.addr - r.config.addr;
+
+        let bit_index = (offset / r.config.block_size as u64) as usize;
+        if !MemEpollHandler::virtio_mem_check_bitmap(bit_index, r.nb_blocks, r.mem_state, !r.plug) {
+            return VIRTIO_MEM_RESP_ERROR;
+        }
+
+        if !r.plug {
+            if let Some(fd) = r.host_fd {
+                let res = unsafe {
+                    libc::fallocate64(
+                        fd,
+                        libc::FALLOC_FL_PUNCH_HOLE | libc::FALLOC_FL_KEEP_SIZE,
+                        offset as libc::off64_t,
+                        r.size as libc::off64_t,
+                    )
+                };
+                if res != 0 {
+                    error!("fallocate64 get error {}", io::Error::last_os_error());
+                    return VIRTIO_MEM_RESP_ERROR;
+                }
+            }
+            let res = unsafe {
+                libc::madvise(
+                    (r.host_addr + offset) as *mut libc::c_void,
+                    r.size as libc::size_t,
+                    libc::MADV_DONTNEED,
+                )
+            };
+            if res != 0 {
+                error!("madvise get error {}", io::Error::last_os_error());
+                return VIRTIO_MEM_RESP_ERROR;
+            }
+        }
+
+        MemEpollHandler::virtio_mem_set_bitmap(bit_index, r.nb_blocks, r.mem_state, r.plug);
+
+        VIRTIO_MEM_RESP_ACK
+    }
+
+    fn virtio_mem_unplug_all(
+        config: VirtioMemConfig,
+        mem_state: &mut Vec<bool>,
+        host_addr: u64,
+        host_fd: Option<RawFd>,
+    ) -> u16 {
+        for x in 0..(config.region_size / config.block_size as u64) as usize {
+            if mem_state[x] {
+                let resp_type =
+                    MemEpollHandler::virtio_mem_state_change_request(StateChangeRequest {
+                        config,
+                        addr: config.addr + x as u64 * config.block_size as u64,
+                        size: config.block_size as u64,
+                        nb_blocks: 1,
+                        mem_state,
+                        host_addr,
+                        host_fd,
+                        plug: false,
+                    });
+                if resp_type != VIRTIO_MEM_RESP_ACK {
+                    return resp_type;
+                }
+                mem_state[x] = false;
+            }
+        }
+
+        VIRTIO_MEM_RESP_ACK
+    }
+
+    fn virtio_mem_state_request(
+        config: VirtioMemConfig,
+        addr: u64,
+        nb_blocks: u16,
+        mem_state: &mut Vec<bool>,
+    ) -> (u16, u16) {
+        let size: u64 = nb_blocks as u64 * config.block_size as u64;
+        let resp_type = if MemEpollHandler::virtio_mem_valid_range(&config, addr, size) {
+            VIRTIO_MEM_RESP_ACK
+        } else {
+            VIRTIO_MEM_RESP_ERROR
+        };
+
+        let offset = addr - config.addr;
+        let bit_index = (offset / config.block_size as u64) as usize;
+        let resp_state =
+            if MemEpollHandler::virtio_mem_check_bitmap(bit_index, nb_blocks, mem_state, true) {
+                VIRTIO_MEM_STATE_PLUGGED
+            } else if MemEpollHandler::virtio_mem_check_bitmap(
+                bit_index, nb_blocks, mem_state, false,
+            ) {
+                VIRTIO_MEM_STATE_UNPLUGGED
+            } else {
+                VIRTIO_MEM_STATE_MIXED
+            };
+
+        (resp_type, resp_state)
+    }
+
+    fn virtio_mem_send_response(
+        mem: &GuestMemoryMmap,
+        resp_type: u16,
+        resp_state: u16,
+        status_addr: GuestAddress,
+    ) -> u32 {
+        let mut resp = VirtioMemResp::default();
+        resp.resp_type = resp_type;
+        resp.state.state = resp_state;
+        match mem.write_obj(resp, status_addr) {
+            Ok(_) => size_of::<VirtioMemResp>() as u32,
+            Err(e) => {
+                error!("bad guest memory address: {}", e);
+                0
+            }
+        }
+    }
+
+    fn signal(&self, int_type: &VirtioInterruptType) -> result::Result<(), DeviceError> {
+        self.interrupt_cb
+            .trigger(int_type, Some(&self.queue))
+            .map_err(|e| {
+                error!("Failed to signal used queue: {:?}", e);
+                DeviceError::FailedSignalingUsedQueue(e)
+            })
+    }
+
+    fn process_queue(&mut self) -> bool {
+        let mut used_desc_heads = [(0, 0); QUEUE_SIZE as usize];
+        let mut used_count = 0;
+        let mem = self.mem.memory();
+        for avail_desc in self.queue.iter(&mem) {
+            let len = match Request::parse(&avail_desc, &mem) {
+                Err(e) => {
+                    error!("failed parse VirtioMemReq: {:?}", e);
+                    0
+                }
+                Ok(r) => {
+                    let mut config = self.config.lock().unwrap();
+                    match r.req.req_type {
+                        VIRTIO_MEM_REQ_PLUG => {
+                            let size: u64 = r.req.nb_blocks as u64 * config.block_size as u64;
+                            let resp_type = MemEpollHandler::virtio_mem_state_change_request(
+                                StateChangeRequest {
+                                    config: *config,
+                                    addr: r.req.addr,
+                                    size,
+                                    nb_blocks: r.req.nb_blocks,
+                                    mem_state: &mut self.mem_state,
+                                    host_addr: self.host_addr,
+                                    host_fd: self.host_fd,
+                                    plug: true,
+                                },
+                            );
+                            if resp_type == VIRTIO_MEM_RESP_ACK {
+                                config.plugged_size += size;
+                            }
+                            MemEpollHandler::virtio_mem_send_response(
+                                &mem,
+                                resp_type,
+                                0u16,
+                                r.status_addr,
+                            )
+                        }
+                        VIRTIO_MEM_REQ_UNPLUG => {
+                            let size: u64 = r.req.nb_blocks as u64 * config.block_size as u64;
+                            let resp_type = MemEpollHandler::virtio_mem_state_change_request(
+                                StateChangeRequest {
+                                    config: *config,
+                                    addr: r.req.addr,
+                                    size,
+                                    nb_blocks: r.req.nb_blocks,
+                                    mem_state: &mut self.mem_state,
+                                    host_addr: self.host_addr,
+                                    host_fd: self.host_fd,
+                                    plug: false,
+                                },
+                            );
+                            if resp_type == VIRTIO_MEM_RESP_ACK {
+                                config.plugged_size -= size;
+                            }
+                            MemEpollHandler::virtio_mem_send_response(
+                                &mem,
+                                resp_type,
+                                0u16,
+                                r.status_addr,
+                            )
+                        }
+                        VIRTIO_MEM_REQ_UNPLUG_ALL => {
+                            let resp_type = MemEpollHandler::virtio_mem_unplug_all(
+                                *config,
+                                &mut self.mem_state,
+                                self.host_addr,
+                                self.host_fd,
+                            );
+                            if resp_type == VIRTIO_MEM_RESP_ACK {
+                                config.plugged_size = 0;
+                                config.usable_region_size = cmp::min(
+                                    config.region_size,
+                                    config.requested_size + VIRTIO_MEM_USABLE_EXTENT,
+                                );
+                            }
+                            MemEpollHandler::virtio_mem_send_response(
+                                &mem,
+                                resp_type,
+                                0u16,
+                                r.status_addr,
+                            )
+                        }
+                        VIRTIO_MEM_REQ_STATE => {
+                            let (resp_type, resp_state) = MemEpollHandler::virtio_mem_state_request(
+                                *config,
+                                r.req.addr,
+                                r.req.nb_blocks,
+                                &mut self.mem_state,
+                            );
+                            MemEpollHandler::virtio_mem_send_response(
+                                &mem,
+                                resp_type,
+                                resp_state,
+                                r.status_addr,
+                            )
+                        }
+                        _ => {
+                            error!("VirtioMemReq unknown request type {:?}", r.req.req_type);
+                            0
+                        }
+                    }
+                }
+            };
+            used_desc_heads[used_count] = (avail_desc.index, len);
+            used_count += 1;
+        }
+
+        for &(desc_index, len) in &used_desc_heads[..used_count] {
+            self.queue.add_used(&mem, desc_index, len);
+        }
+        used_count > 0
+    }
+
+    fn run(&mut self, paused: Arc<AtomicBool>) -> result::Result<(), DeviceError> {
+        // Create the epoll file descriptor
+        let epoll_fd = epoll::create(true).map_err(DeviceError::EpollCreateFd)?;
+        // Use 'File' to enforce closing on 'epoll_fd'
+        let epoll_file = unsafe { File::from_raw_fd(epoll_fd) };
+
+        // Add events
+        epoll::ctl(
+            epoll_file.as_raw_fd(),
+            epoll::ControlOptions::EPOLL_CTL_ADD,
+            self.resize.evt.as_raw_fd(),
+            epoll::Event::new(epoll::Events::EPOLLIN, u64::from(RESIZE_EVENT)),
+        )
+        .map_err(DeviceError::EpollCtl)?;
+
+        epoll::ctl(
+            epoll_file.as_raw_fd(),
+            epoll::ControlOptions::EPOLL_CTL_ADD,
+            self.queue_evt.as_raw_fd(),
+            epoll::Event::new(epoll::Events::EPOLLIN, u64::from(QUEUE_AVAIL_EVENT)),
+        )
+        .map_err(DeviceError::EpollCtl)?;
+
+        epoll::ctl(
+            epoll_file.as_raw_fd(),
+            epoll::ControlOptions::EPOLL_CTL_ADD,
+            self.kill_evt.as_raw_fd(),
+            epoll::Event::new(epoll::Events::EPOLLIN, u64::from(KILL_EVENT)),
+        )
+        .map_err(DeviceError::EpollCtl)?;
+
+        epoll::ctl(
+            epoll_file.as_raw_fd(),
+            epoll::ControlOptions::EPOLL_CTL_ADD,
+            self.pause_evt.as_raw_fd(),
+            epoll::Event::new(epoll::Events::EPOLLIN, u64::from(PAUSE_EVENT)),
+        )
+        .map_err(DeviceError::EpollCtl)?;
+
+        const EPOLL_EVENTS_LEN: usize = 100;
+        let mut events = vec![epoll::Event::new(epoll::Events::empty(), 0); EPOLL_EVENTS_LEN];
+
+        'epoll: loop {
+            let num_events = match epoll::wait(epoll_file.as_raw_fd(), -1, &mut events[..]) {
+                Ok(res) => res,
+                Err(e) => {
+                    if e.kind() == io::ErrorKind::Interrupted {
+                        // It's well defined from the epoll_wait() syscall
+                        // documentation that the epoll loop can be interrupted
+                        // before any of the requested events occurred or the
+                        // timeout expired. In both those cases, epoll_wait()
+                        // returns an error of type EINTR, but this should not
+                        // be considered as a regular error. Instead it is more
+                        // appropriate to retry, by calling into epoll_wait().
+                        continue;
+                    }
+                    return Err(DeviceError::EpollWait(e));
+                }
+            };
+
+            for event in events.iter().take(num_events) {
+                let ev_type = event.data as u16;
+
+                match ev_type {
+                    RESIZE_EVENT => {
+                        if let Err(e) = self.resize.evt.read() {
+                            return Err(DeviceError::EpollHander(format!(
+                                "Failed to get resize event: {:?}",
+                                e
+                            )));
+                        } else {
+                            let size = self.resize.get_size();
+                            let mut config = self.config.lock().unwrap();
+                            let mut signal_error = false;
+                            let r = if config.requested_size == size {
+                                Err(Error::ResizeInval(format!("Virtio-mem resize {} is same with current config.requested_size", size)))
+                            } else if size > config.region_size {
+                                let region_size = config.region_size;
+                                Err(Error::ResizeInval(format!(
+                                    "Virtio-mem resize {} is bigger than config.region_size {}",
+                                    size, region_size
+                                )))
+                            } else if size % (config.block_size as u64) != 0 {
+                                let block_size = config.block_size;
+                                Err(Error::ResizeInval(format!(
+                                    "Virtio-mem resize {} is not aligned with config.block_size {}",
+                                    size, block_size
+                                )))
+                            } else {
+                                config.requested_size = size;
+                                let tmp_size = cmp::min(
+                                    config.region_size,
+                                    config.requested_size + VIRTIO_MEM_USABLE_EXTENT,
+                                );
+                                config.usable_region_size =
+                                    cmp::max(config.usable_region_size, tmp_size);
+                                if let Err(e) = self.signal(&VirtioInterruptType::Config) {
+                                    signal_error = true;
+                                    Err(Error::ResizeTriggerFail(e))
+                                } else {
+                                    Ok(())
+                                }
+                            };
+                            if let Err(e) = &r {
+                                // This error will send back to resize caller.
+                                error!("Handle resize event get error: {:?}", e);
+                            }
+                            if let Err(e) = self.resize.send(r) {
+                                return Err(DeviceError::EpollHander(format!(
+                                    "Sending \"resize\" generated error: {:?}",
+                                    e
+                                )));
+                            }
+                            if signal_error {
+                                return Err(DeviceError::EpollHander(String::from(
+                                    "Signal get error",
+                                )));
+                            }
+                        }
+                    }
+                    QUEUE_AVAIL_EVENT => {
+                        if let Err(e) = self.queue_evt.read() {
+                            return Err(DeviceError::EpollHander(format!(
+                                "Failed to get queue event: {:?}",
+                                e
+                            )));
+                        } else if self.process_queue() {
+                            if let Err(e) = self.signal(&VirtioInterruptType::Queue) {
+                                return Err(DeviceError::EpollHander(format!(
+                                    "Failed to signal used queue: {:?}",
+                                    e
+                                )));
+                            }
+                        }
+                    }
+                    KILL_EVENT => {
+                        debug!("kill_evt received, stopping epoll loop");
+                        break 'epoll;
+                    }
+                    PAUSE_EVENT => {
+                        debug!("PAUSE_EVENT received, pausing virtio-pmem epoll loop");
+                        // We loop here to handle spurious park() returns.
+                        // Until we have not resumed, the paused boolean will
+                        // be true.
+                        while paused.load(Ordering::SeqCst) {
+                            thread::park();
+                        }
+                    }
+                    _ => {
+                        return Err(DeviceError::EpollHander(String::from(
+                            "Unknown event for virtio-mem",
+                        )));
+                    }
+                }
+            }
+        }
+
+        Ok(())
+    }
+}
+
+// Virtio device for exposing entropy to the guest OS through virtio.
+pub struct Mem {
+    id: String,
+    resize: Resize,
+    kill_evt: Option<EventFd>,
+    pause_evt: Option<EventFd>,
+    avail_features: u64,
+    pub acked_features: u64,
+    host_addr: u64,
+    host_fd: Option<RawFd>,
+    config: Arc<Mutex<VirtioMemConfig>>,
+    queue_evts: Option<Vec<EventFd>>,
+    interrupt_cb: Option<Arc<dyn VirtioInterrupt>>,
+    epoll_threads: Option<Vec<thread::JoinHandle<result::Result<(), DeviceError>>>>,
+    paused: Arc<AtomicBool>,
+}
+
+impl Mem {
+    // Create a new virtio-mem device.
+    pub fn new(id: String, region: &Arc<GuestRegionMmap>, resize: Resize) -> io::Result<Mem> {
+        let region_len = region.len();
+
+        if region_len != region_len / VIRTIO_MEM_DEFAULT_BLOCK_SIZE * VIRTIO_MEM_DEFAULT_BLOCK_SIZE
+        {
+            return Err(io::Error::new(
+                io::ErrorKind::Other,
+                format!(
+                    "Virtio-mem size is not aligned with {}",
+                    VIRTIO_MEM_DEFAULT_BLOCK_SIZE
+                ),
+            ));
+        }
+
+        // Fixme: Not support VIRTIO_MEM_F_ACPI_PXM
+        let avail_features = 1u64 << VIRTIO_F_VERSION_1;
+
+        let mut config = VirtioMemConfig::default();
+        config.block_size = VIRTIO_MEM_DEFAULT_BLOCK_SIZE as u32;
+        config.addr = region.start_addr().raw_value();
+        config.region_size = region.len();
+        config.usable_region_size = cmp::min(
+            config.region_size,
+            config.requested_size + VIRTIO_MEM_USABLE_EXTENT,
+        );
+
+        let host_fd = if let Some(f_offset) = region.file_offset() {
+            Some(f_offset.file().as_raw_fd())
+        } else {
+            None
+        };
+
+        Ok(Mem {
+            id,
+            resize,
+            kill_evt: None,
+            pause_evt: None,
+            avail_features,
+            acked_features: 0u64,
+            host_addr: region.as_ptr() as u64,
+            host_fd,
+            config: Arc::new(Mutex::new(config)),
+            queue_evts: None,
+            interrupt_cb: None,
+            epoll_threads: None,
+            paused: Arc::new(AtomicBool::new(false)),
+        })
+    }
+}
+
+impl Drop for Mem {
+    fn drop(&mut self) {
+        if let Some(kill_evt) = self.kill_evt.take() {
+            // Ignore the result because there is nothing we can do about it.
+            let _ = kill_evt.write(1);
+        }
+    }
+}
+
+impl VirtioDevice for Mem {
+    fn device_type(&self) -> u32 {
+        VirtioDeviceType::TYPE_MEM as u32
+    }
+
+    fn queue_max_sizes(&self) -> &[u16] {
+        QUEUE_SIZES
+    }
+
+    fn features(&self) -> u64 {
+        self.avail_features
+    }
+
+    fn ack_features(&mut self, value: u64) {
+        let mut v = value;
+        // Check if the guest is ACK'ing a feature that we didn't claim to have.
+        let unrequested_features = v & !self.avail_features;
+        if unrequested_features != 0 {
+            warn!("Received acknowledge request for unknown feature.");
+
+            // Don't count these features as acked.
+            v &= !unrequested_features;
+        }
+        self.acked_features |= v;
+    }
+
+    fn read_config(&self, offset: u64, mut data: &mut [u8]) {
+        let config = self.config.lock().unwrap();
+        let config_slice = config.as_slice();
+        let config_len = config_slice.len() as u64;
+        if offset >= config_len {
+            error!("Failed to read config space");
+            return;
+        }
+        if let Some(end) = offset.checked_add(data.len() as u64) {
+            // This write can't fail, offset and end are checked against config_len.
+            data.write_all(&config_slice[offset as usize..cmp::min(end, config_len) as usize])
+                .unwrap();
+        }
+    }
+
+    fn write_config(&mut self, _offset: u64, _data: &[u8]) {
+        warn!("virtio-mem device configuration is read-only");
+    }
+
+    fn activate(
+        &mut self,
+        mem: GuestMemoryAtomic<GuestMemoryMmap>,
+        interrupt_cb: Arc<dyn VirtioInterrupt>,
+        mut queues: Vec<Queue>,
+        mut queue_evts: Vec<EventFd>,
+    ) -> ActivateResult {
+        if queues.len() != NUM_QUEUES || queue_evts.len() != NUM_QUEUES {
+            error!(
+                "Cannot perform activate. Expected {} queue(s), got {}",
+                NUM_QUEUES,
+                queues.len()
+            );
+            return Err(ActivateError::BadActivate);
+        }
+
+        let (self_kill_evt, kill_evt) = EventFd::new(EFD_NONBLOCK)
+            .and_then(|e| Ok((e.try_clone()?, e)))
+            .map_err(|e| {
+                error!("failed creating kill EventFd pair: {}", e);
+                ActivateError::BadActivate
+            })?;
+        self.kill_evt = Some(self_kill_evt);
+
+        let (self_pause_evt, pause_evt) = EventFd::new(EFD_NONBLOCK)
+            .and_then(|e| Ok((e.try_clone()?, e)))
+            .map_err(|e| {
+                error!("failed creating pause EventFd pair: {}", e);
+                ActivateError::BadActivate
+            })?;
+        self.pause_evt = Some(self_pause_evt);
+
+        self.interrupt_cb = Some(interrupt_cb.clone());
+
+        let mut tmp_queue_evts: Vec<EventFd> = Vec::new();
+        for queue_evt in queue_evts.iter() {
+            // Save the queue EventFD as we need to return it on reset
+            // but clone it to pass into the thread.
+            tmp_queue_evts.push(queue_evt.try_clone().map_err(|e| {
+                error!("failed to clone queue EventFd: {}", e);
+                ActivateError::BadActivate
+            })?);
+        }
+        self.queue_evts = Some(tmp_queue_evts);
+
+        let config = self.config.lock().unwrap();
+        let mut handler = MemEpollHandler {
+            host_addr: self.host_addr,
+            host_fd: self.host_fd,
+            mem_state: vec![false; config.region_size as usize / config.block_size as usize],
+            config: self.config.clone(),
+            resize: self.resize.try_clone().map_err(|e| {
+                error!("failed to clone resize EventFd: {:?}", e);
+                ActivateError::BadActivate
+            })?,
+            queue: queues.remove(0),
+            mem,
+            interrupt_cb,
+            queue_evt: queue_evts.remove(0),
+            kill_evt,
+            pause_evt,
+        };
+
+        let paused = self.paused.clone();
+        let mut epoll_threads = Vec::new();
+        thread::Builder::new()
+            .name("virtio_mem".to_string())
+            .spawn(move || handler.run(paused))
+            .map(|thread| epoll_threads.push(thread))
+            .map_err(|e| {
+                error!("failed to clone virtio-mem epoll thread: {}", e);
+                ActivateError::BadActivate
+            })?;
+        self.epoll_threads = Some(epoll_threads);
+
+        Ok(())
+    }
+
+    fn reset(&mut self) -> Option<(Arc<dyn VirtioInterrupt>, Vec<EventFd>)> {
+        // We first must resume the virtio thread if it was paused.
+        if self.pause_evt.take().is_some() {
+            self.resume().ok()?;
+        }
+
+        if let Some(kill_evt) = self.kill_evt.take() {
+            // Ignore the result because there is nothing we can do about it.
+            let _ = kill_evt.write(1);
+        }
+
+        // Return the interrupt and queue EventFDs
+        Some((
+            self.interrupt_cb.take().unwrap(),
+            self.queue_evts.take().unwrap(),
+        ))
+    }
+}
+
+virtio_pausable!(Mem);
+impl Snapshottable for Mem {
+    fn id(&self) -> String {
+        self.id.clone()
+    }
+}
+impl Transportable for Mem {}
+impl Migratable for Mem {}

vm-virtio/src/net.rs

@@ -15,23 +15,28 @@ use super::{
     ActivateError, ActivateResult, Queue, VirtioDevice, VirtioDeviceType, VirtioInterruptType,
 };
 use crate::VirtioInterrupt;
-use epoll;
+use anyhow::anyhow;
 use libc::EAGAIN;
 use libc::EFD_NONBLOCK;
 use net_util::{MacAddr, Tap};
 use std::cmp;
+use std::fs::File;
 use std::io::Read;
 use std::io::{self, Write};
 use std::net::Ipv4Addr;
-use std::os::unix::io::{AsRawFd, RawFd};
+use std::os::unix::io::{AsRawFd, FromRawFd, RawFd};
 use std::result;
 use std::sync::atomic::{AtomicBool, Ordering};
 use std::sync::Arc;
 use std::thread;
 use std::vec::Vec;
 use virtio_bindings::bindings::virtio_net::*;
-use vm_device::{Migratable, MigratableError, Pausable, Snapshotable};
+use virtio_bindings::bindings::virtio_ring::VIRTIO_RING_F_EVENT_IDX;
 use vm_memory::{ByteValued, GuestAddressSpace, GuestMemoryAtomic, GuestMemoryMmap};
+use vm_migration::{
+    Migratable, MigratableError, Pausable, Snapshot, SnapshotDataSection, Snapshottable,
+    Transportable,
+};
 use vmm_sys_util::eventfd::EventFd;
 
 #[derive(Debug)]
@@ -42,60 +47,53 @@ pub enum Error {
 
 pub type Result<T> = result::Result<T, Error>;
 
-struct NetEpollHandler {
-    mem: GuestMemoryAtomic<GuestMemoryMmap>,
-    tap: Tap,
-    rx: RxVirtio,
-    tx: TxVirtio,
-    interrupt_cb: Arc<dyn VirtioInterrupt>,
-    kill_evt: EventFd,
-    pause_evt: EventFd,
-    epoll_fd: RawFd,
-    rx_tap_listening: bool,
+pub struct NetQueuePair {
+    pub mem: Option<GuestMemoryAtomic<GuestMemoryMmap>>,
+    pub tap: Tap,
+    pub rx: RxVirtio,
+    pub tx: TxVirtio,
+    pub epoll_fd: Option<RawFd>,
+    pub rx_tap_listening: bool,
 }
 
-impl NetEpollHandler {
-    fn signal_used_queue(&self, queue: &Queue) -> result::Result<(), DeviceError> {
-        self.interrupt_cb
-            .trigger(&VirtioInterruptType::Queue, Some(queue))
-            .map_err(|e| {
-                error!("Failed to signal used queue: {:?}", e);
-                DeviceError::FailedSignalingUsedQueue(e)
-            })
-    }
-
+impl NetQueuePair {
     // Copies a single frame from `self.rx.frame_buf` into the guest. Returns true
     // if a buffer was used, and false if the frame must be deferred until a buffer
     // is made available by the driver.
-    fn rx_single_frame(&mut self, mut queue: &mut Queue) -> bool {
-        let mem = self.mem.memory();
+    fn rx_single_frame(&mut self, mut queue: &mut Queue) -> result::Result<bool, DeviceError> {
+        let mem = self
+            .mem
+            .as_ref()
+            .ok_or(DeviceError::NoMemoryConfigured)
+            .map(|m| m.memory())?;
         let next_desc = queue.iter(&mem).next();
 
         if next_desc.is_none() {
             // Queue has no available descriptors
             if self.rx_tap_listening {
                 unregister_listener(
-                    self.epoll_fd,
+                    self.epoll_fd.unwrap(),
                     self.tap.as_raw_fd(),
                     epoll::Events::EPOLLIN,
                     u64::from(RX_TAP_EVENT),
                 )
-                .unwrap();
+                .map_err(DeviceError::UnregisterListener)?;
                 self.rx_tap_listening = false;
+                info!("Listener unregistered");
             }
-            return false;
+            return Ok(false);
         }
 
-        self.rx.process_desc_chain(&mem, next_desc, &mut queue)
+        Ok(self.rx.process_desc_chain(&mem, next_desc, &mut queue))
     }
 
-    fn process_rx(&mut self, queue: &mut Queue) -> result::Result<(), DeviceError> {
+    fn process_rx(&mut self, queue: &mut Queue) -> result::Result<bool, DeviceError> {
         // Read as many frames as possible.
         loop {
             match self.read_tap() {
                 Ok(count) => {
                     self.rx.bytes_read = count;
-                    if !self.rx_single_frame(queue) {
+                    if !self.rx_single_frame(queue)? {
                         self.rx.deferred_frame = true;
                         break;
                     }
@@ -116,83 +114,152 @@ impl NetEpollHandler {
         }
         if self.rx.deferred_irqs {
             self.rx.deferred_irqs = false;
-            self.signal_used_queue(queue)
+            let mem = self
+                .mem
+                .as_ref()
+                .ok_or(DeviceError::NoMemoryConfigured)
+                .map(|m| m.memory())?;
+            Ok(queue.needs_notification(&mem, queue.next_used))
         } else {
-            Ok(())
+            Ok(false)
         }
     }
 
-    fn resume_rx(&mut self, queue: &mut Queue) -> result::Result<(), DeviceError> {
+    pub fn resume_rx(&mut self, queue: &mut Queue) -> result::Result<bool, DeviceError> {
+        if !self.rx_tap_listening {
+            register_listener(
+                self.epoll_fd.unwrap(),
+                self.tap.as_raw_fd(),
+                epoll::Events::EPOLLIN,
+                u64::from(RX_TAP_EVENT),
+            )
+            .map_err(DeviceError::RegisterListener)?;
+            self.rx_tap_listening = true;
+            info!("Listener registered");
+        }
         if self.rx.deferred_frame {
-            if self.rx_single_frame(queue) {
+            if self.rx_single_frame(queue)? {
                 self.rx.deferred_frame = false;
                 // process_rx() was interrupted possibly before consuming all
                 // packets in the tap; try continuing now.
                 self.process_rx(queue)
             } else if self.rx.deferred_irqs {
                 self.rx.deferred_irqs = false;
-                self.signal_used_queue(queue)
+                let mem = self
+                    .mem
+                    .as_ref()
+                    .ok_or(DeviceError::NoMemoryConfigured)
+                    .map(|m| m.memory())?;
+                Ok(queue.needs_notification(&mem, queue.next_used))
             } else {
-                Ok(())
+                Ok(false)
             }
         } else {
-            Ok(())
+            Ok(false)
         }
     }
 
-    fn process_tx(&mut self, mut queue: &mut Queue) -> result::Result<(), DeviceError> {
-        let mem = self.mem.memory();
-
+    pub fn process_tx(&mut self, mut queue: &mut Queue) -> result::Result<bool, DeviceError> {
+        let mem = self
+            .mem
+            .as_ref()
+            .ok_or(DeviceError::NoMemoryConfigured)
+            .map(|m| m.memory())?;
         self.tx.process_desc_chain(&mem, &mut self.tap, &mut queue);
+        Ok(queue.needs_notification(&mem, queue.next_used))
+    }
 
-        Ok(())
+    pub fn process_rx_tap(&mut self, mut queue: &mut Queue) -> result::Result<bool, DeviceError> {
+        if self.rx.deferred_frame
+        // Process a deferred frame first if available. Don't read from tap again
+        // until we manage to receive this deferred frame.
+        {
+            if self.rx_single_frame(&mut queue)? {
+                self.rx.deferred_frame = false;
+                self.process_rx(&mut queue)
+            } else if self.rx.deferred_irqs {
+                self.rx.deferred_irqs = false;
+                Ok(true)
+            } else {
+                Ok(false)
+            }
+        } else {
+            self.process_rx(&mut queue)
+        }
     }
 
     fn read_tap(&mut self) -> io::Result<usize> {
         self.tap.read(&mut self.rx.frame_buf)
     }
+}
 
-    fn handle_rx_event(&mut self, mut queue: &mut Queue, queue_evt: &EventFd) {
+struct NetEpollHandler {
+    net: NetQueuePair,
+    interrupt_cb: Arc<dyn VirtioInterrupt>,
+    kill_evt: EventFd,
+    pause_evt: EventFd,
+
+    // Always generate interrupts until the driver has signalled to the device.
+    // This mitigates a problem with interrupts from tap events being "lost" upon
+    // a restore as the vCPU thread isn't ready to handle the interrupt. This causes
+    // issues when combined with VIRTIO_RING_F_EVENT_IDX interrupt suppression.
+    driver_awake: bool,
+}
+
+impl NetEpollHandler {
+    fn signal_used_queue(&self, queue: &Queue) -> result::Result<(), DeviceError> {
+        self.interrupt_cb
+            .trigger(&VirtioInterruptType::Queue, Some(queue))
+            .map_err(|e| {
+                error!("Failed to signal used queue: {:?}", e);
+                DeviceError::FailedSignalingUsedQueue(e)
+            })
+    }
+
+    fn handle_rx_event(
+        &mut self,
+        mut queue: &mut Queue,
+        queue_evt: &EventFd,
+    ) -> result::Result<(), DeviceError> {
         if let Err(e) = queue_evt.read() {
             error!("Failed to get rx queue event: {:?}", e);
         }
 
-        self.resume_rx(&mut queue).unwrap();
-        if !self.rx_tap_listening {
-            register_listener(
-                self.epoll_fd,
-                self.tap.as_raw_fd(),
-                epoll::Events::EPOLLIN,
-                u64::from(RX_TAP_EVENT),
-            )
-            .unwrap();
-            self.rx_tap_listening = true;
+        if self.net.resume_rx(&mut queue)? || !self.driver_awake {
+            self.signal_used_queue(queue)?;
+            info!("Signalling RX queue");
+        } else {
+            info!("Not signalling RX queue");
         }
+
+        Ok(())
     }
 
-    fn handle_tx_event(&mut self, mut queue: &mut Queue, queue_evt: &EventFd) {
+    fn handle_tx_event(
+        &mut self,
+        mut queue: &mut Queue,
+        queue_evt: &EventFd,
+    ) -> result::Result<(), DeviceError> {
         if let Err(e) = queue_evt.read() {
             error!("Failed to get tx queue event: {:?}", e);
         }
-
-        self.process_tx(&mut queue).unwrap();
+        if self.net.process_tx(&mut queue)? || !self.driver_awake {
+            self.signal_used_queue(queue)?;
+            info!("Signalling TX queue");
+        } else {
+            info!("Not signalling TX queue");
+        }
+        Ok(())
     }
 
-    fn handle_rx_tap_event(&mut self, mut queue: &mut Queue) {
-        if self.rx.deferred_frame
-        // Process a deferred frame first if available. Don't read from tap again
-        // until we manage to receive this deferred frame.
-        {
-            if self.rx_single_frame(&mut queue) {
-                self.rx.deferred_frame = false;
-                self.process_rx(&mut queue).unwrap();
-            } else if self.rx.deferred_irqs {
-                self.rx.deferred_irqs = false;
-                self.signal_used_queue(&queue).unwrap();
-            }
+    fn handle_rx_tap_event(&mut self, queue: &mut Queue) -> result::Result<(), DeviceError> {
+        if self.net.process_rx_tap(queue)? || !self.driver_awake {
+            self.signal_used_queue(queue)?;
+            info!("Signalling RX queue");
         } else {
-            self.process_rx(&mut queue).unwrap();
+            info!("Not signalling RX queue");
         }
+        Ok(())
     }
 
     fn run(
@@ -202,42 +269,62 @@ impl NetEpollHandler {
         queue_evts: Vec<EventFd>,
     ) -> result::Result<(), DeviceError> {
         // Create the epoll file descriptor
-        self.epoll_fd = epoll::create(true).map_err(DeviceError::EpollCreateFd)?;
-        // Add events
+        let epoll_fd = epoll::create(true).map_err(DeviceError::EpollCreateFd)?;
+        // Use 'File' to enforce closing on 'epoll_fd'
+        let epoll_file = unsafe { File::from_raw_fd(epoll_fd) };
+        self.net.epoll_fd = Some(epoll_fd);
+
         // Add events
         epoll::ctl(
-            self.epoll_fd,
+            epoll_file.as_raw_fd(),
             epoll::ControlOptions::EPOLL_CTL_ADD,
             queue_evts[0].as_raw_fd(),
             epoll::Event::new(epoll::Events::EPOLLIN, u64::from(RX_QUEUE_EVENT)),
         )
         .map_err(DeviceError::EpollCtl)?;
         epoll::ctl(
-            self.epoll_fd,
+            epoll_file.as_raw_fd(),
             epoll::ControlOptions::EPOLL_CTL_ADD,
             queue_evts[1].as_raw_fd(),
             epoll::Event::new(epoll::Events::EPOLLIN, u64::from(TX_QUEUE_EVENT)),
         )
         .map_err(DeviceError::EpollCtl)?;
         epoll::ctl(
-            self.epoll_fd,
+            epoll_file.as_raw_fd(),
             epoll::ControlOptions::EPOLL_CTL_ADD,
             self.kill_evt.as_raw_fd(),
             epoll::Event::new(epoll::Events::EPOLLIN, u64::from(KILL_EVENT)),
         )
         .map_err(DeviceError::EpollCtl)?;
         epoll::ctl(
-            self.epoll_fd,
+            epoll_file.as_raw_fd(),
             epoll::ControlOptions::EPOLL_CTL_ADD,
             self.pause_evt.as_raw_fd(),
             epoll::Event::new(epoll::Events::EPOLLIN, u64::from(PAUSE_EVENT)),
         )
         .map_err(DeviceError::EpollCtl)?;
 
+        // If there are some already available descriptors on the RX queue,
+        // then we can start the thread while listening onto the TAP.
+        if queues[0]
+            .available_descriptors(&self.net.mem.as_ref().unwrap().memory())
+            .unwrap()
+        {
+            epoll::ctl(
+                epoll_file.as_raw_fd(),
+                epoll::ControlOptions::EPOLL_CTL_ADD,
+                self.net.tap.as_raw_fd(),
+                epoll::Event::new(epoll::Events::EPOLLIN, u64::from(RX_TAP_EVENT)),
+            )
+            .map_err(DeviceError::EpollCtl)?;
+            self.net.rx_tap_listening = true;
+            error!("Listener registered at start");
+        }
+
         let mut events = vec![epoll::Event::new(epoll::Events::empty(), 0); NET_EVENTS_COUNT];
 
         'epoll: loop {
-            let num_events = match epoll::wait(self.epoll_fd, -1, &mut events[..]) {
+            let num_events = match epoll::wait(epoll_file.as_raw_fd(), -1, &mut events[..]) {
                 Ok(res) => res,
                 Err(e) => {
                     if e.kind() == io::ErrorKind::Interrupted {
@@ -259,13 +346,15 @@ impl NetEpollHandler {
 
                 match ev_type {
                     RX_QUEUE_EVENT => {
-                        self.handle_rx_event(&mut queues[0], &queue_evts[0]);
+                        self.driver_awake = true;
+                        self.handle_rx_event(&mut queues[0], &queue_evts[0])?;
                     }
                     TX_QUEUE_EVENT => {
-                        self.handle_tx_event(&mut queues[1], &queue_evts[1]);
+                        self.driver_awake = true;
+                        self.handle_tx_event(&mut queues[1], &queue_evts[1])?;
                     }
                     RX_TAP_EVENT => {
-                        self.handle_rx_tap_event(&mut queues[0]);
+                        self.handle_rx_tap_event(&mut queues[0])?;
                     }
                     KILL_EVENT => {
                         debug!("KILL_EVENT received, stopping epoll loop");
@@ -275,6 +364,7 @@ impl NetEpollHandler {
                         // Drain pause event
                         let _ = self.pause_evt.read();
                         debug!("PAUSE_EVENT received, pausing virtio-net epoll loop");
+
                         // We loop here to handle spurious park() returns.
                         // Until we have not resumed, the paused boolean will
                         // be true.
@@ -293,6 +383,7 @@ impl NetEpollHandler {
 }
 
 pub struct Net {
+    id: String,
     kill_evt: Option<EventFd>,
     pause_evt: Option<EventFd>,
     taps: Option<Vec<Tap>>,
@@ -307,9 +398,18 @@ pub struct Net {
     queue_size: Vec<u16>,
 }
 
+#[derive(Serialize, Deserialize)]
+pub struct NetState {
+    pub avail_features: u64,
+    pub acked_features: u64,
+    pub config: VirtioNetConfig,
+    pub queue_size: Vec<u16>,
+}
+
 impl Net {
     /// Create a new virtio network device with the given TAP interface.
     pub fn new_with_tap(
+        id: String,
         taps: Vec<Tap>,
         guest_mac: Option<MacAddr>,
         iommu: bool,
@@ -322,6 +422,7 @@ impl Net {
             | 1 << VIRTIO_NET_F_GUEST_UFO
             | 1 << VIRTIO_NET_F_HOST_TSO4
             | 1 << VIRTIO_NET_F_HOST_UFO
+            | 1 << VIRTIO_RING_F_EVENT_IDX
             | 1 << VIRTIO_F_VERSION_1;
 
         if iommu {
@@ -339,6 +440,7 @@ impl Net {
         }
 
         Ok(Net {
+            id,
             kill_evt: None,
             pause_evt: None,
             taps: Some(taps),
@@ -356,18 +458,40 @@ impl Net {
 
     /// Create a new virtio network device with the given IP address and
     /// netmask.
+    #[allow(clippy::too_many_arguments)]
     pub fn new(
+        id: String,
         if_name: Option<&str>,
         ip_addr: Option<Ipv4Addr>,
         netmask: Option<Ipv4Addr>,
         guest_mac: Option<MacAddr>,
+        host_mac: &mut Option<MacAddr>,
         iommu: bool,
         num_queues: usize,
         queue_size: u16,
     ) -> Result<Self> {
-        let taps = open_tap(if_name, ip_addr, netmask, num_queues / 2).map_err(Error::OpenTap)?;
+        let taps = open_tap(if_name, ip_addr, netmask, host_mac, num_queues / 2)
+            .map_err(Error::OpenTap)?;
 
-        Self::new_with_tap(taps, guest_mac, iommu, num_queues, queue_size)
+        Self::new_with_tap(id, taps, guest_mac, iommu, num_queues, queue_size)
+    }
+
+    fn state(&self) -> NetState {
+        NetState {
+            avail_features: self.avail_features,
+            acked_features: self.acked_features,
+            config: self.config,
+            queue_size: self.queue_size.clone(),
+        }
+    }
+
+    fn set_state(&mut self, state: &NetState) -> Result<()> {
+        self.avail_features = state.avail_features;
+        self.acked_features = state.acked_features;
+        self.config = state.config;
+        self.queue_size = state.queue_size.clone();
+
+        Ok(())
     }
 }
 
@@ -503,6 +627,8 @@ impl VirtioDevice for Net {
                     })?;
             }
 
+            let event_idx = self.acked_features & 1 << VIRTIO_RING_F_EVENT_IDX != 0;
+
             let mut epoll_threads = Vec::new();
             for _ in 0..taps.len() {
                 let rx = RxVirtio::new();
@@ -512,21 +638,26 @@ impl VirtioDevice for Net {
                 let mut queue_pair = Vec::new();
                 queue_pair.push(queues.remove(0));
                 queue_pair.push(queues.remove(0));
+                queue_pair[0].set_event_idx(event_idx);
+                queue_pair[1].set_event_idx(event_idx);
 
                 let mut queue_evt_pair = Vec::new();
                 queue_evt_pair.push(queue_evts.remove(0));
                 queue_evt_pair.push(queue_evts.remove(0));
 
                 let mut handler = NetEpollHandler {
-                    mem: mem.clone(),
-                    tap: taps.remove(0),
-                    rx,
-                    tx,
+                    net: NetQueuePair {
+                        mem: Some(mem.clone()),
+                        tap: taps.remove(0),
+                        rx,
+                        tx,
+                        epoll_fd: None,
+                        rx_tap_listening,
+                    },
                     interrupt_cb: interrupt_cb.clone(),
                     kill_evt: kill_evt.try_clone().unwrap(),
                     pause_evt: pause_evt.try_clone().unwrap(),
-                    epoll_fd: 0,
-                    rx_tap_listening,
+                    driver_awake: false,
                 };
 
                 let paused = self.paused.clone();
@@ -567,5 +698,45 @@ impl VirtioDevice for Net {
 }
 
 virtio_ctrl_q_pausable!(Net);
-impl Snapshotable for Net {}
+impl Snapshottable for Net {
+    fn id(&self) -> String {
+        self.id.clone()
+    }
+
+    fn snapshot(&self) -> std::result::Result<Snapshot, MigratableError> {
+        let snapshot =
+            serde_json::to_vec(&self.state()).map_err(|e| MigratableError::Snapshot(e.into()))?;
+
+        let mut net_snapshot = Snapshot::new(self.id.as_str());
+        net_snapshot.add_data_section(SnapshotDataSection {
+            id: format!("{}-section", self.id),
+            snapshot,
+        });
+
+        Ok(net_snapshot)
+    }
+
+    fn restore(&mut self, snapshot: Snapshot) -> std::result::Result<(), MigratableError> {
+        if let Some(net_section) = snapshot.snapshot_data.get(&format!("{}-section", self.id)) {
+            let net_state = match serde_json::from_slice(&net_section.snapshot) {
+                Ok(state) => state,
+                Err(error) => {
+                    return Err(MigratableError::Restore(anyhow!(
+                        "Could not deserialize NET {}",
+                        error
+                    )))
+                }
+            };
+
+            return self.set_state(&net_state).map_err(|e| {
+                MigratableError::Restore(anyhow!("Could not restore NET state {:?}", e))
+            });
+        }
+
+        Err(MigratableError::Restore(anyhow!(
+            "Could not find NET snapshot section"
+        )))
+    }
+}
+impl Transportable for Net {}
 impl Migratable for Net {}