build, release-notes.md: Document 0.11.0 release

Update release notes and version number for the new release. Signed-off-by: Rob Bradford <robert.bradford@intel.com>
tests: Port more tests to use wait_vm_boot
2026-08-05 02:19:16 +00:00 · 2020-10-29 17:36:02 +00:00 · 2020-10-29 15:30:17 +00:00 · 2020-10-29 15:30:17 +00:00 · 2020-10-29 15:30:17 +00:00 · 2020-10-29 08:19:25 -07:00
161 changed files with 12943 additions and 7961 deletions
--- a/.github/workflows/audit.yaml
+++ b/.github/workflows/audit.yaml
@@ -0,0 +1,15 @@
+name: Cloud Hypervisor Dependency Audit
+on:
+  pull_request:
+    paths: 
+      - '**/Cargo.toml'
+      - '**/Cargo.lock'
+jobs:
+  security_audit:
+    name: Audit
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v1
+      - uses: actions-rs/audit-check@v1
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
--- a/.github/workflows/build.yaml
+++ b/.github/workflows/build.yaml
@@ -18,6 +18,9 @@ jobs:
    steps:
      - name: Code checkout
        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+
      - name: Install Rust toolchain (${{ matrix.rust }})
        uses: actions-rs/toolchain@v1
        with:
@@ -25,5 +28,16 @@ jobs:
            target: ${{ matrix.target }}
            override: true

-      - name: Build
+      - name: Debug Check (default features)
+        run: |
+          git rev-list origin/master..$GITHUB_SHA | xargs -t -I % sh -c 'git checkout %; cargo check --all --target=${{ matrix.target }}'
+          git checkout $GITHUB_SHA
+
+      - name: Build (acpi,kvm)
+        run: cargo rustc --bin cloud-hypervisor --no-default-features --features "acpi,kvm"  -- -D warnings
+
+      - name: Build (kvm)
+        run: cargo rustc --bin cloud-hypervisor --no-default-features --features "kvm"  -- -D warnings
+        
+      - name: Release Build (default features)
        run: cargo build --all --release --target=${{ matrix.target }}
--- a/.github/workflows/cross-build.yaml
+++ b/.github/workflows/cross-build.yaml
@@ -9,7 +9,7 @@ jobs:
    strategy:
      matrix:
        rust:
-          - stable
+          - 1.46.0
        target:
          - aarch64-unknown-linux-gnu
          - aarch64-unknown-linux-musl
@@ -29,4 +29,4 @@ jobs:
        with:
          use-cross: true
          command: build
-          args: --target=${{ matrix.target }} --no-default-features --features "mmio,kvm"
+          args: --target=${{ matrix.target }} --no-default-features --features "kvm"
--- a/.github/workflows/docker-image.yaml
+++ b/.github/workflows/docker-image.yaml
@@ -0,0 +1,36 @@
+name: Cloud-Hypervisor's Docker image update
+
+on:
+  push:
+    branches: master
+    paths: resources/Dockerfile
+
+jobs:
+  main:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Code checkout
+        uses: actions/checkout@v2
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Login to DockerHub
+        uses: docker/login-action@v1 
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Build and push
+        uses: docker/build-push-action@v2
+        with:
+          file: ./resources/Dockerfile
+          platforms: linux/amd64,linux/arm64
+          push: true
+          tags: cloudhypervisor/dev:latest
+
+      - name: Image digest
+        run: echo ${{ steps.docker_build.outputs.digest }}
--- a/.github/workflows/fuzz-build.yaml
+++ b/.github/workflows/fuzz-build.yaml
@@ -0,0 +1,27 @@
+name: Cloud Hypervisor Cargo Fuzz Build
+on: [pull_request, create]
+
+jobs:
+  build:
+    if: github.event_name == 'pull_request'
+    name: Cargo Fuzz Build
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        rust:
+          - nightly
+        target:
+          - x86_64-unknown-linux-gnu
+    steps:
+      - name: Code checkout
+        uses: actions/checkout@v2
+      - name: Install Rust toolchain (${{ matrix.rust }})
+        uses: actions-rs/toolchain@v1
+        with:
+            toolchain: ${{ matrix.rust }}
+            target: ${{ matrix.target }}
+            override: true
+      - name: Install Cargo fuzz
+        run: cargo install -f cargo-fuzz
+      - name: Cargo Fuzz Build
+        run: cargo fuzz build
--- a/.github/workflows/quality-aarch64.yaml
+++ b/.github/workflows/quality-aarch64.yaml
@@ -0,0 +1,34 @@
+name: Cloud Hypervisor Quality Checks
+on: [pull_request, create]
+
+jobs:
+  build:
+    if: github.event_name == 'pull_request'
+    name: Quality (clippy, rustfmt)
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        rust:
+          - stable
+        target:
+          - aarch64-unknown-linux-gnu
+    steps:
+      - name: Code checkout
+        uses: actions/checkout@v2
+      - name: Install Rust toolchain (${{ matrix.rust }})
+        uses: actions-rs/toolchain@v1
+        with:
+            toolchain: ${{ matrix.rust }}
+            target: ${{ matrix.target }}
+            override: true
+            components: rustfmt, clippy
+      - name: Install arm64 libfdt
+        run: wget http://ftp.us.debian.org/debian/pool/main/d/device-tree-compiler/libfdt-dev_1.6.0-1_arm64.deb && dpkg-deb -xv libfdt-dev_1.6.0-1_arm64.deb ./tlibfdtdev && sudo mkdir /tmmmp && mkdir target && mkdir target/debug && mkdir target/debug/deps && sudo cp ./tlibfdtdev/usr/lib/aarch64-linux-gnu/libfdt.a target/debug/deps/libfdt.a && echo "libfdt installed"
+      - name: Formatting (rustfmt)
+        run: cargo fmt -- --check
+      - name: Clippy (kvm)
+        uses: actions-rs/cargo@v1
+        with:
+          use-cross: true
+          command: clippy
+          args: --target=${{ matrix.target }} --no-default-features --features "kvm" -- -D warnings
--- a/.github/workflows/quality.yaml
+++ b/.github/workflows/quality.yaml
@@ -0,0 +1,36 @@
+name: Cloud Hypervisor Quality Checks
+on: [pull_request, create]
+
+jobs:
+  build:
+    if: github.event_name == 'pull_request'
+    name: Quality (clippy, rustfmt)
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        rust:
+          - stable
+        target:
+          - x86_64-unknown-linux-gnu
+    steps:
+      - name: Code checkout
+        uses: actions/checkout@v2
+      - name: Install Rust toolchain (${{ matrix.rust }})
+        uses: actions-rs/toolchain@v1
+        with:
+            toolchain: ${{ matrix.rust }}
+            target: ${{ matrix.target }}
+            override: true
+            components: rustfmt, clippy
+      
+      - name: Formatting (rustfmt)
+        run: cargo fmt -- --check
+
+      - name: Clippy (all features)
+        run: cargo clippy --all-targets --all-features -- -D warnings
+      
+      - name: Clippy (acpi,kvm)
+        run: cargo clippy --all-targets --no-default-features --features "acpi,kvm" -- -D warnings
+      
+      - name: Clippy (kvm)
+        run: cargo clippy --all-targets --no-default-features --features "kvm" -- -D warnings
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -88,5 +88,5 @@ Fixes #88
 Signed-off-by: Sebastien Boeuf <sebastien.boeuf@intel.com>
 ```

-Then, after the correspoding PR is merged, Github will automatically close that issue when parsing the
+Then, after the corresponding PR is merged, Github will automatically close that issue when parsing the
 [commit message](https://help.github.com/articles/closing-issues-via-commit-messages/).
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -24,9 +24,9 @@ checksum = "ee2a4ec343196209d6594e19543ae87a39f96d5534d7174822a3ad825dd6ed7e"

 [[package]]
 name = "aho-corasick"
-version = "0.7.13"
+version = "0.7.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "043164d8ba5c4c3035fec9bbee8647c0261d788f3474306f93bb65901cae0e86"
+checksum = "b476ce7103678b0c6d3d395dbbae31d48ff910bd28be979ba5d48c6351131d0d"
 dependencies = [
 "memchr",
 ]
@@ -42,9 +42,13 @@ dependencies = [

 [[package]]
 name = "anyhow"
-version = "1.0.32"
+version = "1.0.33"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6b602bfe940d21c130f3895acd65221e8a61270debe89d628b9cb4e3ccb8569b"
+checksum = "a1fd36ffbb1fb7c834eac128ea8d0e310c5aeb635548f9d58861e1308d46e71c"
+
+[[package]]
+name = "api_client"
+version = "0.1.0"

 [[package]]
 name = "arc-swap"
@@ -65,7 +69,11 @@ dependencies = [
 "linux-loader",
 "log 0.4.11",
 "rand 0.7.3",
+ "serde",
+ "serde_derive",
+ "serde_json",
 "vm-memory",
+ "vm-migration",
 ]

 [[package]]
@@ -80,9 +88,9 @@ checksum = "a4c527152e37cf757a3f78aae5a06fbeefdb07ccc535c980a3208ee3060dd544"

 [[package]]
 name = "arrayvec"
-version = "0.5.1"
+version = "0.5.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cff77d8686867eceff3105329d4698d96c2391c176d5d03adc90c7389162b5b8"
+checksum = "23b62fc65de8e4e7f52534fb52b0f3ed04746ae267519eef2a83941e8085068b"

 [[package]]
 name = "atty"
@@ -97,18 +105,18 @@ dependencies = [

 [[package]]
 name = "autocfg"
-version = "1.0.0"
+version = "1.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f8aac770f1885fd7e387acedd76065302551364496e46b3dd00860b2f8359b9d"
+checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a"

 [[package]]
 name = "backtrace"
-version = "0.3.50"
+version = "0.3.53"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "46254cf2fdcdf1badb5934448c1bcbe046a56537b3987d96c51a7afc5d03f293"
+checksum = "707b586e0e2f247cbde68cdd2c3ce69ea7b7be43e1c5b426e37c9319c4b9838e"
 dependencies = [
 "addr2line",
- "cfg-if",
+ "cfg-if 1.0.0",
 "libc",
 "miniz_oxide",
 "object",
@@ -117,9 +125,9 @@ dependencies = [

 [[package]]
 name = "base64"
-version = "0.11.0"
+version = "0.12.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b41b7ea54a0c9d92199de89e20e58d49f02f8e699814ef3fdf266f6f748d15c7"
+checksum = "3441f0f7b02788e948e47f457ca01f1d7e6d92c693bc132c22b087d3141c03ff"

 [[package]]
 name = "bitflags"
@@ -168,9 +176,9 @@ checksum = "08c48aae112d48ed9f069b33538ea9e3e90aa263cfa3d1c24309612b1f7472de"

 [[package]]
 name = "cc"
-version = "1.0.58"
+version = "1.0.61"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f9a06fb2e53271d7c279ec1efea6ab691c35a2ae67ec0d91d7acec0caf13b518"
+checksum = "ed67cbde08356238e75fc4656be4749481eeffb09e19f320a25237d5221c985d"

 [[package]]
 name = "cfg-if"
@@ -179,10 +187,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822"

 [[package]]
-name = "clap"
-version = "2.33.2"
+name = "cfg-if"
+version = "1.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "10040cdf04294b565d9e0319955430099ec3813a64c952b86a41200ad714ae48"
+checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
+
+[[package]]
+name = "clap"
+version = "2.33.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "37e58ac78573c40708d45522f0d80fa2f01cc4f9b4e2bf749807255454312002"
 dependencies = [
 "ansi_term",
 "atty",
@@ -196,25 +210,31 @@ dependencies = [

 [[package]]
 name = "cloud-hypervisor"
-version = "0.9.0"
+version = "0.11.0"
 dependencies = [
+ "anyhow",
+ "api_client",
 "clap",
 "credibility",
 "dirs",
+ "epoll",
 "hypervisor",
 "lazy_static",
 "libc",
 "log 0.4.11",
 "net_util",
+ "option_parser",
 "seccomp",
 "serde_json",
 "ssh2",
 "tempdir",
 "tempfile",
+ "thiserror",
 "vhost_user_block",
 "vhost_user_net",
 "vmm",
 "vmm-sys-util",
+ "wait-timeout",
 ]

 [[package]]
@@ -249,7 +269,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c3c7c73a2d1e9fc0886a08b93e98eb643461230d5f1925e4036204d5f2e261a8"
 dependencies = [
 "autocfg",
- "cfg-if",
+ "cfg-if 0.1.10",
 "lazy_static",
 ]

@@ -334,9 +354,9 @@ checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba"

 [[package]]
 name = "futures"
-version = "0.3.5"
+version = "0.3.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1e05b85ec287aac0dc34db7d4a569323df697f9c55b99b15d6b4ef8cde49f613"
+checksum = "95314d38584ffbfda215621d723e0a3906f032e03ae5551e650058dac83d4797"
 dependencies = [
 "futures-channel",
 "futures-core",
@@ -349,9 +369,9 @@ dependencies = [

 [[package]]
 name = "futures-channel"
-version = "0.3.5"
+version = "0.3.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f366ad74c28cca6ba456d95e6422883cfb4b252a83bed929c83abfdbbf2967d5"
+checksum = "0448174b01148032eed37ac4aed28963aaaa8cfa93569a08e5b479bbc6c2c151"
 dependencies = [
 "futures-core",
 "futures-sink",
@@ -359,15 +379,15 @@ dependencies = [

 [[package]]
 name = "futures-core"
-version = "0.3.5"
+version = "0.3.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "59f5fff90fd5d971f936ad674802482ba441b6f09ba5e15fd8b39145582ca399"
+checksum = "18eaa56102984bed2c88ea39026cff3ce3b4c7f508ca970cedf2450ea10d4e46"

 [[package]]
 name = "futures-executor"
-version = "0.3.5"
+version = "0.3.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "10d6bb888be1153d3abeb9006b11b02cf5e9b209fda28693c31ae1e4e012e314"
+checksum = "f5f8e0c9258abaea85e78ebdda17ef9666d390e987f006be6080dfe354b708cb"
 dependencies = [
 "futures-core",
 "futures-task",
@@ -377,15 +397,15 @@ dependencies = [

 [[package]]
 name = "futures-io"
-version = "0.3.5"
+version = "0.3.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "de27142b013a8e869c14957e6d2edeef89e97c289e69d042ee3a49acd8b51789"
+checksum = "6e1798854a4727ff944a7b12aa999f58ce7aa81db80d2dfaaf2ba06f065ddd2b"

 [[package]]
 name = "futures-macro"
-version = "0.3.5"
+version = "0.3.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d0b5a30a4328ab5473878237c447333c093297bded83a4983d10f4deea240d39"
+checksum = "e36fccf3fc58563b4a14d265027c627c3b665d7fed489427e88e7cc929559efe"
 dependencies = [
 "proc-macro-hack",
 "proc-macro2",
@@ -395,24 +415,24 @@ dependencies = [

 [[package]]
 name = "futures-sink"
-version = "0.3.5"
+version = "0.3.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3f2032893cb734c7a05d85ce0cc8b8c4075278e93b24b66f9de99d6eb0fa8acc"
+checksum = "0e3ca3f17d6e8804ae5d3df7a7d35b2b3a6fe89dac84b31872720fc3060a0b11"

 [[package]]
 name = "futures-task"
-version = "0.3.5"
+version = "0.3.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bdb66b5f09e22019b1ab0830f7785bcea8e7a42148683f99214f73f8ec21a626"
+checksum = "96d502af37186c4fef99453df03e374683f8a1eec9dcc1e66b3b82dc8278ce3c"
 dependencies = [
 "once_cell",
 ]

 [[package]]
 name = "futures-util"
-version = "0.3.5"
+version = "0.3.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8764574ff08b701a084482c3c7031349104b07ac897393010494beaa18ce32c6"
+checksum = "abcb44342f62e6f3e8ac427b8aa815f724fd705dfad060b18ac7866c15bb8e34"
 dependencies = [
 "futures-channel",
 "futures-core",
@@ -430,11 +450,11 @@ dependencies = [

 [[package]]
 name = "getrandom"
-version = "0.1.14"
+version = "0.1.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7abc8dd8451921606d809ba32e95b6111925cd2906060d2dcc29c070220503eb"
+checksum = "fc587bc0ec293155d5bfa6b9891ec18a1e330c234f896ea47fbada4cadbe47e6"
 dependencies = [
- "cfg-if",
+ "cfg-if 0.1.10",
 "libc",
 "wasi",
 ]
@@ -453,9 +473,9 @@ checksum = "8be18de09a56b60ed0edf84bc9df007e30040691af7acd1c41874faac5895bfb"

 [[package]]
 name = "hermit-abi"
-version = "0.1.15"
+version = "0.1.17"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3deed196b6e7f9e44a2ae8d94225d80302d81208b1bb673fd21fe634645c85a9"
+checksum = "5aca5565f760fb5b220e499d72710ed156fdb74e631659e99377d9ebfbd13ae8"
 dependencies = [
 "libc",
 ]
@@ -465,6 +485,7 @@ name = "hypervisor"
 version = "0.1.0"
 dependencies = [
 "anyhow",
+ "arc-swap",
 "kvm-bindings",
 "kvm-ioctls",
 "libc",
@@ -491,8 +512,9 @@ dependencies = [

 [[package]]
 name = "io-uring"
-version = "0.3.5"
-source = "git+https://github.com/tokio-rs/io-uring.git?branch=0.4#c00d968b038263a02a72d6510edaf438b0c7b4f3"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2f7589adca0ddd74f56ed83a5098b45e3abf264dc27e150a8bec3397fcc34338"
 dependencies = [
 "bitflags 1.2.1",
 "libc",
@@ -526,7 +548,7 @@ dependencies = [
 [[package]]
 name = "kvm-bindings"
 version = "0.2.0"
-source = "git+https://github.com/cloud-hypervisor/kvm-bindings?branch=ch#6453253aff8b804796bce547c7a23e09464b7491"
+source = "git+https://github.com/cloud-hypervisor/kvm-bindings?branch=ch#2164129ff03bb4d7a4243de5e2078845064d922c"
 dependencies = [
 "serde",
 "serde_derive",
@@ -536,7 +558,7 @@ dependencies = [
 [[package]]
 name = "kvm-ioctls"
 version = "0.5.0"
-source = "git+https://github.com/cloud-hypervisor/kvm-ioctls?branch=ch#37953e968b9064f42d6b71b7afc13872ac4171ce"
+source = "git+https://github.com/cloud-hypervisor/kvm-ioctls?branch=ch#64427395b60e00403aed3026362a1b19873230e6"
 dependencies = [
 "kvm-bindings",
 "libc",
@@ -551,15 +573,15 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"

 [[package]]
 name = "libc"
-version = "0.2.74"
+version = "0.2.80"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a2f02823cf78b754822df5f7f268fb59822e7296276d3e069d8e8cb26a14bd10"
+checksum = "4d58d1b70b004888f764dfbf6a26a3b0342a1632d33968e4a179d8011c760614"

 [[package]]
 name = "libssh2-sys"
-version = "0.2.18"
+version = "0.2.19"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "eafa907407504b0e683786d4aba47acf250f114d37357d56608333fd167dd0fc"
+checksum = "ca46220853ba1c512fc82826d0834d87b06bcd3c2a42241b7de72f3d2fe17056"
 dependencies = [
 "cc",
 "libc",
@@ -571,9 +593,9 @@ dependencies = [

 [[package]]
 name = "libz-sys"
-version = "1.0.25"
+version = "1.1.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2eb5e43362e38e2bca2fd5f5134c4d4564a23a5c28e9b95411652021a8675ebe"
+checksum = "602113192b08db8f38796c4e85c39e960c145965140e918018bcde1952429655"
 dependencies = [
 "cc",
 "libc",
@@ -583,8 +605,8 @@ dependencies = [

 [[package]]
 name = "linux-loader"
-version = "0.1.0"
-source = "git+https://github.com/rust-vmm/linux-loader#b309c9df171bb6c3eba361382c54e82716c6824c"
+version = "0.2.0"
+source = "git+https://github.com/rust-vmm/linux-loader#3cf96c53ed6bfed9c7ca7dcfdbbcd04c86b6dd09"
 dependencies = [
 "vm-memory",
 ]
@@ -613,7 +635,7 @@ version = "0.4.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "4fabed175da42fed1fa0746b0ea71f412aa9d35e76e95e59b192c64b9dc2bf8b"
 dependencies = [
- "cfg-if",
+ "cfg-if 0.1.10",
 ]

 [[package]]
@@ -624,9 +646,9 @@ checksum = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08"

 [[package]]
 name = "memchr"
-version = "2.3.3"
+version = "2.3.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3728d817d99e5ac407411fa471ff9800a778d88a24685968b36824eaf4bee400"
+checksum = "0ee1c47aaa256ecabcaea351eae4a9b01ef39ed810004e298d2511ed284b1525"

 [[package]]
 name = "micro_http"
@@ -638,11 +660,12 @@ dependencies = [

 [[package]]
 name = "miniz_oxide"
-version = "0.4.0"
+version = "0.4.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "be0f75932c1f6cfae3c04000e40114adf955636e19040f9c0a2c380702aa1c7f"
+checksum = "0f2d26ec3309788e423cfbf68ad1800f061638098d76a83681af979dc4eda19d"
 dependencies = [
 "adler",
+ "autocfg",
 ]

 [[package]]
@@ -683,15 +706,15 @@ dependencies = [

 [[package]]
 name = "object"
-version = "0.20.0"
+version = "0.21.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1ab52be62400ca80aa00285d25253d7f7c437b7375c4de678f5405d3afe82ca5"
+checksum = "37fd5004feb2ce328a52b0b3d01dbf4ffff72583493900ed15f22d4111c51693"

 [[package]]
 name = "once_cell"
-version = "1.4.0"
+version = "1.4.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0b631f7e854af39a1739f401cf34a8a013dfe09eac4fa4dba91e9768bd28168d"
+checksum = "260e51e7efe62b592207e9e13a68e43692a7a279171d6ba57abd208bf23645ad"

 [[package]]
 name = "openssl-sys"
@@ -726,7 +749,7 @@ version = "0.7.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d58c7c768d4ba344e3e8d72518ac13e259d7c7ade24167003b8488e10b6740a3"
 dependencies = [
- "cfg-if",
+ "cfg-if 0.1.10",
 "cloudabi",
 "libc",
 "redox_syscall",
@@ -740,7 +763,6 @@ version = "0.1.0"
 dependencies = [
 "anyhow",
 "byteorder",
- "devices",
 "hypervisor",
 "libc",
 "log 0.4.11",
@@ -764,18 +786,18 @@ checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e"

 [[package]]
 name = "pin-project"
-version = "0.4.23"
+version = "1.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ca4433fff2ae79342e497d9f8ee990d174071408f28f726d6d83af93e58e48aa"
+checksum = "ee41d838744f60d959d7074e3afb6b35c7456d0f61cad38a24e35e6553f73841"
 dependencies = [
 "pin-project-internal",
 ]

 [[package]]
 name = "pin-project-internal"
-version = "0.4.23"
+version = "1.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2c0e815c3ee9a031fdf5af21c10aa17c573c9c6a566328d99e3936c34e36461f"
+checksum = "81a4ffa594b66bff340084d4081df649a7dc049ac8d7fc458d8e628bfbbb2f86"
 dependencies = [
 "proc-macro2",
 "quote",
@@ -790,9 +812,9 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"

 [[package]]
 name = "pkg-config"
-version = "0.3.18"
+version = "0.3.19"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d36492546b6af1463394d46f0c834346f31548646f6ba10849802c9c9a27ac33"
+checksum = "3831453b3449ceb48b6d9c7ad7c96d5ea673e9b470a1dc578c2ce6521230884c"

 [[package]]
 name = "pnet"
@@ -885,15 +907,15 @@ dependencies = [

 [[package]]
 name = "ppv-lite86"
-version = "0.2.8"
+version = "0.2.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "237a5ed80e274dbc66f86bd59c1e25edc039660be53194b5fe0a482e0f2612ea"
+checksum = "c36fa947111f5c62a733b652544dd0016a43ce89619538a8ef92724a6f501a20"

 [[package]]
 name = "proc-macro-hack"
-version = "0.5.18"
+version = "0.5.19"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "99c605b9a0adc77b7211c6b1f722dcb613d68d66859a44f3d485a6da332b0598"
+checksum = "dbf0c48bc1d91375ae5c3cd81e3722dff1abcf81a30960240640d223f59fe0e5"

 [[package]]
 name = "proc-macro-nested"
@@ -903,9 +925,9 @@ checksum = "eba180dafb9038b050a4c280019bbedf9f2467b61e5d892dcad585bb57aadc5a"

 [[package]]
 name = "proc-macro2"
-version = "1.0.19"
+version = "1.0.24"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "04f5f085b5d71e2188cb8271e5da0161ad52c3f227a661a3c135fdf28e258b12"
+checksum = "1e0704ee1a7e00d7bb417d0770ea303c1bccbabf0ef1667dae92b5967f5f8a71"
 dependencies = [
 "unicode-xid 0.2.1",
 ]
@@ -1017,9 +1039,9 @@ checksum = "41cc0f7e4d5d4544e8861606a285bb08d3e70712ccc7d2b84d7c0ccfaf4b05ce"

 [[package]]
 name = "redox_users"
-version = "0.3.4"
+version = "0.3.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "09b23093265f8d200fa7b4c2c76297f47e681c655f6f1285a8780d6a022f7431"
+checksum = "de0737333e7a9502c789a36d7c7fa6092a49895d4faa31ca5df163857ded2e9d"
 dependencies = [
 "getrandom",
 "redox_syscall",
@@ -1028,9 +1050,9 @@ dependencies = [

 [[package]]
 name = "regex"
-version = "1.3.9"
+version = "1.4.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9c3780fcf44b193bc4d09f36d2a3c87b251da4a046c87795a0d35f4f927ad8e6"
+checksum = "8963b85b8ce3074fecffde43b4b0dded83ce2f367dc8d363afc56679f3ee820b"
 dependencies = [
 "aho-corasick",
 "memchr",
@@ -1040,9 +1062,9 @@ dependencies = [

 [[package]]
 name = "regex-syntax"
-version = "0.6.18"
+version = "0.6.20"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "26412eb97c6b088a6997e05f69403a802a92d520de2f8e63c2b65f9e0f47c4e8"
+checksum = "8cab7a364d15cde1e505267766a2d3c4e22a843e1a601f0fa7564c0f82ced11c"

 [[package]]
 name = "remain"
@@ -1066,9 +1088,9 @@ dependencies = [

 [[package]]
 name = "rust-argon2"
-version = "0.7.0"
+version = "0.8.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2bc8af4bda8e1ff4932523b94d3dd20ee30a87232323eda55903ffd71d2fb017"
+checksum = "9dab61250775933275e84053ac235621dfb739556d5c54a2f2e9313b7cf43a19"
 dependencies = [
 "base64",
 "blake2b_simd",
@@ -1078,9 +1100,9 @@ dependencies = [

 [[package]]
 name = "rustc-demangle"
-version = "0.1.16"
+version = "0.1.18"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4c691c0e608126e00913e33f0ccf3727d5fc84573623b8d65b2df340b5201783"
+checksum = "6e3bad0ee36814ca07d7968269dd4b7ec89ec2da10c4bb613928d3077083c232"

 [[package]]
 name = "rustc-serialize"
@@ -1110,15 +1132,15 @@ dependencies = [

 [[package]]
 name = "serde"
-version = "1.0.114"
+version = "1.0.117"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5317f7588f0a5078ee60ef675ef96735a1442132dc645eb1d12c018620ed8cd3"
+checksum = "b88fa983de7720629c9387e9f517353ed404164b1e482c970a90c1a4aaf7dc1a"

 [[package]]
 name = "serde_derive"
-version = "1.0.114"
+version = "1.0.117"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2a0be94b04690fbaed37cddffc5c134bf537c8e3329d53e982fe04c374978f8e"
+checksum = "cbd1ae72adb44aab48f325a02444a5fc079349a8d804c1fc922aed3f7454c74e"
 dependencies = [
 "proc-macro2",
 "quote",
@@ -1127,9 +1149,9 @@ dependencies = [

 [[package]]
 name = "serde_json"
-version = "1.0.57"
+version = "1.0.59"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "164eacbdb13512ec2745fb09d51fd5b22b0d65ed294a1dcf7285a360c80a675c"
+checksum = "dcac07dbffa1c65e7f816ab9eba78eb142c6d44410f4eeba1e26e4f5dfa56b95"
 dependencies = [
 "itoa",
 "ryu",
@@ -1164,9 +1186,9 @@ checksum = "c111b5bd5695e56cffe5129854aa230b39c93a305372fdbb2668ca2394eea9f8"

 [[package]]
 name = "smallvec"
-version = "1.4.1"
+version = "1.4.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3757cb9d89161a2f24e1cf78efa0c1fcff485d18e3f55e0aa3480824ddaa0f3f"
+checksum = "fbee7696b84bbf3d89a1c2eccff0850e3047ed46bfcd2e92c29a2d074d57e252"

 [[package]]
 name = "ssh2"
@@ -1188,9 +1210,9 @@ checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a"

 [[package]]
 name = "syn"
-version = "1.0.38"
+version = "1.0.48"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e69abc24912995b3038597a7a593be5053eb0fb44f3cc5beec0deb421790c1f4"
+checksum = "cc371affeffc477f42a221a1e4297aedcea33d47d19b61455588bd9d8f6b19ac"
 dependencies = [
 "proc-macro2",
 "quote",
@@ -1274,7 +1296,7 @@ version = "3.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7a6e24d9338a0a5be79593e2fa15a648add6138caa803e2d5bc782c371732ca9"
 dependencies = [
- "cfg-if",
+ "cfg-if 0.1.10",
 "libc",
 "rand 0.7.3",
 "redox_syscall",
@@ -1314,18 +1336,18 @@ dependencies = [

 [[package]]
 name = "thiserror"
-version = "1.0.20"
+version = "1.0.21"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7dfdd070ccd8ccb78f4ad66bf1982dc37f620ef696c6b5028fe2ed83dd3d0d08"
+checksum = "318234ffa22e0920fe9a40d7b8369b5f649d490980cf7aadcf1eb91594869b42"
 dependencies = [
 "thiserror-impl",
 ]

 [[package]]
 name = "thiserror-impl"
-version = "1.0.20"
+version = "1.0.21"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bd80fc12f73063ac132ac92aceea36734f04a1d93c1240c6944e23a3b8841793"
+checksum = "cae2447b6282786c3493999f40a9be2a6ad20cb8bd268b0a0dbf5a065535c0ab"
 dependencies = [
 "proc-macro2",
 "quote",
@@ -1343,9 +1365,9 @@ dependencies = [

 [[package]]
 name = "tinyvec"
-version = "0.3.3"
+version = "0.3.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "53953d2d3a5ad81d9f844a32f14ebb121f50b650cd59d0ee2a07cf13c617efed"
+checksum = "238ce071d267c5710f9d31451efec16c5ee22de34df17cc05e56cbc92e967117"

 [[package]]
 name = "unicode-bidi"
@@ -1432,7 +1454,7 @@ dependencies = [
 [[package]]
 name = "vhost"
 version = "0.1.0"
-source = "git+https://github.com/cloud-hypervisor/vhost?branch=dragonball#422964150a69afd4611708076c60cb34c64a9856"
+source = "git+https://github.com/cloud-hypervisor/vhost?branch=ch#6fc980bf5083d67586ab91d6a965c3a593c33a78"
 dependencies = [
 "bitflags 1.2.1",
 "libc",
@@ -1523,7 +1545,6 @@ dependencies = [
 "arc-swap",
 "block_util",
 "byteorder",
- "devices",
 "epoll",
 "io-uring",
 "libc",
@@ -1571,8 +1592,8 @@ dependencies = [

 [[package]]
 name = "vm-memory"
-version = "0.2.2"
-source = "git+https://github.com/cloud-hypervisor/vm-memory?branch=ch#b28148d201b464acdef3124116ea43bb645cfd2f"
+version = "0.3.0"
+source = "git+https://github.com/cloud-hypervisor/vm-memory?branch=ch#6f2f7107f29a8f2b66b64a849aeace20783ec8f3"
 dependencies = [
 "arc-swap",
 "libc",
@@ -1610,6 +1631,7 @@ dependencies = [
 "anyhow",
 "arc-swap",
 "arch",
+ "bitflags 1.2.1",
 "block_util",
 "clap",
 "credibility",
@@ -1631,6 +1653,7 @@ dependencies = [
 "serde_json",
 "signal-hook",
 "tempfile",
+ "thiserror",
 "url",
 "vfio-ioctls",
 "virtio-devices",
@@ -1654,6 +1677,15 @@ dependencies = [
 "serde_derive",
 ]

+[[package]]
+name = "wait-timeout"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9f200f5b12eb75f8c1ed65abd4b2db8a6e1b138a20de009dacee265a2498f3f6"
+dependencies = [
+ "libc",
+]
+
 [[package]]
 name = "wasi"
 version = "0.9.0+wasi-snapshot-preview1"
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -1,28 +1,47 @@
 [package]
 name = "cloud-hypervisor"
-version = "0.9.0"
+version = "0.11.0"
 authors = ["The Cloud Hypervisor Authors"]
 edition = "2018"
 default-run = "cloud-hypervisor"
 build = "build.rs"
+license = "LICENSE-APACHE & LICENSE-BSD-3-Clause"
+description = "Open source Virtual Machine Monitor (VMM) that runs on top of KVM"
+homepage = "https://github.com/cloud-hypervisor/cloud-hypervisor"

 [profile.release]
 lto = true

 [dependencies]
-clap = { version = "2.33.2", features = ["wrap_help"] }
+anyhow = "1.0"
+api_client = { path = "api_client" }
+clap = { version = "2.33.3", features = ["wrap_help"] }
+epoll = ">=4.0.1"
 hypervisor = { path = "hypervisor" }
-libc = "0.2.74"
+libc = "0.2.80"
 log = { version = "0.4.11", features = ["std"] }
+option_parser = { path = "option_parser" }
 seccomp = { git = "https://github.com/firecracker-microvm/firecracker", tag = "v0.22.0" }
-serde_json = "1.0.57"
+serde_json = "1.0.59"
+thiserror = "1.0"
 vhost_user_block = { path = "vhost_user_block"}
 vhost_user_net = { path = "vhost_user_net"}
 vmm = { path = "vmm" }
 vmm-sys-util = "0.6.1"
+wait-timeout = "0.2.0"

+[build-dependencies]
+clap = { version = "2.33.3", features = ["wrap_help"] }
+
+# List of patched crates
 [patch.crates-io]
 vm-memory = { git = "https://github.com/cloud-hypervisor/vm-memory", branch = "ch" }
+[patch.'https://github.com/rust-vmm/vhost']
+vhost_rs = { git = "https://github.com/cloud-hypervisor/vhost", branch = "ch", package = "vhost", features = ["vhost-user-master", "vhost-user-slave"] }
+[patch.'https://github.com/rust-vmm/kvm-ioctls']
+kvm-ioctls = { git = "https://github.com/cloud-hypervisor/kvm-ioctls", branch = "ch" }
+[patch.'https://github.com/rust-vmm/kvm-bindings']
+kvm-bindings = { git = "https://github.com/cloud-hypervisor/kvm-bindings", branch = "ch", features = ["with-serde", "fam-wrappers"] }

 [dev-dependencies]
 ssh2 = "0.8.2"
@@ -31,14 +50,12 @@ credibility = "0.1.3"
 tempdir = "0.3.7"
 lazy_static= "1.4.0"
 tempfile = "3.1.0"
-serde_json = "1.0.57"
+serde_json = "1.0.59"
 net_util = { path = "net_util" }

 [features]
-default = ["acpi", "pci", "cmos", "kvm"]
+default = ["acpi", "cmos", "io_uring", "kvm"]
 acpi = ["vmm/acpi"]
-pci = ["vmm/pci_support"]
-mmio = ["vmm/mmio_support"]
 cmos = ["vmm/cmos"]
 fwdebug = ["vmm/fwdebug"]
 kvm = ["vmm/kvm"]
@@ -50,6 +67,7 @@ integration_tests = []
 [workspace]
 members = [
    "acpi_tables",
+    "api_client",
    "arch",
    "arch_gen",
    "block_util",
--- a/110
+++ b/110
@@ -24,28 +24,36 @@ pipeline{
 		stage ('Build') {
 			failFast true
            parallel {
-				stage ('Master build') {
-					agent { node { label 'master' } }
+				stage ('Worker build') {
+					agent { node { label 'groovy' } }
+					options {
+						timeout(time: 1, unit: 'HOURS')
+					}
 					stages {
 						stage ('Checkout') {
 							steps {
 								checkout scm
 							}
 						}
-						stage ('Run Cargo tests') {
-							steps {
-								sh "scripts/dev_cli.sh tests --cargo"
-							}
-						}
 						stage ('Run OpenAPI tests') {
 							steps {
 								sh "scripts/run_openapi_tests.sh"
 							}
 						}
+						stage ('Run unit tests') {
+							steps {
+								sh "scripts/dev_cli.sh tests --unit"
+							}
+						}
+						stage ('Run integration tests') {
+							steps {
+								sh "scripts/dev_cli.sh tests --integration"
+							}
+						}
 					}
-				}	
-				stage ('Worker build') {
-					agent { node { label 'bionic' } }
+				}
+				stage ('AArch64 worker build') {
+					agent { node { label 'bionic-arm64' } }
 					options {
 						timeout(time: 1, unit: 'HOURS')
 					}
@@ -61,17 +69,20 @@ pipeline{
 							}
 						}
 						stage ('Run integration tests') {
-							options {
-								retry(3)
-							}
 							steps {
 								sh "scripts/dev_cli.sh tests --integration"
 							}
 						}
 					}
+					post {
+						always {
+							sh "sudo chown -R jenkins.jenkins ${WORKSPACE}"
+							deleteDir()
+						}
+					}
 				}
 				stage ('Worker build (musl)') {
-					agent { node { label 'bionic' } }
+					agent { node { label 'groovy' } }
 					options {
 						timeout(time: 1, unit: 'HOURS')
 					}
@@ -87,15 +98,80 @@ pipeline{
 							}
 						}
 						stage ('Run integration tests for musl') {
-							options {
-								retry(3)
-							}
 							steps {
 								sh "scripts/dev_cli.sh tests --integration --libc musl"
 							}
 						}
 					}
 				}
+				stage ('Worker build SGX') {
+					agent { node { label 'bionic-sgx' } }
+					options {
+						timeout(time: 1, unit: 'HOURS')
+					}
+					when { branch 'master' }
+					stages {
+						stage ('Checkout') {
+							steps {
+								checkout scm
+							}
+						}
+						stage ('Run SGX integration tests') {
+							steps {
+								sh "scripts/dev_cli.sh tests --integration-sgx"
+							}
+						}
+						stage ('Run SGX integration tests for musl') {
+							steps {
+								sh "scripts/dev_cli.sh tests --integration-sgx --libc musl"
+							}
+						}
+					}
+					post {
+						always {
+							sh "sudo chown -R jenkins.jenkins ${WORKSPACE}"
+							deleteDir()
+						}
+					}
+				}
+				stage ('Worker build - Windows guest') {
+					agent { node { label 'groovy-win' } }
+					options {
+						timeout(time: 1, unit: 'HOURS')
+					}
+					stages {
+						stage ('Checkout') {
+							steps {
+								checkout scm
+							}
+						}
+						stage ('Download assets') {
+							steps {
+								sh "mkdir ${env.HOME}/workloads"
+								azureDownload(storageCredentialId: 'ch-image-store',
+											  containerName: 'private-images',
+											  includeFilesPattern: 'OVMF.fd',
+											  downloadType: 'container',
+											  downloadDirLoc: "${env.HOME}/workloads")
+								azureDownload(storageCredentialId: 'ch-image-store',
+											  containerName: 'private-images',
+											  includeFilesPattern: 'windows-server-2019.raw',
+											  downloadType: 'container',
+											  downloadDirLoc: "${env.HOME}/workloads")
+							}
+						}
+						stage ('Run Windows guest integration tests') {
+							steps {
+								sh "scripts/dev_cli.sh tests --integration-windows"
+							}
+						}
+						stage ('Run Windows guest integration tests for musl') {
+							steps {
+								sh "scripts/dev_cli.sh tests --integration-windows --libc musl"
+							}
+						}
+					}
+				}
 			}
 		}
 	}
--- a/README.md
+++ b/README.md
@@ -1,36 +1,34 @@
 [![Build Status](https://travis-ci.com/cloud-hypervisor/cloud-hypervisor.svg?branch=master)](https://travis-ci.com/cloud-hypervisor/cloud-hypervisor)

-1. [What is Cloud Hypervisor?](#1-what-is-cloud-hypervisor)
-   * [Requirements](#requirements)
-	 + [High Level](#high-level)
-     + [Architectures](#architectures)
-	 + [Guest OS](#guest-os)
-2. [Getting Started](#2-getting-started)
-   * [Clone and build](#clone-and-build)
-   * [Run](#run)
-	 + [Cloud image](#cloud-image)
-	 + [Custom kernel and disk image](#custom-kernel-and-disk-image)
-		 - [Building your kernel](#building-your-kernel)
-		 - [Disk image](#disk-image)
-		 - [Booting the guest VM](#booting-the-guest-vm)
-3. [Status](#2-status)
-	* [Device Model](#device-model)
-	* [TODO](#todo)
-4. [rust-vmm dependency](#4-rust-vmm-dependency)
-	* [Firecracker and crosvm](#firecracker-and-crosvm)
-5. [Community](#5-community)
-	* [Contribute](#contribute)
-	* [Join us](#join-us)
-6. [Security](#6-security)
+- [1. What is Cloud Hypervisor?](#1-what-is-cloud-hypervisor)
+  - [Objectives](#objectives)
+    - [High Level](#high-level)
+    - [Architectures](#architectures)
+    - [Guest OS](#guest-os)
+- [2. Getting Started](#2-getting-started)
+  - [Clone and build](#clone-and-build)
+    - [Containerized builds and tests](#containerized-builds-and-tests)
+  - [Run](#run)
+    - [Cloud image](#cloud-image)
+    - [Custom kernel and disk image](#custom-kernel-and-disk-image)
+      - [Building your kernel](#building-your-kernel)
+      - [Disk image](#disk-image)
+      - [Booting the guest VM](#booting-the-guest-vm)
+- [3. Status](#3-status)
+  - [Hot Plug](#hot-plug)
+  - [Device Model](#device-model)
+  - [TODO](#todo)
+- [4. `rust-vmm` project dependency](#4-rust-vmm-project-dependency)
+  - [Firecracker and crosvm](#firecracker-and-crosvm)
+- [5. Community](#5-community)
+  - [Contribute](#contribute)
+  - [Join us](#join-us)

 # 1. What is Cloud Hypervisor?

-**This project is an experiment and should not be used with production workloads.**
-
 Cloud Hypervisor is an open source Virtual Machine Monitor (VMM) that runs on top of [KVM](https://www.kernel.org/doc/Documentation/virtual/kvm/api.txt).
 The project focuses on exclusively running modern, cloud workloads, on top of a limited set of hardware architectures and platforms.
-Cloud workloads refers to those that are usually run by customers inside a cloud provider. For our purposes this means modern
-Linux* distributions with most I/O handled by paravirtualised devices (i.e. virtio), no requirement for legacy devices and recent CPUs and KVM.
+Cloud workloads refers to those that are usually run by customers inside a cloud provider. For our purposes this means modern operating systems with most I/O handled by paravirtualised devices (i.e. virtio), no requirement for legacy devices, and 64-bit CPUs.

 Cloud Hypervisor is implemented in [Rust](https://www.rust-lang.org/) and is based on the [rust-vmm](https://github.com/rust-vmm) crates.

@@ -38,27 +36,24 @@ Cloud Hypervisor is implemented in [Rust](https://www.rust-lang.org/) and is bas

 ### High Level

-* KVM and KVM only based
-* Minimal emulation
-* Low latency
-* Low memory footprint
-* Low complexity
-* High performance
-* Small attack surface
-* 64-bit support only
-* Build time configurable CPU, memory, PCI and NVDIMM hotplug
-* Machine to machine migration
+- KVM based
+- Minimal emulation
+- Low latency
+- Low memory footprint
+- Low complexity
+- High performance
+- Small attack surface
+- 64-bit support only
+- CPU, memory, PCI hotplug
+- Machine to machine migration

 ### Architectures

-`cloud-hypervisor` only supports the `x86-64` CPU architecture for now.
-
-We're planning to add support for the `AArch64` architecture in the future.
+Cloud Hypervisor supports the `x86-64` and `AArch64` architectures. There are some small differences in functionality between the two architectures (see [#1125](https://github.com/cloud-hypervisor/cloud-hypervisor/issues/1125)).

 ### Guest OS
-* `64-bit Linux`

-Support for *modern* 64-bit Windows guest is being evaluated.
+Cloud Hypervisor supports `64-bit Linux` with support for _modern_ 64-bit Windows guests currently under development.

 # 2. Getting Started

@@ -69,6 +64,23 @@ $ export CLOUDH=$HOME/cloud-hypervisor
 $ mkdir $CLOUDH
 ```

+## Install prerequisites
+
+You need to install some prerequisite packages in order to build and test Cloud Hypervisor.
+Here, all the steps are based on Ubuntu, for other Linux distributions please replace the
+package manager and package name.
+
+```shell
+# Install git
+$ sudo apt install git
+# Install rust tool chain
+$ curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh
+# Install build-essential
+$ sudo apt install build-essential
+# If you want to build statically linked binary please add musl target
+$ rustup target add x86_64-unknown-linux-musl
+```
+
 ## Clone and build

 First you need to clone and build the cloud-hypervisor repo:
@@ -82,6 +94,8 @@ $ cargo build --release
 # We need to give the cloud-hypervisor binary the NET_ADMIN capabilities for it to set TAP interfaces up on the host.
 $ sudo setcap cap_net_admin+ep ./target/release/cloud-hypervisor

+# If you want to build statically linked binary
+$ cargo build --release --target=x86_64-unknown-linux-musl --all
 $ popd
 ```

@@ -121,8 +135,8 @@ You can run a guest VM by either using an existing cloud image or booting into y

 ### Cloud image

-`cloud-hypervisor` supports booting disk images containing all needed
-components to run cloud workloads, a.k.a. cloud images.  To do that we rely on
+Cloud Hypervisor supports booting disk images containing all needed
+components to run cloud workloads, a.k.a. cloud images. To do that we rely on
 the [Rust Hypervisor
 Firmware](https://github.com/cloud-hypervisor/rust-hypervisor-firmware) project to provide
 an ELF
@@ -157,8 +171,7 @@ Multiple arguments can be given to the `--disk` parameter.

 #### Building your kernel

-`cloud-hypervisor` also supports direct kernel boot into a `vmlinux` ELF kernel
-image. In order to support virtio-fs and virtio-iommu we have our own development branch. You are of course able to use your own kernel but these instructions will continue with the version that we develop and test against.
+Cloud Hypervisor also supports direct kernel boot into a `vmlinux` ELF kernel or `bzImage`. In order to support virtio-fs and virtio-iommu we have our own development branch. You are of course able to use your own kernel but these instructions will continue with the version that we develop and test against.

 To build the kernel:

@@ -228,51 +241,52 @@ $ ./cloud-hypervisor/target/release/cloud-hypervisor \

 # 3. Status

-`cloud-hypervisor` is in a very early, pre-alpha stage. Use at your own risk!
+Cloud Hypervisor is under active development. No API or feature stability is guaranteed.

 As of 2020-07-02, the following cloud images are supported:
-* [Ubuntu Bionic](https://cloud-images.ubuntu.com/bionic/current/) (cloudimg)
-* [Ubuntu Focal](https://cloud-images.ubuntu.com/focal/current/) (cloudimg)

-Direct kernel boot to userspace should work with most rootfs.
+- [Ubuntu Bionic](https://cloud-images.ubuntu.com/bionic/current/) (cloudimg)
+- [Ubuntu Focal](https://cloud-images.ubuntu.com/focal/current/) (cloudimg)
+
+Direct kernel boot to userspace should work with a rootfs from most distributions.

 ## Hot Plug

-This [document](https://github.com/cloud-hypervisor/cloud-hypervisor/blob/master/docs/hotplug.md) details how to add devices to
-a running VM. Currently only CPU hot plug is supported.
+Cloud Hypervisor supports hotplug of CPUs, passthrough devices (VFIO), `virtio-{net,block,pmem,fs,vsock}` and memory resizing. This [document](https://github.com/cloud-hypervisor/cloud-hypervisor/blob/master/docs/hotplug.md) details how to add devices to
+a running VM.

 ## Device Model

-Follow this [documentation](https://github.com/cloud-hypervisor/cloud-hypervisor/blob/master/docs/device_model.md).
+Details of the device model can be found in this [documentation](https://github.com/cloud-hypervisor/cloud-hypervisor/blob/master/docs/device_model.md).

 ## TODO

-We are not tracking the `cloud-hypervisor` TODO list from a specific git tracked file but through
+We are not tracking the Cloud Hypervisor TODO list from a specific git tracked file but through
 [github issues](https://github.com/cloud-hypervisor/cloud-hypervisor/issues/new) instead.

 # 4. `rust-vmm` project dependency

 In order to satisfy the design goal of having a high-performance, security-focused hypervisor the decision
 was made to use the [Rust](https://www.rust-lang.org/) programming language.
-The language's strong focus on memory and thread safety makes it an ideal candidate for implementing VMMs
+The language's strong focus on memory and thread safety makes it an ideal candidate for implementing VMMs.

-Instead of implementing the VMM components from scratch, `cloud-hypervisor` is importing the [rust-vmm](https://github.com/rust-vmm)
+Instead of implementing the VMM components from scratch, Cloud Hypervisor is importing the [rust-vmm](https://github.com/rust-vmm)
 crates, and sharing code and architecture together with other VMMs like e.g. Amazon's [Firecracker](https://firecracker-microvm.github.io/)
 and Google's [crosvm](https://chromium.googlesource.com/chromiumos/platform/crosvm/).

-`cloud-hypervisor` embraces the rust-vmm project goals, which is to be able to share and re-use
-as many virtualization crates as possible. As such, the `cloud-hypervisor` relationship with the rust-vmm
+Cloud Hypervisor embraces the rust-vmm project goals, which is to be able to share and re-use
+as many virtualization crates as possible. As such, the Cloud Hypervisor relationship with the rust-vmm
 project is twofold:

 1. It will use as much of the rust-vmm code as possible. Any new rust-vmm crate that's relevant to the project
   goals will be integrated as soon as possible.
-2. As it is likely that the rust-vmm project will lack some of the features that `cloud-hypervisor` needs (e.g. ACPI,
-   VFIO, vhost-user, etc), we will be using the `cloud-hypervisor` VMM to implement and test them, and contribute them
+2. As it is likely that the rust-vmm project will lack some of the features that Cloud Hypervisor needs (e.g. ACPI,
+   VFIO, vhost-user, etc), we will be using the Cloud Hypervisor VMM to implement and test them, and contribute them
   back to the rust-vmm project.

 ## Firecracker and crosvm

-A large part of the `cloud-hypervisor` code is based on either the Firecracker or the crosvm projects implementations.
+A large part of the Cloud Hypervisor code is based on either the Firecracker or the crosvm projects implementations.
 Both of these are VMMs written in Rust with a focus on safety and security, like Cloud Hypervisor.

 However we want to emphasize that the Cloud Hypervisor project is neither a fork nor a reimplementation of any of those
@@ -303,27 +317,3 @@ etc, are all equal and welcome means of contribution. See the [CONTRIBUTING](CON

 Get an [invite to our Slack channel](https://join.slack.com/t/cloud-hypervisor/shared_invite/enQtNjY3MTE3MDkwNDQ4LWQ1MTA1ZDVmODkwMWQ1MTRhYzk4ZGNlN2UwNTI3ZmFlODU0OTcwOWZjMTkwZDExYWE3YjFmNzgzY2FmNDAyMjI)
 and [join us on Slack](https://cloud-hypervisor.slack.com/).
-
-# 6. Security
-
-**Reporting a Potential Security Vulnerability**: If you have discovered
-potential security vulnerability in this project, please send an e-mail to
-secure@intel.com. For issues related to Intel Products, please visit
-https://security-center.intel.com.
-
-It is important to include the following details:
-  - The projects and versions affected
-  - Detailed description of the vulnerability
-  - Information on known exploits
-
-Vulnerability information is extremely sensitive. Please encrypt all security
-vulnerability reports using our *PGP key*
-
-A member of the Intel Product Security Team will review your e-mail and
-contact you to to collaborate on resolving the issue. For more information on
-how Intel works to resolve security issues, see: *Vulnerability Handling
-Guidelines*
-
-PGP Key: https://www.intel.com/content/www/us/en/security-center/pgp-public-key.html
-
-Vulnerability Handling Guidelines: https://www.intel.com/content/www/us/en/security-center/vulnerability-handling-guidelines.html
--- a/acpi_tables/Cargo.toml
+++ b/acpi_tables/Cargo.toml
@@ -5,5 +5,5 @@ authors = ["The Cloud Hypervisor Authors"]
 edition = "2018"

 [dependencies]
-vm-memory = "0.2.1"
+vm-memory = "0.3.0"

--- a/api_client/Cargo.toml
+++ b/api_client/Cargo.toml
@@ -0,0 +1,5 @@
+[package]
+name = "api_client"
+version = "0.1.0"
+authors = ["The Cloud Hypervisor Authors"]
+edition = "2018"
--- a/api_client/src/lib.rs
+++ b/api_client/src/lib.rs
@@ -0,0 +1,175 @@
+// Copyright © 2020 Intel Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+use std::fmt;
+use std::io::{Read, Write};
+
+#[derive(Debug)]
+pub enum Error {
+    Socket(std::io::Error),
+    StatusCodeParsing(std::num::ParseIntError),
+    MissingProtocol,
+    ContentLengthParsing(std::num::ParseIntError),
+    ServerResponse(StatusCode, Option<String>),
+}
+
+impl fmt::Display for Error {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        use Error::*;
+        match self {
+            Socket(e) => write!(f, "Error writing to or reading from HTTP socket: {}", e),
+            StatusCodeParsing(e) => write!(f, "Error parsing HTTP status code: {}", e),
+            MissingProtocol => write!(f, "HTTP output is missing protocol statement"),
+            ContentLengthParsing(e) => write!(f, "Error parsing HTTP Content-Length field: {}", e),
+            ServerResponse(s, o) => {
+                if let Some(o) = o {
+                    write!(f, "Server responded with an error: {:?}: {}", s, o)
+                } else {
+                    write!(f, "Server responded with an error: {:?}", s)
+                }
+            }
+        }
+    }
+}
+
+#[derive(Clone, Copy, Debug)]
+pub enum StatusCode {
+    Continue,
+    OK,
+    NoContent,
+    BadRequest,
+    NotFound,
+    InternalServerError,
+    NotImplemented,
+    Unknown,
+}
+
+impl StatusCode {
+    fn from_raw(code: usize) -> StatusCode {
+        match code {
+            100 => StatusCode::Continue,
+            200 => StatusCode::OK,
+            204 => StatusCode::NoContent,
+            400 => StatusCode::BadRequest,
+            404 => StatusCode::NotFound,
+            500 => StatusCode::InternalServerError,
+            501 => StatusCode::NotImplemented,
+            _ => StatusCode::Unknown,
+        }
+    }
+
+    fn parse(code: &str) -> Result<StatusCode, Error> {
+        Ok(StatusCode::from_raw(
+            code.trim().parse().map_err(Error::StatusCodeParsing)?,
+        ))
+    }
+
+    fn is_server_error(self) -> bool {
+        !matches!(
+            self,
+            StatusCode::OK | StatusCode::Continue | StatusCode::NoContent
+        )
+    }
+}
+
+fn get_header<'a>(res: &'a str, header: &'a str) -> Option<&'a str> {
+    let header_str = format!("{}: ", header);
+    if let Some(o) = res.find(&header_str) {
+        Some(&res[o + header_str.len()..o + res[o..].find('\r').unwrap()])
+    } else {
+        None
+    }
+}
+
+fn get_status_code(res: &str) -> Result<StatusCode, Error> {
+    if let Some(o) = res.find("HTTP/1.1") {
+        Ok(StatusCode::parse(
+            &res[o + "HTTP/1.1 ".len()..res[o..].find('\r').unwrap()],
+        )?)
+    } else {
+        Err(Error::MissingProtocol)
+    }
+}
+
+fn parse_http_response(socket: &mut dyn Read) -> Result<Option<String>, Error> {
+    let mut res = String::new();
+    let mut body_offset = None;
+    let mut content_length: Option<usize> = None;
+    loop {
+        let mut bytes = vec![0; 256];
+        let count = socket.read(&mut bytes).map_err(Error::Socket)?;
+        res.push_str(std::str::from_utf8(&bytes[0..count]).unwrap());
+
+        // End of headers
+        if let Some(o) = res.find("\r\n\r\n") {
+            body_offset = Some(o + "\r\n\r\n".len());
+
+            // With all headers available we can see if there is any body
+            content_length = if let Some(length) = get_header(&res, "Content-Length") {
+                Some(length.trim().parse().map_err(Error::ContentLengthParsing)?)
+            } else {
+                None
+            };
+
+            if content_length.is_none() {
+                break;
+            }
+        }
+
+        if let Some(body_offset) = body_offset {
+            if let Some(content_length) = content_length {
+                if res.len() >= content_length + body_offset {
+                    break;
+                }
+            }
+        }
+    }
+    let body_string = content_length.and(Some(String::from(&res[body_offset.unwrap()..])));
+    let status_code = get_status_code(&res)?;
+
+    if status_code.is_server_error() {
+        Err(Error::ServerResponse(status_code, body_string))
+    } else {
+        Ok(body_string)
+    }
+}
+
+pub fn simple_api_command<T: Read + Write>(
+    socket: &mut T,
+    method: &str,
+    c: &str,
+    request_body: Option<&str>,
+) -> Result<(), Error> {
+    socket
+        .write_all(
+            format!(
+                "{} /api/v1/vm.{} HTTP/1.1\r\nHost: localhost\r\nAccept: */*\r\n",
+                method, c
+            )
+            .as_bytes(),
+        )
+        .map_err(Error::Socket)?;
+
+    if let Some(request_body) = request_body {
+        socket
+            .write_all(format!("Content-Length: {}\r\n", request_body.len()).as_bytes())
+            .map_err(Error::Socket)?;
+    }
+
+    socket.write_all(b"\r\n").map_err(Error::Socket)?;
+
+    if let Some(request_body) = request_body {
+        socket
+            .write_all(request_body.as_bytes())
+            .map_err(Error::Socket)?;
+    }
+
+    socket.flush().map_err(Error::Socket)?;
+
+    if let Some(body) = parse_http_response(socket)? {
+        println!("{}", body);
+    }
+    Ok(())
+}
--- a/arch/Cargo.toml
+++ b/arch/Cargo.toml
@@ -11,9 +11,13 @@ acpi = ["acpi_tables"]
 anyhow = "1.0"
 byteorder = "1.3.4"
 hypervisor = { path = "../hypervisor" }
-libc = "0.2.74"
+libc = "0.2.80"
 log = "0.4.11"
-vm-memory = { version = "0.2.1", features = ["backend-mmap"] }
+serde = {version = ">=1.0.27", features = ["rc"] }
+serde_derive = ">=1.0.27"
+serde_json = ">=1.0.9"
+vm-memory = { version = "0.3.0", features = ["backend-mmap"] }
+vm-migration = { path = "../vm-migration" }
 acpi_tables = { path = "../acpi_tables", optional = true }
 arch_gen = { path = "../arch_gen" }

--- a/arch/src/aarch64/fdt.rs
+++ b/arch/src/aarch64/fdt.rs
@@ -90,16 +90,16 @@ pub enum Error {
 type Result<T> = result::Result<T, Error>;

 /// Creates the flattened device tree for this aarch64 VM.
-pub fn create_fdt<T: DeviceInfoForFDT + Clone + Debug>(
+pub fn create_fdt<T: DeviceInfoForFDT + Clone + Debug, S: ::std::hash::BuildHasher>(
    guest_mem: &GuestMemoryMmap,
    cmdline: &CStr,
    vcpu_mpidr: Vec<u64>,
-    device_info: &HashMap<(DeviceType, String), T>,
-    gic_device: &Box<dyn GICDevice>,
+    device_info: &HashMap<(DeviceType, String), T, S>,
+    gic_device: &dyn GICDevice,
    initrd: &Option<InitramfsConfig>,
-    pci_space_address: &Option<(u64, u64)>,
+    pci_space_address: &(u64, u64),
 ) -> Result<Vec<u8>> {
-    // Alocate stuff necessary for the holding the blob.
+    // Allocate stuff necessary for the holding the blob.
    let mut fdt = vec![0; FDT_MAX_SIZE];

    allocate_fdt(&mut fdt)?;
@@ -126,9 +126,7 @@ pub fn create_fdt<T: DeviceInfoForFDT + Clone + Debug>(
    create_clock_node(&mut fdt)?;
    create_psci_node(&mut fdt)?;
    create_devices_node(&mut fdt, device_info)?;
-    if let Some((pci_device_base, pci_device_size)) = pci_space_address {
-        create_pci_nodes(&mut fdt, *pci_device_base, *pci_device_size)?;
-    }
+    create_pci_nodes(&mut fdt, pci_space_address.0, pci_space_address.1)?;

    // End Header node.
    append_end_node(&mut fdt)?;
@@ -311,7 +309,7 @@ fn generate_prop64(cells: &[u64]) -> Vec<u8> {
 }

 // Following are the auxiliary function for creating the different nodes that we append to our FDT.
-fn create_cpu_nodes(fdt: &mut Vec<u8>, vcpu_mpidr: &Vec<u64>) -> Result<()> {
+fn create_cpu_nodes(fdt: &mut Vec<u8>, vcpu_mpidr: &[u64]) -> Result<()> {
    // See https://github.com/torvalds/linux/blob/master/Documentation/devicetree/bindings/arm/cpus.yaml.
    append_begin_node(fdt, "cpus")?;
    // As per documentation, on ARM v8 64-bit systems value should be set to 2.
@@ -319,7 +317,7 @@ fn create_cpu_nodes(fdt: &mut Vec<u8>, vcpu_mpidr: &Vec<u64>) -> Result<()> {
    append_property_u32(fdt, "#size-cells", 0x0)?;
    let num_cpus = vcpu_mpidr.len();

-    for cpu_index in 0..num_cpus {
+    for (cpu_index, mpidr) in vcpu_mpidr.iter().enumerate().take(num_cpus) {
        let cpu_name = format!("cpu@{:x}", cpu_index);
        append_begin_node(fdt, &cpu_name)?;
        append_property_string(fdt, "device_type", "cpu")?;
@@ -330,7 +328,7 @@ fn create_cpu_nodes(fdt: &mut Vec<u8>, vcpu_mpidr: &Vec<u64>) -> Result<()> {
        }
        // Set the field to first 24 bits of the MPIDR - Multiprocessor Affinity Register.
        // See http://infocenter.arm.com/help/index.jsp?topic=/com.arm.doc.ddi0488c/BABHBJCI.html.
-        append_property_u64(fdt, "reg", vcpu_mpidr[cpu_index] & 0x7FFFFF)?;
+        append_property_u64(fdt, "reg", mpidr & 0x7FFFFF)?;
        append_end_node(fdt)?;
    }
    append_end_node(fdt)?;
@@ -376,7 +374,7 @@ fn create_chosen_node(
    Ok(())
 }

-fn create_gic_node(fdt: &mut Vec<u8>, gic_device: &Box<dyn GICDevice>) -> Result<()> {
+fn create_gic_node(fdt: &mut Vec<u8>, gic_device: &dyn GICDevice) -> Result<()> {
    let gic_reg_prop = generate_prop64(gic_device.device_properties());

    append_begin_node(fdt, "intc")?;
@@ -402,7 +400,7 @@ fn create_gic_node(fdt: &mut Vec<u8>, gic_device: &Box<dyn GICDevice>) -> Result

    if gic_device.msi_compatible() {
        append_begin_node(fdt, "msic")?;
-        append_property_string(fdt, "compatible", gic_device.msi_compatiblility())?;
+        append_property_string(fdt, "compatible", gic_device.msi_compatibility())?;
        append_property_null(fdt, "msi-controller")?;
        append_property_u32(fdt, "phandle", MSI_PHANDLE)?;
        let msi_reg_prop = generate_prop64(gic_device.msi_properties());
@@ -521,9 +519,9 @@ fn create_rtc_node<T: DeviceInfoForFDT + Clone + Debug>(
    Ok(())
 }

-fn create_devices_node<T: DeviceInfoForFDT + Clone + Debug>(
+fn create_devices_node<T: DeviceInfoForFDT + Clone + Debug, S: ::std::hash::BuildHasher>(
    fdt: &mut Vec<u8>,
-    dev_info: &HashMap<(DeviceType, String), T>,
+    dev_info: &HashMap<(DeviceType, String), T, S>,
 ) -> Result<()> {
    // Create one temp Vec to store all virtio devices
    let mut ordered_virtio_device: Vec<&T> = Vec::new();
@@ -539,7 +537,7 @@ fn create_devices_node<T: DeviceInfoForFDT + Clone + Debug>(
    }

    // Sort out virtio devices by address from low to high and insert them into fdt table.
-    ordered_virtio_device.sort_by(|a, b| a.addr().cmp(&b.addr()));
+    ordered_virtio_device.sort_by_key(|&a| a.addr());
    // Current address allocation strategy in cloud-hypervisor is: the first created device
    // will be allocated to higher address. Here we reverse the vector to make sure that
    // the older created device will appear in front of the newer created device in FDT.
--- a/arch/src/aarch64/gic/dist_regs.rs
+++ b/arch/src/aarch64/gic/dist_regs.rs
@@ -0,0 +1,182 @@
+// Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+use super::{Error, Result};
+use crate::layout::IRQ_BASE;
+use hypervisor::kvm::kvm_bindings::{
+    kvm_device_attr, KVM_DEV_ARM_VGIC_GRP_DIST_REGS, KVM_DEV_ARM_VGIC_GRP_NR_IRQS,
+};
+use std::sync::Arc;
+
+/*
+ Distributor registers as detailed at page 456 from
+ https://static.docs.arm.com/ihi0069/c/IHI0069C_gic_architecture_specification.pdf.
+ Address offsets are relative to the Distributor base address defined
+by the system memory map. Unless otherwise stated in the register description,
+all GIC registers are 32-bits wide.
+ */
+const GICD_CTLR: u32 = 0x0;
+const GICD_STATUSR: u32 = 0x0010;
+const GICD_IGROUPR: u32 = 0x0080;
+const GICD_ISENABLER: u32 = 0x0100;
+const GICD_ICENABLER: u32 = 0x0180;
+const GICD_ISPENDR: u32 = 0x0200;
+const GICD_ICPENDR: u32 = 0x0280;
+const GICD_ISACTIVER: u32 = 0x0300;
+const GICD_ICACTIVER: u32 = 0x0380;
+const GICD_IPRIORITYR: u32 = 0x0400;
+const GICD_ICFGR: u32 = 0x0C00;
+const GICD_IROUTER: u32 = 0x6000;
+
+/// This is how we represent the registers of the vgic's distributor.
+/// Some of the distributor register )(i.e GICD_STATUSR) are simple
+/// registers (i.e they are associated to a 32 bit value).
+/// However, there are other registers that have variable lengths since
+/// they dedicate some of the 32 bits to some specific interrupt. So, their length
+/// depends on the number of interrupts (i.e the ones that are represented as GICD_REG<n>)
+/// in the documentation mentioned above.
+struct DistReg {
+    /// Offset from distributor address.
+    base: u32,
+    /// Bits per interrupt.
+    /// Relevant for registers that DO share IRQs.
+    bpi: u8,
+    /// Length of the register.
+    /// Relevant for registers that DO NOT share IRQs.
+    length: u16,
+}
+
+// All or at least the registers we are interested in are 32 bit, so
+// we use a constant for size(u32).
+const REG_SIZE: u8 = 4;
+
+// Creates a vgic distributor register.
+macro_rules! VGIC_DIST_REG {
+    ($base:expr, $bpi:expr, $length:expr) => {
+        DistReg {
+            base: $base,
+            bpi: $bpi,
+            length: $length,
+        }
+    };
+}
+
+// List with relevant distributor registers that we will be restoring.
+// Order is taken from qemu.
+static VGIC_DIST_REGS: &'static [DistReg] = &[
+    VGIC_DIST_REG!(GICD_STATUSR, 0, 4),
+    VGIC_DIST_REG!(GICD_ICENABLER, 1, 0),
+    VGIC_DIST_REG!(GICD_ISENABLER, 1, 0),
+    VGIC_DIST_REG!(GICD_IGROUPR, 1, 0),
+    VGIC_DIST_REG!(GICD_IROUTER, 64, 0),
+    VGIC_DIST_REG!(GICD_ICFGR, 2, 0),
+    VGIC_DIST_REG!(GICD_ICPENDR, 1, 0),
+    VGIC_DIST_REG!(GICD_ISPENDR, 1, 0),
+    VGIC_DIST_REG!(GICD_ICACTIVER, 1, 0),
+    VGIC_DIST_REG!(GICD_ISACTIVER, 1, 0),
+    VGIC_DIST_REG!(GICD_IPRIORITYR, 8, 0),
+];
+
+fn dist_attr_access(
+    gic: &Arc<dyn hypervisor::Device>,
+    offset: u32,
+    val: &u32,
+    set: bool,
+) -> Result<()> {
+    let mut gic_dist_attr = kvm_device_attr {
+        group: KVM_DEV_ARM_VGIC_GRP_DIST_REGS,
+        attr: offset as u64,
+        addr: val as *const u32 as u64,
+        flags: 0,
+    };
+    if set {
+        gic.set_device_attr(&gic_dist_attr)
+            .map_err(Error::SetDeviceAttribute)?;
+    } else {
+        gic.get_device_attr(&mut gic_dist_attr)
+            .map_err(Error::GetDeviceAttribute)?;
+    }
+    Ok(())
+}
+
+/// Get the distributor control register.
+pub fn read_ctlr(gic: &Arc<dyn hypervisor::Device>) -> Result<u32> {
+    let val: u32 = 0;
+    dist_attr_access(gic, GICD_CTLR, &val, false)?;
+    Ok(val)
+}
+
+/// Set the distributor control register.
+pub fn write_ctlr(gic: &Arc<dyn hypervisor::Device>, val: u32) -> Result<()> {
+    dist_attr_access(gic, GICD_CTLR, &val, true)
+}
+
+fn get_interrupts_num(gic: &Arc<dyn hypervisor::Device>) -> Result<u32> {
+    let num_irq = 0;
+
+    let mut nr_irqs_attr = kvm_device_attr {
+        group: KVM_DEV_ARM_VGIC_GRP_NR_IRQS,
+        attr: 0,
+        addr: &num_irq as *const u32 as u64,
+        flags: 0,
+    };
+    gic.get_device_attr(&mut nr_irqs_attr)
+        .map_err(Error::GetDeviceAttribute)?;
+    Ok(num_irq)
+}
+
+fn compute_reg_len(gic: &Arc<dyn hypervisor::Device>, reg: &DistReg, base: u32) -> Result<u32> {
+    let mut end = base;
+    let num_irq = get_interrupts_num(gic)?;
+    if reg.length > 0 {
+        // This is the single type register (i.e one that is not DIST_X<n>) and for which
+        // the bpi is 0.
+        // Look in the kernel for REGISTER_DESC_WITH_LENGTH.
+        end = base + reg.length as u32;
+    }
+    if reg.bpi > 0 {
+        // This is the type of register that takes into account the number of interrupts
+        // that the model has. It is also the type of register where
+        // a register relates to multiple interrupts.
+        end = base + (reg.bpi as u32 * (num_irq - IRQ_BASE) / 8);
+        if reg.bpi as u32 * (num_irq - IRQ_BASE) % 8 > 0 {
+            end += REG_SIZE as u32;
+        }
+    }
+    Ok(end)
+}
+
+/// Set distributor registers of the GIC.
+pub fn set_dist_regs(gic: &Arc<dyn hypervisor::Device>, state: &[u32]) -> Result<()> {
+    let mut idx = 0;
+
+    for dreg in VGIC_DIST_REGS {
+        let mut base = dreg.base + REG_SIZE as u32 * dreg.bpi as u32;
+        let end = compute_reg_len(gic, &dreg, base)?;
+
+        while base < end {
+            let val = state[idx];
+            dist_attr_access(gic, base, &val, true)?;
+            idx += 1;
+            base += REG_SIZE as u32;
+        }
+    }
+    Ok(())
+}
+/// Get distributor registers of the GIC.
+pub fn get_dist_regs(gic: &Arc<dyn hypervisor::Device>) -> Result<Vec<u32>> {
+    let mut state = Vec::new();
+
+    for dreg in VGIC_DIST_REGS {
+        let mut base = dreg.base + REG_SIZE as u32 * dreg.bpi as u32;
+        let end = compute_reg_len(gic, &dreg, base)?;
+
+        while base < end {
+            let val: u32 = 0;
+            dist_attr_access(gic, base, &val, false)?;
+            state.push(val);
+            base += REG_SIZE as u32;
+        }
+    }
+    Ok(state)
+}
--- a/arch/src/aarch64/gic/gicv3.rs
+++ b/arch/src/aarch64/gic/gicv3.rs
@@ -0,0 +1,275 @@
+// Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+pub mod kvm {
+    use crate::aarch64::gic::dist_regs::{get_dist_regs, read_ctlr, set_dist_regs, write_ctlr};
+    use crate::aarch64::gic::icc_regs::{get_icc_regs, set_icc_regs};
+    use crate::aarch64::gic::kvm::{save_pending_tables, KvmGICDevice};
+    use crate::aarch64::gic::redist_regs::{get_redist_regs, set_redist_regs};
+    use crate::aarch64::gic::GICDevice;
+    use crate::layout;
+    use anyhow::anyhow;
+    use hypervisor::kvm::kvm_bindings;
+    use std::any::Any;
+    use std::convert::TryInto;
+    use std::sync::Arc;
+    use std::{boxed::Box, result};
+    use vm_migration::{
+        Migratable, MigratableError, Pausable, Snapshot, SnapshotDataSection, Snapshottable,
+        Transportable,
+    };
+
+    /// Errors thrown while saving/restoring the GICv3.
+    #[derive(Debug)]
+    pub enum Error {
+        /// Error in saving RDIST pending tables into guest RAM.
+        SavePendingTables(crate::aarch64::gic::Error),
+        /// Error in saving GIC distributor registers.
+        SaveDistributorRegisters(crate::aarch64::gic::Error),
+        /// Error in restoring GIC distributor registers.
+        RestoreDistributorRegisters(crate::aarch64::gic::Error),
+        /// Error in saving GIC distributor control registers.
+        SaveDistributorCtrlRegisters(crate::aarch64::gic::Error),
+        /// Error in restoring GIC distributor control registers.
+        RestoreDistributorCtrlRegisters(crate::aarch64::gic::Error),
+        /// Error in saving GIC redistributor registers.
+        SaveRedistributorRegisters(crate::aarch64::gic::Error),
+        /// Error in restoring GIC redistributor registers.
+        RestoreRedistributorRegisters(crate::aarch64::gic::Error),
+        /// Error in saving GIC CPU interface registers.
+        SaveICCRegisters(crate::aarch64::gic::Error),
+        /// Error in restoring GIC CPU interface registers.
+        RestoreICCRegisters(crate::aarch64::gic::Error),
+    }
+
+    type Result<T> = result::Result<T, Error>;
+
+    pub struct KvmGICv3 {
+        /// The hypervisor agnostic device
+        device: Arc<dyn hypervisor::Device>,
+
+        /// Vector holding values of GICR_TYPER for each vCPU
+        gicr_typers: Vec<u64>,
+
+        /// GIC device properties, to be used for setting up the fdt entry
+        properties: [u64; 4],
+
+        /// Number of CPUs handled by the device
+        vcpu_count: u64,
+    }
+
+    #[derive(Serialize, Deserialize)]
+    pub struct Gicv3State {
+        dist: Vec<u32>,
+        rdist: Vec<u32>,
+        icc: Vec<u32>,
+        // special register that enables interrupts and affinity routing
+        gicd_ctlr: u32,
+    }
+
+    impl KvmGICv3 {
+        // Unfortunately bindgen omits defines that are based on other defines.
+        // See arch/arm64/include/uapi/asm/kvm.h file from the linux kernel.
+        pub const SZ_64K: u64 = 0x0001_0000;
+        const KVM_VGIC_V3_DIST_SIZE: u64 = KvmGICv3::SZ_64K;
+        const KVM_VGIC_V3_REDIST_SIZE: u64 = (2 * KvmGICv3::SZ_64K);
+
+        // Device trees specific constants
+        pub const ARCH_GIC_V3_MAINT_IRQ: u32 = 9;
+
+        /// Get the address of the GIC distributor.
+        pub fn get_dist_addr() -> u64 {
+            layout::MAPPED_IO_START - KvmGICv3::KVM_VGIC_V3_DIST_SIZE
+        }
+
+        /// Get the size of the GIC distributor.
+        pub fn get_dist_size() -> u64 {
+            KvmGICv3::KVM_VGIC_V3_DIST_SIZE
+        }
+
+        /// Get the address of the GIC redistributors.
+        pub fn get_redists_addr(vcpu_count: u64) -> u64 {
+            KvmGICv3::get_dist_addr() - KvmGICv3::get_redists_size(vcpu_count)
+        }
+
+        /// Get the size of the GIC redistributors.
+        pub fn get_redists_size(vcpu_count: u64) -> u64 {
+            vcpu_count * KvmGICv3::KVM_VGIC_V3_REDIST_SIZE
+        }
+
+        /// Save the state of GIC.
+        fn state(&self, gicr_typers: &[u64]) -> Result<Gicv3State> {
+            // Flush redistributors pending tables to guest RAM.
+            save_pending_tables(&self.device()).map_err(Error::SavePendingTables)?;
+
+            let gicd_ctlr =
+                read_ctlr(&self.device()).map_err(Error::SaveDistributorCtrlRegisters)?;
+
+            let dist_state =
+                get_dist_regs(&self.device()).map_err(Error::SaveDistributorRegisters)?;
+
+            let rdist_state = get_redist_regs(&self.device(), &gicr_typers)
+                .map_err(Error::SaveRedistributorRegisters)?;
+
+            let icc_state =
+                get_icc_regs(&self.device(), &gicr_typers).map_err(Error::SaveICCRegisters)?;
+
+            Ok(Gicv3State {
+                dist: dist_state,
+                rdist: rdist_state,
+                icc: icc_state,
+                gicd_ctlr,
+            })
+        }
+
+        /// Restore the state of GIC.
+        fn set_state(&mut self, gicr_typers: &[u64], state: &Gicv3State) -> Result<()> {
+            write_ctlr(&self.device(), state.gicd_ctlr)
+                .map_err(Error::RestoreDistributorCtrlRegisters)?;
+
+            set_dist_regs(&self.device(), &state.dist)
+                .map_err(Error::RestoreDistributorRegisters)?;
+
+            set_redist_regs(&self.device(), gicr_typers, &state.rdist)
+                .map_err(Error::RestoreRedistributorRegisters)?;
+
+            set_icc_regs(&self.device(), &gicr_typers, &state.icc)
+                .map_err(Error::RestoreICCRegisters)?;
+
+            Ok(())
+        }
+    }
+
+    impl GICDevice for KvmGICv3 {
+        fn device(&self) -> &Arc<dyn hypervisor::Device> {
+            &self.device
+        }
+
+        fn fdt_compatibility(&self) -> &str {
+            "arm,gic-v3"
+        }
+
+        fn fdt_maint_irq(&self) -> u32 {
+            KvmGICv3::ARCH_GIC_V3_MAINT_IRQ
+        }
+
+        fn device_properties(&self) -> &[u64] {
+            &self.properties
+        }
+
+        fn vcpu_count(&self) -> u64 {
+            self.vcpu_count
+        }
+
+        fn set_gicr_typers(&mut self, gicr_typers: Vec<u64>) {
+            self.gicr_typers = gicr_typers;
+        }
+
+        fn as_any_concrete_mut(&mut self) -> &mut dyn Any {
+            self
+        }
+    }
+
+    impl KvmGICDevice for KvmGICv3 {
+        fn version() -> u32 {
+            kvm_bindings::kvm_device_type_KVM_DEV_TYPE_ARM_VGIC_V3
+        }
+
+        fn create_device(
+            device: Arc<dyn hypervisor::Device>,
+            vcpu_count: u64,
+        ) -> Box<dyn GICDevice> {
+            Box::new(KvmGICv3 {
+                device,
+                gicr_typers: vec![0; vcpu_count.try_into().unwrap()],
+                properties: [
+                    KvmGICv3::get_dist_addr(),
+                    KvmGICv3::get_dist_size(),
+                    KvmGICv3::get_redists_addr(vcpu_count),
+                    KvmGICv3::get_redists_size(vcpu_count),
+                ],
+                vcpu_count,
+            })
+        }
+
+        fn init_device_attributes(
+            _vm: &Arc<dyn hypervisor::Vm>,
+            gic_device: &dyn GICDevice,
+        ) -> crate::aarch64::gic::Result<()> {
+            /* Setting up the distributor attribute.
+             We are placing the GIC below 1GB so we need to substract the size of the distributor.
+            */
+            Self::set_device_attribute(
+                gic_device.device(),
+                kvm_bindings::KVM_DEV_ARM_VGIC_GRP_ADDR,
+                u64::from(kvm_bindings::KVM_VGIC_V3_ADDR_TYPE_DIST),
+                &KvmGICv3::get_dist_addr() as *const u64 as u64,
+                0,
+            )?;
+
+            /* Setting up the redistributors' attribute.
+            We are calculating here the start of the redistributors address. We have one per CPU.
+            */
+            Self::set_device_attribute(
+                gic_device.device(),
+                kvm_bindings::KVM_DEV_ARM_VGIC_GRP_ADDR,
+                u64::from(kvm_bindings::KVM_VGIC_V3_ADDR_TYPE_REDIST),
+                &KvmGICv3::get_redists_addr(gic_device.vcpu_count()) as *const u64 as u64,
+                0,
+            )?;
+
+            Ok(())
+        }
+    }
+
+    pub const GIC_V3_SNAPSHOT_ID: &str = "gic-v3";
+    impl Snapshottable for KvmGICv3 {
+        fn id(&self) -> String {
+            GIC_V3_SNAPSHOT_ID.to_string()
+        }
+
+        fn snapshot(&mut self) -> std::result::Result<Snapshot, MigratableError> {
+            let gicr_typers = self.gicr_typers.clone();
+            let snapshot = serde_json::to_vec(&self.state(&gicr_typers).unwrap())
+                .map_err(|e| MigratableError::Snapshot(e.into()))?;
+
+            let mut gic_v3_snapshot = Snapshot::new(self.id().as_str());
+            gic_v3_snapshot.add_data_section(SnapshotDataSection {
+                id: format!("{}-section", self.id()),
+                snapshot,
+            });
+
+            Ok(gic_v3_snapshot)
+        }
+
+        fn restore(&mut self, snapshot: Snapshot) -> std::result::Result<(), MigratableError> {
+            if let Some(gic_v3_section) = snapshot
+                .snapshot_data
+                .get(&format!("{}-section", self.id()))
+            {
+                let gic_v3_state = match serde_json::from_slice(&gic_v3_section.snapshot) {
+                    Ok(state) => state,
+                    Err(error) => {
+                        return Err(MigratableError::Restore(anyhow!(
+                            "Could not deserialize GICv3 {}",
+                            error
+                        )))
+                    }
+                };
+
+                let gicr_typers = self.gicr_typers.clone();
+                return self.set_state(&gicr_typers, &gic_v3_state).map_err(|e| {
+                    MigratableError::Restore(anyhow!("Could not restore GICv3 state {:?}", e))
+                });
+            }
+
+            Err(MigratableError::Restore(anyhow!(
+                "Could not find GICv3 snapshot section"
+            )))
+        }
+    }
+
+    impl Pausable for KvmGICv3 {}
+    impl Transportable for KvmGICv3 {}
+    impl Migratable for KvmGICv3 {}
+}
--- a/arch/src/aarch64/gic/gicv3_its.rs
+++ b/arch/src/aarch64/gic/gicv3_its.rs
@@ -2,18 +2,23 @@
 // SPDX-License-Identifier: Apache-2.0

 pub mod kvm {
+    use std::any::Any;
+    use std::convert::TryInto;
    use std::sync::Arc;
    use std::{boxed::Box, result};
    type Result<T> = result::Result<T, Error>;
+    use crate::aarch64::gic::gicv3::kvm::KvmGICv3;
    use crate::aarch64::gic::kvm::KvmGICDevice;
    use crate::aarch64::gic::{Error, GICDevice};
-    use crate::aarch64::gicv3::kvm::KvmGICv3;
    use hypervisor::kvm::kvm_bindings;

    pub struct KvmGICv3ITS {
        /// The hypervisor agnostic device
        device: Arc<dyn hypervisor::Device>,

+        /// Vector holding values of GICR_TYPER for each vCPU
+        gicr_typers: Vec<u64>,
+
        /// GIC device properties, to be used for setting up the fdt entry
        gic_properties: [u64; 4],

@@ -49,7 +54,7 @@ pub mod kvm {
            true
        }

-        fn msi_compatiblility(&self) -> &str {
+        fn msi_compatibility(&self) -> &str {
            "arm,gic-v3-its"
        }

@@ -68,6 +73,14 @@ pub mod kvm {
        fn vcpu_count(&self) -> u64 {
            self.vcpu_count
        }
+
+        fn set_gicr_typers(&mut self, gicr_typers: Vec<u64>) {
+            self.gicr_typers = gicr_typers;
+        }
+
+        fn as_any_concrete_mut(&mut self) -> &mut dyn Any {
+            self
+        }
    }

    impl KvmGICDevice for KvmGICv3ITS {
@@ -80,7 +93,8 @@ pub mod kvm {
            vcpu_count: u64,
        ) -> Box<dyn GICDevice> {
            Box::new(KvmGICv3ITS {
-                device: device,
+                device,
+                gicr_typers: vec![0; vcpu_count.try_into().unwrap()],
                gic_properties: [
                    KvmGICv3::get_dist_addr(),
                    KvmGICv3::get_dist_size(),
@@ -91,13 +105,13 @@ pub mod kvm {
                    KvmGICv3ITS::get_msi_addr(vcpu_count),
                    KvmGICv3ITS::get_msi_size(),
                ],
-                vcpu_count: vcpu_count,
+                vcpu_count,
            })
        }

        fn init_device_attributes(
            vm: &Arc<dyn hypervisor::Vm>,
-            gic_device: &Box<dyn GICDevice>,
+            gic_device: &dyn GICDevice,
        ) -> Result<()> {
            KvmGICv3::init_device_attributes(vm, gic_device)?;

@@ -115,7 +129,7 @@ pub mod kvm {
                &its_fd,
                kvm_bindings::KVM_DEV_ARM_VGIC_GRP_ADDR,
                u64::from(kvm_bindings::KVM_VGIC_ITS_ADDR_TYPE),
-                &KvmGICv3ITS::get_msi_addr(u64::from(gic_device.vcpu_count())) as *const u64 as u64,
+                &KvmGICv3ITS::get_msi_addr(gic_device.vcpu_count()) as *const u64 as u64,
                0,
            )?;

--- a/arch/src/aarch64/gic/icc_regs.rs
+++ b/arch/src/aarch64/gic/icc_regs.rs
@@ -0,0 +1,196 @@
+// Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+use super::{Error, Result};
+use hypervisor::kvm::kvm_bindings::{
+    kvm_device_attr, KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS, KVM_REG_ARM64_SYSREG_CRM_MASK,
+    KVM_REG_ARM64_SYSREG_CRM_SHIFT, KVM_REG_ARM64_SYSREG_CRN_MASK, KVM_REG_ARM64_SYSREG_CRN_SHIFT,
+    KVM_REG_ARM64_SYSREG_OP0_MASK, KVM_REG_ARM64_SYSREG_OP0_SHIFT, KVM_REG_ARM64_SYSREG_OP1_MASK,
+    KVM_REG_ARM64_SYSREG_OP1_SHIFT, KVM_REG_ARM64_SYSREG_OP2_MASK, KVM_REG_ARM64_SYSREG_OP2_SHIFT,
+};
+use std::sync::Arc;
+
+const KVM_DEV_ARM_VGIC_V3_MPIDR_SHIFT: u32 = 32;
+const KVM_DEV_ARM_VGIC_V3_MPIDR_MASK: u64 = 0xffffffff << KVM_DEV_ARM_VGIC_V3_MPIDR_SHIFT as u64;
+
+const ICC_CTLR_EL1_PRIBITS_SHIFT: u32 = 8;
+const ICC_CTLR_EL1_PRIBITS_MASK: u32 = 7 << ICC_CTLR_EL1_PRIBITS_SHIFT;
+
+macro_rules! arm64_vgic_sys_reg {
+    ($name: tt, $op0: tt, $op1: tt, $crn: tt, $crm: tt, $op2: expr) => {
+        const $name: u64 = ((($op0 as u64) << KVM_REG_ARM64_SYSREG_OP0_SHIFT)
+            & KVM_REG_ARM64_SYSREG_OP0_MASK as u64)
+            | ((($op1 as u64) << KVM_REG_ARM64_SYSREG_OP1_SHIFT)
+                & KVM_REG_ARM64_SYSREG_OP1_MASK as u64)
+            | ((($crn as u64) << KVM_REG_ARM64_SYSREG_CRN_SHIFT)
+                & KVM_REG_ARM64_SYSREG_CRN_MASK as u64)
+            | ((($crm as u64) << KVM_REG_ARM64_SYSREG_CRM_SHIFT)
+                & KVM_REG_ARM64_SYSREG_CRM_MASK as u64)
+            | ((($op2 as u64) << KVM_REG_ARM64_SYSREG_OP2_SHIFT)
+                & KVM_REG_ARM64_SYSREG_OP2_MASK as u64);
+    };
+}
+
+macro_rules! SYS_ICC_AP0Rn_EL1 {
+    ($name: tt, $n: tt) => {
+        arm64_vgic_sys_reg!($name, 3, 0, 12, 8, (4 | $n));
+    };
+}
+
+macro_rules! SYS_ICC_AP1Rn_EL1 {
+    ($name: tt, $n: tt) => {
+        arm64_vgic_sys_reg!($name, 3, 0, 12, 9, $n);
+    };
+}
+
+arm64_vgic_sys_reg!(SYS_ICC_SRE_EL1, 3, 0, 12, 12, 5);
+arm64_vgic_sys_reg!(SYS_ICC_CTLR_EL1, 3, 0, 12, 12, 4);
+arm64_vgic_sys_reg!(SYS_ICC_IGRPEN0_EL1, 3, 0, 12, 12, 6);
+arm64_vgic_sys_reg!(SYS_ICC_IGRPEN1_EL1, 3, 0, 12, 12, 7);
+arm64_vgic_sys_reg!(SYS_ICC_PMR_EL1, 3, 0, 4, 6, 0);
+arm64_vgic_sys_reg!(SYS_ICC_BPR0_EL1, 3, 0, 12, 8, 3);
+arm64_vgic_sys_reg!(SYS_ICC_BPR1_EL1, 3, 0, 12, 12, 3);
+SYS_ICC_AP0Rn_EL1!(SYS_ICC_AP0R0_EL1, 0);
+SYS_ICC_AP0Rn_EL1!(SYS_ICC_AP0R1_EL1, 1);
+SYS_ICC_AP0Rn_EL1!(SYS_ICC_AP0R2_EL1, 2);
+SYS_ICC_AP0Rn_EL1!(SYS_ICC_AP0R3_EL1, 3);
+SYS_ICC_AP1Rn_EL1!(SYS_ICC_AP1R0_EL1, 0);
+SYS_ICC_AP1Rn_EL1!(SYS_ICC_AP1R1_EL1, 1);
+SYS_ICC_AP1Rn_EL1!(SYS_ICC_AP1R2_EL1, 2);
+SYS_ICC_AP1Rn_EL1!(SYS_ICC_AP1R3_EL1, 3);
+
+static VGIC_ICC_REGS: &'static [u64] = &[
+    SYS_ICC_SRE_EL1,
+    SYS_ICC_CTLR_EL1,
+    SYS_ICC_IGRPEN0_EL1,
+    SYS_ICC_IGRPEN1_EL1,
+    SYS_ICC_PMR_EL1,
+    SYS_ICC_BPR0_EL1,
+    SYS_ICC_BPR1_EL1,
+    SYS_ICC_AP0R0_EL1,
+    SYS_ICC_AP0R1_EL1,
+    SYS_ICC_AP0R2_EL1,
+    SYS_ICC_AP0R3_EL1,
+    SYS_ICC_AP1R0_EL1,
+    SYS_ICC_AP1R1_EL1,
+    SYS_ICC_AP1R2_EL1,
+    SYS_ICC_AP1R3_EL1,
+];
+
+fn icc_attr_access(
+    gic: &Arc<dyn hypervisor::Device>,
+    offset: u64,
+    typer: u64,
+    val: &u32,
+    set: bool,
+) -> Result<()> {
+    let mut gic_icc_attr = kvm_device_attr {
+        group: KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS,
+        attr: ((typer & KVM_DEV_ARM_VGIC_V3_MPIDR_MASK) | offset), // this needs the mpidr
+        addr: val as *const u32 as u64,
+        flags: 0,
+    };
+    if set {
+        #[allow(clippy::unnecessary_mut_passed)]
+        gic.set_device_attr(&mut gic_icc_attr)
+            .map_err(Error::SetDeviceAttribute)?;
+    } else {
+        gic.get_device_attr(&mut gic_icc_attr)
+            .map_err(Error::GetDeviceAttribute)?;
+    }
+    Ok(())
+}
+
+/// Get ICC registers.
+pub fn get_icc_regs(gic: &Arc<dyn hypervisor::Device>, gicr_typer: &[u64]) -> Result<Vec<u32>> {
+    let mut state: Vec<u32> = Vec::new();
+    // We need this for the ICC_AP<m>R<n>_EL1 registers.
+    let mut num_priority_bits = 0;
+
+    for ix in gicr_typer {
+        let i = *ix;
+        for icc_offset in VGIC_ICC_REGS {
+            let val = 0;
+            if *icc_offset == SYS_ICC_CTLR_EL1 {
+                // calculate priority bits by reading the ctrl_el1 register.
+                icc_attr_access(gic, *icc_offset, i, &val, false)?;
+                // The priority bits are found in the ICC_CTLR_EL1 register (bits from  10:8).
+                // See page 194 from https://static.docs.arm.com/ihi0069/c/IHI0069C_gic_
+                // architecture_specification.pdf.
+                // Citation:
+                // "Priority bits. Read-only and writes are ignored. The number of priority bits
+                // implemented, minus one."
+                num_priority_bits =
+                    ((val & ICC_CTLR_EL1_PRIBITS_MASK) >> ICC_CTLR_EL1_PRIBITS_SHIFT) + 1;
+                state.push(val);
+            }
+            // As per ARMv8 documentation: https://static.docs.arm.com/ihi0069/c/IHI0069C_
+            // gic_architecture_specification.pdf
+            // page 178,
+            // ICC_AP0R1_EL1 is only implemented in implementations that support 6 or more bits of
+            // priority.
+            // ICC_AP0R2_EL1 and ICC_AP0R3_EL1 are only implemented in implementations that support
+            // 7 bits of priority.
+            else if *icc_offset == SYS_ICC_AP0R1_EL1 || *icc_offset == SYS_ICC_AP1R1_EL1 {
+                if num_priority_bits >= 6 {
+                    icc_attr_access(gic, *icc_offset, i, &val, false)?;
+                    state.push(val);
+                }
+            } else if *icc_offset == SYS_ICC_AP0R2_EL1
+                || *icc_offset == SYS_ICC_AP0R3_EL1
+                || *icc_offset == SYS_ICC_AP1R2_EL1
+                || *icc_offset == SYS_ICC_AP1R3_EL1
+            {
+                if num_priority_bits == 7 {
+                    icc_attr_access(gic, *icc_offset, i, &val, false)?;
+                    state.push(val);
+                }
+            } else {
+                icc_attr_access(gic, *icc_offset, i, &val, false)?;
+                state.push(val);
+            }
+        }
+    }
+    Ok(state)
+}
+
+/// Set ICC registers.
+pub fn set_icc_regs(
+    gic: &Arc<dyn hypervisor::Device>,
+    gicr_typer: &[u64],
+    state: &[u32],
+) -> Result<()> {
+    let mut num_priority_bits = 0;
+    let mut idx = 0;
+    for ix in gicr_typer {
+        let i = *ix;
+        for icc_offset in VGIC_ICC_REGS {
+            if *icc_offset == SYS_ICC_CTLR_EL1 {
+                let ctrl_el1 = state[idx];
+                num_priority_bits =
+                    ((ctrl_el1 & ICC_CTLR_EL1_PRIBITS_MASK) >> ICC_CTLR_EL1_PRIBITS_SHIFT) + 1;
+            }
+            if *icc_offset == SYS_ICC_AP0R1_EL1 || *icc_offset == SYS_ICC_AP1R1_EL1 {
+                if num_priority_bits >= 6 {
+                    icc_attr_access(gic, *icc_offset, i, &state[idx], true)?;
+                    idx += 1;
+                }
+                continue;
+            }
+            if *icc_offset == SYS_ICC_AP0R2_EL1
+                || *icc_offset == SYS_ICC_AP0R3_EL1
+                || *icc_offset == SYS_ICC_AP1R2_EL1
+                || *icc_offset == SYS_ICC_AP1R3_EL1
+            {
+                if num_priority_bits == 7 {
+                    icc_attr_access(gic, *icc_offset, i, &state[idx], true)?;
+                    idx += 1;
+                }
+                continue;
+            }
+            icc_attr_access(gic, *icc_offset, i, &state[idx], true)?;
+            idx += 1;
+        }
+    }
+    Ok(())
+}
--- a/arch/src/aarch64/gic/mod.rs
+++ b/arch/src/aarch64/gic/mod.rs
@@ -1,6 +1,16 @@
 // Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
 // SPDX-License-Identifier: Apache-2.0

+pub mod dist_regs;
+pub mod gicv3;
+pub mod gicv3_its;
+pub mod icc_regs;
+pub mod redist_regs;
+
+pub use self::dist_regs::{get_dist_regs, read_ctlr, set_dist_regs, write_ctlr};
+pub use self::icc_regs::{get_icc_regs, set_icc_regs};
+pub use self::redist_regs::{get_redist_regs, set_redist_regs};
+use std::any::Any;
 use std::result;
 use std::sync::Arc;

@@ -11,10 +21,12 @@ pub enum Error {
    CreateGIC(hypervisor::HypervisorVmError),
    /// Error while setting device attributes for the GIC.
    SetDeviceAttribute(hypervisor::HypervisorDeviceError),
+    /// Error while getting device attributes for the GIC.
+    GetDeviceAttribute(hypervisor::HypervisorDeviceError),
 }
 type Result<T> = result::Result<T, Error>;

-pub trait GICDevice {
+pub trait GICDevice: Send {
    /// Returns the hypervisor agnostic Device of the GIC device
    fn device(&self) -> &Arc<dyn hypervisor::Device>;

@@ -36,7 +48,7 @@ pub trait GICDevice {
    }

    /// Returns the MSI compatibility property of the device
-    fn msi_compatiblility(&self) -> &str {
+    fn msi_compatibility(&self) -> &str {
        ""
    }

@@ -44,14 +56,18 @@ pub trait GICDevice {
    fn msi_properties(&self) -> &[u64] {
        &[]
    }
+
+    /// Get the values of GICR_TYPER for each vCPU.
+    fn set_gicr_typers(&mut self, gicr_typers: Vec<u64>);
+
+    /// Downcast the trait object to its concrete type.
+    fn as_any_concrete_mut(&mut self) -> &mut dyn Any;
 }

 pub mod kvm {
    use super::GICDevice;
    use super::Result;
-    use crate::aarch64::gicv2::kvm::KvmGICv2;
-    use crate::aarch64::gicv3::kvm::KvmGICv3;
-    use crate::aarch64::gicv3_its::kvm::KvmGICv3ITS;
+    use crate::aarch64::gic::gicv3_its::kvm::KvmGICv3ITS;
    use crate::layout;
    use hypervisor::kvm::kvm_bindings;
    use std::boxed::Box;
@@ -71,7 +87,7 @@ pub mod kvm {
        /// Setup the device-specific attributes
        fn init_device_attributes(
            vm: &Arc<dyn hypervisor::Vm>,
-            gic_device: &Box<dyn GICDevice>,
+            gic_device: &dyn GICDevice,
        ) -> Result<()>;

        /// Initialize a GIC device
@@ -95,10 +111,10 @@ pub mod kvm {
            flags: u32,
        ) -> Result<()> {
            let attr = kvm_bindings::kvm_device_attr {
-                group: group,
-                attr: attr,
-                addr: addr,
-                flags: flags,
+                group,
+                attr,
+                addr,
+                flags,
            };
            device
                .set_device_attr(&attr)
@@ -107,8 +123,29 @@ pub mod kvm {
            Ok(())
        }

+        /// Get a GIC device attribute
+        fn get_device_attribute(
+            device: &Arc<dyn hypervisor::Device>,
+            group: u32,
+            attr: u64,
+            addr: u64,
+            flags: u32,
+        ) -> Result<()> {
+            let mut attr = kvm_bindings::kvm_device_attr {
+                group,
+                attr,
+                addr,
+                flags,
+            };
+            device
+                .get_device_attr(&mut attr)
+                .map_err(super::Error::GetDeviceAttribute)?;
+
+            Ok(())
+        }
+
        /// Finalize the setup of a GIC device
-        fn finalize_device(gic_device: &Box<dyn GICDevice>) -> Result<()> {
+        fn finalize_device(gic_device: &dyn GICDevice) -> Result<()> {
            /* We need to tell the kernel how many irqs to support with this vgic.
             * See the `layout` module for details.
             */
@@ -142,33 +179,32 @@ pub mod kvm {

            let device = Self::create_device(vgic_fd, vcpu_count);

-            Self::init_device_attributes(vm, &device)?;
+            Self::init_device_attributes(vm, &*device)?;

-            Self::finalize_device(&device)?;
+            Self::finalize_device(&*device)?;

            Ok(device)
        }
    }

-    /// Create a GIC device.
+    /// Create a GICv3-ITS device.
    ///
-    /// It will try to create by default a GICv3 device. If that fails it will try
-    /// to fall-back to a GICv2 device.
-    pub fn create_gic(
-        vm: &Arc<dyn hypervisor::Vm>,
-        vcpu_count: u64,
-        its_required: bool,
-    ) -> Result<Box<dyn GICDevice>> {
-        if its_required {
-            KvmGICv3ITS::new(vm, vcpu_count)
-        } else {
-            KvmGICv3ITS::new(vm, vcpu_count).or_else(|_| {
-                debug!("Failed to create GICv3-ITS, will try GICv3 instead.");
-                KvmGICv3::new(vm, vcpu_count).or_else(|_| {
-                    debug!("Failed to create GICv3, will try GICv2 instead.");
-                    KvmGICv2::new(vm, vcpu_count)
-                })
-            })
-        }
+    pub fn create_gic(vm: &Arc<dyn hypervisor::Vm>, vcpu_count: u64) -> Result<Box<dyn GICDevice>> {
+        debug!("creating a GICv3-ITS");
+        KvmGICv3ITS::new(vm, vcpu_count)
+    }
+
+    /// Function that saves RDIST pending tables into guest RAM.
+    ///
+    /// The tables get flushed to guest RAM whenever the VM gets stopped.
+    pub fn save_pending_tables(gic: &Arc<dyn hypervisor::Device>) -> Result<()> {
+        let init_gic_attr = kvm_bindings::kvm_device_attr {
+            group: kvm_bindings::KVM_DEV_ARM_VGIC_GRP_CTRL,
+            attr: u64::from(kvm_bindings::KVM_DEV_ARM_VGIC_SAVE_PENDING_TABLES),
+            addr: 0,
+            flags: 0,
+        };
+        gic.set_device_attr(&init_gic_attr)
+            .map_err(super::Error::SetDeviceAttribute)
    }
 }
--- a/arch/src/aarch64/gic/redist_regs.rs
+++ b/arch/src/aarch64/gic/redist_regs.rs
@@ -0,0 +1,179 @@
+// Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+use super::{Error, Result};
+use hypervisor::kvm::kvm_bindings::{kvm_device_attr, KVM_DEV_ARM_VGIC_GRP_REDIST_REGS};
+use std::sync::Arc;
+
+// Relevant redistributor registers that we want to save/restore.
+const GICR_CTLR: u32 = 0x0000;
+const GICR_STATUSR: u32 = 0x0010;
+const GICR_WAKER: u32 = 0x0014;
+const GICR_PROPBASER: u32 = 0x0070;
+const GICR_PENDBASER: u32 = 0x0078;
+
+/* SGI and PPI Redistributor registers, offsets from RD_base */
+/*
+ * Redistributor frame offsets from RD_base which is actually SZ_
+ */
+const GICR_SGI_OFFSET: u32 = 0x0001_0000;
+const GICR_IGROUPR0: u32 = GICR_SGI_OFFSET + 0x0080;
+const GICR_ICENABLER0: u32 = GICR_SGI_OFFSET + 0x0180;
+const GICR_ISENABLER0: u32 = GICR_SGI_OFFSET + 0x0100;
+const GICR_ISPENDR0: u32 = GICR_SGI_OFFSET + 0x0200;
+const GICR_ICPENDR0: u32 = GICR_SGI_OFFSET + 0x0280;
+const GICR_ISACTIVER0: u32 = GICR_SGI_OFFSET + 0x0300;
+const GICR_ICACTIVER0: u32 = GICR_SGI_OFFSET + 0x0380;
+const GICR_IPRIORITYR0: u32 = GICR_SGI_OFFSET + 0x0400;
+const GICR_ICFGR0: u32 = GICR_SGI_OFFSET + 0x0C00;
+
+const KVM_DEV_ARM_VGIC_V3_MPIDR_SHIFT: u32 = 32;
+const KVM_DEV_ARM_VGIC_V3_MPIDR_MASK: u64 = 0xffffffff << KVM_DEV_ARM_VGIC_V3_MPIDR_SHIFT as u64;
+
+/// This is how we represent the registers of a distributor.
+/// It is relrvant their offset from the base address of the
+/// distributor.
+/// Each register has a different number
+/// of bits_per_irq and is therefore variable length.
+/// First 32 interrupts (0-32) are private to each CPU (SGIs and PPIs).
+/// and so we save the first irq to identify between the type of the interrupt
+/// that the respective register deals with.
+struct RdistReg {
+    /// Offset from distributor address.
+    base: u32,
+    /// Length of the register.
+    length: u8,
+}
+
+// All or at least the registers we are interested in are 32 bit, so
+// we use a constant for size(u32).
+const REG_SIZE: u8 = 4;
+
+// Creates a vgic redistributor register.
+macro_rules! VGIC_RDIST_REG {
+    ($base:expr, $len:expr) => {
+        RdistReg {
+            base: $base,
+            length: $len,
+        }
+    };
+}
+
+// List with relevant distributor registers that we will be restoring.
+static VGIC_RDIST_REGS: &'static [RdistReg] = &[
+    VGIC_RDIST_REG!(GICR_CTLR, 4),
+    VGIC_RDIST_REG!(GICR_STATUSR, 4),
+    VGIC_RDIST_REG!(GICR_WAKER, 4),
+    VGIC_RDIST_REG!(GICR_PROPBASER, 8),
+    VGIC_RDIST_REG!(GICR_PENDBASER, 8),
+];
+
+// List with relevant distributor registers that we will be restoring.
+static VGIC_SGI_REGS: &'static [RdistReg] = &[
+    VGIC_RDIST_REG!(GICR_IGROUPR0, 4),
+    VGIC_RDIST_REG!(GICR_ICENABLER0, 4),
+    VGIC_RDIST_REG!(GICR_ISENABLER0, 4),
+    VGIC_RDIST_REG!(GICR_ICFGR0, 8),
+    VGIC_RDIST_REG!(GICR_ICPENDR0, 4),
+    VGIC_RDIST_REG!(GICR_ISPENDR0, 4),
+    VGIC_RDIST_REG!(GICR_ICACTIVER0, 4),
+    VGIC_RDIST_REG!(GICR_ISACTIVER0, 4),
+    VGIC_RDIST_REG!(GICR_IPRIORITYR0, 32),
+];
+
+fn redist_attr_access(
+    gic: &Arc<dyn hypervisor::Device>,
+    offset: u32,
+    typer: u64,
+    val: &u32,
+    set: bool,
+) -> Result<()> {
+    let mut gic_dist_attr = kvm_device_attr {
+        group: KVM_DEV_ARM_VGIC_GRP_REDIST_REGS,
+        attr: (typer & KVM_DEV_ARM_VGIC_V3_MPIDR_MASK) | (offset as u64), // this needs the mpidr
+        addr: val as *const u32 as u64,
+        flags: 0,
+    };
+    if set {
+        #[allow(clippy::unnecessary_mut_passed)]
+        gic.set_device_attr(&mut gic_dist_attr)
+            .map_err(Error::SetDeviceAttribute)?;
+    } else {
+        gic.get_device_attr(&mut gic_dist_attr)
+            .map_err(Error::GetDeviceAttribute)?;
+    }
+    Ok(())
+}
+
+fn access_redists_aux(
+    gic: &Arc<dyn hypervisor::Device>,
+    gicr_typer: &[u64],
+    state: &mut Vec<u32>,
+    reg_list: &'static [RdistReg],
+    idx: &mut usize,
+    set: bool,
+) -> Result<()> {
+    for i in gicr_typer {
+        for rdreg in reg_list {
+            let mut base = rdreg.base;
+            let end = base + rdreg.length as u32;
+
+            while base < end {
+                let mut val = 0;
+                if set {
+                    val = state[*idx];
+                    redist_attr_access(gic, base, *i, &val, true)?;
+                    *idx += 1;
+                } else {
+                    redist_attr_access(gic, base, *i, &val, false)?;
+                    state.push(val);
+                }
+                base += REG_SIZE as u32;
+            }
+        }
+    }
+    Ok(())
+}
+
+/// Get redistributor registers.
+pub fn get_redist_regs(gic: &Arc<dyn hypervisor::Device>, gicr_typer: &[u64]) -> Result<Vec<u32>> {
+    let mut state = Vec::new();
+    let mut idx: usize = 0;
+    access_redists_aux(
+        gic,
+        &gicr_typer,
+        &mut state,
+        VGIC_RDIST_REGS,
+        &mut idx,
+        false,
+    )?;
+
+    access_redists_aux(gic, &gicr_typer, &mut state, VGIC_SGI_REGS, &mut idx, false)?;
+    Ok(state)
+}
+
+/// Set redistributor registers.
+pub fn set_redist_regs(
+    gic: &Arc<dyn hypervisor::Device>,
+    gicr_typer: &[u64],
+    state: &[u32],
+) -> Result<()> {
+    let mut idx: usize = 0;
+    let mut mut_state = state.to_owned();
+    access_redists_aux(
+        gic,
+        gicr_typer,
+        &mut mut_state,
+        VGIC_RDIST_REGS,
+        &mut idx,
+        true,
+    )?;
+    access_redists_aux(
+        gic,
+        gicr_typer,
+        &mut mut_state,
+        VGIC_SGI_REGS,
+        &mut idx,
+        true,
+    )
+}
--- a/arch/src/aarch64/gicv2.rs
+++ b/arch/src/aarch64/gicv2.rs
@@ -1,124 +0,0 @@
-// Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
-// SPDX-License-Identifier: Apache-2.0
-
-pub mod kvm {
-    use crate::aarch64::gic::kvm::KvmGICDevice;
-    use crate::aarch64::gic::{Error, GICDevice};
-    use std::{boxed::Box, result};
-    type Result<T> = result::Result<T, Error>;
-    use crate::layout;
-    use hypervisor::kvm::kvm_bindings;
-    use std::sync::Arc;
-
-    /// Represent a GIC v2 device
-    pub struct KvmGICv2 {
-        /// The hypervisor agnostic device
-        device: Arc<dyn hypervisor::Device>,
-
-        /// GIC device properties, to be used for setting up the fdt entry
-        properties: [u64; 4],
-
-        /// Number of CPUs handled by the device
-        vcpu_count: u64,
-    }
-
-    impl KvmGICv2 {
-        // Unfortunately bindgen omits defines that are based on other defines.
-        // See arch/arm64/include/uapi/asm/kvm.h file from the linux kernel.
-        const KVM_VGIC_V2_DIST_SIZE: u64 = 0x1000;
-        const KVM_VGIC_V2_CPU_SIZE: u64 = 0x2000;
-
-        // Device trees specific constants
-        const ARCH_GIC_V2_MAINT_IRQ: u32 = 8;
-
-        /// Get the address of the GICv2 distributor.
-        const fn get_dist_addr() -> u64 {
-            layout::MAPPED_IO_START - KvmGICv2::KVM_VGIC_V2_DIST_SIZE
-        }
-
-        /// Get the size of the GIC_v2 distributor.
-        const fn get_dist_size() -> u64 {
-            KvmGICv2::KVM_VGIC_V2_DIST_SIZE
-        }
-
-        /// Get the address of the GIC_v2 CPU.
-        const fn get_cpu_addr() -> u64 {
-            KvmGICv2::get_dist_addr() - KvmGICv2::KVM_VGIC_V2_CPU_SIZE
-        }
-
-        /// Get the size of the GIC_v2 CPU.
-        const fn get_cpu_size() -> u64 {
-            KvmGICv2::KVM_VGIC_V2_CPU_SIZE
-        }
-    }
-
-    impl GICDevice for KvmGICv2 {
-        fn device(&self) -> &Arc<dyn hypervisor::Device> {
-            &self.device
-        }
-
-        fn device_properties(&self) -> &[u64] {
-            &self.properties
-        }
-
-        fn fdt_compatibility(&self) -> &str {
-            "arm,gic-400"
-        }
-
-        fn fdt_maint_irq(&self) -> u32 {
-            KvmGICv2::ARCH_GIC_V2_MAINT_IRQ
-        }
-
-        fn vcpu_count(&self) -> u64 {
-            self.vcpu_count
-        }
-    }
-
-    impl KvmGICDevice for KvmGICv2 {
-        fn version() -> u32 {
-            kvm_bindings::kvm_device_type_KVM_DEV_TYPE_ARM_VGIC_V2
-        }
-
-        fn create_device(
-            device: Arc<dyn hypervisor::Device>,
-            vcpu_count: u64,
-        ) -> Box<dyn GICDevice> {
-            Box::new(KvmGICv2 {
-                device: device,
-                properties: [
-                    KvmGICv2::get_dist_addr(),
-                    KvmGICv2::get_dist_size(),
-                    KvmGICv2::get_cpu_addr(),
-                    KvmGICv2::get_cpu_size(),
-                ],
-                vcpu_count: vcpu_count,
-            })
-        }
-
-        fn init_device_attributes(
-            _vm: &Arc<dyn hypervisor::Vm>,
-            gic_device: &Box<dyn GICDevice>,
-        ) -> Result<()> {
-            /* Setting up the distributor attribute.
-            We are placing the GIC below 1GB so we need to substract the size of the distributor. */
-            Self::set_device_attribute(
-                gic_device.device(),
-                kvm_bindings::KVM_DEV_ARM_VGIC_GRP_ADDR,
-                u64::from(kvm_bindings::KVM_VGIC_V2_ADDR_TYPE_DIST),
-                &KvmGICv2::get_dist_addr() as *const u64 as u64,
-                0,
-            )?;
-
-            /* Setting up the CPU attribute. */
-            Self::set_device_attribute(
-                gic_device.device(),
-                kvm_bindings::KVM_DEV_ARM_VGIC_GRP_ADDR,
-                u64::from(kvm_bindings::KVM_VGIC_V2_ADDR_TYPE_CPU),
-                &KvmGICv2::get_cpu_addr() as *const u64 as u64,
-                0,
-            )?;
-
-            Ok(())
-        }
-    }
-}
--- a/arch/src/aarch64/gicv3.rs
+++ b/arch/src/aarch64/gicv3.rs
@@ -1,128 +0,0 @@
-// Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
-// SPDX-License-Identifier: Apache-2.0
-
-pub mod kvm {
-    use crate::aarch64::gic::kvm::KvmGICDevice;
-    use crate::aarch64::gic::{Error, GICDevice};
-    use crate::layout;
-    use hypervisor::kvm::kvm_bindings;
-    use std::sync::Arc;
-    use std::{boxed::Box, result};
-    type Result<T> = result::Result<T, Error>;
-
-    pub struct KvmGICv3 {
-        /// The hypervisor agnostic device
-        device: Arc<dyn hypervisor::Device>,
-
-        /// GIC device properties, to be used for setting up the fdt entry
-        properties: [u64; 4],
-
-        /// Number of CPUs handled by the device
-        vcpu_count: u64,
-    }
-
-    impl KvmGICv3 {
-        // Unfortunately bindgen omits defines that are based on other defines.
-        // See arch/arm64/include/uapi/asm/kvm.h file from the linux kernel.
-        pub const SZ_64K: u64 = 0x0001_0000;
-        const KVM_VGIC_V3_DIST_SIZE: u64 = KvmGICv3::SZ_64K;
-        const KVM_VGIC_V3_REDIST_SIZE: u64 = (2 * KvmGICv3::SZ_64K);
-
-        // Device trees specific constants
-        pub const ARCH_GIC_V3_MAINT_IRQ: u32 = 9;
-
-        /// Get the address of the GIC distributor.
-        pub fn get_dist_addr() -> u64 {
-            layout::MAPPED_IO_START - KvmGICv3::KVM_VGIC_V3_DIST_SIZE
-        }
-
-        /// Get the size of the GIC distributor.
-        pub fn get_dist_size() -> u64 {
-            KvmGICv3::KVM_VGIC_V3_DIST_SIZE
-        }
-
-        /// Get the address of the GIC redistributors.
-        pub fn get_redists_addr(vcpu_count: u64) -> u64 {
-            KvmGICv3::get_dist_addr() - KvmGICv3::get_redists_size(vcpu_count)
-        }
-
-        /// Get the size of the GIC redistributors.
-        pub fn get_redists_size(vcpu_count: u64) -> u64 {
-            vcpu_count * KvmGICv3::KVM_VGIC_V3_REDIST_SIZE
-        }
-    }
-
-    impl GICDevice for KvmGICv3 {
-        fn device(&self) -> &Arc<dyn hypervisor::Device> {
-            &self.device
-        }
-
-        fn fdt_compatibility(&self) -> &str {
-            "arm,gic-v3"
-        }
-
-        fn fdt_maint_irq(&self) -> u32 {
-            KvmGICv3::ARCH_GIC_V3_MAINT_IRQ
-        }
-
-        fn device_properties(&self) -> &[u64] {
-            &self.properties
-        }
-
-        fn vcpu_count(&self) -> u64 {
-            self.vcpu_count
-        }
-    }
-
-    impl KvmGICDevice for KvmGICv3 {
-        fn version() -> u32 {
-            kvm_bindings::kvm_device_type_KVM_DEV_TYPE_ARM_VGIC_V3
-        }
-
-        fn create_device(
-            device: Arc<dyn hypervisor::Device>,
-            vcpu_count: u64,
-        ) -> Box<dyn GICDevice> {
-            Box::new(KvmGICv3 {
-                device: device,
-                properties: [
-                    KvmGICv3::get_dist_addr(),
-                    KvmGICv3::get_dist_size(),
-                    KvmGICv3::get_redists_addr(vcpu_count),
-                    KvmGICv3::get_redists_size(vcpu_count),
-                ],
-                vcpu_count: vcpu_count,
-            })
-        }
-
-        fn init_device_attributes(
-            _vm: &Arc<dyn hypervisor::Vm>,
-            gic_device: &Box<dyn GICDevice>,
-        ) -> Result<()> {
-            /* Setting up the distributor attribute.
-             We are placing the GIC below 1GB so we need to substract the size of the distributor.
-            */
-            Self::set_device_attribute(
-                gic_device.device(),
-                kvm_bindings::KVM_DEV_ARM_VGIC_GRP_ADDR,
-                u64::from(kvm_bindings::KVM_VGIC_V3_ADDR_TYPE_DIST),
-                &KvmGICv3::get_dist_addr() as *const u64 as u64,
-                0,
-            )?;
-
-            /* Setting up the redistributors' attribute.
-            We are calculating here the start of the redistributors address. We have one per CPU.
-            */
-            Self::set_device_attribute(
-                gic_device.device(),
-                kvm_bindings::KVM_DEV_ARM_VGIC_GRP_ADDR,
-                u64::from(kvm_bindings::KVM_VGIC_V3_ADDR_TYPE_REDIST),
-                &KvmGICv3::get_redists_addr(u64::from(gic_device.vcpu_count())) as *const u64
-                    as u64,
-                0,
-            )?;
-
-            Ok(())
-        }
-    }
-}
--- a/arch/src/aarch64/layout.rs
+++ b/arch/src/aarch64/layout.rs
@@ -37,7 +37,7 @@
 //           |                                                               |
 // 144 M     +---------------------------------------------------------------|
 //           |                                                               |
-//           |                    Reserverd (now GIC is here)                |
+//           |                    Reserved (now GIC is here)                 |
 //           |                                                               |
 // 0GB       +---------------------------------------------------------------+
 //
--- a/arch/src/aarch64/mod.rs
+++ b/arch/src/aarch64/mod.rs
@@ -6,9 +6,6 @@
 pub mod fdt;
 /// Module for the global interrupt controller configuration.
 pub mod gic;
-mod gicv2;
-mod gicv3;
-mod gicv3_its;
 /// Layout for this aarch64 system.
 pub mod layout;
 /// Logic for configuring aarch64 registers.
@@ -17,7 +14,7 @@ pub mod regs;
 pub use self::fdt::DeviceInfoForFDT;
 use crate::DeviceType;
 use crate::RegionType;
-use hypervisor::kvm::kvm_bindings;
+use aarch64::gic::GICDevice;
 use std::collections::HashMap;
 use std::ffi::CStr;
 use std::fmt::Debug;
@@ -42,11 +39,8 @@ pub enum Error {
    /// Error configuring the general purpose registers
    REGSConfiguration(regs::Error),

-    /// Error fetching prefered target
-    VcpuArmPreferredTarget(hypervisor::HypervisorVmError),
-
-    /// Error doing Vcpu Init on Arm.
-    VcpuArmInit(hypervisor::HypervisorCpuError),
+    /// Error configuring the MPIDR register
+    VcpuRegMPIDR(hypervisor::HypervisorCpuError),
 }

 impl From<Error> for super::Error {
@@ -67,23 +61,10 @@ pub struct EntryPoint {
 pub fn configure_vcpu(
    fd: &Arc<dyn hypervisor::Vcpu>,
    id: u8,
-    vm: &Arc<dyn hypervisor::Vm>,
    kernel_entry_point: Option<EntryPoint>,
    vm_memory: &GuestMemoryAtomic<GuestMemoryMmap>,
+    _phys_bits: u8,
 ) -> super::Result<u64> {
-    let mut kvi: kvm_bindings::kvm_vcpu_init = kvm_bindings::kvm_vcpu_init::default();
-
-    // This reads back the kernel's preferred target type.
-    vm.get_preferred_target(&mut kvi)
-        .map_err(Error::VcpuArmPreferredTarget)?;
-    // We already checked that the capability is supported.
-    kvi.features[0] |= 1 << kvm_bindings::KVM_ARM_VCPU_PSCI_0_2;
-    // Non-boot cpus are powered off initially.
-    if id > 0 {
-        kvi.features[0] |= 1 << kvm_bindings::KVM_ARM_VCPU_POWER_OFF;
-    }
-
-    fd.vcpu_init(&kvi).map_err(Error::VcpuArmInit)?;
    if let Some(kernel_entry_point) = kernel_entry_point {
        regs::setup_regs(
            fd,
@@ -94,7 +75,7 @@ pub fn configure_vcpu(
        .map_err(Error::REGSConfiguration)?;
    }

-    let mpidr = regs::read_mpidr(fd).map_err(Error::REGSConfiguration)?;
+    let mpidr = fd.read_mpidr().map_err(Error::VcpuRegMPIDR)?;
    Ok(mpidr)
 }

@@ -136,34 +117,31 @@ pub fn arch_memory_regions(size: GuestUsize) -> Vec<(GuestAddress, usize, Region
 ///
 /// * `guest_mem` - The memory to be used by the guest.
 /// * `num_cpus` - Number of virtual CPUs the guest will have.
-pub fn configure_system<T: DeviceInfoForFDT + Clone + Debug>(
+#[allow(clippy::too_many_arguments)]
+pub fn configure_system<T: DeviceInfoForFDT + Clone + Debug, S: ::std::hash::BuildHasher>(
    vm: &Arc<dyn hypervisor::Vm>,
    guest_mem: &GuestMemoryMmap,
    cmdline_cstring: &CStr,
    vcpu_count: u64,
    vcpu_mpidr: Vec<u64>,
-    device_info: &HashMap<(DeviceType, String), T>,
+    device_info: &HashMap<(DeviceType, String), T, S>,
    initrd: &Option<super::InitramfsConfig>,
-    pci_space_address: &Option<(u64, u64)>,
-) -> super::Result<()> {
-    // If pci_space_address is present, it means PCI devices are used ("pci" feature enabled).
-    // Then GITv3-ITS is required for MSI messaging.
-    // Otherwise ("mmio" feature enabled), any version of GIC is OK.
-    let gic_device = gic::kvm::create_gic(vm, vcpu_count, pci_space_address.is_some())
-        .map_err(Error::SetupGIC)?;
+    pci_space_address: &(u64, u64),
+) -> super::Result<Box<dyn GICDevice>> {
+    let gic_device = gic::kvm::create_gic(vm, vcpu_count).map_err(Error::SetupGIC)?;

    fdt::create_fdt(
        guest_mem,
        cmdline_cstring,
        vcpu_mpidr,
        device_info,
-        &gic_device,
+        &*gic_device,
        initrd,
        pci_space_address,
    )
    .map_err(Error::SetupFDT)?;

-    Ok(())
+    Ok(gic_device)
 }

 /// Returns the memory address where the initramfs could be loaded.
@@ -209,10 +187,10 @@ fn get_fdt_addr(mem: &GuestMemoryMmap) -> u64 {
 pub fn get_host_cpu_phys_bits() -> u8 {
    // The value returned here is used to determine the physical address space size
    // for a VM (IPA size).
-    // In recent kernel versions, the maxium IPA size supported by the host can be
+    // In recent kernel versions, the maximum IPA size supported by the host can be
    // known by querying cap KVM_CAP_ARM_VM_IPA_SIZE. And the IPA size for a
    // guest can be configured smaller.
-    // But in Cloud-Hypervisor we simply use the maxium value for the VM.
+    // But in Cloud-Hypervisor we simply use the maximum value for the VM.
    // Reference https://lwn.net/Articles/766767/.
    //
    // The correct way to query KVM_CAP_ARM_VM_IPA_SIZE is via rust-vmm/kvm-ioctls,
--- a/arch/src/aarch64/regs.rs
+++ b/arch/src/aarch64/regs.rs
@@ -7,12 +7,9 @@

 use super::get_fdt_addr;
 use hypervisor::kvm::kvm_bindings::{
-    user_pt_regs, KVM_REG_ARM64, KVM_REG_ARM64_SYSREG, KVM_REG_ARM64_SYSREG_CRM_MASK,
-    KVM_REG_ARM64_SYSREG_CRM_SHIFT, KVM_REG_ARM64_SYSREG_CRN_MASK, KVM_REG_ARM64_SYSREG_CRN_SHIFT,
-    KVM_REG_ARM64_SYSREG_OP0_MASK, KVM_REG_ARM64_SYSREG_OP0_SHIFT, KVM_REG_ARM64_SYSREG_OP1_MASK,
-    KVM_REG_ARM64_SYSREG_OP1_SHIFT, KVM_REG_ARM64_SYSREG_OP2_MASK, KVM_REG_ARM64_SYSREG_OP2_SHIFT,
-    KVM_REG_ARM_CORE, KVM_REG_SIZE_U64,
+    kvm_regs, user_pt_regs, KVM_REG_ARM64, KVM_REG_ARM_CORE, KVM_REG_SIZE_U64,
 };
+use hypervisor::{arm64_core_reg_id, offset__of};
 use std::sync::Arc;
 use std::{mem, result};
 use vm_memory::GuestMemoryMmap;
@@ -38,80 +35,6 @@ const PSR_D_BIT: u64 = 0x0000_0200;
 // Taken from arch/arm64/kvm/inject_fault.c.
 const PSTATE_FAULT_BITS_64: u64 = PSR_MODE_EL1h | PSR_A_BIT | PSR_F_BIT | PSR_I_BIT | PSR_D_BIT;

-// Following are macros that help with getting the ID of a aarch64 core register.
-// The core register are represented by the user_pt_regs structure. Look for it in
-// arch/arm64/include/uapi/asm/ptrace.h.
-
-// This macro gets the offset of a structure (i.e `str`) member (i.e `field`) without having
-// an instance of that structure.
-// It uses a null pointer to retrieve the offset to the field.
-// Inspired by C solution: `#define offsetof(str, f) ((size_t)(&((str *)0)->f))`.
-// Doing `offset__of!(user_pt_regs, pstate)` in our rust code will trigger the following:
-// unsafe { &(*(0 as *const user_pt_regs)).pstate as *const _ as usize }
-// The dereference expression produces an lvalue, but that lvalue is not actually read from,
-// we're just doing pointer math on it, so in theory, it should safe.
-macro_rules! offset__of {
-    ($str:ty, $field:ident) => {
-        unsafe { &(*(0 as *const $str)).$field as *const _ as usize }
-    };
-}
-
-macro_rules! arm64_core_reg {
-    ($reg: tt) => {
-        // As per `kvm_arm_copy_reg_indices`, the id of a core register can be obtained like this:
-        // `const u64 core_reg = KVM_REG_ARM64 | KVM_REG_SIZE_U64 | KVM_REG_ARM_CORE | i`, where i is obtained with:
-        // `for (i = 0; i < sizeof(struct kvm_regs) / sizeof(__u32); i++) {`
-        // We are using here `user_pt_regs` since this structure contains the core register and it is at
-        // the start of `kvm_regs`.
-        // struct kvm_regs {
-        //	struct user_pt_regs regs;	/* sp = sp_el0 */
-        //
-        //	__u64	sp_el1;
-        //	__u64	elr_el1;
-        //
-        //	__u64	spsr[KVM_NR_SPSR];
-        //
-        //	struct user_fpsimd_state fp_regs;
-        //};
-        // struct user_pt_regs {
-        //	__u64		regs[31];
-        //	__u64		sp;
-        //	__u64		pc;
-        //	__u64		pstate;
-        //};
-        // In our implementation we need: pc, pstate and user_pt_regs->regs[0].
-        KVM_REG_ARM64 as u64
-            | KVM_REG_SIZE_U64 as u64
-            | u64::from(KVM_REG_ARM_CORE)
-            | ((offset__of!(user_pt_regs, $reg) / mem::size_of::<u32>()) as u64)
-    };
-}
-
-// This macro computes the ID of a specific ARM64 system register similar to how
-// the kernel C macro does.
-// https://elixir.bootlin.com/linux/v4.20.17/source/arch/arm64/include/uapi/asm/kvm.h#L203
-macro_rules! arm64_sys_reg {
-    ($name: tt, $op0: tt, $op1: tt, $crn: tt, $crm: tt, $op2: tt) => {
-        const $name: u64 = KVM_REG_ARM64 as u64
-            | KVM_REG_SIZE_U64 as u64
-            | KVM_REG_ARM64_SYSREG as u64
-            | ((($op0 as u64) << KVM_REG_ARM64_SYSREG_OP0_SHIFT)
-                & KVM_REG_ARM64_SYSREG_OP0_MASK as u64)
-            | ((($op1 as u64) << KVM_REG_ARM64_SYSREG_OP1_SHIFT)
-                & KVM_REG_ARM64_SYSREG_OP1_MASK as u64)
-            | ((($crn as u64) << KVM_REG_ARM64_SYSREG_CRN_SHIFT)
-                & KVM_REG_ARM64_SYSREG_CRN_MASK as u64)
-            | ((($crm as u64) << KVM_REG_ARM64_SYSREG_CRM_SHIFT)
-                & KVM_REG_ARM64_SYSREG_CRM_MASK as u64)
-            | ((($op2 as u64) << KVM_REG_ARM64_SYSREG_OP2_SHIFT)
-                & KVM_REG_ARM64_SYSREG_OP2_MASK as u64);
-    };
-}
-
-// Constant imported from the Linux kernel:
-// https://elixir.bootlin.com/linux/v4.20.17/source/arch/arm64/include/asm/sysreg.h#L135
-arm64_sys_reg!(MPIDR_EL1, 3, 0, 0, 0, 5);
-
 /// Configure core registers for a given CPU.
 ///
 /// # Arguments
@@ -126,31 +49,33 @@ pub fn setup_regs(
    boot_ip: u64,
    mem: &GuestMemoryMmap,
 ) -> Result<()> {
+    let kreg_off = offset__of!(kvm_regs, regs);
+
    // Get the register index of the PSTATE (Processor State) register.
-    vcpu.set_one_reg(arm64_core_reg!(pstate), PSTATE_FAULT_BITS_64)
-        .map_err(Error::SetCoreRegister)?;
+    let pstate = offset__of!(user_pt_regs, pstate) + kreg_off;
+    vcpu.set_reg(
+        arm64_core_reg_id!(KVM_REG_SIZE_U64, pstate),
+        PSTATE_FAULT_BITS_64,
+    )
+    .map_err(Error::SetCoreRegister)?;

    // Other vCPUs are powered off initially awaiting PSCI wakeup.
    if cpu_id == 0 {
        // Setting the PC (Processor Counter) to the current program address (kernel address).
-        vcpu.set_one_reg(arm64_core_reg!(pc), boot_ip)
+        let pc = offset__of!(user_pt_regs, pc) + kreg_off;
+        vcpu.set_reg(arm64_core_reg_id!(KVM_REG_SIZE_U64, pc), boot_ip as u64)
            .map_err(Error::SetCoreRegister)?;

        // Last mandatory thing to set -> the address pointing to the FDT (also called DTB).
        // "The device tree blob (dtb) must be placed on an 8-byte boundary and must
        // not exceed 2 megabytes in size." -> https://www.kernel.org/doc/Documentation/arm64/booting.txt.
        // We are choosing to place it the end of DRAM. See `get_fdt_addr`.
-        vcpu.set_one_reg(arm64_core_reg!(regs), get_fdt_addr(mem) as u64)
-            .map_err(Error::SetCoreRegister)?;
+        let regs0 = offset__of!(user_pt_regs, regs) + kreg_off;
+        vcpu.set_reg(
+            arm64_core_reg_id!(KVM_REG_SIZE_U64, regs0),
+            get_fdt_addr(mem) as u64,
+        )
+        .map_err(Error::SetCoreRegister)?;
    }
    Ok(())
 }
-
-/// Read the MPIDR - Multiprocessor Affinity Register.
-///
-/// # Arguments
-///
-/// * `vcpu` - Structure for the VCPU that holds the VCPU's fd.
-pub fn read_mpidr(vcpu: &Arc<dyn hypervisor::Vcpu>) -> Result<u64> {
-    vcpu.get_one_reg(MPIDR_EL1).map_err(Error::GetSysRegister)
-}
--- a/arch/src/lib.rs
+++ b/arch/src/lib.rs
@@ -14,6 +14,7 @@
    clippy::cast_ptr_alignment
 )]

+extern crate anyhow;
 extern crate byteorder;
 extern crate hypervisor;
 extern crate libc;
@@ -23,7 +24,13 @@ extern crate log;
 extern crate acpi_tables;
 extern crate arch_gen;
 extern crate linux_loader;
+extern crate serde;
 extern crate vm_memory;
+extern crate vm_migration;
+#[cfg(target_arch = "aarch64")]
+#[macro_use]
+extern crate serde_derive;
+extern crate serde_json;

 use std::fmt;
 use std::result;
--- a/arch/src/x86_64/mod.rs
+++ b/arch/src/x86_64/mod.rs
@@ -10,7 +10,6 @@ use std::sync::Arc;
 mod gdt;
 pub mod interrupts;
 pub mod layout;
-#[cfg(not(feature = "acpi"))]
 mod mptable;
 pub mod regs;
 use crate::InitramfsConfig;
@@ -26,6 +25,7 @@ use vm_memory::{
    GuestMemoryMmap, GuestMemoryRegion, GuestUsize,
 };
 mod smbios;
+use std::arch::x86_64;

 #[derive(Debug, Copy, Clone)]
 pub enum BootProtocol {
@@ -141,7 +141,7 @@ unsafe impl ByteValued for BootParamsWrapper {}
 pub enum Error {
    /// Invalid e820 setup params.
    E820Configuration,
-    #[cfg(not(feature = "acpi"))]
+
    /// Error writing MP table to memory.
    MpTableSetup(mptable::Error),

@@ -174,6 +174,9 @@ pub enum Error {

    /// Missing SGX_LC CPU feature
    MissingSgxLaunchControlFeature,
+
+    // Error populating Cpuid
+    PopulatingCpuid,
 }

 impl From<Error> for super::Error {
@@ -333,13 +336,95 @@ pub fn configure_vcpu(
    kernel_entry_point: Option<EntryPoint>,
    vm_memory: &GuestMemoryAtomic<GuestMemoryMmap>,
    cpuid: CpuId,
+    kvm_hyperv: bool,
+    phys_bits: u8,
 ) -> super::Result<()> {
    let mut cpuid = cpuid;
    CpuidPatch::set_cpuid_reg(&mut cpuid, 0xb, None, CpuidReg::EDX, u32::from(id));
    CpuidPatch::set_cpuid_reg(&mut cpuid, 0x1f, None, CpuidReg::EDX, u32::from(id));
+
+    if kvm_hyperv {
+        // Remove conflicting entries
+        cpuid.retain(|c| c.function != 0x4000_0000);
+        cpuid.retain(|c| c.function != 0x4000_0001);
+
+        // See "Hypervisor Top Level Functional Specification" for details
+        // Compliance with "Hv#1" requires leaves up to 0x4000_000a
+        cpuid
+            .push(CpuIdEntry {
+                function: 0x40000000,
+                eax: 0x4000000a, // Maximum cpuid leaf
+                ebx: 0x756e694c, // "Linu"
+                ecx: 0x564b2078, // "x KV"
+                edx: 0x7648204d, // "M Hv"
+                ..Default::default()
+            })
+            .map_err(|_| Error::PopulatingCpuid)?;
+        cpuid
+            .push(CpuIdEntry {
+                function: 0x40000001,
+                eax: 0x31237648, // "Hv#1"
+                ..Default::default()
+            })
+            .map_err(|_| Error::PopulatingCpuid)?;
+        cpuid
+            .push(CpuIdEntry {
+                function: 0x40000002,
+                eax: 0x3839,  // "Build number"
+                ebx: 0xa0000, // "Version"
+                ..Default::default()
+            })
+            .map_err(|_| Error::PopulatingCpuid)?;
+        cpuid
+            .push(CpuIdEntry {
+                function: 0x4000_0003,
+                eax: 1 << 1 // AccessPartitionReferenceCounter
+                   | 1 << 2 // AccessSynicRegs
+                   | 1 << 3 // AccessSyntheticTimerRegs
+                   | 1 << 9, // AccessPartitionReferenceTsc
+                ..Default::default()
+            })
+            .map_err(|_| Error::PopulatingCpuid)?;
+        for i in 0x4000_0004..=0x4000_000a {
+            cpuid
+                .push(CpuIdEntry {
+                    function: i,
+                    ..Default::default()
+                })
+                .map_err(|_| Error::PopulatingCpuid)?;
+        }
+    }
+
+    // Copy CPU identification string
+    for i in 0x8000_0002..=0x8000_0004 {
+        cpuid.retain(|c| c.function != i);
+        let leaf = unsafe { x86_64::__cpuid(i) };
+        cpuid
+            .push(CpuIdEntry {
+                function: i,
+                eax: leaf.eax,
+                ebx: leaf.ebx,
+                ecx: leaf.ecx,
+                edx: leaf.edx,
+                ..Default::default()
+            })
+            .map_err(|_| Error::PopulatingCpuid)?;
+    }
+
+    // Set CPU physical bits
+    for entry in cpuid.as_mut_slice().iter_mut() {
+        if entry.function == 0x8000_0008 {
+            entry.eax = (entry.eax & 0xffff_ff00) | (phys_bits as u32 & 0xff);
+        }
+    }
+
    fd.set_cpuid2(&cpuid)
        .map_err(|e| Error::SetSupportedCpusFailed(e.into()))?;

+    if kvm_hyperv {
+        fd.enable_hyperv_synic().unwrap();
+    }
+
    regs::setup_msrs(fd).map_err(Error::MSRSConfiguration)?;
    if let Some(kernel_entry_point) = kernel_entry_point {
        // Safe to unwrap because this method is called after the VM is configured
@@ -426,11 +511,12 @@ pub fn configure_system(
    boot_prot: BootProtocol,
    sgx_epc_region: Option<SgxEpcRegion>,
 ) -> super::Result<()> {
-    smbios::setup_smbios(guest_mem).map_err(Error::SmbiosSetup)?;
+    let size = smbios::setup_smbios(guest_mem).map_err(Error::SmbiosSetup)?;

-    // Note that this puts the mptable at the last 1k of Linux's 640k base RAM
-    #[cfg(not(feature = "acpi"))]
-    mptable::setup_mptable(guest_mem, _num_cpus).map_err(Error::MpTableSetup)?;
+    // Place the MP table after the SMIOS table aligned to 16 bytes
+    let offset = GuestAddress(layout::SMBIOS_START).unchecked_add(size);
+    let offset = GuestAddress((offset.0 + 16) & !0xf);
+    mptable::setup_mptable(offset, guest_mem, _num_cpus).map_err(Error::MpTableSetup)?;

    // Check that the RAM is not smaller than the RSDP start address
    if let Some(rsdp_addr) = rsdp_addr {
@@ -747,7 +833,6 @@ pub fn initramfs_load_addr(
 }

 pub fn get_host_cpu_phys_bits() -> u8 {
-    use std::arch::x86_64;
    unsafe {
        let leaf = x86_64::__cpuid(0x8000_0000);

--- a/arch/src/x86_64/mptable.rs
+++ b/arch/src/x86_64/mptable.rs
@@ -13,8 +13,10 @@ use std::slice;
 use libc::c_char;

 use arch_gen::x86::mpspec;
-use layout::{APIC_START, IOAPIC_START, MPTABLE_START};
-use vm_memory::{Address, ByteValued, Bytes, GuestMemory, GuestMemoryError, GuestMemoryMmap};
+use layout::{APIC_START, HIGH_RAM_START, IOAPIC_START};
+use vm_memory::{
+    Address, ByteValued, Bytes, GuestAddress, GuestMemory, GuestMemoryError, GuestMemoryMmap,
+};

 // This is a workaround to the Rust enforcement specifying that any implementation of a foreign
 // trait (in this case `ByteValued`) where:
@@ -121,16 +123,21 @@ fn compute_mp_size(num_cpus: u8) -> usize {
 }

 /// Performs setup of the MP table for the given `num_cpus`.
-pub fn setup_mptable(mem: &GuestMemoryMmap, num_cpus: u8) -> Result<()> {
+pub fn setup_mptable(offset: GuestAddress, mem: &GuestMemoryMmap, num_cpus: u8) -> Result<()> {
    if num_cpus as u32 > MAX_SUPPORTED_CPUS {
        return Err(Error::TooManyCpus);
    }

    // Used to keep track of the next base pointer into the MP table.
-    let mut base_mp = MPTABLE_START;
+    let mut base_mp = offset;

    let mp_size = compute_mp_size(num_cpus);

+    if offset.unchecked_add(mp_size as u64) >= HIGH_RAM_START {
+        warn!("Skipping mptable creation due to insufficient space");
+        return Ok(());
+    }
+
    let mut checksum: u8 = 0;
    let ioapicid: u8 = num_cpus + 1;

@@ -280,6 +287,7 @@ pub fn setup_mptable(mem: &GuestMemoryMmap, num_cpus: u8) -> Result<()> {
 #[cfg(test)]
 mod tests {
    use super::*;
+    use layout::MPTABLE_START;
    use vm_memory::{GuestAddress, GuestUsize};

    fn table_entry_size(type_: u8) -> usize {
@@ -299,7 +307,7 @@ mod tests {
        let mem =
            GuestMemoryMmap::from_ranges(&[(MPTABLE_START, compute_mp_size(num_cpus))]).unwrap();

-        setup_mptable(&mem, num_cpus).unwrap();
+        setup_mptable(MPTABLE_START, &mem, num_cpus).unwrap();
    }

    #[test]
@@ -308,7 +316,7 @@ mod tests {
        let mem = GuestMemoryMmap::from_ranges(&[(MPTABLE_START, compute_mp_size(num_cpus) - 1)])
            .unwrap();

-        assert!(setup_mptable(&mem, num_cpus).is_err());
+        assert!(setup_mptable(MPTABLE_START, &mem, num_cpus).is_err());
    }

    #[test]
@@ -317,7 +325,7 @@ mod tests {
        let mem =
            GuestMemoryMmap::from_ranges(&[(MPTABLE_START, compute_mp_size(num_cpus))]).unwrap();

-        setup_mptable(&mem, num_cpus).unwrap();
+        setup_mptable(MPTABLE_START, &mem, num_cpus).unwrap();

        let mpf_intel: MpfIntelWrapper = mem.read_obj(MPTABLE_START).unwrap();

@@ -333,7 +341,7 @@ mod tests {
        let mem =
            GuestMemoryMmap::from_ranges(&[(MPTABLE_START, compute_mp_size(num_cpus))]).unwrap();

-        setup_mptable(&mem, num_cpus).unwrap();
+        setup_mptable(MPTABLE_START, &mem, num_cpus).unwrap();

        let mpf_intel: MpfIntelWrapper = mem.read_obj(MPTABLE_START).unwrap();
        let mpc_offset = GuestAddress(mpf_intel.0.physptr as GuestUsize);
@@ -367,7 +375,7 @@ mod tests {
        .unwrap();

        for i in 0..MAX_SUPPORTED_CPUS as u8 {
-            setup_mptable(&mem, i).unwrap();
+            setup_mptable(MPTABLE_START, &mem, i).unwrap();

            let mpf_intel: MpfIntelWrapper = mem.read_obj(MPTABLE_START).unwrap();
            let mpc_offset = GuestAddress(mpf_intel.0.physptr as GuestUsize);
@@ -400,7 +408,7 @@ mod tests {
        let mem =
            GuestMemoryMmap::from_ranges(&[(MPTABLE_START, compute_mp_size(cpus as u8))]).unwrap();

-        let result = setup_mptable(&mem, cpus as u8);
+        let result = setup_mptable(MPTABLE_START, &mem, cpus as u8);
        assert!(result.is_err());
    }
 }
--- a/arch/src/x86_64/smbios.rs
+++ b/arch/src/x86_64/smbios.rs
@@ -161,7 +161,7 @@ fn write_string(
    Ok(curptr)
 }

-pub fn setup_smbios(mem: &GuestMemoryMmap) -> Result<()> {
+pub fn setup_smbios(mem: &GuestMemoryMmap) -> Result<u64> {
    let physptr = GuestAddress(SMBIOS_START)
        .checked_add(mem::size_of::<Smbios30Entrypoint>() as u64)
        .ok_or(Error::NotEnoughMemory)?;
@@ -224,7 +224,7 @@ pub fn setup_smbios(mem: &GuestMemoryMmap) -> Result<()> {
            .map_err(|_| Error::WriteSmbiosEp)?;
    }

-    Ok(())
+    Ok(curptr.unchecked_offset_from(physptr))
 }

 #[cfg(test)]
--- a/block_util/Cargo.toml
+++ b/block_util/Cargo.toml
@@ -9,13 +9,13 @@ default = []
 io_uring = []

 [dependencies]
-io-uring = { git = "https://github.com/tokio-rs/io-uring.git", branch = "0.4" }
-libc = "0.2.74"
+io-uring = ">=0.4.0"
+libc = "0.2.80"
 log = "0.4.11"
 serde = ">=1.0.27"
 serde_derive = ">=1.0.27"
 serde_json = ">=1.0.9"
 virtio-bindings = { version = "0.1", features = ["virtio-v5_0_0"]}
-vm-memory = { version = "0.2.1", features = ["backend-mmap", "backend-atomic"] }
+vm-memory = { version = "0.3.0", features = ["backend-mmap", "backend-atomic"] }
 vm-virtio = { path = "../vm-virtio" }
 vmm-sys-util = ">=0.3.1"
--- a/block_util/src/lib.rs
+++ b/block_util/src/lib.rs
@@ -14,13 +14,15 @@ extern crate log;
 extern crate serde_derive;

 #[cfg(feature = "io_uring")]
-use io_uring::{opcode, IoUring, Probe};
+use io_uring::Probe;
+use io_uring::{opcode, squeue, IoUring};
 use serde::ser::{Serialize, SerializeStruct, Serializer};
 use std::cmp;
 use std::io::{self, Read, Seek, SeekFrom, Write};
 use std::os::linux::fs::MetadataExt;
 #[cfg(feature = "io_uring")]
-use std::os::unix::io::{AsRawFd, RawFd};
+use std::os::unix::io::AsRawFd;
+use std::os::unix::io::RawFd;
 use std::path::PathBuf;
 use std::result;
 use virtio_bindings::bindings::virtio_blk::*;
@@ -50,6 +52,8 @@ pub enum Error {
    GetFileMetadata,
    /// The requested operation would cause a seek beyond disk end.
    InvalidOffset,
+    /// The requested operation does not support multiple descriptors.
+    TooManyDescriptors,
 }

 fn build_device_id(disk_path: &PathBuf) -> result::Result<String, Error> {
@@ -147,8 +151,7 @@ fn sector(mem: &GuestMemoryMmap, desc_addr: GuestAddress) -> result::Result<u64,
 pub struct Request {
    pub request_type: RequestType,
    pub sector: u64,
-    pub data_addr: GuestAddress,
-    pub data_len: u32,
+    pub data_descriptors: Vec<(GuestAddress, u32)>,
    pub status_addr: GuestAddress,
    pub writeback: bool,
 }
@@ -166,15 +169,13 @@ impl Request {
        let mut req = Request {
            request_type: request_type(&mem, avail_desc.addr)?,
            sector: sector(&mem, avail_desc.addr)?,
-            data_addr: GuestAddress(0),
-            data_len: 0,
+            data_descriptors: Vec::new(),
            status_addr: GuestAddress(0),
            writeback: true,
        };

-        let data_desc;
        let status_desc;
-        let desc = avail_desc
+        let mut desc = avail_desc
            .next_descriptor()
            .ok_or(Error::DescriptorChainTooShort)?;

@@ -185,23 +186,22 @@ impl Request {
                return Err(Error::DescriptorChainTooShort);
            }
        } else {
-            data_desc = desc;
-            status_desc = data_desc
-                .next_descriptor()
-                .ok_or(Error::DescriptorChainTooShort)?;
-
-            if data_desc.is_write_only() && req.request_type == RequestType::Out {
-                return Err(Error::UnexpectedWriteOnlyDescriptor);
+            while desc.has_next() {
+                if desc.is_write_only() && req.request_type == RequestType::Out {
+                    return Err(Error::UnexpectedWriteOnlyDescriptor);
+                }
+                if !desc.is_write_only() && req.request_type == RequestType::In {
+                    return Err(Error::UnexpectedReadOnlyDescriptor);
+                }
+                if !desc.is_write_only() && req.request_type == RequestType::GetDeviceID {
+                    return Err(Error::UnexpectedReadOnlyDescriptor);
+                }
+                req.data_descriptors.push((desc.addr, desc.len));
+                desc = desc
+                    .next_descriptor()
+                    .ok_or(Error::DescriptorChainTooShort)?;
            }
-            if !data_desc.is_write_only() && req.request_type == RequestType::In {
-                return Err(Error::UnexpectedReadOnlyDescriptor);
-            }
-            if !data_desc.is_write_only() && req.request_type == RequestType::GetDeviceID {
-                return Err(Error::UnexpectedReadOnlyDescriptor);
-            }
-
-            req.data_addr = data_desc.addr;
-            req.data_len = data_desc.len;
+            status_desc = desc;
        }

        // The status MUST always be writable.
@@ -226,47 +226,48 @@ impl Request {
        mem: &GuestMemoryMmap,
        disk_id: &Vec<u8>,
    ) -> result::Result<u32, ExecuteError> {
-        let mut top: u64 = u64::from(self.data_len) / SECTOR_SIZE;
-        if u64::from(self.data_len) % SECTOR_SIZE != 0 {
-            top += 1;
-        }
-        top = top
-            .checked_add(self.sector)
-            .ok_or(ExecuteError::BadRequest(Error::InvalidOffset))?;
-        if top > disk_nsectors {
-            return Err(ExecuteError::BadRequest(Error::InvalidOffset));
-        }
-
        disk.seek(SeekFrom::Start(self.sector << SECTOR_SHIFT))
            .map_err(ExecuteError::Seek)?;
+        let mut len = 0;
+        for (data_addr, data_len) in &self.data_descriptors {
+            let mut top: u64 = u64::from(*data_len) / SECTOR_SIZE;
+            if u64::from(*data_len) % SECTOR_SIZE != 0 {
+                top += 1;
+            }
+            top = top
+                .checked_add(self.sector)
+                .ok_or(ExecuteError::BadRequest(Error::InvalidOffset))?;
+            if top > disk_nsectors {
+                return Err(ExecuteError::BadRequest(Error::InvalidOffset));
+            }

-        match self.request_type {
-            RequestType::In => {
-                mem.read_exact_from(self.data_addr, disk, self.data_len as usize)
-                    .map_err(ExecuteError::Read)?;
-                return Ok(self.data_len);
-            }
-            RequestType::Out => {
-                mem.write_all_to(self.data_addr, disk, self.data_len as usize)
-                    .map_err(ExecuteError::Write)?;
-                if !self.writeback {
-                    disk.flush().map_err(ExecuteError::Flush)?;
+            match self.request_type {
+                RequestType::In => {
+                    mem.read_exact_from(*data_addr, disk, *data_len as usize)
+                        .map_err(ExecuteError::Read)?;
+                    len += data_len;
                }
-            }
-            RequestType::Flush => disk.flush().map_err(ExecuteError::Flush)?,
-            RequestType::GetDeviceID => {
-                if (self.data_len as usize) < disk_id.len() {
-                    return Err(ExecuteError::BadRequest(Error::InvalidOffset));
+                RequestType::Out => {
+                    mem.write_all_to(*data_addr, disk, *data_len as usize)
+                        .map_err(ExecuteError::Write)?;
+                    if !self.writeback {
+                        disk.flush().map_err(ExecuteError::Flush)?;
+                    }
                }
-                mem.write_slice(&disk_id.as_slice(), self.data_addr)
-                    .map_err(ExecuteError::Write)?;
-            }
-            RequestType::Unsupported(t) => return Err(ExecuteError::Unsupported(t)),
-        };
-        Ok(0)
+                RequestType::Flush => disk.flush().map_err(ExecuteError::Flush)?,
+                RequestType::GetDeviceID => {
+                    if (*data_len as usize) < disk_id.len() {
+                        return Err(ExecuteError::BadRequest(Error::InvalidOffset));
+                    }
+                    mem.write_slice(&disk_id.as_slice(), *data_addr)
+                        .map_err(ExecuteError::Write)?;
+                }
+                RequestType::Unsupported(t) => return Err(ExecuteError::Unsupported(t)),
+            };
+        }
+        Ok(len)
    }

-    #[cfg(feature = "io_uring")]
    pub fn execute_io_uring(
        &self,
        mem: &GuestMemoryMmap,
@@ -276,31 +277,37 @@ impl Request {
        disk_id: &[u8],
        user_data: u64,
    ) -> result::Result<bool, ExecuteError> {
-        let data_len = self.data_len;
        let sector = self.sector;
-        let data_addr = self.data_addr;
        let request_type = self.request_type;
-
-        let mut top: u64 = u64::from(data_len) / SECTOR_SIZE;
-        if u64::from(data_len) % SECTOR_SIZE != 0 {
-            top += 1;
-        }
-        top = top
-            .checked_add(sector)
-            .ok_or(ExecuteError::BadRequest(Error::InvalidOffset))?;
-        if top > disk_nsectors {
-            return Err(ExecuteError::BadRequest(Error::InvalidOffset));
-        }
-
-        let buf = mem
-            .get_slice(data_addr, data_len as usize)
-            .map_err(ExecuteError::GetHostAddress)?
-            .as_ptr();
-        let offset = (sector as i64) << SECTOR_SHIFT;
+        let offset = (sector << SECTOR_SHIFT) as libc::off_t;

        let (submitter, sq, _) = io_uring.split();
        let mut avail_sq = sq.available();

+        let mut iovecs = Vec::new();
+        for (data_addr, data_len) in &self.data_descriptors {
+            let mut top: u64 = u64::from(*data_len) / SECTOR_SIZE;
+            if u64::from(*data_len) % SECTOR_SIZE != 0 {
+                top += 1;
+            }
+            top = top
+                .checked_add(sector)
+                .ok_or(ExecuteError::BadRequest(Error::InvalidOffset))?;
+            if top > disk_nsectors {
+                return Err(ExecuteError::BadRequest(Error::InvalidOffset));
+            }
+
+            let buf = mem
+                .get_slice(*data_addr, *data_len as usize)
+                .map_err(ExecuteError::GetHostAddress)?
+                .as_ptr();
+            let iovec = libc::iovec {
+                iov_base: buf as *mut libc::c_void,
+                iov_len: *data_len as libc::size_t,
+            };
+            iovecs.push(iovec);
+        }
+
        // Queue operations expected to be submitted.
        match request_type {
            RequestType::In => {
@@ -308,10 +315,15 @@ impl Request {
                // relied on vm-memory to provide the buffer address.
                let _ = unsafe {
                    avail_sq.push(
-                        opcode::Read::new(opcode::types::Fd(disk_image_fd), buf, data_len)
-                            .offset(offset)
-                            .build()
-                            .user_data(user_data),
+                        opcode::Readv::new(
+                            opcode::types::Fd(disk_image_fd),
+                            iovecs.as_ptr(),
+                            iovecs.len() as u32,
+                        )
+                        .offset(offset)
+                        .build()
+                        .flags(squeue::Flags::ASYNC)
+                        .user_data(user_data),
                    )
                };
            }
@@ -320,10 +332,15 @@ impl Request {
                // relied on vm-memory to provide the buffer address.
                let _ = unsafe {
                    avail_sq.push(
-                        opcode::Write::new(opcode::types::Fd(disk_image_fd), buf, data_len)
-                            .offset(offset)
-                            .build()
-                            .user_data(user_data),
+                        opcode::Writev::new(
+                            opcode::types::Fd(disk_image_fd),
+                            iovecs.as_ptr(),
+                            iovecs.len() as u32,
+                        )
+                        .offset(offset)
+                        .build()
+                        .flags(squeue::Flags::ASYNC)
+                        .user_data(user_data),
                    )
                };
            }
@@ -333,11 +350,17 @@ impl Request {
                    avail_sq.push(
                        opcode::Fsync::new(opcode::types::Fd(disk_image_fd))
                            .build()
+                            .flags(squeue::Flags::ASYNC)
                            .user_data(user_data),
                    )
                };
            }
            RequestType::GetDeviceID => {
+                let (data_addr, data_len) = if self.data_descriptors.len() == 1 {
+                    (self.data_descriptors[0].0, self.data_descriptors[0].1)
+                } else {
+                    return Err(ExecuteError::BadRequest(Error::TooManyDescriptors));
+                };
                if (data_len as usize) < disk_id.len() {
                    return Err(ExecuteError::BadRequest(Error::InvalidOffset));
                }
@@ -543,3 +566,8 @@ pub fn block_io_uring_is_supported() -> bool {

    true
 }
+
+#[cfg(not(feature = "io_uring"))]
+pub fn block_io_uring_is_supported() -> bool {
+    false
+}
--- a/build.rs
+++ b/build.rs
@@ -3,20 +3,25 @@
 // SPDX-License-Identifier: Apache-2.0
 //

+#[macro_use(crate_version)]
+extern crate clap;
+
 use std::process::Command;

 fn main() {
-    let git_out = Command::new("git")
-        .args(&["describe", "--dirty"])
-        .output()
-        .expect("Expect to get git describe output");
+    let mut version = "v".to_owned() + crate_version!();
+
+    if let Ok(git_out) = Command::new("git").args(&["describe", "--dirty"]).output() {
+        if git_out.status.success() {
+            if let Ok(git_out_str) = String::from_utf8(git_out.stdout) {
+                version = git_out_str;
+            }
+        }
+    }

    // This println!() has a special behavior, as it will set the environment
    // variable BUILT_VERSION, so that it can be reused from the binary.
    // Particularly, this is used from src/main.rs to display the exact
    // version.
-    println!(
-        "cargo:rustc-env=BUILT_VERSION={}",
-        String::from_utf8(git_out.stdout).unwrap()
-    );
+    println!("cargo:rustc-env=BUILT_VERSION={}", version);
 }
--- a/devices/Cargo.toml
+++ b/devices/Cargo.toml
@@ -8,14 +8,14 @@ anyhow = "1.0"
 bitflags = ">=1.2.1"
 byteorder = "1.3.4"
 epoll = ">=4.0.1"
-libc = "0.2.74"
+libc = "0.2.80"
 log = "0.4.11"
 serde = {version = ">=1.0.27", features = ["rc"] }
 serde_derive = ">=1.0.27"
 serde_json = ">=1.0.9"
 vm-device = { path = "../vm-device" }
 acpi_tables = { path = "../acpi_tables", optional = true }
-vm-memory = "0.2.1"
+vm-memory = "0.3.0"
 vm-migration = { path = "../vm-migration" }
 vmm-sys-util = ">=0.3.1"

--- a/devices/src/acpi.rs
+++ b/devices/src/acpi.rs
@@ -7,8 +7,8 @@ use acpi_tables::{aml, aml::Aml};
 use std::sync::Arc;
 use std::time::Instant;
 use vm_device::interrupt::InterruptSourceGroup;
+use vm_device::BusDevice;
 use vmm_sys_util::eventfd::EventFd;
-use BusDevice;
 use HotPlugNotificationFlags;

 /// A device for handling ACPI shutdown and reboot
--- a/devices/src/gic.rs
+++ b/devices/src/gic.rs
@@ -8,7 +8,6 @@ use std::sync::Arc;
 use vm_device::interrupt::{
    InterruptIndex, InterruptManager, InterruptSourceGroup, MsiIrqGroupConfig,
 };
-use vm_migration::{Migratable, MigratableError, Pausable, Snapshot, Snapshottable, Transportable};

 type Result<T> = result::Result<T, Error>;

@@ -47,8 +46,7 @@ impl Gic {

 impl InterruptController for Gic {
    fn enable(&self) -> Result<()> {
-        &self
-            .interrupt_source_group
+        self.interrupt_source_group
            .enable()
            .map_err(Error::EnableInterrupt)?;
        Ok(())
@@ -64,22 +62,3 @@ impl InterruptController for Gic {
        Ok(())
    }
 }
-
-const GIC_SNAPSHOT_ID: &str = "gic";
-impl Snapshottable for Gic {
-    fn id(&self) -> String {
-        GIC_SNAPSHOT_ID.to_string()
-    }
-
-    fn snapshot(&self) -> std::result::Result<Snapshot, MigratableError> {
-        unimplemented!();
-    }
-
-    fn restore(&mut self, _snapshot: Snapshot) -> std::result::Result<(), MigratableError> {
-        unimplemented!();
-    }
-}
-
-impl Pausable for Gic {}
-impl Transportable for Gic {}
-impl Migratable for Gic {}
--- a/devices/src/interrupt_controller.rs
+++ b/devices/src/interrupt_controller.rs
@@ -7,8 +7,6 @@ use std::result;

 #[derive(Debug)]
 pub enum Error {
-    /// Invalid destination mode.
-    InvalidDestinationMode,
    /// Invalid trigger mode.
    InvalidTriggerMode,
    /// Invalid delivery mode.
--- a/devices/src/ioapic.rs
+++ b/devices/src/ioapic.rs
@@ -10,7 +10,6 @@
 // See https://pdos.csail.mit.edu/6.828/2016/readings/ia32/ioapic.pdf for a specification.

 use super::interrupt_controller::{Error, InterruptController};
-use crate::BusDevice;
 use anyhow::anyhow;
 use byteorder::{ByteOrder, LittleEndian};
 use std::result;
@@ -19,6 +18,7 @@ use vm_device::interrupt::{
    InterruptIndex, InterruptManager, InterruptSourceConfig, InterruptSourceGroup,
    MsiIrqGroupConfig, MsiIrqSourceConfig,
 };
+use vm_device::BusDevice;
 use vm_memory::GuestAddress;
 use vm_migration::{
    Migratable, MigratableError, Pausable, Snapshot, SnapshotDataSection, Snapshottable,
@@ -67,10 +67,12 @@ fn trigger_mode(entry: RedirectionTableEntry) -> u8 {
 fn interrupt_mask(entry: RedirectionTableEntry) -> u8 {
    ((entry >> 16) & 0x1u64) as u8
 }
-fn destination_field_physical(entry: RedirectionTableEntry) -> u8 {
-    ((entry >> 56) & 0xfu64) as u8
-}
-fn destination_field_logical(entry: RedirectionTableEntry) -> u8 {
+fn destination_field(entry: RedirectionTableEntry) -> u8 {
+    // When the destination mode is physical, the destination field should only
+    // be defined through bits 56-59, as defined in the IOAPIC specification.
+    // But from the APIC specification, the APIC ID is always defined on 8 bits
+    // no matter which destination mode is selected. That's why we always
+    // retrieve the destination field based on bits 56-63.
    ((entry >> 56) & 0xffu64) as u8
 }
 fn set_delivery_status(entry: &mut RedirectionTableEntry, val: u8) {
@@ -100,12 +102,6 @@ const IOWIN_OFF: u8 = 0x10;
 const IOWIN_SCALE: u8 = 0x2;
 const REG_MAX_OFFSET: u8 = IOWIN_OFF + (NUM_IOAPIC_PINS as u8 * 2) - 1;

-#[repr(u8)]
-enum DestinationMode {
-    Physical = 0,
-    Logical = 1,
-}
-
 #[repr(u8)]
 enum TriggerMode {
    Edge = 0,
@@ -201,15 +197,13 @@ impl Ioapic {
            })
            .map_err(Error::CreateInterruptSourceGroup)?;

-        interrupt_source_group
-            .enable()
-            .map_err(Error::EnableInterrupt)?;
-
+        // The IOAPIC is created with entries already masked. The guest will be
+        // in charge of unmasking them if/when necessary.
        Ok(Ioapic {
            id,
            id_reg: 0,
            reg_sel: 0,
-            reg_entries: [0; NUM_IOAPIC_PINS],
+            reg_entries: [0x10000; NUM_IOAPIC_PINS],
            used_entries: [false; NUM_IOAPIC_PINS],
            apic_address,
            interrupt_source_group,
@@ -220,6 +214,16 @@ impl Ioapic {
        debug!("IOAPIC_W reg 0x{:x}, val 0x{:x}", self.reg_sel, val);

        match self.reg_sel as u8 {
+            IOAPIC_REG_VERSION => {
+                if val == 0 {
+                    // Windows writes zero here (see #1791)
+                } else {
+                    error!(
+                        "IOAPIC: invalid write to version register (0x{:x}): 0x{:x}",
+                        self.reg_sel, val
+                    );
+                }
+            }
            IOAPIC_REG_ID => self.id_reg = (val >> 24) & 0xf,
            IOWIN_OFF..=REG_MAX_OFFSET => {
                let (index, is_high_bits) = decode_irq_from_selector(self.reg_sel as u8);
@@ -241,7 +245,10 @@ impl Ioapic {
                // Store the information this IRQ is now being used.
                self.used_entries[index] = true;
            }
-            _ => error!("IOAPIC: invalid write to register offset"),
+            _ => error!(
+                "IOAPIC: invalid write to register offset 0x{:x}",
+                self.reg_sel
+            ),
        }
    }

@@ -260,7 +267,10 @@ impl Ioapic {
                }
            }
            _ => {
-                error!("IOAPIC: invalid read from register offset");
+                error!(
+                    "IOAPIC: invalid read from register offset 0x{:x}",
+                    self.reg_sel
+                );
                0
            }
        }
@@ -296,11 +306,7 @@ impl Ioapic {

        // Validate Destination Mode value, and retrieve Destination ID
        let destination_mode = destination_mode(entry);
-        let destination_id: u8 = match destination_mode {
-            x if x == DestinationMode::Physical as u8 => destination_field_physical(entry),
-            x if x == DestinationMode::Logical as u8 => destination_field_logical(entry),
-            _ => return Err(Error::InvalidDestinationMode),
-        };
+        let destination_id = destination_field(entry);

        // When this bit is set, the message is directed to the processor with
        // the lowest interrupt priority among processors that can receive the
@@ -405,7 +411,7 @@ impl Snapshottable for Ioapic {
        self.id.clone()
    }

-    fn snapshot(&self) -> std::result::Result<Snapshot, MigratableError> {
+    fn snapshot(&mut self) -> std::result::Result<Snapshot, MigratableError> {
        let snapshot =
            serde_json::to_vec(&self.state()).map_err(|e| MigratableError::Snapshot(e.into()))?;

--- a/devices/src/legacy/cmos.rs
+++ b/devices/src/legacy/cmos.rs
@@ -5,8 +5,7 @@
 use libc::{clock_gettime, gmtime_r, time_t, timespec, tm, CLOCK_REALTIME};
 use std::cmp::min;
 use std::mem;
-
-use crate::BusDevice;
+use vm_device::BusDevice;

 const INDEX_MASK: u8 = 0x7f;
 const INDEX_OFFSET: u64 = 0x0;
--- a/devices/src/legacy/fwdebug.rs
+++ b/devices/src/legacy/fwdebug.rs
@@ -7,7 +7,7 @@
 // SPDX-License-Identifier: Apache-2.0 AND BSD-3-Clause
 //

-use BusDevice;
+use vm_device::BusDevice;

 /// Provides firmware debug output via I/O port controls
 #[derive(Default)]
--- a/devices/src/legacy/i8042.rs
+++ b/devices/src/legacy/i8042.rs
@@ -4,7 +4,7 @@

 use vmm_sys_util::eventfd::EventFd;

-use BusDevice;
+use vm_device::BusDevice;

 /// A i8042 PS/2 controller that emulates just enough to shutdown the machine.
 pub struct I8042Device {
--- a/devices/src/legacy/rtc_pl031.rs
+++ b/devices/src/legacy/rtc_pl031.rs
@@ -12,16 +12,15 @@ use std::fmt;
 use std::sync::Arc;
 use std::time::Instant;
 use std::{io, result};
-
-use crate::BusDevice;
 use vm_device::interrupt::InterruptSourceGroup;
+use vm_device::BusDevice;

 // As you can see in https://static.docs.arm.com/ddi0224/c/real_time_clock_pl031_r1p3_technical_reference_manual_DDI0224C.pdf
 // at section 3.2 Summary of RTC registers, the total size occupied by this device is 0x000 -> 0xFFC + 4 = 0x1000.
 // From 0x0 to 0x1C we have following registers:
 const RTCDR: u64 = 0x0; // Data Register.
 const RTCMR: u64 = 0x4; // Match Register.
-const RTCLR: u64 = 0x8; // Load Regiser.
+const RTCLR: u64 = 0x8; // Load Register.
 const RTCCR: u64 = 0xc; // Control Register.
 const RTCIMSC: u64 = 0x10; // Interrupt Mask Set or Clear Register.
 const RTCRIS: u64 = 0x14; // Raw Interrupt Status.
--- a/devices/src/legacy/serial.rs
+++ b/devices/src/legacy/serial.rs
@@ -5,12 +5,12 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE-BSD-3-Clause file.

-use crate::BusDevice;
 use anyhow::anyhow;
 use std::collections::VecDeque;
 use std::sync::Arc;
 use std::{io, result};
 use vm_device::interrupt::InterruptSourceGroup;
+use vm_device::BusDevice;
 use vm_migration::{
    Migratable, MigratableError, Pausable, Snapshot, SnapshotDataSection, Snapshottable,
    Transportable,
@@ -289,7 +289,7 @@ impl Snapshottable for Serial {
        self.id.clone()
    }

-    fn snapshot(&self) -> std::result::Result<Snapshot, MigratableError> {
+    fn snapshot(&mut self) -> std::result::Result<Snapshot, MigratableError> {
        let snapshot =
            serde_json::to_vec(&self.state()).map_err(|e| MigratableError::Snapshot(e.into()))?;

--- a/devices/src/lib.rs
+++ b/devices/src/lib.rs
@@ -25,12 +25,8 @@ extern crate vmm_sys_util;
 extern crate serde_derive;
 extern crate serde_json;

-use std::fs::File;
-use std::io;
-
 #[cfg(feature = "acpi")]
 mod acpi;
-mod bus;
 #[cfg(target_arch = "aarch64")]
 pub mod gic;
 pub mod interrupt_controller;
@@ -40,45 +36,6 @@ pub mod legacy;

 #[cfg(feature = "acpi")]
 pub use self::acpi::{AcpiGEDDevice, AcpiPMTimerDevice, AcpiShutdownDevice};
-pub use self::bus::{Bus, BusDevice, Error as BusError};
-
-pub type DeviceEventT = u16;
-
-/// The payload is used to handle events where the internal state of the VirtIO device
-/// needs to be changed.
-pub enum EpollHandlerPayload {
-    /// DrivePayload(disk_image)
-    DrivePayload(File),
-    /// Events that do not need a payload.
-    Empty,
-}
-
-type Result<T> = std::result::Result<T, Error>;
-
-pub trait EpollHandler: Send {
-    fn handle_event(
-        &mut self,
-        device_event: DeviceEventT,
-        event_flags: u32,
-        payload: EpollHandlerPayload,
-    ) -> Result<()>;
-}
-
-#[derive(Debug)]
-pub enum Error {
-    FailedReadingQueue {
-        event_type: &'static str,
-        underlying: io::Error,
-    },
-    FailedReadTap,
-    FailedSignalingUsedQueue(io::Error),
-    PayloadExpected,
-    UnknownEvent {
-        device: &'static str,
-        event: DeviceEventT,
-    },
-    IoError(io::Error),
-}

 bitflags! {
    pub struct HotPlugNotificationFlags: u8 {
--- a/docs/api.md
+++ b/docs/api.md
@@ -12,7 +12,7 @@
        * [Reboot a Virtual Machine](#reboot-a-virtual-machine)
        * [Shut a Virtual Machine Down](#shut-a-virtual-machine-down)
    + [Command Line Interface](#command-line-interface)
-    + [REST API and CLI Architecural Relationship](#rest-api-and-cli-architectural-relationship)
+    + [REST API and CLI Architectural Relationship](#rest-api-and-cli-architectural-relationship)
  * [Internal API](#internal-api)
    + [Goals and Design](#goals-and-design)
  * [End to End Example](#end-to-end-example)
@@ -88,7 +88,8 @@ Reboot the VM                      | `/vm.reboot`        | N/A
 Pause the VM                       | `/vm.pause`         | N/A                       | N/A                      | The VM is booted
 Resume the VM                      | `/vm.resume`        | N/A                       | N/A                      | The VM is paused
 Add/remove CPUs to/from the VM     | `/vm.resize`        | `/schemas/VmResize`       | N/A                      | The VM is booted
-Remove memory from the VM          | `/vm.resize`        | `/schemas/VmResize`       | N/A                      | The VM is booted
+Add/remove memory from the VM      | `/vm.resize`        | `/schemas/VmResize`       | N/A                      | The VM is booted
+Add/remove memory from a zone      | `/vm.resize-zone`   | `/schemas/VmResizeZone`   | N/A                      | The VM is booted
 Dump the VM information            | `/vm.info`          | N/A                       | `/schemas/VmInfo`        | The VM is created
 Add VFIO PCI device to the VM      | `/vm.add-device`    | `/schemas/VmAddDevice`    | `/schemas/PciDeviceInfo` | The VM is booted
 Add disk device to the VM          | `/vm.add-disk`      | `/schemas/DiskConfig`     | `/schemas/PciDeviceInfo` | The VM is booted
--- a/docs/arm64.md
+++ b/docs/arm64.md
@@ -16,20 +16,10 @@ sudo apt-get install libfdt-dev

 ## Build

-For Virtio devices, you can choose MMIO or PCI as transport option.
-
-### MMIO
+Using PCI devices requires GICv3-ITS for MSI messaging. GICv3-ITS is very common in modern servers.

 ```bash
-cargo build --no-default-features --features mmio,kvm
-```
-
-### PCI
-
-Using PCI devices requires GICv3-ITS for MSI messaging. GICv3-ITS is very common in modern servers, but your machine happen to be old ones with GICv2(M) (like Raspberry Pi 4) or GICv3 without ITS, MMIO can still work.
-
-```bash
-cargo build --no-default-features --features pci,kvm
+cargo build --no-default-features --features kvm
 ```

 ## Image
@@ -51,7 +41,7 @@ To build the development container:
 ./scripts/dev_cli.sh build-container
 ```

-To build Cloud-hypervisor in the container: (The default option for Virtio transport is MMIO.)
+To build Cloud-hypervisor in the container:

 ```bash
 ./scripts/dev_cli.sh build
@@ -62,7 +52,7 @@ To build Cloud-hypervisor in the container: (The default option for Virtio trans
 Assuming you have built Cloud-hypervisor with the development container, a VM can be started with command:

 ```bash
-sudo build/cargo_target/aarch64-unknown-linux-gnu/debug/cloud-hypervisor --kernel kernel.bin --disk path=rootfs.ext4 --cmdline "keep_bootcon console=hvc0 reboot=k panic=1 pci=off root=/dev/vda rw" --cpus boot=4 --memory size=512M --serial file=serial.log --log-file log.log -vvv
+sudo build/cargo_target/aarch64-unknown-linux-gnu/debug/cloud-hypervisor --kernel kernel.bin --disk path=rootfs.ext4 --cmdline "keep_bootcon console=hvc0 reboot=k panic=1 root=/dev/vda rw" --cpus boot=4 --memory size=512M --serial file=serial.log --log-file log.log -vvv
 ```

 If the build was done out of the container, replace the binary path with `target/debug/cloud-hypervisor`.
--- a/docs/device_model.md
+++ b/docs/device_model.md
@@ -75,12 +75,8 @@ feature is enabled by default.

 ## Virtio devices

-For all virtio devices listed below, both `virtio-mmio` and `virtio-pci`
-transport layers are supported, `virtio-pci` being the default.
-
-Both `virtio-mmio` and `virtio-pci` can be compiled out. `virtio-pci` is
-built-in by default, and enabled by default. If both transport layers were
-built at the same time, `virtio-pci` would be the default transport layer.
+For all virtio devices listed below, only `virtio-pci` transport layer is
+supported.

 ### virtio-block

--- a/docs/fs.md
+++ b/docs/fs.md
@@ -56,7 +56,7 @@ Assuming you have `focal-server-cloudimg-amd64.raw` and `custom-vmlinux.bin` on
 ```bash
 ./cloud-hypervisor \
    --cpus 4 \
-    --memory "size=512,file=/dev/shm" \
+    --memory "size=512M,shared=on" \
    --disk path=focal-server-cloudimg-amd64.raw \
    --kernel custom-vmlinux.bin \
    --cmdline "console=ttyS0 console=hvc0 root=/dev/vda1 rw" \
--- a/docs/hotplug.md
+++ b/docs/hotplug.md
@@ -63,7 +63,7 @@ As per adding CPUs to the guest, after a reboot the VM will be running with the

 ## Memory Hot Plug

-Extra memory can be added from a runing Cloud Hypervisor instance. This is controlled by two mechanisms:
+Extra memory can be added from a running Cloud Hypervisor instance. This is controlled by two mechanisms:

 1. Allocating some of the guest physical address space for hotplug memory.
 2. Making a HTTP API request to the VMM to ask for a new amount of RAM to be assigned to the VM. In the case of expanding the memory for the VM the new memory will be hotplugged into the running VM, if reducing the size of the memory then change will take effect after the next reboot.
--- a/docs/intel_sgx.md
+++ b/docs/intel_sgx.md
@@ -8,7 +8,7 @@ the host kernel. The required Linux and KVM changes can be found in the
 [KVM SGX Tree](https://github.com/intel/kvm-sgx).

 Utilizing SGX in the guest requires a kernel/OS with SGX support, e.g. a kernel
-buit using the [SGX Linux Development Tree](https://github.com/jsakkine-intel/linux-sgx.git)
+built using the [SGX Linux Development Tree](https://github.com/jsakkine-intel/linux-sgx.git)
 or the [KVM SGX Tree](https://github.com/intel/kvm-sgx). Running KVM SGX as the
 guest kernel allows nested virtualization of SGX.

--- a/docs/iommu.md
+++ b/docs/iommu.md
@@ -86,7 +86,7 @@ virtual IOMMU:

 ```bash
 ./cloud-hypervisor \
-    --cpus 1 \
+    --cpus boot=1 \
    --memory size=512M \
    --disk path=focal-server-cloudimg-amd64.raw,iommu=on \
    --kernel custom-bzImage \
@@ -163,8 +163,8 @@ be consumed.

 ```bash
 ./cloud-hypervisor \
-    --cpus 1 \
-    --memory size=8G,file=/dev/hugepages \
+    --cpus boot=1 \
+    --memory size=8G,hugepages=on \
    --disk path=focal-server-cloudimg-amd64.raw \
    --kernel custom-bzImage \
    --cmdline "console=ttyS0 console=hvc0 root=/dev/vda1 rw hugepagesz=2M hugepages=2048" \
@@ -180,8 +180,8 @@ passing through is `0000:00:01.0`.

 ```bash
 ./cloud-hypervisor \
-    --cpus 1 \
-    --memory size=8G,file=/dev/hugepages \
+    --cpus boot=1 \
+    --memory size=8G,hugepages=on \
    --disk path=focal-server-cloudimg-amd64.raw \
    --kernel custom-bzImage \
    --cmdline "console=ttyS0 console=hvc0 root=/dev/vda1 rw kvm-intel.nested=1 vfio_iommu_type1.allow_unsafe_interrupts rw hugepagesz=2M hugepages=2048" \
@@ -200,8 +200,8 @@ Last thing is to start the L2 guest with the huge pages memory backend.

 ```bash
 ./cloud-hypervisor \
-    --cpus 1 \
-    --memory size=4G,file=/dev/hugepages \
+    --cpus boot=1 \
+    --memory size=4G,hugepages=on \
    --disk path=focal-server-cloudimg-amd64.raw \
    --kernel custom-bzImage \
    --cmdline "console=ttyS0 console=hvc0 root=/dev/vda1 rw" \
--- a/docs/logging.md
+++ b/docs/logging.md
@@ -0,0 +1,43 @@
+# Logging
+
+The target audience of this document is both:
+
+* Developers who want to understand what log level to use and when,
+* Users who want to debug issues with running their workloads in Cloud Hypervisor
+
+## Control
+
+The number of `-v` parameters passed to the `cloud-hypervisor` binary will determine the log level. Currenly the default is log messages up to `WARN:` (`warn!`) are included by default. The `--log-file` allows the log to be sent to a location other than `stderr`.
+
+## Levels
+
+### `error!()`
+
+For immediate, unrecoverable errors where it does not make sense for the execution to continue as the behaviour of the VM is considerablely impacted.
+
+Cloud Hypervisor should exit shortly after reporting this error (with a non-zero exit code). Generally this should be used during initial construction of the VM state before the virtual CPUs have begun running code.
+
+A typical situation where this might occur is when the user is using command line options that conflict with each other or is trying to use a file that is not present on the filesystem.
+
+Users should react to this error by checking their initial VM configuration.
+
+### `warn!()`
+
+A serious problem has occured but the execution of the VM can continue although some functionality might be impacted.
+
+A typical example of where this level of message should be generated is during an API call request that cannot be fulfilled.
+
+The user should investigate the meaning of this warning and take steps to ensure the correct functionality.
+
+
+### `info!()`
+
+Use `-v` to enable.
+
+This level is for the benefit of developers. It should be used for sporadic and infrequent messages. The same message should not "spam" the logs. The VM should be usable when this level of debugging is enabled and trying to use `stdin/stdout` and the logs are going to `stderr`.
+
+### `debug!()`
+
+Use `-vv` to enable.
+
+For the most verbose of logging messages. It is acceptable to "spam" the log with repeated invocations of the same message. This level of logging would be combined with `--log-file`.
--- a/docs/memory.md
+++ b/docs/memory.md
@@ -0,0 +1,451 @@
+# Memory
+
+Cloud-Hypervisor has many ways to expose memory to the guest VM. This document
+aims to explain what Cloud-Hypervisor is capable of and how it can be used to
+meet the needs of very different use cases.
+
+## Basic Parameters
+
+`MemoryConfig` or what is known as `--memory` from the CLI perspective is the
+easiest way to get started with Cloud-Hypervisor.
+
+```rust
+struct MemoryConfig {
+    size: u64,
+    mergeable: bool,
+    shared: bool,
+    hugepages: bool,
+    hotplug_method: HotplugMethod,
+    hotplug_size: Option<u64>,
+    hotplugged_size: Option<u64>,
+    zones: Option<Vec<MemoryZoneConfig>>,
+}
+```
+
+```
+--memory <memory>	Memory parameters "size=<guest_memory_size>,mergeable=on|off,shared=on|off,hugepages=on|off,hotplug_method=acpi|virtio-mem,hotplug_size=<hotpluggable_memory_size>,hotplugged_size=<hotplugged_memory_size>"
+```
+
+### `size`
+
+Size of the RAM in the guest VM.
+
+This option is mandatory when using the `--memory` parameter.
+
+Value is an unsigned integer of 64 bits.
+
+_Example_
+
+```
+--memory size=1G
+```
+
+### `mergeable`
+
+Specifies if the pages from the guest RAM must be marked as _mergeable_. In
+case this option is `true` or `on`, the pages will be marked with `madvise(2)`
+to let the host kernel know which pages are eligible for being merged by the
+KSM daemon.
+
+This option can be used when trying to reach a higher density of VMs running
+on a single host, as it will reduce the amount of memory consumed by each VM.
+
+By default this option is turned off.
+
+_Example_
+
+```
+--memory size=1G,mergeable=on
+```
+
+### `shared`
+
+Specifies if the memory must be `mmap(2)` with `MAP_SHARED` flag.
+
+By sharing a memory mapping, one can share the guest RAM with other processes
+running on the host. One can use this option when running vhost-user devices
+as part of the VM device model, as they will be driven by standalone daemons
+needing access to the guest RAM content.
+
+By default this option is turned off, which results in performing `mmap(2)`
+with `MAP_PRIVATE` flag.
+
+_Example_
+
+```
+--memory size=1G,shared=on
+```
+
+### `hugepages`
+
+Specifies if the memory must be `mmap(2)` with `MAP_HUGETLB` and `MAP_HUGE_2MB`
+flags. This performs a memory mapping relying on 2MiB pages instead of the
+default 4kiB pages.
+
+By using hugepages, one can improve the overall performance of the VM, assuming
+the guest will allocate hugepages as well. Another interesting use case is VFIO
+as it speeds up the VM's boot time since the amount of IOMMU mappings are
+reduced.
+
+By default this option is turned off.
+
+_Example_
+
+```
+--memory size=1G,hugepages=on
+```
+
+### `hotplug_method`
+
+Selects the way of adding and/or removing memory to/from a booted VM.
+
+Possible values are `acpi` and `virtio-mem`. Default value is `acpi`.
+
+_Example_
+
+```
+--memory size=1G,hotplug_method=acpi
+```
+
+### `hotplug_size`
+
+Amount of memory that can be dynamically added to the VM.
+
+Value is an unsigned integer of 64 bits. A value of 0 is invalid.
+
+_Example_
+
+```
+--memory size=1G,hotplug_size=1G
+```
+
+### `hotplugged_size`
+
+Amount of memory that will be dynamically added to the VM at boot. This option
+allows for starting a VM with a certain amount of memory that can be reduced
+during runtime.
+
+This is only valid when the `hotplug_method` is `virtio-mem` as it does not
+make sense for the `acpi` use case. When using ACPI, the memory can't be
+resized after it has been extended.
+
+This option is only valid when `hotplug_size` is specified, and its value can't
+exceed the value of `hotplug_size`.
+
+Value is an unsigned integer of 64 bits. A value of 0 is invalid.
+
+_Example_
+
+```
+--memory size=1G,hotplug_method=virtio-mem,hotplug_size=1G,hotplugged_size=512M
+```
+
+## Advanced Parameters
+
+`MemoryZoneConfig` or what is known as `--memory-zone` from the CLI perspective
+is a power user parameter. It allows for a full description of the guest RAM,
+describing how every memory region is backed and exposed to the guest.
+
+```rust
+struct MemoryZoneConfig {
+    id: String,
+    size: u64,
+    file: Option<PathBuf>,
+    shared: bool,
+    hugepages: bool,
+    host_numa_node: Option<u32>,
+    hotplug_size: Option<u64>,
+    hotplugged_size: Option<u64>,
+}
+```
+
+```
+--memory-zone <memory-zone>	User defined memory zone parameters "size=<guest_memory_region_size>,file=<backing_file>,shared=on|off,hugepages=on|off,host_numa_node=<node_id>,id=<zone_identifier>,hotplug_size=<hotpluggable_memory_size>,hotplugged_size=<hotplugged_memory_size>"
+```
+
+This parameter expects one or more occurences, allowing for a list of memory
+zones to be defined. It must be used with `--memory size=0`, clearly indicating
+that the memory will be described through advanced parameters.
+
+Each zone is given a list of options which we detail through the following
+sections.
+
+### `id`
+
+Memory zone identifier. This identifier must be unique, otherwise an error will
+be returned.
+
+This option is useful when referring to a memory zone previously created. In
+particular, the `--numa` parameter can associate a memory zone to a specific
+NUMA node based on the memory zone identifier.
+
+This option is mandatory when using the `--memory-zone` parameter.
+
+Value is a string.
+
+_Example_
+
+```
+--memory size=0
+--memory-zone id=mem0,size=1G
+```
+
+### `size`
+
+Size of the memory zone.
+
+This option is mandatory when using the `--memory-zone` parameter.
+
+Value is an unsigned integer of 64 bits.
+
+_Example_
+
+```
+--memory size=0
+--memory-zone id=mem0,size=1G
+```
+
+### `file`
+
+Path to the file backing the memory zone. This can be either a file or a
+directory. In case of a file, it will be opened and used as the backing file
+for the `mmap(2)` operation. In case of a directory, a temporary file with no
+hard link on the filesystem will be created. This file will be used as the
+backing file for the `mmap(2)` operation.
+
+This option can be particularly useful when trying to back a part of the guest
+RAM with a well known file. In the context of the snapshot/restore feature, and
+if the provided path is a file, the snapshot operation will not perform any
+copy of the guest RAM content for this specific memory zone since the user has
+access to it and it would duplicate data already stored on the current
+filesystem.
+
+Value is a string.
+
+_Example_
+
+```
+--memory size=0
+--memory-zone id=mem0,size=1G,file=/foo/bar
+```
+
+### `shared`
+
+Specifies if the memory zone must be `mmap(2)` with `MAP_SHARED` flag.
+
+By sharing a memory zone mapping, one can share part of the guest RAM with
+other processes running on the host. One can use this option when running
+vhost-user devices as part of the VM device model, as they will be driven
+by standalone daemons needing access to the guest RAM content.
+
+By default this option is turned off, which result in performing `mmap(2)`
+with `MAP_PRIVATE` flag.
+
+_Example_
+
+```
+--memory size=0
+--memory-zone id=mem0,size=1G,shared=on
+```
+
+### `hugepages`
+
+Specifies if the memory zone must be `mmap(2)` with `MAP_HUGETLB` and
+`MAP_HUGE_2MB` flags. This performs a memory zone mapping relying on 2MiB
+pages instead of the default 4kiB pages.
+
+By using hugepages, one can improve the overall performance of the VM, assuming
+the guest will allocate hugepages as well. Another interesting use case is VFIO
+as it speeds up the VM's boot time since the amount of IOMMU mappings are
+reduced.
+
+By default this option is turned off.
+
+_Example_
+
+```
+--memory size=0
+--memory-zone id=mem0,size=1G,hugepages=on
+```
+
+### `host_numa_node`
+
+Node identifier of a node present on the host. This option will let the user
+pick a specific NUMA node from which the memory must be allocated. After the
+memory zone is `mmap(2)`, the NUMA policy for this memory mapping will be
+applied through `mbind(2)`, relying on the provided node identifier. If the
+node does not exist on the host, the call to `mbind(2)` will fail.
+
+This option is useful when trying to back a VM memory with a specific type of
+memory from the host. Assuming a host has two types of memory, with one slower
+than the other, each related to a distinct NUMA node, one could create a VM
+with slower memory accesses by backing the entire guest RAM from the furthest
+NUMA node on the host.
+
+This option also gives the opportunity to create a VM with non uniform memory
+accesses as one could define a first memory zone backed by fast memory, and a
+second memory zone backed by slow memory.
+
+Value is an unsigned integer of 32 bits.
+
+_Example_
+
+```
+--memory size=0
+--memory-zone id=mem0,size=1G,host_numa_node=0
+```
+
+### `hotplug_size`
+
+Amount of memory that can be dynamically added to the memory zone. Since
+`virtio-mem` is the only way of resizing a memory zone, one must specify
+the `hotplug_method=virtio-mem` to the `--memory` parameter.
+
+Value is an unsigned integer of 64 bits. A value of 0 is invalid.
+
+_Example_
+
+```
+--memory size=0,hotplug_method=virtio-mem
+--memory-zone id=mem0,size=1G,hotplug_size=1G
+```
+
+### `hotplugged_size`
+
+Amount of memory that will be dynamically added to a memory zone at VM's boot.
+This option allows for starting a VM with a certain amount of memory that can
+be reduced during runtime.
+
+This is only valid when the `hotplug_method` is `virtio-mem` as it does not
+make sense for the `acpi` use case. When using ACPI, the memory can't be
+resized after it has been extended.
+
+This option is only valid when `hotplug_size` is specified, and its value can't
+exceed the value of `hotplug_size`.
+
+Value is an unsigned integer of 64 bits. A value of 0 is invalid.
+
+_Example_
+
+```
+--memory size=0,hotplug_method=virtio-mem
+--memory-zone id=mem0,size=1G,hotplug_size=1G,hotplugged_size=512M
+```
+
+## NUMA settings
+
+`NumaConfig` or what is known as `--numa` from the CLI perspective has been
+introduced to define a guest NUMA topology. It allows for a fine description
+about the CPUs and memory ranges associated with each NUMA node. Additionally
+it allows for specifying the distance between each NUMA node.
+
+```rust
+struct NumaConfig {
+    id: u32,
+    cpus: Option<Vec<u8>>,
+    distances: Option<Vec<NumaDistance>>,
+    memory_zones: Option<Vec<String>>,
+}
+```
+
+```
+--numa <numa>	Settings related to a given NUMA node "id=<node_id>,cpus=<cpus_id>,distances=<list_of_distances_to_destination_nodes>,memory_zones=<list_of_memory_zones>"
+```
+
+### `guest_numa_id`
+
+Node identifier of a guest NUMA node. This identifier must be unique, otherwise
+an error will be returned.
+
+This option is mandatory when using the `--numa` parameter.
+
+Value is an unsigned integer of 32 bits.
+
+_Example_
+
+```
+--numa guest_numa_id=0
+```
+
+### `cpus`
+
+List of virtual CPUs attached to the guest NUMA node identified by the
+`guest_numa_id` option. This allows for describing a list of CPUs which
+must be seen by the guest as belonging to the NUMA node `guest_numa_id`.
+
+One can use this option for a fine grained description of the NUMA topology
+regarding the CPUs associated with it, which might help the guest run more
+efficiently.
+
+Multiple values can be provided to define the list. Each value is an unsigned
+integer of 8 bits.
+
+For instance, if one needs to attach all CPUs from 0 to 4 to a specific node,
+the syntax using `-` will help define a contiguous range with `cpus=0-4`. The
+same example could also be described with `cpus=0:1:2:3:4`.
+
+A combination of both `-` and `:` separators is useful when one might need to
+describe a list containing all CPUs from 0 to 99 and the CPU 255, as it could
+simply be described with `cpus=0-99:255`.
+
+_Example_
+
+```
+--cpus boot=8
+--numa guest_numa_id=0,cpus=1-3:7
+--numa guest_numa_id=1,cpus=0:4-6
+```
+
+### `distances`
+
+List of distances between the current NUMA node referred by `guest_numa_id`
+and the destination NUMA nodes listed along with distances. This option let
+the user choose the distances between guest NUMA nodes. This is important to
+provide an accurate description of the way non uniform memory accesses will
+perform in the guest.
+
+One or more tuple of two values must be provided through this option. The first
+value is an unsigned integer of 32 bits as it represents the destination NUMA
+node. The second value is an unsigned integer of 8 bits as it represents the
+distance between the current NUMA node and the destination NUMA node. The two
+values are separated by `@` (`value1@value2`), meaning the destination NUMA
+node `value1` is located at a distance of `value2`. Each tuple is separated
+from the others with `:` separator.
+
+For instance, if one wants to define 3 NUMA nodes, with each node located at
+different distances, it can be described with the following example.
+
+_Example_
+
+```
+--numa guest_numa_id=0,distances=1@15:2@25
+--numa guest_numa_id=1,distances=0@15:2@20
+--numa guest_numa_id=2,distances=0@25:1@20
+```
+
+### `memory_zones`
+
+List of memory zones attached to the guest NUMA node identified by the
+`guest_numa_id` option. This allows for describing a list of memory ranges
+which must be seen by the guest as belonging to the NUMA node `guest_numa_id`.
+
+This option can be very useful and powerful when combined with `host_numa_node`
+option from `--memory-zone` parameter as it allows for creating a VM with non
+uniform memory accesses, and let the guest know about it. It allows for
+exposing memory zones through different NUMA nodes, which can help the guest
+workload run more efficiently.
+
+Multiple values can be provided to define the list. Each value is a string
+referring to an existing memory zone identifier. Values are separated from
+each other with the `:` separator.
+
+_Example_
+
+```
+--memory size=0
+--memory-zone id=mem0,size=1G
+--memory-zone id=mem1,size=1G
+--memory-zone id=mem2,size=1G
+--numa guest_numa_id=0,memory_zones=mem0:mem2
+--numa guest_numa_id=1,memory_zones=mem1
+```
--- a/docs/networking.md
+++ b/docs/networking.md
@@ -1,6 +1,6 @@
 # How to use networking

-cloud-hypervisor can emulate one or more virtual network interfaces, represented at the hypervisor host by [tap devices](https://www.kernel.org/doc/Documentation/networking/tuntap.txt). This guide briefly describes, in a manual and distribution neutral way, how to setup and use networking with cloud-hypevisor.
+cloud-hypervisor can emulate one or more virtual network interfaces, represented at the hypervisor host by [tap devices](https://www.kernel.org/doc/Documentation/networking/tuntap.txt). This guide briefly describes, in a manual and distribution neutral way, how to setup and use networking with cloud-hypervisor.

 ## Multiple queue support for net devices ##

@@ -14,7 +14,7 @@ Note:

 - Currently, it does not support to use ethtool to change the combined queue numbers in guest.
 - Multiple queue is enabled for vhost-user-net backend in cloud-hypervisor, however, multiple thread is not added to handle mq, thus, the performance for vhost-user-net backend is not supposed to be improved. The multiple thread will be added for backend later.
- Performance test for vhost-user-net will be covered once vhost-user-net backend has mulitple thread supported.
+- Performance test for vhost-user-net will be covered once vhost-user-net backend has multiple thread supported.
 - Performance test for virtio-net is done by comparing 2 queue pairs with 1 queue pairs, that to run 2 iperf3 sessions in the same test environments, throughput is improved about 37%.

 ## Start cloud-hypervisor with net devices
--- a/docs/snapshot_restore.md
+++ b/docs/snapshot_restore.md
@@ -54,7 +54,7 @@ drwxr-xr-x 47 foo bar       4096 Jul 22 11:47 ../
 In this particular example, we can observe that 2 memory region files were
 created. That is explained by the size of the guest RAM, which is 4GiB in this
 case. Because it exceeds 3GiB (which is where we can find a ~1GiB memory hole),
-Cloud-Hypervisor needs 2 distincts memory regions to be created. Each memory
+Cloud-Hypervisor needs 2 distinct memory regions to be created. Each memory
 region's content is stored through a dedicated file, which explains why we end
 up with 2 different files, the first one containing the guest RAM range 0-3GiB
 and the second one containing the guest RAM range 3-4GiB.
@@ -63,7 +63,7 @@ and the second one containing the guest RAM range 3-4GiB.
 and state. The configuration bits are used to create a similar virtual machine
 with the correct amount of CPUs, RAM, and other expected devices. The state
 bits are used to restore each component in the state it was left before the
-snapshot occured.
+snapshot occurred.

 ## Restore a Cloud-Hypervisor VM

--- a/docs/uefi.md
+++ b/docs/uefi.md
@@ -0,0 +1,42 @@
+# UEFI Boot
+
+Cloud Hypervisor supports UEFI boot through the utilization of the EDK II based UEFI firmware. 
+
+## Building UEFI Firmware
+
+To avoid any unnecessary issues, it is recommended to use Ubuntu 18.04 and its default toolset. Any other compatible Linux distribution is otherwise suitable, however it is suggested to use a temporary Docker container with Ubuntu 18.04 for a quick build on an existing Linux machine.
+
+The commands below will compile an OVMF firmware suitable for Cloud Hypervisor.
+
+```shell
+sudo apt-get update
+sudo apt-get install uuid-dev nasm iasl build-essential python3-distutils git
+
+git clone https://github.com/cloud-hypervisor/edk2 -b ch
+cd edk2
+. edksetup.sh
+git submodule update --init
+
+echo "ACTIVE_PLATFORM=OvmfPkg/OvmfCh.dsc" >> Conf/target.txt
+echo "TARGET_ARCH=X64" >> Conf/target.txt
+echo "TOOL_CHAIN_TAG=GCC5" >> Conf/target.txt
+
+make -C ./BaseTools
+build
+```
+
+After the successful build, the resulting firmware binaries are available under `Build/OvmfCh/DEBUG_GCC5/FV` underneath the edk2 checkout.
+
+## Using OVMF Binaries
+
+Any UEFI capable image can be booted using the Cloud Hypervisor specific firmware. Windows guests under Cloud Hypervisor only support UEFI boot, therefore OVMF is mandatory there.
+
+To make Cloud Hypervisor use UEFI boot, pass the `OVMF.fd` file path as an argument to the `--kernel` option. The firmware file will be opened in read only mode.
+
+The same firmware can be used with Cloud Hypervisor or with QEMU. This is particularly useful if using QEMU for the preparation phase.
+
+# Links
+
+- [OVMF wiki](https://github.com/tianocore/tianocore.github.io/wiki/OVMF) 
+- [Cloud Hypervisor specific tree](https://github.com/cloud-hypervisor/edk2/tree/ch)
+
--- a/docs/vhost-user-blk-testing.md
+++ b/docs/vhost-user-blk-testing.md
@@ -73,7 +73,7 @@ VMs run in client mode. They connect to the socket created by the `dpdkvhostuser
 # From the test terminal. We need to create one vhost-user-blk device for the --disk.
 ./cloud-hypervisor \
        --cpus boot=4 \
-        --memory size=1024M,hugepages=on \
+        --memory size=1024M,hugepages=on,shared=true \
        --kernel linux/arch/x86/boot/compressed/vmlinux.bin \
        --cmdline "console=ttyS0 root=/dev/vda1 rw iommu=off" \
        --disk path=images/focal-server-cloudimg-amd64.raw vhost_user=true,socket=/var/tmp/vhost.1,num_queues=4,queue_size=128 \
--- a/docs/vhost-user-net-testing.md
+++ b/docs/vhost-user-net-testing.md
@@ -76,7 +76,7 @@ VMs run in client mode. They connect to the socket created by the `dpdkvhostuser
 # From one terminal. We need to give the cloud-hypervisor binary the NET_ADMIN capabilities for it to set TAP interfaces up on the host.
 ./cloud-hypervisor \
        --cpus boot=2 \
-        --memory size=512M,hugepages=on   \
+        --memory size=512M,hugepages=on,shared=true \
        --kernel vmlinux \
        --cmdline "console=ttyS0 console=hvc0 root=/dev/vda1 rw" \
        --disk path=focal-server-cloudimg-amd64.raw   \
@@ -85,7 +85,7 @@ VMs run in client mode. They connect to the socket created by the `dpdkvhostuser
 # From another terminal. We need to give the cloud-hypervisor binary the NET_ADMIN capabilities for it to set TAP interfaces up on the host.
 ./cloud-hypervisor \
        --cpus boot=2 \
-        --memory size=512M,hugepages=on   \
+        --memory size=512M,hugepages=on,shared=true \
        --kernel vmlinux \
        --cmdline "console=ttyS0 console=hvc0 root=/dev/vda1 rw" \
        --disk path=focal-server-cloudimg-amd64.raw   \
--- a/docs/windows.md
+++ b/docs/windows.md
@@ -0,0 +1,326 @@
+# Windows Support
+
+Starting with the release version [0.10.0](https://github.com/cloud-hypervisor/cloud-hypervisor/releases/tag/v0.10.0), Cloud Hypervisor supports Windows guests.
+
+__Requirements__
+
+- Host with KVM enabled 
+- [UEFI](uefi.md) capable Windows guest image with Virtio drivers integrated
+
+Any modern Windows Server version is compatible. Cloud Hypervisor has been successfully tested with Windows Server 2019 and Windows Server Core 2004.
+
+At the current stage, only UEFI capable Windows images are supported. This implies the presence of the OVMF firmware during the Windows installation and in any subsequent usage. BIOS boot is not supported.
+
+The subsequent sections will tell, in detail, how to prepare an appropriate Windows image.
+
+## Image Preparation
+
+### Installation using the stock Windows ISO
+
+__Prerequisites__
+
+- QEMU, version >=5.0.0 is recommended.
+- Windows installation ISO. Obtained through MSDN, Visual Studio subscription, evaluation center, etc. 
+- [VirtIO driver ISO](https://fedorapeople.org/groups/virt/virtio-win/direct-downloads/stable-virtio/)
+- Suitable [OVMF](uefi.md) firmware 
+- With the suggested image size of 30G, there should be enough free disk space to hold the installation ISO and any other necessary files
+
+This step currently requires QEMU to install Windows onto the guest. QEMU is only used at the preparation stage, the resulting image is then fully functional with Cloud Hypervisor.
+
+Preparing several command parts as these will be used in the follow up sections as well.
+
+```shell
+IMG_FILE=windows-disk.qcow
+WIN_ISO_FILE=en_windows_server_version_2004_updated_may_2020_x64_dvd_1e7f1cfa.iso
+VIRTIO_ISO_FILE=virtio-win-0.1.185.iso
+OVMF_DIR=./FV
+```
+
+Create an empty image file, `qcow` or `raw` is supported.
+```shell
+qemu-img create -f qcow2 $IMG_FILE 30G
+```
+
+Begin the Windows installation process under QEMU
+```shell
+qemu-system-x86_64 \
+	-machine q35,accel=kvm \
+	-cpu host \
+	-m 4G \
+	-bios ./$OVMF_DIR/OVMF_CODE.fd \
+	-cdrom ./$WIN_ISO_FILE \
+	-drive file=./$VIRTIO_ISO_FILE,index=0,media=cdrom
+	-drive if=none,id=root,file=./$IMG_FILE \
+	-device virtio-blk-pci,drive=root,disable-legacy=on \
+	-device virtio-net-pci,netdev=mynet0,disable-legacy=on \
+	-netdev user,id=mynet0 \
+	-vga std
+```
+
+Before the installation can proceed, point the Windows installation program to the VirtIO disk and install the necessary storage controller drivers. After that, the attached hard drive will become visible and the actual installation can commence.
+
+After the installation has completed, proceed further to the configuration section. QEMU will be needed at least once more to enable the Windows Special Administration Console (SAC) and to possibly install extra device drivers.
+
+## Image Usage
+
+The basic command to boot a Windows image. The configuration section should be checked before executing it for the first time.
+
+```shell
+cloud-hypervisor \
+	--kernel ./$OVMF_DIR/OVMF.fd \
+	--disk path=./$IMG_FILE \
+	--cpus boot=1,kvm_hyperv=on \
+	--memory size=4G \
+	--serial tty \
+	--console off \
+	--net tap=
+```
+
+It is necessary to always:
+
+- Carry the OVMF firmware in the `--kernel` option
+- Add `kvm_hyperv=on` to the `--cpus` option
+
+In cases where the host processor supports address space > 39 bits, it might be necessary to limit the address space. It can be done by appending the option `max_phys_bits=X` to the `--cpus` parameter, where `X` is the number of bits to be supported. Windows was tested to support at least 39-bit address space.
+
+To daemonize the Cloud Hypervisor process, `nohup` can be used. Some STDIO redirections might need to be done. In a simple case it is sufficient to just redirect all the output to `/dev/null`.
+
+## Image Configuration
+
+### Device Drivers
+
+After the Windows installation has finished under QEMU, there might be still devices with no drivers installed. This might happen for example, when a device was not used during the installation. In particular it is important to ensure that the VirtIO network device is setup correctly because further steps for the configuration and the usage require network in most case.
+
+Boot once more under QEMU and use the [Device Manager](https://support.microsoft.com/en-in/help/4028443/windows-10-update-drivers), to ensure all the device drivers, and especially the network card, are installed correctly. Also, as Cloud Hypervisor can introduce new devices, it is advisable to repeat the procedure while booted under Cloud Hypervisor, when the RDP access to the image is functional.
+
+### Windows Special Administration Console (SAC) enablement
+
+SAC provides a text based console access to the Windows guest. As Cloud Hypervisor doesn't implement a VGA adaptor, SAC is an important instrument for the Windows guest management.
+
+Boot the Windows image under QEMU and execute the below commands to permanently enable SAC
+
+```cmd
+bcdedit /emssettings emsport:1 emsbaudrate:115200
+bcdedit /ems on
+bcdedit /bootems on
+```
+
+Once SAC is enabled, the image can be booted under Cloud Hypervisor. The SAC prompt will show up 
+
+<pre>
+Computer is booting, SAC started and initialized.                               
+                                                                                
+Use the "ch -?" command for information about using channels.                   
+Use the "?" command for general help.                                           
+                                                                                
+                                                                                
+SAC>
+</pre>
+
+To open a console on the guest, the command sequence below can be used
+<pre>
+SAC>cmd
+The Command Prompt session was successfully launched.
+SAC>
+EVENT:   A new channel has been created.  Use "ch -?" for channel help.
+Channel: Cmd0001
+SAC>ch -si 1
+</pre>
+
+See also the [links](#Links) section for a more extended SAC documentation.
+
+## Network
+
+This section illustrates the Windows specific corner points for the VM network configuration. For the extended networking guide, including bridging for multiple VMs, follow [networking.md](networking.md).
+
+### Basic Networking
+
+As the simplest option, using `--net tap=` in the Cloud Hypervisor command line will create a `vmtapX` device on the host with the default IPv4 adress `192.168.249.1`. After SAC becomes available, the guest configuration can be set with
+
+<pre>
+SAC>i 10 192.168.249.2 255.255.255.0 192.168.249.1
+</pre> 
+
+Where `10` is the device index as shown by the `i` command.
+
+### Guest Internet Connectivity
+
+Additional steps are necessary to provide the guest with internet access.
+
+- On the guest, add the DNS server either by using `netsh` or by opening `Network and Connectivity Center` and editing the adapter properties.
+- On the host, configure the traffic forwarding. Replace the `NET_DEV` with the name of your network device.
+```shell
+NET_DEV=wlp3s0
+sysctl -w net.ipv4.ip_forward=1
+iptables -t nat -A POSTROUTING -o $NET_DEV -j MASQUERADE
+```
+
+### Remote Desktop Protocol (RDP) enablement
+
+#### Using QEMU
+  - Execute `SystemPropertiesRemote`
+  - In the properties window, choose "Allow remote connections to this computer"
+  - Click "Select Users" and add some user to the allow list
+#### Using powershell
+```powershell
+Set-ItemProperty "HKLM:\SYSTEM\CurrentControlSet\Control\Terminal Server\" -Name "fDenyTSConnections" -Value 0
+Enable-NetFirewallRule -DisplayGroup "Remote Desktop"
+Add-LocalGroupMember -Group "Remote Desktop Users" -Member someuser
+```
+ 
+Administrators can always RDP, non administrator users have to be explicitly enabled. 
+
+Once the configuration is set, RDP clients can connect to `192.168.249.2`.
+
+### SSH
+
+#### Enable using powershell
+
+```powershell
+Add-WindowsCapability -Online -Name OpenSSH.Server~~~~0.0.1.0
+Start-Service sshd
+Set-Service -Name sshd -StartupType ‘Automatic’
+```
+
+This allows for SSH login from a remote machine, for example through the `administrator` user: `ssh administrator@192.168.249.2`. For a more detailed OpenSSH guide, please follow the MSDN article from the [links](#links) section.
+
+## Debugging
+
+The Windows guest debugging process relies heavily on QEMU and [socat](http://www.dest-unreach.org/socat/). The procedure requires two Windows VMs:
+
+- A debugger VM running under QEMU.
+- A debuggee, a Windows VM that has been created in the previous steps, running under Cloud Hypervisor or QEMU.
+
+The connection between both guests happens over TCP, whereby on the guest side it is automatically translated to a COM port. Because the VMs are connected through TCP, the debugging infrastructure can be distributed over the network. The serial port, while slowly transferring data, is common enough to support a wide range of cases and tools.
+
+In this excercise, [WinDbg](https://docs.microsoft.com/en-us/windows-hardware/drivers/debugger/) is used. Any other debugger of choice with the ability to use serial connection can be used instead. 
+
+### Debugger and Debuggee
+
+#### WinDbg VM
+
+For simplicity, the debugger VM is supposed to be only running under QEMU. It will require VGA and doesn't neccessarily depend on UEFI. As an OS, it can carry any supported Windows OS where the debugger of choice can be installed. The simplest way is to follow the image preparation instructions from the previous chapter, but avoid using the OVMF firmware. It is also not required to use VirtIO drivers, whereby it might be useful in some case. Though, while creating the image file for the debugger VM, be sure to choose a sufficient disk size that counts in the need to save the corresponding debug symbols and sources.
+
+To create the debugger Windows VM, the following command can be used:
+
+```shell
+qemu-system-x86_64 \
+	-machine q35,accel=kvm \
+	-cpu host \
+	-smp 1 \
+	-m 4G \
+	-cdrom ./$WIN_ISO_FILE \
+	-drive file=./$VIRTIO_ISO_FILE,index=0,media=cdrom
+	-drive if=none,id=root,file=./windbg-disk.qcow \
+	-device virtio-blk-pci,drive=root,disable-legacy=on \
+	-device virtio-net-pci,netdev=mynet0,disable-legacy=on \
+	-netdev user,id=mynet0,net=192.168.178.0/24,host=192.168.178.1,dhcpstart=192.168.178.64,hostname=windbg-host \
+	-vga std
+```
+
+A non server Windows OS like Windows 10 can be used to carry the debugging tools in the debugger VM.
+
+#### Debuggee VM
+
+The debuggee VM is the one that we've learned to configure and run in the first section. There might be various reasons to debug. For example, there could be an issue in the Windows guest with an emulated device or an included driver. Or, we might want to develop a custom feature like a kernel driver to be available in the guest.
+
+Note, that there are several ways to debug Windows, not all of them need to be enabled at the same time. For example, if developing a kernel module, the only useful options would be to configure for the serial debugging and enable the kernel debug. In that case, any crash or misbehavior in the boot loader or kernel would be ignored. The commands below must be run as administrator on the debuggee guest VM.
+
+##### Turn On Serial Debugging
+
+This will configure the debugging to be enabled and instruct to use the serial port for it.
+
+```cmd
+bcdedit /dbgsettings serial debugport:1 baudrate:115200
+```
+
+##### Turn On Kernel Debuging
+
+```cmd
+bcdedit /debug on
+```
+
+##### Turn On Boot Loader Debug
+
+```cmd
+bcdedit /bootdebug on
+```
+
+##### Turn on boot manager debug 
+
+```cmd
+bcdedit /set {bootmgr} bootdebug on
+```
+
+##### Disable Recovery Screen On Boot Failure
+
+There could be a situation, where a crash is debugged. In such cases, the guest could be left in an inconsistent state. The default Windows behavior would be to boot into the recovery screen, however in some cases it might be not desired. To make Windows ignore failures and always proceed to booting the OS, use the command below:
+
+```cmd
+bcdedit /set {default} bootstatuspolicy ignoreallfailures
+```
+
+### Debugging Process
+
+#### Invoke the WinDbg VM
+
+```shell
+qemu-system-x86_64 \
+	-machine q35,accel=kvm \
+	-cpu host \
+	-smp 1 \
+	-m 4G \
+	-drive if=none,id=root,file=./windbg-disk.qcow \
+	-device virtio-blk-pci,drive=root,disable-legacy=on \
+	-serial tcp::4445,server,nowait \
+	-device virtio-net-pci,netdev=mynet0,disable-legacy=on \
+	-netdev user,id=mynet0,net=192.168.178.0/24,host=192.168.178.1,dhcpstart=192.168.178.64,hostname=windbg-host \
+	-vga std
+```
+
+Note, this VM has the networking enabled. It is needed, because symbols and sources might need to be fetched from a network location.
+
+Also, notice the `-serial` parameter - that's what does the magic on exposing the serial port to the guest while connecting the debugger VM with a client VM through the network. SAC/EMS needs to be disabled in the debugger VM, as otherwise the COM device might be blocked.
+
+#### Invoke the Debuggee VM
+
+##### Under QEMU
+
+Essentially it would be the command like depicted in the guest preparation sections, with a few modifications:
+```shell
+qemu-system-x86_64 \
+	-machine q35,accel=kvm \
+	-cpu host \
+	-m 4G \
+	-bios ./$OVMF_DIR/OVMF_CODE.fd \
+	-cdrom ./$WIN_ISO_FILE \
+	-drive file=./$VIRTIO_ISO_FILE,index=0,media=cdrom
+	-drive if=none,id=root,file=./$IMG_FILE \
+	-device virtio-blk-pci,drive=root,disable-legacy=on \
+	-device virtio-net-pci,netdev=mynet0,disable-legacy=on \
+	-netdev user,id=mynet0 \
+	-serial tcp:127.0.0.1:4445 \
+	-vga std
+```
+
+It is to see, that `-serial` parameter is used here, to establish the connection with the debugger VM.
+
+To disable HPET, attach `--no-hpet`. To enable hypervisor reference timer, use `-cpu host,hv-time`. These and other options can be used to achieve better [Hyper-V compatibility](https://archive.fosdem.org/2019/schedule/event/vai_enlightening_kvm/attachments/slides/2860/export/events/attachments/vai_enlightening_kvm/slides/2860/vkuznets_fosdem2019_enlightening_kvm.pdf).
+
+##### Cloud Hypervisor
+
+The `socat` tool is used to establish the QEMU compatible behavior. Here as well, the Cloud Hypervisor command used to run the Windows guest is to be used. Put the command into a shell script:
+
+`socat SYSTEM:"./ch-script",openpty,raw,echo=0 TCP:localhost:4445`
+
+The reason to pack the command into the shell script is that the command might contain a comma. When using SYSTEM, the shell command can't contain `,` or `!!`.
+
+## Links
+
+- [Fedora VirtIO guide for Windows](https://docs.fedoraproject.org/en-US/quick-docs/creating-windows-virtual-machines-using-virtio-drivers/)
+- [VirtIO driver binaries](https://fedorapeople.org/groups/virt/virtio-win/direct-downloads/stable-virtio/)
+- [VirtIO driver sources](https://github.com/virtio-win/kvm-guest-drivers-windows)
+- [Emergency Management Services](https://docs.microsoft.com/en-us/previous-versions/windows/it-pro/windows-server-2003/cc787940(v=ws.10))
+- [OpenSSH server/client configuration](https://docs.microsoft.com/en-us/windows-server/administration/openssh/openssh_install_firstuse)
+- [Windows guest debugging under KVM](https://www.linux-kvm.org/page/WindowsGuestDrivers/GuestDebugging)
+- ["ENLIGHTENING" KVM](https://archive.fosdem.org/2019/schedule/event/vai_enlightening_kvm/attachments/slides/2860/export/events/attachments/vai_enlightening_kvm/slides/2860/vkuznets_fosdem2019_enlightening_kvm.pdf)
--- a/fuzz/Cargo.toml
+++ b/fuzz/Cargo.toml
@@ -10,19 +10,25 @@ edition = "2018"
 cargo-fuzz = true

 [dependencies]
+kvm-ioctls = { git = "https://github.com/rust-vmm/kvm-ioctls", branch = "master" }
 libc = "0.2.72"
 libfuzzer-sys = "0.3"
 qcow = { path = "../qcow" }
+seccomp = { git = "https://github.com/firecracker-microvm/firecracker", tag = "v0.22.0" }
 virtio-devices = { path = "../virtio-devices" }
 vmm-sys-util = ">=0.3.1"
 vm-virtio = { path = "../vm-virtio" }
-vm-memory = "0.2.1"
+vm-memory = "0.3.0"

 [dependencies.cloud-hypervisor]
 path = ".."

 [patch.crates-io]
 vm-memory = { git = "https://github.com/cloud-hypervisor/vm-memory", branch = "ch" }
+[patch.'https://github.com/rust-vmm/kvm-ioctls']
+kvm-ioctls = { git = "https://github.com/cloud-hypervisor/kvm-ioctls", branch = "ch" }
+[patch.'https://github.com/rust-vmm/kvm-bindings']
+kvm-bindings = { git = "https://github.com/cloud-hypervisor/kvm-bindings", branch = "ch", features = ["with-serde", "fam-wrappers"] }

 # Prevent this from interfering with workspaces
 [workspace]
--- a/fuzz/fuzz_targets/block.rs
+++ b/fuzz/fuzz_targets/block.rs
@@ -16,6 +16,7 @@ use virtio_devices::{Block, VirtioDevice, VirtioInterrupt, VirtioInterruptType};
 use vm_memory::{Bytes, GuestAddress, GuestMemoryAtomic, GuestMemoryMmap};
 use vm_virtio::Queue;
 use vmm_sys_util::eventfd::EventFd;
+use seccomp::SeccompAction;

 const MEM_SIZE: u64 = 256 * 1024 * 1024;
 const DESC_SIZE: u64 = 16; // Bytes in one virtio descriptor.
@@ -92,6 +93,7 @@ fuzz_target!(|bytes| {
        false,
        2,
        256,
+        SeccompAction::Allow,
    )
    .unwrap();

--- a/hypervisor/Cargo.toml
+++ b/hypervisor/Cargo.toml
@@ -1,23 +1,25 @@
 [package]
 name = "hypervisor"
 version = "0.1.0"
-authors = ["Muminul Islam <muislam@microsoft.com>"]
+authors = ["Microsoft Authors"]
 edition = "2018"
+license = "Apache-2.0 OR BSD-3-Clause"

 [features]
 kvm = []

 [dependencies]
 anyhow = "1.0"
+arc-swap = ">=0.4.4"
 thiserror = "1.0"
-libc = "0.2.74"
+libc = "0.2.80"
 log = "0.4.11"
-kvm-ioctls = { git = "https://github.com/cloud-hypervisor/kvm-ioctls", branch = "ch" }
-kvm-bindings = { git = "https://github.com/cloud-hypervisor/kvm-bindings", branch = "ch", features = ["with-serde", "fam-wrappers"] }
+kvm-ioctls = { git = "https://github.com/rust-vmm/kvm-ioctls", branch = "master" }
+kvm-bindings = { git = "https://github.com/rust-vmm/kvm-bindings", branch = "master", features = ["with-serde", "fam-wrappers"] }
 serde = {version = ">=1.0.27", features = ["rc"] }
 serde_derive = ">=1.0.27"
 serde_json = ">=1.0.9"
-vm-memory = { version = "0.2.0", features = ["backend-mmap", "backend-atomic"] }
+vm-memory = { version = "0.3.0", features = ["backend-mmap", "backend-atomic"] }
 vmm-sys-util = { version = ">=0.5.0", features = ["with-serde"] }

 [dependencies.linux-loader]
--- a/hypervisor/src/cpu.rs
+++ b/hypervisor/src/cpu.rs
@@ -10,6 +10,8 @@

 #[cfg(target_arch = "aarch64")]
 use crate::aarch64::VcpuInit;
+#[cfg(target_arch = "aarch64")]
+use crate::aarch64::{RegList, Register, StandardRegisters};
 use crate::{CpuState, MpState};

 #[cfg(target_arch = "x86_64")]
@@ -137,17 +139,47 @@ pub enum HypervisorCpuError {
    /// Setting one reg error
    ///
    #[error("Failed to init vcpu: {0}")]
-    SetOneReg(#[source] anyhow::Error),
+    SetRegister(#[source] anyhow::Error),
    ///
    /// Getting one reg error
    ///
    #[error("Failed to init vcpu: {0}")]
-    GetOneReg(#[source] anyhow::Error),
+    GetRegister(#[source] anyhow::Error),
    ///
    /// Getting guest clock paused error
    ///
    #[error("Failed to notify guest its clock was paused: {0}")]
    NotifyGuestClockPaused(#[source] anyhow::Error),
+    ///
+    /// Enabling HyperV SynIC error
+    ///
+    #[error("Failed to enable HyperV SynIC")]
+    EnableHyperVSynIC(#[source] anyhow::Error),
+    ///
+    /// Getting AArch64 core register error
+    ///
+    #[error("Failed to get core register: {0}")]
+    GetCoreRegister(#[source] anyhow::Error),
+    ///
+    /// Setting AArch64 core register error
+    ///
+    #[error("Failed to set core register: {0}")]
+    SetCoreRegister(#[source] anyhow::Error),
+    ///
+    /// Getting AArch64 registers list error
+    ///
+    #[error("Failed to retrieve list of registers: {0}")]
+    GetRegList(#[source] anyhow::Error),
+    ///
+    /// Getting AArch64 system register error
+    ///
+    #[error("Failed to get system register: {0}")]
+    GetSysRegister(#[source] anyhow::Error),
+    ///
+    /// Setting AArch64 system register error
+    ///
+    #[error("Failed to set system register: {0}")]
+    SetSysRegister(#[source] anyhow::Error),
 }

 #[derive(Debug)]
@@ -162,6 +194,7 @@ pub enum VmExit<'a> {
    MmioWrite(u64 /* address */, &'a [u8]),
    Ignore,
    Reset,
+    Hyperv,
 }

 ///
@@ -210,6 +243,11 @@ pub trait Vcpu: Send + Sync {
    fn set_cpuid2(&self, cpuid: &CpuId) -> Result<()>;
    #[cfg(target_arch = "x86_64")]
    ///
+    /// X86 specific call to enable HyperV SynIC
+    ///
+    fn enable_hyperv_synic(&self) -> Result<()>;
+    #[cfg(target_arch = "x86_64")]
+    ///
    /// X86 specific call to retrieve the CPUID registers.
    ///
    fn get_cpuid2(&self, num_entries: usize) -> Result<CpuId>;
@@ -288,12 +326,43 @@ pub trait Vcpu: Send + Sync {
    /// Sets the value of one register for this vCPU.
    ///
    #[cfg(any(target_arch = "arm", target_arch = "aarch64"))]
-    fn set_one_reg(&self, reg_id: u64, data: u64) -> Result<()>;
+    fn set_reg(&self, reg_id: u64, data: u64) -> Result<()>;
    ///
    /// Sets the value of one register for this vCPU.
    ///
    #[cfg(any(target_arch = "arm", target_arch = "aarch64"))]
-    fn get_one_reg(&self, reg_id: u64) -> Result<u64>;
+    fn get_reg(&self, reg_id: u64) -> Result<u64>;
+    ///
+    /// Gets a list of the guest registers that are supported for the
+    /// KVM_GET_ONE_REG/KVM_SET_ONE_REG calls.
+    ///
+    #[cfg(any(target_arch = "arm", target_arch = "aarch64"))]
+    fn get_reg_list(&self, reg_list: &mut RegList) -> Result<()>;
+    ///
+    /// Save the state of the core registers.
+    ///
+    #[cfg(any(target_arch = "arm", target_arch = "aarch64"))]
+    fn core_registers(&self, state: &mut StandardRegisters) -> Result<()>;
+    ///
+    /// Restore the state of the core registers.
+    ///
+    #[cfg(any(target_arch = "arm", target_arch = "aarch64"))]
+    fn set_core_registers(&self, state: &StandardRegisters) -> Result<()>;
+    ///
+    /// Save the state of the system registers.
+    ///
+    #[cfg(any(target_arch = "arm", target_arch = "aarch64"))]
+    fn system_registers(&self, state: &mut Vec<Register>) -> Result<()>;
+    ///
+    /// Restore the state of the system registers.
+    ///
+    #[cfg(any(target_arch = "arm", target_arch = "aarch64"))]
+    fn set_system_registers(&self, state: &[Register]) -> Result<()>;
+    ///
+    /// Read the MPIDR - Multiprocessor Affinity Register.
+    ///
+    #[cfg(any(target_arch = "arm", target_arch = "aarch64"))]
+    fn read_mpidr(&self) -> Result<u64>;
    ///
    /// Retrieve the vCPU state.
    /// This function is necessary to snapshot the VM
@@ -304,7 +373,6 @@ pub trait Vcpu: Send + Sync {
    /// This function is required when restoring the VM
    ///
    fn set_state(&self, state: &CpuState) -> Result<()>;
-
    ///
    /// Triggers the running of the current virtual CPU returning an exit reason.
    ///
--- a/hypervisor/src/device.rs
+++ b/hypervisor/src/device.rs
@@ -22,6 +22,11 @@ pub enum HypervisorDeviceError {
    ///
    #[error("Failed to set device attribute: {0}")]
    SetDeviceAttribute(#[source] anyhow::Error),
+    ///
+    /// Get device attribute error
+    ///
+    #[error("Failed to get device attribute: {0}")]
+    GetDeviceAttribute(#[source] anyhow::Error),
 }

 ///
@@ -37,4 +42,6 @@ pub type Result<T> = std::result::Result<T, HypervisorDeviceError>;
 pub trait Device: Send + Sync + AsRawFd {
    /// Set device attribute.
    fn set_device_attr(&self, attr: &DeviceAttr) -> Result<()>;
+    /// Get device attribute.
+    fn get_device_attr(&self, attr: &mut DeviceAttr) -> Result<()>;
 }
--- a/hypervisor/src/hypervisor.rs
+++ b/hypervisor/src/hypervisor.rs
@@ -46,20 +46,20 @@ pub enum HypervisorError {
    #[error("Failed to get number of max vcpus: {0}")]
    GetMaxVcpu(#[source] anyhow::Error),
    ///
-    /// Recommended Vcpu error
-    ///
-    #[error("Failed to get number of max vcpus: {0}")]
-    GetNrVcpus(#[source] anyhow::Error),
-    ///
    /// CpuId error
    ///
-    #[error("Failed to get number of max vcpus: {0}")]
+    #[error("Failed to get cpuid: {0}")]
    GetCpuId(#[source] anyhow::Error),
    ///
    /// Failed to retrieve list of MSRs.
    ///
    #[error("Failed to get the list of supported MSRs: {0}")]
    GetMsrList(#[source] anyhow::Error),
+    ///
+    /// API version is not compatible
+    ///
+    #[error("Incompatible API version")]
+    IncompatibleApiVersion,
 }

 ///
@@ -79,10 +79,6 @@ pub trait Hypervisor: Send + Sync {
    ///
    fn create_vm(&self) -> Result<Arc<dyn Vm>>;
    ///
-    /// Get the API version of the hypervisor
-    ///
-    fn get_api_version(&self) -> i32;
-    ///
    /// Returns the size of the memory mapping required to use the vcpu's structures
    ///
    fn get_vcpu_mmap_size(&self) -> Result<usize>;
--- a/hypervisor/src/kvm/aarch64/mod.rs
+++ b/hypervisor/src/kvm/aarch64/mod.rs
@@ -12,16 +12,134 @@
 /// Export generically-named wrappers of kvm-bindings for Unix-based platforms
 ///
 use crate::kvm::{KvmError, KvmResult};
-pub use kvm_bindings::kvm_vcpu_init as VcpuInit;
+use kvm_bindings::{
+    kvm_mp_state, kvm_one_reg, kvm_regs, KVM_REG_ARM64, KVM_REG_ARM64_SYSREG,
+    KVM_REG_ARM64_SYSREG_CRM_MASK, KVM_REG_ARM64_SYSREG_CRM_SHIFT, KVM_REG_ARM64_SYSREG_CRN_MASK,
+    KVM_REG_ARM64_SYSREG_CRN_SHIFT, KVM_REG_ARM64_SYSREG_OP0_MASK, KVM_REG_ARM64_SYSREG_OP0_SHIFT,
+    KVM_REG_ARM64_SYSREG_OP1_MASK, KVM_REG_ARM64_SYSREG_OP1_SHIFT, KVM_REG_ARM64_SYSREG_OP2_MASK,
+    KVM_REG_ARM64_SYSREG_OP2_SHIFT, KVM_REG_ARM_COPROC_MASK, KVM_REG_ARM_CORE, KVM_REG_SIZE_MASK,
+    KVM_REG_SIZE_U32, KVM_REG_SIZE_U64,
+};
+pub use kvm_bindings::{
+    kvm_one_reg as Register, kvm_regs as StandardRegisters, kvm_vcpu_init as VcpuInit, RegList,
+};
 use serde_derive::{Deserialize, Serialize};
 pub use {kvm_ioctls::Cap, kvm_ioctls::Kvm};

+// Following are macros that help with getting the ID of a aarch64 core register.
+// The core register are represented by the user_pt_regs structure. Look for it in
+// arch/arm64/include/uapi/asm/ptrace.h.
+
+// This macro gets the offset of a structure (i.e `str`) member (i.e `field`) without having
+// an instance of that structure.
+// It uses a null pointer to retrieve the offset to the field.
+// Inspired by C solution: `#define offsetof(str, f) ((size_t)(&((str *)0)->f))`.
+// Doing `offset__of!(user_pt_regs, pstate)` in our rust code will trigger the following:
+// unsafe { &(*(0 as *const user_pt_regs)).pstate as *const _ as usize }
+// The dereference expression produces an lvalue, but that lvalue is not actually read from,
+// we're just doing pointer math on it, so in theory, it should safe.
+#[macro_export]
+macro_rules! offset__of {
+    ($str:ty, $field:ident) => {
+        unsafe { &(*std::ptr::null::<$str>()).$field as *const _ as usize }
+    };
+}
+
+// Get the ID of a core register
+#[macro_export]
+macro_rules! arm64_core_reg_id {
+    ($size: tt, $offset: tt) => {
+        // The core registers of an arm64 machine are represented
+        // in kernel by the `kvm_regs` structure. This structure is a
+        // mix of 32, 64 and 128 bit fields:
+        // struct kvm_regs {
+        //     struct user_pt_regs      regs;
+        //
+        //     __u64                    sp_el1;
+        //     __u64                    elr_el1;
+        //
+        //     __u64                    spsr[KVM_NR_SPSR];
+        //
+        //     struct user_fpsimd_state fp_regs;
+        // };
+        // struct user_pt_regs {
+        //     __u64 regs[31];
+        //     __u64 sp;
+        //     __u64 pc;
+        //     __u64 pstate;
+        // };
+        // The id of a core register can be obtained like this:
+        // offset = id & ~(KVM_REG_ARCH_MASK | KVM_REG_SIZE_MASK | KVM_REG_ARM_CORE). Thus,
+        // id = KVM_REG_ARM64 | KVM_REG_SIZE_U64/KVM_REG_SIZE_U32/KVM_REG_SIZE_U128 | KVM_REG_ARM_CORE | offset
+        KVM_REG_ARM64 as u64
+            | u64::from(KVM_REG_ARM_CORE)
+            | $size
+            | (($offset / mem::size_of::<u32>()) as u64)
+    };
+}
+
+// This macro computes the ID of a specific ARM64 system register similar to how
+// the kernel C macro does.
+// https://elixir.bootlin.com/linux/v4.20.17/source/arch/arm64/include/uapi/asm/kvm.h#L203
+#[macro_export]
+macro_rules! arm64_sys_reg {
+    ($name: tt, $op0: tt, $op1: tt, $crn: tt, $crm: tt, $op2: tt) => {
+        pub const $name: u64 = KVM_REG_ARM64 as u64
+            | KVM_REG_SIZE_U64 as u64
+            | KVM_REG_ARM64_SYSREG as u64
+            | ((($op0 as u64) << KVM_REG_ARM64_SYSREG_OP0_SHIFT)
+                & KVM_REG_ARM64_SYSREG_OP0_MASK as u64)
+            | ((($op1 as u64) << KVM_REG_ARM64_SYSREG_OP1_SHIFT)
+                & KVM_REG_ARM64_SYSREG_OP1_MASK as u64)
+            | ((($crn as u64) << KVM_REG_ARM64_SYSREG_CRN_SHIFT)
+                & KVM_REG_ARM64_SYSREG_CRN_MASK as u64)
+            | ((($crm as u64) << KVM_REG_ARM64_SYSREG_CRM_SHIFT)
+                & KVM_REG_ARM64_SYSREG_CRM_MASK as u64)
+            | ((($op2 as u64) << KVM_REG_ARM64_SYSREG_OP2_SHIFT)
+                & KVM_REG_ARM64_SYSREG_OP2_MASK as u64);
+    };
+}
+
+// Constant imported from the Linux kernel:
+// https://elixir.bootlin.com/linux/v4.20.17/source/arch/arm64/include/asm/sysreg.h#L135
+arm64_sys_reg!(MPIDR_EL1, 3, 0, 0, 0, 5);
+
+/// Specifies whether a particular register is a system register or not.
+/// The kernel splits the registers on aarch64 in core registers and system registers.
+/// So, below we get the system registers by checking that they are not core registers.
+///
+/// # Arguments
+///
+/// * `regid` - The index of the register we are checking.
+pub fn is_system_register(regid: u64) -> bool {
+    if (regid & KVM_REG_ARM_COPROC_MASK as u64) == KVM_REG_ARM_CORE as u64 {
+        return false;
+    }
+
+    let size = regid & KVM_REG_SIZE_MASK;
+    if size != KVM_REG_SIZE_U32 && size != KVM_REG_SIZE_U64 {
+        panic!("Unexpected register size for system register {}", size);
+    }
+    true
+}
+
 pub fn check_required_kvm_extensions(kvm: &Kvm) -> KvmResult<()> {
    if !kvm.check_extension(Cap::SignalMsi) {
        return Err(KvmError::CapabilityMissing(Cap::SignalMsi));
    }
+    if !kvm.check_extension(Cap::OneReg) {
+        return Err(KvmError::CapabilityMissing(Cap::OneReg));
+    }
    Ok(())
 }

-#[derive(Clone, Serialize, Deserialize)]
-pub struct VcpuKvmState {}
+#[derive(Clone, Default, Serialize, Deserialize)]
+pub struct VcpuKvmState {
+    pub mp_state: kvm_mp_state,
+    pub core_regs: kvm_regs,
+    pub sys_regs: Vec<kvm_one_reg>,
+    // We will be using the mpidr for passing it to the VmState.
+    // The VmState will give this away for saving restoring the icc and redistributor
+    // registers.
+    pub mpidr: u64,
+}
--- a/hypervisor/src/kvm/mod.rs
+++ b/hypervisor/src/kvm/mod.rs
@@ -8,20 +8,28 @@
 //
 //

+#[cfg(target_arch = "aarch64")]
+pub use crate::aarch64::{
+    check_required_kvm_extensions, is_system_register, VcpuInit, VcpuKvmState as CpuState,
+    MPIDR_EL1,
+};
+use crate::cpu;
+use crate::device;
+use crate::hypervisor;
+use crate::vm::{self, VmmOps};
+#[cfg(target_arch = "aarch64")]
+use crate::{arm64_core_reg_id, offset__of};
+use arc_swap::ArcSwapOption;
 use kvm_ioctls::{NoDatamatch, VcpuFd, VmFd};
+use serde_derive::{Deserialize, Serialize};
 use std::os::unix::io::{AsRawFd, RawFd};
 use std::result;
+#[cfg(target_arch = "x86_64")]
+use std::sync::atomic::{AtomicBool, Ordering};
 use std::sync::Arc;
 #[cfg(target_arch = "x86_64")]
 use vm_memory::Address;
 use vmm_sys_util::eventfd::EventFd;
-
-#[cfg(target_arch = "aarch64")]
-pub use crate::aarch64::{check_required_kvm_extensions, VcpuInit, VcpuKvmState as CpuState};
-use crate::cpu;
-use crate::device;
-use crate::hypervisor;
-use crate::vm;
 // x86_64 dependencies
 #[cfg(target_arch = "x86_64")]
 pub mod x86_64;
@@ -31,6 +39,9 @@ use x86_64::{
    check_required_kvm_extensions, FpuState, SpecialRegisters, StandardRegisters, KVM_TSS_ADDRESS,
 };

+#[cfg(target_arch = "aarch64")]
+use aarch64::{RegList, Register, StandardRegisters};
+
 #[cfg(target_arch = "x86_64")]
 pub use x86_64::{
    CpuId, CpuIdEntry, ExtendedControlRegisters, LapicState, MsrEntries, VcpuKvmState as CpuState,
@@ -38,7 +49,9 @@ pub use x86_64::{
 };

 #[cfg(target_arch = "x86_64")]
-use kvm_bindings::{kvm_enable_cap, MsrList, KVM_CAP_SPLIT_IRQCHIP};
+use kvm_bindings::{
+    kvm_enable_cap, kvm_msr_entry, MsrList, KVM_CAP_HYPERV_SYNIC, KVM_CAP_SPLIT_IRQCHIP,
+};

 #[cfg(target_arch = "x86_64")]
 use crate::arch::x86::NUM_IOAPIC_PINS;
@@ -46,6 +59,13 @@ use crate::arch::x86::NUM_IOAPIC_PINS;
 // aarch64 dependencies
 #[cfg(target_arch = "aarch64")]
 pub mod aarch64;
+#[cfg(target_arch = "aarch64")]
+use kvm_bindings::{
+    kvm_regs, user_fpsimd_state, user_pt_regs, KVM_NR_SPSR, KVM_REG_ARM64, KVM_REG_ARM_CORE,
+    KVM_REG_SIZE_U128, KVM_REG_SIZE_U32, KVM_REG_SIZE_U64,
+};
+#[cfg(target_arch = "aarch64")]
+use std::mem;

 pub use kvm_bindings;
 pub use kvm_bindings::{
@@ -66,12 +86,17 @@ pub use {
    kvm_bindings::kvm_vcpu_events as VcpuEvents, kvm_ioctls::DeviceFd, kvm_ioctls::IoEventAddress,
    kvm_ioctls::VcpuExit,
 };
+#[derive(Clone, Copy, Debug, PartialEq, Deserialize, Serialize)]
+pub struct KvmVmState {}

+pub use KvmVmState as VmState;
 /// Wrapper over KVM VM ioctls.
 pub struct KvmVm {
    fd: Arc<VmFd>,
    #[cfg(target_arch = "x86_64")]
    msrs: MsrEntries,
+    state: KvmVmState,
+    vmmops: ArcSwapOption<Box<dyn vm::VmmOps>>,
 }

 // Returns a `Vec<T>` with a size in bytes at least as large as `size_in_bytes`.
@@ -160,6 +185,9 @@ impl vm::Vm for KvmVm {
            fd: vc,
            #[cfg(target_arch = "x86_64")]
            msrs: self.msrs.clone(),
+            vmmops: self.vmmops.clone(),
+            #[cfg(target_arch = "x86_64")]
+            hyperv_synic: AtomicBool::new(false),
        };
        Ok(Arc::new(vcpu))
    }
@@ -314,6 +342,26 @@ impl vm::Vm for KvmVm {
        self.create_device(&mut vfio_dev)
            .map_err(|e| vm::HypervisorVmError::CreatePassthroughDevice(e.into()))
    }
+    ///
+    /// Get the Vm state. Return VM specific data
+    ///
+    fn state(&self) -> vm::Result<VmState> {
+        Ok(self.state)
+    }
+    ///
+    /// Set the VM state
+    ///
+    fn set_state(&self, _state: VmState) -> vm::Result<()> {
+        Ok(())
+    }
+
+    ///
+    /// Set the VmmOps interface
+    ///
+    fn set_vmmops(&self, vmmops: Box<dyn VmmOps>) -> vm::Result<()> {
+        self.vmmops.store(Some(Arc::new(vmmops)));
+        Ok(())
+    }
 }
 /// Wrapper over KVM system ioctls.
 pub struct KvmHypervisor {
@@ -329,6 +377,12 @@ impl KvmHypervisor {
    /// Create a hypervisor based on Kvm
    pub fn new() -> hypervisor::Result<KvmHypervisor> {
        let kvm_obj = Kvm::new().map_err(|e| hypervisor::HypervisorError::VmCreate(e.into()))?;
+        let api_version = kvm_obj.get_api_version();
+
+        if api_version != kvm_bindings::KVM_API_VERSION as i32 {
+            return Err(hypervisor::HypervisorError::IncompatibleApiVersion);
+        }
+
        Ok(KvmHypervisor { kvm: kvm_obj })
    }
 }
@@ -381,12 +435,21 @@ impl hypervisor::Hypervisor for KvmHypervisor {
                msr_entries[pos].index = *index;
            }

-            Ok(Arc::new(KvmVm { fd: vm_fd, msrs }))
+            Ok(Arc::new(KvmVm {
+                fd: vm_fd,
+                msrs,
+                state: VmState {},
+                vmmops: ArcSwapOption::from(None),
+            }))
        }

        #[cfg(any(target_arch = "arm", target_arch = "aarch64"))]
        {
-            Ok(Arc::new(KvmVm { fd: vm_fd }))
+            Ok(Arc::new(KvmVm {
+                fd: vm_fd,
+                state: VmState {},
+                vmmops: ArcSwapOption::from(None),
+            }))
        }
    }

@@ -395,12 +458,6 @@ impl hypervisor::Hypervisor for KvmHypervisor {
        Ok(())
    }

-    ///
-    /// Returns the KVM API version.
-    ///
-    fn get_api_version(&self) -> i32 {
-        self.kvm.get_api_version()
-    }
    ///
    ///  Returns the size of the memory mapping required to use the vcpu's `kvm_run` structure.
    ///
@@ -452,6 +509,9 @@ pub struct KvmVcpu {
    fd: VcpuFd,
    #[cfg(target_arch = "x86_64")]
    msrs: MsrEntries,
+    vmmops: ArcSwapOption<Box<dyn vm::VmmOps>>,
+    #[cfg(target_arch = "x86_64")]
+    hyperv_synic: AtomicBool,
 }
 /// Implementation of Vcpu trait for KVM
 /// Example:
@@ -527,6 +587,21 @@ impl cpu::Vcpu for KvmVcpu {
            .set_cpuid2(cpuid)
            .map_err(|e| cpu::HypervisorCpuError::SetCpuid(e.into()))
    }
+    #[cfg(target_arch = "x86_64")]
+    ///
+    /// X86 specific call to enable HyperV SynIC
+    ///
+    fn enable_hyperv_synic(&self) -> cpu::Result<()> {
+        // Update the information about Hyper-V SynIC being enabled and
+        // emulated as it will influence later which MSRs should be saved.
+        self.hyperv_synic.store(true, Ordering::SeqCst);
+
+        let mut cap: kvm_enable_cap = Default::default();
+        cap.cap = KVM_CAP_HYPERV_SYNIC;
+        self.fd
+            .enable_cap(&cap)
+            .map_err(|e| cpu::HypervisorCpuError::EnableHyperVSynIC(e.into()))
+    }
    ///
    /// X86 specific call to retrieve the CPUID registers.
    ///
@@ -632,9 +707,27 @@ impl cpu::Vcpu for KvmVcpu {
        match self.fd.run() {
            Ok(run) => match run {
                #[cfg(target_arch = "x86_64")]
-                VcpuExit::IoIn(addr, data) => Ok(cpu::VmExit::IoIn(addr, data)),
+                VcpuExit::IoIn(addr, data) => {
+                    if let Some(vmmops) = self.vmmops.load_full() {
+                        return vmmops
+                            .pio_read(addr.into(), data)
+                            .map(|_| cpu::VmExit::Ignore)
+                            .map_err(|e| cpu::HypervisorCpuError::RunVcpu(e.into()));
+                    }
+
+                    Ok(cpu::VmExit::IoIn(addr, data))
+                }
                #[cfg(target_arch = "x86_64")]
-                VcpuExit::IoOut(addr, data) => Ok(cpu::VmExit::IoOut(addr, data)),
+                VcpuExit::IoOut(addr, data) => {
+                    if let Some(vmmops) = self.vmmops.load_full() {
+                        return vmmops
+                            .pio_write(addr.into(), data)
+                            .map(|_| cpu::VmExit::Ignore)
+                            .map_err(|e| cpu::HypervisorCpuError::RunVcpu(e.into()));
+                    }
+
+                    Ok(cpu::VmExit::IoOut(addr, data))
+                }
                #[cfg(target_arch = "x86_64")]
                VcpuExit::IoapicEoi(vector) => Ok(cpu::VmExit::IoapicEoi(vector)),
                #[cfg(target_arch = "x86_64")]
@@ -642,10 +735,12 @@ impl cpu::Vcpu for KvmVcpu {

                #[cfg(target_arch = "aarch64")]
                VcpuExit::SystemEvent(event_type, flags) => {
-                    use kvm_bindings::KVM_SYSTEM_EVENT_SHUTDOWN;
+                    use kvm_bindings::{KVM_SYSTEM_EVENT_RESET, KVM_SYSTEM_EVENT_SHUTDOWN};
                    // On Aarch64, when the VM is shutdown, run() returns
                    // VcpuExit::SystemEvent with reason KVM_SYSTEM_EVENT_SHUTDOWN
-                    if event_type == KVM_SYSTEM_EVENT_SHUTDOWN {
+                    if event_type == KVM_SYSTEM_EVENT_SHUTDOWN
+                        || event_type == KVM_SYSTEM_EVENT_RESET
+                    {
                        Ok(cpu::VmExit::Reset)
                    } else {
                        Err(cpu::HypervisorCpuError::RunVcpu(anyhow!(
@@ -656,8 +751,27 @@ impl cpu::Vcpu for KvmVcpu {
                    }
                }

-                VcpuExit::MmioRead(addr, data) => Ok(cpu::VmExit::MmioRead(addr, data)),
-                VcpuExit::MmioWrite(addr, data) => Ok(cpu::VmExit::MmioWrite(addr, data)),
+                VcpuExit::MmioRead(addr, data) => {
+                    if let Some(vmmops) = self.vmmops.load_full() {
+                        return vmmops
+                            .mmio_read(addr, data)
+                            .map(|_| cpu::VmExit::Ignore)
+                            .map_err(|e| cpu::HypervisorCpuError::RunVcpu(e.into()));
+                    }
+
+                    Ok(cpu::VmExit::MmioRead(addr, data))
+                }
+                VcpuExit::MmioWrite(addr, data) => {
+                    if let Some(vmmops) = self.vmmops.load_full() {
+                        return vmmops
+                            .mmio_write(addr, data)
+                            .map(|_| cpu::VmExit::Ignore)
+                            .map_err(|e| cpu::HypervisorCpuError::RunVcpu(e.into()));
+                    }
+
+                    Ok(cpu::VmExit::MmioWrite(addr, data))
+                }
+                VcpuExit::Hyperv => Ok(cpu::VmExit::Hyperv),

                r => Err(cpu::HypervisorCpuError::RunVcpu(anyhow!(
                    "Unexpected exit reason on vcpu run: {:?}",
@@ -714,19 +828,263 @@ impl cpu::Vcpu for KvmVcpu {
    /// Sets the value of one register for this vCPU.
    ///
    #[cfg(any(target_arch = "arm", target_arch = "aarch64"))]
-    fn set_one_reg(&self, reg_id: u64, data: u64) -> cpu::Result<()> {
+    fn set_reg(&self, reg_id: u64, data: u64) -> cpu::Result<()> {
        self.fd
            .set_one_reg(reg_id, data)
-            .map_err(|e| cpu::HypervisorCpuError::SetOneReg(e.into()))
+            .map_err(|e| cpu::HypervisorCpuError::SetRegister(e.into()))
    }
    ///
    /// Gets the value of one register for this vCPU.
    ///
    #[cfg(any(target_arch = "arm", target_arch = "aarch64"))]
-    fn get_one_reg(&self, reg_id: u64) -> cpu::Result<u64> {
+    fn get_reg(&self, reg_id: u64) -> cpu::Result<u64> {
        self.fd
            .get_one_reg(reg_id)
-            .map_err(|e| cpu::HypervisorCpuError::GetOneReg(e.into()))
+            .map_err(|e| cpu::HypervisorCpuError::GetRegister(e.into()))
+    }
+    ///
+    /// Gets a list of the guest registers that are supported for the
+    /// KVM_GET_ONE_REG/KVM_SET_ONE_REG calls.
+    ///
+    #[cfg(any(target_arch = "arm", target_arch = "aarch64"))]
+    fn get_reg_list(&self, reg_list: &mut RegList) -> cpu::Result<()> {
+        self.fd
+            .get_reg_list(reg_list)
+            .map_err(|e| cpu::HypervisorCpuError::GetRegList(e.into()))
+    }
+    ///
+    /// Save the state of the core registers.
+    ///
+    #[cfg(any(target_arch = "arm", target_arch = "aarch64"))]
+    fn core_registers(&self, state: &mut StandardRegisters) -> cpu::Result<()> {
+        let mut off = offset__of!(user_pt_regs, regs);
+        // There are 31 user_pt_regs:
+        // https://elixir.free-electrons.com/linux/v4.14.174/source/arch/arm64/include/uapi/asm/ptrace.h#L72
+        // These actually are the general-purpose registers of the Armv8-a
+        // architecture (i.e x0-x30 if used as a 64bit register or w0-30 when used as a 32bit register).
+        for i in 0..31 {
+            state.regs.regs[i] = self
+                .fd
+                .get_one_reg(arm64_core_reg_id!(KVM_REG_SIZE_U64, off))
+                .map_err(|e| cpu::HypervisorCpuError::GetCoreRegister(e.into()))?;
+            off += std::mem::size_of::<u64>();
+        }
+
+        // We are now entering the "Other register" section of the ARMv8-a architecture.
+        // First one, stack pointer.
+        let off = offset__of!(user_pt_regs, sp);
+        state.regs.sp = self
+            .fd
+            .get_one_reg(arm64_core_reg_id!(KVM_REG_SIZE_U64, off))
+            .map_err(|e| cpu::HypervisorCpuError::GetCoreRegister(e.into()))?;
+
+        // Second one, the program counter.
+        let off = offset__of!(user_pt_regs, pc);
+        state.regs.pc = self
+            .fd
+            .get_one_reg(arm64_core_reg_id!(KVM_REG_SIZE_U64, off))
+            .map_err(|e| cpu::HypervisorCpuError::GetCoreRegister(e.into()))?;
+
+        // Next is the processor state.
+        let off = offset__of!(user_pt_regs, pstate);
+        state.regs.pstate = self
+            .fd
+            .get_one_reg(arm64_core_reg_id!(KVM_REG_SIZE_U64, off))
+            .map_err(|e| cpu::HypervisorCpuError::GetCoreRegister(e.into()))?;
+
+        // The stack pointer associated with EL1
+        let off = offset__of!(kvm_regs, sp_el1);
+        state.sp_el1 = self
+            .fd
+            .get_one_reg(arm64_core_reg_id!(KVM_REG_SIZE_U64, off))
+            .map_err(|e| cpu::HypervisorCpuError::GetCoreRegister(e.into()))?;
+
+        // Exception Link Register for EL1, when taking an exception to EL1, this register
+        // holds the address to which to return afterwards.
+        let off = offset__of!(kvm_regs, elr_el1);
+        state.elr_el1 = self
+            .fd
+            .get_one_reg(arm64_core_reg_id!(KVM_REG_SIZE_U64, off))
+            .map_err(|e| cpu::HypervisorCpuError::GetCoreRegister(e.into()))?;
+
+        // Saved Program Status Registers, there are 5 of them used in the kernel.
+        let mut off = offset__of!(kvm_regs, spsr);
+        for i in 0..KVM_NR_SPSR as usize {
+            state.spsr[i] = self
+                .fd
+                .get_one_reg(arm64_core_reg_id!(KVM_REG_SIZE_U64, off))
+                .map_err(|e| cpu::HypervisorCpuError::GetCoreRegister(e.into()))?;
+            off += std::mem::size_of::<u64>();
+        }
+
+        // Now moving on to floting point registers which are stored in the user_fpsimd_state in the kernel:
+        // https://elixir.free-electrons.com/linux/v4.9.62/source/arch/arm64/include/uapi/asm/kvm.h#L53
+        let mut off = offset__of!(kvm_regs, fp_regs) + offset__of!(user_fpsimd_state, vregs);
+        for i in 0..32 {
+            state.fp_regs.vregs[i][0] = self
+                .fd
+                .get_one_reg(arm64_core_reg_id!(KVM_REG_SIZE_U128, off))
+                .map_err(|e| cpu::HypervisorCpuError::GetCoreRegister(e.into()))?;
+            off += mem::size_of::<u128>();
+        }
+
+        // Floating-point Status Register
+        let off = offset__of!(kvm_regs, fp_regs) + offset__of!(user_fpsimd_state, fpsr);
+        state.fp_regs.fpsr = self
+            .fd
+            .get_one_reg(arm64_core_reg_id!(KVM_REG_SIZE_U32, off))
+            .map_err(|e| cpu::HypervisorCpuError::GetCoreRegister(e.into()))?
+            as u32;
+
+        // Floating-point Control Register
+        let off = offset__of!(kvm_regs, fp_regs) + offset__of!(user_fpsimd_state, fpcr);
+        state.fp_regs.fpcr = self
+            .fd
+            .get_one_reg(arm64_core_reg_id!(KVM_REG_SIZE_U32, off))
+            .map_err(|e| cpu::HypervisorCpuError::GetCoreRegister(e.into()))?
+            as u32;
+        Ok(())
+    }
+    ///
+    /// Restore the state of the core registers.
+    ///
+    #[cfg(any(target_arch = "arm", target_arch = "aarch64"))]
+    fn set_core_registers(&self, state: &StandardRegisters) -> cpu::Result<()> {
+        // The function follows the exact identical order from `state`. Look there
+        // for some additional info on registers.
+        let mut off = offset__of!(user_pt_regs, regs);
+        for i in 0..31 {
+            self.fd
+                .set_one_reg(
+                    arm64_core_reg_id!(KVM_REG_SIZE_U64, off),
+                    state.regs.regs[i],
+                )
+                .map_err(|e| cpu::HypervisorCpuError::SetCoreRegister(e.into()))?;
+            off += std::mem::size_of::<u64>();
+        }
+
+        let off = offset__of!(user_pt_regs, sp);
+        self.fd
+            .set_one_reg(arm64_core_reg_id!(KVM_REG_SIZE_U64, off), state.regs.sp)
+            .map_err(|e| cpu::HypervisorCpuError::SetCoreRegister(e.into()))?;
+
+        let off = offset__of!(user_pt_regs, pc);
+        self.fd
+            .set_one_reg(arm64_core_reg_id!(KVM_REG_SIZE_U64, off), state.regs.pc)
+            .map_err(|e| cpu::HypervisorCpuError::SetCoreRegister(e.into()))?;
+
+        let off = offset__of!(user_pt_regs, pstate);
+        self.fd
+            .set_one_reg(arm64_core_reg_id!(KVM_REG_SIZE_U64, off), state.regs.pstate)
+            .map_err(|e| cpu::HypervisorCpuError::SetCoreRegister(e.into()))?;
+
+        let off = offset__of!(kvm_regs, sp_el1);
+        self.fd
+            .set_one_reg(arm64_core_reg_id!(KVM_REG_SIZE_U64, off), state.sp_el1)
+            .map_err(|e| cpu::HypervisorCpuError::SetCoreRegister(e.into()))?;
+
+        let off = offset__of!(kvm_regs, elr_el1);
+        self.fd
+            .set_one_reg(arm64_core_reg_id!(KVM_REG_SIZE_U64, off), state.elr_el1)
+            .map_err(|e| cpu::HypervisorCpuError::SetCoreRegister(e.into()))?;
+
+        let mut off = offset__of!(kvm_regs, spsr);
+        for i in 0..KVM_NR_SPSR as usize {
+            self.fd
+                .set_one_reg(arm64_core_reg_id!(KVM_REG_SIZE_U64, off), state.spsr[i])
+                .map_err(|e| cpu::HypervisorCpuError::SetCoreRegister(e.into()))?;
+            off += std::mem::size_of::<u64>();
+        }
+
+        let mut off = offset__of!(kvm_regs, fp_regs) + offset__of!(user_fpsimd_state, vregs);
+        for i in 0..32 {
+            self.fd
+                .set_one_reg(
+                    arm64_core_reg_id!(KVM_REG_SIZE_U128, off),
+                    state.fp_regs.vregs[i][0],
+                )
+                .map_err(|e| cpu::HypervisorCpuError::SetCoreRegister(e.into()))?;
+            off += mem::size_of::<u128>();
+        }
+
+        let off = offset__of!(kvm_regs, fp_regs) + offset__of!(user_fpsimd_state, fpsr);
+        self.fd
+            .set_one_reg(
+                arm64_core_reg_id!(KVM_REG_SIZE_U32, off),
+                state.fp_regs.fpsr as u64,
+            )
+            .map_err(|e| cpu::HypervisorCpuError::SetCoreRegister(e.into()))?;
+
+        let off = offset__of!(kvm_regs, fp_regs) + offset__of!(user_fpsimd_state, fpcr);
+        self.fd
+            .set_one_reg(
+                arm64_core_reg_id!(KVM_REG_SIZE_U32, off),
+                state.fp_regs.fpcr as u64,
+            )
+            .map_err(|e| cpu::HypervisorCpuError::SetCoreRegister(e.into()))?;
+        Ok(())
+    }
+    ///
+    /// Save the state of the system registers.
+    ///
+    #[cfg(any(target_arch = "arm", target_arch = "aarch64"))]
+    fn system_registers(&self, state: &mut Vec<Register>) -> cpu::Result<()> {
+        // Call KVM_GET_REG_LIST to get all registers available to the guest. For ArmV8 there are
+        // around 500 registers.
+        let mut reg_list = RegList::new(512);
+        self.fd
+            .get_reg_list(&mut reg_list)
+            .map_err(|e| cpu::HypervisorCpuError::GetRegList(e.into()))?;
+
+        // At this point reg_list should contain: core registers and system registers.
+        // The register list contains the number of registers and their ids. We will be needing to
+        // call KVM_GET_ONE_REG on each id in order to save all of them. We carve out from the list
+        // the core registers which are represented in the kernel by kvm_regs structure and for which
+        // we can calculate the id based on the offset in the structure.
+
+        reg_list.retain(|regid| *regid != 0);
+        reg_list.as_slice().to_vec().sort_unstable();
+
+        reg_list.retain(|regid| is_system_register(*regid));
+
+        // Now, for the rest of the registers left in the previously fetched register list, we are
+        // simply calling KVM_GET_ONE_REG.
+        let indices = reg_list.as_slice();
+        for (_pos, index) in indices.iter().enumerate() {
+            if _pos > 230 {
+                break;
+            }
+            state.push(kvm_bindings::kvm_one_reg {
+                id: *index,
+                addr: self
+                    .fd
+                    .get_one_reg(*index)
+                    .map_err(|e| cpu::HypervisorCpuError::GetSysRegister(e.into()))?,
+            });
+        }
+
+        Ok(())
+    }
+    ///
+    /// Restore the state of the system registers.
+    ///
+    #[cfg(any(target_arch = "arm", target_arch = "aarch64"))]
+    fn set_system_registers(&self, state: &[Register]) -> cpu::Result<()> {
+        for reg in state {
+            self.fd
+                .set_one_reg(reg.id, reg.addr)
+                .map_err(|e| cpu::HypervisorCpuError::SetSysRegister(e.into()))?;
+        }
+        Ok(())
+    }
+    ///
+    /// Read the MPIDR - Multiprocessor Affinity Register.
+    ///
+    #[cfg(any(target_arch = "arm", target_arch = "aarch64"))]
+    fn read_mpidr(&self) -> cpu::Result<u64> {
+        self.fd
+            .get_one_reg(MPIDR_EL1)
+            .map_err(|e| cpu::HypervisorCpuError::GetSysRegister(e.into()))
    }
    #[cfg(target_arch = "x86_64")]
    ///
@@ -767,6 +1125,7 @@ impl cpu::Vcpu for KvmVcpu {
    /// let state = vcpu.state().unwrap();
    /// ```
    fn state(&self) -> cpu::Result<CpuState> {
+        let cpuid = self.get_cpuid2(kvm_bindings::KVM_MAX_CPUID_ENTRIES)?;
        let mp_state = self.get_mp_state()?;
        let regs = self.get_regs()?;
        let sregs = self.get_sregs()?;
@@ -781,6 +1140,26 @@ impl cpu::Vcpu for KvmVcpu {
        // by chunks. This is the only way to make sure we try to get as many
        // MSRs as possible, even if some MSRs are not supported.
        let mut msr_entries = self.msrs.clone();
+
+        // Save extra MSRs if the Hyper-V synthetic interrupt controller is
+        // emulated.
+        if self.hyperv_synic.load(Ordering::SeqCst) {
+            let hyperv_synic_msrs = vec![
+                0x40000020, 0x40000021, 0x40000080, 0x40000081, 0x40000082, 0x40000083, 0x40000084,
+                0x40000090, 0x40000091, 0x40000092, 0x40000093, 0x40000094, 0x40000095, 0x40000096,
+                0x40000097, 0x40000098, 0x40000099, 0x4000009a, 0x4000009b, 0x4000009c, 0x4000009d,
+                0x4000009f, 0x400000b0, 0x400000b1, 0x400000b2, 0x400000b3, 0x400000b4, 0x400000b5,
+                0x400000b6, 0x400000b7,
+            ];
+            for index in hyperv_synic_msrs {
+                let msr = kvm_msr_entry {
+                    index,
+                    ..Default::default()
+                };
+                msr_entries.push(msr).unwrap();
+            }
+        }
+
        let expected_num_msrs = msr_entries.as_fam_struct_ref().nmsrs as usize;
        let num_msrs = self.get_msrs(&mut msr_entries)?;
        let msrs = if num_msrs != expected_num_msrs {
@@ -826,6 +1205,7 @@ impl cpu::Vcpu for KvmVcpu {
        let vcpu_events = self.get_vcpu_events()?;

        Ok(CpuState {
+            cpuid,
            msrs,
            vcpu_events,
            regs,
@@ -837,9 +1217,19 @@ impl cpu::Vcpu for KvmVcpu {
            mp_state,
        })
    }
+    ///
+    /// Get the current AArch64 CPU state
+    ///
    #[cfg(target_arch = "aarch64")]
    fn state(&self) -> cpu::Result<CpuState> {
-        unimplemented!();
+        let mut state = CpuState::default();
+        // Get this vCPUs multiprocessing state.
+        state.mp_state = self.get_mp_state()?;
+        self.core_registers(&mut state.core_regs)?;
+        self.system_registers(&mut state.sys_regs)?;
+        state.mpidr = self.read_mpidr()?;
+
+        Ok(state)
    }
    #[cfg(target_arch = "x86_64")]
    ///
@@ -882,6 +1272,7 @@ impl cpu::Vcpu for KvmVcpu {
    /// vcpu.set_state(&state).unwrap();
    /// ```
    fn set_state(&self, state: &CpuState) -> cpu::Result<()> {
+        self.set_cpuid2(&state.cpuid)?;
        self.set_mp_state(state.mp_state)?;
        self.set_regs(&state.regs)?;
        self.set_sregs(&state.sregs)?;
@@ -923,9 +1314,15 @@ impl cpu::Vcpu for KvmVcpu {

        Ok(())
    }
-    #[allow(unused_variables)]
+    ///
+    /// Restore the previously saved AArch64 CPU state
+    ///
    #[cfg(target_arch = "aarch64")]
    fn set_state(&self, state: &CpuState) -> cpu::Result<()> {
+        self.set_core_registers(&state.core_regs)?;
+        self.set_system_registers(&state.sys_regs)?;
+        self.set_mp_state(state.mp_state)?;
+
        Ok(())
    }
 }
@@ -944,6 +1341,14 @@ impl device::Device for KvmDevice {
            .set_device_attr(attr)
            .map_err(|e| device::HypervisorDeviceError::SetDeviceAttribute(e.into()))
    }
+    ///
+    /// Get device attribute
+    ///
+    fn get_device_attr(&self, attr: &mut DeviceAttr) -> device::Result<()> {
+        self.fd
+            .get_device_attr(attr)
+            .map_err(|e| device::HypervisorDeviceError::GetDeviceAttribute(e.into()))
+    }
 }

 impl AsRawFd for KvmDevice {
--- a/hypervisor/src/kvm/x86_64/mod.rs
+++ b/hypervisor/src/kvm/x86_64/mod.rs
@@ -85,6 +85,7 @@ pub fn check_required_kvm_extensions(kvm: &Kvm) -> KvmResult<()> {
 }
 #[derive(Clone, Serialize, Deserialize)]
 pub struct VcpuKvmState {
+    pub cpuid: CpuId,
    pub msrs: MsrEntries,
    pub vcpu_events: VcpuEvents,
    pub regs: StandardRegisters,
--- a/hypervisor/src/lib.rs
+++ b/hypervisor/src/lib.rs
@@ -20,6 +20,7 @@

 #[macro_use]
 extern crate anyhow;
+#[cfg(target_arch = "x86_64")]
 #[macro_use]
 extern crate log;
 extern crate serde;
@@ -52,6 +53,7 @@ pub use kvm::*;
 pub use vm::{DataMatch, HypervisorVmError, Vm};

 use std::sync::Arc;
+
 pub fn new() -> std::result::Result<Arc<dyn Hypervisor>, HypervisorError> {
    #[cfg(feature = "kvm")]
    let hv = kvm::KvmHypervisor::new()?;
--- a/hypervisor/src/vm.rs
+++ b/hypervisor/src/vm.rs
@@ -14,6 +14,7 @@ use crate::cpu::Vcpu;
 use crate::device::Device;
 #[cfg(target_arch = "x86_64")]
 use crate::ClockData;
+use crate::KvmVmState as VmState;
 use crate::{CreateDevice, IoEventAddress, IrqRoutingEntry, MemoryRegion};
 use kvm_ioctls::Cap;
 use std::sync::Arc;
@@ -116,6 +117,36 @@ pub enum HypervisorVmError {
    ///
    #[error("Failed to create passthrough device: {0}")]
    CreatePassthroughDevice(#[source] anyhow::Error),
+    ///
+    /// Write to Guest memory
+    ///
+    #[error("Failed to write to guest memory: {0}")]
+    GuestMemWrite(#[source] anyhow::Error),
+    ///
+    /// Read Guest memory
+    ///
+    #[error("Failed to read guest memory: {0}")]
+    GuestMemRead(#[source] anyhow::Error),
+    ///
+    /// Read from MMIO Bus
+    ///
+    #[error("Failed to read from MMIO Bus: {0}")]
+    MmioBusRead(#[source] anyhow::Error),
+    ///
+    /// Write to MMIO Bus
+    ///
+    #[error("Failed to write to MMIO Bus: {0}")]
+    MmioBusWrite(#[source] anyhow::Error),
+    ///
+    /// Read from IO Bus
+    ///
+    #[error("Failed to read from IO Bus: {0}")]
+    IoBusRead(#[source] anyhow::Error),
+    ///
+    /// Write to IO Bus
+    ///
+    #[error("Failed to write to IO Bus: {0}")]
+    IoBusWrite(#[source] anyhow::Error),
 }
 ///
 /// Result type for returning from a function
@@ -179,4 +210,21 @@ pub trait Vm: Send + Sync {
    fn check_extension(&self, c: Cap) -> bool;
    /// Create a device that is used for passthrough
    fn create_passthrough_device(&self) -> Result<Arc<dyn Device>>;
+    /// Get the Vm state. Return VM specific data
+    fn state(&self) -> Result<VmState>;
+    /// Set the VM state
+    fn set_state(&self, state: VmState) -> Result<()>;
+    /// Set VmmOps interface
+    fn set_vmmops(&self, vmmops: Box<dyn VmmOps>) -> Result<()>;
+}
+
+pub trait VmmOps: Send + Sync {
+    fn guest_mem_write(&self, buf: &[u8], gpa: u64) -> Result<usize>;
+    fn guest_mem_read(&self, buf: &mut [u8], gpa: u64) -> Result<usize>;
+    fn mmio_read(&self, addr: u64, data: &mut [u8]) -> Result<()>;
+    fn mmio_write(&self, addr: u64, data: &[u8]) -> Result<()>;
+    #[cfg(target_arch = "x86_64")]
+    fn pio_read(&self, addr: u64, data: &mut [u8]) -> Result<()>;
+    #[cfg(target_arch = "x86_64")]
+    fn pio_write(&self, addr: u64, data: &[u8]) -> Result<()>;
 }
--- a/net_util/Cargo.toml
+++ b/net_util/Cargo.toml
@@ -5,17 +5,17 @@ authors = ["The Chromium OS Authors"]

 [dependencies]
 epoll = ">=4.0.1"
-libc = "0.2.74"
+libc = "0.2.80"
 log = "0.4.11"
 net_gen = { path = "../net_gen" }
 rand = "0.7.3"
-serde = "1.0.114"
+serde = "1.0.117"
 virtio-bindings = "0.1.0"
-vm-memory = { version = "0.2.1", features = ["backend-mmap", "backend-atomic"] }
+vm-memory = { version = "0.3.0", features = ["backend-mmap", "backend-atomic"] }
 vm-virtio = { path = "../vm-virtio" }
 vmm-sys-util = ">=0.3.1"

 [dev-dependencies]
 lazy_static = "1.3.0"
 pnet = "0.26.0"
-serde_json = "1.0.57"
+serde_json = "1.0.59"
--- a/option_parser/src/lib.rs
+++ b/option_parser/src/lib.rs
@@ -143,6 +143,7 @@ impl FromStr for Toggle {

 pub struct ByteSized(pub u64);

+#[derive(Debug)]
 pub enum ByteSizedParseError {
    InvalidValue(String),
 }
@@ -170,3 +171,99 @@ impl FromStr for ByteSized {
        }))
    }
 }
+
+pub struct IntegerList(pub Vec<u64>);
+
+pub enum IntegerListParseError {
+    InvalidValue(String),
+}
+
+impl FromStr for IntegerList {
+    type Err = IntegerListParseError;
+
+    fn from_str(s: &str) -> std::result::Result<Self, Self::Err> {
+        let mut integer_list = Vec::new();
+        let ranges_list: Vec<&str> = s.trim().split(':').collect();
+
+        for range in ranges_list.iter() {
+            let items: Vec<&str> = range.split('-').collect();
+
+            if items.len() > 2 {
+                return Err(IntegerListParseError::InvalidValue((*range).to_string()));
+            }
+
+            let start_range = items[0]
+                .parse::<u64>()
+                .map_err(|_| IntegerListParseError::InvalidValue(items[0].to_owned()))?;
+
+            integer_list.push(start_range);
+
+            if items.len() == 2 {
+                let end_range = items[1]
+                    .parse::<u64>()
+                    .map_err(|_| IntegerListParseError::InvalidValue(items[1].to_owned()))?;
+                if start_range >= end_range {
+                    return Err(IntegerListParseError::InvalidValue((*range).to_string()));
+                }
+
+                for i in start_range..end_range {
+                    integer_list.push(i + 1);
+                }
+            }
+        }
+
+        Ok(IntegerList(integer_list))
+    }
+}
+
+pub struct TupleTwoIntegers(pub Vec<(u64, u64)>);
+
+pub enum TupleTwoIntegersParseError {
+    InvalidValue(String),
+}
+
+impl FromStr for TupleTwoIntegers {
+    type Err = TupleTwoIntegersParseError;
+
+    fn from_str(s: &str) -> std::result::Result<Self, Self::Err> {
+        let mut list = Vec::new();
+        let tuples_list: Vec<&str> = s.trim().split(':').collect();
+
+        for tuple in tuples_list.iter() {
+            let items: Vec<&str> = tuple.split('@').collect();
+
+            if items.len() != 2 {
+                return Err(TupleTwoIntegersParseError::InvalidValue(
+                    (*tuple).to_string(),
+                ));
+            }
+
+            let item1 = items[0]
+                .parse::<u64>()
+                .map_err(|_| TupleTwoIntegersParseError::InvalidValue(items[0].to_owned()))?;
+            let item2 = items[1]
+                .parse::<u64>()
+                .map_err(|_| TupleTwoIntegersParseError::InvalidValue(items[1].to_owned()))?;
+
+            list.push((item1, item2));
+        }
+
+        Ok(TupleTwoIntegers(list))
+    }
+}
+
+pub struct StringList(pub Vec<String>);
+
+pub enum StringListParseError {
+    InvalidValue(String),
+}
+
+impl FromStr for StringList {
+    type Err = StringListParseError;
+
+    fn from_str(s: &str) -> std::result::Result<Self, Self::Err> {
+        let string_list: Vec<String> = s.trim().split(':').map(|e| e.to_owned()).collect();
+
+        Ok(StringList(string_list))
+    }
+}
--- a/pci/Cargo.toml
+++ b/pci/Cargo.toml
@@ -7,18 +7,17 @@ edition = "2018"
 [dependencies]
 anyhow = "1.0"
 byteorder = "1.3.4"
-devices = { path = "../devices" }
 hypervisor = { path = "../hypervisor" }
 vfio-ioctls = { git = "https://github.com/cloud-hypervisor/vfio-ioctls", branch = "ch" }
 vmm-sys-util = ">=0.3.1"
-libc = "0.2.74"
+libc = "0.2.80"
 log = "0.4.11"
 serde = {version = ">=1.0.27", features = ["rc"] }
 serde_derive = ">=1.0.27"
 serde_json = ">=1.0.9"
 vm-allocator = { path = "../vm-allocator" }
 vm-device = { path = "../vm-device" }
-vm-memory = "0.2.1"
+vm-memory = "0.3.0"
 vm-migration = { path = "../vm-migration" }

 [dependencies.vfio-bindings]
--- a/pci/src/bus.rs
+++ b/pci/src/bus.rs
@@ -7,11 +7,11 @@ use crate::configuration::{
 };
 use crate::device::{DeviceRelocation, Error as PciDeviceError, PciDevice};
 use byteorder::{ByteOrder, LittleEndian};
-use devices::BusDevice;
 use std::any::Any;
 use std::collections::HashMap;
 use std::ops::DerefMut;
 use std::sync::{Arc, Mutex};
+use vm_device::{Bus, BusDevice};
 use vm_memory::{Address, GuestAddress, GuestUsize};

 const VENDOR_ID_INTEL: u16 = 0x8086;
@@ -26,9 +26,9 @@ pub enum PciRootError {
    /// Could not allocate an IRQ number.
    AllocateIrq,
    /// Could not add a device to the port io bus.
-    PioInsert(devices::BusError),
+    PioInsert(vm_device::BusError),
    /// Could not add a device to the mmio bus.
-    MmioInsert(devices::BusError),
+    MmioInsert(vm_device::BusError),
    /// Could not find an available device slot on the PCI bus.
    NoPciDeviceSlotAvailable,
    /// Invalid PCI device identifier provided.
@@ -110,8 +110,8 @@ impl PciBus {
    pub fn register_mapping(
        &self,
        dev: Arc<Mutex<dyn BusDevice>>,
-        #[cfg(target_arch = "x86_64")] io_bus: &devices::Bus,
-        mmio_bus: &devices::Bus,
+        #[cfg(target_arch = "x86_64")] io_bus: &Bus,
+        mmio_bus: &Bus,
        bars: Vec<(GuestAddress, GuestUsize, PciBarRegionType)>,
    ) -> Result<()> {
        for (address, size, type_) in bars {
--- a/pci/src/configuration.rs
+++ b/pci/src/configuration.rs
@@ -64,7 +64,7 @@ impl PciClassCode {
    }
 }

-/// A PCI sublcass. Each class in `PciClassCode` can specify a unique set of subclasses. This trait
+/// A PCI subclass. Each class in `PciClassCode` can specify a unique set of subclasses. This trait
 /// is implemented by each subclass. It allows use of a trait object to generate configurations.
 pub trait PciSubclass {
    /// Convert this subclass to the value used in the PCI specification.
@@ -889,7 +889,7 @@ impl Snapshottable for PciConfiguration {
        String::from("pci_configuration")
    }

-    fn snapshot(&self) -> std::result::Result<Snapshot, MigratableError> {
+    fn snapshot(&mut self) -> std::result::Result<Snapshot, MigratableError> {
        let snapshot =
            serde_json::to_vec(&self.state()).map_err(|e| MigratableError::Snapshot(e.into()))?;

--- a/pci/src/device.rs
+++ b/pci/src/device.rs
@@ -3,11 +3,11 @@
 // found in the LICENSE-BSD-3-Clause file.

 use crate::configuration::{self, PciBarRegionType};
-use devices::BusDevice;
 use std::any::Any;
 use std::fmt::{self, Display};
 use std::{self, io, result};
 use vm_allocator::SystemAllocator;
+use vm_device::BusDevice;
 use vm_memory::{GuestAddress, GuestUsize};

 #[derive(Debug)]
--- a/pci/src/lib.rs
+++ b/pci/src/lib.rs
@@ -5,7 +5,6 @@
 //! Implements pci devices and busses.
 #[macro_use]
 extern crate log;
-extern crate devices;
 extern crate hypervisor;
 extern crate serde;
 #[macro_use]
--- a/pci/src/msix.rs
+++ b/pci/src/msix.rs
@@ -56,7 +56,7 @@ impl Default for MsixTableEntry {
            msg_addr_lo: 0,
            msg_addr_hi: 0,
            msg_data: 0,
-            vector_ctl: 0,
+            vector_ctl: 0x1,
        }
    }
 }
@@ -97,7 +97,7 @@ impl MsixConfig {
            pba_entries,
            devid,
            interrupt_source_group,
-            masked: false,
+            masked: true,
            enabled: false,
        }
    }
@@ -158,7 +158,7 @@ impl MsixConfig {
        self.masked = ((reg >> FUNCTION_MASK_BIT) & 1u16) == 1u16;
        self.enabled = ((reg >> MSIX_ENABLE_BIT) & 1u16) == 1u16;

-        // Update KVM routes
+        // Update interrupt routing
        if old_masked != self.masked || old_enabled != self.enabled {
            if self.enabled && !self.masked {
                for (idx, table_entry) in self.table_entries.iter().enumerate() {
@@ -185,12 +185,6 @@ impl MsixConfig {
                        error!("Failed unmasking vector: {:?}", e);
                    }
                }
-
-                if !old_enabled || old_masked {
-                    if let Err(e) = self.interrupt_source_group.enable() {
-                        error!("Failed enabling irq_fd: {:?}", e);
-                    }
-                }
            } else if old_enabled || !old_masked {
                if let Err(e) = self.interrupt_source_group.disable() {
                    error!("Failed disabling irq_fd: {:?}", e);
@@ -437,7 +431,7 @@ impl Snapshottable for MsixConfig {
        String::from("msix_config")
    }

-    fn snapshot(&self) -> std::result::Result<Snapshot, MigratableError> {
+    fn snapshot(&mut self) -> std::result::Result<Snapshot, MigratableError> {
        let snapshot =
            serde_json::to_vec(&self.state()).map_err(|e| MigratableError::Snapshot(e.into()))?;

--- a/pci/src/vfio.rs
+++ b/pci/src/vfio.rs
@@ -3,7 +3,6 @@
 // SPDX-License-Identifier: Apache-2.0 OR BSD-3-Clause
 //

-extern crate devices;
 extern crate vm_allocator;

 use crate::{
@@ -12,7 +11,6 @@ use crate::{
    PciDevice, PciDeviceError, PciHeaderType, PciSubclass, MSIX_TABLE_ENTRY_SIZE,
 };
 use byteorder::{ByteOrder, LittleEndian};
-use devices::BusDevice;
 use std::any::Any;
 use std::ops::Deref;
 use std::os::unix::io::AsRawFd;
@@ -25,6 +23,7 @@ use vm_allocator::SystemAllocator;
 use vm_device::interrupt::{
    InterruptIndex, InterruptManager, InterruptSourceGroup, MsiIrqGroupConfig,
 };
+use vm_device::BusDevice;
 use vm_memory::{
    Address, GuestAddress, GuestAddressSpace, GuestMemoryAtomic, GuestMemoryMmap, GuestRegionMmap,
    GuestUsize,
@@ -61,7 +60,7 @@ impl fmt::Display for VfioPciError {
            VfioPciError::MapRegionGuest(e) => {
                write!(f, "failed to map VFIO PCI region into guest: {}", e)
            }
-            VfioPciError::SetGsiRouting(e) => write!(f, "failed to set GSI routes for KVM: {}", e),
+            VfioPciError::SetGsiRouting(e) => write!(f, "failed to set GSI routes: {}", e),
            VfioPciError::MsiNotConfigured => write!(f, "MSI interrupt not yet configured"),
            VfioPciError::MsixNotConfigured => write!(f, "MSI-X interrupt not yet configured"),
            VfioPciError::UpdateMemory(e) => write!(f, "failed to update memory: {}", e),
@@ -504,12 +503,9 @@ impl VfioPciDevice {
    ///
    /// # Arguments
    ///
-    /// * `vm` - The KVM VM file descriptor. It is used to set the VFIO MMIO regions
-    ///          as KVM user memory regions.
-    /// * `mem_slot` - The KVM memory slot to set the user memopry regions.
-    /// # Return value
-    ///
-    /// This function returns the updated KVM memory slot id.
+    /// * `vm` - The VM object. It is used to set the VFIO MMIO regions
+    ///          as user memory regions.
+    /// * `mem_slot` - The closure to return a memory slot.
    pub fn map_mmio_regions<F>(&mut self, vm: &Arc<dyn hypervisor::Vm>, mem_slot: F) -> Result<()>
    where
        F: Fn() -> u32,
@@ -587,8 +583,8 @@ impl VfioPciDevice {
            {
                let (mmap_offset, _) = self.device.get_region_mmap(region.index);

-                // Remove region from KVM
-                let kvm_region = self.vm.make_user_memory_region(
+                // Remove region
+                let r = self.vm.make_user_memory_region(
                    mem_slot,
                    region.start.raw_value() + mmap_offset,
                    0,
@@ -596,11 +592,8 @@ impl VfioPciDevice {
                    false,
                );

-                if let Err(e) = self.vm.set_user_memory_region(kvm_region) {
-                    error!(
-                        "Could not remove the userspace memory region from KVM: {}",
-                        e
-                    );
+                if let Err(e) = self.vm.set_user_memory_region(r) {
+                    error!("Could not remove the userspace memory region: {}", e);
                }

                let ret = unsafe { libc::munmap(host_addr as *mut libc::c_void, mmap_size) };
@@ -721,20 +714,17 @@ impl PciDevice for VfioPciDevice {

            // Is this an IO BAR?
            let io_bar = if bar_id != VFIO_PCI_ROM_REGION_INDEX {
-                match lsb_flag & PCI_CONFIG_IO_BAR {
-                    PCI_CONFIG_IO_BAR => true,
-                    _ => false,
-                }
+                matches!(lsb_flag & PCI_CONFIG_IO_BAR, PCI_CONFIG_IO_BAR)
            } else {
                false
            };

            // Is this a 64-bit BAR?
            let is_64bit_bar = if bar_id != VFIO_PCI_ROM_REGION_INDEX {
-                match lsb_flag & PCI_CONFIG_MEMORY_BAR_64BIT {
-                    PCI_CONFIG_MEMORY_BAR_64BIT => true,
-                    _ => false,
-                }
+                matches!(
+                    lsb_flag & PCI_CONFIG_MEMORY_BAR_64BIT,
+                    PCI_CONFIG_MEMORY_BAR_64BIT
+                )
            } else {
                false
            };
@@ -791,8 +781,8 @@ impl PciDevice for VfioPciDevice {

                // We need to allocate a guest MMIO address range for that BAR.
                // In case the BAR is mappable directly, this means it might be
-                // set as KVM user memory region, which expects to deal with 4K
-                // pages. Therefore, the aligment has to be set accordingly.
+                // set as user memory region, which expects to deal with 4K
+                // pages. Therefore, the alignment has to be set accordingly.
                let bar_alignment = if (bar_id == VFIO_PCI_ROM_REGION_INDEX)
                    || (self.device.get_region_flags(bar_id) & VFIO_REGION_INFO_FLAG_MMAP != 0)
                {
@@ -1015,7 +1005,7 @@ impl PciDevice for VfioPciDevice {
                    if let Some(host_addr) = region.host_addr {
                        let (mmap_offset, mmap_size) = self.device.get_region_mmap(region.index);

-                        // Remove old region from KVM
+                        // Remove old region
                        let old_mem_region = self.vm.make_user_memory_region(
                            mem_slot,
                            old_base + mmap_offset,
@@ -1028,7 +1018,7 @@ impl PciDevice for VfioPciDevice {
                            .set_user_memory_region(old_mem_region)
                            .map_err(|e| io::Error::new(io::ErrorKind::Other, e))?;

-                        // Insert new region to KVM
+                        // Insert new region
                        let new_mem_region = self.vm.make_user_memory_region(
                            mem_slot,
                            new_base + mmap_offset,
--- a/qcow/Cargo.toml
+++ b/qcow/Cargo.toml
@@ -10,7 +10,7 @@ path = "src/qcow.rs"

 [dependencies]
 byteorder = "1.3.4"
-libc = "0.2.74"
+libc = "0.2.80"
 log = "0.4.11"
 remain = "0.2.2"
 vmm-sys-util = ">=0.3.1"
--- a/qcow/src/qcow.rs
+++ b/qcow/src/qcow.rs
@@ -769,11 +769,14 @@ impl QcowFile {
            let mut ref_table = vec![0; refcount_table_entries as usize];
            let mut first_free_cluster: u64 = 0;
            for refblock_addr in &mut ref_table {
-                while refcounts[first_free_cluster as usize] != 0 {
-                    first_free_cluster += 1;
+                loop {
                    if first_free_cluster >= refcounts.len() as u64 {
                        return Err(Error::NotEnoughSpaceForRefcounts);
                    }
+                    if refcounts[first_free_cluster as usize] == 0 {
+                        break;
+                    }
+                    first_free_cluster += 1;
                }

                *refblock_addr = first_free_cluster * cluster_size;
@@ -1056,7 +1059,7 @@ impl QcowFile {
            // Free the previously used cluster if one exists. Modified tables are always
            // witten to new clusters so the L1 table can be committed to disk after they
            // are and L1 never points at an invalid table.
-            // The index must be valid from when it was insterted.
+            // The index must be valid from when it was inserted.
            let addr = self.l1_table[l1_index];
            if addr != 0 {
                self.unref_clusters.push(addr);
@@ -1354,7 +1357,7 @@ impl QcowFile {
    fn sync_caches(&mut self) -> std::io::Result<()> {
        // Write out all dirty L2 tables.
        for (l1_index, l2_table) in self.l2_cache.iter_mut().filter(|(_k, v)| v.dirty()) {
-            // The index must be valid from when we insterted it.
+            // The index must be valid from when we inserted it.
            let addr = self.l1_table[*l1_index];
            if addr != 0 {
                self.raw_file.write_pointer_table(
--- a/qcow/src/raw_file.rs
+++ b/qcow/src/raw_file.rs
@@ -58,7 +58,7 @@ impl RawFile {
        }
        RawFile {
            file,
-            alignment: alignment.try_into().unwrap(),
+            alignment,
            position: 0,
        }
    }
@@ -175,7 +175,7 @@ impl Read for RawFile {

            self.seek(SeekFrom::Current(to_copy.try_into().unwrap()))
                .unwrap();
-            Ok(to_copy.try_into().unwrap())
+            Ok(to_copy)
        }
    }
 }
@@ -259,7 +259,7 @@ impl Write for RawFile {

                self.seek(SeekFrom::Current(to_seek.try_into().unwrap()))
                    .unwrap();
-                Ok(to_seek.try_into().unwrap())
+                Ok(to_seek)
            }
        }
    }
--- a/release-notes.md
+++ b/release-notes.md
@@ -1,12 +1,23 @@
- - [v0.8.0](#v080)
-    - [Experimental Snapshot and Restore Support](#experimental-snapshot-and-restore-support)
-    - [Experimental ARM64 Support](#experimental-arm64-support)
-    - [Support for Using 5-level Paging in Guests](#support-for-using-5-level-paging-in-guests)
-    - [Virtio Device Interrupt Suppression for Network Devices](#virtio-device-interrupt-suppression-for-network-devices)
-    - [`vhost_user_fs` Improvements](#vhost_user_fs-improvements)
+- [v0.11.0](#v0110)
+    - [`io_uring` support by default for `virtio-block`](#io_uring-support-by-default-for-virtio-block)
+    - [Windows Guest Support](#windows-guest-support)
+    - [`vhost-user` "Self Spawning" Deprecation](#vhost-user-self-spawning-deprecation)
+    - [`virtio-mmmio` Removal](#virtio-mmmio-removal)
+    - [Snapshot/Restore support for ARM64](#snapshotrestore-support-for-arm64)
+    - [Improved Linux Boot Time](#improved-linux-boot-time)
+    - [`SIGTERM/SIGINT` Interrupt Signal Handling](#sigtermsigint-interrupt-signal-handling)
+    - [Default Log Level Changed](#default-log-level-changed)
+    - [New `--balloon` Parameter Added](#new---balloon-parameter-added)
+    - [Experimental `virtio-watchdog` Support](#experimental-virtio-watchdog-support)
    - [Notable Bug Fixes](#notable-bug-fixes)
-    - [Command Line and API Changes](#command-line-and-api-changes)
    - [Contributors](#contributors)
+- [v0.10.0](#v0100)
+    - [`virtio-block` Support for Multiple Descriptors](#virtio-block-support-for-multiple-descriptors)
+    - [Memory Zones](#memory-zones)
+    - [`Seccomp` Sandbox Improvements](#seccomp-sandbox-improvements)
+    - [Preliminary KVM HyperV Emulation Control](#preliminary-kvm-hyperv-emulation-control)
+    - [Notable Bug Fixes](#notable-bug-fixes-1)
+    - [Contributors](#contributors-1)
 - [v0.9.0](#v090)
    - [`io_uring` Based Block Device Support](#io_uring-based-block-device-support)
    - [Block and Network Device Statistics](#block-and-network-device-statistics)
@@ -18,18 +29,18 @@
    - [Virtio Memory Ballooning Support](#virtio-memory-ballooning-support)
    - [Enhancements to ARM64 Support](#enhancements-to-arm64-support)
    - [Intel SGX Support](#intel-sgx-support)
-    - [`Seccomp` Sandbox Improvements](#seccomp-sandbox-improvements)
-    - [Notable Bug Fixes](#notable-bug-fixes)
-    - [Contributors](#contributors)
+    - [`Seccomp` Sandbox Improvements](#seccomp-sandbox-improvements-1)
+    - [Notable Bug Fixes](#notable-bug-fixes-2)
+    - [Contributors](#contributors-2)
 - [v0.8.0](#v080)
    - [Experimental Snapshot and Restore Support](#experimental-snapshot-and-restore-support)
    - [Experimental ARM64 Support](#experimental-arm64-support)
    - [Support for Using 5-level Paging in Guests](#support-for-using-5-level-paging-in-guests)
    - [Virtio Device Interrupt Suppression for Network Devices](#virtio-device-interrupt-suppression-for-network-devices)
    - [`vhost_user_fs` Improvements](#vhost_user_fs-improvements)
-    - [Notable Bug Fixes](#notable-bug-fixes-1)
+    - [Notable Bug Fixes](#notable-bug-fixes-3)
    - [Command Line and API Changes](#command-line-and-api-changes)
-    - [Contributors](#contributors-1)
+    - [Contributors](#contributors-3)
 - [v0.7.0](#v070)
    - [Block, Network, Persistent Memory (PMEM), VirtioFS and Vsock hotplug](#block-network-persistent-memory-pmem-virtiofs-and-vsock-hotplug)
    - [Alternative `libc` Support](#alternative-libc-support)
@@ -39,14 +50,14 @@
    - [`Seccomp` Sandboxing](#seccomp-sandboxing)
    - [Updated Distribution Support](#updated-distribution-support)
    - [Command Line and API Changes](#command-line-and-api-changes-1)
-    - [Contributors](#contributors-2)
+    - [Contributors](#contributors-4)
 - [v0.6.0](#v060)
    - [Directly Assigned Devices Hotplug](#directly-assigned-devices-hotplug)
    - [Shared Filesystem Improvements](#shared-filesystem-improvements)
    - [Block and Networking IO Self Offloading](#block-and-networking-io-self-offloading)
    - [Command Line Interface](#command-line-interface)
    - [PVH Boot](#pvh-boot)
-    - [Contributors](#contributors-3)
+    - [Contributors](#contributors-5)
 - [v0.5.1](#v051)
 - [v0.5.0](#v050)
    - [Virtual Machine Dynamic Resizing](#virtual-machine-dynamic-resizing)
@@ -54,7 +65,7 @@
    - [New Interrupt Management Framework](#new-interrupt-management-framework)
    - [Development Tools](#development-tools)
    - [Kata Containers Integration](#kata-containers-integration)
-    - [Contributors](#contributors-4)
+    - [Contributors](#contributors-6)
 - [v0.4.0](#v040)
    - [Dynamic virtual CPUs addition](#dynamic-virtual-cpus-addition)
    - [Programmatic firmware tables generation](#programmatic-firmware-tables-generation)
@@ -63,7 +74,7 @@
    - [Userspace IOAPIC by default](#userspace-ioapic-by-default)
    - [PCI BAR reprogramming](#pci-bar-reprogramming)
    - [New `cloud-hypervisor` organization](#new-cloud-hypervisor-organization)
-    - [Contributors](#contributors-5)
+    - [Contributors](#contributors-7)
 - [v0.3.0](#v030)
    - [Block device offloading](#block-device-offloading)
    - [Network device backend](#network-device-backend)
@@ -90,6 +101,154 @@
    - [Unit testing](#unit-testing)
    - [Integration tests parallelization](#integration-tests-parallelization)

+# v0.11.0
+
+This release has been tracked through the [0.11.0 project](https://github.com/cloud-hypervisor/cloud-hypervisor/projects/14).
+
+Highlights for `cloud-hypervisor` version 0.11.0 include:
+
+### `io_uring` support by default for `virtio-block`
+
+Provided that the host OS supports it (Linux kernel 5.8+) then `io_uring` will
+be used for a significantly higher performance block device. 
+
+### Windows Guest Support
+
+This is the first release where we officially support Windows running as a
+guest. Full details of how to setup the image and run Cloud Hypervisor with a
+Windows guest can be found in the dedicated [Windows
+documentation](docs/windows.md).
+
+### `vhost-user` "Self Spawning" Deprecation
+
+Automatically spawning a `vhost-user-net` or `vhost-user-block` backend is now
+deprecated. Users of this functionality will receive a warning and should make
+adjustments. The functionality will be removed in the next release.
+
+### `virtio-mmmio` Removal
+
+Support for using the `virtio-mmio` transport, rather than using PCI, has been
+removed. This has been to simplify the code and significantly
+reduce the testing burden of the project.
+
+### Snapshot/Restore support for ARM64
+
+When running on the ARM64 architecture snapshot and restore has now been
+implemented.
+
+### Improved Linux Boot Time
+
+The time to boot the Linux kernel has been significantly improved by the
+identifying some areas of delays around PCI bus probing, IOAPIC programming and
+MPTABLE issues. Full details can be seen in #1728.
+
+### `SIGTERM/SIGINT` Interrupt Signal Handling
+
+When the VMM process receives the `SIGTERM` or `SIGINT` signals then it will
+trigger the VMM process to cleanly deallocate resources before exiting. The
+guest VM will not be cleanly shutdown but the VMM process will clean up its
+resources.
+
+### Default Log Level Changed
+
+The default logging level was changed to include warnings which should make it
+easier to see potential issues. New [logging
+documentation](docs/logging) was also added.
+
+### New `--balloon` Parameter Added
+
+Control of the setup of `virtio-balloon` has been moved from `--memory` to its
+own dedicated parameter. This makes it easier to add more balloon specific
+controls without overloading `--memory`.
+
+### Experimental `virtio-watchdog` Support
+
+Support for using a new `virtio-watchdog` has been added which can be used to
+have the VMM reboot the guest if the guest userspace fails to ping the
+watchdog. This is enabled with `--watchdog` and requires kernel support.
+
+### Notable Bug Fixes
+
+* MTRR bit was missing from CPUID advertised to guest
+* "Return" key could not be used under `CMD.EXE` under Windows SAC (#1170)
+* CPU identification string is now exposed to the guest
+* `virtio-pmem` with`discard_writes=on` no longer marks the guest memory as
+  read only so avoids excessive VM exits (#1795)
+* PCI device hotplug after an unplug was fixed (#1802)
+* When using the ACPI method to resize the guest memory the full reserved size
+  can be used (#1803)
+* Snapshot and restore followed by a second snapshot and restore now works
+  correctly
+* Snapshot and restore of VMs with more than 2GiB in one region now work
+  correctly
+
+### Contributors
+
+Many thanks to everyone who has contributed to our 0.11.0 release including some new faces.
+
+* Anatol Belski <anbelski@linux.microsoft.com>
+* Bo Chen <chen.bo@intel.com>
+* Daniel Verkamp <dverkamp@chromium.org>
+* Henry Wang <Henry.Wang@arm.com>
+* Hui Zhu <teawater@antfin.com>
+* Jiangbo Wu <jiangbo.wu@intel.com>
+* Josh Soref <jsoref@users.noreply.github.com>
+* Julio Montes <julio.montes@intel.com>
+* Michael Zhao <michael.zhao@arm.com>
+* Muminul Islam <muislam@microsoft.com>
+* pierwill <19642016+pierwill@users.noreply.github.com>
+* Praveen Paladugu <prapal@microsoft.com>
+* Rob Bradford <robert.bradford@intel.com>
+* Sebastien Boeuf <sebastien.boeuf@intel.com>
+* Wei Liu <liuwe@microsoft.com>
+
+
+# v0.10.0
+
+This release has been tracked through the [0.10.0 project](https://github.com/cloud-hypervisor/cloud-hypervisor/projects/13).
+
+Highlights for `cloud-hypervisor` version 0.10.0 include:
+
+### `virtio-block` Support for Multiple Descriptors
+
+Some `virtio-block` device drivers may generate requests with multiple descriptors and support has been added for those drivers.
+
+### Memory Zones
+
+Support has been added for fine grained control of memory allocation for the guest. This includes controlling the backing of sections of guest memory, assigning to specific host NUMA nodes and assigning memory and vCPUs to specific memory nodes inside the guest. Full details of this can be found in the [memory documentation](docs/memory.md).
+
+### `Seccomp` Sandbox Improvements
+
+All the remaining threads and devices are now isolated within their own `seccomp` filters. This provides a layer of sandboxing and enhances the security model of `cloud-hypervisor`.
+
+### Preliminary KVM HyperV Emulation Control
+
+A new option (`kvm_hyperv`) has been added to `--cpus` to provide an option to toggle on KVM's HyperV emulation support. This enables progress towards booting Windows without adding extra emulated devices.
+
+### Notable Bug Fixes
+
+- When using `ch-remote` to resize the VM parameter now accepts the standard sizes suffices (#1596)
+- `cloud-hypervisor` no longer panics when started with `--memory hotplug_method=virtio-mem` and no `hotplug_size` (#1564)
+- After a reboot memory can remove when using `--memory hotplug_method=virtio-mem` (#1593)
+- `--version` shows the version for released binaries (#1669)
+- Errors generated by worker threads for `virtio` devices are now printed out (#1551)
+
+### Contributors
+
+Many thanks to everyone who has contributed to our 0.10.0 release including some new faces.
+
+* Alyssa Ross <hi@alyssa.is>
+* Amey Narkhede <ameynarkhede02@gmail.com>
+* Anatol Belski <ab@php.net>
+* Bo Chen <chen.bo@intel.com>
+* Hui Zhu <teawater@antfin.com>
+* Michael Zhao <michael.zhao@arm.com>
+* Muminul Islam <muislam@microsoft.com>
+* Rob Bradford <robert.bradford@intel.com>
+* Samuel Ortiz <sameo@linux.intel.com>
+* Sebastien Boeuf <sebastien.boeuf@intel.com>
+* Wei Liu <liuwe@microsoft.com>
+
 # v0.9.0

 This release has been tracked through the [0.9.0 project](https://github.com/cloud-hypervisor/cloud-hypervisor/projects/12).
@@ -274,7 +433,7 @@ Highlights for `cloud-hypervisor` version 0.7.0 include:
 Further to our effort to support modifying a running guest we now support
 hotplug and unplug of the following virtio backed devices: block, network,
 pmem, virtio-fs and vsock. This functionality is available on the (default) PCI
-based tranport and is exposed through the HTTP API. The `ch-remote` utility
+based transport and is exposed through the HTTP API. The `ch-remote` utility
 provides a CLI for adding or removing these device types after the VM has
 booted. User can use the `id` parameter on the devices to choose names for
 devices to ease their removal.
@@ -390,7 +549,7 @@ When choosing to offload the paravirtualized block and networking I/O to an
 external process (through the `vhost-user` protocol), Cloud Hypervisor now
 automatically spawns its default `vhost-user-blk` and `vhost-user-net` backends
 into their own, separate processes.
-This provides a seamless parvirtualized I/O user experience for those who want
+This provides a seamless paravirtualized I/O user experience for those who want
 to run their guest I/O into separate executions contexts.

 ### Command Line Interface
@@ -536,7 +695,7 @@ default Cloud Hypervisor I/O architecture.

 ### Guest pause and resume

-As an initial requiremnt for enabling live migration, we added support for
+As an initial requirement for enabling live migration, we added support for
 pausing and resuming any VMM components. As an intermediate step towards live
 migration, the upcoming guest snapshotting feature will be based on the pause
 and resume capabilities.
--- a/resources/Dockerfile
+++ b/resources/Dockerfile
@@ -1,7 +1,7 @@
 FROM ubuntu:18.04 as dev

-ARG TARGETARCH="x86_64"
-ARG RUST_TOOLCHAIN="1.43.0"
+ARG TARGETARCH
+ARG RUST_TOOLCHAIN="1.46.0"
 ARG CLH_SRC_DIR="/cloud-hypervisor"
 ARG CLH_BUILD_DIR="$CLH_SRC_DIR/build"
 ARG CARGO_REGISTRY_DIR="$CLH_BUILD_DIR/cargo_registry"
@@ -39,17 +39,18 @@ RUN apt-get update \
 	cpio \
 	bsdtar \
 	libfdt-dev \
+	python3-setuptools \
    && apt-get clean \
    && rm -rf /var/lib/apt/lists/*

-RUN if [ "$TARGETARCH" = "x86_64" ]; then \
+RUN if [ "$TARGETARCH" = "amd64" ]; then \
 	apt-get update \
 	&& apt-get -yq upgrade \
 	&& DEBIAN_FRONTEND=noninteractive apt-get install -yq gcc-multilib \
 	&& apt-get clean \
    && rm -rf /var/lib/apt/lists/*;	fi

-RUN if [ "$TARGETARCH" = "aarch64" ]; then \
+RUN if [ "$TARGETARCH" = "arm64" ]; then \
        # On AArch64, `setcap` binary should be installed via `libcap2-bin`.
        # The `setcap` binary is used in integration tests.
        apt-get update \
@@ -66,17 +67,21 @@ RUN if [ "$TARGETARCH" = "aarch64" ]; then \
        && rm -rf /dtc;	fi

 # Fix the libssl-dev install
-RUN cp /usr/include/"$TARGETARCH"-linux-gnu/openssl/opensslconf.h /usr/include/openssl/
-ENV OPENSSL_DIR=/usr/lib/"$TARGETARCH"-linux-gnu/
-ENV OPENSSL_LIB_DIR=/usr/lib/"$TARGETARCH"-linux-gnu/
+RUN export ARCH="$(uname -m)" \
+    && cp /usr/include/$ARCH-linux-gnu/openssl/opensslconf.h /usr/include/openssl/
+ENV X86_64_UNKNOWN_LINUX_GNU_OPENSSL_LIB_DIR=/usr/lib/x86_64-linux-gnu/
+ENV X86_64_UNKNOWN_LINUX_MUSL_OPENSSL_LIB_DIR=/usr/lib/x86_64-linux-gnu/
+ENV AARCH64_UNKNOWN_LINUX_GNU_OPENSSL_LIB_DIR=/usr/lib/aarch64-linux-gnu/
+ENV AARCH64_UNKNOWN_LINUX_MUSL_OPENSSL_LIB_DIR=/usr/lib/aarch64-linux-gnu/
 ENV OPENSSL_INCLUDE_DIR=/usr/include/

 # Install the rust toolchain
-RUN nohup curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain "$RUST_TOOLCHAIN" \
-    && rustup target add $TARGETARCH-unknown-linux-musl --toolchain "$RUST_TOOLCHAIN" \
-    && if [ "$TARGETARCH" = "x86_64" ]; then rustup toolchain add $RUST_TOOLCHAIN-x86_64-unknown-linux-musl; fi \
-    && if [ "$TARGETARCH" = "x86_64" ]; then rustup component add rustfmt; fi \
-    && if [ "$TARGETARCH" = "x86_64" ]; then rustup component add clippy; fi \
+RUN export ARCH="$(uname -m)" \
+    && nohup curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain "$RUST_TOOLCHAIN" \
+    && rustup target add $ARCH-unknown-linux-musl --toolchain "$RUST_TOOLCHAIN" \
+    && if [ "$TARGETARCH" = "amd64" ]; then rustup toolchain add $RUST_TOOLCHAIN-x86_64-unknown-linux-musl; fi \
+    && if [ "$TARGETARCH" = "amd64" ]; then rustup component add rustfmt; fi \
+    && if [ "$TARGETARCH" = "amd64" ]; then rustup component add clippy; fi \
    && cargo install cargo-audit \
    && rm -rf "$CARGO_HOME/registry" \
    && ln -s "$CARGO_REGISTRY_DIR" "$CARGO_HOME/registry" \
--- a/resources/linux-config-x86_64
+++ b/resources/linux-config-x86_64
@@ -266,10 +266,8 @@ CONFIG_ARCH_MMAP_RND_BITS_MIN=28
 CONFIG_ARCH_MMAP_RND_BITS_MAX=32
 CONFIG_ARCH_MMAP_RND_COMPAT_BITS_MIN=8
 CONFIG_ARCH_MMAP_RND_COMPAT_BITS_MAX=16
-CONFIG_GENERIC_ISA_DMA=y
 CONFIG_GENERIC_BUG=y
 CONFIG_GENERIC_BUG_RELATIVE_POINTERS=y
-CONFIG_ARCH_MAY_HAVE_PC_FDC=y
 CONFIG_GENERIC_CALIBRATE_DELAY=y
 CONFIG_ARCH_HAS_CPU_RELAX=y
 CONFIG_ARCH_HAS_CACHE_LINE_SIZE=y
@@ -571,7 +569,7 @@ CONFIG_PCI_MMCONFIG=y
 CONFIG_MMCONF_FAM10H=y
 # CONFIG_PCI_CNB20LE_QUIRK is not set
 # CONFIG_ISA_BUS is not set
-CONFIG_ISA_DMA_API=y
+# CONFIG_ISA_DMA_API is not set
 CONFIG_AMD_NB=y
 # CONFIG_X86_SYSFB is not set
 # end of Bus options (PCI etc.)
@@ -838,7 +836,7 @@ CONFIG_MEMORY_ISOLATION=y
 CONFIG_HAVE_BOOTMEM_INFO_NODE=y
 CONFIG_MEMORY_HOTPLUG=y
 CONFIG_MEMORY_HOTPLUG_SPARSE=y
-# CONFIG_MEMORY_HOTPLUG_DEFAULT_ONLINE is not set
+CONFIG_MEMORY_HOTPLUG_DEFAULT_ONLINE=y
 CONFIG_MEMORY_HOTREMOVE=y
 CONFIG_SPLIT_PTLOCK_CPUS=4
 CONFIG_MEMORY_BALLOON=y
@@ -1150,7 +1148,6 @@ CONFIG_PNP=y
 CONFIG_PNPACPI=y
 CONFIG_BLK_DEV=y
 CONFIG_BLK_DEV_NULL_BLK=y
-# CONFIG_BLK_DEV_FD is not set
 # CONFIG_BLK_DEV_PCIESSD_MTIP32XX is not set
 CONFIG_ZRAM=y
 # CONFIG_ZRAM_WRITEBACK is not set
@@ -1447,7 +1444,7 @@ CONFIG_DEVPORT=y
 # CONFIG_HPET is not set
 CONFIG_HANGCHECK_TIMER=y
 # CONFIG_TCG_TPM is not set
-CONFIG_TELCLOCK=y
+# CONFIG_TELCLOCK is not set
 # CONFIG_XILLYBUS is not set
 # end of Character devices

@@ -1465,14 +1462,14 @@ CONFIG_TELCLOCK=y
 # CONFIG_SPMI is not set
 # CONFIG_HSI is not set
 CONFIG_PPS=y
-CONFIG_PPS_DEBUG=y
+# CONFIG_PPS_DEBUG is not set

 #
 # PPS clients support
 #
-CONFIG_PPS_CLIENT_KTIMER=y
-CONFIG_PPS_CLIENT_LDISC=y
-CONFIG_PPS_CLIENT_GPIO=y
+# CONFIG_PPS_CLIENT_KTIMER is not set
+# CONFIG_PPS_CLIENT_LDISC is not set
+# CONFIG_PPS_CLIENT_GPIO is not set

 #
 # PPS generators support
@@ -1525,7 +1522,68 @@ CONFIG_THERMAL_GOV_USER_SPACE=y
 # CONFIG_INTEL_PCH_THERMAL is not set
 # end of Intel thermal drivers

-# CONFIG_WATCHDOG is not set
+CONFIG_WATCHDOG=y
+CONFIG_WATCHDOG_CORE=y
+# CONFIG_WATCHDOG_NOWAYOUT is not set
+CONFIG_WATCHDOG_HANDLE_BOOT_ENABLED=y
+CONFIG_WATCHDOG_OPEN_TIMEOUT=0
+# CONFIG_WATCHDOG_SYSFS is not set
+
+#
+# Watchdog Pretimeout Governors
+#
+# CONFIG_WATCHDOG_PRETIMEOUT_GOV is not set
+
+#
+# Watchdog Device Drivers
+#
+# CONFIG_SOFT_WATCHDOG is not set
+# CONFIG_WDAT_WDT is not set
+# CONFIG_XILINX_WATCHDOG is not set
+# CONFIG_CADENCE_WATCHDOG is not set
+# CONFIG_DW_WATCHDOG is not set
+# CONFIG_MAX63XX_WATCHDOG is not set
+# CONFIG_ACQUIRE_WDT is not set
+# CONFIG_ADVANTECH_WDT is not set
+# CONFIG_ALIM1535_WDT is not set
+# CONFIG_ALIM7101_WDT is not set
+# CONFIG_EBC_C384_WDT is not set
+# CONFIG_F71808E_WDT is not set
+# CONFIG_SP5100_TCO is not set
+# CONFIG_SBC_FITPC2_WATCHDOG is not set
+# CONFIG_EUROTECH_WDT is not set
+# CONFIG_IB700_WDT is not set
+# CONFIG_IBMASR is not set
+# CONFIG_WAFER_WDT is not set
+# CONFIG_I6300ESB_WDT is not set
+# CONFIG_IE6XX_WDT is not set
+# CONFIG_ITCO_WDT is not set
+# CONFIG_IT8712F_WDT is not set
+# CONFIG_IT87_WDT is not set
+# CONFIG_HP_WATCHDOG is not set
+# CONFIG_SC1200_WDT is not set
+# CONFIG_PC87413_WDT is not set
+# CONFIG_NV_TCO is not set
+# CONFIG_60XX_WDT is not set
+# CONFIG_CPU5_WDT is not set
+# CONFIG_SMSC_SCH311X_WDT is not set
+# CONFIG_SMSC37B787_WDT is not set
+# CONFIG_TQMX86_WDT is not set
+# CONFIG_VIA_WDT is not set
+# CONFIG_W83627HF_WDT is not set
+# CONFIG_W83877F_WDT is not set
+# CONFIG_W83977F_WDT is not set
+# CONFIG_MACHZ_WDT is not set
+# CONFIG_SBC_EPX_C3_WATCHDOG is not set
+# CONFIG_NI903X_WDT is not set
+# CONFIG_NIC7018_WDT is not set
+CONFIG_VIRTIO_WDT=y
+
+#
+# PCI-based Watchdog Cards
+#
+# CONFIG_PCIPCWATCHDOG is not set
+# CONFIG_WDTPCI is not set
 CONFIG_SSB_POSSIBLE=y
 # CONFIG_SSB is not set
 CONFIG_BCMA_POSSIBLE=y
--- a/scripts/dev_cli.sh
+++ b/scripts/dev_cli.sh
@@ -7,7 +7,7 @@
 CLI_NAME="Cloud Hypervisor"

 CTR_IMAGE_TAG="cloudhypervisor/dev"
-CTR_IMAGE_VERSION="v4"
+CTR_IMAGE_VERSION="latest"
 CTR_IMAGE="${CTR_IMAGE_TAG}:${CTR_IMAGE_VERSION}"

 DOCKER_RUNTIME="docker"
@@ -136,13 +136,32 @@ fix_dir_perms() {
 	--workdir "$CTR_CLH_ROOT_DIR" \
 	   --rm \
 	   --volume /dev:/dev \
-	   --volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR" \
+	   --volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR" $exported_volumes \
 	   "$CTR_IMAGE" \
           chown -R "$(id -u):$(id -g)" "$CTR_CLH_ROOT_DIR"

    return $1
 }
-
+# Process exported volumes argument, separate the volumes and make docker compatible
+# Sample input: --volumes /a:/a#/b:/b
+# Sample output: --volume /a:/a --volume /b:/b
+#
+process_volumes_args() {
+    if [ -z "$arg_vols" ]; then
+        return
+    fi
+    exported_volumes=""
+    arr_vols=(${arg_vols//#/ })
+    for var in "${arr_vols[@]}"
+    do
+        parts=(${var//:/ })
+        if [[ ! -e "${parts[0]}" ]]; then
+            echo "The volume ${parts[0]} does not exist."
+            exit 1
+        fi
+        exported_volumes="$exported_volumes --volume $var"
+    done
+}
 cmd_help() {
    echo ""
    echo "Cloud Hypervisor $(basename $0)"
@@ -155,14 +174,18 @@ cmd_help() {
    echo "        --debug               Build the debug binaries. This is the default."
    echo "        --release             Build the release binaries."
    echo "        --libc                Select the C library Cloud Hypervisor will be built against. Default is gnu"
+    echo "        --volumes             Hash separated volumes to be exported. Example --volumes /mnt:/mnt#/myvol:/myvol"
    echo ""
    echo "    tests [--unit|--cargo|--all] [--libc musl|gnu] [-- [<cargo test args>]]"
    echo "        Run the Cloud Hypervisor tests."
-    echo "        --unit               Run the unit tests."
-    echo "        --cargo              Run the cargo tests."
-    echo "        --integration        Run the integration tests."
-    echo "        --libc               Select the C library Cloud Hypervisor will be built against. Default is gnu"
-    echo "        --all                Run all tests."
+    echo "        --unit                Run the unit tests."
+    echo "        --cargo               Run the cargo tests."
+    echo "        --integration         Run the integration tests."
+    echo "        --integration-sgx     Run the SGX integration tests."
+    echo "        --integration-windows Run the Windows guest integration tests."
+    echo "        --libc                Select the C library Cloud Hypervisor will be built against. Default is gnu"
+    echo "        --volumes             Hash separated volumes to be exported. Example --volumes /mnt:/mnt#/myvol:/myvol"
+    echo "        --all                 Run all tests."
    echo ""
    echo "    build-container [--type]"
    echo "        Build the Cloud Hypervisor container."
@@ -194,6 +217,10 @@ cmd_build() {
                    die "Invalid libc: $1. Valid options are \"musl\" and \"gnu\"."
                libc="$1"
                ;;
+            "--volumes")
+                shift
+                arg_vols="$1"
+                ;;
            "--")           { shift; break;         } ;;
            *)
 		die "Unknown build argument: $1. Please use --help for help."
@@ -201,6 +228,7 @@ cmd_build() {
 	esac
 	shift
    done
+    process_volumes_args

    target="$(uname -m)-unknown-linux-${libc}"

@@ -208,7 +236,7 @@ cmd_build() {
    [ $build = "release" ] && cargo_args+=("--release")
    cargo_args+=(--target "$target")
    [ $(uname -m) = "aarch64" ] && cargo_args+=("--no-default-features")
-    [ $(uname -m) = "aarch64" ] && cargo_args+=(--features "mmio,kvm")
+    [ $(uname -m) = "aarch64" ] && cargo_args+=(--features "kvm")

    rustflags=""
    if [ $(uname -m) = "aarch64" ] && [ $libc = "musl" ] ; then
@@ -220,7 +248,7 @@ cmd_build() {
 	   --workdir "$CTR_CLH_ROOT_DIR" \
 	   --rm \
 	   --volume /dev:/dev \
-	   --volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR" \
+	   --volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR" $exported_volumes \
 	   --env RUSTFLAGS="$rustflags" \
 	   "$CTR_IMAGE" \
 	   cargo build --all \
@@ -235,7 +263,7 @@ cmd_clean() {
 	   --user "$(id -u):$(id -g)" \
 	   --workdir "$CTR_CLH_ROOT_DIR" \
 	   --rm \
-	   --volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR" \
+	   --volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR" $exported_volumes \
 	   "$CTR_IMAGE" \
 	   cargo clean \
 	         --target-dir "$CTR_CLH_CARGO_TARGET" \
@@ -246,7 +274,10 @@ cmd_tests() {
    unit=false
    cargo=false
    integration=false
+    integration_sgx=false
+    integration_windows=false
    libc="gnu"
+    arg_vols=""

    while [ $# -gt 0 ]; do
 	case "$1" in
@@ -254,12 +285,18 @@ cmd_tests() {
            "--unit")                { unit=true;      } ;;
            "--cargo")               { cargo=true;    } ;;
            "--integration")         { integration=true;    } ;;
+            "--integration-sgx")     { integration_sgx=true;    } ;;
+            "--integration-windows") { integration_windows=true;    } ;;
            "--libc")
                shift
                [[ "$1" =~ ^(musl|gnu)$ ]] || \
                    die "Invalid libc: $1. Valid options are \"musl\" and \"gnu\"."
                libc="$1"
                ;;
+            "--volumes")
+                shift
+                arg_vols="$1"
+                ;;
 	    "--all")                 { cargo=true; unit=true; integration=true;  } ;;
            "--")                    { shift; break;         } ;;
            *)
@@ -269,6 +306,7 @@ cmd_tests() {
 	shift
    done

+    process_volumes_args
    target="$(uname -m)-unknown-linux-${libc}"
    cflags=""
    target_cc=""
@@ -285,7 +323,7 @@ cmd_tests() {
 	       --device /dev/kvm \
 	       --device /dev/net/tun \
 	       --cap-add net_admin \
-	       --volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR" \
+	       --volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR" $exported_volumes \
 	       --env BUILD_TARGET="$target" \
 	       --env CFLAGS="$cflags" \
 	       --env TARGET_CC="$target_cc" \
@@ -298,7 +336,7 @@ cmd_tests() {
 	$DOCKER_RUNTIME run \
 	       --workdir "$CTR_CLH_ROOT_DIR" \
 	       --rm \
-	       --volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR" \
+	       --volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR" $exported_volumes \
 	       "$CTR_IMAGE" \
 	       ./scripts/run_cargo_tests.sh || fix_dir_perms $? || exit $?
    fi
@@ -314,7 +352,7 @@ cmd_tests() {
 	       --net="$CTR_CLH_NET" \
 	       --mount type=tmpfs,destination=/tmp \
 	       --volume /dev:/dev \
-	       --volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR" \
+	       --volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR" $exported_volumes \
 	       --volume "$CLH_INTEGRATION_WORKLOADS:$CTR_CLH_INTEGRATION_WORKLOADS" \
 	       --env USER="root" \
 	       --env CH_LIBC="${libc}" \
@@ -322,6 +360,43 @@ cmd_tests() {
 	       ./scripts/run_integration_tests_$(uname -m).sh "$@" || fix_dir_perms $? || exit $?
    fi

+    if [ "$integration_sgx" = true ] ;  then
+	say "Running SGX integration tests for $target..."
+	$DOCKER_RUNTIME run \
+	       --workdir "$CTR_CLH_ROOT_DIR" \
+	       --rm \
+	       --privileged \
+	       --security-opt seccomp=unconfined \
+	       --ipc=host \
+	       --net="$CTR_CLH_NET" \
+	       --mount type=tmpfs,destination=/tmp \
+	       --volume /dev:/dev \
+	       --volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR" $exported_volumes \
+	       --volume "$CLH_INTEGRATION_WORKLOADS:$CTR_CLH_INTEGRATION_WORKLOADS" \
+	       --env USER="root" \
+	       --env CH_LIBC="${libc}" \
+	       "$CTR_IMAGE" \
+	       ./scripts/run_integration_tests_sgx.sh "$@" || fix_dir_perms $? || exit $?
+    fi
+
+    if [ "$integration_windows" = true ] ;  then
+	say "Running Windows integration tests for $target..."
+	$DOCKER_RUNTIME run \
+	       --workdir "$CTR_CLH_ROOT_DIR" \
+	       --rm \
+	       --privileged \
+	       --security-opt seccomp=unconfined \
+	       --ipc=host \
+	       --net="$CTR_CLH_NET" \
+	       --mount type=tmpfs,destination=/tmp \
+	       --volume /dev:/dev \
+	       --volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR" \
+	       --volume "$CLH_INTEGRATION_WORKLOADS:$CTR_CLH_INTEGRATION_WORKLOADS" \
+	       --env USER="root" \
+	       --env CH_LIBC="${libc}" \
+	       "$CTR_IMAGE" \
+	       ./scripts/run_integration_tests_windows.sh "$@" || fix_dir_perms $? || exit $?
+    fi
    fix_dir_perms $?
 }

@@ -345,11 +420,14 @@ cmd_build-container() {
    mkdir -p $BUILD_DIR
    cp $CLH_DOCKERFILE $BUILD_DIR

+    [ $(uname -m) = "aarch64" ] && TARGETARCH="arm64"
+    [ $(uname -m) = "x86_64" ] && TARGETARCH="amd64"
+
    $DOCKER_RUNTIME build \
 	   --target $container_type \
 	   -t $CTR_IMAGE \
 	   -f $BUILD_DIR/Dockerfile \
-	   --build-arg TARGETARCH="$(uname -m)" \
+	   --build-arg TARGETARCH=$TARGETARCH \
 	   $BUILD_DIR
 }

@@ -366,7 +444,7 @@ cmd_shell() {
 	   --net="$CTR_CLH_NET" \
 	   --tmpfs /tmp:exec \
 	   --volume /dev:/dev \
-	   --volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR" \
+	   --volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR" $exported_volumes \
 	   --volume "$CLH_INTEGRATION_WORKLOADS:$CTR_CLH_INTEGRATION_WORKLOADS" \
 	   --env USER="root" \
 	   --entrypoint bash \
@@ -405,9 +483,4 @@ if [ $(uname -m) = "x86_64" ]; then
    ensure_latest_ctr
 fi

-# Before a public image for AArch64 ready, we build the container if needed.
-if [ $(uname -m) = "aarch64" ]; then
-    cmd_build-container
-fi
-
 $cmd "$@"
--- a/scripts/run_cargo_tests.sh
+++ b/scripts/run_cargo_tests.sh
@@ -15,11 +15,9 @@ time cargo rustc --bin cloud-hypervisor -- -D warnings
 time cargo rustc -p vhost_user_net --bin vhost_user_net -- -D warnings
 time cargo test
 time cargo audit
-time cargo clippy --all-targets --no-default-features --features "pci,acpi,kvm" -- -D warnings
-time cargo rustc --bin cloud-hypervisor --no-default-features --features "pci,acpi,kvm"  -- -D warnings
-time cargo clippy --all-targets --no-default-features --features "pci,kvm" -- -D warnings
-time cargo rustc --bin cloud-hypervisor --no-default-features --features "pci,kvm"  -- -D warnings
-time cargo clippy --all-targets --no-default-features --features "mmio,kvm" -- -D warnings
-time cargo rustc --bin cloud-hypervisor --no-default-features --features "mmio,kvm"  -- -D warnings
+time cargo clippy --all-targets --no-default-features --features "acpi,kvm" -- -D warnings
+time cargo rustc --bin cloud-hypervisor --no-default-features --features "acpi,kvm"  -- -D warnings
+time cargo clippy --all-targets --no-default-features --features "kvm" -- -D warnings
+time cargo rustc --bin cloud-hypervisor --no-default-features --features "kvm"  -- -D warnings
 time cargo fmt -- --check
 time cargo build --all --release
--- a/scripts/run_integration_tests_aarch64.sh
+++ b/scripts/run_integration_tests_aarch64.sh
@@ -126,14 +126,13 @@ update_workloads() {

    if [ ! -f "$VIRTIOFSD" ]; then
        pushd $WORKLOADS_DIR
-        git clone --depth 1 "https://gitlab.com/virtio-fs/qemu.git" -b "virtio-fs-dev" $QEMU_DIR
+        git clone --depth 1 "https://gitlab.com/virtio-fs/qemu.git" -b "qemu5.0-virtiofs-dax" $QEMU_DIR
        pushd $QEMU_DIR
        time ./configure --prefix=$PWD --target-list=aarch64-softmmu
        time make virtiofsd -j `nproc`
        cp virtiofsd $VIRTIOFSD || exit 1
        popd
        rm -rf $QEMU_DIR
-        sudo setcap cap_chown,cap_dac_override,cap_dac_read_search,cap_fowner,cap_fsetid,cap_setgid,cap_setuid,cap_mknod,cap_setfcap,cap_sys_admin+epi "virtiofsd" || exit 1
        popd
    fi

@@ -173,9 +172,9 @@ if [ $RES -ne 0 ]; then
    exit 1
 fi

-# Create tap interface without multipe queues support for vhost_user_net test.
+# Create tap interface without multiple queues support for vhost_user_net test.
 sudo ip tuntap add name vunet-tap0 mode tap
-# Create tap interface with multipe queues support for vhost_user_net test.
+# Create tap interface with multiple queues support for vhost_user_net test.
 sudo ip tuntap add name vunet-tap1 mode tap multi_queue

 BUILD_TARGET="aarch64-unknown-linux-${CH_LIBC}"
@@ -186,52 +185,21 @@ TARGET_CC="musl-gcc"
 CFLAGS="-I /usr/include/aarch64-linux-musl/ -idirafter /usr/include/"
 fi

-cargo build --all --release --no-default-features --features pci,kvm --target $BUILD_TARGET
+cargo build --all --release --no-default-features --features kvm --target $BUILD_TARGET
 strip target/$BUILD_TARGET/release/cloud-hypervisor
 strip target/$BUILD_TARGET/release/vhost_user_net
 strip target/$BUILD_TARGET/release/ch-remote

-# Copy for non-privileged net test
-cp target/$BUILD_TARGET/release/cloud-hypervisor target/$BUILD_TARGET/release/cloud-hypervisor-unprivileged
-
-sudo setcap cap_net_admin+ep target/$BUILD_TARGET/release/cloud-hypervisor
-sudo setcap cap_net_admin+ep target/$BUILD_TARGET/release/vhost_user_net
-
 # Enable KSM with some reasonable parameters so that it won't take too long
 # for the memory to be merged between two processes.
 sudo bash -c "echo 1000000 > /sys/kernel/mm/ksm/pages_to_scan"
 sudo bash -c "echo 10 > /sys/kernel/mm/ksm/sleep_millisecs"
 sudo bash -c "echo 1 > /sys/kernel/mm/ksm/run"

-sudo adduser $USER kvm
-newgrp kvm << EOF
 export RUST_BACKTRACE=1
-time cargo test --no-default-features --features "integration_tests,pci,kvm" "tests::parallel::$@" -- --nocapture
-EOF
+time cargo test --no-default-features --features "integration_tests,kvm" "tests::parallel::$@" -- --skip test_snapshot_restore
 RES=$?

-if [ $RES -eq 0 ]; then
-    # virtio-mmio based testing
-    cargo build --release --target $BUILD_TARGET --no-default-features --features "mmio,kvm"
-    strip target/$BUILD_TARGET/release/cloud-hypervisor
-    strip target/$BUILD_TARGET/release/vhost_user_net
-    strip target/$BUILD_TARGET/release/ch-remote
-
-    sudo setcap cap_net_admin+ep target/$BUILD_TARGET/release/cloud-hypervisor
-    sudo setcap cap_net_admin+ep target/$BUILD_TARGET/release/vhost_user_net
-
-    # Ensure test binary has the same caps as the cloud-hypervisor one
-    time cargo test --no-run --no-default-features --features "integration_tests,mmio,kvm" -- --nocapture || exit 1
-    ls target/debug/deps/cloud_hypervisor-* | xargs -n 1 sudo setcap cap_net_admin+ep
-
-    newgrp kvm << EOF
-export RUST_BACKTRACE=1
-time cargo test --no-default-features --features "integration_tests,mmio,kvm" "tests::parallel::$@" -- --nocapture
-EOF
-
-    RES=$?
-fi
-
 # Tear vhost_user_net test network down
 sudo ip link del vunet-tap0
 sudo ip link del vunet-tap1
--- a/scripts/run_integration_tests_sgx.sh
+++ b/scripts/run_integration_tests_sgx.sh
@@ -0,0 +1,22 @@
+#!/bin/bash
+set -x
+
+source $HOME/.cargo/env
+
+BUILD_TARGET="$(uname -m)-unknown-linux-${CH_LIBC}"
+CFLAGS=""
+TARGET_CC=""
+if [[ "${BUILD_TARGET}" == "x86_64-unknown-linux-musl" ]]; then
+TARGET_CC="musl-gcc"
+CFLAGS="-I /usr/include/x86_64-linux-musl/ -idirafter /usr/include/"
+fi
+
+cargo build --all --release --target $BUILD_TARGET
+strip target/$BUILD_TARGET/release/cloud-hypervisor
+
+export RUST_BACKTRACE=1
+
+time cargo test --features "integration_tests" "tests::sgx::$@"
+RES=$?
+
+exit $RES
--- a/scripts/run_integration_tests_windows.sh
+++ b/scripts/run_integration_tests_windows.sh
@@ -0,0 +1,24 @@
+#!/bin/bash
+set -x
+
+source $HOME/.cargo/env
+
+BUILD_TARGET="$(uname -m)-unknown-linux-${CH_LIBC}"
+CFLAGS=""
+TARGET_CC=""
+if [[ "${BUILD_TARGET}" == "x86_64-unknown-linux-musl" ]]; then
+TARGET_CC="musl-gcc"
+CFLAGS="-I /usr/include/x86_64-linux-musl/ -idirafter /usr/include/"
+fi
+
+cargo build --all --release --target $BUILD_TARGET
+strip target/$BUILD_TARGET/release/cloud-hypervisor
+
+export RUST_BACKTRACE=1
+
+# Only run with 1 thread to avoid tests interfering with one another because
+# Windows has a static IP configured
+time cargo test --features "integration_tests" "tests::windows::$@" -- --test-threads=1
+RES=$?
+
+exit $RES
--- a/scripts/run_integration_tests_x86_64.sh
+++ b/scripts/run_integration_tests_x86_64.sh
@@ -127,14 +127,13 @@ VIRTIOFSD="$WORKLOADS_DIR/virtiofsd"
 QEMU_DIR="qemu_build"
 if [ ! -f "$VIRTIOFSD" ]; then
    pushd $WORKLOADS_DIR
-    git clone --depth 1 "https://gitlab.com/virtio-fs/qemu.git" -b "virtio-fs-dev" $QEMU_DIR
+    git clone --depth 1 "https://gitlab.com/virtio-fs/qemu.git" -b "qemu5.0-virtiofs-dax" $QEMU_DIR
    pushd $QEMU_DIR
    time ./configure --prefix=$PWD --target-list=x86_64-softmmu
    time make virtiofsd -j `nproc`
    cp virtiofsd $VIRTIOFSD || exit 1
    popd
    rm -rf $QEMU_DIR
-    sudo setcap cap_chown,cap_dac_override,cap_dac_read_search,cap_fowner,cap_fsetid,cap_setgid,cap_setuid,cap_mknod,cap_setfcap,cap_sys_admin+epi "virtiofsd" || exit 1
    popd
 fi

@@ -160,7 +159,8 @@ if [ ! -d "$SHARED_DIR" ]; then
 fi

 VFIO_DIR="$WORKLOADS_DIR/vfio"
-rm -rf $VFIO_DIR
+VFIO_DISK_IMAGE="$WORKLOADS_DIR/vfio.img"
+rm -rf $VFIO_DIR $VFIO_DISK_IMAGE
 mkdir -p $VFIO_DIR
 cp $FOCAL_OS_IMAGE $VFIO_DIR
 cp $FW $VFIO_DIR
@@ -188,9 +188,9 @@ sudo ip tuntap add vfio-tap3 mode tap
 sudo ip link set vfio-tap3 master vfio-br0
 sudo ip link set vfio-tap3 up

-# Create tap interface without multipe queues support for vhost_user_net test.
+# Create tap interface without multiple queues support for vhost_user_net test.
 sudo ip tuntap add name vunet-tap0 mode tap
-# Create tap interface with multipe queues support for vhost_user_net test.
+# Create tap interface with multiple queues support for vhost_user_net test.
 sudo ip tuntap add name vunet-tap1 mode tap multi_queue

 BUILD_TARGET="$(uname -m)-unknown-linux-${CH_LIBC}"
@@ -206,12 +206,6 @@ strip target/$BUILD_TARGET/release/cloud-hypervisor
 strip target/$BUILD_TARGET/release/vhost_user_net
 strip target/$BUILD_TARGET/release/ch-remote

-# Copy for non-privileged net test
-cp target/$BUILD_TARGET/release/cloud-hypervisor target/$BUILD_TARGET/release/cloud-hypervisor-unprivileged
-
-sudo setcap cap_net_admin+ep target/$BUILD_TARGET/release/cloud-hypervisor
-sudo setcap cap_net_admin+ep target/$BUILD_TARGET/release/vhost_user_net
-
 # We always copy a fresh version of our binary for our L2 guest.
 cp target/$BUILD_TARGET/release/cloud-hypervisor $VFIO_DIR
 cp target/$BUILD_TARGET/release/ch-remote $VFIO_DIR
@@ -226,58 +220,18 @@ sudo bash -c "echo 1 > /sys/kernel/mm/ksm/run"
 echo 4096 | sudo tee /proc/sys/vm/nr_hugepages
 sudo chmod a+rwX /dev/hugepages

-# Ensure test binary has the same caps as the cloud-hypervisor one
-time cargo test --no-run --features "integration_tests" -- --nocapture || exit 1
-ls target/debug/deps/cloud_hypervisor-* | xargs -n 1 sudo setcap cap_net_admin+ep
-
-sudo adduser $USER kvm
-newgrp kvm << EOF
 export RUST_BACKTRACE=1
-time cargo test --features "integration_tests" "tests::parallel::$@" -- --nocapture
-EOF
+time cargo test --features "integration_tests" "tests::parallel::$@"
 RES=$?

 # Run some tests in sequence since the result could be affected by other tests
 # running in parallel.
 if [ $RES -eq 0 ]; then
-    newgrp kvm << EOF
-export RUST_BACKTRACE=1
-time cargo test --features "integration_tests" "tests::sequential::$@" -- --nocapture --test-threads=1
-EOF
+    export RUST_BACKTRACE=1
+    time cargo test --features "integration_tests" "tests::sequential::$@" -- --test-threads=1
    RES=$?
 fi

-if [ $RES -eq 0 ]; then
-    # virtio-mmio based testing
-    cargo build --all --release --target $BUILD_TARGET --no-default-features --features "mmio,kvm"
-    strip target/$BUILD_TARGET/release/cloud-hypervisor
-    strip target/$BUILD_TARGET/release/vhost_user_net
-    strip target/$BUILD_TARGET/release/ch-remote
-
-    sudo setcap cap_net_admin+ep target/$BUILD_TARGET/release/cloud-hypervisor
-
-    # Ensure test binary has the same caps as the cloud-hypervisor one
-    time cargo test --no-run --features "integration_tests,mmio" -- --nocapture || exit 1
-    ls target/debug/deps/cloud_hypervisor-* | xargs -n 1 sudo setcap cap_net_admin+ep
-
-    newgrp kvm << EOF
-export RUST_BACKTRACE=1
-time cargo test --features "integration_tests,mmio" "tests::parallel::$@" -- --nocapture
-EOF
-
-    RES=$?
-
-    # Run some tests in sequence since the result could be affected by other tests
-    # running in parallel.
-    if [ $RES -eq 0 ]; then
-        newgrp kvm << EOF
-export RUST_BACKTRACE=1
-time cargo test --features "integration_tests,mmio" "tests::sequential::$@" -- --nocapture --test-threads=1
-EOF
-        RES=$?
-    fi
-fi
-
 # Tear VFIO test network down
 sudo ip link del vfio-br0
 sudo ip link del vfio-tap0
--- a/scripts/run_unit_tests.sh
+++ b/scripts/run_unit_tests.sh
@@ -5,16 +5,7 @@ source $HOME/.cargo/env
 BUILD_TARGET=${BUILD_TARGET-x86_64-unknown-linux-gnu}
 cargo_args=("$@")
 [ $(uname -m) = "aarch64" ] && cargo_args+=("--no-default-features")
-[ $(uname -m) = "aarch64" ] && cargo_args+=("--features mmio,kvm")
-[ $(uname -m) = "aarch64" ] && sed -i 's/"with-serde",\ //g' hypervisor/Cargo.toml
+[ $(uname -m) = "aarch64" ] && cargo_args+=("--features kvm")

-cargo test --target $BUILD_TARGET --workspace --no-run ${cargo_args[@]}
-pushd target/$BUILD_TARGET/debug
-ls  | grep net_util | grep -v "\.d" | xargs -n 1 sudo setcap cap_net_admin,cap_net_raw+ep
-popd
-
-sudo adduser $USER kvm
-newgrp kvm << EOF || exit 1
-  export RUST_BACKTRACE=1
-  cargo test --target $BUILD_TARGET --workspace ${cargo_args[@]} || exit 1;
-EOF
+export RUST_BACKTRACE=1
+cargo test --target $BUILD_TARGET --workspace ${cargo_args[@]} || exit 1;
--- a/src/bin/ch-remote.rs
+++ b/src/bin/ch-remote.rs
@@ -5,25 +5,25 @@

 #[macro_use(crate_authors)]
 extern crate clap;
+extern crate api_client;
 extern crate serde_json;
 extern crate vmm;

+use api_client::simple_api_command;
+use api_client::Error as ApiClientError;
 use clap::{App, AppSettings, Arg, ArgMatches, SubCommand};
+use option_parser::{ByteSized, ByteSizedParseError};
 use std::fmt;
-use std::io::{Read, Write};
 use std::os::unix::net::UnixStream;
 use std::process;

 #[derive(Debug)]
 enum Error {
-    Socket(std::io::Error),
-    StatusCodeParsing(std::num::ParseIntError),
-    MissingProtocol,
-    ContentLengthParsing(std::num::ParseIntError),
-    ServerResponse(StatusCode, Option<String>),
+    Connect(std::io::Error),
+    ApiClient(ApiClientError),
    InvalidCPUCount(std::num::ParseIntError),
-    InvalidMemorySize(std::num::ParseIntError),
-    InvalidBalloonSize(std::num::ParseIntError),
+    InvalidMemorySize(ByteSizedParseError),
+    InvalidBalloonSize(ByteSizedParseError),
    AddDeviceConfig(vmm::config::Error),
    AddDiskConfig(vmm::config::Error),
    AddFsConfig(vmm::config::Error),
@@ -37,20 +37,11 @@ impl fmt::Display for Error {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        use Error::*;
        match self {
-            Socket(e) => write!(f, "Error writing to HTTP socket: {}", e),
-            StatusCodeParsing(e) => write!(f, "Error parsing HTTP status code: {}", e),
-            MissingProtocol => write!(f, "HTTP output is missing protocol statement"),
-            ContentLengthParsing(e) => write!(f, "Error parsing HTTP Content-Length field: {}", e),
-            ServerResponse(s, o) => {
-                if let Some(o) = o {
-                    write!(f, "Server responded with an error: {:?}: {}", s, o)
-                } else {
-                    write!(f, "Server responded with an error: {:?}", s)
-                }
-            }
+            ApiClient(e) => e.fmt(f),
+            Connect(e) => write!(f, "Error openning HTTP socket: {}", e),
            InvalidCPUCount(e) => write!(f, "Error parsing CPU count: {}", e),
-            InvalidMemorySize(e) => write!(f, "Error parsing memory size: {}", e),
-            InvalidBalloonSize(e) => write!(f, "Error parsing balloon size: {}", e),
+            InvalidMemorySize(e) => write!(f, "Error parsing memory size: {:?}", e),
+            InvalidBalloonSize(e) => write!(f, "Error parsing balloon size: {:?}", e),
            AddDeviceConfig(e) => write!(f, "Error parsing device syntax: {}", e),
            AddDiskConfig(e) => write!(f, "Error parsing disk syntax: {}", e),
            AddFsConfig(e) => write!(f, "Error parsing filesystem syntax: {}", e),
@@ -62,146 +53,6 @@ impl fmt::Display for Error {
    }
 }

-#[derive(Clone, Copy, Debug)]
-pub enum StatusCode {
-    Continue,
-    OK,
-    NoContent,
-    BadRequest,
-    NotFound,
-    InternalServerError,
-    NotImplemented,
-    Unknown,
-}
-
-impl StatusCode {
-    fn from_raw(code: usize) -> StatusCode {
-        match code {
-            100 => StatusCode::Continue,
-            200 => StatusCode::OK,
-            204 => StatusCode::NoContent,
-            400 => StatusCode::BadRequest,
-            404 => StatusCode::NotFound,
-            500 => StatusCode::InternalServerError,
-            501 => StatusCode::NotImplemented,
-            _ => StatusCode::Unknown,
-        }
-    }
-
-    fn parse(code: &str) -> Result<StatusCode, Error> {
-        Ok(StatusCode::from_raw(
-            code.trim().parse().map_err(Error::StatusCodeParsing)?,
-        ))
-    }
-
-    fn is_server_error(self) -> bool {
-        match self {
-            StatusCode::OK | StatusCode::Continue | StatusCode::NoContent => false,
-            _ => true,
-        }
-    }
-}
-
-fn get_header<'a>(res: &'a str, header: &'a str) -> Option<&'a str> {
-    let header_str = format!("{}: ", header);
-    if let Some(o) = res.find(&header_str) {
-        Some(&res[o + header_str.len()..o + res[o..].find('\r').unwrap()])
-    } else {
-        None
-    }
-}
-
-fn get_status_code(res: &str) -> Result<StatusCode, Error> {
-    if let Some(o) = res.find("HTTP/1.1") {
-        Ok(StatusCode::parse(
-            &res[o + "HTTP/1.1 ".len()..res[o..].find('\r').unwrap()],
-        )?)
-    } else {
-        Err(Error::MissingProtocol)
-    }
-}
-
-fn parse_http_response(socket: &mut UnixStream) -> Result<Option<String>, Error> {
-    let mut res = String::new();
-    let mut body_offset = None;
-    let mut content_length: Option<usize> = None;
-    loop {
-        let mut bytes = vec![0; 256];
-        let count = socket.read(&mut bytes).map_err(Error::Socket)?;
-        res.push_str(std::str::from_utf8(&bytes[0..count]).unwrap());
-
-        // End of headers
-        if let Some(o) = res.find("\r\n\r\n") {
-            body_offset = Some(o + "\r\n\r\n".len());
-
-            // With all headers available we can see if there is any body
-            content_length = if let Some(length) = get_header(&res, "Content-Length") {
-                Some(length.trim().parse().map_err(Error::ContentLengthParsing)?)
-            } else {
-                None
-            };
-
-            if content_length.is_none() {
-                break;
-            }
-        }
-
-        if let Some(body_offset) = body_offset {
-            if let Some(content_length) = content_length {
-                if res.len() >= content_length + body_offset {
-                    break;
-                }
-            }
-        }
-    }
-    let body_string = content_length.and(Some(String::from(&res[body_offset.unwrap()..])));
-    let status_code = get_status_code(&res)?;
-
-    if status_code.is_server_error() {
-        Err(Error::ServerResponse(status_code, body_string))
-    } else {
-        Ok(body_string)
-    }
-}
-
-fn simple_api_command(
-    socket: &mut UnixStream,
-    method: &str,
-    c: &str,
-    request_body: Option<&str>,
-) -> Result<(), Error> {
-    socket
-        .write_all(
-            format!(
-                "{} /api/v1/vm.{} HTTP/1.1\r\nHost: localhost\r\nAccept: */*\r\n",
-                method, c
-            )
-            .as_bytes(),
-        )
-        .map_err(Error::Socket)?;
-
-    if let Some(request_body) = request_body {
-        socket
-            .write_all(format!("Content-Length: {}\r\n", request_body.len()).as_bytes())
-            .map_err(Error::Socket)?;
-    }
-
-    socket.write_all(b"\r\n").map_err(Error::Socket)?;
-
-    if let Some(request_body) = request_body {
-        socket
-            .write_all(request_body.as_bytes())
-            .map_err(Error::Socket)?;
-    }
-
-    socket.flush().map_err(Error::Socket)?;
-
-    if let Some(body) = parse_http_response(socket)? {
-        println!("{}", body);
-    }
-    Ok(())
-}
-
 fn resize_api_command(
    socket: &mut UnixStream,
    cpus: Option<&str>,
@@ -215,13 +66,23 @@ fn resize_api_command(
    };

    let desired_ram: Option<u64> = if let Some(memory) = memory {
-        Some(memory.parse().map_err(Error::InvalidMemorySize)?)
+        Some(
+            memory
+                .parse::<ByteSized>()
+                .map_err(Error::InvalidMemorySize)?
+                .0,
+        )
    } else {
        None
    };

-    let desired_ram_w_balloon: Option<u64> = if let Some(balloon) = balloon {
-        Some(balloon.parse().map_err(Error::InvalidBalloonSize)?)
+    let desired_balloon: Option<u64> = if let Some(balloon) = balloon {
+        Some(
+            balloon
+                .parse::<ByteSized>()
+                .map_err(Error::InvalidBalloonSize)?
+                .0,
+        )
    } else {
        None
    };
@@ -229,7 +90,7 @@ fn resize_api_command(
    let resize = vmm::api::VmResizeData {
        desired_vcpus,
        desired_ram,
-        desired_ram_w_balloon,
+        desired_balloon,
    };

    simple_api_command(
@@ -238,6 +99,25 @@ fn resize_api_command(
        "resize",
        Some(&serde_json::to_string(&resize).unwrap()),
    )
+    .map_err(Error::ApiClient)
+}
+
+fn resize_zone_api_command(socket: &mut UnixStream, id: &str, size: &str) -> Result<(), Error> {
+    let resize_zone = vmm::api::VmResizeZoneData {
+        id: id.to_owned(),
+        desired_ram: size
+            .parse::<ByteSized>()
+            .map_err(Error::InvalidMemorySize)?
+            .0,
+    };
+
+    simple_api_command(
+        socket,
+        "PUT",
+        "resize-zone",
+        Some(&serde_json::to_string(&resize_zone).unwrap()),
+    )
+    .map_err(Error::ApiClient)
 }

 fn add_device_api_command(socket: &mut UnixStream, config: &str) -> Result<(), Error> {
@@ -249,6 +129,7 @@ fn add_device_api_command(socket: &mut UnixStream, config: &str) -> Result<(), E
        "add-device",
        Some(&serde_json::to_string(&device_config).unwrap()),
    )
+    .map_err(Error::ApiClient)
 }

 fn remove_device_api_command(socket: &mut UnixStream, id: &str) -> Result<(), Error> {
@@ -260,6 +141,7 @@ fn remove_device_api_command(socket: &mut UnixStream, id: &str) -> Result<(), Er
        "remove-device",
        Some(&serde_json::to_string(&remove_device_data).unwrap()),
    )
+    .map_err(Error::ApiClient)
 }

 fn add_disk_api_command(socket: &mut UnixStream, config: &str) -> Result<(), Error> {
@@ -271,6 +153,7 @@ fn add_disk_api_command(socket: &mut UnixStream, config: &str) -> Result<(), Err
        "add-disk",
        Some(&serde_json::to_string(&disk_config).unwrap()),
    )
+    .map_err(Error::ApiClient)
 }

 fn add_fs_api_command(socket: &mut UnixStream, config: &str) -> Result<(), Error> {
@@ -282,6 +165,7 @@ fn add_fs_api_command(socket: &mut UnixStream, config: &str) -> Result<(), Error
        "add-fs",
        Some(&serde_json::to_string(&fs_config).unwrap()),
    )
+    .map_err(Error::ApiClient)
 }

 fn add_pmem_api_command(socket: &mut UnixStream, config: &str) -> Result<(), Error> {
@@ -293,6 +177,7 @@ fn add_pmem_api_command(socket: &mut UnixStream, config: &str) -> Result<(), Err
        "add-pmem",
        Some(&serde_json::to_string(&pmem_config).unwrap()),
    )
+    .map_err(Error::ApiClient)
 }

 fn add_net_api_command(socket: &mut UnixStream, config: &str) -> Result<(), Error> {
@@ -304,6 +189,7 @@ fn add_net_api_command(socket: &mut UnixStream, config: &str) -> Result<(), Erro
        "add-net",
        Some(&serde_json::to_string(&net_config).unwrap()),
    )
+    .map_err(Error::ApiClient)
 }

 fn add_vsock_api_command(socket: &mut UnixStream, config: &str) -> Result<(), Error> {
@@ -315,6 +201,7 @@ fn add_vsock_api_command(socket: &mut UnixStream, config: &str) -> Result<(), Er
        "add-vsock",
        Some(&serde_json::to_string(&vsock_config).unwrap()),
    )
+    .map_err(Error::ApiClient)
 }

 fn snapshot_api_command(socket: &mut UnixStream, url: &str) -> Result<(), Error> {
@@ -328,6 +215,7 @@ fn snapshot_api_command(socket: &mut UnixStream, url: &str) -> Result<(), Error>
        "snapshot",
        Some(&serde_json::to_string(&snapshot_config).unwrap()),
    )
+    .map_err(Error::ApiClient)
 }

 fn restore_api_command(socket: &mut UnixStream, config: &str) -> Result<(), Error> {
@@ -339,15 +227,20 @@ fn restore_api_command(socket: &mut UnixStream, config: &str) -> Result<(), Erro
        "restore",
        Some(&serde_json::to_string(&restore_config).unwrap()),
    )
+    .map_err(Error::ApiClient)
 }

 fn do_command(matches: &ArgMatches) -> Result<(), Error> {
    let mut socket =
-        UnixStream::connect(matches.value_of("api-socket").unwrap()).map_err(Error::Socket)?;
+        UnixStream::connect(matches.value_of("api-socket").unwrap()).map_err(Error::Connect)?;

    match matches.subcommand_name() {
-        Some("info") => simple_api_command(&mut socket, "GET", "info", None),
-        Some("counters") => simple_api_command(&mut socket, "GET", "counters", None),
+        Some("info") => {
+            simple_api_command(&mut socket, "GET", "info", None).map_err(Error::ApiClient)
+        }
+        Some("counters") => {
+            simple_api_command(&mut socket, "GET", "counters", None).map_err(Error::ApiClient)
+        }
        Some("resize") => resize_api_command(
            &mut socket,
            matches
@@ -363,6 +256,19 @@ fn do_command(matches: &ArgMatches) -> Result<(), Error> {
                .unwrap()
                .value_of("balloon"),
        ),
+        Some("resize-zone") => resize_zone_api_command(
+            &mut socket,
+            matches
+                .subcommand_matches("resize-zone")
+                .unwrap()
+                .value_of("id")
+                .unwrap(),
+            matches
+                .subcommand_matches("resize-zone")
+                .unwrap()
+                .value_of("size")
+                .unwrap(),
+        ),
        Some("add-device") => add_device_api_command(
            &mut socket,
            matches
@@ -435,7 +341,7 @@ fn do_command(matches: &ArgMatches) -> Result<(), Error> {
                .value_of("restore_config")
                .unwrap(),
        ),
-        Some(c) => simple_api_command(&mut socket, "PUT", c, None),
+        Some(c) => simple_api_command(&mut socket, "PUT", c, None).map_err(Error::ApiClient),
        None => unreachable!(),
    }
 }
@@ -529,14 +435,32 @@ fn main() {
                .arg(
                    Arg::with_name("memory")
                        .long("memory")
-                        .help("New memory size (in MiB)")
+                        .help("New memory size in bytes (supports K/M/G suffix)")
                        .takes_value(true)
                        .number_of_values(1),
                )
                .arg(
                    Arg::with_name("balloon")
                        .long("balloon")
-                        .help("New memory with balloon size")
+                        .help("New balloon size in bytes (supports K/M/G suffix)")
+                        .takes_value(true)
+                        .number_of_values(1),
+                ),
+        )
+        .subcommand(
+            SubCommand::with_name("resize-zone")
+                .about("Resize a memory zone")
+                .arg(
+                    Arg::with_name("id")
+                        .long("id")
+                        .help("Memory zone identifier")
+                        .takes_value(true)
+                        .number_of_values(1),
+                )
+                .arg(
+                    Arg::with_name("size")
+                        .long("size")
+                        .help("New memory zone size in bytes (supports K/M/G suffix)")
                        .takes_value(true)
                        .number_of_values(1),
                ),
--- a/src/main.rs
+++ b/src/main.rs
@@ -3,6 +3,7 @@
 // SPDX-License-Identifier: Apache-2.0
 //

+extern crate anyhow;
 extern crate vmm;
 extern crate vmm_sys_util;

@@ -13,14 +14,39 @@ use clap::{App, Arg, ArgGroup, ArgMatches};
 use libc::EFD_NONBLOCK;
 use log::LevelFilter;
 use seccomp::SeccompAction;
+use std::env;
 use std::sync::mpsc::channel;
 use std::sync::{Arc, Mutex};
-use std::{env, process};
+use thiserror::Error;
 use vhost_user_block::start_block_backend;
 use vhost_user_net::start_net_backend;
 use vmm::config;
 use vmm_sys_util::eventfd::EventFd;

+#[derive(Error, Debug)]
+enum Error {
+    #[error("Failed to create API EventFd: {0}")]
+    CreateAPIEventFd(#[source] std::io::Error),
+    #[error("Failed to open hypervisor interface (is /dev/kvm available?): {0}")]
+    CreateHypervisor(#[source] hypervisor::HypervisorError),
+    #[error("Failed to start the VMM thread: {0}")]
+    StartVMMThread(#[source] vmm::Error),
+    #[error("Error parsing config: {0}")]
+    ParsingConfig(vmm::config::Error),
+    #[error("Error creating VM: {0:?}")]
+    VmCreate(vmm::api::ApiError),
+    #[error("Error booting VM: {0:?}")]
+    VmBoot(vmm::api::ApiError),
+    #[error("Error restoring VM: {0:?}")]
+    VmRestore(vmm::api::ApiError),
+    #[error("Error parsing restore: {0}")]
+    ParsingRestore(vmm::config::Error),
+    #[error("Failed to join on VMM thread: {0:?}")]
+    ThreadJoin(std::boxed::Box<dyn std::any::Any + std::marker::Send>),
+    #[error("VMM thread exited with error: {0}")]
+    VmmThread(#[source] vmm::Error),
+}
+
 struct Logger {
    output: Mutex<Box<dyn std::io::Write + Send>>,
    start: std::time::Instant,
@@ -97,7 +123,8 @@ fn create_app<'a, 'b>(
                .long("cpus")
                .help(
                    "boot=<boot_vcpus>,max=<max_vcpus>,\
-                    topology=<threads_per_core>:<cores_per_die>:<dies_per_package>:<packages>",
+                    topology=<threads_per_core>:<cores_per_die>:<dies_per_package>:<packages>,\
+                    kvm_hyperv=on|off,max_phys_bits=<maximum_number_of_physical_bits>",
                )
                .default_value(&default_vcpus)
                .group("vm-config"),
@@ -109,15 +136,33 @@ fn create_app<'a, 'b>(
                    "Memory parameters \
                     \"size=<guest_memory_size>,mergeable=on|off,shared=on|off,hugepages=on|off,\
                     hotplug_method=acpi|virtio-mem,\
-                     hotplug_size=<hotpluggable_memory_size>\"",
+                     hotplug_size=<hotpluggable_memory_size>,\
+                     hotplugged_size=<hotplugged_memory_size>\"",
                )
                .default_value(&default_memory)
                .group("vm-config"),
        )
+        .arg(
+            Arg::with_name("memory-zone")
+                .long("memory-zone")
+                .help(
+                    "User defined memory zone parameters \
+                     \"size=<guest_memory_region_size>,file=<backing_file>,\
+                     shared=on|off,hugepages=on|off,host_numa_node=<node_id>,\
+                     id=<zone_identifier>,hotplug_size=<hotpluggable_memory_size>,\
+                     hotplugged_size=<hotplugged_memory_size>\"",
+                )
+                .takes_value(true)
+                .min_values(1)
+                .group("vm-config"),
+        )
        .arg(
            Arg::with_name("kernel")
                .long("kernel")
-                .help("Path to kernel image (vmlinux)")
+                .help(
+                    "Path to loaded kernel. This may be a kernel or firmware that supports a PVH \
+                entry point, a vmlinux ELF file or a Linux bzImage or achitecture equivalent",
+                )
                .takes_value(true)
                .group("vm-config"),
        )
@@ -160,6 +205,13 @@ fn create_app<'a, 'b>(
                .default_value(&default_rng)
                .group("vm-config"),
        )
+        .arg(
+            Arg::with_name("balloon")
+                .long("balloon")
+                .help(config::BalloonConfig::SYNTAX)
+                .takes_value(true)
+                .group("vm-config"),
+        )
        .arg(
            Arg::with_name("fs")
                .long("fs")
@@ -208,6 +260,21 @@ fn create_app<'a, 'b>(
                .number_of_values(1)
                .group("vm-config"),
        )
+        .arg(
+            Arg::with_name("numa")
+                .long("numa")
+                .help(config::NumaConfig::SYNTAX)
+                .takes_value(true)
+                .min_values(1)
+                .group("vm-config"),
+        )
+        .arg(
+            Arg::with_name("watchdog")
+                .long("watchdog")
+                .help("Enable virtio-watchdog")
+                .takes_value(false)
+                .group("vm-config"),
+        )
        .arg(
            Arg::with_name("v")
                .short("v")
@@ -279,13 +346,9 @@ fn create_app<'a, 'b>(
    app
 }

-fn start_vmm(cmd_arguments: ArgMatches) {
-    let api_socket_path = cmd_arguments
-        .value_of("api-socket")
-        .expect("Missing argument: api-socket");
-
+fn start_vmm(cmd_arguments: ArgMatches, api_socket_path: &str) -> Result<(), Error> {
    let (api_request_sender, api_request_receiver) = channel();
-    let api_evt = EventFd::new(EFD_NONBLOCK).expect("Cannot create API EventFd");
+    let api_evt = EventFd::new(EFD_NONBLOCK).map_err(Error::CreateAPIEventFd)?;

    let http_sender = api_request_sender.clone();
    let seccomp_action = if let Some(seccomp_value) = cmd_arguments.value_of("seccomp") {
@@ -294,15 +357,15 @@ fn start_vmm(cmd_arguments: ArgMatches) {
            "false" => SeccompAction::Allow,
            "log" => SeccompAction::Log,
            _ => {
-                eprintln!("Invalid parameter {} for \"--seccomp\" flag", seccomp_value);
-                process::exit(1);
+                // The user providing an invalid value will be rejected by clap
+                panic!("Invalid parameter {} for \"--seccomp\" flag", seccomp_value);
            }
        }
    } else {
        SeccompAction::Trap
    };
-    let hypervisor = hypervisor::new().unwrap();
-    let vmm_thread = match vmm::start_vmm_thread(
+    let hypervisor = hypervisor::new().map_err(Error::CreateHypervisor)?;
+    let vmm_thread = vmm::start_vmm_thread(
        env!("CARGO_PKG_VERSION").to_string(),
        api_socket_path,
        api_evt.try_clone().unwrap(),
@@ -310,25 +373,14 @@ fn start_vmm(cmd_arguments: ArgMatches) {
        api_request_receiver,
        &seccomp_action,
        hypervisor,
-    ) {
-        Ok(t) => t,
-        Err(e) => {
-            eprintln!("Failed spawning the VMM thread {:?}", e);
-            process::exit(1);
-        }
-    };
+    )
+    .map_err(Error::StartVMMThread)?;

    // Can't test for "vm-config" group as some have default values. The kernel
    // is the only required option for booting the VM.
    if cmd_arguments.is_present("kernel") {
        let vm_params = config::VmParams::from_arg_matches(&cmd_arguments);
-        let vm_config = match config::VmConfig::parse(vm_params) {
-            Ok(config) => config,
-            Err(e) => {
-                eprintln!("{}", e);
-                process::exit(1);
-            }
-        };
+        let vm_config = config::VmConfig::parse(vm_params).map_err(Error::ParsingConfig)?;

        println!(
            "Cloud Hypervisor Guest\n\tAPI server: {}\n\tvCPUs: {}\n\tMemory: {} MB\n\tKernel: \
@@ -349,36 +401,21 @@ fn start_vmm(cmd_arguments: ArgMatches) {
            api_request_sender,
            Arc::new(Mutex::new(vm_config)),
        )
-        .expect("Could not create the VM");
-        vmm::api::vm_boot(api_evt.try_clone().unwrap(), sender).expect("Could not boot the VM");
+        .map_err(Error::VmCreate)?;
+        vmm::api::vm_boot(api_evt.try_clone().unwrap(), sender).map_err(Error::VmBoot)?;
    } else if let Some(restore_params) = cmd_arguments.value_of("restore") {
        vmm::api::vm_restore(
            api_evt.try_clone().unwrap(),
            api_request_sender,
-            Arc::new(match config::RestoreConfig::parse(restore_params) {
-                Ok(config) => config,
-                Err(e) => {
-                    println!("{}", e);
-                    process::exit(1);
-                }
-            }),
+            Arc::new(config::RestoreConfig::parse(restore_params).map_err(Error::ParsingRestore)?),
        )
-        .expect("Could not restore the VM");
+        .map_err(Error::VmRestore)?;
    }

-    match vmm_thread.join() {
-        Ok(res) => match res {
-            Ok(_) => (),
-            Err(e) => {
-                eprintln!("VMM thread failed {:?}", e);
-                process::exit(1);
-            }
-        },
-        Err(e) => {
-            eprintln!("Could not join VMM thread {:?}", e);
-            process::exit(1);
-        }
-    }
+    vmm_thread
+        .join()
+        .map_err(Error::ThreadJoin)?
+        .map_err(Error::VmmThread)
 }

 fn main() {
@@ -412,10 +449,9 @@ fn main() {
    .get_matches();

    let log_level = match cmd_arguments.occurrences_of("v") {
-        0 => LevelFilter::Error,
-        1 => LevelFilter::Warn,
-        2 => LevelFilter::Info,
-        3 => LevelFilter::Debug,
+        0 => LevelFilter::Warn,
+        1 => LevelFilter::Info,
+        2 => LevelFilter::Debug,
        _ => LevelFilter::Trace,
    };

@@ -440,7 +476,17 @@ fn main() {
    } else if let Some(backend_command) = cmd_arguments.value_of("block-backend") {
        start_block_backend(backend_command);
    } else {
-        start_vmm(cmd_arguments);
+        let api_socket_path = cmd_arguments
+            .value_of("api-socket")
+            .expect("Missing argument: api-socket")
+            .to_string();
+
+        if let Err(e) = start_vmm(cmd_arguments, &api_socket_path) {
+            eprintln!("{}", e);
+            std::fs::remove_file(api_socket_path).ok();
+            std::process::exit(1);
+        }
+        std::fs::remove_file(api_socket_path).ok();
    }
 }

@@ -511,17 +557,18 @@ mod unit_tests {
                    boot_vcpus: 1,
                    max_vcpus: 1,
                    topology: None,
+                    kvm_hyperv: false,
+                    max_phys_bits: None,
                },
                memory: MemoryConfig {
                    size: 536_870_912,
-                    file: None,
                    mergeable: false,
                    hotplug_method: HotplugMethod::Acpi,
                    hotplug_size: None,
+                    hotplugged_size: None,
                    shared: false,
                    hugepages: false,
-                    balloon: false,
-                    balloon_size: 0,
+                    zones: None,
                },
                kernel: Some(KernelConfig {
                    path: PathBuf::from("/path/to/kernel"),
@@ -536,6 +583,7 @@ mod unit_tests {
                    src: PathBuf::from("/dev/urandom"),
                    iommu: false,
                },
+                balloon: None,
                fs: None,
                pmem: None,
                serial: ConsoleConfig {
@@ -553,6 +601,8 @@ mod unit_tests {
                iommu: false,
                #[cfg(target_arch = "x86_64")]
                sgx_epc: None,
+                numa: None,
+                watchdog: false,
            };

            aver_eq!(tb, expected_vm_config, result_vm_config);
@@ -631,18 +681,6 @@ mod unit_tests {
                }"#,
                true,
            ),
-            (
-                vec![
-                    "cloud-hypervisor", "--kernel", "/path/to/kernel",
-                    "--memory",
-                    "size=1G,file=/path/to/shared/file",
-                ],
-                r#"{
-                    "kernel": {"path": "/path/to/kernel"},
-                    "memory": {"size": 1073741824, "file": "/path/to/shared/file"}
-                }"#,
-                true,
-            ),
            (
                vec!["cloud-hypervisor", "--kernel", "/path/to/kernel", "--memory", "size=1G,mergeable=on"],
                r#"{
--- a/test_data/cloud-init/ubuntu/user-data
+++ b/test_data/cloud-init/ubuntu/user-data
@@ -9,6 +9,11 @@ users:

 ssh_pwauth: True

+runcmd:
+  - [ systemctl, daemon-reload]
+  - [ systemctl, enable, notify-booted.service]
+  - [ systemctl, start, --no-block, notify-booted.service ]
+
 write_files:
  -
    path: /etc/systemd/system/vfio.service
@@ -30,12 +35,35 @@ write_files:
    content: |
        #!/bin/bash

-        mount -t virtiofs -o dax myfs /mnt
-        bash -c "echo 0000:00:05.0 > /sys/bus/pci/devices/0000\:00\:05.0/driver/unbind"
-        bash -c "echo 1af4 1041 > /sys/bus/pci/drivers/vfio-pci/new_id"
+        mount /dev/vdc /mnt
        bash -c "echo 0000:00:06.0 > /sys/bus/pci/devices/0000\:00\:06.0/driver/unbind"
        bash -c "echo 1af4 1041 > /sys/bus/pci/drivers/vfio-pci/new_id"
+        bash -c "echo 0000:00:07.0 > /sys/bus/pci/devices/0000\:00\:07.0/driver/unbind"
+        bash -c "echo 1af4 1041 > /sys/bus/pci/drivers/vfio-pci/new_id"
        # 1G ram requires 512 pages
        echo 512 | sudo tee /proc/sys/vm/nr_hugepages
        sudo chmod a+rwX /dev/hugepages
-        /mnt/cloud-hypervisor --kernel /mnt/vmlinux --cmdline "console=hvc0 reboot=k panic=1 nomodules i8042.noaux i8042.nomux i8042.nopnp i8042.dumbkbd root=/dev/vda1 VFIOTAG" --disk path=/mnt/focal-server-cloudimg-amd64-custom.qcow2 path=/mnt/cloudinit.img --cpus boot=1 --memory size=512M,hotplug_size=1G,file=/dev/hugepages --device path=/sys/bus/pci/devices/0000:00:05.0/ path=/sys/bus/pci/devices/0000:00:06.0/ --api-socket /tmp/ch_api.sock
+        /mnt/cloud-hypervisor --kernel /mnt/vmlinux --cmdline "console=hvc0 reboot=k panic=1 nomodules i8042.noaux i8042.nomux i8042.nopnp i8042.dumbkbd root=/dev/vda1 VFIOTAG" --disk path=/mnt/focal-server-cloudimg-amd64-custom.qcow2 path=/mnt/cloudinit.img --cpus boot=1 --memory size=512M,hotplug_size=1G,hugepages=on --device path=/sys/bus/pci/devices/0000:00:06.0/ path=/sys/bus/pci/devices/0000:00:07.0/ --api-socket /tmp/ch_api.sock
+
+  -
+    path: /etc/systemd/system/notify-booted.service
+    permissions: 0644
+    content: |
+        [Unit]
+        Description=Notify the tcp listener on the host that the guest is booted
+        After=sshd.service
+
+        [Service]
+        Type=simple
+        ExecStart=/usr/bin/cloud-hypervisor-notify-booted.sh
+
+        [Install]
+        WantedBy=multi-user.target
+
+  -
+    path: /usr/bin/cloud-hypervisor-notify-booted.sh
+    permissions: 0755
+    content: |
+        #!/bin/bash
+
+        echo -n "@DEFAULT_TCP_LISTENER_MESSAGE" | nc -w0 @HOST_IP @TCP_LISTENER_PORT
--- a/tests/integration.rs
+++ b/tests/integration.rs
--- a/vhost_user_backend/Cargo.toml
+++ b/vhost_user_backend/Cargo.toml
@@ -9,10 +9,10 @@ default = []

 [dependencies]
 epoll = ">=4.0.1"
-libc = "0.2.74"
+libc = "0.2.80"
 log = "0.4.11"
 virtio-bindings = "0.1.0"
-vm-memory = "0.2.1"
+vm-memory = "0.3.0"
 vm-virtio = { path = "../vm-virtio" }
 vmm-sys-util = ">=0.3.1"
-vhost_rs = { git = "https://github.com/cloud-hypervisor/vhost", branch = "dragonball", package = "vhost", features = ["vhost-user-slave"] }
+vhost_rs = { git = "https://github.com/rust-vmm/vhost", branch = "master", package = "vhost", features = ["vhost-user-slave"] }
--- a/vhost_user_block/Cargo.toml
+++ b/vhost_user_block/Cargo.toml
@@ -6,14 +6,14 @@ edition = "2018"

 [dependencies]
 block_util = { path = "../block_util" }
-clap = { version = "2.33.2", features=["wrap_help"] }
+clap = { version = "2.33.3", features=["wrap_help"] }
 epoll = ">=4.0.1"
-libc = "0.2.74"
+libc = "0.2.80"
 log = "0.4.11"
 option_parser = { path = "../option_parser" }
 qcow = { path = "../qcow" }
 vhost_user_backend = { path = "../vhost_user_backend" }
-vhost_rs = { git = "https://github.com/cloud-hypervisor/vhost", branch = "dragonball", package = "vhost", features = ["vhost-user-slave"] }
+vhost_rs = { git = "https://github.com/rust-vmm/vhost", branch = "master", package = "vhost", features = ["vhost-user-slave"] }
 virtio-bindings = "0.1.0"
-vm-memory = "0.2.1"
+vm-memory = "0.3.0"
 vmm-sys-util = ">=0.3.1"
--- a/Show More
+++ b/Show More