build: Release v15.0

The Rust manifiest requires a three part version number however we continue with our plan to use two digits so the last part will not be used. Signed-off-by: Rob Bradford <robert.bradford@intel.com>
tests: Fix reboot by allowing specifying the custom timeout
2026-08-05 02:19:16 +00:00 · 2021-04-29 16:07:45 +01:00 · 2021-04-29 11:10:40 +01:00 · 2021-04-29 11:10:32 +01:00 · 2021-04-29 09:44:40 +01:00 · 2021-04-29 09:44:40 +01:00
164 changed files with 10174 additions and 5163 deletions
--- a/.github/ISSUE_TEMPLATE/bug_report.md
+++ b/.github/ISSUE_TEMPLATE/bug_report.md
@@ -0,0 +1,34 @@
+---
+name: Bug report
+about: File a bug report
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+**Describe the bug**
+A clear and concise description of what the bug is.
+
+**To Reproduce**
+Steps to reproduce the behaviour:
+
+**Version**
+
+Output of `cloud-hypervisor --version`:
+
+Did you build from source, if so build command line (e.g. features):
+
+**VM configuration**
+
+What command line did you run (or JSON config data):
+
+Guest OS version details:
+
+Host OS version details:
+
+**Logs**
+
+Output of `cloud-hypervisor -v` from either standard error or via `--log-file`:
+
+Linux kernel output:
--- a/.github/workflows/cross-build.yaml
+++ b/.github/workflows/cross-build.yaml
@@ -1,32 +0,0 @@
-name: Cloud Hypervisor Cross Build
-on: [pull_request, create]
-
-jobs:
-  build:
-    if: github.event_name == 'pull_request'
-    name: Build
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        rust:
-          - 1.46.0
-        target:
-          - aarch64-unknown-linux-gnu
-          - aarch64-unknown-linux-musl
-    steps:
-      - name: Code checkout
-        uses: actions/checkout@v2
-      - name: Install Rust toolchain (${{ matrix.rust }})
-        uses: actions-rs/toolchain@v1
-        with:
-            toolchain: ${{ matrix.rust }}
-            target: ${{ matrix.target }}
-            override: true
-      - name: Install arm64 libfdt
-        run: wget http://ftp.us.debian.org/debian/pool/main/d/device-tree-compiler/libfdt-dev_1.6.0-1_arm64.deb && dpkg-deb -xv libfdt-dev_1.6.0-1_arm64.deb ./tlibfdtdev && mkdir -p target/debug/deps &&  cp ./tlibfdtdev/usr/lib/aarch64-linux-gnu/libfdt.a target/debug/deps/ && echo "libfdt copied into build tree"
-      - name: Build
-        uses: actions-rs/cargo@v1
-        with:
-          use-cross: true
-          command: build
-          args: --target=${{ matrix.target }} --no-default-features --features "kvm"
--- a/.github/workflows/quality-aarch64.yaml
+++ b/.github/workflows/quality-aarch64.yaml
@@ -6,12 +6,19 @@ jobs:
    if: github.event_name == 'pull_request'
    name: Quality (clippy, rustfmt)
    runs-on: ubuntu-latest
+    continue-on-error: ${{ matrix.experimental }}
    strategy:
+      fail-fast: false
      matrix:
        rust:
          - stable
        target:
          - aarch64-unknown-linux-gnu
+        experimental: [false]
+        include:
+          - rust: beta
+            target: aarch64-unknown-linux-gnu
+            experimental: true
    steps:
      - name: Code checkout
        uses: actions/checkout@v2
@@ -23,7 +30,7 @@ jobs:
            override: true
            components: rustfmt, clippy
      - name: Install arm64 libfdt
-        run: wget http://ftp.us.debian.org/debian/pool/main/d/device-tree-compiler/libfdt-dev_1.6.0-1_arm64.deb && dpkg-deb -xv libfdt-dev_1.6.0-1_arm64.deb ./tlibfdtdev && sudo mkdir /tmmmp && mkdir target && mkdir target/debug && mkdir target/debug/deps && sudo cp ./tlibfdtdev/usr/lib/aarch64-linux-gnu/libfdt.a target/debug/deps/libfdt.a && echo "libfdt installed"
+        run: wget http://ftp.us.debian.org/debian/pool/main/d/device-tree-compiler/libfdt-dev_1.6.0-1_arm64.deb && dpkg-deb -xv libfdt-dev_1.6.0-1_arm64.deb ./tlibfdtdev && mkdir -p target/debug/deps && sudo cp ./tlibfdtdev/usr/lib/aarch64-linux-gnu/libfdt.a target/debug/deps/libfdt.a && echo "libfdt installed"
      - name: Formatting (rustfmt)
        run: cargo fmt -- --check
      - name: Clippy (kvm)
--- a/.github/workflows/quality.yaml
+++ b/.github/workflows/quality.yaml
@@ -6,12 +6,19 @@ jobs:
    if: github.event_name == 'pull_request'
    name: Quality (clippy, rustfmt)
    runs-on: ubuntu-latest
+    continue-on-error: ${{ matrix.experimental }}
    strategy:
+      fail-fast: false
      matrix:
        rust:
          - stable
        target:
          - x86_64-unknown-linux-gnu
+        experimental: [false]
+        include:
+          - rust: beta
+            target: x86_64-unknown-linux-gnu
+            experimental: true
    steps:
      - name: Code checkout
        uses: actions/checkout@v2
@@ -35,6 +42,9 @@ jobs:
      - name: Clippy (acpi,kvm)
        run: cargo clippy --all --all-targets --no-default-features --tests --features "acpi,kvm" -- -D warnings

+      - name: Clippy (acpi,kvm,tdx)
+        run: cargo clippy --all --all-targets --no-default-features --tests --features "acpi,kvm,tdx" -- -D warnings
+
      - name: Clippy (kvm)
        run: cargo clippy --all --all-targets --no-default-features --tests --features "kvm" -- -D warnings

--- a/Cargo.lock
+++ b/Cargo.lock
@@ -18,9 +18,9 @@ dependencies = [

 [[package]]
 name = "adler"
-version = "0.2.3"
+version = "1.0.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ee2a4ec343196209d6594e19543ae87a39f96d5534d7174822a3ad825dd6ed7e"
+checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe"

 [[package]]
 name = "aho-corasick"
@@ -42,9 +42,9 @@ dependencies = [

 [[package]]
 name = "anyhow"
-version = "1.0.38"
+version = "1.0.40"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "afddf7f520a80dbf76e6f50a35bca42a2331ef227a28b3b6dc5c2e2338d114b1"
+checksum = "28b2cd92db5cbd74e8e5028f7e27dd7aa3090e89e4f2a197cc7c8dfb69c7063b"

 [[package]]
 name = "api_client"
@@ -68,10 +68,10 @@ dependencies = [
 "libc",
 "linux-loader",
 "log 0.4.14",
- "rand 0.8.3",
 "serde",
 "serde_derive",
 "serde_json",
+ "thiserror",
 "vm-memory",
 "vm-migration",
 ]
@@ -80,18 +80,6 @@ dependencies = [
 name = "arch_gen"
 version = "0.1.0"

-[[package]]
-name = "arrayref"
-version = "0.3.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a4c527152e37cf757a3f78aae5a06fbeefdb07ccc535c980a3208ee3060dd544"
-
-[[package]]
-name = "arrayvec"
-version = "0.5.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "23b62fc65de8e4e7f52534fb52b0f3ed04746ae267519eef2a83941e8085068b"
-
 [[package]]
 name = "atty"
 version = "0.2.14"
@@ -111,11 +99,12 @@ checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a"

 [[package]]
 name = "backtrace"
-version = "0.3.56"
+version = "0.3.58"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9d117600f438b1707d4e4ae15d3595657288f8235a0eb593e80ecc98ab34e1bc"
+checksum = "88fb5a785d6b44fd9d6700935608639af1b8356de1e55d5f7c2740f4faa15d82"
 dependencies = [
 "addr2line",
+ "cc",
 "cfg-if 1.0.0",
 "libc",
 "miniz_oxide",
@@ -123,12 +112,6 @@ dependencies = [
 "rustc-demangle",
 ]

-[[package]]
-name = "base64"
-version = "0.13.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd"
-
 [[package]]
 name = "bitflags"
 version = "0.5.0"
@@ -141,17 +124,6 @@ version = "1.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693"

-[[package]]
-name = "blake2b_simd"
-version = "0.5.11"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "afa748e348ad3be8263be728124b24a24f268266f6f5d58af9d75f6a40b5c587"
-dependencies = [
- "arrayref",
- "arrayvec",
- "constant_time_eq",
-]
-
 [[package]]
 name = "block_util"
 version = "0.1.0"
@@ -163,7 +135,6 @@ dependencies = [
 "serde",
 "serde_derive",
 "serde_json",
- "tempfile",
 "thiserror",
 "virtio-bindings",
 "vm-memory",
@@ -173,15 +144,15 @@ dependencies = [

 [[package]]
 name = "byteorder"
-version = "1.3.4"
+version = "1.4.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "08c48aae112d48ed9f069b33538ea9e3e90aa263cfa3d1c24309612b1f7472de"
+checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610"

 [[package]]
 name = "cc"
-version = "1.0.66"
+version = "1.0.67"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4c0496836a84f8d0495758516b8621a622beb77c0fed418570e50764093ced48"
+checksum = "e3c69b077ad434294d3ce9f1f6143a2a4b89a8a2d54ef813d85003a4fd1137fd"

 [[package]]
 name = "cfg-if"
@@ -213,7 +184,7 @@ dependencies = [

 [[package]]
 name = "cloud-hypervisor"
-version = "0.13.0"
+version = "15.0.0"
 dependencies = [
 "anyhow",
 "api_client",
@@ -221,6 +192,7 @@ dependencies = [
 "credibility",
 "dirs",
 "epoll",
+ "event_monitor",
 "hypervisor",
 "lazy_static",
 "libc",
@@ -230,9 +202,7 @@ dependencies = [
 "seccomp",
 "serde_json",
 "signal-hook",
- "ssh2",
- "tempdir",
- "tempfile",
+ "test_infra",
 "thiserror",
 "vm-memory",
 "vmm",
@@ -249,12 +219,6 @@ dependencies = [
 "bitflags 1.2.1",
 ]

-[[package]]
-name = "constant_time_eq"
-version = "0.1.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc"
-
 [[package]]
 name = "credibility"
 version = "0.1.3"
@@ -265,17 +229,6 @@ dependencies = [
 "failure_derive",
 ]

-[[package]]
-name = "crossbeam-utils"
-version = "0.8.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "02d96d1e189ef58269ebe5b97953da3274d83a93af647c2ddd6f9dab28cedb8d"
-dependencies = [
- "autocfg",
- "cfg-if 1.0.0",
- "lazy_static",
-]
-
 [[package]]
 name = "devices"
 version = "0.1.0"
@@ -290,7 +243,6 @@ dependencies = [
 "serde",
 "serde_derive",
 "serde_json",
- "tempfile",
 "vm-device",
 "vm-memory",
 "vm-migration",
@@ -299,18 +251,18 @@ dependencies = [

 [[package]]
 name = "dirs"
-version = "3.0.1"
+version = "3.0.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "142995ed02755914747cc6ca76fc7e4583cd18578746716d0508ea6ed558b9ff"
+checksum = "30baa043103c9d0c2a57cf537cc2f35623889dc0d405e6c3cccfadbc81c71309"
 dependencies = [
 "dirs-sys",
 ]

 [[package]]
 name = "dirs-sys"
-version = "0.3.5"
+version = "0.3.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8e93d7f5705de3e49895a2b5e0b8855a1c27f080192ae9c32a6432d50741a57a"
+checksum = "03d86534ed367a67548dc68113a0f5db55432fdfbb6e6f9d77704397d95d5780"
 dependencies = [
 "libc",
 "redox_users",
@@ -340,6 +292,16 @@ dependencies = [
 "libc",
 ]

+[[package]]
+name = "event_monitor"
+version = "0.1.0"
+dependencies = [
+ "libc",
+ "serde",
+ "serde_derive",
+ "serde_json",
+]
+
 [[package]]
 name = "failure"
 version = "0.1.8"
@@ -362,44 +324,17 @@ dependencies = [
 "synstructure",
 ]

-[[package]]
-name = "form_urlencoded"
-version = "1.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ece68d15c92e84fa4f19d3780f1294e5ca82a78a6d515f1efaabcc144688be00"
-dependencies = [
- "matches",
- "percent-encoding",
-]
-
-[[package]]
-name = "fuchsia-cprng"
-version = "0.1.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba"
-
 [[package]]
 name = "getrandom"
-version = "0.1.16"
+version = "0.2.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce"
+checksum = "c9495705279e7140bf035dde1f6e750c162df8b625267cd52cc44e0b156732c8"
 dependencies = [
 "cfg-if 1.0.0",
 "libc",
 "wasi",
 ]

-[[package]]
-name = "getrandom"
-version = "0.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ee8025cf36f917e6a52cce185b7c7177689b838b7ec138364e50cc2277a56cf4"
-dependencies = [
- "cfg-if 0.1.10",
- "libc",
- "wasi",
-]
-
 [[package]]
 name = "gimli"
 version = "0.23.0"
@@ -451,31 +386,19 @@ dependencies = [

 [[package]]
 name = "iced-x86"
-version = "1.10.3"
+version = "1.11.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "644fd8d7e49d1ccae568a8fbb873a7ee7fcb3bc9214a474a5e55c03b6d1fa5ac"
+checksum = "97b85a147f5dcb22ece887da5b99187954cc046939f22416e953dd7c336e85a1"
 dependencies = [
 "lazy_static",
- "rustc_version",
 "static_assertions",
 ]

-[[package]]
-name = "idna"
-version = "0.2.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "de910d521f7cc3135c4de8db1cb910e0b5ed1dc6f57c381cd07e8e661ce10094"
-dependencies = [
- "matches",
- "unicode-bidi",
- "unicode-normalization",
-]
-
 [[package]]
 name = "io-uring"
-version = "0.4.0"
+version = "0.5.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2f7589adca0ddd74f56ed83a5098b45e3abf264dc27e150a8bec3397fcc34338"
+checksum = "fb82832e05cc4ca298f198a8db108837b4f7b7b1248e3cba8e48f151aece80cf"
 dependencies = [
 "bitflags 1.2.1",
 "libc",
@@ -508,8 +431,8 @@ dependencies = [

 [[package]]
 name = "kvm-bindings"
-version = "0.2.0"
-source = "git+https://github.com/cloud-hypervisor/kvm-bindings?branch=ch#2164129ff03bb4d7a4243de5e2078845064d922c"
+version = "0.4.0"
+source = "git+https://github.com/cloud-hypervisor/kvm-bindings?branch=ch-v0.4.0#1a68725639283e622f4bb64584885b30bfe8be44"
 dependencies = [
 "serde",
 "serde_derive",
@@ -518,8 +441,9 @@ dependencies = [

 [[package]]
 name = "kvm-ioctls"
-version = "0.6.0"
-source = "git+https://github.com/cloud-hypervisor/kvm-ioctls?branch=ch#b7f21758bf8e46ac55a28a1b2ed008b5732bbb44"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "74058b16912c6723db02d4ca3ec919b73cd41512ccd3b6202cf91ae8d6c9dce5"
 dependencies = [
 "kvm-bindings",
 "libc",
@@ -534,9 +458,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"

 [[package]]
 name = "libc"
-version = "0.2.86"
+version = "0.2.94"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b7282d924be3275cec7f6756ff4121987bc6481325397dde6ba3e7802b1a8b1c"
+checksum = "18794a8ad5b29321f790b55d93dfba91e125cb1a9edbd4f8e3150acc771c1a5e"

 [[package]]
 name = "libssh2-sys"
@@ -566,8 +490,9 @@ dependencies = [

 [[package]]
 name = "linux-loader"
-version = "0.2.0"
-source = "git+https://github.com/rust-vmm/linux-loader#2855be15a725e32df0b5198487d7b52f4a90ac0f"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c819cc8275b0f2c1ed9feec455ca288b45d82932384a6a5f7a86812ee3427459"
 dependencies = [
 "vm-memory",
 ]
@@ -599,12 +524,6 @@ dependencies = [
 "cfg-if 1.0.0",
 ]

-[[package]]
-name = "matches"
-version = "0.1.8"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08"
-
 [[package]]
 name = "memchr"
 version = "2.3.4"
@@ -614,16 +533,17 @@ checksum = "0ee1c47aaa256ecabcaea351eae4a9b01ef39ed810004e298d2511ed284b1525"
 [[package]]
 name = "micro_http"
 version = "0.1.0"
-source = "git+https://github.com/firecracker-microvm/micro-http?branch=master#59ab64440a95f7b16253fef67b5201c2ec4adaf7"
+source = "git+https://github.com/firecracker-microvm/micro-http?branch=main#9b605a8b61df602cda62076d30d9f70307ea336f"
 dependencies = [
- "epoll",
+ "libc",
+ "vmm-sys-util",
 ]

 [[package]]
 name = "miniz_oxide"
-version = "0.4.3"
+version = "0.4.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0f2d26ec3309788e423cfbf68ad1800f061638098d76a83681af979dc4eda19d"
+checksum = "a92518e98c078586bc6c934028adcca4c92a53d6a958196de835170a01d84e4b"
 dependencies = [
 "adler",
 "autocfg",
@@ -632,7 +552,7 @@ dependencies = [
 [[package]]
 name = "mshv-bindings"
 version = "0.1.0"
-source = "git+https://github.com/cloud-hypervisor/mshv?branch=master#3a7d7ba07e12409a3b23794c72c30b5052d221d9"
+source = "git+https://github.com/cloud-hypervisor/mshv?branch=master#936e2e34af0af50f4383ab8a208e03dfba7ed206"
 dependencies = [
 "libc",
 "serde",
@@ -644,7 +564,7 @@ dependencies = [
 [[package]]
 name = "mshv-ioctls"
 version = "0.1.0"
-source = "git+https://github.com/cloud-hypervisor/mshv?branch=master#3a7d7ba07e12409a3b23794c72c30b5052d221d9"
+source = "git+https://github.com/cloud-hypervisor/mshv?branch=master#936e2e34af0af50f4383ab8a208e03dfba7ed206"
 dependencies = [
 "libc",
 "mshv-bindings",
@@ -668,7 +588,7 @@ dependencies = [
 "log 0.4.14",
 "net_gen",
 "pnet",
- "rand 0.8.3",
+ "rate_limiter",
 "serde",
 "serde_json",
 "virtio-bindings",
@@ -683,17 +603,11 @@ version = "0.23.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a9a7ab5d64814df0fe4a4b5ead45ed6c5f181ee3ff04ba344313a6c80446c5d4"

-[[package]]
-name = "once_cell"
-version = "1.5.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "13bd41f508810a131401606d54ac32a467c97172d74ba7662562ebba5ad07fa0"
-
 [[package]]
 name = "openssl-sys"
-version = "0.9.60"
+version = "0.9.62"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "921fc71883267538946025deffb622905ecad223c28efbfdef9bb59a0175f3e6"
+checksum = "fa52160d45fa2e7608d504b7c3a3355afed615e6d8b627a74458634ba21b69bd"
 dependencies = [
 "autocfg",
 "cc",
@@ -751,12 +665,6 @@ dependencies = [
 "vmm-sys-util",
 ]

-[[package]]
-name = "percent-encoding"
-version = "2.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e"
-
 [[package]]
 name = "pkg-config"
 version = "0.3.19"
@@ -851,17 +759,11 @@ dependencies = [
 "pnet_sys",
 ]

-[[package]]
-name = "ppv-lite86"
-version = "0.2.10"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ac74c624d6b2d21f425f752262f42188365d7b8ff1aff74c82e45136510a4857"
-
 [[package]]
 name = "proc-macro2"
-version = "1.0.24"
+version = "1.0.26"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1e0704ee1a7e00d7bb417d0770ea303c1bccbabf0ef1667dae92b5967f5f8a71"
+checksum = "a152013215dca273577e18d2bf00fa862b89b24169fb78c4c95aeb07992c9cec"
 dependencies = [
 "unicode-xid 0.2.1",
 ]
@@ -874,7 +776,6 @@ dependencies = [
 "libc",
 "log 0.4.14",
 "remain",
- "tempfile",
 "vmm-sys-util",
 ]

@@ -888,80 +789,12 @@ dependencies = [
 ]

 [[package]]
-name = "rand"
-version = "0.4.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "552840b97013b1a26992c11eac34bdd778e464601a4c2054b5f0bff7c6761293"
-dependencies = [
- "fuchsia-cprng",
- "libc",
- "rand_core 0.3.1",
- "rdrand",
- "winapi 0.3.9",
-]
-
-[[package]]
-name = "rand"
-version = "0.8.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0ef9e7e66b4468674bfcb0c81af8b7fa0bb154fa9f28eb840da5c447baeb8d7e"
+name = "rate_limiter"
+version = "0.1.0"
 dependencies = [
 "libc",
- "rand_chacha",
- "rand_core 0.6.0",
- "rand_hc",
-]
-
-[[package]]
-name = "rand_chacha"
-version = "0.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e12735cf05c9e10bf21534da50a147b924d555dc7a547c42e6bb2d5b6017ae0d"
-dependencies = [
- "ppv-lite86",
- "rand_core 0.6.0",
-]
-
-[[package]]
-name = "rand_core"
-version = "0.3.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7a6fdeb83b075e8266dcc8762c22776f6877a63111121f5f8c7411e5be7eed4b"
-dependencies = [
- "rand_core 0.4.2",
-]
-
-[[package]]
-name = "rand_core"
-version = "0.4.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9c33a3c44ca05fa6f1807d8e6743f3824e8509beca625669633be0acbdf509dc"
-
-[[package]]
-name = "rand_core"
-version = "0.6.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a8b34ba8cfb21243bd8df91854c830ff0d785fff2e82ebd4434c2644cb9ada18"
-dependencies = [
- "getrandom 0.2.0",
-]
-
-[[package]]
-name = "rand_hc"
-version = "0.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3190ef7066a446f2e7f42e239d161e905420ccab01eb967c9eb27d21b2322a73"
-dependencies = [
- "rand_core 0.6.0",
-]
-
-[[package]]
-name = "rdrand"
-version = "0.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "678054eb77286b51581ba43620cc911abf02758c91f93f479767aed0f90458b2"
-dependencies = [
- "rand_core 0.3.1",
+ "log 0.4.14",
+ "vmm-sys-util",
 ]

 [[package]]
@@ -972,41 +805,39 @@ checksum = "41cc0f7e4d5d4544e8861606a285bb08d3e70712ccc7d2b84d7c0ccfaf4b05ce"

 [[package]]
 name = "redox_syscall"
-version = "0.2.4"
+version = "0.2.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "05ec8ca9416c5ea37062b502703cd7fcb207736bc294f6e0cf367ac6fc234570"
+checksum = "8270314b5ccceb518e7e578952f0b72b88222d02e8f77f5ecf7abbb673539041"
 dependencies = [
 "bitflags 1.2.1",
 ]

 [[package]]
 name = "redox_users"
-version = "0.3.5"
+version = "0.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "de0737333e7a9502c789a36d7c7fa6092a49895d4faa31ca5df163857ded2e9d"
+checksum = "528532f3d801c87aec9def2add9ca802fe569e44a544afe633765267840abe64"
 dependencies = [
- "getrandom 0.1.16",
- "redox_syscall 0.1.57",
- "rust-argon2",
+ "getrandom",
+ "redox_syscall 0.2.6",
 ]

 [[package]]
 name = "regex"
-version = "1.4.3"
+version = "1.4.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d9251239e129e16308e70d853559389de218ac275b515068abc96829d05b948a"
+checksum = "2a26af418b574bd56588335b3a3659a65725d4e636eb1016c2f9e3b38c7cc759"
 dependencies = [
 "aho-corasick",
 "memchr",
 "regex-syntax",
- "thread_local",
 ]

 [[package]]
 name = "regex-syntax"
-version = "0.6.22"
+version = "0.6.23"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b5eb417147ba9860a96cfe72a0b93bf88fee1744b5636ec99ab20c1aa9376581"
+checksum = "24d5f089152e60f62d28b835fbff2cd2e8dc0baf1ac13343bef92ab7eed84548"

 [[package]]
 name = "remain"
@@ -1019,27 +850,6 @@ dependencies = [
 "syn",
 ]

-[[package]]
-name = "remove_dir_all"
-version = "0.5.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7"
-dependencies = [
- "winapi 0.3.9",
-]
-
-[[package]]
-name = "rust-argon2"
-version = "0.8.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4b18820d944b33caa75a71378964ac46f58517c92b6ae5f762636247c09e78fb"
-dependencies = [
- "base64",
- "blake2b_simd",
- "constant_time_eq",
- "crossbeam-utils",
-]
-
 [[package]]
 name = "rustc-demangle"
 version = "0.1.18"
@@ -1052,15 +862,6 @@ version = "0.3.24"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "dcf128d1287d2ea9d80910b5f1120d0b8eede3fbf1abe91c40d39ea7d51e6fda"

-[[package]]
-name = "rustc_version"
-version = "0.2.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a"
-dependencies = [
- "semver",
-]
-
 [[package]]
 name = "ryu"
 version = "1.0.5"
@@ -1076,37 +877,22 @@ checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
 [[package]]
 name = "seccomp"
 version = "0.1.0"
-source = "git+https://github.com/firecracker-microvm/firecracker?tag=v0.22.0#cc5387637c132e500b4857b80b5a239d8d732b77"
+source = "git+https://github.com/firecracker-microvm/firecracker?tag=v0.24.2#5ba819d7b7a684107f5434bcbd1617bc94565478"
 dependencies = [
 "libc",
 ]

-[[package]]
-name = "semver"
-version = "0.9.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403"
-dependencies = [
- "semver-parser",
-]
-
-[[package]]
-name = "semver-parser"
-version = "0.7.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3"
-
 [[package]]
 name = "serde"
-version = "1.0.123"
+version = "1.0.125"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "92d5161132722baa40d802cc70b15262b98258453e85e5d1d365c757c73869ae"
+checksum = "558dc50e1a5a5fa7112ca2ce4effcb321b0300c0d4ccf0776a9f60cd89031171"

 [[package]]
 name = "serde_derive"
-version = "1.0.123"
+version = "1.0.125"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9391c295d64fc0abb2c556bad848f33cb8296276b1ad2677d1ae1ace4f258f31"
+checksum = "b093b7a2bb58203b5da3056c05b4ec1fed827dcfdb37347a8841695263b3d06d"
 dependencies = [
 "proc-macro2",
 "quote",
@@ -1115,9 +901,9 @@ dependencies = [

 [[package]]
 name = "serde_json"
-version = "1.0.62"
+version = "1.0.64"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ea1c6153794552ea7cf7cf63b1231a25de00ec90db326ba6264440fa08e31486"
+checksum = "799e97dc9fdae36a5c8b8f2cae9ce2ee9fdce2058c57a93e6099d919fd982f79"
 dependencies = [
 "itoa",
 "ryu",
@@ -1126,9 +912,9 @@ dependencies = [

 [[package]]
 name = "signal-hook"
-version = "0.3.4"
+version = "0.3.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "780f5e3fe0c66f67197236097d89de1e86216f1f6fdeaf47c442f854ab46c240"
+checksum = "ef33d6d0cd06e0840fba9985aab098c147e67e05cee14d412d3345ed14ff30ac"
 dependencies = [
 "libc",
 "signal-hook-registry",
@@ -1163,9 +949,9 @@ dependencies = [

 [[package]]
 name = "static_assertions"
-version = "0.3.4"
+version = "1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7f3eb36b47e512f8f1c9e3d10c2c1965bc992bd9cdb024fa581e2194501c83d3"
+checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f"

 [[package]]
 name = "strsim"
@@ -1175,9 +961,9 @@ checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a"

 [[package]]
 name = "syn"
-version = "1.0.60"
+version = "1.0.71"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c700597eca8a5a762beb35753ef6b94df201c81cca676604f547495a0d7f0081"
+checksum = "ad184cc9470f9117b2ac6817bfe297307418819ba40552f9b3846f05c33d5373"
 dependencies = [
 "proc-macro2",
 "quote",
@@ -1245,30 +1031,6 @@ dependencies = [
 "unicode-xid 0.0.3",
 ]

-[[package]]
-name = "tempdir"
-version = "0.3.7"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "15f2b5fb00ccdf689e0149d1b1b3c03fead81c2b37735d812fa8bddbbf41b6d8"
-dependencies = [
- "rand 0.4.6",
- "remove_dir_all",
-]
-
-[[package]]
-name = "tempfile"
-version = "3.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dac1c663cfc93810f88aed9b8941d48cabf856a1b111c29a40439018d870eb22"
-dependencies = [
- "cfg-if 1.0.0",
- "libc",
- "rand 0.8.3",
- "redox_syscall 0.2.4",
- "remove_dir_all",
- "winapi 0.3.9",
-]
-
 [[package]]
 name = "term"
 version = "0.4.6"
@@ -1298,6 +1060,18 @@ dependencies = [
 "winapi-util",
 ]

+[[package]]
+name = "test_infra"
+version = "0.1.0"
+dependencies = [
+ "dirs",
+ "epoll",
+ "libc",
+ "ssh2",
+ "vmm-sys-util",
+ "wait-timeout",
+]
+
 [[package]]
 name = "textwrap"
 version = "0.11.0"
@@ -1310,66 +1084,24 @@ dependencies = [

 [[package]]
 name = "thiserror"
-version = "1.0.23"
+version = "1.0.24"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "76cc616c6abf8c8928e2fdcc0dbfab37175edd8fb49a4641066ad1364fdab146"
+checksum = "e0f4a65597094d4483ddaed134f409b2cb7c1beccf25201a9f73c719254fa98e"
 dependencies = [
 "thiserror-impl",
 ]

 [[package]]
 name = "thiserror-impl"
-version = "1.0.23"
+version = "1.0.24"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9be73a2caec27583d0046ef3796c3794f868a5bc813db689eed00c7631275cd1"
+checksum = "7765189610d8241a44529806d6fd1f2e0a08734313a35d5b3a556f92b381f3c0"
 dependencies = [
 "proc-macro2",
 "quote",
 "syn",
 ]

-[[package]]
-name = "thread_local"
-version = "1.1.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8018d24e04c95ac8790716a5987d0fec4f8b27249ffa0f7d33f1369bdfb88cbd"
-dependencies = [
- "once_cell",
-]
-
-[[package]]
-name = "tinyvec"
-version = "1.1.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "317cca572a0e89c3ce0ca1f1bdc9369547fe318a683418e42ac8f59d14701023"
-dependencies = [
- "tinyvec_macros",
-]
-
-[[package]]
-name = "tinyvec_macros"
-version = "0.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c"
-
-[[package]]
-name = "unicode-bidi"
-version = "0.3.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "49f2bd0c6468a8230e1db229cff8029217cf623c767ea5d60bfbd42729ea54d5"
-dependencies = [
- "matches",
-]
-
-[[package]]
-name = "unicode-normalization"
-version = "0.1.17"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "07fbfce1c8a97d547e8b5334978438d9d6ec8c20e38f56d4a4374d181493eaef"
-dependencies = [
- "tinyvec",
-]
-
 [[package]]
 name = "unicode-width"
 version = "0.1.8"
@@ -1388,23 +1120,11 @@ version = "0.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f7fe0bb3479651439c9112f72b6c505038574c9fbb575ed1bf3b797fa39dd564"

-[[package]]
-name = "url"
-version = "2.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5909f2b0817350449ed73e8bcd81c8c3c8d9a7a5d8acba4b27db277f1868976e"
-dependencies = [
- "form_urlencoded",
- "idna",
- "matches",
- "percent-encoding",
-]
-
 [[package]]
 name = "vcpkg"
-version = "0.2.11"
+version = "0.2.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b00bca6106a5e23f3eee943593759b7fcddb00554332e856d990c893966879fb"
+checksum = "cbdbff6266a24120518560b5dc983096efb98462e51d0d68169895b237be3e5d"

 [[package]]
 name = "vec_map"
@@ -1424,7 +1144,7 @@ dependencies = [
 [[package]]
 name = "vfio-ioctls"
 version = "0.1.0"
-source = "git+https://github.com/cloud-hypervisor/vfio-ioctls?branch=ch#6ea1b934bbff9102bbdc80c23a357a48645cd722"
+source = "git+https://github.com/rust-vmm/vfio-ioctls?branch=master#ed8d2534576371000361e615eeb91b4266c55713"
 dependencies = [
 "byteorder",
 "kvm-bindings",
@@ -1438,7 +1158,7 @@ dependencies = [
 [[package]]
 name = "vhost"
 version = "0.1.0"
-source = "git+https://github.com/cloud-hypervisor/vhost?branch=ch#6fc980bf5083d67586ab91d6a965c3a593c33a78"
+source = "git+https://github.com/rust-vmm/vhost?branch=master#c4cd1d375e386069d0b781174de9f62ef41812e2"
 dependencies = [
 "bitflags 1.2.1",
 "libc",
@@ -1509,18 +1229,18 @@ dependencies = [
 "block_util",
 "byteorder",
 "epoll",
+ "event_monitor",
 "io-uring",
 "libc",
 "log 0.4.14",
 "net_gen",
 "net_util",
 "pci",
+ "rate_limiter",
 "seccomp",
 "serde",
 "serde_derive",
 "serde_json",
- "tempfile",
- "vfio-ioctls",
 "vhost",
 "virtio-bindings",
 "vm-allocator",
@@ -1549,6 +1269,7 @@ dependencies = [
 "serde_derive",
 "serde_json",
 "thiserror",
+ "vfio-ioctls",
 "vm-memory",
 "vmm-sys-util",
 ]
@@ -1602,6 +1323,7 @@ dependencies = [
 "credibility",
 "devices",
 "epoll",
+ "event_monitor",
 "hypervisor",
 "lazy_static",
 "libc",
@@ -1617,9 +1339,7 @@ dependencies = [
 "serde_derive",
 "serde_json",
 "signal-hook",
- "tempfile",
 "thiserror",
- "url",
 "vfio-ioctls",
 "virtio-devices",
 "vm-allocator",
@@ -1632,9 +1352,9 @@ dependencies = [

 [[package]]
 name = "vmm-sys-util"
-version = "0.7.0"
+version = "0.8.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d1cdd1d72e262bbfb014de65ada24c1ac50e10a2e3b1e8ec052df188c2ee5dfa"
+checksum = "01cf11afbc4ebc0d5c7a7748a77d19e2042677fc15faa2f4ccccb27c18a60605"
 dependencies = [
 "bitflags 1.2.1",
 "libc",
@@ -1653,9 +1373,9 @@ dependencies = [

 [[package]]
 name = "wasi"
-version = "0.9.0+wasi-snapshot-preview1"
+version = "0.10.2+wasi-snapshot-preview1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519"
+checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6"

 [[package]]
 name = "winapi"
@@ -1712,9 +1432,9 @@ dependencies = [

 [[package]]
 name = "zerocopy-derive"
-version = "0.2.0"
+version = "0.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d498dbd1fd7beb83c86709ae1c33ca50942889473473d287d56ce4770a18edfb"
+checksum = "dc9c39e6d503229ffa00cc2954af4a751e6bbedf2a2c18e856eb3ece93d32495"
 dependencies = [
 "proc-macro2",
 "syn",
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "cloud-hypervisor"
-version = "0.13.0"
+version = "15.0.0"
 authors = ["The Cloud Hypervisor Authors"]
 edition = "2018"
 default-run = "cloud-hypervisor"
@@ -13,39 +13,38 @@ homepage = "https://github.com/cloud-hypervisor/cloud-hypervisor"
 lto = true

 [dependencies]
-anyhow = "1.0"
+anyhow = "1.0.40"
 api_client = { path = "api_client" }
 clap = { version = "2.33.3", features = ["wrap_help"] }
-epoll = ">=4.0.1"
+epoll = "4.3.1"
+event_monitor = { path = "event_monitor" }
 hypervisor = { path = "hypervisor" }
-libc = "0.2.86"
+libc = "0.2.94"
 log = { version = "0.4.14", features = ["std"] }
 option_parser = { path = "option_parser" }
-seccomp = { git = "https://github.com/firecracker-microvm/firecracker", tag = "v0.22.0" }
-serde_json = "1.0.62"
-signal-hook = "0.3.4"
-thiserror = "1.0"
+seccomp = { git = "https://github.com/firecracker-microvm/firecracker", tag = "v0.24.2" }
+serde_json = "1.0.64"
+signal-hook = "0.3.8"
+thiserror = "1.0.24"
 vmm = { path = "vmm" }
-vmm-sys-util = "0.7.0"
+vmm-sys-util = "0.8.0"
 vm-memory = "0.5.0"
-wait-timeout = "0.2.0"

 [build-dependencies]
 clap = { version = "2.33.3", features = ["wrap_help"] }

 # List of patched crates
-[patch.'https://github.com/rust-vmm/vhost']
-vhost_rs = { git = "https://github.com/cloud-hypervisor/vhost", branch = "ch", package = "vhost", features = ["vhost-user-master", "vhost-user-slave"] }
+[patch.crates-io]
+kvm-bindings = { git = "https://github.com/cloud-hypervisor/kvm-bindings", branch = "ch-v0.4.0", features = ["with-serde", "fam-wrappers"] }

 [dev-dependencies]
-ssh2 = "0.9.1"
-dirs = "3.0.1"
 credibility = "0.1.3"
-tempdir = "0.3.7"
+dirs = "3.0.2"
 lazy_static= "1.4.0"
-tempfile = "3.2.0"
-serde_json = "1.0.62"
 net_util = { path = "net_util" }
+serde_json = "1.0.64"
+test_infra = { path = "test_infra" }
+wait-timeout = "0.2.0"

 [features]
 default = ["acpi", "cmos", "io_uring", "kvm"]
@@ -57,6 +56,7 @@ fwdebug = ["vmm/fwdebug"]
 kvm = ["vmm/kvm"]
 mshv = ["vmm/mshv"]
 io_uring = ["vmm/io_uring"]
+tdx = ["vmm/tdx"]

 # Integration tests require a special environment to run in
 integration_tests = []
@@ -69,12 +69,14 @@ members = [
    "arch_gen",
    "block_util",
    "devices",
+    "event_monitor",
    "hypervisor",
    "net_gen",
    "net_util",
    "option_parser",
    "pci",
    "qcow",
+    "rate_limiter",
    "vhost_user_backend",
    "vhost_user_block",
    "vhost_user_net",
@@ -85,3 +87,4 @@ members = [
    "vm-migration",
    "vm-virtio"
 ]
+exclude = ["test_infra"]
--- a/35
+++ b/35
@@ -136,6 +136,39 @@ pipeline{
 						}
 					}
 				}
+				stage ('Worker build VFIO') {
+					agent { node { label 'bionic-vfio' } }
+					when { branch 'master' }
+					stages {
+						stage ('Checkout') {
+							steps {
+								checkout scm
+							}
+						}
+						stage ('Run VFIO integration tests') {
+							options {
+								timeout(time: 1, unit: 'HOURS')
+							}
+							steps {
+								sh "scripts/dev_cli.sh tests --integration-vfio"
+							}
+						}
+						stage ('Run VFIO integration tests for musl') {
+							options {
+								timeout(time: 1, unit: 'HOURS')
+							}
+							steps {
+								sh "scripts/dev_cli.sh tests --integration-vfio --libc musl"
+							}
+						}
+					}
+					post {
+						always {
+							sh "sudo chown -R jenkins.jenkins ${WORKSPACE}"
+							deleteDir()
+						}
+					}
+				}
 				stage ('Worker build - Windows guest') {
 					agent { node { label 'groovy-win' } }
 					stages {
@@ -149,7 +182,7 @@ pipeline{
 								sh "mkdir ${env.HOME}/workloads"
 								azureDownload(storageCredentialId: 'ch-image-store',
 											  containerName: 'private-images',
-											  includeFilesPattern: 'OVMF.fd',
+											  includeFilesPattern: 'OVMF-4b47d0c6c8.fd',
 											  downloadType: 'container',
 											  downloadDirLoc: "${env.HOME}/workloads")
 								azureDownload(storageCredentialId: 'ch-image-store',
--- a/README.md
+++ b/README.md
@@ -26,7 +26,8 @@

 # 1. What is Cloud Hypervisor?

-Cloud Hypervisor is an open source Virtual Machine Monitor (VMM) that runs on top of [KVM](https://www.kernel.org/doc/Documentation/virtual/kvm/api.txt).
+Cloud Hypervisor is an open source Virtual Machine Monitor (VMM) that runs on top of [KVM](https://www.kernel.org/doc/Documentation/virtual/kvm/api.txt) and the MSHV hypervisors .
+
 The project focuses on exclusively running modern, cloud workloads, on top of a limited set of hardware architectures and platforms.
 Cloud workloads refers to those that are usually run by customers inside a cloud provider. For our purposes this means modern operating systems with most I/O handled by paravirtualised devices (i.e. virtio), no requirement for legacy devices, and 64-bit CPUs.

@@ -36,7 +37,7 @@ Cloud Hypervisor is implemented in [Rust](https://www.rust-lang.org/) and is bas

 ### High Level

- KVM based
+- Runs on KVM or MSHV
 - Minimal emulation
 - Low latency
 - Low memory footprint
@@ -53,7 +54,7 @@ Cloud Hypervisor supports the `x86-64` and `AArch64` architectures. There are so

 ### Guest OS

-Cloud Hypervisor supports `64-bit Linux` with support for _modern_ 64-bit Windows guests currently under development.
+Cloud Hypervisor supports `64-bit Linux` and Windows 10/Windows Server 2019.

 # 2. Getting Started

@@ -148,7 +149,7 @@ We need to get the latest `rust-hypervisor-firmware` release and also a working
 $ pushd $CLOUDH
 $ wget https://cloud-images.ubuntu.com/focal/current/focal-server-cloudimg-amd64.img
 $ qemu-img convert -p -f qcow2 -O raw focal-server-cloudimg-amd64.img focal-server-cloudimg-amd64.raw
-$ wget https://github.com/cloud-hypervisor/rust-hypervisor-firmware/releases/download/0.2.8/hypervisor-fw
+$ wget https://github.com/cloud-hypervisor/rust-hypervisor-firmware/releases/download/0.3.1/hypervisor-fw
 $ popd
 ```

@@ -171,7 +172,7 @@ Multiple arguments can be given to the `--disk` parameter.

 #### Building your kernel

-Cloud Hypervisor also supports direct kernel boot into a `vmlinux` ELF kernel or `bzImage`. In order to support virtio-fs and virtio-iommu we have our own development branch. You are of course able to use your own kernel but these instructions will continue with the version that we develop and test against.
+Cloud Hypervisor also supports direct kernel boot into a `vmlinux` ELF kernel. In order to support virtio-iommu we have our own development branch. You are of course able to use your own kernel but these instructions will continue with the version that we develop and test against.

 To build the kernel:

@@ -179,7 +180,7 @@ To build the kernel:

 # Clone the Cloud Hypervisor Linux branch
 $ pushd $CLOUDH
-$ git clone --depth 1 https://github.com/cloud-hypervisor/linux.git -b virtio-fs-virtio-iommu-virtio-mem-5.6-rc4 linux-cloud-hypervisor
+$ git clone --depth 1 https://github.com/cloud-hypervisor/linux.git -b ch-5.12 linux-cloud-hypervisor
 $ pushd linux-cloud-hypervisor

 # Use the cloud-hypervisor kernel config to build your kernel
@@ -241,23 +242,40 @@ $ ./cloud-hypervisor/target/release/cloud-hypervisor \

 # 3. Status

-Cloud Hypervisor is under active development. No API or feature stability is guaranteed.
+Cloud Hypervisor is under active development. The following stability guarantees are currently made:

-As of 2020-07-02, the following cloud images are supported:
+* The API (including command line options) will not be removed or changed in a
+  breaking way without a minimum of 2 releases notice. Where possible warnings
+  will be given about the use of deprecated functionality and the deprecations
+  will be documented in the release notes.
+* Point releases will be made between individual releases where there are
+  substantial bug fixes or security issues that need to be fixed.
+
+Currently the following items are **not** guaranteed across updates:
+
+* Snapshot/restore is not supported across different versions
+* Live migration is not supported across different versions
+* The following features are considered experimental and may change
+  substantially between releases: TDX, SGX.
+
+
+As of 2021-04-29, the following cloud images are supported:

 - [Ubuntu Bionic](https://cloud-images.ubuntu.com/bionic/current/) (cloudimg)
 - [Ubuntu Focal](https://cloud-images.ubuntu.com/focal/current/) (cloudimg)
+- [Ubuntu Groovy](https://cloud-images.ubuntu.com/groovy/current/) (cloudimg)
+- [Ubuntu Hirsute](https://cloud-images.ubuntu.com/hirsute/current/) (cloudimg)

 Direct kernel boot to userspace should work with a rootfs from most distributions.

 ## Hot Plug

-Cloud Hypervisor supports hotplug of CPUs, passthrough devices (VFIO), `virtio-{net,block,pmem,fs,vsock}` and memory resizing. This [document](https://github.com/cloud-hypervisor/cloud-hypervisor/blob/master/docs/hotplug.md) details how to add devices to
+Cloud Hypervisor supports hotplug of CPUs, passthrough devices (VFIO), `virtio-{net,block,pmem,fs,vsock}` and memory resizing. This [document](docs/hotplug.md) details how to add devices to
 a running VM.

 ## Device Model

-Details of the device model can be found in this [documentation](https://github.com/cloud-hypervisor/cloud-hypervisor/blob/master/docs/device_model.md).
+Details of the device model can be found in this [documentation](docs/device_model.md).

 ## TODO

@@ -317,3 +335,7 @@ etc, are all equal and welcome means of contribution. See the [CONTRIBUTING](CON

 Get an [invite to our Slack channel](https://join.slack.com/t/cloud-hypervisor/shared_invite/enQtNjY3MTE3MDkwNDQ4LWQ1MTA1ZDVmODkwMWQ1MTRhYzk4ZGNlN2UwNTI3ZmFlODU0OTcwOWZjMTkwZDExYWE3YjFmNzgzY2FmNDAyMjI)
 and [join us on Slack](https://cloud-hypervisor.slack.com/).
+
+## Security issues
+
+Please use the GitHub security advisories feature for reporting issues: https://github.com/cloud-hypervisor/cloud-hypervisor/security/advisories/new
--- a/acpi_tables/src/aml.rs
+++ b/acpi_tables/src/aml.rs
@@ -95,8 +95,7 @@ pub type Byte = u8;

 impl Aml for Byte {
    fn to_aml_bytes(&self) -> Vec<u8> {
-        let mut bytes = Vec::new();
-        bytes.push(0x0a); /* BytePrefix */
+        let mut bytes = vec![0x0a]; /* BytePrefix */
        bytes.push(*self);
        bytes
    }
@@ -106,8 +105,7 @@ pub type Word = u16;

 impl Aml for Word {
    fn to_aml_bytes(&self) -> Vec<u8> {
-        let mut bytes = Vec::new();
-        bytes.push(0x0bu8); /* WordPrefix */
+        let mut bytes = vec![0x0bu8]; /* WordPrefix */
        bytes.append(&mut self.to_le_bytes().to_vec());
        bytes
    }
@@ -117,8 +115,7 @@ pub type DWord = u32;

 impl Aml for DWord {
    fn to_aml_bytes(&self) -> Vec<u8> {
-        let mut bytes = Vec::new();
-        bytes.push(0x0c); /* DWordPrefix */
+        let mut bytes = vec![0x0c]; /* DWordPrefix */
        bytes.append(&mut self.to_le_bytes().to_vec());
        bytes
    }
@@ -128,8 +125,7 @@ pub type QWord = u64;

 impl Aml for QWord {
    fn to_aml_bytes(&self) -> Vec<u8> {
-        let mut bytes = Vec::new();
-        bytes.push(0x0e); /* QWordPrefix */
+        let mut bytes = vec![0x0e]; /* QWordPrefix */
        bytes.append(&mut self.to_le_bytes().to_vec());
        bytes
    }
@@ -147,8 +143,7 @@ impl Aml for Name {

 impl Name {
    pub fn new(path: Path, inner: &dyn Aml) -> Self {
-        let mut bytes = Vec::new();
-        bytes.push(0x08); /* NameOp */
+        let mut bytes = vec![0x08]; /* NameOp */
        bytes.append(&mut path.to_aml_bytes());
        bytes.append(&mut inner.to_aml_bytes());
        Name { bytes }
@@ -161,8 +156,7 @@ pub struct Package<'a> {

 impl<'a> Aml for Package<'a> {
    fn to_aml_bytes(&self) -> Vec<u8> {
-        let mut bytes = Vec::new();
-        bytes.push(self.children.len() as u8);
+        let mut bytes = vec![self.children.len() as u8];
        for child in &self.children {
            bytes.append(&mut child.to_aml_bytes());
        }
@@ -239,11 +233,11 @@ fn create_pkg_length(data: &[u8], include_self: bool) -> Vec<u8> {
    result
 }

-pub struct EISAName {
+pub struct EisaName {
    value: DWord,
 }

-impl EISAName {
+impl EisaName {
    pub fn new(name: &str) -> Self {
        assert_eq!(name.len(), 7);

@@ -258,11 +252,11 @@ impl EISAName {
            | name.chars().nth(6).unwrap().to_digit(16).unwrap())
        .swap_bytes();

-        EISAName { value }
+        EisaName { value }
    }
 }

-impl Aml for EISAName {
+impl Aml for EisaName {
    fn to_aml_bytes(&self) -> Vec<u8> {
        self.value.to_aml_bytes()
    }
@@ -289,8 +283,7 @@ impl Aml for Usize {
 }

 fn create_aml_string(v: &str) -> Vec<u8> {
-    let mut data = Vec::new();
-    data.push(0x0D); /* String Op */
+    let mut data = vec![0x0D]; /* String Op */
    data.extend_from_slice(v.as_bytes());
    data.push(0x0); /* NullChar */
    data
@@ -374,9 +367,7 @@ impl Memory32Fixed {

 impl Aml for Memory32Fixed {
    fn to_aml_bytes(&self) -> Vec<u8> {
-        let mut bytes = Vec::new();
-
-        bytes.push(0x86); /* Memory32Fixed */
+        let mut bytes = vec![0x86]; /* Memory32Fixed */
        bytes.append(&mut 9u16.to_le_bytes().to_vec());

        // 9 bytes of payload
@@ -390,7 +381,7 @@ impl Aml for Memory32Fixed {
 #[derive(Copy, Clone)]
 enum AddressSpaceType {
    Memory,
-    IO,
+    Io,
    BusNumber,
 }

@@ -421,7 +412,7 @@ impl<T> AddressSpace<T> {

    pub fn new_io(min: T, max: T) -> Self {
        AddressSpace {
-            r#type: AddressSpaceType::IO,
+            r#type: AddressSpaceType::Io,
            min,
            max,
            type_flags: 3, /* EntireRange */
@@ -510,16 +501,16 @@ impl Aml for AddressSpace<u64> {
    }
 }

-pub struct IO {
+pub struct Io {
    min: u16,
    max: u16,
    alignment: u8,
    length: u8,
 }

-impl IO {
+impl Io {
    pub fn new(min: u16, max: u16, alignment: u8, length: u8) -> Self {
-        IO {
+        Io {
            min,
            max,
            alignment,
@@ -528,11 +519,10 @@ impl IO {
    }
 }

-impl Aml for IO {
+impl Aml for Io {
    fn to_aml_bytes(&self) -> Vec<u8> {
-        let mut bytes = Vec::new();
+        let mut bytes = vec![0x47]; /* Io Port Descriptor */

-        bytes.push(0x47); /* IO Port Descriptor */
        bytes.push(1); /* IODecode16 */
        bytes.append(&mut self.min.to_le_bytes().to_vec());
        bytes.append(&mut self.max.to_le_bytes().to_vec());
@@ -571,9 +561,7 @@ impl Interrupt {

 impl Aml for Interrupt {
    fn to_aml_bytes(&self) -> Vec<u8> {
-        let mut bytes = Vec::new();
-
-        bytes.push(0x89); /* Extended IRQ Descriptor */
+        let mut bytes = vec![0x89]; /* Extended IRQ Descriptor */
        bytes.append(&mut 6u16.to_le_bytes().to_vec());
        let flags = (self.shared as u8) << 3
            | (self.active_low as u8) << 2
@@ -699,8 +687,7 @@ impl<'a> Return<'a> {

 impl<'a> Aml for Return<'a> {
    fn to_aml_bytes(&self) -> Vec<u8> {
-        let mut bytes = Vec::new();
-        bytes.push(0xa4); /* ReturnOp */
+        let mut bytes = vec![0xa4]; /* ReturnOp */
        bytes.append(&mut self.value.to_aml_bytes());
        bytes
    }
@@ -745,9 +732,9 @@ impl Field {
    ) -> Self {
        Field {
            path,
+            fields,
            access_type,
            update_rule,
-            fields,
        }
    }
 }
@@ -788,14 +775,14 @@ impl Aml for Field {
 #[derive(Clone, Copy)]
 pub enum OpRegionSpace {
    SystemMemory,
-    SystemIO,
-    PCIConfig,
+    SystemIo,
+    PConfig,
    EmbeddedControl,
-    SMBus,
-    SystemCMOS,
+    Smbus,
+    SystemCmos,
    PciBarTarget,
-    IPMI,
-    GeneralPurposeIO,
+    Ipmi,
+    GeneralPurposeIo,
    GenericSerialBus,
 }

@@ -864,8 +851,8 @@ impl<'a> Aml for If<'a> {
 }

 pub struct Equal<'a> {
-    right: &'a dyn Aml,
    left: &'a dyn Aml,
+    right: &'a dyn Aml,
 }

 impl<'a> Equal<'a> {
@@ -876,8 +863,7 @@ impl<'a> Equal<'a> {

 impl<'a> Aml for Equal<'a> {
    fn to_aml_bytes(&self) -> Vec<u8> {
-        let mut bytes = Vec::new();
-        bytes.push(0x93); /* LEqualOp */
+        let mut bytes = vec![0x93]; /* LEqualOp */
        bytes.extend_from_slice(&self.left.to_aml_bytes());
        bytes.extend_from_slice(&self.right.to_aml_bytes());
        bytes
@@ -885,8 +871,8 @@ impl<'a> Aml for Equal<'a> {
 }

 pub struct LessThan<'a> {
-    right: &'a dyn Aml,
    left: &'a dyn Aml,
+    right: &'a dyn Aml,
 }

 impl<'a> LessThan<'a> {
@@ -897,8 +883,7 @@ impl<'a> LessThan<'a> {

 impl<'a> Aml for LessThan<'a> {
    fn to_aml_bytes(&self) -> Vec<u8> {
-        let mut bytes = Vec::new();
-        bytes.push(0x95); /* LLessOp */
+        let mut bytes = vec![0x95]; /* LLessOp */
        bytes.extend_from_slice(&self.left.to_aml_bytes());
        bytes.extend_from_slice(&self.right.to_aml_bytes());
        bytes
@@ -940,8 +925,7 @@ impl<'a> Store<'a> {

 impl<'a> Aml for Store<'a> {
    fn to_aml_bytes(&self) -> Vec<u8> {
-        let mut bytes = Vec::new();
-        bytes.push(0x70); /* StoreOp */
+        let mut bytes = vec![0x70]; /* StoreOp */
        bytes.extend_from_slice(&self.value.to_aml_bytes());
        bytes.extend_from_slice(&self.name.to_aml_bytes());
        bytes
@@ -961,8 +945,7 @@ impl Mutex {

 impl Aml for Mutex {
    fn to_aml_bytes(&self) -> Vec<u8> {
-        let mut bytes = Vec::new();
-        bytes.push(0x5b); /* ExtOpPrefix */
+        let mut bytes = vec![0x5b]; /* ExtOpPrefix */
        bytes.push(0x01); /* MutexOp */
        bytes.extend_from_slice(&self.path.to_aml_bytes());
        bytes.push(self.sync_level);
@@ -983,8 +966,7 @@ impl Acquire {

 impl Aml for Acquire {
    fn to_aml_bytes(&self) -> Vec<u8> {
-        let mut bytes = Vec::new();
-        bytes.push(0x5b); /* ExtOpPrefix */
+        let mut bytes = vec![0x5b]; /* ExtOpPrefix */
        bytes.push(0x23); /* AcquireOp */
        bytes.extend_from_slice(&self.mutex.to_aml_bytes());
        bytes.extend_from_slice(&self.timeout.to_le_bytes());
@@ -1004,8 +986,7 @@ impl Release {

 impl Aml for Release {
    fn to_aml_bytes(&self) -> Vec<u8> {
-        let mut bytes = Vec::new();
-        bytes.push(0x5b); /* ExtOpPrefix */
+        let mut bytes = vec![0x5b]; /* ExtOpPrefix */
        bytes.push(0x27); /* ReleaseOp */
        bytes.extend_from_slice(&self.mutex.to_aml_bytes());
        bytes
@@ -1025,8 +1006,7 @@ impl<'a> Notify<'a> {

 impl<'a> Aml for Notify<'a> {
    fn to_aml_bytes(&self) -> Vec<u8> {
-        let mut bytes = Vec::new();
-        bytes.push(0x86); /* NotifyOp */
+        let mut bytes = vec![0x86]; /* NotifyOp */
        bytes.extend_from_slice(&self.object.to_aml_bytes());
        bytes.extend_from_slice(&self.value.to_aml_bytes());
        bytes
@@ -1076,14 +1056,13 @@ macro_rules! binary_op {

        impl<'a> $name<'a> {
            pub fn new(target: &'a dyn Aml, a: &'a dyn Aml, b: &'a dyn Aml) -> Self {
-                $name { target, a, b }
+                $name { a, b, target }
            }
        }

        impl<'a> Aml for $name<'a> {
            fn to_aml_bytes(&self) -> Vec<u8> {
-                let mut bytes = Vec::new();
-                bytes.push($opcode); /* Op for the binary operator */
+                let mut bytes = vec![$opcode]; /* Op for the binary operator */
                bytes.extend_from_slice(&self.a.to_aml_bytes());
                bytes.extend_from_slice(&self.b.to_aml_bytes());
                bytes.extend_from_slice(&self.target.to_aml_bytes());
@@ -1180,8 +1159,7 @@ impl<'a, T> CreateField<'a, T> {

 impl<'a> Aml for CreateField<'a, u64> {
    fn to_aml_bytes(&self) -> Vec<u8> {
-        let mut bytes = Vec::new();
-        bytes.push(0x8f); /* CreateQWordFieldOp */
+        let mut bytes = vec![0x8f]; /* CreateQWordFieldOp */
        bytes.extend_from_slice(&self.buffer.to_aml_bytes());
        bytes.extend_from_slice(&self.offset.to_aml_bytes());
        bytes.extend_from_slice(&self.field.to_aml_bytes());
@@ -1191,8 +1169,7 @@ impl<'a> Aml for CreateField<'a, u64> {

 impl<'a> Aml for CreateField<'a, u32> {
    fn to_aml_bytes(&self) -> Vec<u8> {
-        let mut bytes = Vec::new();
-        bytes.push(0x8a); /* CreateDWordFieldOp */
+        let mut bytes = vec![0x8a]; /* CreateDWordFieldOp */
        bytes.extend_from_slice(&self.buffer.to_aml_bytes());
        bytes.extend_from_slice(&self.offset.to_aml_bytes());
        bytes.extend_from_slice(&self.field.to_aml_bytes());
@@ -1235,12 +1212,12 @@ mod tests {
            Device::new(
                "_SB_.COM1".into(),
                vec![
-                    &Name::new("_HID".into(), &EISAName::new("PNP0501")),
+                    &Name::new("_HID".into(), &EisaName::new("PNP0501")),
                    &Name::new(
                        "_CRS".into(),
                        &ResourceTemplate::new(vec![
                            &Interrupt::new(true, true, false, false, 4),
-                            &IO::new(0x3f8, 0x3f8, 0, 0x8)
+                            &Io::new(0x3f8, 0x3f8, 0, 0x8)
                        ])
                    )
                ]
@@ -1476,7 +1453,7 @@ mod tests {
                "_CRS".into(),
                &ResourceTemplate::new(vec![
                    &Interrupt::new(true, true, false, false, 4),
-                    &IO::new(0x3f8, 0x3f8, 0, 0x8)
+                    &Io::new(0x3f8, 0x3f8, 0, 0x8)
                ])
            )
            .to_aml_bytes(),
@@ -1519,7 +1496,7 @@ mod tests {
    #[test]
    fn test_eisa_name() {
        assert_eq!(
-            Name::new("_HID".into(), &EISAName::new("PNP0501")).to_aml_bytes(),
+            Name::new("_HID".into(), &EisaName::new("PNP0501")).to_aml_bytes(),
            [0x08, 0x5F, 0x48, 0x49, 0x44, 0x0C, 0x41, 0xD0, 0x05, 0x01],
        )
    }
@@ -1665,14 +1642,14 @@ mod tests {
    #[test]
    fn test_op_region() {
        /*
-            OperationRegion (PRST, SystemIO, 0x0CD8, 0x0C)
+            OperationRegion (PRST, SystemIo, 0x0CD8, 0x0C)
        */
        let op_region_data = [
            0x5Bu8, 0x80, 0x50, 0x52, 0x53, 0x54, 0x01, 0x0B, 0xD8, 0x0C, 0x0A, 0x0C,
        ];

        assert_eq!(
-            OpRegion::new("PRST".into(), OpRegionSpace::SystemIO, 0xcd8, 0xc).to_aml_bytes(),
+            OpRegion::new("PRST".into(), OpRegionSpace::SystemIo, 0xcd8, 0xc).to_aml_bytes(),
            &op_region_data[..]
        );
    }
@@ -1766,7 +1743,7 @@ mod tests {
            Device::new(
                "_SB_.MHPC".into(),
                vec![
-                    &Name::new("_HID".into(), &EISAName::new("PNP0A06")),
+                    &Name::new("_HID".into(), &EisaName::new("PNP0A06")),
                    &mutex,
                    &Method::new(
                        "TEST".into(),
@@ -1807,7 +1784,7 @@ mod tests {
            Device::new(
                "_SB_.MHPC".into(),
                vec![
-                    &Name::new("_HID".into(), &EISAName::new("PNP0A06")),
+                    &Name::new("_HID".into(), &EisaName::new("PNP0A06")),
                    &Method::new(
                        "TEST".into(),
                        0,
@@ -1848,7 +1825,7 @@ mod tests {
            Device::new(
                "_SB_.MHPC".into(),
                vec![
-                    &Name::new("_HID".into(), &EISAName::new("PNP0A06")),
+                    &Name::new("_HID".into(), &EisaName::new("PNP0A06")),
                    &Method::new(
                        "TEST".into(),
                        0,
--- a/acpi_tables/src/rsdp.rs
+++ b/acpi_tables/src/rsdp.rs
@@ -7,7 +7,7 @@ use vm_memory::ByteValued;

 #[repr(packed)]
 #[derive(Clone, Copy, Default)]
-pub struct RSDP {
+pub struct Rsdp {
    pub signature: [u8; 8],
    pub checksum: u8,
    pub oem_id: [u8; 6],
@@ -19,17 +19,17 @@ pub struct RSDP {
    _reserved: [u8; 3],
 }

-unsafe impl ByteValued for RSDP {}
+unsafe impl ByteValued for Rsdp {}

-impl RSDP {
+impl Rsdp {
    pub fn new(oem_id: [u8; 6], xsdt_addr: u64) -> Self {
-        let mut rsdp = RSDP {
+        let mut rsdp = Rsdp {
            signature: *b"RSD PTR ",
            checksum: 0,
            oem_id,
            revision: 2,
            _rsdt_addr: 0,
-            length: std::mem::size_of::<RSDP>() as u32,
+            length: std::mem::size_of::<Rsdp>() as u32,
            xsdt_addr,
            extended_checksum: 0,
            _reserved: [0; 3],
@@ -41,18 +41,18 @@ impl RSDP {
    }

    pub fn len() -> usize {
-        std::mem::size_of::<RSDP>()
+        std::mem::size_of::<Rsdp>()
    }
 }

 #[cfg(test)]
 mod tests {
-    use super::RSDP;
+    use super::Rsdp;
    use vm_memory::bytes::ByteValued;

    #[test]
    fn test_rsdp() {
-        let rsdp = RSDP::new(*b"CHYPER", 0xdead_beef);
+        let rsdp = Rsdp::new(*b"CHYPER", 0xdead_beef);
        let sum = rsdp
            .as_slice()
            .iter()
--- a/acpi_tables/src/sdt.rs
+++ b/acpi_tables/src/sdt.rs
@@ -24,12 +24,12 @@ impl GenericAddress {
    }
 }

-pub struct SDT {
+pub struct Sdt {
    data: Vec<u8>,
 }

 #[allow(clippy::len_without_is_empty)]
-impl SDT {
+impl Sdt {
    pub fn new(
        signature: [u8; 4],
        length: u32,
@@ -53,7 +53,7 @@ impl SDT {
        assert_eq!(data.len(), 36);

        data.resize(length as usize, 0);
-        let mut sdt = SDT { data };
+        let mut sdt = Sdt { data };

        sdt.update_checksum();
        sdt
@@ -117,11 +117,11 @@ impl SDT {

 #[cfg(test)]
 mod tests {
-    use super::SDT;
+    use super::Sdt;

    #[test]
    fn test_sdt() {
-        let mut sdt = SDT::new(*b"TEST", 40, 1, *b"CLOUDH", *b"TESTTEST", 1);
+        let mut sdt = Sdt::new(*b"TEST", 40, 1, *b"CLOUDH", *b"TESTTEST", 1);
        let sum: u8 = sdt
            .as_slice()
            .iter()
--- a/api_client/src/lib.rs
+++ b/api_client/src/lib.rs
@@ -37,7 +37,7 @@ impl fmt::Display for Error {
 #[derive(Clone, Copy, Debug)]
 pub enum StatusCode {
    Continue,
-    OK,
+    Ok,
    NoContent,
    BadRequest,
    NotFound,
@@ -50,7 +50,7 @@ impl StatusCode {
    fn from_raw(code: usize) -> StatusCode {
        match code {
            100 => StatusCode::Continue,
-            200 => StatusCode::OK,
+            200 => StatusCode::Ok,
            204 => StatusCode::NoContent,
            400 => StatusCode::BadRequest,
            404 => StatusCode::NotFound,
@@ -69,18 +69,15 @@ impl StatusCode {
    fn is_server_error(self) -> bool {
        !matches!(
            self,
-            StatusCode::OK | StatusCode::Continue | StatusCode::NoContent
+            StatusCode::Ok | StatusCode::Continue | StatusCode::NoContent
        )
    }
 }

 fn get_header<'a>(res: &'a str, header: &'a str) -> Option<&'a str> {
    let header_str = format!("{}: ", header);
-    if let Some(o) = res.find(&header_str) {
-        Some(&res[o + header_str.len()..o + res[o..].find('\r').unwrap()])
-    } else {
-        None
-    }
+    res.find(&header_str)
+        .map(|o| &res[o + header_str.len()..o + res[o..].find('\r').unwrap()])
 }

 fn get_status_code(res: &str) -> Result<StatusCode, Error> {
--- a/arch/Cargo.toml
+++ b/arch/Cargo.toml
@@ -6,24 +6,21 @@ authors = ["The Chromium OS Authors"]
 [features]
 default = []
 acpi = ["acpi_tables"]
+tdx = []

 [dependencies]
+acpi_tables = { path = "../acpi_tables", optional = true }
 anyhow = "1.0"
-byteorder = "1.3.4"
+arch_gen = { path = "../arch_gen" }
+byteorder = "1.4.3"
 hypervisor = { path = "../hypervisor" }
-libc = "0.2.86"
+libc = "0.2.94"
+linux-loader = { version = "0.3.0", features = ["elf", "bzimage", "pe"] }
 log = "0.4.14"
 serde = {version = ">=1.0.27", features = ["rc"] }
 serde_derive = ">=1.0.27"
 serde_json = ">=1.0.9"
+thiserror = "1.0"
 vm-memory = { version = "0.5.0", features = ["backend-mmap"] }
 vm-migration = { path = "../vm-migration" }
-acpi_tables = { path = "../acpi_tables", optional = true }
-arch_gen = { path = "../arch_gen" }

-[dependencies.linux-loader]
-git = "https://github.com/rust-vmm/linux-loader"
-features = ["elf", "bzimage"]
-
-[dev-dependencies]
-rand = "0.8.3"
--- a/arch/src/aarch64/fdt.rs
+++ b/arch/src/aarch64/fdt.rs
@@ -16,12 +16,12 @@ use std::{io, result};
 use super::super::DeviceType;
 use super::super::InitramfsConfig;
 use super::get_fdt_addr;
-use super::gic::GICDevice;
+use super::gic::GicDevice;
 use super::layout::{
    FDT_MAX_SIZE, MEM_32BIT_DEVICES_SIZE, MEM_32BIT_DEVICES_START, PCI_MMCONFIG_SIZE,
    PCI_MMCONFIG_START,
 };
-use crate::aarch64::fdt::Error::CstringFDTTransform;
+use crate::aarch64::fdt::Error::CstringFdtTransform;
 use vm_memory::{Address, Bytes, GuestAddress, GuestMemory, GuestMemoryError, GuestMemoryMmap};

 // This is a value for uniquely identifying the FDT node declaring the interrupt controller.
@@ -30,6 +30,8 @@ const GIC_PHANDLE: u32 = 1;
 const MSI_PHANDLE: u32 = 2;
 // This is a value for uniquely identifying the FDT node containing the clock definition.
 const CLOCK_PHANDLE: u32 = 3;
+// This is a value for uniquely identifying the FDT node containing the gpio controller.
+const GPIO_PHANDLE: u32 = 4;

 // Read the documentation specified when appending the root node to the FDT.
 const ADDRESS_CELLS: u32 = 0x2;
@@ -45,6 +47,10 @@ const GIC_FDT_IRQ_TYPE_PPI: u32 = 1;
 const IRQ_TYPE_EDGE_RISING: u32 = 1;
 const IRQ_TYPE_LEVEL_HI: u32 = 4;

+// Keys and Buttons
+// System Power Down
+const KEY_POWER: u32 = 116;
+
 // This links to libfdt which handles the creation of the binary blob
 // flattened device tree (fdt) that is passed to the kernel and indicates
 // the hardware configuration of the machine.
@@ -62,7 +68,7 @@ extern "C" {
 }

 /// Trait for devices to be added to the Flattened Device Tree.
-pub trait DeviceInfoForFDT {
+pub trait DeviceInfoForFdt {
    /// Returns the address where this device will be loaded.
    fn addr(&self) -> u64;
    /// Returns the associated interrupt for this device.
@@ -75,27 +81,27 @@ pub trait DeviceInfoForFDT {
 #[derive(Debug)]
 pub enum Error {
    /// Failed to append node to the FDT.
-    AppendFDTNode(io::Error),
+    AppendFdtNode(io::Error),
    /// Failed to append a property to the FDT.
-    AppendFDTProperty(io::Error),
+    AppendFdtProperty(io::Error),
    /// Syscall for creating FDT failed.
-    CreateFDT(io::Error),
+    CreateFdt(io::Error),
    /// Failed to obtain a C style string.
-    CstringFDTTransform(NulError),
+    CstringFdtTransform(NulError),
    /// Failure in calling syscall for terminating this FDT.
-    FinishFDTReserveMap(io::Error),
+    FinishFdtReserveMap(io::Error),
    /// Failure in writing FDT in memory.
-    WriteFDTToMemory(GuestMemoryError),
+    WriteFdtToMemory(GuestMemoryError),
 }
 type Result<T> = result::Result<T, Error>;

 /// Creates the flattened device tree for this aarch64 VM.
-pub fn create_fdt<T: DeviceInfoForFDT + Clone + Debug, S: ::std::hash::BuildHasher>(
+pub fn create_fdt<T: DeviceInfoForFdt + Clone + Debug, S: ::std::hash::BuildHasher>(
    guest_mem: &GuestMemoryMmap,
    cmdline: &CStr,
    vcpu_mpidr: Vec<u64>,
    device_info: &HashMap<(DeviceType, String), T, S>,
-    gic_device: &dyn GICDevice,
+    gic_device: &dyn GicDevice,
    initrd: &Option<InitramfsConfig>,
    pci_space_address: &(u64, u64),
 ) -> Result<Vec<u8>> {
@@ -139,7 +145,7 @@ pub fn create_fdt<T: DeviceInfoForFDT + Clone + Debug, S: ::std::hash::BuildHash
    let fdt_address = GuestAddress(get_fdt_addr(&guest_mem));
    guest_mem
        .write_slice(fdt_final.as_slice(), fdt_address)
-        .map_err(Error::WriteFDTToMemory)?;
+        .map_err(Error::WriteFdtToMemory)?;
    Ok(fdt_final)
 }

@@ -149,7 +155,7 @@ fn allocate_fdt(fdt: &mut Vec<u8>) -> Result<()> {
    let mut fdt_ret = unsafe { fdt_create(fdt.as_mut_ptr() as *mut c_void, FDT_MAX_SIZE as c_int) };

    if fdt_ret != 0 {
-        return Err(Error::CreateFDT(io::Error::last_os_error()));
+        return Err(Error::CreateFdt(io::Error::last_os_error()));
    }

    // The flattened device trees created with fdt_create() contains a list of
@@ -160,7 +166,7 @@ fn allocate_fdt(fdt: &mut Vec<u8>) -> Result<()> {
    // Safe since we previously allocated this array.
    fdt_ret = unsafe { fdt_finish_reservemap(fdt.as_mut_ptr() as *mut c_void) };
    if fdt_ret != 0 {
-        return Err(Error::FinishFDTReserveMap(io::Error::last_os_error()));
+        return Err(Error::FinishFdtReserveMap(io::Error::last_os_error()));
    }
    Ok(())
 }
@@ -169,7 +175,7 @@ fn finish_fdt(from_fdt: &mut Vec<u8>, to_fdt: &mut Vec<u8>) -> Result<()> {
    // Safe since we allocated `fdt_final` and previously passed in its size.
    let mut fdt_ret = unsafe { fdt_finish(from_fdt.as_mut_ptr() as *mut c_void) };
    if fdt_ret != 0 {
-        return Err(Error::FinishFDTReserveMap(io::Error::last_os_error()));
+        return Err(Error::FinishFdtReserveMap(io::Error::last_os_error()));
    }

    // Safe because we allocated both arrays with the correct size.
@@ -181,25 +187,25 @@ fn finish_fdt(from_fdt: &mut Vec<u8>, to_fdt: &mut Vec<u8>) -> Result<()> {
        )
    };
    if fdt_ret != 0 {
-        return Err(Error::FinishFDTReserveMap(io::Error::last_os_error()));
+        return Err(Error::FinishFdtReserveMap(io::Error::last_os_error()));
    }

    // Safe since we allocated `to_fdt`.
    fdt_ret = unsafe { fdt_pack(to_fdt.as_mut_ptr() as *mut c_void) };
    if fdt_ret != 0 {
-        return Err(Error::FinishFDTReserveMap(io::Error::last_os_error()));
+        return Err(Error::FinishFdtReserveMap(io::Error::last_os_error()));
    }
    Ok(())
 }

 // Following are auxiliary functions for appending nodes to FDT.
 fn append_begin_node(fdt: &mut Vec<u8>, name: &str) -> Result<()> {
-    let cstr_name = CString::new(name).map_err(CstringFDTTransform)?;
+    let cstr_name = CString::new(name).map_err(CstringFdtTransform)?;

    // Safe because we allocated fdt and converted name to a CString
    let fdt_ret = unsafe { fdt_begin_node(fdt.as_mut_ptr() as *mut c_void, cstr_name.as_ptr()) };
    if fdt_ret != 0 {
-        return Err(Error::AppendFDTNode(io::Error::last_os_error()));
+        return Err(Error::AppendFdtNode(io::Error::last_os_error()));
    }
    Ok(())
 }
@@ -208,7 +214,7 @@ fn append_end_node(fdt: &mut Vec<u8>) -> Result<()> {
    // Safe because we allocated fdt.
    let fdt_ret = unsafe { fdt_end_node(fdt.as_mut_ptr() as *mut c_void) };
    if fdt_ret != 0 {
-        return Err(Error::AppendFDTNode(io::Error::last_os_error()));
+        return Err(Error::AppendFdtNode(io::Error::last_os_error()));
    }
    Ok(())
 }
@@ -223,13 +229,13 @@ fn append_property_u64(fdt: &mut Vec<u8>, name: &str, val: u64) -> Result<()> {
 }

 fn append_property_string(fdt: &mut Vec<u8>, name: &str, value: &str) -> Result<()> {
-    let cstr_value = CString::new(value).map_err(CstringFDTTransform)?;
+    let cstr_value = CString::new(value).map_err(CstringFdtTransform)?;
    append_property_cstring(fdt, name, &cstr_value)
 }

 fn append_property_cstring(fdt: &mut Vec<u8>, name: &str, cstr_value: &CStr) -> Result<()> {
    let value_bytes = cstr_value.to_bytes_with_nul();
-    let cstr_name = CString::new(name).map_err(CstringFDTTransform)?;
+    let cstr_name = CString::new(name).map_err(CstringFdtTransform)?;
    // Safe because we allocated fdt, converted name and value to CStrings
    let fdt_ret = unsafe {
        fdt_property(
@@ -240,13 +246,13 @@ fn append_property_cstring(fdt: &mut Vec<u8>, name: &str, cstr_value: &CStr) ->
        )
    };
    if fdt_ret != 0 {
-        return Err(Error::AppendFDTProperty(io::Error::last_os_error()));
+        return Err(Error::AppendFdtProperty(io::Error::last_os_error()));
    }
    Ok(())
 }

 fn append_property_null(fdt: &mut Vec<u8>, name: &str) -> Result<()> {
-    let cstr_name = CString::new(name).map_err(CstringFDTTransform)?;
+    let cstr_name = CString::new(name).map_err(CstringFdtTransform)?;

    // Safe because we allocated fdt, converted name to a CString
    let fdt_ret = unsafe {
@@ -258,13 +264,13 @@ fn append_property_null(fdt: &mut Vec<u8>, name: &str) -> Result<()> {
        )
    };
    if fdt_ret != 0 {
-        return Err(Error::AppendFDTProperty(io::Error::last_os_error()));
+        return Err(Error::AppendFdtProperty(io::Error::last_os_error()));
    }
    Ok(())
 }

 fn append_property(fdt: &mut Vec<u8>, name: &str, val: &[u8]) -> Result<()> {
-    let cstr_name = CString::new(name).map_err(CstringFDTTransform)?;
+    let cstr_name = CString::new(name).map_err(CstringFdtTransform)?;
    let val_ptr = val.as_ptr() as *const c_void;

    // Safe because we allocated fdt and converted name to a CString
@@ -277,7 +283,7 @@ fn append_property(fdt: &mut Vec<u8>, name: &str, val: &[u8]) -> Result<()> {
        )
    };
    if fdt_ret != 0 {
-        return Err(Error::AppendFDTProperty(io::Error::last_os_error()));
+        return Err(Error::AppendFdtProperty(io::Error::last_os_error()));
    }
    Ok(())
 }
@@ -374,7 +380,7 @@ fn create_chosen_node(
    Ok(())
 }

-fn create_gic_node(fdt: &mut Vec<u8>, gic_device: &dyn GICDevice) -> Result<()> {
+fn create_gic_node(fdt: &mut Vec<u8>, gic_device: &dyn GicDevice) -> Result<()> {
    let gic_reg_prop = generate_prop64(gic_device.device_properties());

    append_begin_node(fdt, "intc")?;
@@ -466,7 +472,7 @@ fn create_psci_node(fdt: &mut Vec<u8>) -> Result<()> {
    Ok(())
 }

-fn create_virtio_node<T: DeviceInfoForFDT + Clone + Debug>(
+fn create_virtio_node<T: DeviceInfoForFdt + Clone + Debug>(
    fdt: &mut Vec<u8>,
    dev_info: &T,
 ) -> Result<()> {
@@ -483,15 +489,16 @@ fn create_virtio_node<T: DeviceInfoForFDT + Clone + Debug>(
    Ok(())
 }

-fn create_serial_node<T: DeviceInfoForFDT + Clone + Debug>(
+fn create_serial_node<T: DeviceInfoForFdt + Clone + Debug>(
    fdt: &mut Vec<u8>,
    dev_info: &T,
 ) -> Result<()> {
+    let compatible = b"arm,pl011\0arm,primecell\0";
    let serial_reg_prop = generate_prop64(&[dev_info.addr(), dev_info.length()]);
    let irq = generate_prop32(&[GIC_FDT_IRQ_TYPE_SPI, dev_info.irq(), IRQ_TYPE_EDGE_RISING]);

-    append_begin_node(fdt, &format!("uart@{:x}", dev_info.addr()))?;
-    append_property_string(fdt, "compatible", "ns16550a")?;
+    append_begin_node(fdt, &format!("pl011@{:x}", dev_info.addr()))?;
+    append_property(fdt, "compatible", compatible)?;
    append_property(fdt, "reg", &serial_reg_prop)?;
    append_property_u32(fdt, "clocks", CLOCK_PHANDLE)?;
    append_property_string(fdt, "clock-names", "apb_pclk")?;
@@ -501,7 +508,7 @@ fn create_serial_node<T: DeviceInfoForFDT + Clone + Debug>(
    Ok(())
 }

-fn create_rtc_node<T: DeviceInfoForFDT + Clone + Debug>(
+fn create_rtc_node<T: DeviceInfoForFdt + Clone + Debug>(
    fdt: &mut Vec<u8>,
    dev_info: &T,
 ) -> Result<()> {
@@ -519,7 +526,42 @@ fn create_rtc_node<T: DeviceInfoForFDT + Clone + Debug>(
    Ok(())
 }

-fn create_devices_node<T: DeviceInfoForFDT + Clone + Debug, S: ::std::hash::BuildHasher>(
+fn create_gpio_node<T: DeviceInfoForFdt + Clone + Debug>(
+    fdt: &mut Vec<u8>,
+    dev_info: &T,
+) -> Result<()> {
+    // PL061 GPIO controller node
+    let compatible = b"arm,pl061\0arm,primecell\0";
+    let gpio_reg_prop = generate_prop64(&[dev_info.addr(), dev_info.length()]);
+    let irq = generate_prop32(&[GIC_FDT_IRQ_TYPE_SPI, dev_info.irq(), IRQ_TYPE_EDGE_RISING]);
+    append_begin_node(fdt, &format!("pl061@{:x}", dev_info.addr()))?;
+    append_property(fdt, "compatible", compatible)?;
+    append_property(fdt, "reg", &gpio_reg_prop)?;
+    append_property(fdt, "interrupts", &irq)?;
+    append_property_null(fdt, "gpio-controller")?;
+    append_property_u32(fdt, "#gpio-cells", 2)?;
+    append_property_u32(fdt, "clocks", CLOCK_PHANDLE)?;
+    append_property_string(fdt, "clock-names", "apb_pclk")?;
+    append_property_u32(fdt, "phandle", GPIO_PHANDLE)?;
+    append_end_node(fdt)?;
+
+    // gpio-keys node
+    append_begin_node(fdt, "/gpio-keys")?;
+    append_property_string(fdt, "compatible", "gpio-keys")?;
+    append_property_u32(fdt, "#size-cells", 0)?;
+    append_property_u32(fdt, "#address-cells", 1)?;
+    append_begin_node(fdt, "/gpio-keys/poweroff")?;
+    append_property_string(fdt, "label", "GPIO Key Poweroff")?;
+    append_property_u32(fdt, "linux,code", KEY_POWER)?;
+    let gpios = generate_prop32(&[GPIO_PHANDLE, 3, 0]);
+    append_property(fdt, "gpios", &gpios)?;
+    append_end_node(fdt)?;
+    append_end_node(fdt)?;
+
+    Ok(())
+}
+
+fn create_devices_node<T: DeviceInfoForFdt + Clone + Debug, S: ::std::hash::BuildHasher>(
    fdt: &mut Vec<u8>,
    dev_info: &HashMap<(DeviceType, String), T, S>,
 ) -> Result<()> {
@@ -528,7 +570,8 @@ fn create_devices_node<T: DeviceInfoForFDT + Clone + Debug, S: ::std::hash::Buil

    for ((device_type, _device_id), info) in dev_info {
        match device_type {
-            DeviceType::RTC => create_rtc_node(fdt, info)?,
+            DeviceType::Gpio => create_gpio_node(fdt, info)?,
+            DeviceType::Rtc => create_rtc_node(fdt, info)?,
            DeviceType::Serial => create_serial_node(fdt, info)?,
            DeviceType::Virtio(_) => {
                ordered_virtio_device.push(info);
--- a/arch/src/aarch64/gic/gicv3.rs
+++ b/arch/src/aarch64/gic/gicv3.rs
@@ -4,9 +4,9 @@
 pub mod kvm {
    use crate::aarch64::gic::dist_regs::{get_dist_regs, read_ctlr, set_dist_regs, write_ctlr};
    use crate::aarch64::gic::icc_regs::{get_icc_regs, set_icc_regs};
-    use crate::aarch64::gic::kvm::{save_pending_tables, KvmGICDevice};
+    use crate::aarch64::gic::kvm::{save_pending_tables, KvmGicDevice};
    use crate::aarch64::gic::redist_regs::{get_redist_regs, set_redist_regs};
-    use crate::aarch64::gic::GICDevice;
+    use crate::aarch64::gic::GicDevice;
    use crate::layout;
    use anyhow::anyhow;
    use hypervisor::kvm::kvm_bindings;
@@ -15,8 +15,7 @@ pub mod kvm {
    use std::sync::Arc;
    use std::{boxed::Box, result};
    use vm_migration::{
-        Migratable, MigratableError, Pausable, Snapshot, SnapshotDataSection, Snapshottable,
-        Transportable,
+        Migratable, MigratableError, Pausable, Snapshot, Snapshottable, Transportable,
    };

    /// Errors thrown while saving/restoring the GICv3.
@@ -37,14 +36,14 @@ pub mod kvm {
        /// Error in restoring GIC redistributor registers.
        RestoreRedistributorRegisters(crate::aarch64::gic::Error),
        /// Error in saving GIC CPU interface registers.
-        SaveICCRegisters(crate::aarch64::gic::Error),
+        SaveIccRegisters(crate::aarch64::gic::Error),
        /// Error in restoring GIC CPU interface registers.
-        RestoreICCRegisters(crate::aarch64::gic::Error),
+        RestoreIccRegisters(crate::aarch64::gic::Error),
    }

    type Result<T> = result::Result<T, Error>;

-    pub struct KvmGICv3 {
+    pub struct KvmGicV3 {
        /// The hypervisor agnostic device
        device: Arc<dyn hypervisor::Device>,

@@ -67,34 +66,34 @@ pub mod kvm {
        gicd_ctlr: u32,
    }

-    impl KvmGICv3 {
+    impl KvmGicV3 {
        // Unfortunately bindgen omits defines that are based on other defines.
        // See arch/arm64/include/uapi/asm/kvm.h file from the linux kernel.
        pub const SZ_64K: u64 = 0x0001_0000;
-        const KVM_VGIC_V3_DIST_SIZE: u64 = KvmGICv3::SZ_64K;
-        const KVM_VGIC_V3_REDIST_SIZE: u64 = (2 * KvmGICv3::SZ_64K);
+        const KVM_VGIC_V3_DIST_SIZE: u64 = KvmGicV3::SZ_64K;
+        const KVM_VGIC_V3_REDIST_SIZE: u64 = (2 * KvmGicV3::SZ_64K);

        // Device trees specific constants
        pub const ARCH_GIC_V3_MAINT_IRQ: u32 = 9;

        /// Get the address of the GIC distributor.
        pub fn get_dist_addr() -> u64 {
-            layout::MAPPED_IO_START - KvmGICv3::KVM_VGIC_V3_DIST_SIZE
+            layout::MAPPED_IO_START - KvmGicV3::KVM_VGIC_V3_DIST_SIZE
        }

        /// Get the size of the GIC distributor.
        pub fn get_dist_size() -> u64 {
-            KvmGICv3::KVM_VGIC_V3_DIST_SIZE
+            KvmGicV3::KVM_VGIC_V3_DIST_SIZE
        }

        /// Get the address of the GIC redistributors.
        pub fn get_redists_addr(vcpu_count: u64) -> u64 {
-            KvmGICv3::get_dist_addr() - KvmGICv3::get_redists_size(vcpu_count)
+            KvmGicV3::get_dist_addr() - KvmGicV3::get_redists_size(vcpu_count)
        }

        /// Get the size of the GIC redistributors.
        pub fn get_redists_size(vcpu_count: u64) -> u64 {
-            vcpu_count * KvmGICv3::KVM_VGIC_V3_REDIST_SIZE
+            vcpu_count * KvmGicV3::KVM_VGIC_V3_REDIST_SIZE
        }

        /// Save the state of GIC.
@@ -112,7 +111,7 @@ pub mod kvm {
                .map_err(Error::SaveRedistributorRegisters)?;

            let icc_state =
-                get_icc_regs(&self.device(), &gicr_typers).map_err(Error::SaveICCRegisters)?;
+                get_icc_regs(&self.device(), &gicr_typers).map_err(Error::SaveIccRegisters)?;

            Ok(Gicv3State {
                dist: dist_state,
@@ -134,13 +133,13 @@ pub mod kvm {
                .map_err(Error::RestoreRedistributorRegisters)?;

            set_icc_regs(&self.device(), &gicr_typers, &state.icc)
-                .map_err(Error::RestoreICCRegisters)?;
+                .map_err(Error::RestoreIccRegisters)?;

            Ok(())
        }
    }

-    impl GICDevice for KvmGICv3 {
+    impl GicDevice for KvmGicV3 {
        fn device(&self) -> &Arc<dyn hypervisor::Device> {
            &self.device
        }
@@ -150,7 +149,7 @@ pub mod kvm {
        }

        fn fdt_maint_irq(&self) -> u32 {
-            KvmGICv3::ARCH_GIC_V3_MAINT_IRQ
+            KvmGicV3::ARCH_GIC_V3_MAINT_IRQ
        }

        fn device_properties(&self) -> &[u64] {
@@ -170,7 +169,7 @@ pub mod kvm {
        }
    }

-    impl KvmGICDevice for KvmGICv3 {
+    impl KvmGicDevice for KvmGicV3 {
        fn version() -> u32 {
            kvm_bindings::kvm_device_type_KVM_DEV_TYPE_ARM_VGIC_V3
        }
@@ -178,15 +177,15 @@ pub mod kvm {
        fn create_device(
            device: Arc<dyn hypervisor::Device>,
            vcpu_count: u64,
-        ) -> Box<dyn GICDevice> {
-            Box::new(KvmGICv3 {
+        ) -> Box<dyn GicDevice> {
+            Box::new(KvmGicV3 {
                device,
                gicr_typers: vec![0; vcpu_count.try_into().unwrap()],
                properties: [
-                    KvmGICv3::get_dist_addr(),
-                    KvmGICv3::get_dist_size(),
-                    KvmGICv3::get_redists_addr(vcpu_count),
-                    KvmGICv3::get_redists_size(vcpu_count),
+                    KvmGicV3::get_dist_addr(),
+                    KvmGicV3::get_dist_size(),
+                    KvmGicV3::get_redists_addr(vcpu_count),
+                    KvmGicV3::get_redists_size(vcpu_count),
                ],
                vcpu_count,
            })
@@ -194,7 +193,7 @@ pub mod kvm {

        fn init_device_attributes(
            _vm: &Arc<dyn hypervisor::Vm>,
-            gic_device: &dyn GICDevice,
+            gic_device: &dyn GicDevice,
        ) -> crate::aarch64::gic::Result<()> {
            /* Setting up the distributor attribute.
             We are placing the GIC below 1GB so we need to substract the size of the distributor.
@@ -203,7 +202,7 @@ pub mod kvm {
                gic_device.device(),
                kvm_bindings::KVM_DEV_ARM_VGIC_GRP_ADDR,
                u64::from(kvm_bindings::KVM_VGIC_V3_ADDR_TYPE_DIST),
-                &KvmGICv3::get_dist_addr() as *const u64 as u64,
+                &KvmGicV3::get_dist_addr() as *const u64 as u64,
                0,
            )?;

@@ -214,7 +213,7 @@ pub mod kvm {
                gic_device.device(),
                kvm_bindings::KVM_DEV_ARM_VGIC_GRP_ADDR,
                u64::from(kvm_bindings::KVM_VGIC_V3_ADDR_TYPE_REDIST),
-                &KvmGICv3::get_redists_addr(gic_device.vcpu_count()) as *const u64 as u64,
+                &KvmGicV3::get_redists_addr(gic_device.vcpu_count()) as *const u64 as u64,
                0,
            )?;

@@ -223,53 +222,26 @@ pub mod kvm {
    }

    pub const GIC_V3_SNAPSHOT_ID: &str = "gic-v3";
-    impl Snapshottable for KvmGICv3 {
+    impl Snapshottable for KvmGicV3 {
        fn id(&self) -> String {
            GIC_V3_SNAPSHOT_ID.to_string()
        }

        fn snapshot(&mut self) -> std::result::Result<Snapshot, MigratableError> {
            let gicr_typers = self.gicr_typers.clone();
-            let snapshot = serde_json::to_vec(&self.state(&gicr_typers).unwrap())
-                .map_err(|e| MigratableError::Snapshot(e.into()))?;
-
-            let mut gic_v3_snapshot = Snapshot::new(self.id().as_str());
-            gic_v3_snapshot.add_data_section(SnapshotDataSection {
-                id: format!("{}-section", self.id()),
-                snapshot,
-            });
-
-            Ok(gic_v3_snapshot)
+            Snapshot::new_from_state(&self.id(), &self.state(&gicr_typers).unwrap())
        }

        fn restore(&mut self, snapshot: Snapshot) -> std::result::Result<(), MigratableError> {
-            if let Some(gic_v3_section) = snapshot
-                .snapshot_data
-                .get(&format!("{}-section", self.id()))
-            {
-                let gic_v3_state = match serde_json::from_slice(&gic_v3_section.snapshot) {
-                    Ok(state) => state,
-                    Err(error) => {
-                        return Err(MigratableError::Restore(anyhow!(
-                            "Could not deserialize GICv3 {}",
-                            error
-                        )))
-                    }
-                };
-
-                let gicr_typers = self.gicr_typers.clone();
-                return self.set_state(&gicr_typers, &gic_v3_state).map_err(|e| {
+            let gicr_typers = self.gicr_typers.clone();
+            self.set_state(&gicr_typers, &snapshot.to_state(&self.id())?)
+                .map_err(|e| {
                    MigratableError::Restore(anyhow!("Could not restore GICv3 state {:?}", e))
-                });
-            }
-
-            Err(MigratableError::Restore(anyhow!(
-                "Could not find GICv3 snapshot section"
-            )))
+                })
        }
    }

-    impl Pausable for KvmGICv3 {}
-    impl Transportable for KvmGICv3 {}
-    impl Migratable for KvmGICv3 {}
+    impl Pausable for KvmGicV3 {}
+    impl Transportable for KvmGicV3 {}
+    impl Migratable for KvmGicV3 {}
 }
--- a/arch/src/aarch64/gic/gicv3_its.rs
+++ b/arch/src/aarch64/gic/gicv3_its.rs
@@ -3,22 +3,18 @@

 pub mod kvm {
    use std::any::Any;
-    use std::convert::TryInto;
    use std::sync::Arc;
    use std::{boxed::Box, result};
    type Result<T> = result::Result<T, Error>;
-    use crate::aarch64::gic::gicv3::kvm::KvmGICv3;
-    use crate::aarch64::gic::kvm::KvmGICDevice;
-    use crate::aarch64::gic::{Error, GICDevice};
+    use crate::aarch64::gic::gicv3::kvm::KvmGicV3;
+    use crate::aarch64::gic::kvm::KvmGicDevice;
+    use crate::aarch64::gic::{Error, GicDevice};
    use hypervisor::kvm::kvm_bindings;

-    pub struct KvmGICv3ITS {
+    pub struct KvmGicV3Its {
        /// The hypervisor agnostic device
        device: Arc<dyn hypervisor::Device>,

-        /// Vector holding values of GICR_TYPER for each vCPU
-        gicr_typers: Vec<u64>,
-
        /// GIC device properties, to be used for setting up the fdt entry
        gic_properties: [u64; 4],

@@ -29,19 +25,19 @@ pub mod kvm {
        vcpu_count: u64,
    }

-    impl KvmGICv3ITS {
-        const KVM_VGIC_V3_ITS_SIZE: u64 = (2 * KvmGICv3::SZ_64K);
+    impl KvmGicV3Its {
+        const KVM_VGIC_V3_ITS_SIZE: u64 = (2 * KvmGicV3::SZ_64K);

        fn get_msi_size() -> u64 {
-            KvmGICv3ITS::KVM_VGIC_V3_ITS_SIZE
+            KvmGicV3Its::KVM_VGIC_V3_ITS_SIZE
        }

        fn get_msi_addr(vcpu_count: u64) -> u64 {
-            KvmGICv3::get_redists_addr(vcpu_count) - KvmGICv3ITS::get_msi_size()
+            KvmGicV3::get_redists_addr(vcpu_count) - KvmGicV3Its::get_msi_size()
        }
    }

-    impl GICDevice for KvmGICv3ITS {
+    impl GicDevice for KvmGicV3Its {
        fn device(&self) -> &Arc<dyn hypervisor::Device> {
            &self.device
        }
@@ -59,7 +55,7 @@ pub mod kvm {
        }

        fn fdt_maint_irq(&self) -> u32 {
-            KvmGICv3::ARCH_GIC_V3_MAINT_IRQ
+            KvmGicV3::ARCH_GIC_V3_MAINT_IRQ
        }

        fn msi_properties(&self) -> &[u64] {
@@ -74,36 +70,33 @@ pub mod kvm {
            self.vcpu_count
        }

-        fn set_gicr_typers(&mut self, gicr_typers: Vec<u64>) {
-            self.gicr_typers = gicr_typers;
-        }
+        fn set_gicr_typers(&mut self, _gicr_typers: Vec<u64>) {}

        fn as_any_concrete_mut(&mut self) -> &mut dyn Any {
            self
        }
    }

-    impl KvmGICDevice for KvmGICv3ITS {
+    impl KvmGicDevice for KvmGicV3Its {
        fn version() -> u32 {
-            KvmGICv3::version()
+            KvmGicV3::version()
        }

        fn create_device(
            device: Arc<dyn hypervisor::Device>,
            vcpu_count: u64,
-        ) -> Box<dyn GICDevice> {
-            Box::new(KvmGICv3ITS {
+        ) -> Box<dyn GicDevice> {
+            Box::new(KvmGicV3Its {
                device,
-                gicr_typers: vec![0; vcpu_count.try_into().unwrap()],
                gic_properties: [
-                    KvmGICv3::get_dist_addr(),
-                    KvmGICv3::get_dist_size(),
-                    KvmGICv3::get_redists_addr(vcpu_count),
-                    KvmGICv3::get_redists_size(vcpu_count),
+                    KvmGicV3::get_dist_addr(),
+                    KvmGicV3::get_dist_size(),
+                    KvmGicV3::get_redists_addr(vcpu_count),
+                    KvmGicV3::get_redists_size(vcpu_count),
                ],
                msi_properties: [
-                    KvmGICv3ITS::get_msi_addr(vcpu_count),
-                    KvmGICv3ITS::get_msi_size(),
+                    KvmGicV3Its::get_msi_addr(vcpu_count),
+                    KvmGicV3Its::get_msi_size(),
                ],
                vcpu_count,
            })
@@ -111,9 +104,9 @@ pub mod kvm {

        fn init_device_attributes(
            vm: &Arc<dyn hypervisor::Vm>,
-            gic_device: &dyn GICDevice,
+            gic_device: &dyn GicDevice,
        ) -> Result<()> {
-            KvmGICv3::init_device_attributes(vm, gic_device)?;
+            KvmGicV3::init_device_attributes(vm, gic_device)?;

            let mut its_device = kvm_bindings::kvm_create_device {
                type_: kvm_bindings::kvm_device_type_KVM_DEV_TYPE_ARM_VGIC_ITS,
@@ -123,13 +116,13 @@ pub mod kvm {

            let its_fd = vm
                .create_device(&mut its_device)
-                .map_err(Error::CreateGIC)?;
+                .map_err(Error::CreateGic)?;

            Self::set_device_attribute(
                &its_fd,
                kvm_bindings::KVM_DEV_ARM_VGIC_GRP_ADDR,
                u64::from(kvm_bindings::KVM_VGIC_ITS_ADDR_TYPE),
-                &KvmGICv3ITS::get_msi_addr(gic_device.vcpu_count()) as *const u64 as u64,
+                &KvmGicV3Its::get_msi_addr(gic_device.vcpu_count()) as *const u64 as u64,
                0,
            )?;

--- a/arch/src/aarch64/gic/mod.rs
+++ b/arch/src/aarch64/gic/mod.rs
@@ -18,7 +18,7 @@ use std::sync::Arc;
 #[derive(Debug)]
 pub enum Error {
    /// Error while calling KVM ioctl for setting up the global interrupt controller.
-    CreateGIC(hypervisor::HypervisorVmError),
+    CreateGic(hypervisor::HypervisorVmError),
    /// Error while setting device attributes for the GIC.
    SetDeviceAttribute(hypervisor::HypervisorDeviceError),
    /// Error while getting device attributes for the GIC.
@@ -26,7 +26,7 @@ pub enum Error {
 }
 type Result<T> = result::Result<T, Error>;

-pub trait GICDevice: Send {
+pub trait GicDevice: Send {
    /// Returns the hypervisor agnostic Device of the GIC device
    fn device(&self) -> &Arc<dyn hypervisor::Device>;

@@ -65,16 +65,16 @@ pub trait GICDevice: Send {
 }

 pub mod kvm {
-    use super::GICDevice;
+    use super::GicDevice;
    use super::Result;
-    use crate::aarch64::gic::gicv3_its::kvm::KvmGICv3ITS;
+    use crate::aarch64::gic::gicv3_its::kvm::KvmGicV3Its;
    use crate::layout;
    use hypervisor::kvm::kvm_bindings;
    use std::boxed::Box;
    use std::sync::Arc;

    /// Trait for GIC devices.
-    pub trait KvmGICDevice: Send + Sync + GICDevice {
+    pub trait KvmGicDevice: Send + Sync + GicDevice {
        /// Returns the GIC version of the device
        fn version() -> u32;

@@ -82,12 +82,12 @@ pub mod kvm {
        fn create_device(
            device: Arc<dyn hypervisor::Device>,
            vcpu_count: u64,
-        ) -> Box<dyn GICDevice>;
+        ) -> Box<dyn GicDevice>;

        /// Setup the device-specific attributes
        fn init_device_attributes(
            vm: &Arc<dyn hypervisor::Vm>,
-            gic_device: &dyn GICDevice,
+            gic_device: &dyn GicDevice,
        ) -> Result<()>;

        /// Initialize a GIC device
@@ -99,7 +99,7 @@ pub mod kvm {
            };

            vm.create_device(&mut gic_device)
-                .map_err(super::Error::CreateGIC)
+                .map_err(super::Error::CreateGic)
        }

        /// Set a GIC device attribute
@@ -111,10 +111,10 @@ pub mod kvm {
            flags: u32,
        ) -> Result<()> {
            let attr = kvm_bindings::kvm_device_attr {
+                flags,
                group,
                attr,
                addr,
-                flags,
            };
            device
                .set_device_attr(&attr)
@@ -132,10 +132,10 @@ pub mod kvm {
            flags: u32,
        ) -> Result<()> {
            let mut attr = kvm_bindings::kvm_device_attr {
+                flags,
                group,
                attr,
                addr,
-                flags,
            };
            device
                .get_device_attr(&mut attr)
@@ -145,7 +145,7 @@ pub mod kvm {
        }

        /// Finalize the setup of a GIC device
-        fn finalize_device(gic_device: &dyn GICDevice) -> Result<()> {
+        fn finalize_device(gic_device: &dyn GicDevice) -> Result<()> {
            /* We need to tell the kernel how many irqs to support with this vgic.
             * See the `layout` module for details.
             */
@@ -175,7 +175,7 @@ pub mod kvm {

        /// Method to initialize the GIC device
        #[allow(clippy::new_ret_no_self)]
-        fn new(vm: &Arc<dyn hypervisor::Vm>, vcpu_count: u64) -> Result<Box<dyn GICDevice>> {
+        fn new(vm: &Arc<dyn hypervisor::Vm>, vcpu_count: u64) -> Result<Box<dyn GicDevice>> {
            let vgic_fd = Self::init_device(vm)?;

            let device = Self::create_device(vgic_fd, vcpu_count);
@@ -190,9 +190,9 @@ pub mod kvm {

    /// Create a GICv3-ITS device.
    ///
-    pub fn create_gic(vm: &Arc<dyn hypervisor::Vm>, vcpu_count: u64) -> Result<Box<dyn GICDevice>> {
+    pub fn create_gic(vm: &Arc<dyn hypervisor::Vm>, vcpu_count: u64) -> Result<Box<dyn GicDevice>> {
        debug!("creating a GICv3-ITS");
-        KvmGICv3ITS::new(vm, vcpu_count)
+        KvmGicV3Its::new(vm, vcpu_count)
    }

    /// Function that saves RDIST pending tables into guest RAM.
--- a/arch/src/aarch64/layout.rs
+++ b/arch/src/aarch64/layout.rs
@@ -19,15 +19,14 @@
 // memory)   |                                                               |
 //           |                            DRAM                               |
 //           |                                                               |
-// 2GB       +---------------------------------------------------------------+
 //           |                                                               |
-//           |                           Reserved                            |
 //           |                                                               |
-// 1G+256M   +---------------------------------------------------------------+
+// 1GB       +---------------------------------------------------------------+
 //           |                                                               |
 //           |                        PCI MMCONFIG space                     |
 //           |                                                               |
-// 1GB       +---------------------------------------------------------------+
+// 768 M     +---------------------------------------------------------------+
+//           |                                                               |
 //           |                                                               |
 //           |                           PCI MMIO space                      |
 //           |                                                               |
@@ -36,8 +35,10 @@
 //           |                        Legacy devices space                   |
 //           |                                                               |
 // 144 M     +---------------------------------------------------------------|
-//           |                                                               |
 //           |                    Reserved (now GIC is here)                 |
+// 128 M     +---------------------------------------------------------------+
+//           |                                                               |
+//           |                          UEFI space                           |
 //           |                                                               |
 // 0GB       +---------------------------------------------------------------+
 //
@@ -45,26 +46,36 @@

 use vm_memory::GuestAddress;

+/// 0x0 ~ 0x800_0000 is reserved to uefi
+pub const UEFI_START: u64 = 0x0;
+pub const MEM_UEFI_START: GuestAddress = GuestAddress(0);
+pub const UEFI_SIZE: u64 = 0x0800_0000;
+
 /// Below this address will reside the GIC, above this address will reside the MMIO devices.
 pub const MAPPED_IO_START: u64 = 0x0900_0000;

 /// Space 0x0900_0000 ~ 0x1000_0000 is reserved for legacy devices.
 pub const LEGACY_SERIAL_MAPPED_IO_START: u64 = 0x0900_0000;
 pub const LEGACY_RTC_MAPPED_IO_START: u64 = 0x0901_0000;
+pub const LEGACY_GPIO_MAPPED_IO_START: u64 = 0x0902_0000;
+
+/// Space 0x0902_0000 ~ 0x903_0000 is reserved for pcie io address
+pub const MEM_PCI_IO_START: GuestAddress = GuestAddress(0x0902_0000);
+pub const MEM_PCI_IO_SIZE: u64 = 0x10000;

 /// Legacy space will be allocated at once whiling setting up legacy devices.
 pub const LEGACY_DEVICES_MAPPED_IO_SIZE: u64 = 0x0700_0000;

-/// Starting from 0x1000_0000 (256MiB), the 768MiB (ends at 1 GiB) is used for PCIE MMIO
+/// Starting from 0x1000_0000 (256MiB) to 0x3000_0000 (768MiB) is used for PCIE MMIO
 pub const MEM_32BIT_DEVICES_START: GuestAddress = GuestAddress(0x1000_0000);
-pub const MEM_32BIT_DEVICES_SIZE: u64 = 0x3000_0000;
+pub const MEM_32BIT_DEVICES_SIZE: u64 = 0x2000_0000;

 /// PCI MMCONFIG space (start: after the device space at 1 GiB, length: 256MiB)
-pub const PCI_MMCONFIG_START: GuestAddress = GuestAddress(0x4000_0000);
+pub const PCI_MMCONFIG_START: GuestAddress = GuestAddress(0x3000_0000);
 pub const PCI_MMCONFIG_SIZE: u64 = 256 << 20;

 /// Start of RAM on 64 bit ARM.
-pub const RAM_64BIT_START: u64 = 0x8000_0000;
+pub const RAM_64BIT_START: u64 = 0x4000_0000;

 /// Kernel command line maximum size.
 /// As per `arch/arm64/include/uapi/asm/setup.h`.
@@ -73,14 +84,18 @@ pub const CMDLINE_MAX_SIZE: usize = 2048;
 /// Maximum size of the device tree blob as specified in https://www.kernel.org/doc/Documentation/arm64/booting.txt.
 pub const FDT_MAX_SIZE: usize = 0x20_0000;

+/// Put ACPI table above dtb
+pub const ACPI_START: u64 = RAM_64BIT_START + FDT_MAX_SIZE as u64;
+pub const RSDP_POINTER: GuestAddress = GuestAddress(ACPI_START);
+
 // As per virt/kvm/arm/vgic/vgic-kvm-device.c we need
 // the number of interrupts our GIC will support to be:
 // * bigger than 32
 // * less than 1023 and
 // * a multiple of 32.
-// We are setting up our interrupt controller to support a maximum of 128 interrupts.
+// We are setting up our interrupt controller to support a maximum of 256 interrupts.
 /// First usable interrupt on aarch64.
-pub const IRQ_BASE: u32 = 32;
+pub const IRQ_BASE: u32 = 0;

 /// Last usable interrupt on aarch64.
-pub const IRQ_MAX: u32 = 159;
+pub const IRQ_MAX: u32 = 255;
--- a/arch/src/aarch64/mod.rs
+++ b/arch/src/aarch64/mod.rs
@@ -11,10 +11,10 @@ pub mod layout;
 /// Logic for configuring aarch64 registers.
 pub mod regs;

-pub use self::fdt::DeviceInfoForFDT;
+pub use self::fdt::DeviceInfoForFdt;
 use crate::DeviceType;
 use crate::RegionType;
-use aarch64::gic::GICDevice;
+use aarch64::gic::GicDevice;
 use std::collections::HashMap;
 use std::ffi::CStr;
 use std::fmt::Debug;
@@ -28,19 +28,19 @@ use vm_memory::{
 #[derive(Debug)]
 pub enum Error {
    /// Failed to create a FDT.
-    SetupFDT(fdt::Error),
+    SetupFdt(fdt::Error),

    /// Failed to create a GIC.
-    SetupGIC(gic::Error),
+    SetupGic(gic::Error),

    /// Failed to compute the initramfs address.
    InitramfsAddress,

    /// Error configuring the general purpose registers
-    REGSConfiguration(regs::Error),
+    RegsConfiguration(regs::Error),

    /// Error configuring the MPIDR register
-    VcpuRegMPIDR(hypervisor::HypervisorCpuError),
+    VcpuRegMpidr(hypervisor::HypervisorCpuError),
 }

 impl From<Error> for super::Error {
@@ -71,43 +71,39 @@ pub fn configure_vcpu(
            kernel_entry_point.entry_addr.raw_value(),
            &vm_memory.memory(),
        )
-        .map_err(Error::REGSConfiguration)?;
+        .map_err(Error::RegsConfiguration)?;
    }

-    let mpidr = fd.read_mpidr().map_err(Error::VcpuRegMPIDR)?;
+    let mpidr = fd.read_mpidr().map_err(Error::VcpuRegMpidr)?;
    Ok(mpidr)
 }

 pub fn arch_memory_regions(size: GuestUsize) -> Vec<(GuestAddress, usize, RegionType)> {
-    let mut regions = Vec::new();
-    // 0 ~ 256 MiB: Reserved
-    regions.push((
-        GuestAddress(0),
-        layout::MEM_32BIT_DEVICES_START.0 as usize,
-        RegionType::Reserved,
-    ));
-
-    // 256 MiB ~ 1 G: MMIO space
-    regions.push((
-        layout::MEM_32BIT_DEVICES_START,
-        layout::MEM_32BIT_DEVICES_SIZE as usize,
-        RegionType::SubRegion,
-    ));
-
-    // 1G  ~ 2G: reserved. The leading 256M for PCIe MMCONFIG space
-    regions.push((
-        layout::PCI_MMCONFIG_START,
-        (layout::RAM_64BIT_START - layout::PCI_MMCONFIG_START.0) as usize,
-        RegionType::Reserved,
-    ));
-
-    regions.push((
-        GuestAddress(layout::RAM_64BIT_START),
-        size as usize,
-        RegionType::Ram,
-    ));
-
-    regions
+    vec![
+        // 0 ~ 256 MiB: Reserved
+        (
+            GuestAddress(0),
+            layout::MEM_32BIT_DEVICES_START.0 as usize,
+            RegionType::Reserved,
+        ),
+        // 256 MiB ~ 1 G: MMIO space
+        (
+            layout::MEM_32BIT_DEVICES_START,
+            layout::MEM_32BIT_DEVICES_SIZE as usize,
+            RegionType::SubRegion,
+        ),
+        // 1G  ~ 2G: reserved. The leading 256M for PCIe MMCONFIG space
+        (
+            layout::PCI_MMCONFIG_START,
+            (layout::RAM_64BIT_START - layout::PCI_MMCONFIG_START.0) as usize,
+            RegionType::Reserved,
+        ),
+        (
+            GuestAddress(layout::RAM_64BIT_START),
+            size as usize,
+            RegionType::Ram,
+        ),
+    ]
 }

 /// Configures the system and should be called once per vm before starting vcpu threads.
@@ -117,7 +113,7 @@ pub fn arch_memory_regions(size: GuestUsize) -> Vec<(GuestAddress, usize, Region
 /// * `guest_mem` - The memory to be used by the guest.
 /// * `num_cpus` - Number of virtual CPUs the guest will have.
 #[allow(clippy::too_many_arguments)]
-pub fn configure_system<T: DeviceInfoForFDT + Clone + Debug, S: ::std::hash::BuildHasher>(
+pub fn configure_system<T: DeviceInfoForFdt + Clone + Debug, S: ::std::hash::BuildHasher>(
    vm: &Arc<dyn hypervisor::Vm>,
    guest_mem: &GuestMemoryMmap,
    cmdline_cstring: &CStr,
@@ -126,8 +122,8 @@ pub fn configure_system<T: DeviceInfoForFDT + Clone + Debug, S: ::std::hash::Bui
    device_info: &HashMap<(DeviceType, String), T, S>,
    initrd: &Option<super::InitramfsConfig>,
    pci_space_address: &(u64, u64),
-) -> super::Result<Box<dyn GICDevice>> {
-    let gic_device = gic::kvm::create_gic(vm, vcpu_count).map_err(Error::SetupGIC)?;
+) -> super::Result<Box<dyn GicDevice>> {
+    let gic_device = gic::kvm::create_gic(vm, vcpu_count).map_err(Error::SetupGic)?;

    fdt::create_fdt(
        guest_mem,
@@ -138,7 +134,7 @@ pub fn configure_system<T: DeviceInfoForFDT + Clone + Debug, S: ::std::hash::Bui
        initrd,
        pci_space_address,
    )
-    .map_err(Error::SetupFDT)?;
+    .map_err(Error::SetupFdt)?;

    Ok(gic_device)
 }
--- a/arch/src/lib.rs
+++ b/arch/src/lib.rs
@@ -6,13 +6,7 @@

 //! Implements platform specific functionality.
 //! Supported platforms: x86_64, aarch64.
-#![allow(
-    clippy::unreadable_literal,
-    clippy::redundant_static_lifetimes,
-    clippy::cast_lossless,
-    clippy::transmute_ptr_to_ptr,
-    clippy::cast_ptr_alignment
-)]
+#![allow(clippy::transmute_ptr_to_ptr, clippy::redundant_static_lifetimes)]

 extern crate anyhow;
 extern crate byteorder;
@@ -31,6 +25,7 @@ extern crate vm_migration;
 #[macro_use]
 extern crate serde_derive;
 extern crate serde_json;
+extern crate thiserror;

 use std::fmt;
 use std::result;
@@ -61,7 +56,7 @@ pub enum Error {
    /// Error writing module entry to guest memory.
    ModlistSetup(vm_memory::GuestMemoryError),
    /// RSDP Beyond Guest Memory
-    RSDPPastRamEnd,
+    RsdpPastRamEnd,
 }

 /// Type for returning public functions outcome.
@@ -92,7 +87,7 @@ pub mod aarch64;

 #[cfg(target_arch = "aarch64")]
 pub use aarch64::{
-    arch_memory_regions, configure_system, configure_vcpu, fdt::DeviceInfoForFDT,
+    arch_memory_regions, configure_system, configure_vcpu, fdt::DeviceInfoForFdt,
    get_host_cpu_phys_bits, get_kernel_start, initramfs_load_addr, layout,
    layout::CMDLINE_MAX_SIZE, layout::IRQ_BASE, layout::IRQ_MAX, EntryPoint,
 };
@@ -133,7 +128,10 @@ pub enum DeviceType {
    Serial,
    /// Device Type: RTC.
    #[cfg(target_arch = "aarch64")]
-    RTC,
+    Rtc,
+    /// Device Type: GPIO.
+    #[cfg(target_arch = "aarch64")]
+    Gpio,
 }

 /// Default (smallest) memory page size for the supported architectures.
@@ -148,13 +146,13 @@ impl fmt::Display for DeviceType {
 /// Structure to describe MMIO device information
 #[derive(Clone, Debug)]
 #[cfg(target_arch = "aarch64")]
-pub struct MMIODeviceInfo {
+pub struct MmioDeviceInfo {
    pub addr: u64,
    pub irq: u32,
 }

 #[cfg(target_arch = "aarch64")]
-impl DeviceInfoForFDT for MMIODeviceInfo {
+impl DeviceInfoForFdt for MmioDeviceInfo {
    fn addr(&self) -> u64 {
        self.addr
    }
--- a/arch/src/x86_64/interrupts.rs
+++ b/arch/src/x86_64/interrupts.rs
@@ -5,15 +5,13 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE-BSD-3-Clause file.

+use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt};
+use hypervisor::x86_64::LapicState;
 use std::io::Cursor;
 use std::mem;
 use std::result;
 use std::sync::Arc;

-use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt};
-
-use hypervisor::x86_64::LapicState;
-
 #[derive(Debug)]
 pub enum Error {
    GetLapic(anyhow::Error),
@@ -82,12 +80,7 @@ pub fn set_lint(vcpu: &Arc<dyn hypervisor::Vcpu>) -> Result<()> {
 }

 #[cfg(test)]
-#[cfg(feature = "kvm")]
 mod tests {
-
-    extern crate rand;
-    use self::rand::Rng;
-
    use super::*;

    const KVM_APIC_REG_SIZE: usize = 0x400;
@@ -111,8 +104,15 @@ mod tests {

    #[test]
    fn test_apic_delivery_mode() {
-        let mut rng = rand::thread_rng();
-        let mut v: Vec<u32> = (0..20).map(|_| rng.gen::<u32>()).collect();
+        let mut v: Vec<u32> = Vec::new();
+        v.resize(20, 0);
+
+        unsafe {
+            assert_eq!(
+                libc::getrandom(v.as_mut_ptr() as *mut _ as *mut libc::c_void, 80, 0),
+                80
+            );
+        }

        v.iter_mut()
            .for_each(|x| *x = set_apic_delivery_mode(*x, 2));
--- a/arch/src/x86_64/mod.rs
+++ b/arch/src/x86_64/mod.rs
@@ -25,6 +25,8 @@ use vm_memory::{
 };
 mod smbios;
 use std::arch::x86_64;
+#[cfg(feature = "tdx")]
+pub mod tdx;

 #[derive(Debug, Copy, Clone)]
 pub enum BootProtocol {
@@ -145,16 +147,16 @@ pub enum Error {
    MpTableSetup(mptable::Error),

    /// Error configuring the general purpose registers
-    REGSConfiguration(regs::Error),
+    RegsConfiguration(regs::Error),

    /// Error configuring the special registers
-    SREGSConfiguration(regs::Error),
+    SregsConfiguration(regs::Error),

    /// Error configuring the floating point related registers
-    FPUConfiguration(regs::Error),
+    FpuConfiguration(regs::Error),

    /// Error configuring the MSR registers
-    MSRSConfiguration(regs::Error),
+    MsrsConfiguration(regs::Error),

    /// Failed to set supported CPUs.
    SetSupportedCpusFailed(anyhow::Error),
@@ -181,7 +183,7 @@ impl From<Error> for super::Error {
    }
 }

-#[allow(dead_code)]
+#[allow(dead_code, clippy::upper_case_acronyms)]
 #[derive(Copy, Clone)]
 pub enum CpuidReg {
    EAX,
@@ -346,7 +348,7 @@ pub fn configure_vcpu(
        fd.enable_hyperv_synic().unwrap();
    }

-    regs::setup_msrs(fd).map_err(Error::MSRSConfiguration)?;
+    regs::setup_msrs(fd).map_err(Error::MsrsConfiguration)?;
    if let Some(kernel_entry_point) = kernel_entry_point {
        // Safe to unwrap because this method is called after the VM is configured
        regs::setup_regs(
@@ -356,10 +358,10 @@ pub fn configure_vcpu(
            layout::ZERO_PAGE_START.raw_value(),
            kernel_entry_point.protocol,
        )
-        .map_err(Error::REGSConfiguration)?;
-        regs::setup_fpu(fd).map_err(Error::FPUConfiguration)?;
+        .map_err(Error::RegsConfiguration)?;
+        regs::setup_fpu(fd).map_err(Error::FpuConfiguration)?;
        regs::setup_sregs(&vm_memory.memory(), fd, kernel_entry_point.protocol)
-            .map_err(Error::SREGSConfiguration)?;
+            .map_err(Error::SregsConfiguration)?;
    }
    interrupts::set_lint(fd).map_err(|e| Error::LocalIntConfiguration(e.into()))?;
    Ok(())
@@ -442,7 +444,7 @@ pub fn configure_system(
    // Check that the RAM is not smaller than the RSDP start address
    if let Some(rsdp_addr) = rsdp_addr {
        if rsdp_addr.0 > guest_mem.last_addr().0 {
-            return Err(super::Error::RSDPPastRamEnd);
+            return Err(super::Error::RsdpPastRamEnd);
        }
    }

@@ -457,6 +459,7 @@ pub fn configure_system(
            )?;
        }
        BootProtocol::LinuxBoot => {
+            error!("Using deprecated LinuxBoot protocol: Please configure your kernel with CONFIG_PVH=y and supply the `vmlinux` file to `--kernel`");
            configure_64bit_boot(
                guest_mem,
                cmdline_addr,
--- a/arch/src/x86_64/regs.rs
+++ b/arch/src/x86_64/regs.rs
@@ -25,25 +25,25 @@ pub enum Error {
    /// Failed to set base registers for this CPU.
    SetBaseRegisters(hypervisor::HypervisorCpuError),
    /// Failed to configure the FPU.
-    SetFPURegisters(hypervisor::HypervisorCpuError),
+    SetFpuRegisters(hypervisor::HypervisorCpuError),
    /// Setting up MSRs failed.
    SetModelSpecificRegisters(hypervisor::HypervisorCpuError),
    /// Failed to set SREGs for this CPU.
    SetStatusRegisters(hypervisor::HypervisorCpuError),
    /// Checking the GDT address failed.
-    CheckGDTAddr,
+    CheckGdtAddr,
    /// Writing the GDT to RAM failed.
-    WriteGDT(GuestMemoryError),
+    WriteGdt(GuestMemoryError),
    /// Writing the IDT to RAM failed.
-    WriteIDT(GuestMemoryError),
+    WriteIdt(GuestMemoryError),
    /// Writing PDPTE to RAM failed.
-    WritePDPTEAddress(GuestMemoryError),
+    WritePdpteAddress(GuestMemoryError),
    /// Writing PDE to RAM failed.
-    WritePDEAddress(GuestMemoryError),
+    WritePdeAddress(GuestMemoryError),
    /// Writing PML4 to RAM failed.
-    WritePML4Address(GuestMemoryError),
+    WritePml4Address(GuestMemoryError),
    /// Writing PML5 to RAM failed.
-    WritePML5Address(GuestMemoryError),
+    WritePml5Address(GuestMemoryError),
 }

 pub type Result<T> = result::Result<T, Error>;
@@ -60,7 +60,7 @@ pub fn setup_fpu(vcpu: &Arc<dyn hypervisor::Vcpu>) -> Result<()> {
        ..Default::default()
    };

-    vcpu.set_fpu(&fpu).map_err(Error::SetFPURegisters)
+    vcpu.set_fpu(&fpu).map_err(Error::SetFpuRegisters)
 }

 /// Configure Model Specific Registers (MSRs) for a given CPU.
@@ -140,8 +140,8 @@ fn write_gdt_table(table: &[u64], guest_mem: &GuestMemoryMmap) -> Result<()> {
    for (index, entry) in table.iter().enumerate() {
        let addr = guest_mem
            .checked_offset(boot_gdt_addr, index * mem::size_of::<u64>())
-            .ok_or(Error::CheckGDTAddr)?;
-        guest_mem.write_obj(*entry, addr).map_err(Error::WriteGDT)?;
+            .ok_or(Error::CheckGdtAddr)?;
+        guest_mem.write_obj(*entry, addr).map_err(Error::WriteGdt)?;
    }
    Ok(())
 }
@@ -150,7 +150,7 @@ fn write_idt_value(val: u64, guest_mem: &GuestMemoryMmap) -> Result<()> {
    let boot_idt_addr = BOOT_IDT_START;
    guest_mem
        .write_obj(val, boot_idt_addr)
-        .map_err(Error::WriteIDT)
+        .map_err(Error::WriteIdt)
 }

 pub fn configure_segments_and_sregs(
@@ -220,7 +220,7 @@ pub fn setup_page_tables(mem: &GuestMemoryMmap, sregs: &mut SpecialRegisters) ->
    if unsafe { std::arch::x86_64::__cpuid(7).ecx } & (1 << 16) != 0 {
        // Entry covering VA [0..256TB)
        mem.write_obj(PML4_START.raw_value() | 0x03, PML5_START)
-            .map_err(Error::WritePML5Address)?;
+            .map_err(Error::WritePml5Address)?;

        sregs.cr3 = PML5_START.raw_value();
        sregs.cr4 |= CR4_LA57;
@@ -230,17 +230,17 @@ pub fn setup_page_tables(mem: &GuestMemoryMmap, sregs: &mut SpecialRegisters) ->

    // Entry covering VA [0..512GB)
    mem.write_obj(PDPTE_START.raw_value() | 0x03, PML4_START)
-        .map_err(Error::WritePML4Address)?;
+        .map_err(Error::WritePml4Address)?;

    // Entry covering VA [0..1GB)
    mem.write_obj(PDE_START.raw_value() | 0x03, PDPTE_START)
-        .map_err(Error::WritePDPTEAddress)?;
+        .map_err(Error::WritePdpteAddress)?;

    // 512 2MB entries together covering VA [0..1GB). Note we are assuming
    // CPU supports 2MB pages (/proc/cpuinfo has 'pse'). All modern CPUs do.
    for i in 0..512 {
        mem.write_obj((i << 21) + 0x83u64, PDE_START.unchecked_add(i * 8))
-            .map_err(Error::WritePDEAddress)?;
+            .map_err(Error::WritePdeAddress)?;
    }

    sregs.cr4 |= CR4_PAE;
--- a/arch/src/x86_64/tdx/mod.rs
+++ b/arch/src/x86_64/tdx/mod.rs
@@ -0,0 +1,318 @@
+// Copyright © 2021 Intel Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+use std::fs::File;
+use std::io::{Read, Seek, SeekFrom};
+use thiserror::Error;
+use vm_memory::{ByteValued, Bytes, GuestAddress, GuestMemoryError, GuestMemoryMmap};
+
+#[derive(Error, Debug)]
+pub enum TdvfError {
+    #[error("Failed read TDVF descriptor: {0}")]
+    ReadDescriptor(#[source] std::io::Error),
+    #[error("Failed read TDVF descriptor offset: {0}")]
+    ReadDescriptorOffset(#[source] std::io::Error),
+    #[error("Invalid descriptor signature")]
+    InvalidDescriptorSignature,
+    #[error("Invalid descriptor size")]
+    InvalidDescriptorSize,
+    #[error("Invalid descriptor version")]
+    InvalidDescriptorVersion,
+    #[error("Failed to write HOB details to guest memory: {0}")]
+    GuestMemoryWriteHob(#[source] GuestMemoryError),
+}
+
+// TDVF_DESCRIPTOR
+#[repr(packed)]
+pub struct TdvfDescriptor {
+    signature: [u8; 4],
+    length: u32,
+    version: u32,
+    num_sections: u32, // NumberOfSectionEntry
+}
+
+// TDVF_SECTION
+#[repr(packed)]
+#[derive(Clone, Copy, Default, Debug)]
+pub struct TdvfSection {
+    pub data_offset: u32,
+    pub data_size: u32, // RawDataSize
+    pub address: u64,   // MemoryAddress
+    pub size: u64,      // MemoryDataSize
+    pub r#type: TdvfSectionType,
+    pub attributes: u32,
+}
+
+#[repr(u32)]
+#[derive(Clone, Copy, Debug)]
+pub enum TdvfSectionType {
+    Bfv,
+    Cfv,
+    TdHob,
+    TempMem,
+    Reserved = 0xffffffff,
+}
+
+impl Default for TdvfSectionType {
+    fn default() -> Self {
+        TdvfSectionType::Reserved
+    }
+}
+
+pub fn parse_tdvf_sections(file: &mut File) -> Result<Vec<TdvfSection>, TdvfError> {
+    // The 32-bit offset to the TDVF metadata is located 32 bytes from
+    // the end of the file.
+    // See "TDVF Metadata Pointer" in "TDX Virtual Firmware Design Guide
+    file.seek(SeekFrom::End(-0x20))
+        .map_err(TdvfError::ReadDescriptorOffset)?;
+
+    let mut descriptor_offset: [u8; 4] = [0; 4];
+    file.read_exact(&mut descriptor_offset)
+        .map_err(TdvfError::ReadDescriptorOffset)?;
+    let descriptor_offset = u32::from_le_bytes(descriptor_offset) as u64;
+
+    file.seek(SeekFrom::Start(descriptor_offset))
+        .map_err(TdvfError::ReadDescriptor)?;
+
+    let mut descriptor: TdvfDescriptor = unsafe { std::mem::zeroed() };
+    // Safe as we read exactly the size of the descriptor header
+    file.read_exact(unsafe {
+        std::slice::from_raw_parts_mut(
+            &mut descriptor as *mut _ as *mut u8,
+            std::mem::size_of::<TdvfDescriptor>(),
+        )
+    })
+    .map_err(TdvfError::ReadDescriptor)?;
+
+    if &descriptor.signature != b"TDVF" {
+        return Err(TdvfError::InvalidDescriptorSignature);
+    }
+
+    if descriptor.length as usize
+        != std::mem::size_of::<TdvfDescriptor>()
+            + std::mem::size_of::<TdvfSection>() * descriptor.num_sections as usize
+    {
+        return Err(TdvfError::InvalidDescriptorSize);
+    }
+
+    if descriptor.version != 1 {
+        return Err(TdvfError::InvalidDescriptorVersion);
+    }
+
+    let mut sections = Vec::new();
+    sections.resize_with(descriptor.num_sections as usize, TdvfSection::default);
+
+    // Safe as we read exactly the advertised sections
+    file.read_exact(unsafe {
+        std::slice::from_raw_parts_mut(
+            sections.as_mut_ptr() as *mut u8,
+            descriptor.num_sections as usize * std::mem::size_of::<TdvfSection>(),
+        )
+    })
+    .map_err(TdvfError::ReadDescriptor)?;
+
+    Ok(sections)
+}
+
+#[repr(u16)]
+#[derive(Copy, Clone, Debug)]
+enum HobType {
+    Handoff = 0x1,
+    ResourceDescriptor = 0x3,
+    Unused = 0xfffe,
+    EndOfHobList = 0xffff,
+}
+
+impl Default for HobType {
+    fn default() -> Self {
+        HobType::Unused
+    }
+}
+
+#[repr(C)]
+#[derive(Copy, Clone, Default, Debug)]
+struct HobHeader {
+    r#type: HobType,
+    length: u16,
+    reserved: u32,
+}
+unsafe impl ByteValued for HobHeader {}
+
+#[repr(C)]
+#[derive(Copy, Clone, Default, Debug)]
+struct HobHandoffInfoTable {
+    header: HobHeader,
+    version: u32,
+    efi_memory_top: u64,
+    efi_memory_bottom: u64,
+    efi_free_memory_top: u64,
+    efi_free_memory_bottom: u64,
+    efi_end_of_hob_list: u64,
+}
+unsafe impl ByteValued for HobHandoffInfoTable {}
+
+#[repr(C)]
+#[derive(Copy, Clone, Default, Debug)]
+struct EfiGuid {
+    data1: u32,
+    data2: u16,
+    data3: u16,
+    data4: [u8; 8],
+}
+
+#[repr(C)]
+#[derive(Copy, Clone, Default, Debug)]
+struct HobResourceDescriptor {
+    header: HobHeader,
+    owner: EfiGuid,
+    resource_type: u32,
+    resource_attribute: u32,
+    physical_start: u64,
+    resource_length: u64,
+}
+unsafe impl ByteValued for HobResourceDescriptor {}
+
+pub struct TdHob {
+    start_offset: u64,
+    current_offset: u64,
+}
+
+fn align_hob(v: u64) -> u64 {
+    (v + 7) / 8 * 8
+}
+
+impl TdHob {
+    fn update_offset<T>(&mut self) {
+        self.current_offset = align_hob(self.current_offset + std::mem::size_of::<T>() as u64)
+    }
+
+    pub fn start(offset: u64) -> TdHob {
+        // Leave a gap to place the HandoffTable at the start as it can only be filled in later
+        let mut hob = TdHob {
+            start_offset: offset,
+            current_offset: offset,
+        };
+        hob.update_offset::<HobHandoffInfoTable>();
+        hob
+    }
+
+    pub fn finish(&mut self, mem: &GuestMemoryMmap) -> Result<(), TdvfError> {
+        // Write end
+        let end = HobHeader {
+            r#type: HobType::EndOfHobList,
+            length: std::mem::size_of::<HobHeader>() as u16,
+            reserved: 0,
+        };
+        info!("Writing HOB end {:x} {:x?}", self.current_offset, end);
+        mem.write_obj(end, GuestAddress(self.current_offset))
+            .map_err(TdvfError::GuestMemoryWriteHob)?;
+        self.update_offset::<HobHeader>();
+
+        // Write handoff, delayed as it needs end of HOB list
+        let efi_end_of_hob_list = self.current_offset;
+        let handoff = HobHandoffInfoTable {
+            header: HobHeader {
+                r#type: HobType::Handoff,
+                length: std::mem::size_of::<HobHandoffInfoTable>() as u16,
+                reserved: 0,
+            },
+            version: 0x9,
+            efi_memory_top: 0,
+            efi_memory_bottom: 0,
+            efi_free_memory_top: 0,
+            efi_free_memory_bottom: 0,
+            efi_end_of_hob_list,
+        };
+        info!("Writing HOB start {:x} {:x?}", self.start_offset, handoff);
+        mem.write_obj(handoff, GuestAddress(self.start_offset))
+            .map_err(TdvfError::GuestMemoryWriteHob)
+    }
+
+    pub fn add_resource(
+        &mut self,
+        mem: &GuestMemoryMmap,
+        physical_start: u64,
+        resource_length: u64,
+        resource_type: u32,
+        resource_attribute: u32,
+    ) -> Result<(), TdvfError> {
+        let resource_descriptor = HobResourceDescriptor {
+            header: HobHeader {
+                r#type: HobType::ResourceDescriptor,
+                length: std::mem::size_of::<HobResourceDescriptor>() as u16,
+                reserved: 0,
+            },
+            owner: EfiGuid::default(),
+            resource_type,
+            resource_attribute,
+            physical_start,
+            resource_length,
+        };
+        info!(
+            "Writing HOB resource {:x} {:x?}",
+            self.current_offset, resource_descriptor
+        );
+        mem.write_obj(resource_descriptor, GuestAddress(self.current_offset))
+            .map_err(TdvfError::GuestMemoryWriteHob)?;
+        self.update_offset::<HobResourceDescriptor>();
+        Ok(())
+    }
+
+    pub fn add_memory_resource(
+        &mut self,
+        mem: &GuestMemoryMmap,
+        physical_start: u64,
+        resource_length: u64,
+        ram: bool,
+    ) -> Result<(), TdvfError> {
+        self.add_resource(
+            mem,
+            physical_start,
+            resource_length,
+            if ram {
+                0 /* EFI_RESOURCE_SYSTEM_MEMORY */
+            } else {
+                0x5 /*EFI_RESOURCE_MEMORY_RESERVED */
+            },
+            /* TODO:
+             * QEMU currently fills it in like this:
+             * EFI_RESOURCE_ATTRIBUTE_PRESENT | EFI_RESOURCE_ATTRIBUTE_INITIALIZED | EFI_RESOURCE_ATTRIBUTE_TESTED
+             * which differs from the spec (due to TDVF implementation issue?)
+             */
+            0x7,
+        )
+    }
+
+    pub fn add_mmio_resource(
+        &mut self,
+        mem: &GuestMemoryMmap,
+        physical_start: u64,
+        resource_length: u64,
+    ) -> Result<(), TdvfError> {
+        self.add_resource(
+            mem,
+            physical_start,
+            resource_length,
+            0x1, /* EFI_RESOURCE_MEMORY_MAPPED_IO */
+            /*
+             * EFI_RESOURCE_ATTRIBUTE_PRESENT | EFI_RESOURCE_ATTRIBUTE_INITIALIZED | EFI_RESOURCE_ATTRIBUTE_UNCACHEABLE
+             */
+            0x403,
+        )
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    #[ignore]
+    fn test_parse_tdvf_sections() {
+        let mut f = std::fs::File::open("tdvf.fd").unwrap();
+        let sections = parse_tdvf_sections(&mut f).unwrap();
+        for section in sections {
+            eprintln!("{:x?}", section)
+        }
+    }
+}
--- a/arch_gen/src/x86/mod.rs
+++ b/arch_gen/src/x86/mod.rs
@@ -8,14 +8,6 @@
 #[allow(non_upper_case_globals)]
 #[allow(non_camel_case_types)]
 #[allow(non_snake_case)]
-#[allow(
-    clippy::unreadable_literal,
-    clippy::redundant_static_lifetimes,
-    clippy::trivially_copy_pass_by_ref,
-    clippy::useless_transmute,
-    clippy::should_implement_trait,
-    clippy::transmute_ptr_to_ptr
-)]
 #[allow(non_camel_case_types)]
 #[allow(non_upper_case_globals)]
 #[allow(clippy::unreadable_literal, clippy::redundant_static_lifetimes)]
--- a/block_util/Cargo.toml
+++ b/block_util/Cargo.toml
@@ -10,7 +10,7 @@ io_uring = []

 [dependencies]
 io-uring = ">=0.4.0"
-libc = "0.2.86"
+libc = "0.2.94"
 log = "0.4.14"
 qcow = { path = "../qcow" }
 serde = ">=1.0.27"
@@ -22,5 +22,3 @@ vm-memory = { version = "0.5.0", features = ["backend-mmap", "backend-atomic"] }
 vm-virtio = { path = "../vm-virtio" }
 vmm-sys-util = ">=0.3.1"

-[dev-dependencies]
-tempfile = "3.2.0"
--- a/block_util/src/lib.rs
+++ b/block_util/src/lib.rs
@@ -24,7 +24,6 @@ pub mod vhd;
 use crate::async_io::{AsyncIo, AsyncIoError, AsyncIoResult, DiskFileError, DiskFileResult};
 #[cfg(feature = "io_uring")]
 use io_uring::{opcode, IoUring, Probe};
-use serde::ser::{Serialize, SerializeStruct, Serializer};
 use std::cmp;
 use std::convert::TryInto;
 use std::fs::File;
@@ -32,7 +31,7 @@ use std::io::{self, IoSlice, IoSliceMut, Read, Seek, SeekFrom, Write};
 use std::os::linux::fs::MetadataExt;
 #[cfg(feature = "io_uring")]
 use std::os::unix::io::AsRawFd;
-use std::path::PathBuf;
+use std::path::Path;
 use std::result;
 use std::sync::{Arc, Mutex};
 use virtio_bindings::bindings::virtio_blk::*;
@@ -65,7 +64,7 @@ pub enum Error {
    TooManyDescriptors,
 }

-fn build_device_id(disk_path: &PathBuf) -> result::Result<String, Error> {
+fn build_device_id(disk_path: &Path) -> result::Result<String, Error> {
    let blk_metadata = match disk_path.metadata() {
        Err(_) => return Err(Error::GetFileMetadata),
        Ok(m) => m,
@@ -80,7 +79,7 @@ fn build_device_id(disk_path: &PathBuf) -> result::Result<String, Error> {
    Ok(device_id)
 }

-pub fn build_disk_image_id(disk_path: &PathBuf) -> Vec<u8> {
+pub fn build_disk_image_id(disk_path: &Path) -> Vec<u8> {
    let mut default_disk_image_id = vec![0; VIRTIO_BLK_ID_BYTES as usize];
    match build_device_id(disk_path) {
        Err(_) => {
@@ -135,7 +134,7 @@ pub enum RequestType {
    In,
    Out,
    Flush,
-    GetDeviceID,
+    GetDeviceId,
    Unsupported(u32),
 }

@@ -148,7 +147,7 @@ pub fn request_type(
        VIRTIO_BLK_T_IN => Ok(RequestType::In),
        VIRTIO_BLK_T_OUT => Ok(RequestType::Out),
        VIRTIO_BLK_T_FLUSH => Ok(RequestType::Flush),
-        VIRTIO_BLK_T_GET_ID => Ok(RequestType::GetDeviceID),
+        VIRTIO_BLK_T_GET_ID => Ok(RequestType::GetDeviceId),
        t => Ok(RequestType::Unsupported(t)),
    }
 }
@@ -163,6 +162,7 @@ fn sector(mem: &GuestMemoryMmap, desc_addr: GuestAddress) -> result::Result<u64,
    mem.read_obj(addr).map_err(Error::GuestMemory)
 }

+#[derive(Debug)]
 pub struct Request {
    pub request_type: RequestType,
    pub sector: u64,
@@ -192,12 +192,17 @@ impl Request {
        let status_desc;
        let mut desc = avail_desc
            .next_descriptor()
-            .ok_or(Error::DescriptorChainTooShort)?;
+            .ok_or(Error::DescriptorChainTooShort)
+            .map_err(|e| {
+                error!("Only head descriptor present: request = {:?}", req);
+                e
+            })?;

        if !desc.has_next() {
            status_desc = desc;
            // Only flush requests are allowed to skip the data descriptor.
            if req.request_type != RequestType::Flush {
+                error!("Need a data descriptor: request = {:?}", req);
                return Err(Error::DescriptorChainTooShort);
            }
        } else {
@@ -208,13 +213,17 @@ impl Request {
                if !desc.is_write_only() && req.request_type == RequestType::In {
                    return Err(Error::UnexpectedReadOnlyDescriptor);
                }
-                if !desc.is_write_only() && req.request_type == RequestType::GetDeviceID {
+                if !desc.is_write_only() && req.request_type == RequestType::GetDeviceId {
                    return Err(Error::UnexpectedReadOnlyDescriptor);
                }
                req.data_descriptors.push((desc.addr, desc.len));
                desc = desc
                    .next_descriptor()
-                    .ok_or(Error::DescriptorChainTooShort)?;
+                    .ok_or(Error::DescriptorChainTooShort)
+                    .map_err(|e| {
+                        error!("DescriptorChain corrupted: request = {:?}", req);
+                        e
+                    })?;
            }
            status_desc = desc;
        }
@@ -270,7 +279,7 @@ impl Request {
                    }
                }
                RequestType::Flush => disk.flush().map_err(ExecuteError::Flush)?,
-                RequestType::GetDeviceID => {
+                RequestType::GetDeviceId => {
                    if (*data_len as usize) < disk_id.len() {
                        return Err(ExecuteError::BadRequest(Error::InvalidOffset));
                    }
@@ -336,7 +345,7 @@ impl Request {
                    .fsync(Some(user_data))
                    .map_err(ExecuteError::AsyncFlush)?;
            }
-            RequestType::GetDeviceID => {
+            RequestType::GetDeviceId => {
                let (data_addr, data_len) = if self.data_descriptors.len() == 1 {
                    (self.data_descriptors[0].0, self.data_descriptors[0].1)
                } else {
@@ -360,7 +369,7 @@ impl Request {
    }
 }

-#[derive(Copy, Clone, Debug, Default, Deserialize)]
+#[derive(Copy, Clone, Debug, Default, Deserialize, Serialize)]
 #[repr(C, packed)]
 pub struct VirtioBlockConfig {
    pub capacity: u64,
@@ -383,95 +392,15 @@ pub struct VirtioBlockConfig {
    pub write_zeroes_may_unmap: u8,
    pub unused1: [u8; 3],
 }
-
-// We must explicitly implement Serialize since the structure is packed and
-// it's unsafe to borrow from a packed structure. And by default, if we derive
-// Serialize from serde, it will borrow the values from the structure.
-// That's why this implementation copies each field separately before it
-// serializes the entire structure field by field.
-impl Serialize for VirtioBlockConfig {
-    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
-    where
-        S: Serializer,
-    {
-        let capacity = self.capacity;
-        let size_max = self.size_max;
-        let seg_max = self.seg_max;
-        let geometry = self.geometry;
-        let blk_size = self.blk_size;
-        let physical_block_exp = self.physical_block_exp;
-        let alignment_offset = self.alignment_offset;
-        let min_io_size = self.min_io_size;
-        let opt_io_size = self.opt_io_size;
-        let writeback = self.writeback;
-        let unused = self.unused;
-        let num_queues = self.num_queues;
-        let max_discard_sectors = self.max_discard_sectors;
-        let max_discard_seg = self.max_discard_seg;
-        let discard_sector_alignment = self.discard_sector_alignment;
-        let max_write_zeroes_sectors = self.max_write_zeroes_sectors;
-        let max_write_zeroes_seg = self.max_write_zeroes_seg;
-        let write_zeroes_may_unmap = self.write_zeroes_may_unmap;
-        let unused1 = self.unused1;
-
-        let mut virtio_block_config = serializer.serialize_struct("VirtioBlockConfig", 60)?;
-        virtio_block_config.serialize_field("capacity", &capacity)?;
-        virtio_block_config.serialize_field("size_max", &size_max)?;
-        virtio_block_config.serialize_field("seg_max", &seg_max)?;
-        virtio_block_config.serialize_field("geometry", &geometry)?;
-        virtio_block_config.serialize_field("blk_size", &blk_size)?;
-        virtio_block_config.serialize_field("physical_block_exp", &physical_block_exp)?;
-        virtio_block_config.serialize_field("alignment_offset", &alignment_offset)?;
-        virtio_block_config.serialize_field("min_io_size", &min_io_size)?;
-        virtio_block_config.serialize_field("opt_io_size", &opt_io_size)?;
-        virtio_block_config.serialize_field("writeback", &writeback)?;
-        virtio_block_config.serialize_field("unused", &unused)?;
-        virtio_block_config.serialize_field("num_queues", &num_queues)?;
-        virtio_block_config.serialize_field("max_discard_sectors", &max_discard_sectors)?;
-        virtio_block_config.serialize_field("max_discard_seg", &max_discard_seg)?;
-        virtio_block_config
-            .serialize_field("discard_sector_alignment", &discard_sector_alignment)?;
-        virtio_block_config
-            .serialize_field("max_write_zeroes_sectors", &max_write_zeroes_sectors)?;
-        virtio_block_config.serialize_field("max_write_zeroes_seg", &max_write_zeroes_seg)?;
-        virtio_block_config.serialize_field("write_zeroes_may_unmap", &write_zeroes_may_unmap)?;
-        virtio_block_config.serialize_field("unused1", &unused1)?;
-        virtio_block_config.end()
-    }
-}
-
 unsafe impl ByteValued for VirtioBlockConfig {}

-#[derive(Copy, Clone, Debug, Default, Deserialize)]
+#[derive(Copy, Clone, Debug, Default, Serialize, Deserialize)]
 #[repr(C, packed)]
 pub struct VirtioBlockGeometry {
    pub cylinders: u16,
    pub heads: u8,
    pub sectors: u8,
 }
-
-// We must explicitly implement Serialize since the structure is packed and
-// it's unsafe to borrow from a packed structure. And by default, if we derive
-// Serialize from serde, it will borrow the values from the structure.
-// That's why this implementation copies each field separately before it
-// serializes the entire structure field by field.
-impl Serialize for VirtioBlockGeometry {
-    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
-    where
-        S: Serializer,
-    {
-        let cylinders = self.cylinders;
-        let heads = self.heads;
-        let sectors = self.sectors;
-
-        let mut virtio_block_geometry = serializer.serialize_struct("VirtioBlockGeometry", 4)?;
-        virtio_block_geometry.serialize_field("cylinders", &cylinders)?;
-        virtio_block_geometry.serialize_field("heads", &heads)?;
-        virtio_block_geometry.serialize_field("sectors", &sectors)?;
-        virtio_block_geometry.end()
-    }
-}
-
 unsafe impl ByteValued for VirtioBlockGeometry {}

 /// Check if io_uring for block device can be used on the current system, as
--- a/block_util/src/raw_async.rs
+++ b/block_util/src/raw_async.rs
@@ -5,7 +5,7 @@
 use crate::async_io::{
    AsyncIo, AsyncIoError, AsyncIoResult, DiskFile, DiskFileError, DiskFileResult,
 };
-use io_uring::{opcode, squeue, IoUring};
+use io_uring::{opcode, squeue, types, IoUring};
 use std::fs::File;
 use std::io::{Seek, SeekFrom};
 use std::os::unix::io::{AsRawFd, RawFd};
@@ -71,28 +71,23 @@ impl AsyncIo for RawFileAsync {
        iovecs: Vec<libc::iovec>,
        user_data: u64,
    ) -> AsyncIoResult<()> {
-        let (submitter, sq, _) = self.io_uring.split();
-        let mut avail_sq = sq.available();
+        let (submitter, mut sq, _) = self.io_uring.split();

        // Safe because we know the file descriptor is valid and we
        // relied on vm-memory to provide the buffer address.
        let _ = unsafe {
-            avail_sq.push(
-                opcode::Readv::new(
-                    opcode::types::Fd(self.fd),
-                    iovecs.as_ptr(),
-                    iovecs.len() as u32,
-                )
-                .offset(offset)
-                .build()
-                .flags(squeue::Flags::ASYNC)
-                .user_data(user_data),
+            sq.push(
+                &opcode::Readv::new(types::Fd(self.fd), iovecs.as_ptr(), iovecs.len() as u32)
+                    .offset(offset)
+                    .build()
+                    .flags(squeue::Flags::ASYNC)
+                    .user_data(user_data),
            )
        };

        // Update the submission queue and submit new operations to the
        // io_uring instance.
-        avail_sq.sync();
+        sq.sync();
        submitter.submit().map_err(AsyncIoError::ReadVectored)?;

        Ok(())
@@ -104,28 +99,23 @@ impl AsyncIo for RawFileAsync {
        iovecs: Vec<libc::iovec>,
        user_data: u64,
    ) -> AsyncIoResult<()> {
-        let (submitter, sq, _) = self.io_uring.split();
-        let mut avail_sq = sq.available();
+        let (submitter, mut sq, _) = self.io_uring.split();

        // Safe because we know the file descriptor is valid and we
        // relied on vm-memory to provide the buffer address.
        let _ = unsafe {
-            avail_sq.push(
-                opcode::Writev::new(
-                    opcode::types::Fd(self.fd),
-                    iovecs.as_ptr(),
-                    iovecs.len() as u32,
-                )
-                .offset(offset)
-                .build()
-                .flags(squeue::Flags::ASYNC)
-                .user_data(user_data),
+            sq.push(
+                &opcode::Writev::new(types::Fd(self.fd), iovecs.as_ptr(), iovecs.len() as u32)
+                    .offset(offset)
+                    .build()
+                    .flags(squeue::Flags::ASYNC)
+                    .user_data(user_data),
            )
        };

        // Update the submission queue and submit new operations to the
        // io_uring instance.
-        avail_sq.sync();
+        sq.sync();
        submitter.submit().map_err(AsyncIoError::WriteVectored)?;

        Ok(())
@@ -133,13 +123,12 @@ impl AsyncIo for RawFileAsync {

    fn fsync(&mut self, user_data: Option<u64>) -> AsyncIoResult<()> {
        if let Some(user_data) = user_data {
-            let (submitter, sq, _) = self.io_uring.split();
-            let mut avail_sq = sq.available();
+            let (submitter, mut sq, _) = self.io_uring.split();

            // Safe because we know the file descriptor is valid.
            let _ = unsafe {
-                avail_sq.push(
-                    opcode::Fsync::new(opcode::types::Fd(self.fd))
+                sq.push(
+                    &opcode::Fsync::new(types::Fd(self.fd))
                        .build()
                        .flags(squeue::Flags::ASYNC)
                        .user_data(user_data),
@@ -148,7 +137,7 @@ impl AsyncIo for RawFileAsync {

            // Update the submission queue and submit new operations to the
            // io_uring instance.
-            avail_sq.sync();
+            sq.sync();
            submitter.submit().map_err(AsyncIoError::Fsync)?;
        } else {
            unsafe { libc::fsync(self.fd) };
@@ -161,7 +150,7 @@ impl AsyncIo for RawFileAsync {
        let mut completion_list = Vec::new();

        let cq = self.io_uring.completion();
-        for cq_entry in cq.available() {
+        for cq_entry in cq {
            completion_list.push((cq_entry.user_data(), cq_entry.result()));
        }

--- a/block_util/src/vhd.rs
+++ b/block_util/src/vhd.rs
@@ -124,7 +124,7 @@ mod tests {
    use super::{is_fixed_vhd, VhdFooter};
    use std::fs::File;
    use std::io::{Seek, SeekFrom, Write};
-    use tempfile::tempfile;
+    use vmm_sys_util::tempfile::TempFile;

    fn valid_fixed_vhd_footer() -> Vec<u8> {
        vec![
@@ -172,7 +172,7 @@ mod tests {
    where
        F: FnMut(File),
    {
-        let mut disk_file: File = tempfile().unwrap();
+        let mut disk_file: File = TempFile::new().unwrap().into_file();
        disk_file.set_len(0x1000_0200).unwrap();
        disk_file.seek(SeekFrom::Start(0x1000_0000)).unwrap();
        disk_file.write_all(&footer).unwrap();
--- a/devices/Cargo.toml
+++ b/devices/Cargo.toml
@@ -6,9 +6,9 @@ authors = ["The Chromium OS Authors"]
 [dependencies]
 anyhow = "1.0"
 bitflags = ">=1.2.1"
-byteorder = "1.3.4"
+byteorder = "1.4.3"
 epoll = ">=4.0.1"
-libc = "0.2.86"
+libc = "0.2.94"
 log = "0.4.14"
 serde = {version = ">=1.0.27", features = ["rc"] }
 serde_derive = ">=1.0.27"
@@ -19,9 +19,6 @@ vm-memory = "0.5.0"
 vm-migration = { path = "../vm-migration" }
 vmm-sys-util = ">=0.3.1"

-[dev-dependencies]
-tempfile = "3.2.0"
-
 [features]
 default = []
 acpi = ["acpi_tables"]
--- a/devices/src/acpi.rs
+++ b/devices/src/acpi.rs
@@ -62,20 +62,20 @@ impl BusDevice for AcpiShutdownDevice {
 }

 /// A device for handling ACPI GED event generation
-pub struct AcpiGEDDevice {
+pub struct AcpiGedDevice {
    interrupt: Arc<Box<dyn InterruptSourceGroup>>,
    notification_type: AcpiNotificationFlags,
    ged_irq: u32,
    address: GuestAddress,
 }

-impl AcpiGEDDevice {
+impl AcpiGedDevice {
    pub fn new(
        interrupt: Arc<Box<dyn InterruptSourceGroup>>,
        ged_irq: u32,
        address: GuestAddress,
-    ) -> AcpiGEDDevice {
-        AcpiGEDDevice {
+    ) -> AcpiGedDevice {
+        AcpiGedDevice {
            interrupt,
            notification_type: AcpiNotificationFlags::NO_DEVICES_CHANGED,
            ged_irq,
@@ -97,7 +97,7 @@ impl AcpiGEDDevice {
 }

 // I/O port reports what type of notification was made
-impl BusDevice for AcpiGEDDevice {
+impl BusDevice for AcpiGedDevice {
    // Spec has all fields as zero
    fn read(&mut self, _base: u64, _offset: u64, data: &mut [u8]) {
        data[0] = self.notification_type.bits();
@@ -106,7 +106,7 @@ impl BusDevice for AcpiGEDDevice {
 }

 #[cfg(feature = "acpi")]
-impl Aml for AcpiGEDDevice {
+impl Aml for AcpiGedDevice {
    fn to_aml_bytes(&self) -> Vec<u8> {
        aml::Device::new(
            "_SB_.GED_".into(),
@@ -172,11 +172,11 @@ impl Aml for AcpiGEDDevice {
    }
 }

-pub struct AcpiPMTimerDevice {
+pub struct AcpiPmTimerDevice {
    start: Instant,
 }

-impl AcpiPMTimerDevice {
+impl AcpiPmTimerDevice {
    pub fn new() -> Self {
        Self {
            start: Instant::now(),
@@ -184,13 +184,13 @@ impl AcpiPMTimerDevice {
    }
 }

-impl Default for AcpiPMTimerDevice {
+impl Default for AcpiPmTimerDevice {
    fn default() -> Self {
        Self::new()
    }
 }

-impl BusDevice for AcpiPMTimerDevice {
+impl BusDevice for AcpiPmTimerDevice {
    fn read(&mut self, _base: u64, _offset: u64, data: &mut [u8]) {
        let now = Instant::now();
        let since = now.duration_since(self.start);
--- a/devices/src/gic.rs
+++ b/devices/src/gic.rs
@@ -6,16 +6,16 @@ use super::interrupt_controller::{Error, InterruptController};
 use std::result;
 use std::sync::Arc;
 use vm_device::interrupt::{
-    InterruptIndex, InterruptManager, InterruptSourceGroup, MsiIrqGroupConfig,
+    InterruptIndex, InterruptManager, InterruptSourceConfig, InterruptSourceGroup,
+    LegacyIrqSourceConfig, MsiIrqGroupConfig,
 };
 use vmm_sys_util::eventfd::EventFd;

 type Result<T> = result::Result<T, Error>;

-// Reserve 32 IRQs (GSI 32 ~ 64) for legacy device.
-// GsiAllocator should allocate beyond this: from 64 on
+// Reserve 32 IRQs for legacy device.
+pub const IRQ_LEGACY_BASE: usize = 0;
 pub const IRQ_LEGACY_COUNT: usize = 32;
-pub const IRQ_SPI_OFFSET: usize = 32;

 // This Gic struct implements InterruptController to provide interrupt delivery service.
 // The Gic source files in arch/ folder maintain the Aarch64 specific Gic device.
@@ -34,7 +34,7 @@ impl Gic {
    ) -> Result<Gic> {
        let interrupt_source_group = interrupt_manager
            .create_group(MsiIrqGroupConfig {
-                base: IRQ_SPI_OFFSET as InterruptIndex,
+                base: IRQ_LEGACY_BASE as InterruptIndex,
                count: IRQ_LEGACY_COUNT as InterruptIndex,
            })
            .map_err(Error::CreateInterruptSourceGroup)?;
@@ -47,9 +47,26 @@ impl Gic {

 impl InterruptController for Gic {
    fn enable(&self) -> Result<()> {
+        // Set irqfd for legacy interrupts
        self.interrupt_source_group
            .enable()
            .map_err(Error::EnableInterrupt)?;
+
+        // Set irq_routing for legacy interrupts.
+        //   irqchip: Hardcode to 0 as we support only 1 GIC
+        //   pin: Use irq number as pin
+        for i in IRQ_LEGACY_BASE..(IRQ_LEGACY_BASE + IRQ_LEGACY_COUNT) {
+            let config = LegacyIrqSourceConfig {
+                irqchip: 0,
+                pin: i as u32,
+            };
+            self.interrupt_source_group
+                .update(
+                    i as InterruptIndex,
+                    InterruptSourceConfig::LegacyIrq(config),
+                )
+                .map_err(Error::EnableInterrupt)?;
+        }
        Ok(())
    }

--- a/devices/src/ioapic.rs
+++ b/devices/src/ioapic.rs
@@ -20,16 +20,9 @@ use vm_device::interrupt::{
 };
 use vm_device::BusDevice;
 use vm_memory::GuestAddress;
-use vm_migration::{
-    Migratable, MigratableError, Pausable, Snapshot, SnapshotDataSection, Snapshottable,
-    Transportable,
-};
+use vm_migration::{Migratable, MigratableError, Pausable, Snapshot, Snapshottable, Transportable};
 use vmm_sys_util::eventfd::EventFd;

-#[derive(Serialize, Deserialize)]
-#[serde(remote = "GuestAddress")]
-pub struct GuestAddressDef(pub u64);
-
 type Result<T> = result::Result<T, Error>;

 // I/O REDIRECTION TABLE REGISTER
@@ -113,9 +106,9 @@ enum TriggerMode {
 enum DeliveryMode {
    Fixed = 0b000,
    Lowest = 0b001,
-    SMI = 0b010,        // System management interrupt
+    Smi = 0b010,        // System management interrupt
    RemoteRead = 0b011, // This is no longer supported by intel.
-    NMI = 0b100,        // Non maskable interrupt
+    Nmi = 0b100,        // Non maskable interrupt
    Init = 0b101,
    Startup = 0b110,
    External = 0b111,
@@ -146,8 +139,7 @@ pub struct IoapicState {
    reg_sel: u32,
    reg_entries: [RedirectionTableEntry; NUM_IOAPIC_PINS],
    used_entries: [bool; NUM_IOAPIC_PINS],
-    #[serde(with = "GuestAddressDef")]
-    apic_address: GuestAddress,
+    apic_address: u64,
 }

 impl BusDevice for Ioapic {
@@ -284,7 +276,7 @@ impl Ioapic {
            reg_sel: self.reg_sel,
            reg_entries: self.reg_entries,
            used_entries: self.used_entries,
-            apic_address: self.apic_address,
+            apic_address: self.apic_address.0,
        }
    }

@@ -293,7 +285,7 @@ impl Ioapic {
        self.reg_sel = state.reg_sel;
        self.reg_entries = state.reg_entries;
        self.used_entries = state.used_entries;
-        self.apic_address = state.apic_address;
+        self.apic_address = GuestAddress(state.apic_address);
        for (irq, entry) in self.used_entries.iter().enumerate() {
            if *entry {
                self.update_entry(irq)?;
@@ -333,9 +325,9 @@ impl Ioapic {
        match delivery_mode {
            x if (x == DeliveryMode::Fixed as u8)
                || (x == DeliveryMode::Lowest as u8)
-                || (x == DeliveryMode::SMI as u8)
+                || (x == DeliveryMode::Smi as u8)
                || (x == DeliveryMode::RemoteRead as u8)
-                || (x == DeliveryMode::NMI as u8)
+                || (x == DeliveryMode::Nmi as u8)
                || (x == DeliveryMode::Init as u8)
                || (x == DeliveryMode::Startup as u8)
                || (x == DeliveryMode::External as u8) => {}
@@ -418,38 +410,13 @@ impl Snapshottable for Ioapic {
    }

    fn snapshot(&mut self) -> std::result::Result<Snapshot, MigratableError> {
-        let snapshot =
-            serde_json::to_vec(&self.state()).map_err(|e| MigratableError::Snapshot(e.into()))?;
-
-        let mut ioapic_snapshot = Snapshot::new(self.id.as_str());
-        ioapic_snapshot.add_data_section(SnapshotDataSection {
-            id: format!("{}-section", self.id),
-            snapshot,
-        });
-
-        Ok(ioapic_snapshot)
+        Snapshot::new_from_state(&self.id, &self.state())
    }

    fn restore(&mut self, snapshot: Snapshot) -> std::result::Result<(), MigratableError> {
-        if let Some(ioapic_section) = snapshot.snapshot_data.get(&format!("{}-section", self.id)) {
-            let ioapic_state = match serde_json::from_slice(&ioapic_section.snapshot) {
-                Ok(state) => state,
-                Err(error) => {
-                    return Err(MigratableError::Restore(anyhow!(
-                        "Could not deserialize IOAPIC {}",
-                        error
-                    )))
-                }
-            };
-
-            return self.set_state(&ioapic_state).map_err(|e| {
-                MigratableError::Restore(anyhow!("Could not restore IOAPIC state {:?}", e))
-            });
-        }
-
-        Err(MigratableError::Restore(anyhow!(
-            "Could not find IOAPIC snapshot section"
-        )))
+        self.set_state(&snapshot.to_state(&self.id)?).map_err(|e| {
+            MigratableError::Restore(anyhow!("Could not restore state for {}: {:?}", self.id, e))
+        })
    }
 }

--- a/devices/src/legacy/gpio_pl061.rs
+++ b/devices/src/legacy/gpio_pl061.rs
@@ -0,0 +1,454 @@
+// Copyright 2021 Arm Limited (or its affiliates). All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+//! ARM PrimeCell General Purpose Input/Output(PL061)
+//!
+//! This module implements an ARM PrimeCell General Purpose Input/Output(PL061) to support gracefully poweroff microvm from external.
+//!
+
+use crate::{read_le_u32, write_le_u32};
+use std::result;
+use std::sync::{Arc, Barrier};
+use std::{fmt, io};
+use vm_device::interrupt::InterruptSourceGroup;
+use vm_device::BusDevice;
+use vm_migration::{Migratable, MigratableError, Pausable, Snapshot, Snapshottable, Transportable};
+
+const OFS_DATA: u64 = 0x400; // Data Register
+const GPIODIR: u64 = 0x400; // Direction Register
+const GPIOIS: u64 = 0x404; // Interrupt Sense Register
+const GPIOIBE: u64 = 0x408; // Interrupt Both Edges Register
+const GPIOIEV: u64 = 0x40c; // Interrupt Event Register
+const GPIOIE: u64 = 0x410; // Interrupt Mask Register
+const GPIORIE: u64 = 0x414; // Raw Interrupt Status Register
+const GPIOMIS: u64 = 0x418; // Masked Interrupt Status Register
+const GPIOIC: u64 = 0x41c; // Interrupt Clear Register
+const GPIOAFSEL: u64 = 0x420; // Mode Control Select Register
+                              // From 0x424 to 0xFDC => reserved space.
+                              // From 0xFE0 to 0xFFC => Peripheral and PrimeCell Identification Registers which are Read Only registers.
+                              // Thses registers can conceptually be treated as a 32-bit register, and PartNumber[11:0] is used to identify the peripheral.
+                              // We are putting the expected values (look at 'Reset value' column from above mentioned document) in an array.
+const GPIO_ID: [u8; 8] = [0x61, 0x10, 0x14, 0x00, 0x0d, 0xf0, 0x05, 0xb1];
+// ID Margins
+const GPIO_ID_LOW: u64 = 0xfe0;
+const GPIO_ID_HIGH: u64 = 0x1000;
+
+const N_GPIOS: u32 = 8;
+
+#[derive(Debug)]
+pub enum Error {
+    BadWriteOffset(u64),
+    GpioInterruptDisabled,
+    GpioInterruptFailure(io::Error),
+    GpioTriggerKeyFailure(u32),
+}
+
+impl fmt::Display for Error {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        match self {
+            Error::BadWriteOffset(offset) => write!(f, "Bad Write Offset: {}", offset),
+            Error::GpioInterruptDisabled => write!(f, "GPIO interrupt disabled by guest driver.",),
+            Error::GpioInterruptFailure(ref e) => {
+                write!(f, "Could not trigger GPIO interrupt: {}.", e)
+            }
+            Error::GpioTriggerKeyFailure(key) => {
+                write!(f, "Invalid GPIO Input key triggerd: {}.", key)
+            }
+        }
+    }
+}
+
+type Result<T> = result::Result<T, Error>;
+
+/// A GPIO device following the PL061 specification.
+pub struct Gpio {
+    id: String,
+    // Data Register
+    data: u32,
+    old_in_data: u32,
+    // Direction Register
+    dir: u32,
+    // Interrupt Sense Register
+    isense: u32,
+    // Interrupt Both Edges Register
+    ibe: u32,
+    // Interrupt Event Register
+    iev: u32,
+    // Interrupt Mask Register
+    im: u32,
+    // Raw Interrupt Status Register
+    istate: u32,
+    // Mode Control Select Register
+    afsel: u32,
+    // GPIO irq_field
+    interrupt: Arc<Box<dyn InterruptSourceGroup>>,
+}
+
+#[derive(Serialize, Deserialize)]
+pub struct GpioState {
+    data: u32,
+    old_in_data: u32,
+    dir: u32,
+    isense: u32,
+    ibe: u32,
+    iev: u32,
+    im: u32,
+    istate: u32,
+    afsel: u32,
+}
+
+impl Gpio {
+    /// Constructs an PL061 GPIO device.
+    pub fn new(id: String, interrupt: Arc<Box<dyn InterruptSourceGroup>>) -> Self {
+        Self {
+            id,
+            data: 0,
+            old_in_data: 0,
+            dir: 0,
+            isense: 0,
+            ibe: 0,
+            iev: 0,
+            im: 0,
+            istate: 0,
+            afsel: 0,
+            interrupt,
+        }
+    }
+
+    fn state(&self) -> GpioState {
+        GpioState {
+            data: self.data,
+            old_in_data: self.old_in_data,
+            dir: self.dir,
+            isense: self.isense,
+            ibe: self.ibe,
+            iev: self.iev,
+            im: self.im,
+            istate: self.istate,
+            afsel: self.afsel,
+        }
+    }
+
+    fn set_state(&mut self, state: &GpioState) {
+        self.data = state.data;
+        self.old_in_data = state.old_in_data;
+        self.dir = state.dir;
+        self.isense = state.isense;
+        self.ibe = state.ibe;
+        self.iev = state.iev;
+        self.im = state.im;
+        self.istate = state.istate;
+        self.afsel = state.afsel;
+    }
+
+    fn pl061_internal_update(&mut self) {
+        // FIXME:
+        //  Missing Output Interrupt Emulation.
+
+        // Input Edging Interrupt Emulation.
+        let changed = ((self.old_in_data ^ self.data) & !self.dir) as u32;
+        if changed > 0 {
+            self.old_in_data = self.data;
+            for i in 0..N_GPIOS {
+                let mask = (1 << i) as u32;
+                if (changed & mask) > 0 {
+                    // Bits set high in GPIOIS(Interrupt sense register) configure the corresponding
+                    // pins to detect levels, otherwise, detect edges.
+                    if (self.isense & mask) == 0 {
+                        if (self.ibe & mask) > 0 {
+                            // Bits set high in GPIOIBE(Interrupt both-edges register) configure the corresponding
+                            // pins to detect both falling and rising edges.
+                            // Clearing a bit configures the pin to be controlled by GPIOIEV.
+                            self.istate |= mask;
+                        } else {
+                            // Bits set to high in GPIOIEV(Interrupt event register) configure the
+                            // corresponding pin to detect rising edges, otherwise, detect falling edges.
+                            self.istate |= !(self.data ^ self.iev) & mask;
+                        }
+                    }
+                }
+            }
+        }
+
+        // Input Level Interrupt Emulation.
+        self.istate |= !(self.data ^ self.iev) & self.isense;
+    }
+
+    fn handle_write(&mut self, offset: u64, val: u32) -> Result<()> {
+        if offset < OFS_DATA {
+            // In order to write to data register, the corresponding bits in the mask, resulting
+            // from the offsite[9:2], must be HIGH. otherwise the bit values remain unchanged.
+            let mask = (offset >> 2) as u32 & self.dir;
+            self.data = (self.data & !mask) | (val & mask);
+        } else {
+            match offset {
+                GPIODIR => {
+                    /* Direction Register */
+                    self.dir = val & 0xff;
+                }
+                GPIOIS => {
+                    /* Interrupt Sense Register */
+                    self.isense = val & 0xff;
+                }
+                GPIOIBE => {
+                    /* Interrupt Both Edges Register */
+                    self.ibe = val & 0xff;
+                }
+                GPIOIEV => {
+                    /* Interrupt Event Register */
+                    self.iev = val & 0xff;
+                }
+                GPIOIE => {
+                    /* Interrupt Mask Register */
+                    self.im = val & 0xff;
+                }
+                GPIOIC => {
+                    /* Interrupt Clear Register */
+                    self.istate &= !val;
+                }
+                GPIOAFSEL => {
+                    /* Mode Control Select Register */
+                    self.afsel = val & 0xff;
+                }
+                o => {
+                    return Err(Error::BadWriteOffset(o));
+                }
+            }
+        }
+        Ok(())
+    }
+
+    pub fn trigger_key(&mut self, key: u32) -> Result<()> {
+        let mask = (1 << key) as u32;
+        if (!self.dir & mask) > 0 {
+            // emulate key event
+            // By default, Input Pin is configured to detect both rising and falling edges.
+            // So reverse the input pin data to generate a pulse.
+            self.data |= !(self.data & mask) & mask;
+            self.pl061_internal_update();
+
+            match self.trigger_gpio_interrupt() {
+                Ok(_) | Err(Error::GpioInterruptDisabled) => return Ok(()),
+                Err(e) => return Err(e),
+            }
+        }
+
+        Err(Error::GpioTriggerKeyFailure(key))
+    }
+
+    fn trigger_gpio_interrupt(&self) -> Result<()> {
+        // Bits set to high in GPIOIE(Interrupt mask register) allow the corresponding pins to
+        // trigger their individual interrupts and then the combined GPIOINTR line.
+        if (self.istate & self.im) == 0 {
+            warn!("Failed to trigger GPIO input interrupt (disabled by guest OS)");
+            return Err(Error::GpioInterruptDisabled);
+        }
+        self.interrupt
+            .trigger(0)
+            .map_err(Error::GpioInterruptFailure)?;
+        Ok(())
+    }
+}
+
+impl BusDevice for Gpio {
+    fn read(&mut self, _base: u64, offset: u64, data: &mut [u8]) {
+        let value;
+        let mut read_ok = true;
+
+        if (GPIO_ID_LOW..GPIO_ID_HIGH).contains(&offset) {
+            let index = ((offset - GPIO_ID_LOW) >> 2) as usize;
+            value = u32::from(GPIO_ID[index]);
+        } else if offset < OFS_DATA {
+            value = self.data & ((offset >> 2) as u32)
+        } else {
+            value = match offset {
+                GPIODIR => self.dir,
+                GPIOIS => self.isense,
+                GPIOIBE => self.ibe,
+                GPIOIEV => self.iev,
+                GPIOIE => self.im,
+                GPIORIE => self.istate,
+                GPIOMIS => self.istate & self.im,
+                GPIOAFSEL => self.afsel,
+                _ => {
+                    read_ok = false;
+                    0
+                }
+            };
+        }
+
+        if read_ok && data.len() <= 4 {
+            write_le_u32(data, value);
+        } else {
+            warn!(
+                "Invalid GPIO PL061 read: offset {}, data length {}",
+                offset,
+                data.len()
+            );
+        }
+    }
+
+    fn write(&mut self, _base: u64, offset: u64, data: &[u8]) -> Option<Arc<Barrier>> {
+        if data.len() <= 4 {
+            let value = read_le_u32(&data);
+            if let Err(e) = self.handle_write(offset, value) {
+                warn!("Failed to write to GPIO PL061 device: {}", e);
+            }
+        } else {
+            warn!(
+                "Invalid GPIO PL061 write: offset {}, data length {}",
+                offset,
+                data.len()
+            );
+        }
+
+        None
+    }
+}
+
+impl Snapshottable for Gpio {
+    fn id(&self) -> String {
+        self.id.clone()
+    }
+
+    fn snapshot(&mut self) -> std::result::Result<Snapshot, MigratableError> {
+        Snapshot::new_from_state(&self.id, &self.state())
+    }
+
+    fn restore(&mut self, snapshot: Snapshot) -> std::result::Result<(), MigratableError> {
+        self.set_state(&snapshot.to_state(&self.id)?);
+        Ok(())
+    }
+}
+
+impl Pausable for Gpio {}
+impl Transportable for Gpio {}
+impl Migratable for Gpio {}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::{read_le_u32, write_le_u32};
+    use std::sync::Arc;
+    use vm_device::interrupt::{InterruptIndex, InterruptSourceConfig};
+    use vmm_sys_util::eventfd::EventFd;
+
+    const GPIO_NAME: &str = "gpio";
+    const LEGACY_GPIO_MAPPED_IO_START: u64 = 0x0902_0000;
+
+    struct TestInterrupt {
+        event_fd: EventFd,
+    }
+
+    impl InterruptSourceGroup for TestInterrupt {
+        fn trigger(&self, _index: InterruptIndex) -> result::Result<(), std::io::Error> {
+            self.event_fd.write(1)
+        }
+
+        fn update(
+            &self,
+            _index: InterruptIndex,
+            _config: InterruptSourceConfig,
+        ) -> result::Result<(), std::io::Error> {
+            Ok(())
+        }
+
+        fn notifier(&self, _index: InterruptIndex) -> Option<EventFd> {
+            Some(self.event_fd.try_clone().unwrap())
+        }
+    }
+
+    impl TestInterrupt {
+        fn new(event_fd: EventFd) -> Self {
+            TestInterrupt { event_fd }
+        }
+    }
+
+    #[test]
+    fn test_gpio_read_write_and_event() {
+        let intr_evt = EventFd::new(libc::EFD_NONBLOCK).unwrap();
+        let mut gpio = Gpio::new(
+            String::from(GPIO_NAME),
+            Arc::new(Box::new(TestInterrupt::new(intr_evt.try_clone().unwrap()))),
+        );
+        let mut data = [0; 4];
+
+        // Read and write to the GPIODIR register.
+        // Set pin 0 output pin.
+        write_le_u32(&mut data, 1);
+        gpio.write(LEGACY_GPIO_MAPPED_IO_START, GPIODIR, &mut data);
+        gpio.read(LEGACY_GPIO_MAPPED_IO_START, GPIODIR, &mut data);
+        let v = read_le_u32(&data);
+        assert_eq!(v, 1);
+
+        // Read and write to the GPIODATA register.
+        write_le_u32(&mut data, 1);
+        // Set pin 0 high.
+        let offset = 0x00000004 as u64;
+        gpio.write(LEGACY_GPIO_MAPPED_IO_START, offset, &mut data);
+        gpio.read(LEGACY_GPIO_MAPPED_IO_START, offset, &mut data);
+        let v = read_le_u32(&data);
+        assert_eq!(v, 1);
+
+        // Read and write to the GPIOIS register.
+        // Configure pin 0 detecting level interrupt.
+        write_le_u32(&mut data, 1);
+        gpio.write(LEGACY_GPIO_MAPPED_IO_START, GPIOIS, &mut data);
+        gpio.read(LEGACY_GPIO_MAPPED_IO_START, GPIOIS, &mut data);
+        let v = read_le_u32(&data);
+        assert_eq!(v, 1);
+
+        // Read and write to the GPIOIBE register.
+        // Configure pin 1 detecting both falling and rising edges.
+        write_le_u32(&mut data, 2);
+        gpio.write(LEGACY_GPIO_MAPPED_IO_START, GPIOIBE, &mut data);
+        gpio.read(LEGACY_GPIO_MAPPED_IO_START, GPIOIBE, &mut data);
+        let v = read_le_u32(&data);
+        assert_eq!(v, 2);
+
+        // Read and write to the GPIOIEV register.
+        // Configure pin 2 detecting both falling and rising edges.
+        write_le_u32(&mut data, 4);
+        gpio.write(LEGACY_GPIO_MAPPED_IO_START, GPIOIEV, &mut data);
+        gpio.read(LEGACY_GPIO_MAPPED_IO_START, GPIOIEV, &mut data);
+        let v = read_le_u32(&data);
+        assert_eq!(v, 4);
+
+        // Read and write to the GPIOIE register.
+        // Configure pin 0...2 capable of triggering their individual interrupts
+        // and then the combined GPIOINTR line.
+        write_le_u32(&mut data, 7);
+        gpio.write(LEGACY_GPIO_MAPPED_IO_START, GPIOIE, &mut data);
+        gpio.read(LEGACY_GPIO_MAPPED_IO_START, GPIOIE, &mut data);
+        let v = read_le_u32(&data);
+        assert_eq!(v, 7);
+
+        let mask = 0x00000002 as u32;
+        // emulate an rising pulse in pin 1.
+        gpio.data |= !(gpio.data & mask) & mask;
+        gpio.pl061_internal_update();
+        // The interrupt line on pin 1 should be on.
+        // Read the GPIOMIS register.
+        gpio.read(LEGACY_GPIO_MAPPED_IO_START, GPIOMIS, &mut data);
+        let v = read_le_u32(&data);
+        assert_eq!(v, 2);
+
+        // Read and Write to the GPIOIC register.
+        // clear interrupt in pin 1.
+        write_le_u32(&mut data, 2);
+        gpio.write(LEGACY_GPIO_MAPPED_IO_START, GPIOIC, &mut data);
+        gpio.read(LEGACY_GPIO_MAPPED_IO_START, GPIOIC, &mut data);
+        let v = read_le_u32(&data);
+        assert_eq!(v, 2);
+
+        // Attempts to write beyond the writable space.
+        write_le_u32(&mut data, 0);
+        gpio.write(LEGACY_GPIO_MAPPED_IO_START, GPIO_ID_LOW, &mut data);
+
+        let mut data = [0; 4];
+        gpio.read(LEGACY_GPIO_MAPPED_IO_START, GPIO_ID_LOW, &mut data);
+        let index = GPIO_ID_LOW + 3;
+        assert_eq!(data[0], GPIO_ID[((index - GPIO_ID_LOW) >> 2) as usize]);
+    }
+}
--- a/devices/src/legacy/mod.rs
+++ b/devices/src/legacy/mod.rs
@@ -9,10 +9,14 @@
 mod cmos;
 #[cfg(feature = "fwdebug")]
 mod fwdebug;
+#[cfg(target_arch = "aarch64")]
+mod gpio_pl061;
 mod i8042;
 #[cfg(target_arch = "aarch64")]
 mod rtc_pl031;
 mod serial;
+#[cfg(target_arch = "aarch64")]
+mod uart_pl011;

 #[cfg(feature = "cmos")]
 pub use self::cmos::Cmos;
@@ -22,4 +26,10 @@ pub use self::i8042::I8042Device;
 pub use self::serial::Serial;

 #[cfg(target_arch = "aarch64")]
-pub use self::rtc_pl031::RTC;
+pub use self::gpio_pl061::Error as GpioDeviceError;
+#[cfg(target_arch = "aarch64")]
+pub use self::gpio_pl061::Gpio;
+#[cfg(target_arch = "aarch64")]
+pub use self::rtc_pl031::Rtc;
+#[cfg(target_arch = "aarch64")]
+pub use self::uart_pl011::Pl011;
--- a/devices/src/legacy/rtc_pl031.rs
+++ b/devices/src/legacy/rtc_pl031.rs
@@ -8,6 +8,7 @@
 //! This is achieved by generating an interrupt signal after counting for a programmed number of cycles of
 //! a real-time clock input.
 //!
+use crate::{read_le_u32, write_le_u32};
 use std::fmt;
 use std::sync::{Arc, Barrier};
 use std::time::Instant;
@@ -38,52 +39,6 @@ const AMBA_ID_HIGH: u64 = 0x1000;
 /// Constant to convert seconds to nanoseconds.
 pub const NANOS_PER_SECOND: u64 = 1_000_000_000;

-#[allow(unused_macros)]
-macro_rules! generate_read_fn {
-    ($fn_name: ident, $data_type: ty, $byte_type: ty, $type_size: expr, $endian_type: ident) => {
-        #[allow(dead_code)]
-        pub fn $fn_name(input: &[$byte_type]) -> $data_type {
-            assert!($type_size == std::mem::size_of::<$data_type>());
-            let mut array = [0u8; $type_size];
-            for (byte, read) in array.iter_mut().zip(input.iter().cloned()) {
-                *byte = read as u8;
-            }
-            <$data_type>::$endian_type(array)
-        }
-    };
-}
-
-#[allow(unused_macros)]
-macro_rules! generate_write_fn {
-    ($fn_name: ident, $data_type: ty, $byte_type: ty, $endian_type: ident) => {
-        #[allow(dead_code)]
-        pub fn $fn_name(buf: &mut [$byte_type], n: $data_type) {
-            for (byte, read) in buf
-                .iter_mut()
-                .zip(<$data_type>::$endian_type(n).iter().cloned())
-            {
-                *byte = read as $byte_type;
-            }
-        }
-    };
-}
-
-generate_read_fn!(read_le_u16, u16, u8, 2, from_le_bytes);
-generate_read_fn!(read_le_u32, u32, u8, 4, from_le_bytes);
-generate_read_fn!(read_le_u64, u64, u8, 8, from_le_bytes);
-generate_read_fn!(read_le_i32, i32, i8, 4, from_le_bytes);
-
-generate_read_fn!(read_be_u16, u16, u8, 2, from_be_bytes);
-generate_read_fn!(read_be_u32, u32, u8, 4, from_be_bytes);
-
-generate_write_fn!(write_le_u16, u16, u8, to_le_bytes);
-generate_write_fn!(write_le_u32, u32, u8, to_le_bytes);
-generate_write_fn!(write_le_u64, u64, u8, to_le_bytes);
-generate_write_fn!(write_le_i32, i32, i8, to_le_bytes);
-
-generate_write_fn!(write_be_u16, u16, u8, to_be_bytes);
-generate_write_fn!(write_be_u32, u32, u8, to_be_bytes);
-
 #[derive(Debug)]
 pub enum Error {
    BadWriteOffset(u64),
@@ -115,9 +70,9 @@ pub enum ClockType {
    ThreadCpu,
 }

-impl Into<libc::clockid_t> for ClockType {
-    fn into(self) -> libc::clockid_t {
-        match self {
+impl From<ClockType> for libc::clockid_t {
+    fn from(ct: ClockType) -> libc::clockid_t {
+        match ct {
            ClockType::Monotonic => libc::CLOCK_MONOTONIC,
            ClockType::Real => libc::CLOCK_REALTIME,
            ClockType::ProcessCpu => libc::CLOCK_PROCESS_CPUTIME_ID,
@@ -260,7 +215,7 @@ pub fn seconds_to_nanoseconds(value: i64) -> Option<i64> {
 }

 /// A RTC device following the PL031 specification..
-pub struct RTC {
+pub struct Rtc {
    previous_now: Instant,
    tick_offset: i64,
    // This is used for implementing the RTC alarm. However, in Firecracker we do not need it.
@@ -272,10 +227,10 @@ pub struct RTC {
    interrupt: Arc<Box<dyn InterruptSourceGroup>>,
 }

-impl RTC {
+impl Rtc {
    /// Constructs an AMBA PL031 RTC device.
-    pub fn new(interrupt: Arc<Box<dyn InterruptSourceGroup>>) -> RTC {
-        RTC {
+    pub fn new(interrupt: Arc<Box<dyn InterruptSourceGroup>>) -> Self {
+        Self {
            // This is used only for duration measuring purposes.
            previous_now: Instant::now(),
            tick_offset: get_time(ClockType::Real) as i64,
@@ -334,7 +289,7 @@ impl RTC {
    }
 }

-impl BusDevice for RTC {
+impl BusDevice for Rtc {
    fn read(&mut self, _base: u64, offset: u64, data: &mut [u8]) {
        let v;
        let mut read_ok = true;
@@ -373,7 +328,7 @@ impl BusDevice for RTC {

    fn write(&mut self, _base: u64, offset: u64, data: &[u8]) -> Option<Arc<Barrier>> {
        if data.len() <= 4 {
-            let v = read_le_u32(&data[..]);
+            let v = read_le_u32(&data);
            if let Err(e) = self.handle_write(offset, v) {
                warn!("Failed to write to RTC PL031 device: {}", e);
            }
@@ -392,6 +347,10 @@ impl BusDevice for RTC {
 #[cfg(test)]
 mod tests {
    use super::*;
+    use crate::{
+        read_be_u16, read_be_u32, read_le_i32, read_le_u16, read_le_u32, read_le_u64, write_be_u16,
+        write_be_u32, write_le_i32, write_le_u16, write_le_u32, write_le_u64,
+    };
    use std::sync::Arc;
    use vm_device::interrupt::{InterruptIndex, InterruptSourceConfig};
    use vmm_sys_util::eventfd::EventFd;
@@ -491,7 +450,7 @@ mod tests {
    fn test_rtc_read_write_and_event() {
        let intr_evt = EventFd::new(libc::EFD_NONBLOCK).unwrap();

-        let mut rtc = RTC::new(Arc::new(Box::new(TestInterrupt::new(
+        let mut rtc = Rtc::new(Arc::new(Box::new(TestInterrupt::new(
            intr_evt.try_clone().unwrap(),
        ))));
        let mut data = [0; 4];
@@ -500,7 +459,7 @@ mod tests {
        write_le_u32(&mut data, 123);
        rtc.write(LEGACY_RTC_MAPPED_IO_START, RTCMR, &mut data);
        rtc.read(LEGACY_RTC_MAPPED_IO_START, RTCMR, &mut data);
-        let v = read_le_u32(&data[..]);
+        let v = read_le_u32(&data);
        assert_eq!(v, 123);

        // Read and write to the LR register.
@@ -512,7 +471,7 @@ mod tests {
        assert!(rtc.previous_now > previous_now_before);

        rtc.read(LEGACY_RTC_MAPPED_IO_START, RTCLR, &mut data);
-        let v_read = read_le_u32(&data[..]);
+        let v_read = read_le_u32(&data);
        assert_eq!((v / NANOS_PER_SECOND) as u32, v_read);

        // Read and write to IMSC register.
@@ -523,14 +482,14 @@ mod tests {
        // The interrupt line should be on.
        assert!(rtc.interrupt.notifier(0).unwrap().read().unwrap() == 1);
        rtc.read(LEGACY_RTC_MAPPED_IO_START, RTCIMSC, &mut data);
-        let v = read_le_u32(&data[..]);
+        let v = read_le_u32(&data);
        assert_eq!(non_zero & 1, v);

        // Now test with 0.
        write_le_u32(&mut data, 0);
        rtc.write(LEGACY_RTC_MAPPED_IO_START, RTCIMSC, &mut data);
        rtc.read(LEGACY_RTC_MAPPED_IO_START, RTCIMSC, &mut data);
-        let v = read_le_u32(&data[..]);
+        let v = read_le_u32(&data);
        assert_eq!(0, v);

        // Read and write to the ICR register.
@@ -538,10 +497,10 @@ mod tests {
        rtc.write(LEGACY_RTC_MAPPED_IO_START, RTCICR, &mut data);
        // The interrupt line should be on.
        assert!(rtc.interrupt.notifier(0).unwrap().read().unwrap() > 1);
-        let v_before = read_le_u32(&data[..]);
+        let v_before = read_le_u32(&data);

        rtc.read(LEGACY_RTC_MAPPED_IO_START, RTCICR, &mut data);
-        let v = read_le_u32(&data[..]);
+        let v = read_le_u32(&data);
        // ICR is a  write only register. Data received should stay equal to data sent.
        assert_eq!(v, v_before);

@@ -549,7 +508,7 @@ mod tests {
        write_le_u32(&mut data, 0);
        rtc.write(LEGACY_RTC_MAPPED_IO_START, RTCCR, &mut data);
        rtc.read(LEGACY_RTC_MAPPED_IO_START, RTCCR, &mut data);
-        let v = read_le_u32(&data[..]);
+        let v = read_le_u32(&data);
        assert_eq!(v, 1);

        // Attempts to write beyond the writable space. Using here the space used to read
--- a/devices/src/legacy/serial.rs
+++ b/devices/src/legacy/serial.rs
@@ -5,16 +5,12 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE-BSD-3-Clause file.

-use anyhow::anyhow;
 use std::collections::VecDeque;
 use std::sync::{Arc, Barrier};
 use std::{io, result};
 use vm_device::interrupt::InterruptSourceGroup;
 use vm_device::BusDevice;
-use vm_migration::{
-    Migratable, MigratableError, Pausable, Snapshot, SnapshotDataSection, Snapshottable,
-    Transportable,
-};
+use vm_migration::{Migratable, MigratableError, Pausable, Snapshot, Snapshottable, Transportable};
 use vmm_sys_util::errno::Result;

 const LOOP_SIZE: usize = 0x40;
@@ -84,7 +80,7 @@ pub struct SerialState {
    modem_status: u8,
    scratch: u8,
    baud_divisor: u16,
-    in_buffer: VecDeque<u8>,
+    in_buffer: Vec<u8>,
 }

 impl Serial {
@@ -227,7 +223,7 @@ impl Serial {
            modem_status: self.modem_status,
            scratch: self.scratch,
            baud_divisor: self.baud_divisor,
-            in_buffer: self.in_buffer.clone(),
+            in_buffer: self.in_buffer.clone().into(),
        }
    }

@@ -240,7 +236,7 @@ impl Serial {
        self.modem_status = state.modem_status;
        self.scratch = state.scratch;
        self.baud_divisor = state.baud_divisor;
-        self.in_buffer = state.in_buffer.clone();
+        self.in_buffer = state.in_buffer.clone().into();
    }
 }

@@ -292,38 +288,12 @@ impl Snapshottable for Serial {
    }

    fn snapshot(&mut self) -> std::result::Result<Snapshot, MigratableError> {
-        let snapshot =
-            serde_json::to_vec(&self.state()).map_err(|e| MigratableError::Snapshot(e.into()))?;
-
-        let mut serial_snapshot = Snapshot::new(self.id.as_str());
-        serial_snapshot.add_data_section(SnapshotDataSection {
-            id: format!("{}-section", self.id),
-            snapshot,
-        });
-
-        Ok(serial_snapshot)
+        Snapshot::new_from_state(&self.id, &self.state())
    }

    fn restore(&mut self, snapshot: Snapshot) -> std::result::Result<(), MigratableError> {
-        if let Some(serial_section) = snapshot.snapshot_data.get(&format!("{}-section", self.id)) {
-            let serial_state = match serde_json::from_slice(&serial_section.snapshot) {
-                Ok(state) => state,
-                Err(error) => {
-                    return Err(MigratableError::Restore(anyhow!(
-                        "Could not deserialize SERIAL {}",
-                        error
-                    )))
-                }
-            };
-
-            self.set_state(&serial_state);
-
-            return Ok(());
-        }
-
-        Err(MigratableError::Restore(anyhow!(
-            "Could not find the serial snapshot section"
-        )))
+        self.set_state(&snapshot.to_state(&self.id)?);
+        Ok(())
    }
 }

--- a/devices/src/legacy/uart_pl011.rs
+++ b/devices/src/legacy/uart_pl011.rs
@@ -0,0 +1,476 @@
+// Copyright 2021 Arm Limited (or its affiliates). All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+//! ARM PrimeCell UART(PL011)
+//!
+//! This module implements an ARM PrimeCell UART(PL011).
+//!
+
+use crate::{read_le_u32, write_le_u32};
+use std::collections::VecDeque;
+use std::fmt;
+use std::sync::{Arc, Barrier};
+use std::{io, result};
+use vm_device::interrupt::InterruptSourceGroup;
+use vm_device::BusDevice;
+use vm_migration::{Migratable, MigratableError, Pausable, Snapshot, Snapshottable, Transportable};
+
+/* Registers */
+const UARTDR: u64 = 0;
+const UARTRSR_UARTECR: u64 = 1;
+const UARTFR: u64 = 6;
+const UARTILPR: u64 = 8;
+const UARTIBRD: u64 = 9;
+const UARTFBRD: u64 = 10;
+const UARTLCR_H: u64 = 11;
+const UARTCR: u64 = 12;
+const UARTIFLS: u64 = 13;
+const UARTIMSC: u64 = 14;
+const UARTRIS: u64 = 15;
+const UARTMIS: u64 = 16;
+const UARTICR: u64 = 17;
+const UARTDMACR: u64 = 18;
+
+const PL011_INT_TX: u32 = 0x20;
+const PL011_INT_RX: u32 = 0x10;
+
+const PL011_FLAG_RXFF: u32 = 0x40;
+const PL011_FLAG_RXFE: u32 = 0x10;
+
+const PL011_ID: [u8; 8] = [0x11, 0x10, 0x14, 0x00, 0x0d, 0xf0, 0x05, 0xb1];
+// We are only interested in the margins.
+const AMBA_ID_LOW: u64 = 0x3f8;
+const AMBA_ID_HIGH: u64 = 0x401;
+
+#[derive(Debug)]
+pub enum Error {
+    BadWriteOffset(u64),
+    DmaNotImplemented,
+    InterruptFailure(io::Error),
+    WriteAllFailure(io::Error),
+    FlushFailure(io::Error),
+}
+
+impl fmt::Display for Error {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        match self {
+            Error::BadWriteOffset(offset) => write!(f, "pl011_write: Bad Write Offset: {}", offset),
+            Error::DmaNotImplemented => write!(f, "pl011: DMA not implemented."),
+            Error::InterruptFailure(e) => write!(f, "Failed to trigger interrupt: {}", e),
+            Error::WriteAllFailure(e) => write!(f, "Failed to write: {}", e),
+            Error::FlushFailure(e) => write!(f, "Failed to flush: {}", e),
+        }
+    }
+}
+
+type Result<T> = result::Result<T, Error>;
+
+/// A PL011 device following the PL011 specification.
+pub struct Pl011 {
+    id: String,
+    flags: u32,
+    lcr: u32,
+    rsr: u32,
+    cr: u32,
+    dmacr: u32,
+    int_enabled: u32,
+    int_level: u32,
+    read_fifo: VecDeque<u8>,
+    ilpr: u32,
+    ibrd: u32,
+    fbrd: u32,
+    ifl: u32,
+    read_count: u32,
+    read_trigger: u32,
+    irq: Arc<Box<dyn InterruptSourceGroup>>,
+    out: Option<Box<dyn io::Write + Send>>,
+}
+
+#[derive(Serialize, Deserialize)]
+pub struct Pl011State {
+    flags: u32,
+    lcr: u32,
+    rsr: u32,
+    cr: u32,
+    dmacr: u32,
+    int_enabled: u32,
+    int_level: u32,
+    read_fifo: Vec<u8>,
+    ilpr: u32,
+    ibrd: u32,
+    fbrd: u32,
+    ifl: u32,
+    read_count: u32,
+    read_trigger: u32,
+}
+
+impl Pl011 {
+    /// Constructs an AMBA PL011 UART device.
+    pub fn new(
+        id: String,
+        irq: Arc<Box<dyn InterruptSourceGroup>>,
+        out: Option<Box<dyn io::Write + Send>>,
+    ) -> Self {
+        Self {
+            id,
+            flags: 0x90u32,
+            lcr: 0u32,
+            rsr: 0u32,
+            cr: 0x300u32,
+            dmacr: 0u32,
+            int_enabled: 0u32,
+            int_level: 0u32,
+            read_fifo: VecDeque::new(),
+            ilpr: 0u32,
+            ibrd: 0u32,
+            fbrd: 0u32,
+            ifl: 0x12u32,
+            read_count: 0u32,
+            read_trigger: 1u32,
+            irq,
+            out,
+        }
+    }
+
+    fn state(&self) -> Pl011State {
+        Pl011State {
+            flags: self.flags,
+            lcr: self.lcr,
+            rsr: self.rsr,
+            cr: self.cr,
+            dmacr: self.dmacr,
+            int_enabled: self.int_enabled,
+            int_level: self.int_level,
+            read_fifo: self.read_fifo.clone().into(),
+            ilpr: self.ilpr,
+            ibrd: self.ibrd,
+            fbrd: self.fbrd,
+            ifl: self.ifl,
+            read_count: self.read_count,
+            read_trigger: self.read_trigger,
+        }
+    }
+
+    fn set_state(&mut self, state: &Pl011State) {
+        self.flags = state.flags;
+        self.lcr = state.lcr;
+        self.rsr = state.rsr;
+        self.cr = state.cr;
+        self.dmacr = state.dmacr;
+        self.int_enabled = state.int_enabled;
+        self.int_level = state.int_level;
+        self.read_fifo = state.read_fifo.clone().into();
+        self.ilpr = state.ilpr;
+        self.ibrd = state.ibrd;
+        self.fbrd = state.fbrd;
+        self.ifl = state.ifl;
+        self.read_count = state.read_count;
+        self.read_trigger = state.read_trigger;
+    }
+
+    /// Queues raw bytes for the guest to read and signals the interrupt
+    pub fn queue_input_bytes(&mut self, c: &[u8]) -> vmm_sys_util::errno::Result<()> {
+        self.read_fifo.extend(c);
+        self.read_count += c.len() as u32;
+        self.flags &= !PL011_FLAG_RXFE;
+
+        if ((self.lcr & 0x10) == 0) || (self.read_count == 16) {
+            self.flags |= PL011_FLAG_RXFF;
+        }
+
+        if self.read_count >= self.read_trigger {
+            self.int_level |= PL011_INT_RX;
+            self.trigger_interrupt()?;
+        }
+
+        Ok(())
+    }
+
+    fn pl011_get_baudrate(&self) -> u32 {
+        if self.fbrd == 0 {
+            return 0;
+        }
+
+        let clk = 24_000_000; // We set the APB_PLCK to 24M in device tree
+        (clk / ((self.ibrd << 6) + self.fbrd)) << 2
+    }
+
+    fn pl011_trace_baudrate_change(&self) {
+        debug!(
+            "=== New baudrate: {:#?} (clk: {:#?}Hz, ibrd: {:#?}, fbrd: {:#?}) ===",
+            self.pl011_get_baudrate(),
+            24_000_000, // We set the APB_PLCK to 24M in device tree
+            self.ibrd,
+            self.fbrd
+        );
+    }
+
+    fn pl011_set_read_trigger(&mut self) {
+        self.read_trigger = 1;
+    }
+
+    fn handle_write(&mut self, offset: u64, val: u32) -> Result<()> {
+        match offset >> 2 {
+            UARTDR => {
+                self.int_level |= PL011_INT_TX;
+                if let Some(out) = self.out.as_mut() {
+                    out.write_all(&[val.to_le_bytes()[0]])
+                        .map_err(Error::WriteAllFailure)?;
+                    out.flush().map_err(Error::FlushFailure)?;
+                }
+            }
+            UARTRSR_UARTECR => {
+                self.rsr = 0;
+            }
+            UARTFR => { /* Writes to Flag register are ignored.*/ }
+            UARTILPR => {
+                self.ilpr = val;
+            }
+            UARTIBRD => {
+                self.ibrd = val;
+                self.pl011_trace_baudrate_change();
+            }
+            UARTFBRD => {
+                self.fbrd = val;
+                self.pl011_trace_baudrate_change();
+            }
+            UARTLCR_H => {
+                /* Reset the FIFO state on FIFO enable or disable */
+                if ((self.lcr ^ val) & 0x10) != 0 {
+                    self.read_count = 0;
+                }
+                self.lcr = val;
+                self.pl011_set_read_trigger();
+            }
+            UARTCR => {
+                self.cr = val;
+            }
+            UARTIFLS => {
+                self.ifl = val;
+                self.pl011_set_read_trigger();
+            }
+            UARTIMSC => {
+                self.int_enabled = val;
+                self.trigger_interrupt().map_err(Error::InterruptFailure)?;
+            }
+            UARTICR => {
+                self.int_level &= !val;
+                self.trigger_interrupt().map_err(Error::InterruptFailure)?;
+            }
+            UARTDMACR => {
+                self.dmacr = val;
+                if (val & 3) != 0 {
+                    return Err(Error::DmaNotImplemented);
+                }
+            }
+            off => {
+                return Err(Error::BadWriteOffset(off));
+            }
+        }
+        Ok(())
+    }
+
+    fn trigger_interrupt(&mut self) -> result::Result<(), io::Error> {
+        self.irq.trigger(0)
+    }
+}
+
+impl BusDevice for Pl011 {
+    fn read(&mut self, _base: u64, offset: u64, data: &mut [u8]) {
+        let v;
+        let mut read_ok = true;
+        if (AMBA_ID_LOW..AMBA_ID_HIGH).contains(&(offset >> 2)) {
+            let index = ((offset - 0xfe0) >> 2) as usize;
+            v = u32::from(PL011_ID[index]);
+        } else {
+            v = match offset >> 2 {
+                UARTDR => {
+                    let c: u32;
+                    let r: u32;
+
+                    self.flags &= !PL011_FLAG_RXFF;
+                    c = self.read_fifo.pop_front().unwrap_or_default().into();
+                    if self.read_count > 0 {
+                        self.read_count -= 1;
+                    }
+                    if self.read_count == 0 {
+                        self.flags |= PL011_FLAG_RXFE;
+                    }
+                    if self.read_count == (self.read_trigger - 1) {
+                        self.int_level &= !PL011_INT_RX;
+                    }
+                    self.rsr = c >> 8;
+                    r = c;
+                    r
+                }
+                UARTRSR_UARTECR => self.rsr,
+                UARTFR => self.flags,
+                UARTILPR => self.ilpr,
+                UARTIBRD => self.ibrd,
+                UARTFBRD => self.fbrd,
+                UARTLCR_H => self.lcr,
+                UARTCR => self.cr,
+                UARTIFLS => self.ifl,
+                UARTIMSC => self.int_enabled,
+                UARTRIS => self.int_level,
+                UARTMIS => (self.int_level & self.int_enabled),
+                UARTDMACR => self.dmacr,
+                _ => {
+                    read_ok = false;
+                    0
+                }
+            }
+        }
+
+        if read_ok && data.len() <= 4 {
+            write_le_u32(data, v);
+        } else {
+            warn!(
+                "Invalid PL011 read: offset {}, data length {}",
+                offset,
+                data.len()
+            );
+        }
+    }
+
+    fn write(&mut self, _base: u64, offset: u64, data: &[u8]) -> Option<Arc<Barrier>> {
+        if data.len() <= 4 {
+            let v = read_le_u32(&data);
+            if let Err(e) = self.handle_write(offset, v) {
+                warn!("Failed to write to PL011 device: {}", e);
+            }
+        } else {
+            warn!(
+                "Invalid PL011 write: offset {}, data length {}",
+                offset,
+                data.len()
+            );
+        }
+
+        None
+    }
+}
+
+impl Snapshottable for Pl011 {
+    fn id(&self) -> String {
+        self.id.clone()
+    }
+
+    fn snapshot(&mut self) -> std::result::Result<Snapshot, MigratableError> {
+        Snapshot::new_from_state(&self.id, &self.state())
+    }
+
+    fn restore(&mut self, snapshot: Snapshot) -> std::result::Result<(), MigratableError> {
+        self.set_state(&snapshot.to_state(&self.id)?);
+        Ok(())
+    }
+}
+
+impl Pausable for Pl011 {}
+impl Transportable for Pl011 {}
+impl Migratable for Pl011 {}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use std::io;
+    use std::sync::{Arc, Mutex};
+    use vm_device::interrupt::{InterruptIndex, InterruptSourceConfig};
+    use vmm_sys_util::eventfd::EventFd;
+
+    const SERIAL_NAME: &str = "serial";
+
+    struct TestInterrupt {
+        event_fd: EventFd,
+    }
+
+    impl InterruptSourceGroup for TestInterrupt {
+        fn trigger(&self, _index: InterruptIndex) -> result::Result<(), std::io::Error> {
+            self.event_fd.write(1)
+        }
+        fn update(
+            &self,
+            _index: InterruptIndex,
+            _config: InterruptSourceConfig,
+        ) -> result::Result<(), std::io::Error> {
+            Ok(())
+        }
+        fn notifier(&self, _index: InterruptIndex) -> Option<EventFd> {
+            Some(self.event_fd.try_clone().unwrap())
+        }
+    }
+
+    impl TestInterrupt {
+        fn new(event_fd: EventFd) -> Self {
+            TestInterrupt { event_fd }
+        }
+    }
+
+    #[derive(Clone)]
+    struct SharedBuffer {
+        buf: Arc<Mutex<Vec<u8>>>,
+    }
+
+    impl SharedBuffer {
+        fn new() -> SharedBuffer {
+            SharedBuffer {
+                buf: Arc::new(Mutex::new(Vec::new())),
+            }
+        }
+    }
+
+    impl io::Write for SharedBuffer {
+        fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
+            self.buf.lock().unwrap().write(buf)
+        }
+        fn flush(&mut self) -> io::Result<()> {
+            self.buf.lock().unwrap().flush()
+        }
+    }
+
+    #[test]
+    fn pl011_output() {
+        let intr_evt = EventFd::new(0).unwrap();
+        let pl011_out = SharedBuffer::new();
+        let mut pl011 = Pl011::new(
+            String::from(SERIAL_NAME),
+            Arc::new(Box::new(TestInterrupt::new(intr_evt.try_clone().unwrap()))),
+            Some(Box::new(pl011_out.clone())),
+        );
+
+        pl011.write(0, UARTDR as u64, &[b'x', b'y']);
+        pl011.write(0, UARTDR as u64, &[b'a']);
+        pl011.write(0, UARTDR as u64, &[b'b']);
+        pl011.write(0, UARTDR as u64, &[b'c']);
+        assert_eq!(
+            pl011_out.buf.lock().unwrap().as_slice(),
+            &[b'x', b'a', b'b', b'c']
+        );
+    }
+
+    #[test]
+    fn pl011_input() {
+        let intr_evt = EventFd::new(0).unwrap();
+        let pl011_out = SharedBuffer::new();
+        let mut pl011 = Pl011::new(
+            String::from(SERIAL_NAME),
+            Arc::new(Box::new(TestInterrupt::new(intr_evt.try_clone().unwrap()))),
+            Some(Box::new(pl011_out)),
+        );
+
+        // write 1 to the interrupt event fd, so that read doesn't block in case the event fd
+        // counter doesn't change (for 0 it blocks)
+        assert!(intr_evt.write(1).is_ok());
+        pl011.queue_input_bytes(&[b'a', b'b', b'c']).unwrap();
+
+        assert_eq!(intr_evt.read().unwrap(), 2);
+
+        let mut data = [0u8];
+        pl011.read(0, UARTDR as u64, &mut data);
+        assert_eq!(data[0], b'a');
+        pl011.read(0, UARTDR as u64, &mut data);
+        assert_eq!(data[0], b'b');
+        pl011.read(0, UARTDR as u64, &mut data);
+        assert_eq!(data[0], b'c');
+    }
+}
--- a/devices/src/lib.rs
+++ b/devices/src/lib.rs
@@ -35,7 +35,7 @@ pub mod ioapic;
 pub mod legacy;

 #[cfg(feature = "acpi")]
-pub use self::acpi::{AcpiGEDDevice, AcpiPMTimerDevice, AcpiShutdownDevice};
+pub use self::acpi::{AcpiGedDevice, AcpiPmTimerDevice, AcpiShutdownDevice};

 bitflags! {
    pub struct AcpiNotificationFlags: u8 {
@@ -46,3 +46,63 @@ bitflags! {
        const POWER_BUTTON_CHANGED = 0b1000;
    }
 }
+
+#[allow(unused_macros)]
+#[cfg(target_arch = "aarch64")]
+macro_rules! generate_read_fn {
+    ($fn_name: ident, $data_type: ty, $byte_type: ty, $type_size: expr, $endian_type: ident) => {
+        #[allow(dead_code)]
+        pub fn $fn_name(input: &[$byte_type]) -> $data_type {
+            assert!($type_size == std::mem::size_of::<$data_type>());
+            let mut array = [0u8; $type_size];
+            for (byte, read) in array.iter_mut().zip(input.iter().cloned()) {
+                *byte = read as u8;
+            }
+            <$data_type>::$endian_type(array)
+        }
+    };
+}
+
+#[allow(unused_macros)]
+#[cfg(target_arch = "aarch64")]
+macro_rules! generate_write_fn {
+    ($fn_name: ident, $data_type: ty, $byte_type: ty, $endian_type: ident) => {
+        #[allow(dead_code)]
+        pub fn $fn_name(buf: &mut [$byte_type], n: $data_type) {
+            for (byte, read) in buf
+                .iter_mut()
+                .zip(<$data_type>::$endian_type(n).iter().cloned())
+            {
+                *byte = read as $byte_type;
+            }
+        }
+    };
+}
+
+#[cfg(target_arch = "aarch64")]
+generate_read_fn!(read_le_u16, u16, u8, 2, from_le_bytes);
+#[cfg(target_arch = "aarch64")]
+generate_read_fn!(read_le_u32, u32, u8, 4, from_le_bytes);
+#[cfg(target_arch = "aarch64")]
+generate_read_fn!(read_le_u64, u64, u8, 8, from_le_bytes);
+#[cfg(target_arch = "aarch64")]
+generate_read_fn!(read_le_i32, i32, i8, 4, from_le_bytes);
+
+#[cfg(target_arch = "aarch64")]
+generate_read_fn!(read_be_u16, u16, u8, 2, from_be_bytes);
+#[cfg(target_arch = "aarch64")]
+generate_read_fn!(read_be_u32, u32, u8, 4, from_be_bytes);
+
+#[cfg(target_arch = "aarch64")]
+generate_write_fn!(write_le_u16, u16, u8, to_le_bytes);
+#[cfg(target_arch = "aarch64")]
+generate_write_fn!(write_le_u32, u32, u8, to_le_bytes);
+#[cfg(target_arch = "aarch64")]
+generate_write_fn!(write_le_u64, u64, u8, to_le_bytes);
+#[cfg(target_arch = "aarch64")]
+generate_write_fn!(write_le_i32, i32, i8, to_le_bytes);
+
+#[cfg(target_arch = "aarch64")]
+generate_write_fn!(write_be_u16, u16, u8, to_be_bytes);
+#[cfg(target_arch = "aarch64")]
+generate_write_fn!(write_be_u32, u32, u8, to_be_bytes);
--- a/docs/device_model.md
+++ b/docs/device_model.md
@@ -28,9 +28,13 @@ This document describes the device model supported by `cloud-hypervisor`.
 ### Serial port

 Simple emulation of a serial port by reading and writing to specific port I/O
-addresses. Used as the default console for Linux when booting with the option
-`console=ttyS0`, the serial port can be very useful to gather early logs from
-the operating system booted inside the VM.
+addresses. The serial port can be very useful to gather early logs from the
+operating system booted inside the VM.
+
+For x86_64, The default serial port is from an emulated 16550A device. It can
+be used as the default console for Linux when booting with the option
+`console=ttyS0`. For AArch64, the default serial port is from an emulated
+PL011 UART device. The related command line for AArch64 is `console=ttyAMA0`.

 This device is always built-in, and it is disabled by default. It can be
 enabled with the `--serial` option, as long as its parameter is not `off`.
@@ -44,6 +48,10 @@ This device is built-in by default, but it can be compiled out with Rust
 features. When compiled in, it is always enabled, and cannot be disabled
 from the command line.

+For AArch64 machines, an ARM PrimeCell Real Time Clock(PL031) is implemented.
+This device is built-in by default for the AArch64 platform, and it is always
+enabled, and cannot be disabled from the command line.
+
 ### I/O APIC

 `cloud-hypervisor` supports a so-called split IRQ chip implementation by
@@ -65,6 +73,11 @@ enabled by default, the handling of reboot/shutdown goes through the dedicated
 ACPI device. In case ACPI is disabled, this device is enabled to bring to the
 VM some reboot/shutdown support.

+### ARM PrimeCell General Purpose Input/Output (PL061)
+
+Simplified ARM PrimeCell GPIO (PL061) implementation. Only supports key 3 to
+trigger a graceful shutdown of the AArch64 guest.
+
 ### ACPI device

 This is a dedicated device for handling ACPI shutdown and reboot when ACPI is
@@ -100,9 +113,8 @@ selecting `--serial tty --console off` from the command line.
 ### virtio-iommu

 As we want to improve our nested guests support, we added support for exposing
-a [paravirtualized IOMMU](https://github.com/cloud-hypervisor/cloud-hypervisor/blob/master/docs/iommu.md)
-device through virtio. This allows for a safer nested virtio and directly
-assigned devices support.
+a [paravirtualized IOMMU](iommu.md) device through virtio. This allows for a
+safer nested virtio and directly assigned devices support.

 This device is always built-in, and it is enabled based on the presence of the
 parameter `iommu=on` in any of the virtio or VFIO devices. If at least one of
@@ -171,8 +183,8 @@ This device is always built-in, and it is enabled when `vhost_user=true` and
 shared file system, allowing for an efficient and reliable way of sharing
 a filesystem between the host and the cloud-hypervisor guest.

-See our [filesystem sharing](https://github.com/cloud-hypervisor/cloud-hypervisor/blob/master/docs/fs.md)
-documentation for more details on how to use virtio-fs with cloud-hypervisor.
+See our [filesystem sharing](fs.md) documentation for more details on how to
+use virtio-fs with cloud-hypervisor.

 This device is always built-in, and it is enabled based on the presence of the
 flag `--fs`.
@@ -193,9 +205,8 @@ VFIO (Virtual Function I/O) is a kernel framework that exposes direct device
 access to userspace. `cloud-hypervisor` uses VFIO to directly assign host
 physical devices into its guest.

-See our [VFIO documentation](https://github.com/cloud-hypervisor/cloud-hypervisor/blob/master/docs/vfio.md)
-for more details on how to directly assign host devices to `cloud-hypervisor`
-guests.
+See our [VFIO documentation](vfio.md) for more details on how to directly
+assign host devices to `cloud-hypervisor` guests.

 Because VFIO implies `vfio-pci` in the `cloud-hypervisor` context, the VFIO
 support is built-in when the `pci` feature is selected. And because the `pci`
--- a/docs/fs.md
+++ b/docs/fs.md
@@ -12,7 +12,7 @@ This virtual device relies on the _vhost-user_ protocol, which assumes the backe

 _Build virtiofsd_
 ```bash
-git clone --depth 1 "https://github.com/sboeuf/qemu.git" -b "virtio-fs" $VIRTIOFSD_DIR
+git clone --depth 1 "https://gitlab.com/virtio-fs/qemu.git" -b "qemu5.0-virtiofs-dax" $VIRTIOFSD_DIR
 cd $VIRTIOFSD_DIR
 ./configure --prefix=$PWD --target-list=x86_64-softmmu
 make virtiofsd -j `nproc`
@@ -35,45 +35,42 @@ The `cache=none` option should be the default when using `virtiofsd` with the __

 The `cache=always` option will allow for the guest page cache to be used, which will increase the memory footprint of the guest. This option should be used only for specific use cases where a single VM is going to be running on a host.

-### The kernel
+### Kernel support

-In order to leverage __virtio-fs__ support from within the guest, and because the code has not been merged in upstream Linux kernel yet, it is required to build a custom kernel embedding the patches.
-
-The following branch `virtio-fs-virtio-iommu` on the repository https://github.com/cloud-hypervisor/linux.git includes all the needed patches to support __virtio-fs__.
-
-Make sure to build a kernel out of this branch that can be then used to boot the VM.
+Modern Linux kernels starting (at least v5.10) have support for virtio-fs. Use
+of older kernels, with additional patches, are not supported.

 ## How to share directories with cloud-hypervisor

 ### Start the VM
 Once the daemon is running, the option `--fs` from __cloud-hypervisor__ needs to be used.

-Direct kernel boot option is preferred since we need to provide the custom kernel including the __virtio-fs__ patches. We could boot from `hypervisor-fw` if we had previously edited the image to replace the kernel binary.
+Direct kernel boot is the preferred option, but we can boot from an EFI cloud image if it contains a recent enough kernel.

-Because _vhost-user_ expects a dedicated process (__virtiofsd__ in this case) to be able to access the guest RAM to communicate through the _virtqueues_ with the driver running in the guest, `--memory` option needs to be slightly modified. It needs to specify a backing file for the memory so that an external process can access it.
+Because _vhost-user_ expects a dedicated process (__virtiofsd__ in this case) to be able to access the guest RAM to communicate through the _virtqueues_ with the driver running in the guest, `--memory` option needs to be slightly modified. It must specify `shared=on` to share the memory pages so that an external process can access them.

-Assuming you have `focal-server-cloudimg-amd64.raw` and `custom-vmlinux.bin` on your system, here is the __cloud-hypervisor__ command you need to run:
+Assuming you have `focal-server-cloudimg-amd64.raw` and `vmlinux` on your system, here is the __cloud-hypervisor__ command you need to run:
 ```bash
 ./cloud-hypervisor \
-    --cpus 4 \
-    --memory "size=512M,shared=on" \
+    --cpus boot=1 \
+    --memory size=1G,shared=on \
    --disk path=focal-server-cloudimg-amd64.raw \
-    --kernel custom-vmlinux.bin \
-    --cmdline "console=ttyS0 console=hvc0 root=/dev/vda1 rw" \
+    --kernel vmlinux \
+    --cmdline "console=hvc0 root=/dev/vda1 rw" \
    --fs tag=myfs,socket=/tmp/virtiofs,num_queues=1,queue_size=512
 ```

 By default, DAX is enabled with a cache window of 8GiB. You can specify a custom size (let's say 4GiB for this example) for the cache by explicitly setting DAX and the cache size:

 ```bash
--fs tag=virtiofs,socket=/tmp/virtiofs,num_queues=1,queue_size=512,dax=on,cache_size=4G
+--fs tag=myfs,socket=/tmp/virtiofs,num_queues=1,queue_size=512,dax=on,cache_size=4G

 ```

 In case you don't want to use a shared window of cache to pass the shared files content, this means you will have to explicitly disable DAX with `dax=off`. Note that in this case, the `cache_size` parameter will be ignored.

 ```bash
--fs tag=virtiofs,socket=/tmp/virtiofs,num_queues=1,queue_size=512,dax=off
+--fs tag=myfs,socket=/tmp/virtiofs,num_queues=1,queue_size=512,dax=off

 ```

--- a/docs/hotplug.md
+++ b/docs/hotplug.md
@@ -27,7 +27,7 @@ $ ./cloud-hypervisor/target/release/cloud-hypervisor \
 	--memory size=1024M \
 	--net "tap=,mac=,ip=,mask=" \
 	--rng \
-    --api-socket=/tmp/ch-socket
+	--api-socket=/tmp/ch-socket
 $ popd
 ```

@@ -36,7 +36,7 @@ Notice the addition of `--api-socket=/tmp/ch-socket` and a `max` parameter on `-
 To ask the VMM to add additional vCPUs then use the resize API:

 ```shell
-curl -H "Accept: application/json" -H "Content-Type: application/json" -i -XPUT --unix-socket /tmp/ch-socket -d "{ \"desired_vcpus\":8}" http://localhost/api/v1/vm.resize
+./ch-remote --api-socket=/tmp/ch-socket resize --cpus 8
 ```

 The extra vCPU threads will be created and advertised to the running kernel. The kernel does not bring up the CPUs immediately and instead the user must "online" them from inside the VM:
@@ -56,19 +56,23 @@ After a reboot the added CPUs will remain.
 Removing CPUs works similarly by reducing the number in the "desired_vcpus" field of the reisze API. The CPUs will be automatically offlined inside the guest so there is no need to run any commands inside the guest:

 ```shell
-curl -H "Accept: application/json" -H "Content-Type: application/json" -i -XPUT --unix-socket /tmp/ch-socket -d "{ \"desired_vcpus\":2}" http://localhost/api/v1/vm.resize
+./ch-remote --api-socket=/tmp/ch-socket resize --cpus 2
 ```

 As per adding CPUs to the guest, after a reboot the VM will be running with the reduced number of vCPUs.

 ## Memory Hot Plug

+### ACPI method
+
 Extra memory can be added from a running Cloud Hypervisor instance. This is controlled by two mechanisms:

 1. Allocating some of the guest physical address space for hotplug memory.
 2. Making a HTTP API request to the VMM to ask for a new amount of RAM to be assigned to the VM. In the case of expanding the memory for the VM the new memory will be hotplugged into the running VM, if reducing the size of the memory then change will take effect after the next reboot.

-To use memory hotplug start the VM specifying some size RAM in the "hotplug_size" parameter to the memory configuration. Not all the memory specified in this parameter will be available to hotplug as there are spacing and alignment requirements so it is recommended to make it larger than the hotplug RAM needed.
+To use memory hotplug start the VM specifying some size RAM in the `hotplug_size` parameter to the memory configuration. Not all the memory specified in this parameter will be available to hotplug as there are spacing and alignment requirements so it is recommended to make it larger than the hotplug RAM needed.
+
+Because the ACPI method is the default, there is no need to add the extra option `hotplug_method=acpi`.

 ```shell
 $ pushd $CLOUDH
@@ -81,7 +85,7 @@ $ ./cloud-hypervisor/target/release/cloud-hypervisor \
 	--memory size=1024M,hotplug_size=8192M \
 	--net "tap=,mac=,ip=,mask=" \
 	--rng \
-    --api-socket=/tmp/ch-socket
+	--api-socket=/tmp/ch-socket
 $ popd
 ```

@@ -91,10 +95,10 @@ Before issuing the API request it is necessary to run the following command insi
 root@ch-guest ~ # echo online | sudo tee /sys/devices/system/memory/auto_online_blocks
 ```

-To ask the VMM to add expand the RAM for the VM  (request is in bytes):
+To ask the VMM to expand the RAM for the VM:

 ```shell
-curl -H "Accept: application/json" -H "Content-Type: application/json" -i -XPUT --unix-socket /tmp/ch-socket -d "{ \"desired_vcpus\": 4, \"desired_ram\" : 3221225472}" http://localhost/api/v1/vm.resize
+./ch-remote --api-socket=/tmp/ch-socket resize --memory 3G
 ```

 The new memory is now available to use inside the VM:
@@ -111,3 +115,134 @@ Due to guest OS limitations is is necessary to ensure that amount of memory adde
 The same API can also be used to reduce the desired RAM for a VM but the change will not be applied until the VM is rebooted.

 Memory and CPU resizing can be combined together into the same HTTP API request.
+
+### virtio-mem method
+
+Extra memory can be added and removed from a running Cloud Hypervisor instance. This is controlled by two mechanisms:
+
+1. Allocating some of the guest physical address space for hotplug memory.
+2. Making a HTTP API request to the VMM to ask for a new amount of RAM to be assigned to the VM.
+
+To use memory hotplug start the VM specifying some size RAM in the `hotplug_size` parameter along with `hotplug_method=virtio-mem` to the memory configuration.
+
+```shell
+$ pushd $CLOUDH
+$ sudo setcap cap_net_admin+ep ./cloud-hypervisor/target/release/cloud-hypervisor
+$ ./cloud-hypervisor/target/release/cloud-hypervisor \
+	--kernel custom-vmlinux.bin \
+	--cmdline "console=ttyS0 console=hvc0 root=/dev/vda1 rw" \
+	--disk path=focal-server-cloudimg-amd64.raw \
+	--memory size=1024M,hotplug_size=8192M,hotplug_method=virtio-mem \
+	--net "tap=,mac=,ip=,mask=" \
+	--api-socket=/tmp/ch-socket
+$ popd
+```
+
+To ask the VMM to expand the RAM for the VM (request is in bytes):
+
+```shell
+./ch-remote --api-socket=/tmp/ch-socket resize --memory 3G
+```
+
+The new memory is now available to use inside the VM:
+
+```shell
+free -h
+              total        used        free      shared  buff/cache   available
+Mem:          3.0Gi        71Mi       2.8Gi       0.0Ki        47Mi       2.8Gi
+Swap:          32Mi          0B        32Mi
+```
+
+The same API can also be used to reduce the desired RAM for a VM. It is important to note that reducing RAM size might only partially work, as the guest might be using some of it.
+
+## PCI Device Hot Plug
+
+Extra PCI devices can be added and removed from a running Cloud Hypervisor instance. This is controlled by making a HTTP API request to the VMM to ask for the additional device to be added, or for the existing device to be removed.
+
+To use PCI device hotplug start the VM with the HTTP server.
+
+```shell
+$ sudo setcap cap_net_admin+ep ./cloud-hypervisor/target/release/cloud-hypervisor
+$ ./cloud-hypervisor/target/release/cloud-hypervisor \
+	--kernel custom-vmlinux.bin \
+	--cmdline "console=ttyS0 console=hvc0 root=/dev/vda1 rw" \
+	--disk path=focal-server-cloudimg-amd64.raw \
+	--cpus boot=4 \
+	--memory size=1024M \
+	--net "tap=,mac=,ip=,mask=" \
+	--api-socket=/tmp/ch-socket
+```
+
+Notice the addition of `--api-socket=/tmp/ch-socket`.
+
+### Add VFIO Device
+
+To ask the VMM to add additional VFIO device then use the `add-device` API.
+
+```shell
+./ch-remote --api-socket=/tmp/ch-socket add-device path=/sys/bus/pci/devices/0000:01:00.0/
+```
+
+### Add Disk Device
+
+To ask the VMM to add additional disk device then use the `add-disk` API.
+
+```shell
+./ch-remote --api-socket=/tmp/ch-socket add-disk path=/foo/bar/cloud.img
+```
+
+### Add Fs Device
+
+To ask the VMM to add additional fs device then use the `add-fs` API.
+
+```shell
+./ch-remote --api-socket=/tmp/ch-socket add-fs tag=myfs,socket=/foo/bar/virtiofs.sock
+```
+
+### Add Net Device
+
+To ask the VMM to add additional network device then use the `add-net` API.
+
+```shell
+./ch-remote --api-socket=/tmp/ch-socket add-net tap=chtap0
+```
+
+### Add Pmem Device
+
+To ask the VMM to add additional PMEM device then use the `add-pmem` API.
+
+```shell
+./ch-remote --api-socket=/tmp/ch-socket add-pmem file=/foo/bar.cloud.img
+```
+
+### Add Vsock Device
+
+To ask the VMM to add additional vsock device then use the `add-vsock` API.
+
+```shell
+./ch-remote --api-socket=/tmp/ch-socket add-vsock cid=3,socket=/foo/bar/vsock.sock
+```
+
+### Common Across All PCI Devices
+
+The extra PCI device will be created and advertised to the running kernel. The new device can be found by checking the list of PCI devices.
+
+```shell
+root@ch-guest ~ # lspci
+00:00.0 Host bridge: Intel Corporation Device 0d57
+00:01.0 Unassigned class [ffff]: Red Hat, Inc. Virtio console (rev 01)
+00:02.0 Mass storage controller: Red Hat, Inc. Virtio block device (rev 01)
+00:03.0 Unassigned class [ffff]: Red Hat, Inc. Virtio RNG (rev 01)
+```
+
+After a reboot the added PCI device will remain.
+
+### Remove PCI device
+
+Removing a PCI device works the same way for all kind of PCI devices. The unique identifier related to the device must be provided. This identifier can be provided by the user when adding the new device, or by default Cloud Hypervisor will assign one.
+
+```shell
+./ch-remote --api-socket=/tmp/ch-socket remove-device _disk0
+```
+
+As per adding a PCI device to the guest, after a reboot the VM will be running without the removed PCI device.
--- a/docs/intel_sgx.md
+++ b/docs/intel_sgx.md
@@ -32,7 +32,7 @@ memory, the second one being 32MiB with no pre-allocated memory.
    --cpus boot=1 \
    --memory size=1G \
    --disk path=focal-server-cloudimg-amd64.raw \
-    --kernel bzImage \
+    --kernel vmlinux \
    --cmdline "console=ttyS0 console=hvc0 root=/dev/vda1 rw" \
    --sgx-epc size=64M,prefault=on size=32M,prefault=off
 ```
--- a/docs/io_throttling.md
+++ b/docs/io_throttling.md
@@ -0,0 +1,44 @@
+# I/O Throttling
+
+Cloud Hypervisor now supports I/O throttling on virtio-block and virtio-net
+devices. This support is based on the [`rate-limiter` module](https://github.com/firecracker-microvm/firecracker/tree/master/src/rate_limiter)
+from Firecracker. This document explains the user interface of this
+feature, and highlights some internal implementations that can help users
+better understand the expected behavior of I/O throttling in practice.
+
+Cloud Hypervisor allows to limit both the I/O bandwidth (e.g. bytes/s)
+and I/O operations (ops/s) independently. For virtio-net devices, while
+sharing the same "rate limit" from user inputs (on both bandwidth and
+operations), the RX and TX queues are throttled independently.
+To limit the I/O bandwidth, Cloud Hypervisor
+provides three user options, i.e., `bw_size` (bytes), `bw_one_time_burst`
+(bytes), and `bw_refill_time` (ms). Both `bw_size` and `bw_refill_time`
+are required, while `bw_one_time_burst` is optional.
+Internally, these options define a TokenBucket with a maximum capacity
+(`bw_size` bytes), an initial burst size (`bw_one_time_burst`) and an
+interval for refilling purposes (`bw_refill_time`). The "refill-rate" is
+`bw_size` bytes per `bw_refill_time` ms, and it is the constant rate at
+which the tokens replenish. The refill process only starts happening
+after the initial burst budget is consumed. Consumption from the token
+bucket is unbounded in speed which allows for bursts bound in size by
+the amount of tokens available. Once the token bucket is empty,
+consumption speed is bound by the "refill-rate". Similarly, Cloud
+Hypervisor provides another three options for limiting I/O operations,
+i.e., `ops_size` (I/O operations), `bw_one_time_burst` (I/O operations),
+and `bw_refill_time` (ms).
+
+One caveat in the I/O throttling is that every-time the bucket gets
+empty, it will stop I/O operations for a fixed amount of time
+(`cool_down_time`). The `cool_down_time` now is fixed at `100 ms`, it
+can have big implications to the actual rate limit (which can be a lot
+different the expected "refill-rate" derived from user inputs). For
+example, to have a 1000 IOPS limit on a virtio-blk device, users should
+be able to provide either of the following two options:
+`ops_size=1000,ops_refill_time=1000` or
+`ops_size=10,ops_refill_time=10`. However, the actual IOPS limits are
+likely to be ~1000 IOPS and ~100 IOPS respectively. The reason is the
+actual rate limit users get can be as low as
+`ops_size/(ops_refill_time+cool_down_time)`. As a result, it is
+generally advisable to keep `bw/ops_refill_time` larger than `100 ms`
+(`cool_down_time`) to make sure the actual rate limit is close to users'
+expectation ("refill-rate").
--- a/docs/iommu.md
+++ b/docs/iommu.md
@@ -89,7 +89,7 @@ virtual IOMMU:
    --cpus boot=1 \
    --memory size=512M \
    --disk path=focal-server-cloudimg-amd64.raw,iommu=on \
-    --kernel custom-bzImage \
+    --kernel custom-vmlinux \
    --cmdline "console=ttyS0 console=hvc0 root=/dev/vda1 rw" \
 ```

@@ -166,7 +166,7 @@ be consumed.
    --cpus boot=1 \
    --memory size=8G,hugepages=on \
    --disk path=focal-server-cloudimg-amd64.raw \
-    --kernel custom-bzImage \
+    --kernel custom-vmlinux \
    --cmdline "console=ttyS0 console=hvc0 root=/dev/vda1 rw hugepagesz=2M hugepages=2048" \
    --net tap=,mac=,iommu=on
 ```
@@ -183,7 +183,7 @@ passing through is `0000:00:01.0`.
    --cpus boot=1 \
    --memory size=8G,hugepages=on \
    --disk path=focal-server-cloudimg-amd64.raw \
-    --kernel custom-bzImage \
+    --kernel custom-vmlinux \
    --cmdline "console=ttyS0 console=hvc0 root=/dev/vda1 rw kvm-intel.nested=1 vfio_iommu_type1.allow_unsafe_interrupts rw hugepagesz=2M hugepages=2048" \
    --device path=/sys/bus/pci/devices/0000:00:01.0,iommu=on
 ```
@@ -194,6 +194,7 @@ guest, and bind it to VFIO (it should appear as `0000:00:04.0`).
 ```bash
 echo 0000:00:04.0 > /sys/bus/pci/devices/0000\:00\:04.0/driver/unbind
 echo 8086 1502 > /sys/bus/pci/drivers/vfio-pci/new_id
+echo 0000:00:04.0 > /sys/bus/pci/drivers/vfio-pci/bind
 ```

 Last thing is to start the L2 guest with the huge pages memory backend.
@@ -203,7 +204,7 @@ Last thing is to start the L2 guest with the huge pages memory backend.
    --cpus boot=1 \
    --memory size=4G,hugepages=on \
    --disk path=focal-server-cloudimg-amd64.raw \
-    --kernel custom-bzImage \
+    --kernel custom-vmlinux \
    --cmdline "console=ttyS0 console=hvc0 root=/dev/vda1 rw" \
    --device path=/sys/bus/pci/devices/0000:00:04.0
 ```
--- a/docs/networking.md
+++ b/docs/networking.md
@@ -2,7 +2,7 @@

 cloud-hypervisor can emulate one or more virtual network interfaces, represented at the hypervisor host by [tap devices](https://www.kernel.org/doc/Documentation/networking/tuntap.txt). This guide briefly describes, in a manual and distribution neutral way, how to setup and use networking with cloud-hypervisor.

-## Multiple queue support for net devices ##
+## Multiple queue support for net devices

 While multiple vcpus defined for guest, to gain the benefit of vcpu scalable to improve performance, it suggests to define multiple queue pairs for net devices, one Tx/Rx queue pair per one vcpu, that means the number of queue pairs at least is equal to the vcpu count. In that case, after virtnet driver set cpu affinity for virtqueues in guest kernel, vcpus could handle interrupt from different virtqueue pairs in parallel.

@@ -23,7 +23,7 @@ Use one `--net` command-line argument from cloud-hypervisor to specify the emula

 ```bash
 ./cloud-hypervisor \
-    --cpus 4 \
+    --cpus boot=4 \
    --memory "size=512M" \
    --disk path=focal-server-cloudimg-amd64.raw \
    --kernel my-vmlinux.bin \
@@ -34,14 +34,14 @@ Use one `--net` command-line argument from cloud-hypervisor to specify the emula

 The `--net` argument takes 1 or more space-separated strings of key value pairs containing the following 4 keys or fields:

-| Name       | Purpose                    | Optional  |
-| -----------|----------------------------| ----------|
-| tap        | tap device name            | Yes       |
-| mac        | vNIC mac address           | Yes       |
-| ip         | tap IP IP address          | yes       |
-| mask       | tap IP netmask             | Yes       |
-| num_queues | the number of queues       | yes       |
-| queue_size | the size of each queue     | Yes       |
+| Name       | Purpose                | Optional |
+| ---------- | ---------------------- | -------- |
+| tap        | tap device name        | Yes      |
+| mac        | vNIC mac address       | Yes      |
+| ip         | tap IP IP address      | yes      |
+| mask       | tap IP netmask         | Yes      |
+| num_queues | the number of queues   | yes      |
+| queue_size | the size of each queue | Yes      |

 num_queues is the total number of tx and rx queues, the default value is 2, and it could be increased by multiples of 2. Additionally, num_queues is suggested to be as 2 times of vcpu count. The default value for queue_size is 256.

@@ -104,6 +104,7 @@ bridge name     bridge id               STP enabled     interfaces
 ich-dpl         8000.067afc1b9a67       no              ich1
 ich-int         8000.725412ffce6f       no              ich0
 ```
+
 This completes the layer 2 wiring: The cloud-hypervisor is now connected to the hypervisor host via the 2 linux bridges.

 ## IP (Layer 3) provisioning
@@ -116,6 +117,7 @@ On the hypervisor host add the network gateway IP address of each network to the
 root@host:~# ip addr add 192.168.4.1/24 dev ich-int
 root@host:~# ip addr add 10.0.1.1/24 dev ich-dpl
 ```
+
 The routing table of the hypervisor host should now also have corresponding routing entries:

 ```bash
@@ -127,9 +129,10 @@ Destination     Gateway         Genmask         Flags Metric Ref    Use Iface
 192.168.4.0     0.0.0.0         255.255.255.0   U     0      0        0 ich-int
 192.168.178.0   0.0.0.0         255.255.255.0   U     600    0        0 wlan1
 ```
+
 ### Virtual Machine

-Within the virtual machine set the vNIC's to up state and provision the corresponding IP addresses on the 2 vNIC's. The steps outlined below use the ip command as an example. Alternative distribution specific procedures can also apply.   
+Within the virtual machine set the vNIC's to up state and provision the corresponding IP addresses on the 2 vNIC's. The steps outlined below use the ip command as an example. Alternative distribution specific procedures can also apply.

 ```bash
 root@guest:~# ip link set up enp0s2
@@ -165,7 +168,7 @@ root@192.168.4.2's password:
 Linux cloud-hypervisor 5.2.0 #2 SMP Thu Jul 11 08:08:16 CEST 2019 x86_64

 Debian GNU/Linux comes with ABSOLUTELY NO WARRANTY, to the extent
-permitted by applicable law.                                                         
+permitted by applicable law.


 Last login: Fri Jul 12 13:27:56 2019 from 192.168.4.1
@@ -189,7 +192,7 @@ nameserver 192.168.178.1
 make sure that the default gateway of the hypervisor host (in this example host 192.168.178.1 which is an adsl router) has an entry in the routing table for the 192.168.4.0/24 network otherwise IP connectivity will not work.

 ```bash
-root@guest:~# nslookup  ftp.nl.debian.org       
+root@guest:~# nslookup  ftp.nl.debian.org
 Server:         192.168.178.1
 Address:        192.168.178.1#53

--- a/docs/seccomp.md
+++ b/docs/seccomp.md
@@ -0,0 +1,68 @@
+# Seccomp filtering
+
+As a means to harden Cloud Hypervisor's security, the project leverages seccomp
+filtering.
+
+## What is seccomp filtering
+
+A seccomp filter is a way for a process to tell the kernel which system calls
+are authorized.
+In case this process calls into a prohibited system call, the kernel will kill
+the process right away.
+
+## How does it apply to Cloud Hypervisor
+
+Cloud Hypervisor is a multi threaded application. It spawns dedicated threads
+for virtual CPUs, virtio devices and HTTP server, along with the main thread
+representing the VMM.
+
+Each of these threads has a limited scope of what it is expected to perform,
+which is why different filters are applied to each of them.
+
+By default, Cloud Hypervisor enables seccomp filtering as the project believes
+that security should not be an option.
+
+For development and debugging purposes, one might want to disable this feature
+or log the faulty system call.
+
+### Disabling seccomp filters
+
+Append `--seccomp false` to Cloud Hypervisor's command line to prevent seccomp
+filtering from being applied.
+
+### Logging prohibited system calls
+
+In the context of debug, one alternative to disabling seccomp filtering is to
+log faulty system calls that would have caused the application to be killed by
+the kernel.
+
+Append `--seccomp log` to Cloud Hypervisor's command line to enable faulty
+system calls to be logged.
+
+The kernel running on the host machine must have the `audit` parameter enabled.
+If this is not the case, update kernel boot options by appending `audit=1`.
+
+Unauthorized system calls will be logged to the journal similarly to the
+following example
+
+```
+type=SECCOMP msg=audit(1423263412.694:7878): auid=1000 uid=1000 gid=1000 ses=3 subj=unconfined_u:unconfined_r:cloud_hypervisor:s0-s0:c0.c1023 pid=1193 comm="cloud-hypervisor" exe="/usr/bin/cloud-hypervisor" sig=0 arch=c000003e syscall=47 compat=0 ip=0x7f4f63982604 code=0x50000
+```
+
+Provided `ausyscall` has been installed on the host, the system call can be
+identified with
+
+```
+$ ausyscall 47
+recvmsg
+```
+
+### Further debug with `strace`
+
+One more way of debugging seccomp related issues is to use the `strace` tool as
+it will log every system call issued by the process. It is important to use
+`-f` option in order to trace each and every thread belonging to the process.
+
+```
+strace -f ./cloud-hypervisor ...
+```
--- a/docs/snapshot_restore.md
+++ b/docs/snapshot_restore.md
@@ -20,7 +20,7 @@ First thing, we must run a Cloud-Hypervisor VM:
    --api-socket /tmp/cloud-hypervisor.sock \
    --cpus boot=4 \
    --memory size=4G \
-    --kernel bzImage \
+    --kernel vmlinux \
    --cmdline "root=/dev/vda1 console=hvc0 rw" \
    --disk path=focal-server-cloudimg-amd64.raw
 ```
--- a/docs/vfio.md
+++ b/docs/vfio.md
@@ -54,6 +54,7 @@ $ lspci -n -s 01:00.0
 01:00.0 ff00: 10ec:525a (rev 01)

 $ echo 10ec 525a > /sys/bus/pci/drivers/vfio-pci/new_id
+$ echo 0000:01:00.0 > /sys/bus/pci/drivers/vfio-pci/bind
 ```

 Now the device is managed by the VFIO framework.
--- a/docs/virtiofs-root.md
+++ b/docs/virtiofs-root.md
@@ -0,0 +1,92 @@
+# HOWTO VirtioFS rootfs
+
+A quick guide for using virtiofs as a cloud-hypervisor guest's rootfs (i.e.
+with no root block device). This document is a quick getting started guide.
+There are many more steps to take to make this a production ready, secure
+setup.
+
+## Prerequisites
+
+1. virtiofsd from the qemu project
+   * We are using the Qemu version for now
+   * There is a Rust version being worked on that may be a better option in the future
+   * Part of the qemu-system-common package on Ubuntu
+   * Part of the qemu-common package on Fedora
+2. cloud-hypervisor - the newer the better, but I tested with 0.12
+3. a rootfs - This howto uses an alpine rootfs available here:
+   * https://dl-cdn.alpinelinux.org/alpine/v3.13/releases/x86_64/alpine-minirootfs-3.13.2-x86_64.tar.gz
+   * Others should work
+
+## To create the VM rootfs
+
+```bash
+mkdir rootfs/
+cd rootfs
+# this needs sudo to be able to set root permissions on fs components
+sudo tar -xf /path/to/alpine-minirootfs-3.13.1-x86_64.tar.gz
+# this will get created when the VM actually boots by the dhcp client
+# but we need it in the chroot to download packages
+sudo cp /etc/resolv.conf etc/
+# the alpine mini rootfs is meant for docker containers, we need a few extra
+# things for a working rootfs
+sudo chroot $PWD apk add openrc busybox-initscripts
+# we are using the paravirt console in cloud-hypervisor, so enable it in init
+# append it after the other console since it doesn't work just appending it
+sudo sed -i '/vt100/a \n# paravirt console\nhvc0::respawn:/sbin/getty -L hvc0 115200 vt100' etc/inittab
+# set no password for root user... you obviously don't want to do this for
+# any sort of production setup
+sudo sed -i 's/root:!::0:::::/root:::0:::::/' etc/shadow
+# set up init scripts
+for i in acpid crond
+    sudo ln -sf /etc/init.d/$i etc/runlevels/default/$i
+end
+for i in bootmisc hostname hwclock loadkmap modules networking swap sysctl syslog urandom
+    sudo ln -sf /etc/init.d/$i etc/runlevels/boot/$i
+end
+
+for i in killprocs mount-ro savecache
+    sudo ln -sf /etc/init.d/$i etc/runlevels/shutdown/$i
+end
+
+for i in devfs dmesg hwdrivers mdev
+    sudo ln -sf /etc/init.d/$i etc/runlevels/sysinit/$i
+end
+# setup network config
+echo 'auto lo
+iface lo inet loopback
+
+auto eth0
+iface eth0 inet dhcp
+' | sudo tee etc/network/interfaces
+
+```
+
+## To run the VM
+
+```bash
+# starting in the directory above rootfs
+sudo virtiofsd --socket-path=$PWD/virtiofs-rootfs.sock -o source=$PWD/rootfs -o cache=none &
+sudo cloud-hypervisor \
+    --cpus boot=1,max=1 \
+    --kernel vmlinux \
+    --fs tag=/dev/root,socket=$PWD/virtiofs-rootfs.sock \
+    --memory size=2G,shared=on \
+    --cmdline "console=hvc0 rootfstype=virtiofs root=/dev/root ro debug" \
+    --api-socket $PWD/ch.sock \
+    --rng \
+    --net ...
+```
+
+Note: an important part of the above is the `tag=/dev/root` and
+`root=/dev/root` parts. For whatever reason, it would only work with that as
+the tag.
+
+Note: another important bit is that the memory is shared. This is required for
+virtiofs
+
+## Message from the author
+
+If you find any issues or have suggestions, feel free to reach out to @iggy on
+the cloud-hypervisor slack. Also if this works for you, I'd like to know as
+well. It would also be nice to get steps for preparing other distribution root
+filesystems.
--- a/docs/windows.md
+++ b/docs/windows.md
@@ -184,6 +184,14 @@ Set-Service -Name sshd -StartupType ‘Automatic’

 This allows for SSH login from a remote machine, for example through the `administrator` user: `ssh administrator@192.168.249.2`. For a more detailed OpenSSH guide, please follow the MSDN article from the [links](#links) section.

+## Hotplug capability
+
+CPU hotplug is supported. The VM operating system needs to support hotplug and be appropriately licensed. SKU limitations like constraints on the number of cores are to be taken into consideration. Note, that Windows doesn't support CPU hot-remove. When `ch-remote` is invoked to reduce the number of CPUs, the result will be visible after the OS reboot within the same hypervisor instance.
+
+RAM hotplug is supported. Note, that while the `pnpmem.sys` driver in use supports RAM hot-remove, the RAM being unplugged has to be not in use and have no reserved pages. In most cases it means, hot-remove won't work. Same as with the CPU hot-remove, when `ch-remote` is invoked to reduce the RAM size, the result will be visible after the OS reboot.
+
+Network device hotplug and hot-remove are supported.
+
 ## Debugging

 The Windows guest debugging process relies heavily on QEMU and [socat](http://www.dest-unreach.org/socat/). The procedure requires two Windows VMs:
--- a/event_monitor/Cargo.toml
+++ b/event_monitor/Cargo.toml
@@ -0,0 +1,11 @@
+[package]
+name = "event_monitor"
+version = "0.1.0"
+authors = ["The Cloud Hypervisor Authors"]
+edition = "2018"
+
+[dependencies]
+libc = "0.2.94"
+serde = {version = ">=1.0.27", features = ["rc"] }
+serde_derive = ">=1.0.27"
+serde_json = ">=1.0.9"
--- a/event_monitor/src/lib.rs
+++ b/event_monitor/src/lib.rs
@@ -0,0 +1,77 @@
+// Copyright © 2021 Intel Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+#[macro_use]
+extern crate serde_derive;
+
+use std::borrow::Cow;
+use std::collections::HashMap;
+use std::fs::File;
+use std::os::unix::io::AsRawFd;
+use std::time::{Duration, Instant};
+
+static mut MONITOR: Option<(File, Instant)> = None;
+
+/// This function must only be called once from the main process before any threads
+/// are created to avoid race conditions
+pub fn set_monitor(file: File) -> Result<(), std::io::Error> {
+    assert!(unsafe { MONITOR.is_none() });
+    let fd = file.as_raw_fd();
+    let ret = unsafe {
+        let mut flags = libc::fcntl(fd, libc::F_GETFL);
+        flags |= libc::O_NONBLOCK;
+        libc::fcntl(fd, libc::F_SETFL, flags)
+    };
+    if ret < 0 {
+        return Err(std::io::Error::last_os_error());
+    }
+    unsafe {
+        MONITOR = Some((file, Instant::now()));
+    };
+    Ok(())
+}
+
+#[derive(Serialize)]
+struct Event<'a> {
+    timestamp: Duration,
+    source: &'a str,
+    event: &'a str,
+    properties: Option<&'a HashMap<Cow<'a, str>, Cow<'a, str>>>,
+}
+
+pub fn event_log(source: &str, event: &str, properties: Option<&HashMap<Cow<str>, Cow<str>>>) {
+    if let Some((file, start)) = unsafe { MONITOR.as_ref() } {
+        let e = Event {
+            timestamp: start.elapsed(),
+            source,
+            event,
+            properties,
+        };
+        serde_json::to_writer_pretty(file, &e).ok();
+    }
+}
+
+/*
+    Through the use of Cow<'a, str> it is possible to use String as well as
+    &str as the parameters:
+    e.g.
+    event!("cpu_manager", "create_vcpu", "id", cpu_id.to_string());
+*/
+#[macro_export]
+macro_rules! event {
+    ($source:expr, $event:expr) => {
+        $crate::event_log($source, $event, None)
+    };
+    ($source:expr, $event:expr, $($key:expr, $value:expr),*) => {
+        {
+            let mut properties = ::std::collections::HashMap::new();
+            $(
+                properties.insert($key.into(), $value.into());
+            )+
+            $crate::event_log($source, $event, Some(&properties))
+        }
+     };
+
+}
--- a/fuzz/Cargo.lock
+++ b/fuzz/Cargo.lock
@@ -0,0 +1,777 @@
+# This file is automatically @generated by Cargo.
+# It is not intended for manual editing.
+version = 3
+
+[[package]]
+name = "acpi_tables"
+version = "0.1.0"
+dependencies = [
+ "vm-memory",
+]
+
+[[package]]
+name = "ansi_term"
+version = "0.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b"
+dependencies = [
+ "winapi",
+]
+
+[[package]]
+name = "anyhow"
+version = "1.0.40"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "28b2cd92db5cbd74e8e5028f7e27dd7aa3090e89e4f2a197cc7c8dfb69c7063b"
+
+[[package]]
+name = "api_client"
+version = "0.1.0"
+
+[[package]]
+name = "arbitrary"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "698b65a961a9d730fb45b6b0327e20207810c9f61ee421b082b27ba003f49e2b"
+
+[[package]]
+name = "arc-swap"
+version = "1.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d4d7d63395147b81a9e570bcc6243aaf71c017bd666d4909cfef0085bdda8d73"
+
+[[package]]
+name = "arch"
+version = "0.1.0"
+dependencies = [
+ "acpi_tables",
+ "anyhow",
+ "arch_gen",
+ "byteorder",
+ "hypervisor",
+ "libc",
+ "linux-loader",
+ "log",
+ "serde",
+ "serde_derive",
+ "serde_json",
+ "thiserror",
+ "vm-memory",
+ "vm-migration",
+]
+
+[[package]]
+name = "arch_gen"
+version = "0.1.0"
+
+[[package]]
+name = "atty"
+version = "0.2.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8"
+dependencies = [
+ "hermit-abi",
+ "libc",
+ "winapi",
+]
+
+[[package]]
+name = "bitflags"
+version = "1.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693"
+
+[[package]]
+name = "block_util"
+version = "0.1.0"
+dependencies = [
+ "io-uring",
+ "libc",
+ "log",
+ "qcow",
+ "serde",
+ "serde_derive",
+ "serde_json",
+ "thiserror",
+ "virtio-bindings",
+ "vm-memory",
+ "vm-virtio",
+ "vmm-sys-util",
+]
+
+[[package]]
+name = "byteorder"
+version = "1.4.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610"
+
+[[package]]
+name = "cc"
+version = "1.0.67"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e3c69b077ad434294d3ce9f1f6143a2a4b89a8a2d54ef813d85003a4fd1137fd"
+
+[[package]]
+name = "cfg-if"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
+
+[[package]]
+name = "clap"
+version = "2.33.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "37e58ac78573c40708d45522f0d80fa2f01cc4f9b4e2bf749807255454312002"
+dependencies = [
+ "ansi_term",
+ "atty",
+ "bitflags",
+ "strsim",
+ "term_size",
+ "textwrap",
+ "unicode-width",
+ "vec_map",
+]
+
+[[package]]
+name = "cloud-hypervisor"
+version = "0.14.0"
+dependencies = [
+ "anyhow",
+ "api_client",
+ "clap",
+ "epoll",
+ "event_monitor",
+ "hypervisor",
+ "libc",
+ "log",
+ "option_parser",
+ "seccomp",
+ "serde_json",
+ "signal-hook",
+ "thiserror",
+ "vm-memory",
+ "vmm",
+ "vmm-sys-util",
+]
+
+[[package]]
+name = "cloud-hypervisor-fuzz"
+version = "0.0.0"
+dependencies = [
+ "block_util",
+ "cloud-hypervisor",
+ "libc",
+ "libfuzzer-sys",
+ "qcow",
+ "seccomp",
+ "virtio-devices",
+ "vm-memory",
+ "vm-virtio",
+ "vmm-sys-util",
+]
+
+[[package]]
+name = "devices"
+version = "0.1.0"
+dependencies = [
+ "acpi_tables",
+ "anyhow",
+ "bitflags",
+ "byteorder",
+ "epoll",
+ "libc",
+ "log",
+ "serde",
+ "serde_derive",
+ "serde_json",
+ "vm-device",
+ "vm-memory",
+ "vm-migration",
+ "vmm-sys-util",
+]
+
+[[package]]
+name = "epoll"
+version = "4.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "20df693c700404f7e19d4d6fae6b15215d2913c27955d2b9d6f2c0f537511cd0"
+dependencies = [
+ "bitflags",
+ "libc",
+]
+
+[[package]]
+name = "event_monitor"
+version = "0.1.0"
+dependencies = [
+ "libc",
+ "serde",
+ "serde_derive",
+ "serde_json",
+]
+
+[[package]]
+name = "hermit-abi"
+version = "0.1.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "322f4de77956e22ed0e5032c359a0f1273f1f7f0d79bfa3b8ffbc730d7fbcc5c"
+dependencies = [
+ "libc",
+]
+
+[[package]]
+name = "hypervisor"
+version = "0.1.0"
+dependencies = [
+ "anyhow",
+ "epoll",
+ "iced-x86",
+ "kvm-bindings",
+ "kvm-ioctls",
+ "libc",
+ "log",
+ "serde",
+ "serde_derive",
+ "serde_json",
+ "thiserror",
+ "vm-memory",
+ "vmm-sys-util",
+]
+
+[[package]]
+name = "iced-x86"
+version = "1.11.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "97b85a147f5dcb22ece887da5b99187954cc046939f22416e953dd7c336e85a1"
+dependencies = [
+ "lazy_static",
+ "static_assertions",
+]
+
+[[package]]
+name = "io-uring"
+version = "0.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fb82832e05cc4ca298f198a8db108837b4f7b7b1248e3cba8e48f151aece80cf"
+dependencies = [
+ "bitflags",
+ "libc",
+]
+
+[[package]]
+name = "itoa"
+version = "0.4.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dd25036021b0de88a0aff6b850051563c6516d0bf53f8638938edbb9de732736"
+
+[[package]]
+name = "kvm-bindings"
+version = "0.4.0"
+source = "git+https://github.com/cloud-hypervisor/kvm-bindings?branch=ch-v0.4.0#1a68725639283e622f4bb64584885b30bfe8be44"
+dependencies = [
+ "serde",
+ "serde_derive",
+ "vmm-sys-util",
+]
+
+[[package]]
+name = "kvm-ioctls"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "74058b16912c6723db02d4ca3ec919b73cd41512ccd3b6202cf91ae8d6c9dce5"
+dependencies = [
+ "kvm-bindings",
+ "libc",
+ "vmm-sys-util",
+]
+
+[[package]]
+name = "lazy_static"
+version = "1.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
+
+[[package]]
+name = "libc"
+version = "0.2.94"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "18794a8ad5b29321f790b55d93dfba91e125cb1a9edbd4f8e3150acc771c1a5e"
+
+[[package]]
+name = "libfuzzer-sys"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "86c975d637bc2a2f99440932b731491fc34c7f785d239e38af3addd3c2fd0e46"
+dependencies = [
+ "arbitrary",
+ "cc",
+]
+
+[[package]]
+name = "linux-loader"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c819cc8275b0f2c1ed9feec455ca288b45d82932384a6a5f7a86812ee3427459"
+dependencies = [
+ "vm-memory",
+]
+
+[[package]]
+name = "log"
+version = "0.4.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "51b9bbe6c47d51fc3e1a9b945965946b4c44142ab8792c50835a980d362c2710"
+dependencies = [
+ "cfg-if",
+]
+
+[[package]]
+name = "micro_http"
+version = "0.1.0"
+source = "git+https://github.com/firecracker-microvm/micro-http?branch=main#9b605a8b61df602cda62076d30d9f70307ea336f"
+dependencies = [
+ "libc",
+ "vmm-sys-util",
+]
+
+[[package]]
+name = "net_gen"
+version = "0.1.0"
+dependencies = [
+ "vmm-sys-util",
+]
+
+[[package]]
+name = "net_util"
+version = "0.1.0"
+dependencies = [
+ "epoll",
+ "libc",
+ "log",
+ "net_gen",
+ "rate_limiter",
+ "serde",
+ "virtio-bindings",
+ "vm-memory",
+ "vm-virtio",
+ "vmm-sys-util",
+]
+
+[[package]]
+name = "option_parser"
+version = "0.1.0"
+
+[[package]]
+name = "pci"
+version = "0.1.0"
+dependencies = [
+ "anyhow",
+ "byteorder",
+ "hypervisor",
+ "libc",
+ "log",
+ "serde",
+ "serde_derive",
+ "serde_json",
+ "vfio-bindings",
+ "vfio-ioctls",
+ "vm-allocator",
+ "vm-device",
+ "vm-memory",
+ "vm-migration",
+ "vmm-sys-util",
+]
+
+[[package]]
+name = "proc-macro2"
+version = "1.0.26"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a152013215dca273577e18d2bf00fa862b89b24169fb78c4c95aeb07992c9cec"
+dependencies = [
+ "unicode-xid",
+]
+
+[[package]]
+name = "qcow"
+version = "0.1.0"
+dependencies = [
+ "byteorder",
+ "libc",
+ "log",
+ "remain",
+ "vmm-sys-util",
+]
+
+[[package]]
+name = "quote"
+version = "1.0.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c3d0b9745dc2debf507c8422de05d7226cc1f0644216dfdfead988f9b1ab32a7"
+dependencies = [
+ "proc-macro2",
+]
+
+[[package]]
+name = "rate_limiter"
+version = "0.1.0"
+dependencies = [
+ "libc",
+ "log",
+ "vmm-sys-util",
+]
+
+[[package]]
+name = "remain"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "70ba1e78fa68412cb93ef642fd4d20b9a941be49ee9333875ebaf13112673ea7"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "ryu"
+version = "1.0.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e"
+
+[[package]]
+name = "seccomp"
+version = "0.1.0"
+source = "git+https://github.com/firecracker-microvm/firecracker?tag=v0.24.2#5ba819d7b7a684107f5434bcbd1617bc94565478"
+dependencies = [
+ "libc",
+]
+
+[[package]]
+name = "serde"
+version = "1.0.125"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "558dc50e1a5a5fa7112ca2ce4effcb321b0300c0d4ccf0776a9f60cd89031171"
+
+[[package]]
+name = "serde_derive"
+version = "1.0.125"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b093b7a2bb58203b5da3056c05b4ec1fed827dcfdb37347a8841695263b3d06d"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "serde_json"
+version = "1.0.64"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "799e97dc9fdae36a5c8b8f2cae9ce2ee9fdce2058c57a93e6099d919fd982f79"
+dependencies = [
+ "itoa",
+ "ryu",
+ "serde",
+]
+
+[[package]]
+name = "signal-hook"
+version = "0.3.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ef33d6d0cd06e0840fba9985aab098c147e67e05cee14d412d3345ed14ff30ac"
+dependencies = [
+ "libc",
+ "signal-hook-registry",
+]
+
+[[package]]
+name = "signal-hook-registry"
+version = "1.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "16f1d0fef1604ba8f7a073c7e701f213e056707210e9020af4528e0101ce11a6"
+dependencies = [
+ "libc",
+]
+
+[[package]]
+name = "static_assertions"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f"
+
+[[package]]
+name = "strsim"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a"
+
+[[package]]
+name = "syn"
+version = "1.0.71"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ad184cc9470f9117b2ac6817bfe297307418819ba40552f9b3846f05c33d5373"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "unicode-xid",
+]
+
+[[package]]
+name = "term_size"
+version = "0.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1e4129646ca0ed8f45d09b929036bafad5377103edd06e50bf574b353d2b08d9"
+dependencies = [
+ "libc",
+ "winapi",
+]
+
+[[package]]
+name = "textwrap"
+version = "0.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060"
+dependencies = [
+ "term_size",
+ "unicode-width",
+]
+
+[[package]]
+name = "thiserror"
+version = "1.0.24"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e0f4a65597094d4483ddaed134f409b2cb7c1beccf25201a9f73c719254fa98e"
+dependencies = [
+ "thiserror-impl",
+]
+
+[[package]]
+name = "thiserror-impl"
+version = "1.0.24"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7765189610d8241a44529806d6fd1f2e0a08734313a35d5b3a556f92b381f3c0"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "unicode-width"
+version = "0.1.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9337591893a19b88d8d87f2cec1e73fad5cdfd10e5a6f349f498ad6ea2ffb1e3"
+
+[[package]]
+name = "unicode-xid"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f7fe0bb3479651439c9112f72b6c505038574c9fbb575ed1bf3b797fa39dd564"
+
+[[package]]
+name = "vec_map"
+version = "0.8.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191"
+
+[[package]]
+name = "vfio-bindings"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4a21f546f2bda37f5a8cfb138c87f95b8e34d2d78d6a7a92ba3785f4e08604a7"
+dependencies = [
+ "vmm-sys-util",
+]
+
+[[package]]
+name = "vfio-ioctls"
+version = "0.1.0"
+source = "git+https://github.com/rust-vmm/vfio-ioctls?branch=master#a87b13bdec026e8144b91f30f35451a966d8c1ca"
+dependencies = [
+ "byteorder",
+ "kvm-bindings",
+ "kvm-ioctls",
+ "log",
+ "vfio-bindings",
+ "vm-memory",
+ "vmm-sys-util",
+]
+
+[[package]]
+name = "vhost"
+version = "0.1.0"
+source = "git+https://github.com/rust-vmm/vhost?branch=master#ee3e8722706c984b3dfe12d3a130e92101b78e8f"
+dependencies = [
+ "bitflags",
+ "libc",
+ "vmm-sys-util",
+]
+
+[[package]]
+name = "virtio-bindings"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3ff512178285488516ed85f15b5d0113a7cdb89e9e8a760b269ae4f02b84bd6b"
+
+[[package]]
+name = "virtio-devices"
+version = "0.1.0"
+dependencies = [
+ "anyhow",
+ "arc-swap",
+ "block_util",
+ "byteorder",
+ "epoll",
+ "event_monitor",
+ "io-uring",
+ "libc",
+ "log",
+ "net_gen",
+ "net_util",
+ "pci",
+ "rate_limiter",
+ "seccomp",
+ "serde",
+ "serde_derive",
+ "serde_json",
+ "vhost",
+ "virtio-bindings",
+ "vm-allocator",
+ "vm-device",
+ "vm-memory",
+ "vm-migration",
+ "vm-virtio",
+ "vmm-sys-util",
+]
+
+[[package]]
+name = "vm-allocator"
+version = "0.1.0"
+dependencies = [
+ "arch",
+ "libc",
+ "vm-memory",
+]
+
+[[package]]
+name = "vm-device"
+version = "0.1.0"
+dependencies = [
+ "anyhow",
+ "serde",
+ "serde_derive",
+ "serde_json",
+ "thiserror",
+ "vfio-ioctls",
+ "vm-memory",
+ "vmm-sys-util",
+]
+
+[[package]]
+name = "vm-memory"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "625f401b1b8b3ac3d43f53903cd138cfe840bd985f8581e553027b31d2bb8ae8"
+dependencies = [
+ "arc-swap",
+ "libc",
+ "winapi",
+]
+
+[[package]]
+name = "vm-migration"
+version = "0.1.0"
+dependencies = [
+ "anyhow",
+ "serde",
+ "serde_derive",
+ "serde_json",
+ "thiserror",
+ "vm-memory",
+]
+
+[[package]]
+name = "vm-virtio"
+version = "0.1.0"
+dependencies = [
+ "log",
+ "serde",
+ "serde_derive",
+ "serde_json",
+ "virtio-bindings",
+ "vm-memory",
+]
+
+[[package]]
+name = "vmm"
+version = "0.1.0"
+dependencies = [
+ "acpi_tables",
+ "anyhow",
+ "arc-swap",
+ "arch",
+ "bitflags",
+ "block_util",
+ "clap",
+ "devices",
+ "epoll",
+ "event_monitor",
+ "hypervisor",
+ "lazy_static",
+ "libc",
+ "linux-loader",
+ "log",
+ "micro_http",
+ "net_util",
+ "option_parser",
+ "pci",
+ "qcow",
+ "seccomp",
+ "serde",
+ "serde_derive",
+ "serde_json",
+ "signal-hook",
+ "thiserror",
+ "vfio-ioctls",
+ "virtio-devices",
+ "vm-allocator",
+ "vm-device",
+ "vm-memory",
+ "vm-migration",
+ "vm-virtio",
+ "vmm-sys-util",
+]
+
+[[package]]
+name = "vmm-sys-util"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "01cf11afbc4ebc0d5c7a7748a77d19e2042677fc15faa2f4ccccb27c18a60605"
+dependencies = [
+ "bitflags",
+ "libc",
+ "serde",
+ "serde_derive",
+]
+
+[[package]]
+name = "winapi"
+version = "0.3.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419"
+dependencies = [
+ "winapi-i686-pc-windows-gnu",
+ "winapi-x86_64-pc-windows-gnu",
+]
+
+[[package]]
+name = "winapi-i686-pc-windows-gnu"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
+
+[[package]]
+name = "winapi-x86_64-pc-windows-gnu"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
--- a/fuzz/Cargo.toml
+++ b/fuzz/Cargo.toml
@@ -10,15 +10,18 @@ cargo-fuzz = true

 [dependencies]
 block_util = { path = "../block_util" }
-libc = "0.2.72"
-libfuzzer-sys = "0.3"
+libc = "0.2.94"
+libfuzzer-sys = "0.4"
 qcow = { path = "../qcow" }
-seccomp = { git = "https://github.com/firecracker-microvm/firecracker", tag = "v0.22.0" }
+seccomp = { git = "https://github.com/firecracker-microvm/firecracker", tag = "v0.24.2" }
 virtio-devices = { path = "../virtio-devices" }
-vmm-sys-util = ">=0.3.1"
+vmm-sys-util = "0.8.0"
 vm-virtio = { path = "../vm-virtio" }
 vm-memory = "0.5.0"

+[patch.crates-io]
+kvm-bindings = { git = "https://github.com/cloud-hypervisor/kvm-bindings", branch = "ch-v0.4.0", features = ["with-serde", "fam-wrappers"] }
+
 [dependencies.cloud-hypervisor]
 path = ".."

--- a/fuzz/fuzz_targets/block.rs
+++ b/fuzz/fuzz_targets/block.rs
@@ -95,6 +95,7 @@ fuzz_target!(|bytes| {
        2,
        256,
        SeccompAction::Allow,
+        None,
    )
    .unwrap();

--- a/hypervisor/Cargo.toml
+++ b/hypervisor/Cargo.toml
@@ -6,21 +6,20 @@ edition = "2018"
 license = "Apache-2.0 OR BSD-3-Clause"

 [features]
-kvm = []
-mshv = []
+kvm = ["kvm-ioctls", "kvm-bindings"]
+mshv = ["mshv-ioctls", "mshv-bindings"]
+tdx = []

 [dependencies]
 anyhow = "1.0"
 epoll = ">=4.0.1"
 thiserror = "1.0"
-libc = "0.2.86"
+libc = "0.2.94"
 log = "0.4.14"
-kvm-ioctls = { git = "https://github.com/cloud-hypervisor/kvm-ioctls", branch = "ch" }
-kvm-bindings = { git = "https://github.com/cloud-hypervisor/kvm-bindings", branch = "ch", features = ["with-serde", "fam-wrappers"] }
-
-mshv-bindings = {git = "https://github.com/cloud-hypervisor/mshv", branch = "master", features = ["with-serde", "fam-wrappers"] }
-mshv-ioctls = { git = "https://github.com/cloud-hypervisor/mshv", branch = "master" }
-
+kvm-ioctls = {  version = "0.8.0", optional = true }
+kvm-bindings = { git = "https://github.com/cloud-hypervisor/kvm-bindings", branch = "ch-v0.4.0", features = ["with-serde", "fam-wrappers"], optional  = true }
+mshv-bindings = {git = "https://github.com/cloud-hypervisor/mshv", branch = "master", features = ["with-serde", "fam-wrappers"], optional  = true }
+mshv-ioctls = { git = "https://github.com/cloud-hypervisor/mshv", branch = "master", optional  = true}
 serde = {version = ">=1.0.27", features = ["rc"] }
 serde_derive = ">=1.0.27"
 serde_json = ">=1.0.9"
@@ -28,7 +27,7 @@ vm-memory = { version = "0.5.0", features = ["backend-mmap", "backend-atomic"] }
 vmm-sys-util = { version = ">=0.5.0", features = ["with-serde"] }

 [target.'cfg(target_arch = "x86_64")'.dependencies.iced-x86]
-version = "1.10"
+version = "1.11"
 default-features = false
 features = ["std", "decoder", "op_code_info", "instr_info", "fast_fmt"]

--- a/hypervisor/src/arch/emulator/mod.rs
+++ b/hypervisor/src/arch/emulator/mod.rs
@@ -47,8 +47,8 @@ pub enum PlatformError {
    #[error("Set CPU state failure: {0}")]
    SetCpuStateFailure(#[source] anyhow::Error),

-    #[error("Unmapped virtual address: {0}")]
-    UnmappedGVA(#[source] anyhow::Error),
+    #[error("Translate virtual address: {0}")]
+    TranslateVirtualAddress(#[source] anyhow::Error),

    #[error("Unsupported CPU Mode: {0}")]
    UnsupportedCpuMode(#[source] anyhow::Error),
--- a/hypervisor/src/arch/x86/emulator/instructions/cmp.rs
+++ b/hypervisor/src/arch/x86/emulator/instructions/cmp.rs
@@ -4,7 +4,7 @@
 // SPDX-License-Identifier: Apache-2.0
 //

-#![allow(non_camel_case_types)]
+#![allow(non_camel_case_types, clippy::upper_case_acronyms)]

 //
 // CMP-Compare Two Operands
@@ -229,7 +229,7 @@ mod tests {
        let ip: u64 = 0x1000;
        let cpu_id = 0;
        let insn = [0x38, 0xc4]; // cmp ah,al
-        let mut vmm = MockVMM::new(ip, vec![(Register::RAX, rax)], None);
+        let mut vmm = MockVmm::new(ip, vec![(Register::RAX, rax)], None);
        assert!(vmm.emulate_first_insn(cpu_id, &insn).is_ok());

        let rflags: u64 = vmm.cpu_state(cpu_id).unwrap().flags() & FLAGS_MASK;
@@ -243,7 +243,7 @@ mod tests {
        let ip: u64 = 0x1000;
        let cpu_id = 0;
        let insn = [0x83, 0xf8, 0x64]; // cmp eax,100
-        let mut vmm = MockVMM::new(ip, vec![(Register::RAX, rax)], None);
+        let mut vmm = MockVmm::new(ip, vec![(Register::RAX, rax)], None);
        assert!(vmm.emulate_first_insn(cpu_id, &insn).is_ok());

        let rflags: u64 = vmm.cpu_state(cpu_id).unwrap().flags() & FLAGS_MASK;
@@ -257,7 +257,7 @@ mod tests {
        let ip: u64 = 0x1000;
        let cpu_id = 0;
        let insn = [0x83, 0xf8, 0xff]; // cmp eax,-1
-        let mut vmm = MockVMM::new(ip, vec![(Register::RAX, rax)], None);
+        let mut vmm = MockVmm::new(ip, vec![(Register::RAX, rax)], None);
        assert!(vmm.emulate_first_insn(cpu_id, &insn).is_ok());

        let rflags: u64 = vmm.cpu_state(cpu_id).unwrap().flags() & FLAGS_MASK;
@@ -272,7 +272,7 @@ mod tests {
        let ip: u64 = 0x1000;
        let cpu_id = 0;
        let insn = [0x48, 0x39, 0xd8, 0x00, 0xc3]; // cmp rax,rbx + two bytes garbage
-        let mut vmm = MockVMM::new(ip, vec![(Register::RAX, rax), (Register::RBX, rbx)], None);
+        let mut vmm = MockVmm::new(ip, vec![(Register::RAX, rax), (Register::RBX, rbx)], None);
        assert!(vmm.emulate_first_insn(cpu_id, &insn).is_ok());

        let rflags: u64 = vmm.cpu_state(cpu_id).unwrap().flags() & FLAGS_MASK;
@@ -295,7 +295,7 @@ mod tests {
            let rax = d.0;
            let rbx = d.1;
            let insn = [0x48, 0x39, 0xd8]; // cmp rax,rbx
-            let mut vmm = MockVMM::new(
+            let mut vmm = MockVmm::new(
                0x1000,
                vec![(Register::RAX, rax), (Register::RBX, rbx)],
                None,
@@ -323,7 +323,7 @@ mod tests {
            let rax = d.0;
            let rbx = d.1;
            let insn = [0x39, 0xd8]; // cmp eax,ebx
-            let mut vmm = MockVMM::new(
+            let mut vmm = MockVmm::new(
                0x1000,
                vec![(Register::RAX, rax), (Register::RBX, rbx)],
                None,
--- a/hypervisor/src/arch/x86/emulator/instructions/mod.rs
+++ b/hypervisor/src/arch/x86/emulator/instructions/mod.rs
@@ -13,6 +13,7 @@ use iced_x86::*;

 pub mod cmp;
 pub mod mov;
+pub mod movs;

 fn get_op<T: CpuStateManager>(
    insn: &Instruction,
@@ -130,7 +131,7 @@ fn memory_operand_address<T: CpuStateManager>(
        address += index;
    }

-    address += insn.memory_displacement() as u64;
+    address += insn.memory_displacement64();

    // Translate to a linear address.
    state.linearize(insn.memory_segment(), address, write)
--- a/hypervisor/src/arch/x86/emulator/instructions/mov.rs
+++ b/hypervisor/src/arch/x86/emulator/instructions/mov.rs
@@ -71,7 +71,7 @@ macro_rules! mov_rm_imm {
 }

 macro_rules! movzx {
-    ($src_op_size:ty, $dest_op_size:ty) => {
+    ($dest_op_size:ty, $src_op_size:ty) => {
        fn emulate(
            &self,
            insn: &Instruction,
@@ -254,7 +254,7 @@ mod tests {
        let ip: u64 = 0x1000;
        let cpu_id = 0;
        let insn = [0x48, 0x89, 0xd8];
-        let mut vmm = MockVMM::new(ip, vec![(Register::RBX, rbx)], None);
+        let mut vmm = MockVmm::new(ip, vec![(Register::RBX, rbx)], None);
        assert!(vmm.emulate_first_insn(cpu_id, &insn).is_ok());

        let rax: u64 = vmm
@@ -272,7 +272,7 @@ mod tests {
        let ip: u64 = 0x1000;
        let cpu_id = 0;
        let insn = [0x48, 0xb8, 0x44, 0x33, 0x22, 0x11, 0x44, 0x33, 0x22, 0x11];
-        let mut vmm = MockVMM::new(ip, vec![], None);
+        let mut vmm = MockVmm::new(ip, vec![], None);
        assert!(vmm.emulate_first_insn(cpu_id, &insn).is_ok());

        let rax: u64 = vmm
@@ -292,7 +292,7 @@ mod tests {
        let cpu_id = 0;
        let memory: [u8; 8] = target_rax.to_le_bytes();
        let insn = [0x48, 0x8b, 0x04, 0x00];
-        let mut vmm = MockVMM::new(ip, vec![(Register::RAX, rax)], Some((rax + rax, &memory)));
+        let mut vmm = MockVmm::new(ip, vec![(Register::RAX, rax)], Some((rax + rax, &memory)));
        assert!(vmm.emulate_first_insn(cpu_id, &insn).is_ok());

        rax = vmm
@@ -310,7 +310,7 @@ mod tests {
        let ip: u64 = 0x1000;
        let cpu_id = 0;
        let insn = [0xb0, 0x11];
-        let mut vmm = MockVMM::new(ip, vec![], None);
+        let mut vmm = MockVmm::new(ip, vec![], None);
        assert!(vmm.emulate_first_insn(cpu_id, &insn).is_ok());

        let al = vmm
@@ -328,7 +328,7 @@ mod tests {
        let ip: u64 = 0x1000;
        let cpu_id = 0;
        let insn = [0xb8, 0x11, 0x00, 0x00, 0x00];
-        let mut vmm = MockVMM::new(ip, vec![], None);
+        let mut vmm = MockVmm::new(ip, vec![], None);
        assert!(vmm.emulate_first_insn(cpu_id, &insn).is_ok());

        let eax = vmm
@@ -346,7 +346,7 @@ mod tests {
        let ip: u64 = 0x1000;
        let cpu_id = 0;
        let insn = [0x48, 0xc7, 0xc0, 0x44, 0x33, 0x22, 0x11];
-        let mut vmm = MockVMM::new(ip, vec![], None);
+        let mut vmm = MockVmm::new(ip, vec![], None);
        assert!(vmm.emulate_first_insn(cpu_id, &insn).is_ok());

        let rax: u64 = vmm
@@ -365,7 +365,7 @@ mod tests {
        let ip: u64 = 0x1000;
        let cpu_id = 0;
        let insn = [0x88, 0x30];
-        let mut vmm = MockVMM::new(
+        let mut vmm = MockVmm::new(
            ip,
            vec![(Register::RAX, rax), (Register::DH, dh.into())],
            None,
@@ -386,7 +386,7 @@ mod tests {
        let ip: u64 = 0x1000;
        let cpu_id = 0;
        let insn = [0x89, 0x30];
-        let mut vmm = MockVMM::new(
+        let mut vmm = MockVmm::new(
            ip,
            vec![(Register::RAX, rax), (Register::ESI, esi.into())],
            None,
@@ -408,7 +408,7 @@ mod tests {
        let ip: u64 = 0x1000;
        let cpu_id = 0;
        let insn = [0x89, 0x3c, 0x05, 0x01, 0x00, 0x00, 0x00];
-        let mut vmm = MockVMM::new(
+        let mut vmm = MockVmm::new(
            ip,
            vec![(Register::RAX, rax), (Register::EDI, edi.into())],
            None,
@@ -431,7 +431,7 @@ mod tests {
        let ip: u64 = 0x1000;
        let cpu_id = 0;
        let insn = [0x8b, 0x40, 0x10];
-        let mut vmm = MockVMM::new(
+        let mut vmm = MockVmm::new(
            ip,
            vec![(Register::RAX, rax)],
            Some((rax + displacement, &memory)),
@@ -456,7 +456,7 @@ mod tests {
        let cpu_id = 0;
        let insn = [0x8a, 0x40, 0x10];
        let memory: [u8; 1] = al.to_le_bytes();
-        let mut vmm = MockVMM::new(
+        let mut vmm = MockVmm::new(
            ip,
            vec![(Register::RAX, rax)],
            Some((rax + displacement, &memory)),
@@ -485,7 +485,7 @@ mod tests {
            0x48, 0xc7, 0xc0, 0x00, 0x01, 0x00, 0x00, // mov rax, 0x100
            0x48, 0x8b, 0x58, 0x10, // mov rbx, qword ptr [rax+10h]
        ];
-        let mut vmm = MockVMM::new(ip, vec![], Some((rax + displacement, &memory)));
+        let mut vmm = MockVmm::new(ip, vec![], Some((rax + displacement, &memory)));
        assert!(vmm.emulate_insn(cpu_id, &insn, Some(2)).is_ok());

        let rbx: u64 = vmm
@@ -511,7 +511,7 @@ mod tests {
            0x48, 0x8b, 0x58, 0x10, // mov rbx, qword ptr [rax+10h]
        ];

-        let mut vmm = MockVMM::new(ip, vec![], Some((rax + displacement, &memory)));
+        let mut vmm = MockVmm::new(ip, vec![], Some((rax + displacement, &memory)));
        // Only run the first instruction.
        assert!(vmm.emulate_first_insn(cpu_id, &insn).is_ok());

@@ -542,7 +542,7 @@ mod tests {
            0x48, 0xc7, 0xc0, 0x00, 0x02, 0x00, 0x00, // mov rax, 0x200
        ];

-        let mut vmm = MockVMM::new(ip, vec![], Some((rax + displacement, &memory)));
+        let mut vmm = MockVmm::new(ip, vec![], Some((rax + displacement, &memory)));
        // Run the 2 first instructions.
        assert!(vmm.emulate_insn(cpu_id, &insn, Some(2)).is_ok());

@@ -571,7 +571,7 @@ mod tests {
        let ip: u64 = 0x1000;
        let cpu_id = 0;
        let insn = [0x0f, 0xb6, 0xc3];
-        let mut vmm = MockVMM::new(ip, vec![(Register::BX, bx as u64)], None);
+        let mut vmm = MockVmm::new(ip, vec![(Register::BX, bx as u64)], None);
        assert!(vmm.emulate_first_insn(cpu_id, &insn).is_ok());

        let eax: u64 = vmm
@@ -589,7 +589,7 @@ mod tests {
        let ip: u64 = 0x1000;
        let cpu_id = 0;
        let insn = [0x0f, 0xb6, 0xc7];
-        let mut vmm = MockVMM::new(ip, vec![(Register::BX, bx as u64)], None);
+        let mut vmm = MockVmm::new(ip, vec![(Register::BX, bx as u64)], None);
        assert!(vmm.emulate_first_insn(cpu_id, &insn).is_ok());

        let eax: u64 = vmm
@@ -609,7 +609,7 @@ mod tests {
        let cpu_id = 0;
        let insn = [0x0f, 0xb7, 0x03];
        let memory: [u8; 1] = value.to_le_bytes();
-        let mut vmm = MockVMM::new(ip, vec![(Register::RBX, rbx)], Some((rbx, &memory)));
+        let mut vmm = MockVmm::new(ip, vec![(Register::RBX, rbx)], Some((rbx, &memory)));
        assert!(vmm.emulate_first_insn(cpu_id, &insn).is_ok());

        let eax: u64 = vmm
--- a/hypervisor/src/arch/x86/emulator/instructions/movs.rs
+++ b/hypervisor/src/arch/x86/emulator/instructions/movs.rs
@@ -0,0 +1,147 @@
+//
+// Copyright © 2021 Microsoft
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+#![allow(non_camel_case_types)]
+
+//
+// MOVS - Move Data from String to String
+//
+
+extern crate iced_x86;
+
+use crate::arch::emulator::{EmulationError, PlatformEmulator};
+use crate::arch::x86::emulator::instructions::*;
+use crate::arch::x86::regs::DF;
+use crate::arch::x86::Exception;
+
+pub struct Movsd_m32_m32;
+impl<T: CpuStateManager> InstructionHandler<T> for Movsd_m32_m32 {
+    fn emulate(
+        &self,
+        insn: &Instruction,
+        state: &mut T,
+        platform: &mut dyn PlatformEmulator<CpuState = T>,
+    ) -> Result<(), EmulationError<Exception>> {
+        let mut count: u64 = if insn.has_rep_prefix() {
+            state
+                .read_reg(Register::ECX)
+                .map_err(|e| EmulationError::InvalidOperand(anyhow!(e)))?
+        } else {
+            1
+        };
+
+        let mut rsi = state
+            .read_reg(Register::RSI)
+            .map_err(|e| EmulationError::InvalidOperand(anyhow!(e)))?;
+        let mut rdi = state
+            .read_reg(Register::RDI)
+            .map_err(|e| EmulationError::InvalidOperand(anyhow!(e)))?;
+
+        let df = (state.flags() & DF) != 0;
+        let len = std::mem::size_of::<u32>();
+
+        while count > 0 {
+            let mut memory: [u8; 4] = [0; 4];
+
+            let src = state
+                .linearize(Register::DS, rsi, false)
+                .map_err(|e| EmulationError::InvalidOperand(anyhow!(e)))?;
+            let dst = state
+                .linearize(Register::ES, rdi, true)
+                .map_err(|e| EmulationError::InvalidOperand(anyhow!(e)))?;
+
+            platform
+                .read_memory(src, &mut memory[0..len])
+                .map_err(EmulationError::PlatformEmulationError)?;
+            platform
+                .write_memory(dst, &memory[0..len])
+                .map_err(EmulationError::PlatformEmulationError)?;
+
+            if df {
+                rsi = rsi.wrapping_sub(len as u64);
+                rdi = rdi.wrapping_sub(len as u64);
+            } else {
+                rsi = rsi.wrapping_add(len as u64);
+                rdi = rdi.wrapping_add(len as u64);
+            }
+            count -= 1;
+        }
+
+        state
+            .write_reg(Register::RSI, rsi)
+            .map_err(|e| EmulationError::InvalidOperand(anyhow!(e)))?;
+        state
+            .write_reg(Register::RDI, rdi)
+            .map_err(|e| EmulationError::InvalidOperand(anyhow!(e)))?;
+        if insn.has_rep_prefix() {
+            state
+                .write_reg(Register::ECX, 0)
+                .map_err(|e| EmulationError::InvalidOperand(anyhow!(e)))?;
+        }
+
+        Ok(())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    #![allow(unused_mut)]
+    use super::*;
+    use crate::arch::x86::emulator::mock_vmm::*;
+
+    #[test]
+    fn test_rep_movsd_m32_m32() {
+        let ip: u64 = 0x1000;
+        let memory: [u8; 24] = [
+            0x78, 0x56, 0x34, 0x12, // 0x12345678
+            0xdd, 0xcc, 0xbb, 0xaa, // 0xaabbccdd
+            0xa5, 0x5a, 0xa5, 0x5a, // 0x5aa55aa5
+            0x00, 0x00, 0x00, 0x00, // 0x00000000
+            0x00, 0x00, 0x00, 0x00, // 0x00000000
+            0x00, 0x00, 0x00, 0x00, // 0x00000000
+        ];
+        let insn = [0xf3, 0xa5]; // rep movsd
+        let regs = vec![(Register::ECX, 3), (Register::ESI, 0), (Register::EDI, 0xc)];
+        let mut data = [0u8; 4];
+
+        let mut vmm = MockVmm::new(ip, regs, Some((0, &memory)));
+
+        assert!(vmm.emulate_first_insn(0, &insn).is_ok());
+
+        vmm.read_memory(0xc, &mut data).unwrap();
+        assert_eq!(0x12345678, <u32>::from_le_bytes(data));
+        vmm.read_memory(0xc + 4, &mut data).unwrap();
+        assert_eq!(0xaabbccdd, <u32>::from_le_bytes(data));
+        vmm.read_memory(0xc + 8, &mut data).unwrap();
+        assert_eq!(0x5aa55aa5, <u32>::from_le_bytes(data));
+        // The rest should be default value 0 from MockVmm
+        vmm.read_memory(0xc + 12, &mut data).unwrap();
+        assert_eq!(0x0, <u32>::from_le_bytes(data));
+    }
+
+    #[test]
+    fn test_movsd_m32_m32() {
+        let ip: u64 = 0x1000;
+        let memory: [u8; 4] = [
+            0x78, 0x56, 0x34, 0x12, // 0x12345678
+        ];
+        let insn = [0xa5]; // movsd
+        let regs = vec![(Register::ESI, 0), (Register::EDI, 0x8)];
+        let mut data = [0u8; 4];
+
+        let mut vmm = MockVmm::new(ip, regs, Some((0, &memory)));
+
+        assert!(vmm.emulate_first_insn(0, &insn).is_ok());
+
+        vmm.read_memory(0x8, &mut data).unwrap();
+        assert_eq!(0x12345678, <u32>::from_le_bytes(data));
+        // The rest should be default value 0 from MockVmm
+        vmm.read_memory(0x4, &mut data).unwrap();
+        assert_eq!(0x0, <u32>::from_le_bytes(data));
+        vmm.read_memory(0x8 + 8, &mut data).unwrap();
+        assert_eq!(0x0, <u32>::from_le_bytes(data));
+    }
+}
--- a/hypervisor/src/arch/x86/emulator/mod.rs
+++ b/hypervisor/src/arch/x86/emulator/mod.rs
@@ -524,7 +524,9 @@ impl<'a, T: CpuStateManager> Emulator<'a, T> {
            (mov, Movzx_r32_rm8),
            (mov, Movzx_r64_rm8),
            (mov, Movzx_r32_rm16),
-            (mov, Movzx_r64_rm16)
+            (mov, Movzx_r64_rm16),
+            // MOVS
+            (movs, Movsd_m32_m32)
        );

        handler
@@ -648,17 +650,17 @@ mod mock_vmm {
    use std::sync::{Arc, Mutex};

    #[derive(Debug, Clone)]
-    pub struct MockVMM {
+    pub struct MockVmm {
        memory: Vec<u8>,
        state: Arc<Mutex<CpuState>>,
    }

-    unsafe impl Sync for MockVMM {}
+    unsafe impl Sync for MockVmm {}

    pub type MockResult = Result<(), EmulationError<Exception>>;

-    impl MockVMM {
-        pub fn new(ip: u64, regs: Vec<(Register, u64)>, memory: Option<(u64, &[u8])>) -> MockVMM {
+    impl MockVmm {
+        pub fn new(ip: u64, regs: Vec<(Register, u64)>, memory: Option<(u64, &[u8])>) -> MockVmm {
            let _ = env_logger::try_init();
            let cs_reg = segment_from_gdt(gdt_entry(0xc09b, 0, 0xffffffff), 1);
            let ds_reg = segment_from_gdt(gdt_entry(0xc093, 0, 0xffffffff), 2);
@@ -672,7 +674,7 @@ mod mock_vmm {
                initial_state.write_reg(reg, value).unwrap();
            }

-            let mut vmm = MockVMM {
+            let mut vmm = MockVmm {
                memory: vec![0; 8192],
                state: Arc::new(Mutex::new(initial_state)),
            };
@@ -708,7 +710,7 @@ mod mock_vmm {
        }
    }

-    impl PlatformEmulator for MockVMM {
+    impl PlatformEmulator for MockVmm {
        type CpuState = CpuState;

        fn read_memory(&self, gva: u64, data: &mut [u8]) -> Result<(), PlatformError> {
@@ -790,7 +792,7 @@ mod tests {
            0x48, 0xc7, 0xc0, 0x00, // mov rax, 0x1000 -- Missing bytes: 0x00, 0x10, 0x00, 0x00,
        ];

-        let mut vmm = MockVMM::new(ip, vec![], Some((ip, &memory)));
+        let mut vmm = MockVmm::new(ip, vec![], Some((ip, &memory)));
        assert!(vmm.emulate_insn(cpu_id, &insn, Some(2)).is_ok());

        let rax: u64 = vmm
@@ -825,7 +827,7 @@ mod tests {
            0x48, 0x8b, // Truncated mov rbx, qword ptr [rax+10h] -- missing [0x58, 0x10]
        ];

-        let mut vmm = MockVMM::new(ip, vec![], Some((ip, &memory)));
+        let mut vmm = MockVmm::new(ip, vec![], Some((ip, &memory)));
        assert!(vmm.emulate_insn(cpu_id, &insn, Some(2)).is_ok());

        let rbx: u64 = vmm
@@ -855,7 +857,7 @@ mod tests {
            0x48, 0xc7, 0xc0, 0x00, // mov rax, 0x1000 -- Missing bytes: 0x00, 0x10, 0x00, 0x00,
        ];

-        let mut vmm = MockVMM::new(ip, vec![], Some((ip, &memory)));
+        let mut vmm = MockVmm::new(ip, vec![], Some((ip, &memory)));
        assert!(vmm.emulate_first_insn(cpu_id, &insn).is_err());
    }
 }
--- a/hypervisor/src/arch/x86/mod.rs
+++ b/hypervisor/src/arch/x86/mod.rs
@@ -11,27 +11,14 @@
 // Copyright © 2020, Microsoft Corporation
 //

+pub mod emulator;
 pub mod gdt;
-
 #[allow(non_upper_case_globals)]
 #[allow(non_camel_case_types)]
 #[allow(non_snake_case)]
 #[allow(non_upper_case_globals)]
-#[allow(unused)]
-#[allow(
-    clippy::unreadable_literal,
-    clippy::redundant_static_lifetimes,
-    clippy::trivially_copy_pass_by_ref,
-    clippy::useless_transmute,
-    clippy::should_implement_trait,
-    clippy::transmute_ptr_to_ptr,
-    clippy::unreadable_literal,
-    clippy::redundant_static_lifetimes
-)]
 pub mod msr_index;

-pub mod emulator;
-
 // MTRR constants
 pub const MTRR_ENABLE: u64 = 0x800; // IA32_MTRR_DEF_TYPE MSR: E (MTRRs enabled) flag, bit 11
 pub const MTRR_MEM_TYPE_WB: u64 = 0x6;
@@ -40,7 +27,7 @@ pub const MTRR_MEM_TYPE_WB: u64 = 0x6;
 pub const NUM_IOAPIC_PINS: usize = 24;

 // X86 Exceptions
-#[allow(dead_code)]
+#[allow(dead_code, clippy::upper_case_acronyms)]
 #[derive(Clone, Debug)]
 pub enum Exception {
    DE = 0,  // Divide Error
--- a/hypervisor/src/arch/x86/regs.rs
+++ b/hypervisor/src/arch/x86/regs.rs
@@ -22,6 +22,7 @@ pub const PF_SHIFT: usize = 2;
 pub const AF_SHIFT: usize = 4;
 pub const ZF_SHIFT: usize = 6;
 pub const SF_SHIFT: usize = 7;
+pub const DF_SHIFT: usize = 10;
 pub const OF_SHIFT: usize = 11;

 pub const CF: u64 = 1 << CF_SHIFT;
@@ -29,4 +30,5 @@ pub const PF: u64 = 1 << PF_SHIFT;
 pub const AF: u64 = 1 << AF_SHIFT;
 pub const ZF: u64 = 1 << ZF_SHIFT;
 pub const SF: u64 = 1 << SF_SHIFT;
+pub const DF: u64 = 1 << DF_SHIFT;
 pub const OF: u64 = 1 << OF_SHIFT;
--- a/hypervisor/src/cpu.rs
+++ b/hypervisor/src/cpu.rs
@@ -23,6 +23,8 @@ use crate::CpuState;
 use crate::MpState;
 #[cfg(target_arch = "x86_64")]
 use crate::Xsave;
+#[cfg(feature = "mshv")]
+use mshv_bindings::*;
 use thiserror::Error;

 #[derive(Error, Debug)]
@@ -172,7 +174,7 @@ pub enum HypervisorCpuError {
    /// Enabling HyperV SynIC error
    ///
    #[error("Failed to enable HyperV SynIC")]
-    EnableHyperVSynIC(#[source] anyhow::Error),
+    EnableHyperVSyncIc(#[source] anyhow::Error),
    ///
    /// Getting AArch64 core register error
    ///
@@ -198,6 +200,17 @@ pub enum HypervisorCpuError {
    ///
    #[error("Failed to set system register: {0}")]
    SetSysRegister(#[source] anyhow::Error),
+    ///
+    /// GVA translation error
+    ///
+    #[error("Failed to translate GVA: {0}")]
+    TranslateVirtualAddress(#[source] anyhow::Error),
+    ///
+    /// Failed to initialize TDX on CPU
+    ///
+    #[cfg(feature = "tdx")]
+    #[error("Failed to initialize TDX: {0}")]
+    InitializeTdx(#[source] std::io::Error),
 }

 #[derive(Debug)]
@@ -398,4 +411,14 @@ pub trait Vcpu: Send + Sync {
    /// Triggers the running of the current virtual CPU returning an exit reason.
    ///
    fn run(&self) -> std::result::Result<VmExit, HypervisorCpuError>;
+    #[cfg(all(feature = "mshv", target_arch = "x86_64"))]
+    ///
+    /// Translate guest virtual address to guest physical address
+    ///
+    fn translate_gva(&self, gva: u64, flags: u64) -> Result<(u64, hv_translate_gva_result)>;
+    ///
+    /// Initialize TDX support on the vCPU
+    ///
+    #[cfg(feature = "tdx")]
+    fn tdx_init(&self, hob_address: u64) -> Result<()>;
 }
--- a/hypervisor/src/hypervisor.rs
+++ b/hypervisor/src/hypervisor.rs
@@ -15,7 +15,6 @@ use crate::x86_64::MsrList;
 #[cfg(all(feature = "kvm", target_arch = "x86_64"))]
 use kvm_ioctls::Cap;
 use std::sync::Arc;
-
 use thiserror::Error;

 #[derive(Error, Debug)]
@@ -87,6 +86,12 @@ pub trait Hypervisor: Send + Sync {
    fn create_vm(&self) -> Result<Arc<dyn Vm>>;
    #[cfg(feature = "kvm")]
    ///
+    /// Create a Vm of a specific type using the underlying hypervisor
+    /// Return a hypervisor-agnostic Vm trait object
+    ///
+    fn create_vm_with_type(&self, vm_type: u64) -> Result<Arc<dyn Vm>>;
+    #[cfg(feature = "kvm")]
+    ///
    /// Returns the size of the memory mapping required to use the vcpu's structures
    ///
    fn get_vcpu_mmap_size(&self) -> Result<usize>;
--- a/hypervisor/src/kvm/mod.rs
+++ b/hypervisor/src/kvm/mod.rs
@@ -16,6 +16,7 @@ pub use crate::aarch64::{
 use crate::cpu;
 use crate::device;
 use crate::hypervisor;
+use crate::vec_with_array_field;
 use crate::vm::{self, VmmOps};
 #[cfg(target_arch = "aarch64")]
 use crate::{arm64_core_reg_id, offset__of};
@@ -32,48 +33,45 @@ use vmm_sys_util::eventfd::EventFd;
 // x86_64 dependencies
 #[cfg(target_arch = "x86_64")]
 pub mod x86_64;
-
+#[cfg(target_arch = "x86_64")]
+use crate::arch::x86::NUM_IOAPIC_PINS;
+#[cfg(target_arch = "aarch64")]
+use aarch64::{RegList, Register, StandardRegisters};
+#[cfg(target_arch = "x86_64")]
+use kvm_bindings::{
+    kvm_enable_cap, kvm_msr_entry, MsrList, KVM_CAP_HYPERV_SYNIC, KVM_CAP_SPLIT_IRQCHIP,
+};
 #[cfg(target_arch = "x86_64")]
 use x86_64::{
    check_required_kvm_extensions, FpuState, SpecialRegisters, StandardRegisters, KVM_TSS_ADDRESS,
 };
-
-#[cfg(target_arch = "aarch64")]
-use aarch64::{RegList, Register, StandardRegisters};
-
 #[cfg(target_arch = "x86_64")]
 pub use x86_64::{
    CpuId, CpuIdEntry, ExtendedControlRegisters, LapicState, MsrEntries, VcpuKvmState as CpuState,
    Xsave, CPUID_FLAG_VALID_INDEX,
 };
-
-#[cfg(target_arch = "x86_64")]
-use kvm_bindings::{
-    kvm_enable_cap, kvm_msr_entry, MsrList, KVM_CAP_HYPERV_SYNIC, KVM_CAP_SPLIT_IRQCHIP,
-};
-
-#[cfg(target_arch = "x86_64")]
-use crate::arch::x86::NUM_IOAPIC_PINS;
-
 // aarch64 dependencies
 #[cfg(target_arch = "aarch64")]
 pub mod aarch64;
+pub use kvm_bindings;
+#[cfg(feature = "tdx")]
+use kvm_bindings::KVMIO;
+pub use kvm_bindings::{
+    kvm_create_device, kvm_device_type_KVM_DEV_TYPE_VFIO, kvm_irq_routing, kvm_irq_routing_entry,
+    kvm_userspace_memory_region, KVM_IRQ_ROUTING_IRQCHIP, KVM_IRQ_ROUTING_MSI,
+    KVM_MEM_LOG_DIRTY_PAGES, KVM_MEM_READONLY, KVM_MSI_VALID_DEVID,
+};
 #[cfg(target_arch = "aarch64")]
 use kvm_bindings::{
    kvm_regs, user_fpsimd_state, user_pt_regs, KVM_NR_SPSR, KVM_REG_ARM64, KVM_REG_ARM_CORE,
    KVM_REG_SIZE_U128, KVM_REG_SIZE_U32, KVM_REG_SIZE_U64,
 };
-#[cfg(target_arch = "aarch64")]
-use std::mem;
-
-pub use kvm_bindings;
-pub use kvm_bindings::{
-    kvm_create_device, kvm_device_type_KVM_DEV_TYPE_VFIO, kvm_irq_routing, kvm_irq_routing_entry,
-    kvm_userspace_memory_region, KVM_IRQ_ROUTING_MSI, KVM_MEM_LOG_DIRTY_PAGES, KVM_MEM_READONLY,
-    KVM_MSI_VALID_DEVID,
-};
 pub use kvm_ioctls;
 pub use kvm_ioctls::{Cap, Kvm};
+#[cfg(target_arch = "aarch64")]
+use std::mem;
+#[cfg(feature = "tdx")]
+use vmm_sys_util::{ioctl::ioctl_with_val, ioctl_expr, ioctl_ioc_nr, ioctl_iowr_nr};

 ///
 /// Export generically-named wrappers of kvm-bindings for Unix-based platforms
@@ -86,6 +84,21 @@ pub use {
    kvm_bindings::kvm_vcpu_events as VcpuEvents, kvm_ioctls::DeviceFd, kvm_ioctls::IoEventAddress,
    kvm_ioctls::VcpuExit,
 };
+
+#[cfg(feature = "tdx")]
+ioctl_iowr_nr!(KVM_MEMORY_ENCRYPT_OP, KVMIO, 0xba, std::os::raw::c_ulong);
+
+#[cfg(feature = "tdx")]
+#[repr(u32)]
+enum TdxCommand {
+    #[allow(dead_code)]
+    Capabilities = 0,
+    InitVm,
+    InitVcpu,
+    InitMemRegion,
+    Finalize,
+}
+
 #[derive(Clone, Copy, Debug, PartialEq, Deserialize, Serialize)]
 pub struct KvmVmState {}

@@ -98,36 +111,6 @@ pub struct KvmVm {
    state: KvmVmState,
 }

-// Returns a `Vec<T>` with a size in bytes at least as large as `size_in_bytes`.
-fn vec_with_size_in_bytes<T: Default>(size_in_bytes: usize) -> Vec<T> {
-    let rounded_size = (size_in_bytes + size_of::<T>() - 1) / size_of::<T>();
-    let mut v = Vec::with_capacity(rounded_size);
-    v.resize_with(rounded_size, T::default);
-    v
-}
-
-// The kvm API has many structs that resemble the following `Foo` structure:
-//
-// ```
-// #[repr(C)]
-// struct Foo {
-//    some_data: u32
-//    entries: __IncompleteArrayField<__u32>,
-// }
-// ```
-//
-// In order to allocate such a structure, `size_of::<Foo>()` would be too small because it would not
-// include any space for `entries`. To make the allocation large enough while still being aligned
-// for `Foo`, a `Vec<Foo>` is created. Only the first element of `Vec<Foo>` would actually be used
-// as a `Foo`. The remaining memory in the `Vec<Foo>` is for `entries`, which must be contiguous
-// with `Foo`. This function is used to make the `Vec<Foo>` with enough space for `count` entries.
-use std::mem::size_of;
-fn vec_with_array_field<T: Default, F>(count: usize) -> Vec<T> {
-    let element_space = count * size_of::<F>();
-    let vec_size_bytes = size_of::<T>() + element_space;
-    vec_with_size_in_bytes(vec_size_bytes)
-}
-
 ///
 /// Implementation of Vm trait for KVM
 /// Example:
@@ -374,7 +357,109 @@ impl vm::Vm for KvmVm {
            .get_dirty_log(slot, memory_size as usize)
            .map_err(|e| vm::HypervisorVmError::GetDirtyLog(e.into()))
    }
+
+    ///
+    /// Initialize TDX for this VM
+    ///
+    #[cfg(feature = "tdx")]
+    fn tdx_init(&self, cpuid: &CpuId, max_vcpus: u32) -> vm::Result<()> {
+        #[repr(C)]
+        struct TdxInitVm {
+            max_vcpus: u32,
+            reserved: u32,
+            attributes: u64,
+            cpuid: u64,
+        }
+        let data = TdxInitVm {
+            max_vcpus,
+            reserved: 0,
+            attributes: 0,
+            cpuid: cpuid.as_fam_struct_ptr() as u64,
+        };
+
+        tdx_command(
+            &self.fd.as_raw_fd(),
+            TdxCommand::InitVm,
+            0,
+            &data as *const _ as u64,
+        )
+        .map_err(vm::HypervisorVmError::InitializeTdx)
+    }
+
+    ///
+    /// Finalize the TDX setup for this VM
+    ///
+    #[cfg(feature = "tdx")]
+    fn tdx_finalize(&self) -> vm::Result<()> {
+        tdx_command(&self.fd.as_raw_fd(), TdxCommand::Finalize, 0, 0)
+            .map_err(vm::HypervisorVmError::FinalizeTdx)
+    }
+
+    ///
+    /// Initialize memory regions for the TDX VM
+    ///
+    #[cfg(feature = "tdx")]
+    fn tdx_init_memory_region(
+        &self,
+        host_address: u64,
+        guest_address: u64,
+        size: u64,
+        measure: bool,
+    ) -> vm::Result<()> {
+        #[repr(C)]
+        struct TdxInitMemRegion {
+            host_address: u64,
+            guest_address: u64,
+            pages: u64,
+        }
+        let data = TdxInitMemRegion {
+            host_address,
+            guest_address,
+            pages: size / 4096,
+        };
+
+        tdx_command(
+            &self.fd.as_raw_fd(),
+            TdxCommand::InitMemRegion,
+            if measure { 1 } else { 0 },
+            &data as *const _ as u64,
+        )
+        .map_err(vm::HypervisorVmError::InitMemRegionTdx)
+    }
 }
+
+#[cfg(feature = "tdx")]
+fn tdx_command(
+    fd: &RawFd,
+    command: TdxCommand,
+    metadata: u32,
+    data: u64,
+) -> std::result::Result<(), std::io::Error> {
+    #[repr(C)]
+    struct TdxIoctlCmd {
+        command: TdxCommand,
+        metadata: u32,
+        data: u64,
+    }
+    let cmd = TdxIoctlCmd {
+        command,
+        metadata,
+        data,
+    };
+    let ret = unsafe {
+        ioctl_with_val(
+            fd,
+            KVM_MEMORY_ENCRYPT_OP(),
+            &cmd as *const TdxIoctlCmd as std::os::raw::c_ulong,
+        )
+    };
+
+    if ret < 0 {
+        return Err(std::io::Error::last_os_error());
+    }
+    Ok(())
+}
+
 /// Wrapper over KVM system ioctls.
 pub struct KvmHypervisor {
    kvm: Kvm,
@@ -407,18 +492,18 @@ impl KvmHypervisor {
 /// let vm = hypervisor.create_vm().expect("new VM fd creation failed");
 ///
 impl hypervisor::Hypervisor for KvmHypervisor {
-    /// Create a KVM vm object and return the object as Vm trait object
+    /// Create a KVM vm object of a specific VM type and return the object as Vm trait object
    /// Example
    /// # extern crate hypervisor;
    /// # use hypervisor::KvmHypervisor;
    /// use hypervisor::KvmVm;
    /// let hypervisor = KvmHypervisor::new().unwrap();
-    /// let vm = hypervisor.create_vm().unwrap()
+    /// let vm = hypervisor.create_vm_with_type(KvmVmType::LegacyVm).unwrap()
    ///
-    fn create_vm(&self) -> hypervisor::Result<Arc<dyn vm::Vm>> {
+    fn create_vm_with_type(&self, vm_type: u64) -> hypervisor::Result<Arc<dyn vm::Vm>> {
        let fd: VmFd;
        loop {
-            match self.kvm.create_vm() {
+            match self.kvm.create_vm_with_type(vm_type) {
                Ok(res) => fd = res,
                Err(e) => {
                    if e.errno() == libc::EINTR {
@@ -440,7 +525,7 @@ impl hypervisor::Hypervisor for KvmHypervisor {
        {
            let msr_list = self.get_msr_list()?;
            let num_msrs = msr_list.as_fam_struct_ref().nmsrs as usize;
-            let mut msrs = MsrEntries::new(num_msrs);
+            let mut msrs = MsrEntries::new(num_msrs).unwrap();
            let indices = msr_list.as_slice();
            let msr_entries = msrs.as_mut_slice();
            for (pos, index) in indices.iter().enumerate() {
@@ -463,6 +548,18 @@ impl hypervisor::Hypervisor for KvmHypervisor {
        }
    }

+    /// Create a KVM vm object and return the object as Vm trait object
+    /// Example
+    /// # extern crate hypervisor;
+    /// # use hypervisor::KvmHypervisor;
+    /// use hypervisor::KvmVm;
+    /// let hypervisor = KvmHypervisor::new().unwrap();
+    /// let vm = hypervisor.create_vm().unwrap()
+    ///
+    fn create_vm(&self) -> hypervisor::Result<Arc<dyn vm::Vm>> {
+        self.create_vm_with_type(0) // Create with default platform type
+    }
+
    fn check_required_extensions(&self) -> hypervisor::Result<()> {
        check_required_kvm_extensions(&self.kvm).expect("Missing KVM capabilities");
        Ok(())
@@ -612,7 +709,7 @@ impl cpu::Vcpu for KvmVcpu {
        };
        self.fd
            .enable_cap(&cap)
-            .map_err(|e| cpu::HypervisorCpuError::EnableHyperVSynIC(e.into()))
+            .map_err(|e| cpu::HypervisorCpuError::EnableHyperVSyncIc(e.into()))
    }
    ///
    /// X86 specific call to retrieve the CPUID registers.
@@ -1043,7 +1140,7 @@ impl cpu::Vcpu for KvmVcpu {
    fn system_registers(&self, state: &mut Vec<Register>) -> cpu::Result<()> {
        // Call KVM_GET_REG_LIST to get all registers available to the guest. For ArmV8 there are
        // around 500 registers.
-        let mut reg_list = RegList::new(512);
+        let mut reg_list = RegList::new(500).unwrap();
        self.fd
            .get_reg_list(&mut reg_list)
            .map_err(|e| cpu::HypervisorCpuError::GetRegList(e.into()))?;
@@ -1177,7 +1274,7 @@ impl cpu::Vcpu for KvmVcpu {
        let msrs = if num_msrs != expected_num_msrs {
            let mut faulty_msr_index = num_msrs;
            let mut msr_entries_tmp =
-                MsrEntries::from_entries(&msr_entries.as_slice()[..faulty_msr_index]);
+                MsrEntries::from_entries(&msr_entries.as_slice()[..faulty_msr_index]).unwrap();

            loop {
                warn!(
@@ -1187,7 +1284,7 @@ impl cpu::Vcpu for KvmVcpu {

                let start_pos = faulty_msr_index + 1;
                let mut sub_msr_entries =
-                    MsrEntries::from_entries(&msr_entries.as_slice()[start_pos..]);
+                    MsrEntries::from_entries(&msr_entries.as_slice()[start_pos..]).unwrap();
                let expected_num_msrs = sub_msr_entries.as_fam_struct_ref().nmsrs as usize;
                let num_msrs = self.get_msrs(&mut sub_msr_entries)?;

@@ -1311,7 +1408,8 @@ impl cpu::Vcpu for KvmVcpu {
                );

                let start_pos = faulty_msr_index + 1;
-                let sub_msr_entries = MsrEntries::from_entries(&state.msrs.as_slice()[start_pos..]);
+                let sub_msr_entries =
+                    MsrEntries::from_entries(&state.msrs.as_slice()[start_pos..]).unwrap();
                let expected_num_msrs = sub_msr_entries.as_fam_struct_ref().nmsrs as usize;
                let num_msrs = self.set_msrs(&sub_msr_entries)?;

@@ -1338,6 +1436,15 @@ impl cpu::Vcpu for KvmVcpu {

        Ok(())
    }
+
+    ///
+    /// Initialize TDX for this CPU
+    ///
+    #[cfg(feature = "tdx")]
+    fn tdx_init(&self, hob_address: u64) -> cpu::Result<()> {
+        tdx_command(&self.fd.as_raw_fd(), TdxCommand::InitVcpu, 0, hob_address)
+            .map_err(cpu::HypervisorCpuError::InitializeTdx)
+    }
 }

 /// Device struct for KVM
--- a/hypervisor/src/kvm/x86_64/mod.rs
+++ b/hypervisor/src/kvm/x86_64/mod.rs
@@ -111,6 +111,7 @@ pub fn boot_msr_entries() -> MsrEntries {
        ),
        msr_data!(msr_index::MSR_MTRRdefType, MTRR_ENABLE | MTRR_MEM_TYPE_WB),
    ])
+    .unwrap()
 }

 ///
--- a/hypervisor/src/lib.rs
+++ b/hypervisor/src/lib.rs
@@ -72,3 +72,33 @@ pub fn new() -> std::result::Result<Arc<dyn Hypervisor>, HypervisorError> {

    Ok(Arc::new(hv))
 }
+
+// Returns a `Vec<T>` with a size in bytes at least as large as `size_in_bytes`.
+fn vec_with_size_in_bytes<T: Default>(size_in_bytes: usize) -> Vec<T> {
+    let rounded_size = (size_in_bytes + size_of::<T>() - 1) / size_of::<T>();
+    let mut v = Vec::with_capacity(rounded_size);
+    v.resize_with(rounded_size, T::default);
+    v
+}
+
+// The kvm API has many structs that resemble the following `Foo` structure:
+//
+// ```
+// #[repr(C)]
+// struct Foo {
+//    some_data: u32
+//    entries: __IncompleteArrayField<__u32>,
+// }
+// ```
+//
+// In order to allocate such a structure, `size_of::<Foo>()` would be too small because it would not
+// include any space for `entries`. To make the allocation large enough while still being aligned
+// for `Foo`, a `Vec<Foo>` is created. Only the first element of `Vec<Foo>` would actually be used
+// as a `Foo`. The remaining memory in the `Vec<Foo>` is for `entries`, which must be contiguous
+// with `Foo`. This function is used to make the `Vec<Foo>` with enough space for `count` entries.
+use std::mem::size_of;
+pub fn vec_with_array_field<T: Default, F>(count: usize) -> Vec<T> {
+    let element_space = count * size_of::<F>();
+    let vec_size_bytes = size_of::<T>() + element_space;
+    vec_with_size_in_bytes(vec_size_bytes)
+}
--- a/hypervisor/src/mshv/mod.rs
+++ b/hypervisor/src/mshv/mod.rs
@@ -5,17 +5,16 @@

 use crate::arch::emulator::{PlatformEmulator, PlatformError};

-use thiserror::Error;
-
 #[cfg(target_arch = "x86_64")]
 use crate::arch::x86::emulator::{Emulator, EmulatorCpuState};
 use crate::cpu;
 use crate::cpu::Vcpu;
 use crate::hypervisor;
+use crate::vec_with_array_field;
 use crate::vm::{self, VmmOps};
 pub use mshv_bindings::*;
 pub use mshv_ioctls::IoEventAddress;
-use mshv_ioctls::{set_registers_64, InterruptRequest, Mshv, NoDatamatch, VcpuFd, VmFd};
+use mshv_ioctls::{set_registers_64, Mshv, NoDatamatch, VcpuFd, VmFd};
 use serde_derive::{Deserialize, Serialize};
 use std::sync::Arc;
 use vm::DataMatch;
@@ -29,7 +28,6 @@ pub use x86_64::VcpuMshvState as CpuState;
 #[cfg(target_arch = "x86_64")]
 pub use x86_64::*;

-use std::collections::HashMap;
 use std::os::unix::io::AsRawFd;
 use std::sync::RwLock;

@@ -93,7 +91,7 @@ impl hypervisor::Hypervisor for MshvHypervisor {

        let msr_list = self.get_msr_list()?;
        let num_msrs = msr_list.as_fam_struct_ref().nmsrs as usize;
-        let mut msrs = MsrEntries::new(num_msrs);
+        let mut msrs = MsrEntries::new(num_msrs).unwrap();
        let indices = msr_list.as_slice();
        let msr_entries = msrs.as_mut_slice();
        for (pos, index) in indices.iter().enumerate() {
@@ -101,12 +99,9 @@ impl hypervisor::Hypervisor for MshvHypervisor {
        }
        let vm_fd = Arc::new(fd);

-        let gsi_routes = Arc::new(RwLock::new(HashMap::new()));
-
        Ok(Arc::new(MshvVm {
            fd: vm_fd,
            msrs,
-            gsi_routes,
            hv_state: hv_state_init(),
            vmmops: None,
        }))
@@ -115,7 +110,7 @@ impl hypervisor::Hypervisor for MshvHypervisor {
    /// Get the supported CpuID
    ///
    fn get_cpuid(&self) -> hypervisor::Result<CpuId> {
-        Ok(CpuId::new(1))
+        Ok(CpuId::new(1).unwrap())
    }
    #[cfg(target_arch = "x86_64")]
    ///
@@ -128,38 +123,6 @@ impl hypervisor::Hypervisor for MshvHypervisor {
    }
 }

-#[derive(Clone)]
-// A software emulated TLB.
-// This is mostly used by the instruction emulator to cache gva to gpa translations
-// passed from the hypervisor.
-struct SoftTLB {
-    addr_map: HashMap<u64, u64>,
-}
-
-impl SoftTLB {
-    fn new() -> SoftTLB {
-        SoftTLB {
-            addr_map: HashMap::new(),
-        }
-    }
-
-    // Adds a gva -> gpa mapping into the TLB.
-    fn add_mapping(&mut self, gva: u64, gpa: u64) {
-        *self.addr_map.entry(gva).or_insert(gpa) = gpa;
-    }
-
-    // Do the actual gva -> gpa translation
-    fn translate(&self, gva: u64) -> Result<u64, PlatformError> {
-        self.addr_map
-            .get(&gva)
-            .ok_or_else(|| PlatformError::UnmappedGVA(anyhow!("{:#?}", gva)))
-            .map(|v| *v)
-
-        // TODO Check if we could fallback to e.g. an hypercall for doing
-        // the translation for us.
-    }
-}
-#[allow(clippy::type_complexity)]
 #[allow(dead_code)]
 /// Vcpu struct for Microsoft Hypervisor
 pub struct MshvVcpu {
@@ -167,7 +130,6 @@ pub struct MshvVcpu {
    vp_index: u8,
    cpuid: CpuId,
    msrs: MsrEntries,
-    gsi_routes: Arc<RwLock<HashMap<u32, MshvIrqRoutingEntry>>>,
    hv_state: Arc<RwLock<HvState>>, // Mshv State
    vmmops: Option<Arc<Box<dyn vm::VmmOps>>>,
 }
@@ -322,17 +284,52 @@ impl cpu::Vcpu for MshvVcpu {
                hv_message_type_HVMSG_X64_IO_PORT_INTERCEPT => {
                    let info = x.to_ioport_info().unwrap();
                    let access_info = info.access_info;
+                    let len = unsafe { access_info.__bindgen_anon_1.access_size() } as usize;
+                    let is_write = info.header.intercept_access_type == 1;
+                    let port = info.port_number;
+                    let mut data: [u8; 4] = [0; 4];
+                    let mut ret_rax = info.rax;
+
+                    /*
+                     * XXX: Ignore QEMU fw_cfg (0x5xx) and debug console (0x402) ports.
+                     *
+                     * Cloud Hypervisor doesn't support fw_cfg at the moment. It does support 0x402
+                     * under the "fwdebug" feature flag. But that feature is not enabled by default
+                     * and is considered legacy.
+                     *
+                     * OVMF unconditionally pokes these IO ports with string IO.
+                     *
+                     * Instead of trying to implement string IO support now which does not do much
+                     * now, skip those ports explicitly to avoid panicking.
+                     *
+                     * Proper string IO support can be added once we gain the ability to translate
+                     * guest virtual addresses to guest physical addresses on MSHV.
+                     */
+                    match port {
+                        0x402 | 0x510 | 0x511 | 0x514 => {
+                            let insn_len = info.header.instruction_length() as u64;
+
+                            /* Advance RIP and update RAX */
+                            let arr_reg_name_value = [
+                                (
+                                    hv_register_name::HV_X64_REGISTER_RIP,
+                                    info.header.rip + insn_len,
+                                ),
+                                (hv_register_name::HV_X64_REGISTER_RAX, ret_rax),
+                            ];
+                            set_registers_64!(self.fd, arr_reg_name_value)
+                                .map_err(|e| cpu::HypervisorCpuError::SetRegister(e.into()))?;
+                            return Ok(cpu::VmExit::Ignore);
+                        }
+                        _ => {}
+                    }
+
                    if unsafe { access_info.__bindgen_anon_1.string_op() } == 1 {
                        panic!("String IN/OUT not supported");
                    }
                    if unsafe { access_info.__bindgen_anon_1.rep_prefix() } == 1 {
                        panic!("Rep IN/OUT not supported");
                    }
-                    let len = unsafe { access_info.__bindgen_anon_1.access_size() } as usize;
-                    let is_write = info.header.intercept_access_type == 1;
-                    let port = info.port_number;
-                    let mut data: [u8; 4] = [0; 4];
-                    let mut ret_rax = info.rax;

                    if is_write {
                        let data = (info.rax as u32).to_le_bytes();
@@ -376,14 +373,9 @@ impl cpu::Vcpu for MshvVcpu {

                    let mut context = MshvEmulatorContext {
                        vcpu: self,
-                        tlb: SoftTLB::new(),
+                        map: (info.guest_virtual_address, info.guest_physical_address),
                    };

-                    // Add the GVA <-> GPA mapping.
-                    context
-                        .tlb
-                        .add_mapping(info.guest_virtual_address, info.guest_physical_address);
-
                    // Create a new emulator.
                    let mut emul = Emulator::new(&mut context);

@@ -530,11 +522,47 @@ impl cpu::Vcpu for MshvVcpu {
            xsave,
        })
    }
+    #[cfg(target_arch = "x86_64")]
+    ///
+    /// Translate guest virtual address to guest physical address
+    ///
+    fn translate_gva(&self, gva: u64, flags: u64) -> cpu::Result<(u64, hv_translate_gva_result)> {
+        let r = self
+            .fd
+            .translate_gva(gva, flags)
+            .map_err(|e| cpu::HypervisorCpuError::TranslateVirtualAddress(e.into()))?;
+
+        Ok(r)
+    }
 }

 struct MshvEmulatorContext<'a> {
    vcpu: &'a MshvVcpu,
-    tlb: SoftTLB,
+    map: (u64, u64), // Initial GVA to GPA mapping provided by the hypervisor
+}
+
+impl<'a> MshvEmulatorContext<'a> {
+    // Do the actual gva -> gpa translation
+    #[allow(non_upper_case_globals)]
+    fn translate(&self, gva: u64) -> Result<u64, PlatformError> {
+        if self.map.0 == gva {
+            return Ok(self.map.1);
+        }
+
+        // TODO: More fine-grained control for the flags
+        let flags = HV_TRANSLATE_GVA_VALIDATE_READ | HV_TRANSLATE_GVA_VALIDATE_WRITE;
+
+        let r = self
+            .vcpu
+            .translate_gva(gva, flags.into())
+            .map_err(|e| PlatformError::TranslateVirtualAddress(anyhow!(e)))?;
+
+        let result_code = unsafe { r.1.__bindgen_anon_1.result_code };
+        match result_code {
+            hv_translate_gva_result_code_HvTranslateGvaSuccess => Ok(r.0),
+            _ => Err(PlatformError::TranslateVirtualAddress(anyhow!(result_code))),
+        }
+    }
 }

 /// Platform emulation for Hyper-V
@@ -542,7 +570,7 @@ impl<'a> PlatformEmulator for MshvEmulatorContext<'a> {
    type CpuState = EmulatorCpuState;

    fn read_memory(&self, gva: u64, data: &mut [u8]) -> Result<(), PlatformError> {
-        let gpa = self.tlb.translate(gva)?;
+        let gpa = self.translate(gva)?;
        debug!(
            "mshv emulator: memory read {} bytes from [{:#x} -> {:#x}]",
            data.len(),
@@ -551,16 +579,18 @@ impl<'a> PlatformEmulator for MshvEmulatorContext<'a> {
        );

        if let Some(vmmops) = &self.vcpu.vmmops {
-            vmmops
-                .mmio_read(gpa, data)
-                .map_err(|e| PlatformError::MemoryReadFailure(e.into()))?;
+            if vmmops.guest_mem_read(gpa, data).is_err() {
+                vmmops
+                    .mmio_read(gpa, data)
+                    .map_err(|e| PlatformError::MemoryReadFailure(e.into()))?;
+            }
        }

        Ok(())
    }

    fn write_memory(&mut self, gva: u64, data: &[u8]) -> Result<(), PlatformError> {
-        let gpa = self.tlb.translate(gva)?;
+        let gpa = self.translate(gva)?;
        debug!(
            "mshv emulator: memory write {} bytes at [{:#x} -> {:#x}]",
            data.len(),
@@ -569,9 +599,11 @@ impl<'a> PlatformEmulator for MshvEmulatorContext<'a> {
        );

        if let Some(vmmops) = &self.vcpu.vmmops {
-            vmmops
-                .mmio_write(gpa, data)
-                .map_err(|e| PlatformError::MemoryWriteFailure(e.into()))?;
+            if vmmops.guest_mem_write(gpa, data).is_err() {
+                vmmops
+                    .mmio_write(gpa, data)
+                    .map_err(|e| PlatformError::MemoryWriteFailure(e.into()))?;
+            }
        }

        Ok(())
@@ -622,7 +654,7 @@ impl<'a> PlatformEmulator for MshvEmulatorContext<'a> {
    }

    fn gva_to_gpa(&self, gva: u64) -> Result<u64, PlatformError> {
-        self.tlb.translate(gva)
+        self.translate(gva)
    }

    fn fetch(&self, _ip: u64, _instruction_bytes: &mut [u8]) -> Result<(), PlatformError> {
@@ -630,14 +662,11 @@ impl<'a> PlatformEmulator for MshvEmulatorContext<'a> {
    }
 }

-#[allow(clippy::type_complexity)]
 #[allow(dead_code)]
 /// Wrapper over Mshv VM ioctls.
 pub struct MshvVm {
    fd: Arc<VmFd>,
    msrs: MsrEntries,
-    // GSI routing information
-    gsi_routes: Arc<RwLock<HashMap<u32, MshvIrqRoutingEntry>>>,
    // Hypervisor State
    hv_state: Arc<RwLock<HvState>>,
    vmmops: Option<Arc<Box<dyn vm::VmmOps>>>,
@@ -678,19 +707,9 @@ impl vm::Vm for MshvVm {
    fn register_irqfd(&self, fd: &EventFd, gsi: u32) -> vm::Result<()> {
        debug!("register_irqfd fd {} gsi {}", fd.as_raw_fd(), gsi);

-        let gsi_routes = self.gsi_routes.read().unwrap();
-
-        if let Some(e) = gsi_routes.get(&gsi) {
-            let msi = e
-                .get_msi_routing()
-                .map_err(|e| vm::HypervisorVmError::RegisterIrqFd(e.into()))?;
-            let request = msi.to_interrupt_request();
-            self.fd
-                .register_irqfd(&fd, gsi, &request)
-                .map_err(|e| vm::HypervisorVmError::RegisterIrqFd(e.into()))?;
-        } else {
-            error!("No routing info found for GSI {}", gsi)
-        }
+        self.fd
+            .register_irqfd(&fd, gsi)
+            .map_err(|e| vm::HypervisorVmError::RegisterIrqFd(e.into()))?;

        Ok(())
    }
@@ -721,9 +740,8 @@ impl vm::Vm for MshvVm {
        let vcpu = MshvVcpu {
            fd: vcpu_fd,
            vp_index: id,
-            cpuid: CpuId::new(1),
+            cpuid: CpuId::new(1).unwrap(),
            msrs: self.msrs.clone(),
-            gsi_routes: self.gsi_routes.clone(),
            hv_state: self.hv_state.clone(),
            vmmops,
        };
@@ -807,17 +825,20 @@ impl vm::Vm for MshvVm {
        )))
    }

-    fn set_gsi_routing(&self, irq_routing: &[IrqRoutingEntry]) -> vm::Result<()> {
-        let mut routes = self.gsi_routes.write().unwrap();
+    fn set_gsi_routing(&self, entries: &[IrqRoutingEntry]) -> vm::Result<()> {
+        let mut msi_routing =
+            vec_with_array_field::<mshv_msi_routing, mshv_msi_routing_entry>(entries.len());
+        msi_routing[0].nr = entries.len() as u32;

-        routes.drain();
-
-        for r in irq_routing {
-            debug!("gsi routing {:x?}", r);
-            routes.insert(r.gsi, *r);
+        unsafe {
+            let entries_slice: &mut [mshv_msi_routing_entry] =
+                msi_routing[0].entries.as_mut_slice(entries.len());
+            entries_slice.copy_from_slice(&entries);
        }

-        Ok(())
+        self.fd
+            .set_msi_routing(&msi_routing[0])
+            .map_err(|e| vm::HypervisorVmError::SetGsiRouting(e.into()))
    }
    ///
    /// Get the Vm state. Return VM specific data
@@ -843,104 +864,6 @@ impl vm::Vm for MshvVm {
 }
 pub use hv_cpuid_entry as CpuIdEntry;

-#[derive(Copy, Clone, Debug)]
-pub struct MshvIrqRoutingMsi {
-    pub address_lo: u32,
-    pub address_hi: u32,
-    pub data: u32,
-}
-
-#[derive(Copy, Clone, Debug)]
-pub enum MshvIrqRouting {
-    Msi(MshvIrqRoutingMsi),
-}
-
-#[derive(Error, Debug)]
-pub enum MshvIrqRoutingEntryError {
-    #[error("Invalid MSI address: {0}")]
-    InvalidMsiAddress(#[source] anyhow::Error),
-}
-
-#[derive(Copy, Clone, Debug)]
-pub struct MshvIrqRoutingEntry {
-    pub gsi: u32,
-    pub route: MshvIrqRouting,
-}
-pub type IrqRoutingEntry = MshvIrqRoutingEntry;
-
-impl MshvIrqRoutingEntry {
-    fn get_msi_routing(&self) -> Result<MshvIrqRoutingMsi, MshvIrqRoutingEntryError> {
-        let MshvIrqRouting::Msi(msi) = self.route;
-        if msi.address_hi != 0 {
-            return Err(MshvIrqRoutingEntryError::InvalidMsiAddress(anyhow!(
-                "MSI high address part is not zero"
-            )));
-        }
-        Ok(msi)
-    }
-}
-
-impl MshvIrqRoutingMsi {
-    ///
-    /// See Intel SDM vol3 10.11.1
-    /// We assume APIC ID and Hyper-V Vcpu ID are the same value
-    ///
-
-    fn get_destination(&self) -> u64 {
-        ((self.address_lo >> 12) & 0xff).into()
-    }
-
-    fn get_destination_mode(&self) -> bool {
-        if (self.address_lo >> 2) & 0x1 == 0x1 {
-            return true;
-        }
-        false
-    }
-
-    fn get_vector(&self) -> u8 {
-        (self.data & 0xff) as u8
-    }
-
-    ///
-    ///  True means level triggered
-    ///
-    fn get_trigger_mode(&self) -> bool {
-        if (self.data >> 15) & 0x1 == 0x1 {
-            return true;
-        }
-        false
-    }
-
-    fn get_delivery_mode(&self) -> u8 {
-        ((self.data & 0x700) >> 8) as u8
-    }
-
-    ///
-    ///  Translate from architectural defined delivery mode to Hyper-V type
-    /// See Intel SDM vol3 10.11.2
-    ///
-    fn get_interrupt_type(&self) -> Option<hv_interrupt_type> {
-        match self.get_delivery_mode() {
-            0 => Some(hv_interrupt_type_HV_X64_INTERRUPT_TYPE_FIXED),
-            1 => Some(hv_interrupt_type_HV_X64_INTERRUPT_TYPE_LOWESTPRIORITY),
-            2 => Some(hv_interrupt_type_HV_X64_INTERRUPT_TYPE_SMI),
-            4 => Some(hv_interrupt_type_HV_X64_INTERRUPT_TYPE_NMI),
-            5 => Some(hv_interrupt_type_HV_X64_INTERRUPT_TYPE_INIT),
-            7 => Some(hv_interrupt_type_HV_X64_INTERRUPT_TYPE_EXTINT),
-            _ => None,
-        }
-    }
-
-    pub fn to_interrupt_request(&self) -> InterruptRequest {
-        InterruptRequest {
-            interrupt_type: self.get_interrupt_type().unwrap(),
-            apic_id: self.get_destination(),
-            vector: self.get_vector() as u32,
-            level_triggered: self.get_trigger_mode(),
-            logical_destination_mode: self.get_destination_mode(),
-            long_mode: false,
-        }
-    }
-}
+pub type IrqRoutingEntry = mshv_msi_routing_entry;

 pub const CPUID_FLAG_VALID_INDEX: u32 = 0;
--- a/hypervisor/src/mshv/x86_64/mod.rs
+++ b/hypervisor/src/mshv/x86_64/mod.rs
@@ -119,4 +119,5 @@ pub fn boot_msr_entries() -> MsrEntries {
        msr!(msr_index::MSR_SYSCALL_MASK),
        msr!(msr_index::MSR_IA32_TSC),
    ])
+    .unwrap()
 }
--- a/hypervisor/src/vm.rs
+++ b/hypervisor/src/vm.rs
@@ -12,6 +12,8 @@
 use crate::aarch64::VcpuInit;
 use crate::cpu::Vcpu;
 use crate::device::Device;
+#[cfg(feature = "tdx")]
+use crate::x86_64::CpuId;
 #[cfg(all(feature = "kvm", target_arch = "x86_64"))]
 use crate::ClockData;
 #[cfg(feature = "kvm")]
@@ -36,9 +38,9 @@ pub enum DataMatch {
    DataMatch64(u64),
 }

-impl Into<u64> for DataMatch {
-    fn into(self) -> u64 {
-        match self {
+impl From<DataMatch> for u64 {
+    fn from(dm: DataMatch) -> u64 {
+        match dm {
            DataMatch::DataMatch32(dm) => dm.into(),
            DataMatch::DataMatch64(dm) => dm,
        }
@@ -163,6 +165,25 @@ pub enum HypervisorVmError {
    ///
    #[error("Failed to assert virtual Interrupt: {0}")]
    AsserttVirtualInterrupt(#[source] anyhow::Error),
+
+    #[cfg(feature = "tdx")]
+    ///
+    /// Error initializing TDX on the VM
+    ///
+    #[error("Failed to initialize TDX: {0}")]
+    InitializeTdx(#[source] std::io::Error),
+    #[cfg(feature = "tdx")]
+    ///
+    /// Error finalizing the TDX configuration on the VM
+    ///
+    #[error("Failed to finalize TDX: {0}")]
+    FinalizeTdx(#[source] std::io::Error),
+    #[cfg(feature = "tdx")]
+    ///
+    /// Error initializing the TDX memory region
+    ///
+    #[error("Failed to initialize memory region TDX: {0}")]
+    InitMemRegionTdx(#[source] std::io::Error),
 }
 ///
 /// Result type for returning from a function
@@ -235,6 +256,21 @@ pub trait Vm: Send + Sync {
    fn set_state(&self, state: VmState) -> Result<()>;
    /// Get dirty pages bitmap
    fn get_dirty_log(&self, slot: u32, memory_size: u64) -> Result<Vec<u64>>;
+    #[cfg(feature = "tdx")]
+    /// Initalize TDX on this VM
+    fn tdx_init(&self, cpuid: &CpuId, max_vcpus: u32) -> Result<()>;
+    #[cfg(feature = "tdx")]
+    /// Finalize the configuration of TDX on this VM
+    fn tdx_finalize(&self) -> Result<()>;
+    #[cfg(feature = "tdx")]
+    /// Initalize a TDX memory region for this VM
+    fn tdx_init_memory_region(
+        &self,
+        host_address: u64,
+        guest_address: u64,
+        size: u64,
+        measure: bool,
+    ) -> Result<()>;
 }

 pub trait VmmOps: Send + Sync {
--- a/net_util/Cargo.toml
+++ b/net_util/Cargo.toml
@@ -5,11 +5,11 @@ authors = ["The Chromium OS Authors"]

 [dependencies]
 epoll = ">=4.0.1"
-libc = "0.2.86"
+libc = "0.2.94"
 log = "0.4.14"
 net_gen = { path = "../net_gen" }
-rand = "0.8.3"
-serde = "1.0.123"
+rate_limiter = { path = "../rate_limiter" }
+serde = "1.0.125"
 virtio-bindings = "0.1.0"
 vm-memory = { version = "0.5.0", features = ["backend-mmap", "backend-atomic"] }
 vm-virtio = { path = "../vm-virtio" }
@@ -18,4 +18,4 @@ vmm-sys-util = ">=0.3.1"
 [dev-dependencies]
 lazy_static = "1.3.0"
 pnet = "0.27.2"
-serde_json = "1.0.62"
+serde_json = "1.0.64"
--- a/net_util/src/lib.rs
+++ b/net_util/src/lib.rs
@@ -14,7 +14,7 @@ extern crate libc;
 #[macro_use]
 extern crate log;
 extern crate net_gen;
-extern crate rand;
+extern crate rate_limiter;
 extern crate serde;
 extern crate virtio_bindings;
 extern crate vm_memory;
--- a/net_util/src/mac.rs
+++ b/net_util/src/mac.rs
@@ -5,7 +5,6 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the THIRD-PARTY file.

-use rand::Rng;
 use std::fmt;
 use std::io;
 use std::result::Result;
@@ -58,7 +57,7 @@ impl MacAddr {
        // TODO: using something like std::mem::uninitialized could avoid the extra initialization,
        // if this ever becomes a performance bottleneck.
        let mut bytes = [0u8; MAC_ADDR_LEN];
-        bytes[..].copy_from_slice(&src[..]);
+        bytes[..].copy_from_slice(&src);

        MacAddr { bytes }
    }
@@ -82,7 +81,22 @@ impl MacAddr {

    pub fn local_random() -> MacAddr {
        // Generate a fully random MAC
-        let mut random_bytes = rand::thread_rng().gen::<[u8; MAC_ADDR_LEN]>();
+        let mut random_bytes = [0u8; MAC_ADDR_LEN];
+        unsafe {
+            // Man page says this function will not be interrupted by a signal
+            // for requests less than 256 bytes
+            if libc::getrandom(
+                random_bytes.as_mut_ptr() as *mut _ as *mut libc::c_void,
+                MAC_ADDR_LEN,
+                0,
+            ) < 0
+            {
+                error!(
+                    "Error populating MAC address with random data: {}",
+                    std::io::Error::last_os_error()
+                )
+            }
+        };

        // Set the first byte to make the OUI a locally administered OUI
        random_bytes[0] = 0x2e;
--- a/net_util/src/open_tap.rs
+++ b/net_util/src/open_tap.rs
@@ -70,7 +70,6 @@ pub fn open_tap(
    let mut taps: Vec<Tap> = Vec::new();
    let mut ifname: String = String::new();
    let vnet_hdr_size = vnet_hdr_len() as i32;
-    let flag = net_gen::TUN_F_CSUM | net_gen::TUN_F_UFO | net_gen::TUN_F_TSO4 | net_gen::TUN_F_TSO6;

    // In case the tap interface already exists, check if the number of
    // queues is appropriate. The tap might not support multiqueue while
@@ -98,7 +97,6 @@ pub fn open_tap(
                *host_mac = Some(tap.get_mac_addr().map_err(Error::TapGetMac)?)
            }
            tap.enable().map_err(Error::TapEnable)?;
-            tap.set_offload(flag).map_err(Error::TapSetOffload)?;

            tap.set_vnet_hdr_size(vnet_hdr_size)
                .map_err(Error::TapSetVnetHdrSize)?;
@@ -106,7 +104,6 @@ pub fn open_tap(
            ifname = String::from_utf8(tap.get_if_name()).unwrap();
        } else {
            tap = Tap::open_named(ifname.as_str(), num_rx_q, flags).map_err(Error::TapOpen)?;
-            tap.set_offload(flag).map_err(Error::TapSetOffload)?;

            tap.set_vnet_hdr_size(vnet_hdr_size)
                .map_err(Error::TapSetVnetHdrSize)?;
--- a/net_util/src/queue_pair.rs
+++ b/net_util/src/queue_pair.rs
@@ -2,27 +2,18 @@
 //
 // SPDX-License-Identifier: Apache-2.0 AND BSD-3-Clause

-use super::{register_listener, unregister_listener, vnet_hdr_len, Tap};
-use libc::EAGAIN;
-use std::cmp;
+use super::{unregister_listener, vnet_hdr_len, Tap};
+use rate_limiter::{RateLimiter, TokenType};
 use std::io;
-use std::io::{Read, Write};
 use std::num::Wrapping;
 use std::os::unix::io::{AsRawFd, RawFd};
 use std::sync::atomic::{AtomicU64, Ordering};
 use std::sync::Arc;
-use vm_memory::{Bytes, GuestAddress, GuestAddressSpace, GuestMemoryAtomic, GuestMemoryMmap};
-use vm_virtio::{DescriptorChain, Queue};
-
-/// The maximum buffer size when segmentation offload is enabled. This
-/// includes the 12-byte virtio net header.
-/// http://docs.oasis-open.org/virtio/virtio/v1.0/virtio-v1.0.html#x1-1740003
-const MAX_BUFFER_SIZE: usize = 65562;
+use vm_memory::{Bytes, GuestAddressSpace, GuestMemory, GuestMemoryAtomic, GuestMemoryMmap};
+use vm_virtio::Queue;

 #[derive(Clone)]
 pub struct TxVirtio {
-    pub iovec: Vec<(GuestAddress, usize)>,
-    pub frame_buf: [u8; MAX_BUFFER_SIZE],
    pub counter_bytes: Wrapping<u64>,
    pub counter_frames: Wrapping<u64>,
 }
@@ -36,73 +27,96 @@ impl Default for TxVirtio {
 impl TxVirtio {
    pub fn new() -> Self {
        TxVirtio {
-            iovec: Vec::new(),
-            frame_buf: [0u8; MAX_BUFFER_SIZE],
            counter_bytes: Wrapping(0),
            counter_frames: Wrapping(0),
        }
    }

-    pub fn process_desc_chain(&mut self, mem: &GuestMemoryMmap, tap: &mut Tap, queue: &mut Queue) {
+    pub fn process_desc_chain(
+        &mut self,
+        mem: &GuestMemoryMmap,
+        tap: &mut Tap,
+        queue: &mut Queue,
+        rate_limiter: &mut Option<RateLimiter>,
+    ) -> Result<(), NetQueuePairError> {
        while let Some(avail_desc) = queue.iter(&mem).next() {
            let head_index = avail_desc.index;
-            let mut read_count = 0;
            let mut next_desc = Some(avail_desc);

-            self.iovec.clear();
-            while let Some(desc) = next_desc {
-                if desc.is_write_only() {
+            if let Some(rate_limiter) = rate_limiter {
+                if !rate_limiter.consume(1, TokenType::Ops) {
+                    queue.go_to_previous_position();
                    break;
                }
-                self.iovec.push((desc.addr, desc.len as usize));
-                read_count += desc.len as usize;
+
+                let mut bytes = Wrapping(0);
+                let mut tmp_next_desc = next_desc.clone();
+                while let Some(desc) = tmp_next_desc {
+                    if !desc.is_write_only() {
+                        bytes += Wrapping(desc.len as u64);
+                    }
+                    tmp_next_desc = desc.next_descriptor();
+                }
+                bytes -= Wrapping(vnet_hdr_len() as u64);
+                if !rate_limiter.consume(bytes.0, TokenType::Bytes) {
+                    // Revert the OPS consume().
+                    rate_limiter.manual_replenish(1, TokenType::Ops);
+                    queue.go_to_previous_position();
+                    break;
+                }
+            }
+
+            let mut iovecs = Vec::new();
+            while let Some(desc) = next_desc {
+                if !desc.is_write_only() && desc.len > 0 {
+                    let buf = mem
+                        .get_slice(desc.addr, desc.len as usize)
+                        .map_err(NetQueuePairError::GuestMemory)?
+                        .as_ptr();
+                    let iovec = libc::iovec {
+                        iov_base: buf as *mut libc::c_void,
+                        iov_len: desc.len as libc::size_t,
+                    };
+                    iovecs.push(iovec);
+                }
                next_desc = desc.next_descriptor();
            }

-            read_count = 0;
-            // Copy buffer from across multiple descriptors.
-            // TODO(performance - Issue #420): change this to use `writev()` instead of `write()`
-            // and get rid of the intermediate buffer.
-            for (desc_addr, desc_len) in self.iovec.drain(..) {
-                let limit = cmp::min((read_count + desc_len) as usize, self.frame_buf.len());
+            if !iovecs.is_empty() {
+                let result = unsafe {
+                    libc::writev(
+                        tap.as_raw_fd() as libc::c_int,
+                        iovecs.as_ptr() as *const libc::iovec,
+                        iovecs.len() as libc::c_int,
+                    )
+                };
+                if result < 0 {
+                    let e = std::io::Error::last_os_error();
+                    queue.go_to_previous_position();

-                let read_result =
-                    mem.read_slice(&mut self.frame_buf[read_count..limit as usize], desc_addr);
-                match read_result {
-                    Ok(_) => {
-                        // Increment by number of bytes actually read
-                        read_count += limit - read_count;
-                    }
-                    Err(e) => {
-                        println!("Failed to read slice: {:?}", e);
+                    /* EAGAIN */
+                    if e.kind() == std::io::ErrorKind::WouldBlock {
+                        warn!("net: tx: (recoverable) failed writing to tap: {}", e);
                        break;
                    }
+                    error!("net: tx: failed writing to tap: {}", e);
+                    return Err(NetQueuePairError::WriteTap(e));
                }
+
+                self.counter_bytes += Wrapping(result as u64 - vnet_hdr_len() as u64);
+                self.counter_frames += Wrapping(1);
            }

-            let write_result = tap.write(&self.frame_buf[..read_count]);
-            match write_result {
-                Ok(_) => {}
-                Err(e) => {
-                    println!("net: tx: error failed to write to tap: {}", e);
-                }
-            };
-
-            self.counter_bytes += Wrapping((read_count - vnet_hdr_len()) as u64);
-            self.counter_frames += Wrapping(1);
-
            queue.add_used(&mem, head_index, 0);
            queue.update_avail_event(&mem);
        }
+
+        Ok(())
    }
 }

 #[derive(Clone)]
 pub struct RxVirtio {
-    pub deferred_frame: bool,
-    pub deferred_irqs: bool,
-    pub bytes_read: usize,
-    pub frame_buf: [u8; MAX_BUFFER_SIZE],
    pub counter_bytes: Wrapping<u64>,
    pub counter_frames: Wrapping<u64>,
 }
@@ -116,10 +130,6 @@ impl Default for RxVirtio {
 impl RxVirtio {
    pub fn new() -> Self {
        RxVirtio {
-            deferred_frame: false,
-            deferred_irqs: false,
-            bytes_read: 0,
-            frame_buf: [0u8; MAX_BUFFER_SIZE],
            counter_bytes: Wrapping(0),
            counter_frames: Wrapping(0),
        }
@@ -128,63 +138,89 @@ impl RxVirtio {
    pub fn process_desc_chain(
        &mut self,
        mem: &GuestMemoryMmap,
-        mut next_desc: Option<DescriptorChain>,
+        tap: &mut Tap,
        queue: &mut Queue,
-    ) -> bool {
-        let head_index = next_desc.as_ref().unwrap().index;
-        let mut write_count = 0;
+        rate_limiter: &mut Option<RateLimiter>,
+    ) -> Result<bool, NetQueuePairError> {
+        let mut exhausted_descs = true;
+        let mut rate_limit_reached = false;

-        // Copy from frame into buffer, which may span multiple descriptors.
-        loop {
-            match next_desc {
-                Some(desc) => {
-                    if !desc.is_write_only() {
-                        break;
-                    }
-                    let limit = cmp::min(write_count + desc.len as usize, self.bytes_read);
-                    let source_slice = &self.frame_buf[write_count..limit];
-                    let write_result = mem.write_slice(source_slice, desc.addr);
+        while let Some(avail_desc) = queue.iter(&mem).next() {
+            if rate_limit_reached {
+                exhausted_descs = false;
+                queue.go_to_previous_position();
+                break;
+            }

-                    match write_result {
-                        Ok(_) => {
-                            write_count = limit;
-                        }
-                        Err(e) => {
-                            error!("Failed to write slice: {:?}", e);
-                            break;
-                        }
+            let head_index = avail_desc.index;
+            let num_buffers_addr = mem.checked_offset(avail_desc.addr, 10).unwrap();
+            let mut next_desc = Some(avail_desc);
+
+            let mut iovecs = Vec::new();
+            while let Some(desc) = next_desc {
+                if desc.is_write_only() && desc.len > 0 {
+                    let buf = mem
+                        .get_slice(desc.addr, desc.len as usize)
+                        .map_err(NetQueuePairError::GuestMemory)?
+                        .as_ptr();
+                    let iovec = libc::iovec {
+                        iov_base: buf as *mut libc::c_void,
+                        iov_len: desc.len as libc::size_t,
                    };
+                    iovecs.push(iovec);
+                }
+                next_desc = desc.next_descriptor();
+            }

-                    if write_count >= self.bytes_read {
+            let len = if !iovecs.is_empty() {
+                let result = unsafe {
+                    libc::readv(
+                        tap.as_raw_fd() as libc::c_int,
+                        iovecs.as_ptr() as *const libc::iovec,
+                        iovecs.len() as libc::c_int,
+                    )
+                };
+                if result < 0 {
+                    let e = std::io::Error::last_os_error();
+                    exhausted_descs = false;
+                    queue.go_to_previous_position();
+
+                    /* EAGAIN */
+                    if e.kind() == std::io::ErrorKind::WouldBlock {
                        break;
                    }
-                    next_desc = desc.next_descriptor();
-                }
-                None => {
-                    warn!("Receiving buffer is too small to hold frame of current size");
-                    break;
+
+                    error!("net: rx: failed reading from tap: {}", e);
+                    return Err(NetQueuePairError::ReadTap(e));
                }
+
+                // Write num_buffers to guest memory. We simply write 1 as we
+                // never spread the frame over more than one descriptor chain.
+                mem.write_obj(1u16, num_buffers_addr)
+                    .map_err(NetQueuePairError::GuestMemory)?;
+
+                self.counter_bytes += Wrapping(result as u64 - vnet_hdr_len() as u64);
+                self.counter_frames += Wrapping(1);
+
+                result as u32
+            } else {
+                0
+            };
+
+            queue.add_used(&mem, head_index, len);
+            queue.update_avail_event(&mem);
+
+            // For the sake of simplicity (keeping the handling of RX_QUEUE_EVENT and
+            // RX_TAP_EVENT totally asynchronous), we always let the 'last' descriptor
+            // chain go-through even if it was over the rate limit, and simply stop
+            // processing oncoming `avail_desc` if any.
+            if let Some(rate_limiter) = rate_limiter {
+                rate_limit_reached = !rate_limiter.consume(1, TokenType::Ops)
+                    || !rate_limiter.consume(len as u64, TokenType::Bytes);
            }
        }

-        self.counter_bytes += Wrapping((write_count - vnet_hdr_len()) as u64);
-        self.counter_frames += Wrapping(1);
-
-        queue.add_used(&mem, head_index, write_count as u32);
-        queue.update_avail_event(&mem);
-
-        // Mark that we have at least one pending packet and we need to interrupt the guest.
-        self.deferred_irqs = true;
-
-        // Update the frame_buf buffer.
-        if write_count < self.bytes_read {
-            self.frame_buf.copy_within(write_count..self.bytes_read, 0);
-            self.bytes_read -= write_count;
-            false
-        } else {
-            self.bytes_read = 0;
-            true
-        }
+        Ok(exhausted_descs)
    }
 }

@@ -204,8 +240,12 @@ pub enum NetQueuePairError {
    RegisterListener(io::Error),
    /// Error unregistering listener
    UnregisterListener(io::Error),
+    /// Error writing to the TAP device
+    WriteTap(io::Error),
    /// Error reading from the TAP device
-    FailedReadTap,
+    ReadTap(io::Error),
+    /// Error related to guest memory
+    GuestMemory(vm_memory::GuestMemoryError),
 }

 pub struct NetQueuePair {
@@ -217,131 +257,21 @@ pub struct NetQueuePair {
    pub rx_tap_listening: bool,
    pub counters: NetCounters,
    pub tap_event_id: u16,
+    pub rx_desc_avail: bool,
+    pub rx_rate_limiter: Option<RateLimiter>,
+    pub tx_rate_limiter: Option<RateLimiter>,
 }

 impl NetQueuePair {
-    // Copies a single frame from `self.rx.frame_buf` into the guest. Returns true
-    // if a buffer was used, and false if the frame must be deferred until a buffer
-    // is made available by the driver.
-    fn rx_single_frame(&mut self, mut queue: &mut Queue) -> Result<bool, NetQueuePairError> {
-        let mem = self
-            .mem
-            .as_ref()
-            .ok_or(NetQueuePairError::NoMemoryConfigured)
-            .map(|m| m.memory())?;
-        let next_desc = queue.iter(&mem).next();
-
-        if next_desc.is_none() {
-            // Queue has no available descriptors
-            if self.rx_tap_listening {
-                unregister_listener(
-                    self.epoll_fd.unwrap(),
-                    self.tap.as_raw_fd(),
-                    epoll::Events::EPOLLIN,
-                    u64::from(self.tap_event_id),
-                )
-                .map_err(NetQueuePairError::UnregisterListener)?;
-                self.rx_tap_listening = false;
-                info!("Listener unregistered");
-            }
-            return Ok(false);
-        }
-
-        Ok(self.rx.process_desc_chain(&mem, next_desc, &mut queue))
-    }
-
-    fn process_rx(&mut self, queue: &mut Queue) -> Result<bool, NetQueuePairError> {
-        // Read as many frames as possible.
-        loop {
-            match self.read_tap() {
-                Ok(count) => {
-                    self.rx.bytes_read = count;
-                    if !self.rx_single_frame(queue)? {
-                        self.rx.deferred_frame = true;
-                        break;
-                    }
-                }
-                Err(e) => {
-                    // The tap device is non-blocking, so any error aside from EAGAIN is
-                    // unexpected.
-                    match e.raw_os_error() {
-                        Some(err) if err == EAGAIN => (),
-                        _ => {
-                            error!("Failed to read tap: {:?}", e);
-                            return Err(NetQueuePairError::FailedReadTap);
-                        }
-                    };
-                    break;
-                }
-            }
-        }
-
-        // Consume the counters from the Rx/Tx queues and accumulate into
-        // the counters for the device as whole. This consumption is needed
-        // to handle MQ.
-        self.counters
-            .rx_bytes
-            .fetch_add(self.rx.counter_bytes.0, Ordering::AcqRel);
-        self.counters
-            .rx_frames
-            .fetch_add(self.rx.counter_frames.0, Ordering::AcqRel);
-        self.rx.counter_bytes = Wrapping(0);
-        self.rx.counter_frames = Wrapping(0);
-
-        if self.rx.deferred_irqs {
-            self.rx.deferred_irqs = false;
-            let mem = self
-                .mem
-                .as_ref()
-                .ok_or(NetQueuePairError::NoMemoryConfigured)
-                .map(|m| m.memory())?;
-            Ok(queue.needs_notification(&mem, queue.next_used))
-        } else {
-            Ok(false)
-        }
-    }
-
-    pub fn resume_rx(&mut self, queue: &mut Queue) -> Result<bool, NetQueuePairError> {
-        if !self.rx_tap_listening {
-            register_listener(
-                self.epoll_fd.unwrap(),
-                self.tap.as_raw_fd(),
-                epoll::Events::EPOLLIN,
-                u64::from(self.tap_event_id),
-            )
-            .map_err(NetQueuePairError::RegisterListener)?;
-            self.rx_tap_listening = true;
-            info!("Listener registered");
-        }
-        if self.rx.deferred_frame {
-            if self.rx_single_frame(queue)? {
-                self.rx.deferred_frame = false;
-                // process_rx() was interrupted possibly before consuming all
-                // packets in the tap; try continuing now.
-                self.process_rx(queue)
-            } else if self.rx.deferred_irqs {
-                self.rx.deferred_irqs = false;
-                let mem = self
-                    .mem
-                    .as_ref()
-                    .ok_or(NetQueuePairError::NoMemoryConfigured)
-                    .map(|m| m.memory())?;
-                Ok(queue.needs_notification(&mem, queue.next_used))
-            } else {
-                Ok(false)
-            }
-        } else {
-            Ok(false)
-        }
-    }
-
    pub fn process_tx(&mut self, mut queue: &mut Queue) -> Result<bool, NetQueuePairError> {
        let mem = self
            .mem
            .as_ref()
            .ok_or(NetQueuePairError::NoMemoryConfigured)
            .map(|m| m.memory())?;
-        self.tx.process_desc_chain(&mem, &mut self.tap, &mut queue);
+
+        self.tx
+            .process_desc_chain(&mem, &mut self.tap, &mut queue, &mut self.tx_rate_limiter)?;

        self.counters
            .tx_bytes
@@ -355,26 +285,47 @@ impl NetQueuePair {
        Ok(queue.needs_notification(&mem, queue.next_used))
    }

-    pub fn process_rx_tap(&mut self, mut queue: &mut Queue) -> Result<bool, NetQueuePairError> {
-        if self.rx.deferred_frame
-        // Process a deferred frame first if available. Don't read from tap again
-        // until we manage to receive this deferred frame.
-        {
-            if self.rx_single_frame(&mut queue)? {
-                self.rx.deferred_frame = false;
-                self.process_rx(&mut queue)
-            } else if self.rx.deferred_irqs {
-                self.rx.deferred_irqs = false;
-                Ok(true)
-            } else {
-                Ok(false)
-            }
-        } else {
-            self.process_rx(&mut queue)
-        }
-    }
+    pub fn process_rx(&mut self, mut queue: &mut Queue) -> Result<bool, NetQueuePairError> {
+        let mem = self
+            .mem
+            .as_ref()
+            .ok_or(NetQueuePairError::NoMemoryConfigured)
+            .map(|m| m.memory())?;

-    fn read_tap(&mut self) -> io::Result<usize> {
-        self.tap.read(&mut self.rx.frame_buf)
+        self.rx_desc_avail = !self.rx.process_desc_chain(
+            &mem,
+            &mut self.tap,
+            &mut queue,
+            &mut self.rx_rate_limiter,
+        )?;
+        let rate_limit_reached = self
+            .rx_rate_limiter
+            .as_ref()
+            .map_or(false, |r| r.is_blocked());
+
+        // Stop listening on the `RX_TAP_EVENT` when:
+        // 1) there is no available describles, or
+        // 2) the RX rate limit is reached.
+        if self.rx_tap_listening && (!self.rx_desc_avail || rate_limit_reached) {
+            unregister_listener(
+                self.epoll_fd.unwrap(),
+                self.tap.as_raw_fd(),
+                epoll::Events::EPOLLIN,
+                u64::from(self.tap_event_id),
+            )
+            .map_err(NetQueuePairError::UnregisterListener)?;
+            self.rx_tap_listening = false;
+        }
+
+        self.counters
+            .rx_bytes
+            .fetch_add(self.rx.counter_bytes.0, Ordering::AcqRel);
+        self.counters
+            .rx_frames
+            .fetch_add(self.rx.counter_frames.0, Ordering::AcqRel);
+        self.rx.counter_bytes = Wrapping(0);
+        self.rx.counter_frames = Wrapping(0);
+
+        Ok(queue.needs_notification(&mem, queue.next_used))
    }
 }
--- a/net_util/src/tap.rs
+++ b/net_util/src/tap.rs
@@ -190,11 +190,8 @@ impl Tap {
            return Err(Error::ConfigureTap(IoError::last_os_error()));
        }

-        let tap = Tap { if_name, tap_file };
-        let offload_flags =
-            net_gen::TUN_F_CSUM | net_gen::TUN_F_UFO | net_gen::TUN_F_TSO4 | net_gen::TUN_F_TSO6;
+        let tap = Tap { tap_file, if_name };
        let vnet_hdr_size = vnet_hdr_len() as i32;
-        tap.set_offload(offload_flags)?;
        tap.set_vnet_hdr_size(vnet_hdr_size)?;

        Ok(tap)
--- a/pci/Cargo.toml
+++ b/pci/Cargo.toml
@@ -6,11 +6,11 @@ edition = "2018"

 [dependencies]
 anyhow = "1.0"
-byteorder = "1.3.4"
+byteorder = "1.4.3"
 hypervisor = { path = "../hypervisor" }
-vfio-ioctls = { git = "https://github.com/cloud-hypervisor/vfio-ioctls", branch = "ch" }
+vfio-ioctls = { git = "https://github.com/rust-vmm/vfio-ioctls", branch = "master" }
 vmm-sys-util = ">=0.3.1"
-libc = "0.2.86"
+libc = "0.2.94"
 log = "0.4.14"
 serde = {version = ">=1.0.27", features = ["rc"] }
 serde_derive = ">=1.0.27"
--- a/pci/src/bus.rs
+++ b/pci/src/bus.rs
@@ -122,7 +122,7 @@ impl PciBus {
    ) -> Result<()> {
        for (address, size, type_) in bars {
            match type_ {
-                PciBarRegionType::IORegion => {
+                PciBarRegionType::IoRegion => {
                    #[cfg(target_arch = "x86_64")]
                    io_bus
                        .insert(dev.clone(), address.raw_value(), size)
--- a/pci/src/configuration.rs
+++ b/pci/src/configuration.rs
@@ -4,11 +4,10 @@

 use crate::device::BarReprogrammingParams;
 use crate::{MsixConfig, PciInterruptPin};
-use anyhow::anyhow;
 use byteorder::{ByteOrder, LittleEndian};
 use std::fmt::{self, Display};
 use std::sync::{Arc, Mutex};
-use vm_migration::{MigratableError, Pausable, Snapshot, SnapshotDataSection, Snapshottable};
+use vm_migration::{MigratableError, Pausable, Snapshot, Snapshottable};

 // The number of 32bit registers in the config space, 4096 bytes.
 const NUM_CONFIGURATION_REGISTERS: usize = 1024;
@@ -100,7 +99,7 @@ pub enum PciBridgeSubclass {
    PcmciaBridge = 0x05,
    NuBusBridge = 0x06,
    CardBusBridge = 0x07,
-    RACEwayBridge = 0x08,
+    RacEwayBridge = 0x08,
    PciToPciSemiTransparentBridge = 0x09,
    InfiniBrandToPciHostBridge = 0x0a,
    OtherBridgeDevice = 0x80,
@@ -117,9 +116,9 @@ impl PciSubclass for PciBridgeSubclass {
 #[derive(Copy, Clone)]
 pub enum PciSerialBusSubClass {
    Firewire = 0x00,
-    ACCESSbus = 0x01,
-    SSA = 0x02,
-    USB = 0x03,
+    Accessbus = 0x01,
+    Ssa = 0x02,
+    Usb = 0x03,
 }

 impl PciSubclass for PciSerialBusSubClass {
@@ -132,15 +131,15 @@ impl PciSubclass for PciSerialBusSubClass {
 #[allow(dead_code)]
 #[derive(Copy, Clone)]
 pub enum PciMassStorageSubclass {
-    SCSIStorage = 0x00,
-    IDEInterface = 0x01,
+    ScsiStorage = 0x00,
+    IdeInterface = 0x01,
    FloppyController = 0x02,
-    IPIController = 0x03,
-    RAIDController = 0x04,
-    ATAController = 0x05,
-    SATAController = 0x06,
-    SerialSCSIController = 0x07,
-    NVMController = 0x08,
+    IpiController = 0x03,
+    RaidController = 0x04,
+    AtaController = 0x05,
+    SataController = 0x06,
+    SerialScsiController = 0x07,
+    NvmController = 0x08,
    MassStorage = 0x80,
 }

@@ -156,11 +155,11 @@ impl PciSubclass for PciMassStorageSubclass {
 pub enum PciNetworkControllerSubclass {
    EthernetController = 0x00,
    TokenRingController = 0x01,
-    FDDIController = 0x02,
-    ATMController = 0x03,
-    ISDNController = 0x04,
+    FddiController = 0x02,
+    AtmController = 0x03,
+    IsdnController = 0x04,
    WorldFipController = 0x05,
-    PICMGController = 0x06,
+    PicmgController = 0x06,
    InfinibandController = 0x07,
    FabricController = 0x08,
    NetworkController = 0x80,
@@ -186,55 +185,55 @@ pub trait PciProgrammingInterface {
 #[allow(dead_code)]
 #[allow(non_camel_case_types)]
 #[repr(C)]
-pub enum PciCapabilityID {
-    ListID = 0,
+pub enum PciCapabilityId {
+    ListId = 0,
    PowerManagement = 0x01,
    AcceleratedGraphicsPort = 0x02,
    VitalProductData = 0x03,
    SlotIdentification = 0x04,
    MessageSignalledInterrupts = 0x05,
-    CompactPCIHotSwap = 0x06,
-    PCIX = 0x07,
+    CompactPciHotSwap = 0x06,
+    PciX = 0x07,
    HyperTransport = 0x08,
    VendorSpecific = 0x09,
    Debugport = 0x0A,
-    CompactPCICentralResourceControl = 0x0B,
-    PCIStandardHotPlugController = 0x0C,
-    BridgeSubsystemVendorDeviceID = 0x0D,
-    AGPTargetPCIPCIbridge = 0x0E,
+    CompactPciCentralResourceControl = 0x0B,
+    PciStandardHotPlugController = 0x0C,
+    BridgeSubsystemVendorDeviceId = 0x0D,
+    AgpTargetPciPcibridge = 0x0E,
    SecureDevice = 0x0F,
-    PCIExpress = 0x10,
-    MSIX = 0x11,
-    SATADataIndexConf = 0x12,
-    PCIAdvancedFeatures = 0x13,
-    PCIEnhancedAllocation = 0x14,
+    PciExpress = 0x10,
+    MsiX = 0x11,
+    SataDataIndexConf = 0x12,
+    PciAdvancedFeatures = 0x13,
+    PciEnhancedAllocation = 0x14,
 }

-impl From<u8> for PciCapabilityID {
+impl From<u8> for PciCapabilityId {
    fn from(c: u8) -> Self {
        match c {
-            0 => PciCapabilityID::ListID,
-            0x01 => PciCapabilityID::PowerManagement,
-            0x02 => PciCapabilityID::AcceleratedGraphicsPort,
-            0x03 => PciCapabilityID::VitalProductData,
-            0x04 => PciCapabilityID::SlotIdentification,
-            0x05 => PciCapabilityID::MessageSignalledInterrupts,
-            0x06 => PciCapabilityID::CompactPCIHotSwap,
-            0x07 => PciCapabilityID::PCIX,
-            0x08 => PciCapabilityID::HyperTransport,
-            0x09 => PciCapabilityID::VendorSpecific,
-            0x0A => PciCapabilityID::Debugport,
-            0x0B => PciCapabilityID::CompactPCICentralResourceControl,
-            0x0C => PciCapabilityID::PCIStandardHotPlugController,
-            0x0D => PciCapabilityID::BridgeSubsystemVendorDeviceID,
-            0x0E => PciCapabilityID::AGPTargetPCIPCIbridge,
-            0x0F => PciCapabilityID::SecureDevice,
-            0x10 => PciCapabilityID::PCIExpress,
-            0x11 => PciCapabilityID::MSIX,
-            0x12 => PciCapabilityID::SATADataIndexConf,
-            0x13 => PciCapabilityID::PCIAdvancedFeatures,
-            0x14 => PciCapabilityID::PCIEnhancedAllocation,
-            _ => PciCapabilityID::ListID,
+            0 => PciCapabilityId::ListId,
+            0x01 => PciCapabilityId::PowerManagement,
+            0x02 => PciCapabilityId::AcceleratedGraphicsPort,
+            0x03 => PciCapabilityId::VitalProductData,
+            0x04 => PciCapabilityId::SlotIdentification,
+            0x05 => PciCapabilityId::MessageSignalledInterrupts,
+            0x06 => PciCapabilityId::CompactPciHotSwap,
+            0x07 => PciCapabilityId::PciX,
+            0x08 => PciCapabilityId::HyperTransport,
+            0x09 => PciCapabilityId::VendorSpecific,
+            0x0A => PciCapabilityId::Debugport,
+            0x0B => PciCapabilityId::CompactPciCentralResourceControl,
+            0x0C => PciCapabilityId::PciStandardHotPlugController,
+            0x0D => PciCapabilityId::BridgeSubsystemVendorDeviceId,
+            0x0E => PciCapabilityId::AgpTargetPciPcibridge,
+            0x0F => PciCapabilityId::SecureDevice,
+            0x10 => PciCapabilityId::PciExpress,
+            0x11 => PciCapabilityId::MsiX,
+            0x12 => PciCapabilityId::SataDataIndexConf,
+            0x13 => PciCapabilityId::PciAdvancedFeatures,
+            0x14 => PciCapabilityId::PciEnhancedAllocation,
+            _ => PciCapabilityId::ListId,
        }
    }
 }
@@ -242,7 +241,7 @@ impl From<u8> for PciCapabilityID {
 /// A PCI capability list. Devices can optionally specify capabilities in their configuration space.
 pub trait PciCapability {
    fn bytes(&self) -> &[u8];
-    fn id(&self) -> PciCapabilityID;
+    fn id(&self) -> PciCapabilityId;
 }

 fn encode_32_bits_bar_size(bar_size: u32) -> Option<u32> {
@@ -317,7 +316,7 @@ pub struct PciConfiguration {
 #[derive(Copy, Clone, PartialEq, Serialize, Deserialize)]
 pub enum PciBarRegionType {
    Memory32BitRegion = 0,
-    IORegion = 0x01,
+    IoRegion = 0x01,
    Memory64BitRegion = 0x04,
 }

@@ -572,7 +571,7 @@ impl PciConfiguration {
            .checked_add(config.size - 1)
            .ok_or(Error::BarAddressInvalid(config.addr, config.size))?;
        match config.region_type {
-            PciBarRegionType::Memory32BitRegion | PciBarRegionType::IORegion => {
+            PciBarRegionType::Memory32BitRegion | PciBarRegionType::IoRegion => {
                if end_addr > u64::from(u32::max_value()) {
                    return Err(Error::BarAddressInvalid(config.addr, config.size));
                }
@@ -614,7 +613,7 @@ impl PciConfiguration {
                BAR_MEM_ADDR_MASK,
                config.prefetchable as u32 | config.region_type as u32,
            ),
-            PciBarRegionType::IORegion => (BAR_IO_ADDR_MASK, config.region_type as u32),
+            PciBarRegionType::IoRegion => (BAR_IO_ADDR_MASK, config.region_type as u32),
        };

        self.registers[bar_idx] = ((config.addr as u32) & mask) | lower_bits;
@@ -711,7 +710,7 @@ impl PciConfiguration {
        }
        self.last_capability = Some((cap_offset, total_len));

-        if cap_data.id() == PciCapabilityID::MSIX {
+        if cap_data.id() == PciCapabilityId::MsiX {
            self.msix_cap_reg_idx = Some(cap_offset / 4);
        }

@@ -769,85 +768,77 @@ impl PciConfiguration {

        let mask = self.writable_bits[reg_idx];
        if (BAR0_REG..BAR0_REG + NUM_BAR_REGS).contains(&reg_idx) {
+            // Ignore the case where the BAR size is being asked for.
+            if value == 0xffff_ffff {
+                return None;
+            }
+
            let bar_idx = reg_idx - 4;
-            if (value & mask) != (self.bars[bar_idx].addr & mask) {
-                // Handle special case where the address being written is
-                // different from the address initially provided. This is a
-                // BAR reprogramming case which needs to be properly caught.
-                if let Some(bar_type) = self.bars[bar_idx].r#type {
-                    match bar_type {
-                        PciBarRegionType::Memory64BitRegion => {}
-                        _ => {
-                            // Ignore the case where the BAR size is being
-                            // asked for.
-                            if value == 0xffff_ffff {
-                                return None;
-                            }
-
-                            debug!(
-                                "DETECT BAR REPROG: current 0x{:x}, new 0x{:x}",
-                                self.registers[reg_idx], value
-                            );
-                            let old_base = u64::from(self.bars[bar_idx].addr & mask);
-                            let new_base = u64::from(value & mask);
-                            let len = u64::from(
-                                decode_32_bits_bar_size(self.bars[bar_idx].size)
-                                    .ok_or(Error::Decode32BarSize)
-                                    .unwrap(),
-                            );
-                            let region_type = bar_type;
-
-                            self.bars[bar_idx].addr = value;
-
-                            return Some(BarReprogrammingParams {
-                                old_base,
-                                new_base,
-                                len,
-                                region_type,
-                            });
-                        }
-                    }
-                } else if (reg_idx > BAR0_REG)
-                    && (self.registers[reg_idx - 1] & self.writable_bits[reg_idx - 1])
-                        != (self.bars[bar_idx - 1].addr & self.writable_bits[reg_idx - 1])
-                {
-                    // Ignore the case where the BAR size is being asked for.
-                    // Because we are in the 64bits case here, we have to check
-                    // if the lower 32bits of the current BAR have already been
-                    // asked for the BAR size too.
-                    if value == 0xffff_ffff
-                        && self.registers[reg_idx - 1] & self.writable_bits[reg_idx - 1]
-                            == self.bars[bar_idx - 1].size & self.writable_bits[reg_idx - 1]
-                    {
-                        return None;
-                    }
-
-                    debug!(
-                        "DETECT BAR REPROG: current 0x{:x}, new 0x{:x}",
-                        self.registers[reg_idx], value
-                    );
-                    let old_base = u64::from(self.bars[bar_idx].addr & mask) << 32
-                        | u64::from(self.bars[bar_idx - 1].addr & self.writable_bits[reg_idx - 1]);
-                    let new_base = u64::from(value & mask) << 32
-                        | u64::from(self.registers[reg_idx - 1] & self.writable_bits[reg_idx - 1]);
-                    let len = decode_64_bits_bar_size(
-                        self.bars[bar_idx].size,
-                        self.bars[bar_idx - 1].size,
-                    )
-                    .ok_or(Error::Decode64BarSize)
-                    .unwrap();
-                    let region_type = PciBarRegionType::Memory64BitRegion;
-
-                    self.bars[bar_idx].addr = value;
-                    self.bars[bar_idx - 1].addr = self.registers[reg_idx - 1];
-
-                    return Some(BarReprogrammingParams {
-                        old_base,
-                        new_base,
-                        len,
-                        region_type,
-                    });
+            // Handle special case where the address being written is
+            // different from the address initially provided. This is a
+            // BAR reprogramming case which needs to be properly caught.
+            if let Some(bar_type) = self.bars[bar_idx].r#type {
+                // In case of 64 bits memory BAR, we don't do anything until
+                // the upper BAR is modified, otherwise we would be moving the
+                // BAR to a wrong location in memory.
+                if bar_type == PciBarRegionType::Memory64BitRegion {
+                    return None;
                }
+
+                // Ignore the case where the value is unchanged.
+                if (value & mask) == (self.bars[bar_idx].addr & mask) {
+                    return None;
+                }
+
+                debug!(
+                    "DETECT BAR REPROG: current 0x{:x}, new 0x{:x}",
+                    self.registers[reg_idx], value
+                );
+                let old_base = u64::from(self.bars[bar_idx].addr & mask);
+                let new_base = u64::from(value & mask);
+                let len = u64::from(
+                    decode_32_bits_bar_size(self.bars[bar_idx].size)
+                        .ok_or(Error::Decode32BarSize)
+                        .unwrap(),
+                );
+                let region_type = bar_type;
+
+                self.bars[bar_idx].addr = value;
+
+                return Some(BarReprogrammingParams {
+                    old_base,
+                    new_base,
+                    len,
+                    region_type,
+                });
+            } else if (reg_idx > BAR0_REG)
+                && ((self.registers[reg_idx - 1] & self.writable_bits[reg_idx - 1])
+                    != (self.bars[bar_idx - 1].addr & self.writable_bits[reg_idx - 1])
+                    || (value & mask) != (self.bars[bar_idx].addr & mask))
+            {
+                debug!(
+                    "DETECT BAR REPROG: current 0x{:x}, new 0x{:x}",
+                    self.registers[reg_idx], value
+                );
+                let old_base = u64::from(self.bars[bar_idx].addr & mask) << 32
+                    | u64::from(self.bars[bar_idx - 1].addr & self.writable_bits[reg_idx - 1]);
+                let new_base = u64::from(value & mask) << 32
+                    | u64::from(self.registers[reg_idx - 1] & self.writable_bits[reg_idx - 1]);
+                let len =
+                    decode_64_bits_bar_size(self.bars[bar_idx].size, self.bars[bar_idx - 1].size)
+                        .ok_or(Error::Decode64BarSize)
+                        .unwrap();
+                let region_type = PciBarRegionType::Memory64BitRegion;
+
+                self.bars[bar_idx].addr = value;
+                self.bars[bar_idx - 1].addr = self.registers[reg_idx - 1];
+
+                return Some(BarReprogrammingParams {
+                    old_base,
+                    new_base,
+                    len,
+                    region_type,
+                });
            }
        } else if reg_idx == ROM_BAR_REG && (value & mask) != (self.rom_bar_addr & mask) {
            // Ignore the case where the BAR size is being asked for.
@@ -890,43 +881,12 @@ impl Snapshottable for PciConfiguration {
    }

    fn snapshot(&mut self) -> std::result::Result<Snapshot, MigratableError> {
-        let snapshot =
-            serde_json::to_vec(&self.state()).map_err(|e| MigratableError::Snapshot(e.into()))?;
-
-        let mut config_snapshot = Snapshot::new(self.id().as_str());
-        config_snapshot.add_data_section(SnapshotDataSection {
-            id: format!("{}-section", self.id()),
-            snapshot,
-        });
-
-        Ok(config_snapshot)
+        Snapshot::new_from_state(&self.id(), &self.state())
    }

    fn restore(&mut self, snapshot: Snapshot) -> std::result::Result<(), MigratableError> {
-        if let Some(config_section) = snapshot
-            .snapshot_data
-            .get(&format!("{}-section", self.id()))
-        {
-            let config_state = match serde_json::from_slice(&config_section.snapshot) {
-                Ok(state) => state,
-                Err(error) => {
-                    return Err(MigratableError::Restore(anyhow!(
-                        "Could not deserialize {}: {}",
-                        self.id(),
-                        error
-                    )))
-                }
-            };
-
-            self.set_state(&config_state);
-
-            return Ok(());
-        }
-
-        Err(MigratableError::Restore(anyhow!(
-            "Could not find {} snapshot section",
-            self.id()
-        )))
+        self.set_state(&snapshot.to_state(&self.id())?);
+        Ok(())
    }
 }

@@ -1005,8 +965,8 @@ mod tests {
            self.as_slice()
        }

-        fn id(&self) -> PciCapabilityID {
-            PciCapabilityID::VendorSpecific
+        fn id(&self) -> PciCapabilityId {
+            PciCapabilityId::VendorSpecific
        }
    }

@@ -1056,11 +1016,11 @@ mod tests {
    }

    #[derive(Copy, Clone)]
-    enum TestPI {
+    enum TestPi {
        Test = 0x5a,
    }

-    impl PciProgrammingInterface for TestPI {
+    impl PciProgrammingInterface for TestPi {
        fn get_register_value(&self) -> u8 {
            *self as u8
        }
@@ -1074,7 +1034,7 @@ mod tests {
            0x1,
            PciClassCode::MultimediaController,
            &PciMultimediaSubclass::AudioController,
-            Some(&TestPI::Test),
+            Some(&TestPi::Test),
            PciHeaderType::Device,
            0xABCD,
            0x2468,
--- a/pci/src/lib.rs
+++ b/pci/src/lib.rs
@@ -21,7 +21,7 @@ mod vfio;

 pub use self::bus::{PciBus, PciConfigIo, PciConfigMmio, PciRoot, PciRootError};
 pub use self::configuration::{
-    PciBarConfiguration, PciBarPrefetchable, PciBarRegionType, PciCapability, PciCapabilityID,
+    PciBarConfiguration, PciBarPrefetchable, PciBarRegionType, PciCapability, PciCapabilityId,
    PciClassCode, PciConfiguration, PciHeaderType, PciMassStorageSubclass,
    PciNetworkControllerSubclass, PciProgrammingInterface, PciSerialBusSubClass, PciSubclass,
 };
--- a/pci/src/msix.rs
+++ b/pci/src/msix.rs
@@ -6,7 +6,7 @@
 extern crate byteorder;
 extern crate vm_memory;

-use crate::{PciCapability, PciCapabilityID};
+use crate::{PciCapability, PciCapabilityId};
 use anyhow::anyhow;
 use byteorder::{ByteOrder, LittleEndian};
 use std::io;
@@ -16,7 +16,7 @@ use vm_device::interrupt::{
    InterruptIndex, InterruptSourceConfig, InterruptSourceGroup, MsiIrqSourceConfig,
 };
 use vm_memory::ByteValued;
-use vm_migration::{MigratableError, Pausable, Snapshot, SnapshotDataSection, Snapshottable};
+use vm_migration::{MigratableError, Pausable, Snapshot, Snapshottable};

 const MAX_MSIX_VECTORS_PER_DEVICE: u16 = 2048;
 const MSIX_TABLE_ENTRIES_MODULO: u64 = 16;
@@ -432,41 +432,18 @@ impl Snapshottable for MsixConfig {
    }

    fn snapshot(&mut self) -> std::result::Result<Snapshot, MigratableError> {
-        let snapshot =
-            serde_json::to_vec(&self.state()).map_err(|e| MigratableError::Snapshot(e.into()))?;
-
-        let mut msix_snapshot = Snapshot::new(self.id().as_str());
-        msix_snapshot.add_data_section(SnapshotDataSection {
-            id: format!("{}-section", self.id()),
-            snapshot,
-        });
-
-        Ok(msix_snapshot)
+        Snapshot::new_from_state(&self.id(), &self.state())
    }

    fn restore(&mut self, snapshot: Snapshot) -> std::result::Result<(), MigratableError> {
-        if let Some(msix_section) = snapshot
-            .snapshot_data
-            .get(&format!("{}-section", self.id()))
-        {
-            let msix_state = match serde_json::from_slice(&msix_section.snapshot) {
-                Ok(state) => state,
-                Err(error) => {
-                    return Err(MigratableError::Restore(anyhow!(
-                        "Could not deserialize MSI-X {}",
-                        error
-                    )))
-                }
-            };
-
-            return self.set_state(&msix_state).map_err(|e| {
-                MigratableError::Restore(anyhow!("Could not restore MSI-X state {:?}", e))
-            });
-        }
-
-        Err(MigratableError::Restore(anyhow!(
-            "Could not find MSI-X snapshot section"
-        )))
+        self.set_state(&snapshot.to_state(&self.id())?)
+            .map_err(|e| {
+                MigratableError::Restore(anyhow!(
+                    "Could not restore state for {}: {:?}",
+                    self.id(),
+                    e
+                ))
+            })
    }
 }

@@ -498,8 +475,8 @@ impl PciCapability for MsixCap {
        self.as_slice()
    }

-    fn id(&self) -> PciCapabilityID {
-        PciCapabilityID::MSIX
+    fn id(&self) -> PciCapabilityId {
+        PciCapabilityId::MsiX
    }
 }

--- a/pci/src/vfio.rs
+++ b/pci/src/vfio.rs
@@ -7,32 +7,30 @@ extern crate vm_allocator;

 use crate::{
    msi_num_enabled_vectors, BarReprogrammingParams, MsiConfig, MsixCap, MsixConfig,
-    PciBarConfiguration, PciBarRegionType, PciCapabilityID, PciClassCode, PciConfiguration,
+    PciBarConfiguration, PciBarRegionType, PciCapabilityId, PciClassCode, PciConfiguration,
    PciDevice, PciDeviceError, PciHeaderType, PciSubclass, MSIX_TABLE_ENTRY_SIZE,
 };
 use byteorder::{ByteOrder, LittleEndian};
 use std::any::Any;
-use std::ops::Deref;
 use std::os::unix::io::AsRawFd;
 use std::ptr::null_mut;
 use std::sync::{Arc, Barrier};
 use std::{fmt, io, result};
 use vfio_bindings::bindings::vfio::*;
-use vfio_ioctls::{VfioDevice, VfioError};
+use vfio_ioctls::{VfioContainer, VfioDevice, VfioError};
 use vm_allocator::SystemAllocator;
 use vm_device::interrupt::{
    InterruptIndex, InterruptManager, InterruptSourceGroup, MsiIrqGroupConfig,
 };
 use vm_device::BusDevice;
-use vm_memory::{
-    Address, GuestAddress, GuestAddressSpace, GuestMemoryAtomic, GuestMemoryMmap, GuestRegionMmap,
-    GuestUsize,
-};
+use vm_memory::{Address, GuestAddress, GuestUsize};
 use vmm_sys_util::eventfd::EventFd;

 #[derive(Debug)]
 pub enum VfioPciError {
    AllocateGsi,
+    DmaMap(VfioError),
+    DmaUnmap(VfioError),
    EnableIntx(VfioError),
    EnableMsi(VfioError),
    EnableMsix(VfioError),
@@ -45,7 +43,6 @@ pub enum VfioPciError {
    MsixNotConfigured,
    NewVfioPciDevice,
    SetGsiRouting(hypervisor::HypervisorVmError),
-    UpdateMemory(VfioError),
 }
 pub type Result<T> = std::result::Result<T, VfioPciError>;

@@ -53,6 +50,8 @@ impl fmt::Display for VfioPciError {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        match self {
            VfioPciError::AllocateGsi => write!(f, "failed to allocate GSI"),
+            VfioPciError::DmaMap(e) => write!(f, "failed to DMA map: {}", e),
+            VfioPciError::DmaUnmap(e) => write!(f, "failed to DMA unmap: {}", e),
            VfioPciError::EnableIntx(e) => write!(f, "failed to enable INTx: {}", e),
            VfioPciError::EnableMsi(e) => write!(f, "failed to enable MSI: {}", e),
            VfioPciError::EnableMsix(e) => write!(f, "failed to enable MSI-X: {}", e),
@@ -69,7 +68,6 @@ impl fmt::Display for VfioPciError {
            VfioPciError::MsixNotConfigured => write!(f, "MSI-X interrupt not yet configured"),
            VfioPciError::NewVfioPciDevice => write!(f, "failed to create VFIO PCI device"),
            VfioPciError::SetGsiRouting(e) => write!(f, "failed to set GSI routes: {}", e),
-            VfioPciError::UpdateMemory(e) => write!(f, "failed to update memory: {}", e),
        }
    }
 }
@@ -186,13 +184,13 @@ impl Interrupt {
        None
    }

-    fn accessed(&self, offset: u64) -> Option<(PciCapabilityID, u64)> {
+    fn accessed(&self, offset: u64) -> Option<(PciCapabilityId, u64)> {
        if let Some(msi) = &self.msi {
            if offset >= u64::from(msi.cap_offset)
                && offset < u64::from(msi.cap_offset) + msi.cfg.size()
            {
                return Some((
-                    PciCapabilityID::MessageSignalledInterrupts,
+                    PciCapabilityId::MessageSignalledInterrupts,
                    u64::from(msi.cap_offset),
                ));
            }
@@ -200,7 +198,7 @@ impl Interrupt {

        if let Some(msix) = &self.msix {
            if offset == u64::from(msix.cap_offset) {
-                return Some((PciCapabilityID::MSIX, u64::from(msix.cap_offset)));
+                return Some((PciCapabilityId::MsiX, u64::from(msix.cap_offset)));
            }
        }

@@ -298,11 +296,12 @@ impl VfioPciConfig {
 pub struct VfioPciDevice {
    vm: Arc<dyn hypervisor::Vm>,
    device: Arc<VfioDevice>,
+    container: Arc<VfioContainer>,
    vfio_pci_configuration: VfioPciConfig,
    configuration: PciConfiguration,
    mmio_regions: Vec<MmioRegion>,
    interrupt: Interrupt,
-    mem: GuestMemoryAtomic<GuestMemoryMmap>,
+    iommu_attached: bool,
 }

 impl VfioPciDevice {
@@ -310,9 +309,10 @@ impl VfioPciDevice {
    pub fn new(
        vm: &Arc<dyn hypervisor::Vm>,
        device: VfioDevice,
+        container: Arc<VfioContainer>,
        msi_interrupt_manager: &Arc<dyn InterruptManager<GroupConfig = MsiIrqGroupConfig>>,
        legacy_interrupt_group: Option<Arc<Box<dyn InterruptSourceGroup>>>,
-        mem: GuestMemoryAtomic<GuestMemoryMmap>,
+        iommu_attached: bool,
    ) -> Result<Self> {
        let device = Arc::new(device);
        device.reset();
@@ -335,6 +335,7 @@ impl VfioPciDevice {
        let mut vfio_pci_device = VfioPciDevice {
            vm: vm.clone(),
            device,
+            container,
            configuration,
            vfio_pci_configuration,
            mmio_regions: Vec::new(),
@@ -343,7 +344,7 @@ impl VfioPciDevice {
                msi: None,
                msix: None,
            },
-            mem,
+            iommu_attached,
        };

        vfio_pci_device.parse_capabilities(msi_interrupt_manager);
@@ -353,6 +354,10 @@ impl VfioPciDevice {
        Ok(vfio_pci_device)
    }

+    pub fn iommu_attached(&self) -> bool {
+        self.iommu_attached
+    }
+
    fn enable_intx(&mut self) -> Result<()> {
        if let Some(intx) = &mut self.interrupt.intx {
            if !intx.enabled {
@@ -421,7 +426,7 @@ impl VfioPciDevice {

            self.device
                .enable_msix(irq_fds.iter().collect())
-                .map_err(VfioPciError::EnableMsi)?;
+                .map_err(VfioPciError::EnableMsix)?;
        }

        Ok(())
@@ -535,8 +540,8 @@ impl VfioPciDevice {
                .vfio_pci_configuration
                .read_config_byte(cap_next.into());

-            match PciCapabilityID::from(cap_id) {
-                PciCapabilityID::MessageSignalledInterrupts => {
+            match PciCapabilityId::from(cap_id) {
+                PciCapabilityId::MessageSignalledInterrupts => {
                    if let Some(irq_info) = self.device.get_irq_info(VFIO_PCI_MSI_IRQ_INDEX) {
                        if irq_info.count > 0 {
                            // Parse capability only if the VFIO device
@@ -545,7 +550,7 @@ impl VfioPciDevice {
                        }
                    }
                }
-                PciCapabilityID::MSIX => {
+                PciCapabilityId::MsiX => {
                    if let Some(irq_info) = self.device.get_irq_info(VFIO_PCI_MSIX_IRQ_INDEX) {
                        if irq_info.count > 0 {
                            // Parse capability only if the VFIO device
@@ -722,10 +727,24 @@ impl VfioPciDevice {
        }
    }

-    pub fn update_memory(&self, new_region: &Arc<GuestRegionMmap>) -> Result<()> {
-        self.device
-            .extend_dma_map(new_region)
-            .map_err(VfioPciError::UpdateMemory)
+    pub fn dma_map(&self, iova: u64, size: u64, user_addr: u64) -> Result<()> {
+        if !self.iommu_attached {
+            self.container
+                .vfio_dma_map(iova, size, user_addr)
+                .map_err(VfioPciError::DmaMap)?;
+        }
+
+        Ok(())
+    }
+
+    pub fn dma_unmap(&self, iova: u64, size: u64) -> Result<()> {
+        if !self.iommu_attached {
+            self.container
+                .vfio_dma_unmap(iova, size)
+                .map_err(VfioPciError::DmaUnmap)?;
+        }
+
+        Ok(())
    }

    pub fn mmio_regions(&self) -> Vec<MmioRegion> {
@@ -752,14 +771,6 @@ impl Drop for VfioPciDevice {
        if self.interrupt.intx_in_use() {
            self.disable_intx();
        }
-
-        if self
-            .device
-            .unset_dma_map(self.mem.memory().deref())
-            .is_err()
-        {
-            error!("failed to remove all guest memory regions from iommu table");
-        }
    }
 }

@@ -856,7 +867,7 @@ impl PciDevice for VfioPciDevice {
                #[cfg(target_arch = "x86_64")]
                {
                    // IO BAR
-                    region_type = PciBarRegionType::IORegion;
+                    region_type = PciBarRegionType::IoRegion;

                    // Clear first bit.
                    lsb_size &= 0xffff_fffc;
@@ -899,26 +910,14 @@ impl PciDevice for VfioPciDevice {
                let first_bit = region_size.trailing_zeros();
                region_size = 2u64.pow(first_bit);

-                // We need to allocate a guest MMIO address range for that BAR.
-                // In case the BAR is mappable directly, this means it might be
-                // set as user memory region, which expects to deal with 4K
-                // pages. Therefore, the alignment has to be set accordingly.
-                let bar_alignment = if (bar_id == VFIO_PCI_ROM_REGION_INDEX)
-                    || (self.device.get_region_flags(bar_id) & VFIO_REGION_INFO_FLAG_MMAP != 0)
-                {
-                    // 4K alignment
-                    0x1000
-                } else {
-                    // Default 16 bytes alignment
-                    0x10
-                };
+                // BAR allocation needs to be naturally aligned
                if is_64bit_bar {
                    bar_addr = allocator
-                        .allocate_mmio_addresses(None, region_size, Some(bar_alignment))
+                        .allocate_mmio_addresses(None, region_size, Some(region_size))
                        .ok_or(PciDeviceError::IoAllocationFailed(region_size))?;
                } else {
                    bar_addr = allocator
-                        .allocate_mmio_hole_addresses(None, region_size, Some(bar_alignment))
+                        .allocate_mmio_hole_addresses(None, region_size, Some(region_size))
                        .ok_or(PciDeviceError::IoAllocationFailed(region_size))?;
                }
            }
@@ -963,14 +962,6 @@ impl PciDevice for VfioPciDevice {
            }
        }

-        if self
-            .device
-            .setup_dma_map(self.mem.memory().deref())
-            .is_err()
-        {
-            error!("failed to add all guest memory regions into iommu table");
-        }
-
        Ok(ranges)
    }

@@ -980,7 +971,7 @@ impl PciDevice for VfioPciDevice {
    ) -> std::result::Result<(), PciDeviceError> {
        for region in self.mmio_regions.iter() {
            match region.type_ {
-                PciBarRegionType::IORegion => {
+                PciBarRegionType::IoRegion => {
                    #[cfg(target_arch = "x86_64")]
                    allocator.free_io_addresses(region.start, region.length);
                    #[cfg(target_arch = "aarch64")]
@@ -1026,12 +1017,12 @@ impl PciDevice for VfioPciDevice {
        if let Some((cap_id, cap_base)) = self.interrupt.accessed(reg) {
            let cap_offset: u64 = reg - cap_base + offset;
            match cap_id {
-                PciCapabilityID::MessageSignalledInterrupts => {
+                PciCapabilityId::MessageSignalledInterrupts => {
                    if let Err(e) = self.update_msi_capabilities(cap_offset, data) {
                        error!("Could not update MSI capabilities: {}", e);
                    }
                }
-                PciCapabilityID::MSIX => {
+                PciCapabilityId::MsiX => {
                    if let Err(e) = self.update_msix_capabilities(cap_offset, data) {
                        error!("Could not update MSI-X capabilities: {}", e);
                    }
--- a/qcow/Cargo.toml
+++ b/qcow/Cargo.toml
@@ -9,11 +9,8 @@ license = "BSD-3-Clause"
 path = "src/qcow.rs"

 [dependencies]
-byteorder = "1.3.4"
-libc = "0.2.86"
+byteorder = "1.4.3"
+libc = "0.2.94"
 log = "0.4.14"
 remain = "0.2.2"
 vmm-sys-util = ">=0.3.1"
-
-[dev-dependencies]
-tempfile = "3.2.0"
--- a/qcow/src/qcow.rs
+++ b/qcow/src/qcow.rs
@@ -22,7 +22,7 @@ use std::io::{self, Read, Seek, SeekFrom, Write};
 use std::mem::size_of;
 use vmm_sys_util::{
    file_traits::FileSetLen, file_traits::FileSync, seek_hole::SeekHole, write_zeroes::PunchHole,
-    write_zeroes::WriteZeroes,
+    write_zeroes::WriteZeroesAt,
 };

 pub use crate::raw_file::RawFile;
@@ -1274,8 +1274,7 @@ impl QcowFile {
                // unallocated clusters already read back as zeroes.
                if let Some(offset) = self.file_offset_read(curr_addr)? {
                    // Partial cluster - zero it out.
-                    self.raw_file.file_mut().seek(SeekFrom::Start(offset))?;
-                    self.raw_file.file_mut().write_zeroes(count)?;
+                    self.raw_file.file_mut().write_zeroes_at(offset, count)?;
                }
            }

@@ -1528,6 +1527,13 @@ impl PunchHole for QcowFile {
    }
 }

+impl WriteZeroesAt for QcowFile {
+    fn write_zeroes_at(&mut self, offset: u64, length: usize) -> io::Result<usize> {
+        self.punch_hole(offset, length as u64)?;
+        Ok(length)
+    }
+}
+
 impl SeekHole for QcowFile {
    fn seek_hole(&mut self, offset: u64) -> io::Result<Option<u64>> {
        match self.find_allocated_cluster(offset, false) {
@@ -1704,7 +1710,8 @@ pub fn detect_image_type(file: &mut RawFile) -> Result<ImageType> {
 mod tests {
    use super::*;
    use std::io::{Read, Seek, SeekFrom, Write};
-    use tempfile::tempfile;
+    use vmm_sys_util::tempfile::TempFile;
+    use vmm_sys_util::write_zeroes::WriteZeroes;

    fn valid_header_v3() -> Vec<u8> {
        vec![
@@ -1775,7 +1782,7 @@ mod tests {
    where
        F: FnMut(RawFile),
    {
-        let mut disk_file: RawFile = RawFile::new(tempfile().unwrap(), false);
+        let mut disk_file: RawFile = RawFile::new(TempFile::new().unwrap().into_file(), false);
        disk_file.write_all(&header).unwrap();
        disk_file.set_len(0x1_0000_0000).unwrap();
        disk_file.seek(SeekFrom::Start(0)).unwrap();
@@ -1787,7 +1794,7 @@ mod tests {
    where
        F: FnMut(QcowFile),
    {
-        let tmp: RawFile = RawFile::new(tempfile().unwrap(), direct);
+        let tmp: RawFile = RawFile::new(TempFile::new().unwrap().into_file(), direct);
        let qcow_file = QcowFile::new(tmp, 3, file_size).unwrap();

        testfn(qcow_file); // File closed when the function exits.
@@ -1796,7 +1803,7 @@ mod tests {
    #[test]
    fn default_header_v2() {
        let header = QcowHeader::create_for_size(2, 0x10_0000);
-        let mut disk_file: RawFile = RawFile::new(tempfile().unwrap(), false);
+        let mut disk_file: RawFile = RawFile::new(TempFile::new().unwrap().into_file(), false);
        header
            .write_to(&mut disk_file)
            .expect("Failed to write header to temporary file.");
@@ -1807,7 +1814,7 @@ mod tests {
    #[test]
    fn default_header_v3() {
        let header = QcowHeader::create_for_size(3, 0x10_0000);
-        let mut disk_file: RawFile = RawFile::new(tempfile().unwrap(), false);
+        let mut disk_file: RawFile = RawFile::new(TempFile::new().unwrap().into_file(), false);
        header
            .write_to(&mut disk_file)
            .expect("Failed to write header to temporary file.");
@@ -2040,7 +2047,7 @@ mod tests {
            struct Transfer {
                pub write: bool,
                pub addr: u64,
-            };
+            }

            // Write transactions from mkfs.ext4.
            let xfers: Vec<Transfer> = vec![
--- a/qcow/src/raw_file.rs
+++ b/qcow/src/raw_file.rs
@@ -15,7 +15,7 @@ use std::fs::{File, Metadata};
 use std::io::{self, Read, Seek, SeekFrom, Write};
 use std::os::unix::io::{AsRawFd, RawFd};
 use std::slice;
-use vmm_sys_util::{seek_hole::SeekHole, write_zeroes::PunchHole};
+use vmm_sys_util::{seek_hole::SeekHole, write_zeroes::PunchHole, write_zeroes::WriteZeroesAt};

 #[derive(Debug)]
 pub struct RawFile {
@@ -281,6 +281,12 @@ impl Seek for RawFile {
    }
 }

+impl WriteZeroesAt for RawFile {
+    fn write_zeroes_at(&mut self, offset: u64, length: usize) -> std::io::Result<usize> {
+        self.file.write_zeroes_at(offset, length)
+    }
+}
+
 impl PunchHole for RawFile {
    fn punch_hole(&mut self, offset: u64, length: u64) -> std::io::Result<()> {
        self.file.punch_hole(offset, length)
--- a/rate_limiter/Cargo.toml
+++ b/rate_limiter/Cargo.toml
@@ -0,0 +1,9 @@
+[package]
+name = "rate_limiter"
+version = "0.1.0"
+edition = "2018"
+
+[dependencies]
+libc = "0.2.94"
+log = "0.4.14"
+vmm-sys-util = "0.8.0"
--- a/rate_limiter/src/lib.rs
+++ b/rate_limiter/src/lib.rs
@@ -0,0 +1,899 @@
+// Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+#![deny(missing_docs)]
+//! # Rate Limiter
+//!
+//! Provides a rate limiter written in Rust useful for IO operations that need to
+//! be throttled.
+//!
+//! ## Behavior
+//!
+//! The rate limiter starts off as 'unblocked' with two token buckets configured
+//! with the values passed in the `RateLimiter::new()` constructor.
+//! All subsequent accounting is done independently for each token bucket based
+//! on the `TokenType` used. If any of the buckets runs out of budget, the limiter
+//! goes in the 'blocked' state. At this point an internal timer is set up which
+//! will later 'wake up' the user in order to retry sending data. The 'wake up'
+//! notification will be dispatched as an event on the FD provided by the `AsRawFD`
+//! trait implementation.
+//!
+//! The contract is that the user shall also call the `event_handler()` method on
+//! receipt of such an event.
+//!
+//! The token buckets are replenished every time a `consume()` is called, before
+//! actually trying to consume the requested amount of tokens. The amount of tokens
+//! replenished is automatically calculated to respect the `complete_refill_time`
+//! configuration parameter provided by the user. The token buckets will never
+//! replenish above their respective `size`.
+//!
+//! Each token bucket can start off with a `one_time_burst` initial extra capacity
+//! on top of their `size`. This initial extra credit does not replenish and
+//! can be used for an initial burst of data.
+//!
+//! The granularity for 'wake up' events when the rate limiter is blocked is
+//! currently hardcoded to `100 milliseconds`.
+//!
+//! ## Limitations
+//!
+//! This rate limiter implementation relies on the *Linux kernel's timerfd* so its
+//! usage is limited to Linux systems.
+//!
+//! Another particularity of this implementation is that it is not self-driving.
+//! It is meant to be used in an external event loop and thus implements the `AsRawFd`
+//! trait and provides an *event-handler* as part of its API. This *event-handler*
+//! needs to be called by the user on every event on the rate limiter's `AsRawFd` FD.
+#[macro_use]
+extern crate log;
+
+use std::os::unix::io::{AsRawFd, RawFd};
+use std::time::{Duration, Instant};
+use std::{fmt, io};
+use vmm_sys_util::timerfd::TimerFd;
+
+#[derive(Debug)]
+/// Describes the errors that may occur while handling rate limiter events.
+pub enum Error {
+    /// The event handler was called spuriously.
+    SpuriousRateLimiterEvent(&'static str),
+    /// The event handler encounters while TimerFd::wait()
+    TimerFdWaitError(std::io::Error),
+}
+
+// Interval at which the refill timer will run when limiter is at capacity.
+const REFILL_TIMER_INTERVAL_MS: u64 = 100;
+const TIMER_REFILL_DUR: Duration = Duration::from_millis(REFILL_TIMER_INTERVAL_MS);
+
+const NANOSEC_IN_ONE_MILLISEC: u64 = 1_000_000;
+
+// Euclid's two-thousand-year-old algorithm for finding the greatest common divisor.
+fn gcd(x: u64, y: u64) -> u64 {
+    let mut x = x;
+    let mut y = y;
+    while y != 0 {
+        let t = y;
+        y = x % y;
+        x = t;
+    }
+    x
+}
+
+/// Enum describing the outcomes of a `reduce()` call on a `TokenBucket`.
+#[derive(Clone, Debug, PartialEq)]
+pub enum BucketReduction {
+    /// There are not enough tokens to complete the operation.
+    Failure,
+    /// A part of the available tokens have been consumed.
+    Success,
+    /// A number of tokens `inner` times larger than the bucket size have been consumed.
+    OverConsumption(f64),
+}
+
+/// TokenBucket provides a lower level interface to rate limiting with a
+/// configurable capacity, refill-rate and initial burst.
+#[derive(Clone, Debug, PartialEq)]
+pub struct TokenBucket {
+    // Bucket defining traits.
+    size: u64,
+    // Initial burst size (number of free initial tokens, that can be consumed at no cost)
+    one_time_burst: u64,
+    // Complete refill time in milliseconds.
+    refill_time: u64,
+
+    // Internal state descriptors.
+    budget: u64,
+    last_update: Instant,
+
+    // Fields used for pre-processing optimizations.
+    processed_capacity: u64,
+    processed_refill_time: u64,
+}
+
+impl TokenBucket {
+    /// Creates a `TokenBucket` wrapped in an `Option`.
+    ///
+    /// TokenBucket created is of `size` total capacity and takes `complete_refill_time_ms`
+    /// milliseconds to go from zero tokens to total capacity. The `one_time_burst` is initial
+    /// extra credit on top of total capacity, that does not replenish and which can be used
+    /// for an initial burst of data.
+    ///
+    /// If the `size` or the `complete refill time` are zero, then `None` is returned.
+    pub fn new(size: u64, one_time_burst: u64, complete_refill_time_ms: u64) -> Option<Self> {
+        // If either token bucket capacity or refill time is 0, disable limiting.
+        if size == 0 || complete_refill_time_ms == 0 {
+            return None;
+        }
+        // Formula for computing current refill amount:
+        // refill_token_count = (delta_time * size) / (complete_refill_time_ms * 1_000_000)
+        // In order to avoid overflows, simplify the fractions by computing greatest common divisor.
+
+        let complete_refill_time_ns = complete_refill_time_ms * NANOSEC_IN_ONE_MILLISEC;
+        // Get the greatest common factor between `size` and `complete_refill_time_ns`.
+        let common_factor = gcd(size, complete_refill_time_ns);
+        // The division will be exact since `common_factor` is a factor of `size`.
+        let processed_capacity: u64 = size / common_factor;
+        // The division will be exact since `common_factor` is a factor of `complete_refill_time_ns`.
+        let processed_refill_time: u64 = complete_refill_time_ns / common_factor;
+
+        Some(TokenBucket {
+            size,
+            one_time_burst,
+            refill_time: complete_refill_time_ms,
+            // Start off full.
+            budget: size,
+            // Last updated is now.
+            last_update: Instant::now(),
+            processed_capacity,
+            processed_refill_time,
+        })
+    }
+
+    /// Attempts to consume `tokens` from the bucket and returns whether the action succeeded.
+    // TODO (Issue #259): handle cases where a single request is larger than the full capacity
+    // for such cases we need to support partial fulfilment of requests
+    pub fn reduce(&mut self, mut tokens: u64) -> BucketReduction {
+        // First things first: consume the one-time-burst budget.
+        if self.one_time_burst > 0 {
+            // We still have burst budget for *all* tokens requests.
+            if self.one_time_burst >= tokens {
+                self.one_time_burst -= tokens;
+                self.last_update = Instant::now();
+                // No need to continue to the refill process, we still have burst budget to consume from.
+                return BucketReduction::Success;
+            } else {
+                // We still have burst budget for *some* of the tokens requests.
+                // The tokens left unfulfilled will be consumed from current `self.budget`.
+                tokens -= self.one_time_burst;
+                self.one_time_burst = 0;
+            }
+        }
+
+        // Compute time passed since last refill/update.
+        let time_delta = self.last_update.elapsed().as_nanos() as u64;
+        self.last_update = Instant::now();
+
+        // At each 'time_delta' nanoseconds the bucket should refill with:
+        // refill_amount = (time_delta * size) / (complete_refill_time_ms * 1_000_000)
+        // `processed_capacity` and `processed_refill_time` are the result of simplifying above
+        // fraction formula with their greatest-common-factor.
+        self.budget += (time_delta * self.processed_capacity) / self.processed_refill_time;
+
+        if self.budget >= self.size {
+            self.budget = self.size;
+        }
+
+        if tokens > self.budget {
+            // This operation requests a bandwidth higher than the bucket size
+            if tokens > self.size {
+                error!(
+                    "Consumed {} tokens from bucket of size {}",
+                    tokens, self.size
+                );
+                // Empty the bucket and report an overconsumption of
+                // (remaining tokens / size) times larger than the bucket size
+                tokens -= self.budget;
+                self.budget = 0;
+                return BucketReduction::OverConsumption(tokens as f64 / self.size as f64);
+            }
+            // If not enough tokens consume() fails, return false.
+            return BucketReduction::Failure;
+        }
+
+        self.budget -= tokens;
+        BucketReduction::Success
+    }
+
+    /// "Manually" adds tokens to bucket.
+    pub fn replenish(&mut self, tokens: u64) {
+        // This means we are still during the burst interval.
+        // Of course there is a very small chance  that the last reduce() also used up burst
+        // budget which should now be replenished, but for performance and code-complexity
+        // reasons we're just gonna let that slide since it's practically inconsequential.
+        if self.one_time_burst > 0 {
+            self.one_time_burst += tokens;
+            return;
+        }
+        self.budget = std::cmp::min(self.budget + tokens, self.size);
+    }
+
+    /// Returns the capacity of the token bucket.
+    pub fn capacity(&self) -> u64 {
+        self.size
+    }
+
+    /// Returns the remaining one time burst budget.
+    pub fn one_time_burst(&self) -> u64 {
+        self.one_time_burst
+    }
+
+    /// Returns the time in milliseconds required to to completely fill the bucket.
+    pub fn refill_time_ms(&self) -> u64 {
+        self.refill_time
+    }
+
+    /// Returns the current budget (one time burst allowance notwithstanding).
+    pub fn budget(&self) -> u64 {
+        self.budget
+    }
+}
+
+/// Enum that describes the type of token used.
+pub enum TokenType {
+    /// Token type used for bandwidth limiting.
+    Bytes,
+    /// Token type used for operations/second limiting.
+    Ops,
+}
+
+/// Enum that describes the type of token bucket update.
+pub enum BucketUpdate {
+    /// No Update - same as before.
+    None,
+    /// Rate Limiting is disabled on this bucket.
+    Disabled,
+    /// Rate Limiting enabled with updated bucket.
+    Update(TokenBucket),
+}
+
+/// Rate Limiter that works on both bandwidth and ops/s limiting.
+///
+/// Bandwidth (bytes/s) and ops/s limiting can be used at the same time or individually.
+///
+/// Implementation uses a single timer through TimerFd to refresh either or
+/// both token buckets.
+///
+/// Its internal buckets are 'passively' replenished as they're being used (as
+/// part of `consume()` operations).
+/// A timer is enabled and used to 'actively' replenish the token buckets when
+/// limiting is in effect and `consume()` operations are disabled.
+///
+/// RateLimiters will generate events on the FDs provided by their `AsRawFd` trait
+/// implementation. These events are meant to be consumed by the user of this struct.
+/// On each such event, the user must call the `event_handler()` method.
+pub struct RateLimiter {
+    bandwidth: Option<TokenBucket>,
+    ops: Option<TokenBucket>,
+
+    timer_fd: TimerFd,
+    // Internal flag that quickly determines timer state.
+    timer_active: bool,
+}
+
+impl PartialEq for RateLimiter {
+    fn eq(&self, other: &RateLimiter) -> bool {
+        self.bandwidth == other.bandwidth && self.ops == other.ops
+    }
+}
+
+impl fmt::Debug for RateLimiter {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(
+            f,
+            "RateLimiter {{ bandwidth: {:?}, ops: {:?} }}",
+            self.bandwidth, self.ops
+        )
+    }
+}
+
+impl RateLimiter {
+    /// Creates a new Rate Limiter that can limit on both bytes/s and ops/s.
+    ///
+    /// # Arguments
+    ///
+    /// * `bytes_total_capacity` - the total capacity of the `TokenType::Bytes` token bucket.
+    /// * `bytes_one_time_burst` - initial extra credit on top of `bytes_total_capacity`,
+    /// that does not replenish and which can be used for an initial burst of data.
+    /// * `bytes_complete_refill_time_ms` - number of milliseconds for the `TokenType::Bytes`
+    /// token bucket to go from zero Bytes to `bytes_total_capacity` Bytes.
+    /// * `ops_total_capacity` - the total capacity of the `TokenType::Ops` token bucket.
+    /// * `ops_one_time_burst` - initial extra credit on top of `ops_total_capacity`,
+    /// that does not replenish and which can be used for an initial burst of data.
+    /// * `ops_complete_refill_time_ms` - number of milliseconds for the `TokenType::Ops` token
+    /// bucket to go from zero Ops to `ops_total_capacity` Ops.
+    ///
+    /// If either bytes/ops *size* or *refill_time* are **zero**, the limiter
+    /// is **disabled** for that respective token type.
+    ///
+    /// # Errors
+    ///
+    /// If the timerfd creation fails, an error is returned.
+    pub fn new(
+        bytes_total_capacity: u64,
+        bytes_one_time_burst: u64,
+        bytes_complete_refill_time_ms: u64,
+        ops_total_capacity: u64,
+        ops_one_time_burst: u64,
+        ops_complete_refill_time_ms: u64,
+    ) -> io::Result<Self> {
+        let bytes_token_bucket = TokenBucket::new(
+            bytes_total_capacity,
+            bytes_one_time_burst,
+            bytes_complete_refill_time_ms,
+        );
+
+        let ops_token_bucket = TokenBucket::new(
+            ops_total_capacity,
+            ops_one_time_burst,
+            ops_complete_refill_time_ms,
+        );
+
+        // We'll need a timer_fd, even if our current config effectively disables rate limiting,
+        // because `Self::update_buckets()` might re-enable it later, and we might be
+        // seccomp-blocked from creating the timer_fd at that time.
+        let timer_fd = TimerFd::new()?;
+        // Note: vmm_sys_util::TimerFd::new() open the fd w/o O_NONBLOCK. We manually add this flag
+        // so that `Self::event_handler` won't be blocked with `vmm_sys_util::TimerFd::wait()`.
+        let ret = unsafe {
+            let fd = timer_fd.as_raw_fd();
+            let mut flags = libc::fcntl(fd, libc::F_GETFL);
+            flags |= libc::O_NONBLOCK;
+            libc::fcntl(fd, libc::F_SETFL, flags)
+        };
+        if ret < 0 {
+            return Err(std::io::Error::last_os_error());
+        }
+
+        Ok(RateLimiter {
+            bandwidth: bytes_token_bucket,
+            ops: ops_token_bucket,
+            timer_fd,
+            timer_active: false,
+        })
+    }
+
+    // Arm the timer of the rate limiter with the provided `Duration` (which will fire only once).
+    fn activate_timer(&mut self, dur: Duration) {
+        // Panic when failing to arm the timer (same handling in crate TimerFd::set_state())
+        self.timer_fd
+            .reset(dur, None)
+            .expect("Can't arm the timer (unexpected 'timerfd_settime' failure).");
+        self.timer_active = true;
+    }
+
+    /// Attempts to consume tokens and returns whether that is possible.
+    ///
+    /// If rate limiting is disabled on provided `token_type`, this function will always succeed.
+    pub fn consume(&mut self, tokens: u64, token_type: TokenType) -> bool {
+        // If the timer is active, we can't consume tokens from any bucket and the function fails.
+        if self.timer_active {
+            return false;
+        }
+
+        // Identify the required token bucket.
+        let token_bucket = match token_type {
+            TokenType::Bytes => self.bandwidth.as_mut(),
+            TokenType::Ops => self.ops.as_mut(),
+        };
+        // Try to consume from the token bucket.
+        if let Some(bucket) = token_bucket {
+            let refill_time = bucket.refill_time_ms();
+            match bucket.reduce(tokens) {
+                // When we report budget is over, there will be no further calls here,
+                // register a timer to replenish the bucket and resume processing;
+                // make sure there is only one running timer for this limiter.
+                BucketReduction::Failure => {
+                    if !self.timer_active {
+                        self.activate_timer(TIMER_REFILL_DUR);
+                    }
+                    false
+                }
+                // The operation succeeded and further calls can be made.
+                BucketReduction::Success => true,
+                // The operation succeeded as the tokens have been consumed
+                // but the timer still needs to be armed.
+                BucketReduction::OverConsumption(ratio) => {
+                    // The operation "borrowed" a number of tokens `ratio` times
+                    // greater than the size of the bucket, and since it takes
+                    // `refill_time` milliseconds to fill an empty bucket, in
+                    // order to enforce the bandwidth limit we need to prevent
+                    // further calls to the rate limiter for
+                    // `ratio * refill_time` milliseconds.
+                    self.activate_timer(Duration::from_millis((ratio * refill_time as f64) as u64));
+                    true
+                }
+            }
+        } else {
+            // If bucket is not present rate limiting is disabled on token type,
+            // consume() will always succeed.
+            true
+        }
+    }
+
+    /// Adds tokens of `token_type` to their respective bucket.
+    ///
+    /// Can be used to *manually* add tokens to a bucket. Useful for reverting a
+    /// `consume()` if needed.
+    pub fn manual_replenish(&mut self, tokens: u64, token_type: TokenType) {
+        // Identify the required token bucket.
+        let token_bucket = match token_type {
+            TokenType::Bytes => self.bandwidth.as_mut(),
+            TokenType::Ops => self.ops.as_mut(),
+        };
+        // Add tokens to the token bucket.
+        if let Some(bucket) = token_bucket {
+            bucket.replenish(tokens);
+        }
+    }
+
+    /// Returns whether this rate limiter is blocked.
+    ///
+    /// The limiter 'blocks' when a `consume()` operation fails because there was not enough
+    /// budget for it.
+    /// An event will be generated on the exported FD when the limiter 'unblocks'.
+    pub fn is_blocked(&self) -> bool {
+        self.timer_active
+    }
+
+    /// This function needs to be called every time there is an event on the
+    /// FD provided by this object's `AsRawFd` trait implementation.
+    ///
+    /// # Errors
+    ///
+    /// If the rate limiter is disabled or is not blocked, an error is returned.
+    pub fn event_handler(&mut self) -> Result<(), Error> {
+        loop {
+            // Note: As we manually added the `O_NONBLOCK` flag to the FD, the following
+            // `timer_fd::wait()` won't block (which is different from its default behavior.)
+            match self.timer_fd.wait() {
+                Err(e) => {
+                    let err: std::io::Error = e.into();
+                    match err.kind() {
+                        std::io::ErrorKind::Interrupted => (),
+                        std::io::ErrorKind::WouldBlock => {
+                            return Err(Error::SpuriousRateLimiterEvent(
+                                "Rate limiter event handler called without a present timer",
+                            ))
+                        }
+                        _ => return Err(Error::TimerFdWaitError(err)),
+                    }
+                }
+                _ => {
+                    self.timer_active = false;
+                    return Ok(());
+                }
+            }
+        }
+    }
+
+    /// Updates the parameters of the token buckets associated with this RateLimiter.
+    // TODO: Please note that, right now, the buckets become full after being updated.
+    pub fn update_buckets(&mut self, bytes: BucketUpdate, ops: BucketUpdate) {
+        match bytes {
+            BucketUpdate::Disabled => self.bandwidth = None,
+            BucketUpdate::Update(tb) => self.bandwidth = Some(tb),
+            BucketUpdate::None => (),
+        };
+        match ops {
+            BucketUpdate::Disabled => self.ops = None,
+            BucketUpdate::Update(tb) => self.ops = Some(tb),
+            BucketUpdate::None => (),
+        };
+    }
+
+    /// Returns an immutable view of the inner bandwidth token bucket.
+    pub fn bandwidth(&self) -> Option<&TokenBucket> {
+        self.bandwidth.as_ref()
+    }
+
+    /// Returns an immutable view of the inner ops token bucket.
+    pub fn ops(&self) -> Option<&TokenBucket> {
+        self.ops.as_ref()
+    }
+}
+
+impl AsRawFd for RateLimiter {
+    /// Provides a FD which needs to be monitored for POLLIN events.
+    ///
+    /// This object's `event_handler()` method must be called on such events.
+    ///
+    /// Will return a negative value if rate limiting is disabled on both
+    /// token types.
+    fn as_raw_fd(&self) -> RawFd {
+        self.timer_fd.as_raw_fd()
+    }
+}
+
+impl Default for RateLimiter {
+    /// Default RateLimiter is a no-op limiter with infinite budget.
+    fn default() -> Self {
+        // Safe to unwrap since this will not attempt to create timer_fd.
+        RateLimiter::new(0, 0, 0, 0, 0, 0).expect("Failed to build default RateLimiter")
+    }
+}
+
+#[cfg(test)]
+pub(crate) mod tests {
+    use super::*;
+    use std::thread;
+    use std::time::Duration;
+
+    impl TokenBucket {
+        // Resets the token bucket: budget set to max capacity and last-updated set to now.
+        fn reset(&mut self) {
+            self.budget = self.size;
+            self.last_update = Instant::now();
+        }
+
+        fn get_last_update(&self) -> &Instant {
+            &self.last_update
+        }
+
+        fn get_processed_capacity(&self) -> u64 {
+            self.processed_capacity
+        }
+
+        fn get_processed_refill_time(&self) -> u64 {
+            self.processed_refill_time
+        }
+
+        // After a restore, we cannot be certain that the last_update field has the same value.
+        pub fn partial_eq(&self, other: &TokenBucket) -> bool {
+            (other.capacity() == self.capacity())
+                && (other.one_time_burst() == self.one_time_burst())
+                && (other.refill_time_ms() == self.refill_time_ms())
+                && (other.budget() == self.budget())
+        }
+    }
+
+    impl RateLimiter {
+        fn get_token_bucket(&self, token_type: TokenType) -> Option<&TokenBucket> {
+            match token_type {
+                TokenType::Bytes => self.bandwidth.as_ref(),
+                TokenType::Ops => self.ops.as_ref(),
+            }
+        }
+    }
+
+    #[test]
+    fn test_token_bucket_create() {
+        let before = Instant::now();
+        let tb = TokenBucket::new(1000, 0, 1000).unwrap();
+        assert_eq!(tb.capacity(), 1000);
+        assert_eq!(tb.budget(), 1000);
+        assert!(*tb.get_last_update() >= before);
+        let after = Instant::now();
+        assert!(*tb.get_last_update() <= after);
+        assert_eq!(tb.get_processed_capacity(), 1);
+        assert_eq!(tb.get_processed_refill_time(), 1_000_000);
+
+        // Verify invalid bucket configurations result in `None`.
+        assert!(TokenBucket::new(0, 1234, 1000).is_none());
+        assert!(TokenBucket::new(100, 1234, 0).is_none());
+        assert!(TokenBucket::new(0, 1234, 0).is_none());
+    }
+
+    #[test]
+    fn test_token_bucket_preprocess() {
+        let tb = TokenBucket::new(1000, 0, 1000).unwrap();
+        assert_eq!(tb.get_processed_capacity(), 1);
+        assert_eq!(tb.get_processed_refill_time(), NANOSEC_IN_ONE_MILLISEC);
+
+        let thousand = 1000;
+        let tb = TokenBucket::new(3 * 7 * 11 * 19 * thousand, 0, 7 * 11 * 13 * 17).unwrap();
+        assert_eq!(tb.get_processed_capacity(), 3 * 19);
+        assert_eq!(
+            tb.get_processed_refill_time(),
+            13 * 17 * (NANOSEC_IN_ONE_MILLISEC / thousand)
+        );
+    }
+
+    #[test]
+    fn test_token_bucket_reduce() {
+        // token bucket with capacity 1000 and refill time of 1000 milliseconds
+        // allowing rate of 1 token/ms.
+        let capacity = 1000;
+        let refill_ms = 1000;
+        let mut tb = TokenBucket::new(capacity, 0, refill_ms as u64).unwrap();
+
+        assert_eq!(tb.reduce(123), BucketReduction::Success);
+        assert_eq!(tb.budget(), capacity - 123);
+
+        thread::sleep(Duration::from_millis(123));
+        assert_eq!(tb.reduce(1), BucketReduction::Success);
+        assert_eq!(tb.budget(), capacity - 1);
+        assert_eq!(tb.reduce(100), BucketReduction::Success);
+        assert_eq!(tb.reduce(capacity), BucketReduction::Failure);
+
+        // token bucket with capacity 1000 and refill time of 1000 milliseconds
+        let mut tb = TokenBucket::new(1000, 1100, 1000).unwrap();
+        // safely assuming the thread can run these 3 commands in less than 500ms
+        assert_eq!(tb.reduce(1000), BucketReduction::Success);
+        assert_eq!(tb.one_time_burst(), 100);
+        assert_eq!(tb.reduce(500), BucketReduction::Success);
+        assert_eq!(tb.one_time_burst(), 0);
+        assert_eq!(tb.reduce(500), BucketReduction::Success);
+        assert_eq!(tb.reduce(500), BucketReduction::Failure);
+        thread::sleep(Duration::from_millis(500));
+        assert_eq!(tb.reduce(500), BucketReduction::Success);
+        thread::sleep(Duration::from_millis(1000));
+        assert_eq!(tb.reduce(2500), BucketReduction::OverConsumption(1.5));
+
+        let before = Instant::now();
+        tb.reset();
+        assert_eq!(tb.capacity(), 1000);
+        assert_eq!(tb.budget(), 1000);
+        assert!(*tb.get_last_update() >= before);
+        let after = Instant::now();
+        assert!(*tb.get_last_update() <= after);
+    }
+
+    #[test]
+    fn test_rate_limiter_default() {
+        let mut l = RateLimiter::default();
+
+        // limiter should not be blocked
+        assert!(!l.is_blocked());
+        // limiter should be disabled so consume(whatever) should work
+        assert!(l.consume(u64::max_value(), TokenType::Ops));
+        assert!(l.consume(u64::max_value(), TokenType::Bytes));
+        // calling the handler without there having been an event should error
+        assert!(l.event_handler().is_err());
+        assert_eq!(
+            format!("{:?}", l.event_handler().err().unwrap()),
+            "SpuriousRateLimiterEvent(\
+             \"Rate limiter event handler called without a present timer\")"
+        );
+    }
+
+    #[test]
+    fn test_rate_limiter_new() {
+        let l = RateLimiter::new(1000, 1001, 1002, 1003, 1004, 1005).unwrap();
+
+        let bw = l.bandwidth.unwrap();
+        assert_eq!(bw.capacity(), 1000);
+        assert_eq!(bw.one_time_burst(), 1001);
+        assert_eq!(bw.refill_time_ms(), 1002);
+        assert_eq!(bw.budget(), 1000);
+
+        let ops = l.ops.unwrap();
+        assert_eq!(ops.capacity(), 1003);
+        assert_eq!(ops.one_time_burst(), 1004);
+        assert_eq!(ops.refill_time_ms(), 1005);
+        assert_eq!(ops.budget(), 1003);
+    }
+
+    #[test]
+    fn test_rate_limiter_manual_replenish() {
+        // rate limiter with limit of 1000 bytes/s and 1000 ops/s
+        let mut l = RateLimiter::new(1000, 0, 1000, 1000, 0, 1000).unwrap();
+
+        // consume 123 bytes
+        assert!(l.consume(123, TokenType::Bytes));
+        l.manual_replenish(23, TokenType::Bytes);
+        {
+            let bytes_tb = l.get_token_bucket(TokenType::Bytes).unwrap();
+            assert_eq!(bytes_tb.budget(), 900);
+        }
+        // consume 123 ops
+        assert!(l.consume(123, TokenType::Ops));
+        l.manual_replenish(23, TokenType::Ops);
+        {
+            let bytes_tb = l.get_token_bucket(TokenType::Ops).unwrap();
+            assert_eq!(bytes_tb.budget(), 900);
+        }
+    }
+
+    #[test]
+    fn test_rate_limiter_bandwidth() {
+        // rate limiter with limit of 1000 bytes/s
+        let mut l = RateLimiter::new(1000, 0, 1000, 0, 0, 0).unwrap();
+
+        // limiter should not be blocked
+        assert!(!l.is_blocked());
+        // raw FD for this disabled should be valid
+        assert!(l.as_raw_fd() > 0);
+
+        // ops/s limiter should be disabled so consume(whatever) should work
+        assert!(l.consume(u64::max_value(), TokenType::Ops));
+
+        // do full 1000 bytes
+        assert!(l.consume(1000, TokenType::Bytes));
+        // try and fail on another 100
+        assert!(!l.consume(100, TokenType::Bytes));
+        // since consume failed, limiter should be blocked now
+        assert!(l.is_blocked());
+        // wait half the timer period
+        thread::sleep(Duration::from_millis(REFILL_TIMER_INTERVAL_MS / 2));
+        // limiter should still be blocked
+        assert!(l.is_blocked());
+        // wait the other half of the timer period
+        thread::sleep(Duration::from_millis(REFILL_TIMER_INTERVAL_MS / 2));
+        // the timer_fd should have an event on it by now
+        assert!(l.event_handler().is_ok());
+        // limiter should now be unblocked
+        assert!(!l.is_blocked());
+        // try and succeed on another 100 bytes this time
+        assert!(l.consume(100, TokenType::Bytes));
+    }
+
+    #[test]
+    fn test_rate_limiter_ops() {
+        // rate limiter with limit of 1000 ops/s
+        let mut l = RateLimiter::new(0, 0, 0, 1000, 0, 1000).unwrap();
+
+        // limiter should not be blocked
+        assert!(!l.is_blocked());
+        // raw FD for this disabled should be valid
+        assert!(l.as_raw_fd() > 0);
+
+        // bytes/s limiter should be disabled so consume(whatever) should work
+        assert!(l.consume(u64::max_value(), TokenType::Bytes));
+
+        // do full 1000 ops
+        assert!(l.consume(1000, TokenType::Ops));
+        // try and fail on another 100
+        assert!(!l.consume(100, TokenType::Ops));
+        // since consume failed, limiter should be blocked now
+        assert!(l.is_blocked());
+        // wait half the timer period
+        thread::sleep(Duration::from_millis(REFILL_TIMER_INTERVAL_MS / 2));
+        // limiter should still be blocked
+        assert!(l.is_blocked());
+        // wait the other half of the timer period
+        thread::sleep(Duration::from_millis(REFILL_TIMER_INTERVAL_MS / 2));
+        // the timer_fd should have an event on it by now
+        assert!(l.event_handler().is_ok());
+        // limiter should now be unblocked
+        assert!(!l.is_blocked());
+        // try and succeed on another 100 ops this time
+        assert!(l.consume(100, TokenType::Ops));
+    }
+
+    #[test]
+    fn test_rate_limiter_full() {
+        // rate limiter with limit of 1000 bytes/s and 1000 ops/s
+        let mut l = RateLimiter::new(1000, 0, 1000, 1000, 0, 1000).unwrap();
+
+        // limiter should not be blocked
+        assert!(!l.is_blocked());
+        // raw FD for this disabled should be valid
+        assert!(l.as_raw_fd() > 0);
+
+        // do full 1000 bytes
+        assert!(l.consume(1000, TokenType::Ops));
+        // do full 1000 bytes
+        assert!(l.consume(1000, TokenType::Bytes));
+        // try and fail on another 100 ops
+        assert!(!l.consume(100, TokenType::Ops));
+        // try and fail on another 100 bytes
+        assert!(!l.consume(100, TokenType::Bytes));
+        // since consume failed, limiter should be blocked now
+        assert!(l.is_blocked());
+        // wait half the timer period
+        thread::sleep(Duration::from_millis(REFILL_TIMER_INTERVAL_MS / 2));
+        // limiter should still be blocked
+        assert!(l.is_blocked());
+        // wait the other half of the timer period
+        thread::sleep(Duration::from_millis(REFILL_TIMER_INTERVAL_MS / 2));
+        // the timer_fd should have an event on it by now
+        assert!(l.event_handler().is_ok());
+        // limiter should now be unblocked
+        assert!(!l.is_blocked());
+        // try and succeed on another 100 ops this time
+        assert!(l.consume(100, TokenType::Ops));
+        // try and succeed on another 100 bytes this time
+        assert!(l.consume(100, TokenType::Bytes));
+    }
+
+    #[test]
+    fn test_rate_limiter_overconsumption() {
+        // initialize the rate limiter
+        let mut l = RateLimiter::new(1000, 0, 1000, 1000, 0, 1000).unwrap();
+        // try to consume 2.5x the bucket size
+        // we are "borrowing" 1.5x the bucket size in tokens since
+        // the bucket is full
+        assert!(l.consume(2500, TokenType::Bytes));
+
+        // check that even after a whole second passes, the rate limiter
+        // is still blocked
+        thread::sleep(Duration::from_millis(1000));
+        assert!(l.event_handler().is_err());
+        assert!(l.is_blocked());
+
+        // after 1.5x the replenish time has passed, the rate limiter
+        // is available again
+        thread::sleep(Duration::from_millis(500));
+        assert!(l.event_handler().is_ok());
+        assert!(!l.is_blocked());
+
+        // reset the rate limiter
+        let mut l = RateLimiter::new(1000, 0, 1000, 1000, 0, 1000).unwrap();
+        // try to consume 1.5x the bucket size
+        // we are "borrowing" 1.5x the bucket size in tokens since
+        // the bucket is full, should arm the timer to 0.5x replenish
+        // time, which is 500 ms
+        assert!(l.consume(1500, TokenType::Bytes));
+
+        // check that after more than the minimum refill time,
+        // the rate limiter is still blocked
+        thread::sleep(Duration::from_millis(200));
+        assert!(l.event_handler().is_err());
+        assert!(l.is_blocked());
+
+        // try to consume some tokens, which should fail as the timer
+        // is still active
+        assert!(!l.consume(100, TokenType::Bytes));
+        assert!(l.event_handler().is_err());
+        assert!(l.is_blocked());
+
+        // check that after the minimum refill time, the timer was not
+        // overwritten and the rate limiter is still blocked from the
+        // borrowing we performed earlier
+        thread::sleep(Duration::from_millis(100));
+        assert!(l.event_handler().is_err());
+        assert!(l.is_blocked());
+        assert!(!l.consume(100, TokenType::Bytes));
+
+        // after waiting out the full duration, rate limiter should be
+        // availale again
+        thread::sleep(Duration::from_millis(200));
+        assert!(l.event_handler().is_ok());
+        assert!(!l.is_blocked());
+        assert!(l.consume(100, TokenType::Bytes));
+    }
+
+    #[test]
+    fn test_update_buckets() {
+        let mut x = RateLimiter::new(1000, 2000, 1000, 10, 20, 1000).unwrap();
+
+        let initial_bw = x.bandwidth.clone();
+        let initial_ops = x.ops.clone();
+
+        x.update_buckets(BucketUpdate::None, BucketUpdate::None);
+        assert_eq!(x.bandwidth, initial_bw);
+        assert_eq!(x.ops, initial_ops);
+
+        let new_bw = TokenBucket::new(123, 0, 57).unwrap();
+        let new_ops = TokenBucket::new(321, 12346, 89).unwrap();
+        x.update_buckets(
+            BucketUpdate::Update(new_bw.clone()),
+            BucketUpdate::Update(new_ops.clone()),
+        );
+
+        // We have manually adjust the last_update field, because it changes when update_buckets()
+        // constructs new buckets (and thus gets a different value for last_update). We do this so
+        // it makes sense to test the following assertions.
+        x.bandwidth.as_mut().unwrap().last_update = new_bw.last_update;
+        x.ops.as_mut().unwrap().last_update = new_ops.last_update;
+
+        assert_eq!(x.bandwidth, Some(new_bw));
+        assert_eq!(x.ops, Some(new_ops));
+
+        x.update_buckets(BucketUpdate::Disabled, BucketUpdate::Disabled);
+        assert_eq!(x.bandwidth, None);
+        assert_eq!(x.ops, None);
+    }
+
+    #[test]
+    fn test_rate_limiter_debug() {
+        let l = RateLimiter::new(1, 2, 3, 4, 5, 6).unwrap();
+        assert_eq!(
+            format!("{:?}", l),
+            format!(
+                "RateLimiter {{ bandwidth: {:?}, ops: {:?} }}",
+                l.bandwidth(),
+                l.ops()
+            ),
+        );
+    }
+}
--- a/release-notes.md
+++ b/release-notes.md
@@ -1,17 +1,35 @@
+- [v15.0](#v150)
+    - [Version numbering and stability guarantees](#version-numbering-and-stability-guarantees)
+    - [Network device rate limiting](#network-device-rate-limiting)
+    - [Support for runtime control of `virtio-net` guest offload](#support-for-runtime-control-of-virtio-net-guest-offload)
+    - [`--api-socket` supports file descriptor parameter](#--api-socket-supports-file-descriptor-parameter)
+    - [Bug fixes](#bug-fixes)
+    - [Deprecations](#deprecations)
+    - [Contributors](#contributors)
+- [v0.14.1](#v0141)
+- [v0.14.0](#v0140)
+    - [Structured event monitoring](#structured-event-monitoring)
+    - [MSHV improvements](#mshv-improvements)
+    - [Improved aarch64 platform](#improved-aarch64-platform)
+    - [Updated hotplug documentation](#updated-hotplug-documentation)
+    - [PTY control for serial and `virtio-console`](#pty-control-for-serial-and-virtio-console)
+    - [Block device rate limiting](#block-device-rate-limiting)
+    - [Deprecations](#deprecations-1)
+    - [Contributors](#contributors-1)
 - [v0.13.0](#v0130)
    - [Wider VFIO device support](#wider-vfio-device-support)
-    - [Improve huge page support](#improve-huge-page-support)
+    - [Improved huge page support](#improved-huge-page-support)
    - [MACvTAP support](#macvtap-support)
    - [VHD disk image support](#vhd-disk-image-support)
    - [Improved Virtio device threading](#improved-virtio-device-threading)
    - [Clean shutdown support via synthetic power button](#clean-shutdown-support-via-synthetic-power-button)
-    - [Contributors](#contributors)
+    - [Contributors](#contributors-2)
 - [v0.12.0](#v0120)
    - [ARM64 enhancements](#arm64-enhancements)
    - [Removal of `vhost-user-net` and `vhost-user-block` self spawning](#removal-of-vhost-user-net-and-vhost-user-block-self-spawning)
    - [Migration of `vhost-user-fs` backend](#migration-of-vhost-user-fs-backend)
    - [Enhanced "info" API](#enhanced-info-api)
-    - [Contributors](#contributors-1)
+    - [Contributors](#contributors-3)
 - [v0.11.0](#v0110)
    - [`io_uring` support by default for `virtio-block`](#io_uring-support-by-default-for-virtio-block)
    - [Windows Guest Support](#windows-guest-support)
@@ -24,14 +42,14 @@
    - [New `--balloon` Parameter Added](#new---balloon-parameter-added)
    - [Experimental `virtio-watchdog` Support](#experimental-virtio-watchdog-support)
    - [Notable Bug Fixes](#notable-bug-fixes)
-    - [Contributors](#contributors-2)
+    - [Contributors](#contributors-4)
 - [v0.10.0](#v0100)
    - [`virtio-block` Support for Multiple Descriptors](#virtio-block-support-for-multiple-descriptors)
    - [Memory Zones](#memory-zones)
    - [`Seccomp` Sandbox Improvements](#seccomp-sandbox-improvements)
    - [Preliminary KVM HyperV Emulation Control](#preliminary-kvm-hyperv-emulation-control)
    - [Notable Bug Fixes](#notable-bug-fixes-1)
-    - [Contributors](#contributors-3)
+    - [Contributors](#contributors-5)
 - [v0.9.0](#v090)
    - [`io_uring` Based Block Device Support](#io_uring-based-block-device-support)
    - [Block and Network Device Statistics](#block-and-network-device-statistics)
@@ -45,7 +63,7 @@
    - [Intel SGX Support](#intel-sgx-support)
    - [`Seccomp` Sandbox Improvements](#seccomp-sandbox-improvements-1)
    - [Notable Bug Fixes](#notable-bug-fixes-2)
-    - [Contributors](#contributors-4)
+    - [Contributors](#contributors-6)
 - [v0.8.0](#v080)
    - [Experimental Snapshot and Restore Support](#experimental-snapshot-and-restore-support)
    - [Experimental ARM64 Support](#experimental-arm64-support)
@@ -54,7 +72,7 @@
    - [`vhost_user_fs` Improvements](#vhost_user_fs-improvements)
    - [Notable Bug Fixes](#notable-bug-fixes-3)
    - [Command Line and API Changes](#command-line-and-api-changes)
-    - [Contributors](#contributors-5)
+    - [Contributors](#contributors-7)
 - [v0.7.0](#v070)
    - [Block, Network, Persistent Memory (PMEM), VirtioFS and Vsock hotplug](#block-network-persistent-memory-pmem-virtiofs-and-vsock-hotplug)
    - [Alternative `libc` Support](#alternative-libc-support)
@@ -64,14 +82,14 @@
    - [`Seccomp` Sandboxing](#seccomp-sandboxing)
    - [Updated Distribution Support](#updated-distribution-support)
    - [Command Line and API Changes](#command-line-and-api-changes-1)
-    - [Contributors](#contributors-6)
+    - [Contributors](#contributors-8)
 - [v0.6.0](#v060)
    - [Directly Assigned Devices Hotplug](#directly-assigned-devices-hotplug)
    - [Shared Filesystem Improvements](#shared-filesystem-improvements)
    - [Block and Networking IO Self Offloading](#block-and-networking-io-self-offloading)
    - [Command Line Interface](#command-line-interface)
    - [PVH Boot](#pvh-boot)
-    - [Contributors](#contributors-7)
+    - [Contributors](#contributors-9)
 - [v0.5.1](#v051)
 - [v0.5.0](#v050)
    - [Virtual Machine Dynamic Resizing](#virtual-machine-dynamic-resizing)
@@ -79,7 +97,7 @@
    - [New Interrupt Management Framework](#new-interrupt-management-framework)
    - [Development Tools](#development-tools)
    - [Kata Containers Integration](#kata-containers-integration)
-    - [Contributors](#contributors-8)
+    - [Contributors](#contributors-10)
 - [v0.4.0](#v040)
    - [Dynamic virtual CPUs addition](#dynamic-virtual-cpus-addition)
    - [Programmatic firmware tables generation](#programmatic-firmware-tables-generation)
@@ -88,7 +106,7 @@
    - [Userspace IOAPIC by default](#userspace-ioapic-by-default)
    - [PCI BAR reprogramming](#pci-bar-reprogramming)
    - [New `cloud-hypervisor` organization](#new-cloud-hypervisor-organization)
-    - [Contributors](#contributors-9)
+    - [Contributors](#contributors-11)
 - [v0.3.0](#v030)
    - [Block device offloading](#block-device-offloading)
    - [Network device backend](#network-device-backend)
@@ -115,6 +133,169 @@
    - [Unit testing](#unit-testing)
    - [Integration tests parallelization](#integration-tests-parallelization)

+# v15.0
+
+This release has been tracked through the [v15.0 project](https://github.com/cloud-hypervisor/cloud-hypervisor/projects/18).
+
+Highlights for `cloud-hypervisor` version v15.0 include:
+
+### Version numbering and stability guarantees
+
+This release is the first in a new version numbering scheme to represent that
+we believe Cloud Hypervisor is maturing and entering a period of stability.
+With this new release we are beginning our new stability guarantees:
+
+* The API (including command line options) will not be removed or changed in a
+  breaking way without a minimum of 2 releases notice. Where possible warnings
+  will be given about the use of deprecated functionality and the deprecations
+  will be documented in the release notes.
+* Point releases will be made between individual releases where there are
+  substantial bug fixes or security issues that need to be fixed.
+
+Currently the following items are **not** guaranteed across updates:
+
+* Snapshot/restore is not supported across different versions
+* Live migration is not supported across different versions
+* The following features are considered experimental and may change
+  substantially between releases: TDX, SGX.
+
+### Network device rate limiting
+
+Building on our existing support for rate limiting block activity the network
+device also now supports rate limiting. Full details of the controls are in the
+[IO throttling documentation.](docs/io_throttling.md)
+
+### Support for runtime control of `virtio-net` guest offload
+
+The guest is now able to change the offload settings for the `virtio-net`
+device. As well as providing a useful control this mitigates an issue in the
+Linux kernel where the guest will attempt to reprogram the offload settings
+even if they are not advertised as configurable (#2528).
+
+### `--api-socket` supports file descriptor parameter
+
+The `--api-socket` can now take an `fd=` parameter to specify an existing file
+descriptor to use. This is particularly beneficial for frameworks that need to
+programmatically control Cloud Hypervisor.
+
+### Bug fixes
+
+* A workaround has been put in place to mitigate a Linux kernel issues that
+  results in the CPU thread spinning at 100% when using `virtio-pmem` (#2277).
+* PCI BARs are now correctly aligned removing the need for the guest to
+  reprogram them (#1797,#1798)
+* Handle TAP interface not being writable within virtio-net (due to the buffer
+  exhaustion on the host) (#2517)
+* The recommended Linux kernel is now v5.12.0 as it contains a fix that
+  prevents snapshot & restore working (#2535)
+
+### Deprecations
+
+Deprecated features will be removed in a subsequent release and users should plan to use alternatives
+
+* Support for booting with the "LinuxBoot" protocol for ELF and `bzImage`
+  binaries has been deprecated. When using direct boot users should configure
+  their kernel with `CONFIG_PVH=y`. Will be removed in v16.0.
+
+### Contributors
+
+Many thanks to everyone who has contributed to our release including some new faces.
+
+* Alyssa Ross <hi@alyssa.is>
+* Anatol Belski <anbelski@linux.microsoft.com>
+* Bo Chen <chen.bo@intel.com>
+* Gaelan Steele <gbs@canishe.com>
+* Jianyong Wu <jianyong.wu@arm.com>
+* Michael Zhao <michael.zhao@arm.com>
+* Muminul Islam <muislam@microsoft.com>
+* Rob Bradford <robert.bradford@intel.com>
+* Sebastien Boeuf <sebastien.boeuf@intel.com>
+* Wei Liu <liuwe@microsoft.com>
+* William Douglas <william.douglas@intel.com>
+
+# v0.14.1
+
+Bug fix release branched off the v0.14.0 release. The following bugs were fixed
+in this release:
+
+* CPU hotplug on Windows failed due to misreported CPU state information and
+  the lack of HyperV CPUID bit enabled (#2437, #2449, #2436)
+* A seccomp rule was missing that was triggered on CPU unplug (#2455)
+* A bounds check in VIRTIO queue validation was erroneously generating
+  DescriptorChainTooShort errors in certain circumstances (#2450, #2424)
+
+# v0.14.0
+
+This release has been tracked through the [0.14.0 project](https://github.com/cloud-hypervisor/cloud-hypervisor/projects/17).
+
+Highlights for `cloud-hypervisor` version 0.14.0 include:
+
+### Structured event monitoring
+
+A new option was added to the VMM `--event-monitor` which reports structured
+events (JSON) over a file or file descriptor at key events in the lifecycle of
+the VM. The list of events is limited at the moment but will be further
+extended over subsequent releases. The events exposed form part of the Cloud
+Hypervisor API surface.
+
+### MSHV improvements
+
+Basic support has been added for running Windows guests atop the MSHV
+hypervisor as an alternative to KVM and further improvements have been made to
+the MSHV support.
+
+### Improved aarch64 platform
+
+The aarch64 platform has been enhanced with more devices exposed to the running
+VM including an enhanced serial UART.
+
+### Updated hotplug documentation
+
+The documentation for the hotplug support has been updated to reflect the use
+of the `ch-remote` tool and to include details of `virtio-mem` based hotplug as
+well as documenting hotplug of paravirtualised and VFIO devices.
+
+### PTY control for serial and `virtio-console`
+
+The `--serial` and `--console` parameters can now direct the console to a PTY
+allowing programmatic control of the console from another process through the
+PTY subsystem.
+
+### Block device rate limiting
+
+The block device performance can now be constrained as part of the VM
+configuration allowing rate limiting. Full details of the controls are in the
+[IO throttling documentation.](docs/io_throttling.md)
+
+
+### Deprecations
+
+Deprecated features will be removed in a subsequent release and users should plan to use alternatives
+
+* Support for booting with the "LinuxBoot" protocol for ELF and `bzImage`
+  binaries has been deprecated. When using direct boot users should configure
+  their kernel with `CONFIG_PVH=y`.
+
+
+### Contributors
+
+Many thanks to everyone who has contributed to our 0.14.0 release including
+some new faces.
+
+Bo Chen <chen.bo@intel.com>
+Henry Wang <Henry.Wang@arm.com>
+Iggy Jackson <iggy@theiggy.com>
+Jiachen Zhang <zhangjiachen.jaycee@bytedance.com>
+Michael Zhao <michael.zhao@arm.com>
+Muminul Islam <muislam@microsoft.com>
+Penny Zheng <Penny.Zheng@arm.com>
+Rob Bradford <robert.bradford@intel.com>
+Sebastien Boeuf <sebastien.boeuf@intel.com>
+Vineeth Pillai <viremana@linux.microsoft.com>
+Wei Liu <liuwe@microsoft.com>
+William Douglas <william.r.douglas@gmail.com>
+Zide Chen <zide.chen@intel.com>
+
 # v0.13.0

 This release has been tracked through the [0.13.0 project](https://github.com/cloud-hypervisor/cloud-hypervisor/projects/16).
@@ -128,7 +309,7 @@ devices that do not support MSI or MSI-X and instead rely on INTx interrupts.
 Most notably this widens the support to most NVIDIA cards with the proprietary
 drivers.

-### Improve huge page support
+### Improved huge page support

 Through the addition of `hugepage_size` on `--memory` it is now possible to
 specify the desired size of the huge pages used when allocating the guest
@@ -672,11 +853,11 @@ to run their guest I/O into separate executions contexts.
 ### Command Line Interface

 More and more Cloud Hypervisor services are exposed through the
-[Rest API](https://github.com/cloud-hypervisor/cloud-hypervisor/blob/master/vmm/src/api/openapi/cloud-hypervisor.yaml)
-and thus only accessible via relatively cumbersome HTTP calls. In order
-to abstract those calls into a more user friendly tool, we created a Cloud
-Hypervisor Command Line Interface (CLI) called `ch-remote`.
-The `ch-remote` binary is created with each build and available e.g. at
+[Rest API](vmm/src/api/openapi/cloud-hypervisor.yaml) and thus only
+accessible via relatively cumbersome HTTP calls. In order to abstract
+those calls into a more user friendly tool, we created a Cloud Hypervisor
+Command Line Interface (CLI) called `ch-remote`.  The `ch-remote` binary
+is created with each build and available e.g. at
 `cloud-hypervisor/target/debug/ch-remote` when doing a debug build.

 Please check `ch-remote --help` for a complete description of all available
@@ -752,7 +933,7 @@ In order to provide a better developer experience, we worked on improving our
 build, development and testing tools.
 Somehow similar to the excellent
 [Firecracker's devtool](https://github.com/firecracker-microvm/firecracker/blob/master/tools/devtool),
-we now provide a [dev_cli script](https://github.com/cloud-hypervisor/cloud-hypervisor/blob/master/scripts/dev_cli.sh).
+we now provide a [dev_cli script](scripts/dev_cli.sh).

 With this new tool, our users and contributors will be able to build and test
 Cloud Hypervisor through a containerized environment.
@@ -789,7 +970,7 @@ As a way to vertically scale Cloud-Hypervisor guests, we now support dynamically
 adding virtual CPUs to the guests, a mechanism also known as CPU hot plug.
 Through hardware-reduced ACPI notifications, Cloud Hypervisor can now add CPUs
 to an already running guest and the high level operations for that process are
-documented [here](https://github.com/cloud-hypervisor/cloud-hypervisor/blob/master/docs/hotplug.md)
+documented [here](docs/hotplug.md)

 During the next release cycles we are planning to extend Cloud Hypervisor
 hot plug framework to other resources, namely PCI devices and memory.
@@ -905,9 +1086,8 @@ configurations that do not require a PCI bus emulation.
 ### Paravirtualized IOMMU

 As we want to improve our nested guests support, we added support for exposing
-a [paravirtualized IOMMU](https://github.com/cloud-hypervisor/cloud-hypervisor/blob/master/docs/iommu.md)
-device through virtio. This allows for a safer nested virtio and directly
-assigned devices support.
+a [paravirtualized IOMMU](docs/iommu.md) device through virtio. This allows
+for a safer nested virtio and directly assigned devices support.

 To add the IOMMU support, we had to make some CLI changes for Cloud Hypervisor
 users to be able to specify if devices had to be handled through this virtual
@@ -956,8 +1136,8 @@ Based on the Firecracker idea of using a dedicated I/O port to measure guest
 boot times, we added support for logging guest events through the
 [0x80](https://www.intel.com/content/www/us/en/support/articles/000005500/boards-and-kits.html)
 PC debug port. This allows, among other things, for granular guest boot time
-measurements. See our [debug port documentation](https://github.com/cloud-hypervisor/cloud-hypervisor/blob/master/docs/debug-port.md)
-for more details.
+measurements. See our [debug port documentation](docs/debug-port.md) for more
+details.

 ### Improved direct device assignment

@@ -992,8 +1172,8 @@ We added support for the [virtio-fs](https://virtio-fs.gitlab.io/) shared file
 system, allowing for an efficient and reliable way of sharing a filesystem
 between the host and the `cloud-hypervisor` guest.

-See our [filesystem sharing](https://github.com/cloud-hypervisor/cloud-hypervisor/blob/master/docs/fs.md)
-documentation for more details on how to use virtio-fs with `cloud-hypervisor`.
+See our [filesystem sharing](docs/fs.md) documentation for more details on how
+to use virtio-fs with `cloud-hypervisor`.

 ### Initial direct device assignment support

@@ -1001,9 +1181,8 @@ VFIO (Virtual Function I/O) is a kernel framework that exposes direct device
 access to userspace. `cloud-hypervisor` uses VFIO to directly assign host
 physical devices into its guest.

-See our [VFIO](https://github.com/cloud-hypervisor/cloud-hypervisor/blob/master/docs/vfio.md)
-documentation for more detail on how to directly assign host devices to
-`cloud-hypervisor` guests.
+See our [VFIO](docs/vfio.md) documentation for more detail on how to directly
+assign host devices to `cloud-hypervisor` guests.

 ### Userspace IOAPIC

--- a/resources/Dockerfile
+++ b/resources/Dockerfile
@@ -1,7 +1,7 @@
-FROM ubuntu:18.04 as dev
+FROM ubuntu:20.04 as dev

 ARG TARGETARCH
-ARG RUST_TOOLCHAIN="1.49.0"
+ARG RUST_TOOLCHAIN="1.51.0"
 ARG CLH_SRC_DIR="/cloud-hypervisor"
 ARG CLH_BUILD_DIR="$CLH_SRC_DIR/build"
 ARG CARGO_REGISTRY_DIR="$CLH_BUILD_DIR/cargo_registry"
@@ -37,7 +37,6 @@ RUN apt-get update \
 	socat \
 	dosfstools \
 	cpio \
-	bsdtar \
 	libfdt-dev \
 	python3-setuptools \
    && apt-get clean \
--- a/resources/linux-config-aarch64
+++ b/resources/linux-config-aarch64
@@ -1,12 +1,13 @@
 #
 # Automatically generated file; DO NOT EDIT.
-# Linux/arm64 5.10.0 Kernel Configuration
+# Linux/arm64 5.12.0 Kernel Configuration
 #
 CONFIG_CC_VERSION_TEXT="gcc (Ubuntu 9.3.0-17ubuntu1~20.04) 9.3.0"
 CONFIG_CC_IS_GCC=y
 CONFIG_GCC_VERSION=90300
-CONFIG_LD_VERSION=234000000
 CONFIG_CLANG_VERSION=0
+CONFIG_LD_IS_BFD=y
+CONFIG_LD_VERSION=23400
 CONFIG_LLD_VERSION=0
 CONFIG_CC_CAN_LINK=y
 CONFIG_CC_CAN_LINK_STATIC=y
@@ -210,6 +211,7 @@ CONFIG_ARCH_WANT_DEFAULT_BPF_JIT=y
 # CONFIG_BPF_PRELOAD is not set
 CONFIG_USERFAULTFD=y
 CONFIG_ARCH_HAS_MEMBARRIER_SYNC_CORE=y
+# CONFIG_KCMP is not set
 CONFIG_RSEQ=y
 # CONFIG_DEBUG_RSEQ is not set
 # CONFIG_EMBEDDED is not set
@@ -225,7 +227,6 @@ CONFIG_PERF_EVENTS=y

 CONFIG_VM_EVENT_COUNTERS=y
 CONFIG_SLUB_DEBUG=y
-# CONFIG_SLUB_MEMCG_SYSFS_ON is not set
 # CONFIG_COMPAT_BRK is not set
 # CONFIG_SLAB is not set
 CONFIG_SLUB=y
@@ -273,9 +274,11 @@ CONFIG_ARCH_PROC_KCORE_TEXT=y
 #
 # CONFIG_ARCH_ACTIONS is not set
 # CONFIG_ARCH_AGILEX is not set
+# CONFIG_ARCH_N5X is not set
 # CONFIG_ARCH_SUNXI is not set
 # CONFIG_ARCH_ALPINE is not set
 # CONFIG_ARCH_BCM2835 is not set
+# CONFIG_ARCH_BCM4908 is not set
 # CONFIG_ARCH_BCM_IPROC is not set
 # CONFIG_ARCH_BERLIN is not set
 # CONFIG_ARCH_BITMAIN is not set
@@ -307,7 +310,6 @@ CONFIG_ARCH_PROC_KCORE_TEXT=y
 # CONFIG_ARCH_VEXPRESS is not set
 # CONFIG_ARCH_VISCONTI is not set
 # CONFIG_ARCH_XGENE is not set
-# CONFIG_ARCH_ZX is not set
 # CONFIG_ARCH_ZYNQMP is not set
 # end of Platform selection

@@ -348,6 +350,7 @@ CONFIG_QCOM_FALKOR_ERRATUM_1003=y
 CONFIG_QCOM_FALKOR_ERRATUM_1009=y
 CONFIG_QCOM_QDF2400_ERRATUM_0065=y
 CONFIG_QCOM_FALKOR_ERRATUM_E1041=y
+# CONFIG_NVIDIA_CARMEL_CNP_ERRATUM is not set
 CONFIG_SOCIONEXT_SYNQUACER_PREITS=y
 # end of ARM errata workarounds via the alternatives framework

@@ -377,14 +380,11 @@ CONFIG_HZ_250=y
 # CONFIG_HZ_1000 is not set
 CONFIG_HZ=250
 CONFIG_SCHED_HRTICK=y
-CONFIG_ARCH_SUPPORTS_DEBUG_PAGEALLOC=y
 CONFIG_ARCH_SPARSEMEM_ENABLE=y
 CONFIG_ARCH_SPARSEMEM_DEFAULT=y
 CONFIG_ARCH_SELECT_MEMORY_MODEL=y
-CONFIG_HAVE_ARCH_PFN_VALID=y
 CONFIG_HW_PERF_EVENTS=y
 CONFIG_SYS_SUPPORTS_HUGETLBFS=y
-CONFIG_ARCH_WANT_HUGE_PMD_SHARE=y
 CONFIG_ARCH_HAS_CACHE_LINE_SIZE=y
 CONFIG_ARCH_ENABLE_SPLIT_PMD_PTLOCK=y
 CONFIG_PARAVIRT=y
@@ -393,6 +393,7 @@ CONFIG_PARAVIRT_TIME_ACCOUNTING=y
 CONFIG_KEXEC_FILE=y
 # CONFIG_KEXEC_SIG is not set
 # CONFIG_CRASH_DUMP is not set
+CONFIG_TRANS_TABLE=y
 # CONFIG_XEN is not set
 CONFIG_FORCE_MAX_ZONEORDER=11
 CONFIG_UNMAP_KERNEL_AT_EL0=y
@@ -406,6 +407,8 @@ CONFIG_ARM64_TAGGED_ADDR_ABI=y
 #
 CONFIG_ARM64_HW_AFDBM=y
 CONFIG_ARM64_PAN=y
+CONFIG_AS_HAS_LDAPR=y
+CONFIG_AS_HAS_LSE_ATOMICS=y
 CONFIG_ARM64_LSE_ATOMICS=y
 CONFIG_ARM64_USE_LSE_ATOMICS=y
 CONFIG_ARM64_VHE=y
@@ -414,7 +417,6 @@ CONFIG_ARM64_VHE=y
 #
 # ARMv8.2 architectural features
 #
-CONFIG_ARM64_UAO=y
 # CONFIG_ARM64_PMEM is not set
 CONFIG_ARM64_RAS_EXTN=y
 CONFIG_ARM64_CNP=y
@@ -441,6 +443,7 @@ CONFIG_ARM64_TLB_RANGE=y
 #
 # ARMv8.5 architectural features
 #
+CONFIG_AS_HAS_ARMV8_5=y
 CONFIG_ARM64_BTI=y
 CONFIG_CC_HAS_BRANCH_PROT_PAC_RET_BTI=y
 CONFIG_ARM64_E0PD=y
@@ -635,7 +638,6 @@ CONFIG_HAVE_KVM_ARCH_TLB_FLUSH_ALL=y
 CONFIG_KVM_GENERIC_DIRTYLOG_READ_PROTECT=y
 CONFIG_HAVE_KVM_IRQ_BYPASS=y
 CONFIG_HAVE_KVM_VCPU_RUN_PID_CHANGE=y
-CONFIG_KVM_ARM_PMU=y
 # CONFIG_ARM64_CRYPTO is not set

 #
@@ -643,7 +645,6 @@ CONFIG_KVM_ARM_PMU=y
 #
 CONFIG_CRASH_CORE=y
 CONFIG_KEXEC_CORE=y
-CONFIG_SET_FS=y
 CONFIG_JUMP_LABEL=y
 # CONFIG_STATIC_KEYS_SELFTEST is not set
 CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS=y
@@ -679,16 +680,22 @@ CONFIG_HAVE_ARCH_SECCOMP=y
 CONFIG_HAVE_ARCH_SECCOMP_FILTER=y
 CONFIG_SECCOMP=y
 CONFIG_SECCOMP_FILTER=y
+# CONFIG_SECCOMP_CACHE_DEBUG is not set
 CONFIG_HAVE_ARCH_STACKLEAK=y
 CONFIG_HAVE_STACKPROTECTOR=y
 CONFIG_STACKPROTECTOR=y
 CONFIG_STACKPROTECTOR_STRONG=y
+CONFIG_ARCH_SUPPORTS_LTO_CLANG=y
+CONFIG_ARCH_SUPPORTS_LTO_CLANG_THIN=y
+CONFIG_LTO_NONE=y
 CONFIG_HAVE_CONTEXT_TRACKING=y
 CONFIG_HAVE_VIRT_CPU_ACCOUNTING_GEN=y
 CONFIG_HAVE_IRQ_TIME_ACCOUNTING=y
+CONFIG_HAVE_MOVE_PUD=y
 CONFIG_HAVE_MOVE_PMD=y
 CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE=y
 CONFIG_HAVE_ARCH_HUGE_VMAP=y
+CONFIG_ARCH_WANT_HUGE_PMD_SHARE=y
 CONFIG_MODULES_USE_ELF_RELA=y
 CONFIG_ARCH_HAS_ELF_RANDOMIZE=y
 CONFIG_HAVE_ARCH_MMAP_RND_BITS=y
@@ -707,6 +714,8 @@ CONFIG_ARCH_USE_MEMREMAP_PROT=y
 # CONFIG_LOCK_EVENT_COUNTS is not set
 CONFIG_ARCH_HAS_RELR=y
 CONFIG_ARCH_WANT_LD_ORPHAN_WARN=y
+CONFIG_HAVE_ARCH_PFN_VALID=y
+CONFIG_ARCH_SUPPORTS_DEBUG_PAGEALLOC=y

 #
 # GCOV-based kernel profiling
@@ -916,7 +925,7 @@ CONFIG_DEV_PAGEMAP_OPS=y
 # CONFIG_DEVICE_PRIVATE is not set
 CONFIG_ARCH_USES_HIGH_VMA_FLAGS=y
 CONFIG_PERCPU_STATS=y
-# CONFIG_GUP_BENCHMARK is not set
+# CONFIG_GUP_TEST is not set
 # CONFIG_READ_ONLY_THP_FOR_FS is not set
 CONFIG_ARCH_HAS_PTE_SPECIAL=y
 # end of Memory Management options
@@ -1012,6 +1021,7 @@ CONFIG_VIRTIO_VSOCKETS_COMMON=y
 # CONFIG_NET_NCSI is not set
 CONFIG_RPS=y
 CONFIG_RFS_ACCEL=y
+CONFIG_SOCK_RX_QUEUE_MAPPING=y
 CONFIG_XPS=y
 CONFIG_CGROUP_NET_PRIO=y
 CONFIG_CGROUP_NET_CLASSID=y
@@ -1034,7 +1044,6 @@ CONFIG_NET_FLOW_LIMIT=y
 # CONFIG_AF_KCM is not set
 CONFIG_STREAM_PARSER=y
 # CONFIG_WIRELESS is not set
-# CONFIG_WIMAX is not set
 # CONFIG_RFKILL is not set
 # CONFIG_NET_9P is not set
 # CONFIG_CAIF is not set
@@ -1071,7 +1080,6 @@ CONFIG_PCIEASPM_DEFAULT=y
 CONFIG_PCIE_PME=y
 # CONFIG_PCIE_DPC is not set
 # CONFIG_PCIE_PTM is not set
-# CONFIG_PCIE_BW is not set
 CONFIG_PCI_MSI=y
 CONFIG_PCI_MSI_IRQ_DOMAIN=y
 CONFIG_PCI_QUIRKS=y
@@ -1105,6 +1113,7 @@ CONFIG_PCI_HOST_GENERIC=y
 # CONFIG_PCIE_ALTERA is not set
 # CONFIG_PCI_HOST_THUNDER_PEM is not set
 # CONFIG_PCI_HOST_THUNDER_ECAM is not set
+# CONFIG_PCIE_MICROCHIP_HOST is not set

 #
 # DesignWare PCI Core Support
@@ -1142,6 +1151,7 @@ CONFIG_PCI_HOST_GENERIC=y
 # CONFIG_PCI_SW_SWITCHTEC is not set
 # end of PCI switch controller drivers

+# CONFIG_CXL_BUS is not set
 # CONFIG_PCCARD is not set
 # CONFIG_RAPIDIO is not set

@@ -1174,6 +1184,7 @@ CONFIG_ALLOW_DEV_COREDUMP=y
 CONFIG_GENERIC_CPU_AUTOPROBE=y
 CONFIG_GENERIC_CPU_VULNERABILITIES=y
 CONFIG_GENERIC_ARCH_TOPOLOGY=y
+CONFIG_GENERIC_ARCH_NUMA=y
 # end of Generic Driver Options

 #
@@ -1213,6 +1224,9 @@ CONFIG_BLK_DEV=y
 CONFIG_BLK_DEV_NULL_BLK=y
 # CONFIG_BLK_DEV_PCIESSD_MTIP32XX is not set
 CONFIG_ZRAM=y
+CONFIG_ZRAM_DEF_COMP_LZORLE=y
+# CONFIG_ZRAM_DEF_COMP_LZO is not set
+CONFIG_ZRAM_DEF_COMP="lzo-rle"
 # CONFIG_ZRAM_WRITEBACK is not set
 # CONFIG_ZRAM_MEMORY_TRACKING is not set
 # CONFIG_BLK_DEV_UMEM is not set
@@ -1221,7 +1235,6 @@ CONFIG_BLK_DEV_LOOP_MIN_COUNT=8
 CONFIG_BLK_DEV_CRYPTOLOOP=y
 # CONFIG_BLK_DEV_DRBD is not set
 # CONFIG_BLK_DEV_NBD is not set
-# CONFIG_BLK_DEV_SKD is not set
 # CONFIG_BLK_DEV_SX8 is not set
 CONFIG_BLK_DEV_RAM=y
 CONFIG_BLK_DEV_RAM_COUNT=16
@@ -1265,6 +1278,7 @@ CONFIG_VIRTIO_BLK=y
 #
 # Texas Instruments shared transport line discipline
 #
+# CONFIG_TI_ST is not set
 # end of Texas Instruments shared transport line discipline

 #
@@ -1272,6 +1286,7 @@ CONFIG_VIRTIO_BLK=y
 #
 # CONFIG_GENWQE is not set
 # CONFIG_ECHO is not set
+# CONFIG_BCM_VK is not set
 # CONFIG_MISC_ALCOR_PCI is not set
 # CONFIG_MISC_RTSX_PCI is not set
 # CONFIG_HABANA_AI is not set
@@ -1324,6 +1339,7 @@ CONFIG_VIRTIO_NET=y
 #
 # Distributed Switch Architecture drivers
 #
+# CONFIG_NET_DSA_MV88E6XXX_PTP is not set
 # end of Distributed Switch Architecture drivers

 # CONFIG_ETHERNET is not set
@@ -1345,10 +1361,6 @@ CONFIG_VIRTIO_NET=y
 # Host-side USB support is needed for USB Network Adapter support
 #
 # CONFIG_WLAN is not set
-
-#
-# Enable WiMAX (Networking options) to see the WiMAX drivers
-#
 # CONFIG_WAN is not set
 # CONFIG_VMXNET3 is not set
 # CONFIG_FUJITSU_ES is not set
@@ -1362,7 +1374,6 @@ CONFIG_NET_FAILOVER=y
 #
 CONFIG_INPUT=y
 CONFIG_INPUT_FF_MEMLESS=y
-CONFIG_INPUT_POLLDEV=y
 CONFIG_INPUT_SPARSEKMAP=y
 # CONFIG_INPUT_MATRIXKMAP is not set

@@ -1383,6 +1394,9 @@ CONFIG_INPUT_EVDEV=y
 CONFIG_INPUT_KEYBOARD=y
 # CONFIG_KEYBOARD_ATKBD is not set
 # CONFIG_KEYBOARD_LKKBD is not set
+CONFIG_KEYBOARD_GPIO=y
+CONFIG_KEYBOARD_GPIO_POLLED=y
+# CONFIG_KEYBOARD_MATRIX is not set
 # CONFIG_KEYBOARD_NEWTON is not set
 # CONFIG_KEYBOARD_OPENCORES is not set
 # CONFIG_KEYBOARD_SAMSUNG is not set
@@ -1398,9 +1412,14 @@ CONFIG_INPUT_KEYBOARD=y
 CONFIG_INPUT_MISC=y
 # CONFIG_INPUT_AD714X is not set
 # CONFIG_INPUT_E3X0_BUTTON is not set
+# CONFIG_INPUT_GPIO_BEEPER is not set
+# CONFIG_INPUT_GPIO_DECODER is not set
+# CONFIG_INPUT_GPIO_VIBRA is not set
 CONFIG_INPUT_UINPUT=y
+# CONFIG_INPUT_GPIO_ROTARY_ENCODER is not set
 # CONFIG_INPUT_ADXL34X is not set
 # CONFIG_INPUT_CMA3000 is not set
+# CONFIG_INPUT_SOC_BUTTON_ARRAY is not set
 # CONFIG_RMI4_CORE is not set

 #
@@ -1444,13 +1463,14 @@ CONFIG_SERIAL_8250_RUNTIME_UARTS=1
 CONFIG_SERIAL_8250_FSL=y
 # CONFIG_SERIAL_8250_DW is not set
 # CONFIG_SERIAL_8250_RT288X is not set
-# CONFIG_SERIAL_OF_PLATFORM is not set
+CONFIG_SERIAL_OF_PLATFORM=y

 #
 # Non-8250 serial port support
 #
 # CONFIG_SERIAL_AMBA_PL010 is not set
-# CONFIG_SERIAL_AMBA_PL011 is not set
+CONFIG_SERIAL_AMBA_PL011=y
+CONFIG_SERIAL_AMBA_PL011_CONSOLE=y
 # CONFIG_SERIAL_EARLYCON_ARM_SEMIHOST is not set
 # CONFIG_SERIAL_UARTLITE is not set
 CONFIG_SERIAL_CORE=y
@@ -1458,6 +1478,7 @@ CONFIG_SERIAL_CORE_CONSOLE=y
 # CONFIG_SERIAL_JSM is not set
 # CONFIG_SERIAL_SIFIVE is not set
 # CONFIG_SERIAL_SCCNXP is not set
+# CONFIG_SERIAL_BCM63XX is not set
 # CONFIG_SERIAL_ALTERA_JTAGUART is not set
 # CONFIG_SERIAL_ALTERA_UART is not set
 # CONFIG_SERIAL_XILINX_PS_UART is not set
@@ -1471,6 +1492,7 @@ CONFIG_SERIAL_ARC_NR_PORTS=1
 # CONFIG_SERIAL_SPRD is not set
 # end of Serial drivers

+CONFIG_SERIAL_MCTRL_GPIO=y
 # CONFIG_SERIAL_NONSTANDARD is not set
 # CONFIG_N_GSM is not set
 # CONFIG_NOZOMI is not set
@@ -1487,7 +1509,6 @@ CONFIG_HW_RANDOM=y
 # CONFIG_HW_RANDOM_TIMERIOMEM is not set
 # CONFIG_HW_RANDOM_BA431 is not set
 CONFIG_HW_RANDOM_VIRTIO=y
-CONFIG_HW_RANDOM_HISI_V2=y
 CONFIG_HW_RANDOM_CAVIUM=y
 # CONFIG_HW_RANDOM_CCTRNG is not set
 # CONFIG_HW_RANDOM_XIPHERA is not set
@@ -1535,12 +1556,67 @@ CONFIG_PTP_1588_CLOCK=y
 #
 # Enable PHYLIB and NETWORK_PHY_TIMESTAMPING to see the additional clocks.
 #
+# CONFIG_PTP_1588_CLOCK_OCP is not set
 # end of PTP clock support

 # CONFIG_PINCTRL is not set
-# CONFIG_GPIOLIB is not set
+CONFIG_GPIOLIB=y
+CONFIG_GPIOLIB_FASTPATH_LIMIT=512
+CONFIG_OF_GPIO=y
+CONFIG_GPIO_ACPI=y
+CONFIG_GPIOLIB_IRQCHIP=y
+# CONFIG_DEBUG_GPIO is not set
+# CONFIG_GPIO_SYSFS is not set
+# CONFIG_GPIO_CDEV is not set
+
+#
+# Memory mapped GPIO drivers
+#
+# CONFIG_GPIO_74XX_MMIO is not set
+# CONFIG_GPIO_ALTERA is not set
+# CONFIG_GPIO_AMDPT is not set
+# CONFIG_GPIO_CADENCE is not set
+# CONFIG_GPIO_DWAPB is not set
+# CONFIG_GPIO_EXAR is not set
+# CONFIG_GPIO_FTGPIO010 is not set
+# CONFIG_GPIO_GENERIC_PLATFORM is not set
+# CONFIG_GPIO_GRGPIO is not set
+# CONFIG_GPIO_HISI is not set
+# CONFIG_GPIO_HLWD is not set
+# CONFIG_GPIO_MB86S7X is not set
+CONFIG_GPIO_PL061=y
+# CONFIG_GPIO_SIFIVE is not set
+# CONFIG_GPIO_XGENE is not set
+# CONFIG_GPIO_XILINX is not set
+# CONFIG_GPIO_AMD_FCH is not set
+# end of Memory mapped GPIO drivers
+
+#
+# MFD GPIO expanders
+#
+# end of MFD GPIO expanders
+
+#
+# PCI GPIO expanders
+#
+# CONFIG_GPIO_BT8XX is not set
+# CONFIG_GPIO_PCI_IDIO_16 is not set
+# CONFIG_GPIO_PCIE_IDIO_24 is not set
+# CONFIG_GPIO_RDC321X is not set
+# end of PCI GPIO expanders
+
+#
+# Virtual GPIO drivers
+#
+# CONFIG_GPIO_AGGREGATOR is not set
+# CONFIG_GPIO_MOCKUP is not set
+# end of Virtual GPIO drivers
+
 # CONFIG_W1 is not set
 CONFIG_POWER_RESET=y
+# CONFIG_POWER_RESET_GPIO is not set
+# CONFIG_POWER_RESET_GPIO_RESTART is not set
+# CONFIG_POWER_RESET_LTC2952 is not set
 # CONFIG_POWER_RESET_RESTART is not set
 # CONFIG_POWER_RESET_XGENE is not set
 # CONFIG_POWER_RESET_SYSCON is not set
@@ -1554,6 +1630,8 @@ CONFIG_POWER_SUPPLY=y
 # CONFIG_BATTERY_DS2781 is not set
 # CONFIG_BATTERY_BQ27XXX is not set
 # CONFIG_CHARGER_MAX8903 is not set
+# CONFIG_CHARGER_GPIO is not set
+# CONFIG_CHARGER_LT3651 is not set
 # CONFIG_HWMON is not set
 CONFIG_THERMAL=y
 # CONFIG_THERMAL_NETLINK is not set
@@ -1587,6 +1665,7 @@ CONFIG_WATCHDOG_OPEN_TIMEOUT=0
 # Watchdog Device Drivers
 #
 # CONFIG_SOFT_WATCHDOG is not set
+# CONFIG_GPIO_WATCHDOG is not set
 # CONFIG_WDAT_WDT is not set
 # CONFIG_XILINX_WATCHDOG is not set
 # CONFIG_ARM_SP805_WATCHDOG is not set
@@ -1597,6 +1676,7 @@ CONFIG_WATCHDOG_OPEN_TIMEOUT=0
 # CONFIG_ARM_SMC_WATCHDOG is not set
 # CONFIG_ALIM7101_WDT is not set
 # CONFIG_I6300ESB_WDT is not set
+# CONFIG_MEN_A21_WDT is not set
 CONFIG_VIRTIO_WDT=y

 #
@@ -1619,6 +1699,7 @@ CONFIG_BCMA_POSSIBLE=y
 # CONFIG_HTC_PASIC3 is not set
 # CONFIG_LPC_ICH is not set
 # CONFIG_LPC_SCH is not set
+# CONFIG_MFD_INTEL_PMT is not set
 # CONFIG_MFD_JANZ_CMODIO is not set
 # CONFIG_MFD_KEMPLD is not set
 # CONFIG_MFD_MT6397 is not set
@@ -1706,7 +1787,9 @@ CONFIG_FB=y
 CONFIG_LCD_CLASS_DEVICE=y
 # CONFIG_LCD_PLATFORM is not set
 CONFIG_BACKLIGHT_CLASS_DEVICE=y
+# CONFIG_BACKLIGHT_KTD253 is not set
 # CONFIG_BACKLIGHT_QCOM_WLED is not set
+# CONFIG_BACKLIGHT_GPIO is not set
 # end of Backlight & LCD device support

 #
@@ -1782,6 +1865,7 @@ CONFIG_HID_REDRAGON=y
 # CONFIG_HID_PETALYNX is not set
 # CONFIG_HID_PICOLCD is not set
 # CONFIG_HID_PLANTRONICS is not set
+# CONFIG_HID_PLAYSTATION is not set
 # CONFIG_HID_PRIMAX is not set
 # CONFIG_HID_SAITEK is not set
 # CONFIG_HID_SAMSUNG is not set
@@ -1875,6 +1959,7 @@ CONFIG_RTC_DRV_PL031=y
 #
 # HID Sensor RTC drivers
 #
+# CONFIG_RTC_DRV_GOLDFISH is not set
 CONFIG_DMADEVICES=y
 # CONFIG_DMADEVICES_DEBUG is not set

@@ -1916,6 +2001,7 @@ CONFIG_DMA_OF=y
 #
 # CONFIG_SYNC_FILE is not set
 # CONFIG_DMABUF_MOVE_NOTIFY is not set
+# CONFIG_DMABUF_DEBUG is not set
 # CONFIG_DMABUF_HEAPS is not set
 # end of DMABUF options

@@ -1941,6 +2027,7 @@ CONFIG_VFIO_PCI_INTX=y
 # CONFIG_VFIO_MDEV is not set
 # CONFIG_VIRT_DRIVERS is not set
 CONFIG_VIRTIO=y
+CONFIG_VIRTIO_PCI_LIB=y
 CONFIG_VIRTIO_MENU=y
 CONFIG_VIRTIO_PCI=y
 CONFIG_VIRTIO_PCI_LEGACY=y
@@ -1965,13 +2052,15 @@ CONFIG_VHOST_MENU=y
 # CONFIG_GOLDFISH is not set
 # CONFIG_CHROME_PLATFORMS is not set
 # CONFIG_MELLANOX_PLATFORM is not set
+# CONFIG_SURFACE_PLATFORMS is not set
 CONFIG_HAVE_CLK=y
 CONFIG_CLKDEV_LOOKUP=y
 CONFIG_HAVE_CLK_PREPARE=y
 CONFIG_COMMON_CLK=y
-# CONFIG_CLK_QORIQ is not set
+# CONFIG_COMMON_CLK_AXI_CLKGEN is not set
 # CONFIG_COMMON_CLK_XGENE is not set
 # CONFIG_COMMON_CLK_FIXED_MMIO is not set
+# CONFIG_XILINX_VCU is not set
 # CONFIG_HWSPINLOCK is not set

 #
@@ -1991,6 +2080,7 @@ CONFIG_ARM64_ERRATUM_858921=y

 CONFIG_MAILBOX=y
 # CONFIG_ARM_MHU is not set
+# CONFIG_ARM_MHU_V2 is not set
 # CONFIG_PLATFORM_MHU is not set
 # CONFIG_PL320_MBOX is not set
 CONFIG_PCC=y
@@ -2042,11 +2132,6 @@ CONFIG_ACPI_VIOT=y
 #
 # end of Amlogic SoC drivers

-#
-# Aspeed SoC drivers
-#
-# end of Aspeed SoC drivers
-
 #
 # Broadcom SoC drivers
 #
@@ -2065,6 +2150,12 @@ CONFIG_ACPI_VIOT=y
 #
 # end of i.MX SoC drivers

+#
+# Enable LiteX SoC Builder specific drivers
+#
+# CONFIG_LITEX_SOC_CONTROLLER is not set
+# end of Enable LiteX SoC Builder specific drivers
+
 #
 # Qualcomm SoC drivers
 #
@@ -2075,7 +2166,6 @@ CONFIG_ACPI_VIOT=y
 #
 # Xilinx SoC drivers
 #
-# CONFIG_XILINX_VCU is not set
 # end of Xilinx SoC drivers
 # end of SOC (System On Chip) specific Drivers

@@ -2130,8 +2220,10 @@ CONFIG_PARTITION_PERCPU=y
 # CONFIG_ARM_CMN is not set
 CONFIG_ARM_PMU=y
 CONFIG_ARM_PMU_ACPI=y
+# CONFIG_ARM_SMMU_V3_PMU is not set
 # CONFIG_ARM_DSU_PMU is not set
 # CONFIG_ARM_SPE_PMU is not set
+# CONFIG_ARM_DMC620_PMU is not set
 # CONFIG_HISI_PMU is not set
 # end of Performance monitor support

@@ -2161,6 +2253,7 @@ CONFIG_DEV_DAX_PMEM=y
 CONFIG_DEV_DAX_KMEM=y
 CONFIG_NVMEM=y
 CONFIG_NVMEM_SYSFS=y
+# CONFIG_NVMEM_RMEM is not set

 #
 # HW tracing support
@@ -2395,6 +2488,7 @@ CONFIG_HAVE_HARDENED_USERCOPY_ALLOCATOR=y
 # CONFIG_HARDENED_USERCOPY is not set
 CONFIG_FORTIFY_SOURCE=y
 # CONFIG_STATIC_USERMODEHELPER is not set
+# CONFIG_IMA_SECURE_AND_OR_TRUSTED_BOOT is not set
 CONFIG_DEFAULT_SECURITY_DAC=y
 CONFIG_LSM="yama,loadpin,safesetid,integrity"

@@ -2443,6 +2537,7 @@ CONFIG_CRYPTO_NULL2=y
 # CONFIG_CRYPTO_PCRYPT is not set
 CONFIG_CRYPTO_CRYPTD=y
 # CONFIG_CRYPTO_AUTHENC is not set
+# CONFIG_CRYPTO_TEST is not set

 #
 # Public-key cryptography
@@ -2502,17 +2597,13 @@ CONFIG_CRYPTO_POLY1305=y
 # CONFIG_CRYPTO_MD4 is not set
 CONFIG_CRYPTO_MD5=y
 # CONFIG_CRYPTO_MICHAEL_MIC is not set
-# CONFIG_CRYPTO_RMD128 is not set
 # CONFIG_CRYPTO_RMD160 is not set
-# CONFIG_CRYPTO_RMD256 is not set
-# CONFIG_CRYPTO_RMD320 is not set
 # CONFIG_CRYPTO_SHA1 is not set
 CONFIG_CRYPTO_SHA256=y
 CONFIG_CRYPTO_SHA512=y
 # CONFIG_CRYPTO_SHA3 is not set
 # CONFIG_CRYPTO_SM3 is not set
 # CONFIG_CRYPTO_STREEBOG is not set
-# CONFIG_CRYPTO_TGR192 is not set
 # CONFIG_CRYPTO_WP512 is not set

 #
@@ -2526,7 +2617,6 @@ CONFIG_CRYPTO_AES=y
 # CONFIG_CRYPTO_CAST6 is not set
 CONFIG_CRYPTO_DES=y
 # CONFIG_CRYPTO_FCRYPT is not set
-# CONFIG_CRYPTO_SALSA20 is not set
 # CONFIG_CRYPTO_CHACHA20 is not set
 # CONFIG_CRYPTO_SERPENT is not set
 # CONFIG_CRYPTO_SM4 is not set
@@ -2662,6 +2752,7 @@ CONFIG_DMA_COHERENT_POOL=y
 CONFIG_DMA_REMAP=y
 CONFIG_DMA_DIRECT_REMAP=y
 # CONFIG_DMA_API_DEBUG is not set
+# CONFIG_DMA_MAP_BENCHMARK is not set
 CONFIG_SGL_ALLOC=y
 CONFIG_CPU_RMAP=y
 CONFIG_DQL=y
@@ -2682,6 +2773,8 @@ CONFIG_SBITMAP=y
 # CONFIG_STRING_SELFTEST is not set
 # end of Library routines

+CONFIG_GENERIC_LIB_DEVMEM_IS_ALLOWED=y
+
 #
 # Kernel hacking
 #
@@ -2705,7 +2798,6 @@ CONFIG_SYMBOLIC_ERRNAME=y
 # Compile-time checks and compiler options
 #
 # CONFIG_DEBUG_INFO is not set
-# CONFIG_ENABLE_MUST_CHECK is not set
 CONFIG_FRAME_WARN=2048
 CONFIG_STRIP_ASM_SYMS=y
 # CONFIG_READABLE_ASM is not set
@@ -2766,9 +2858,12 @@ CONFIG_DEBUG_MEMORY_INIT=y
 # CONFIG_DEBUG_PER_CPU_MAPS is not set
 CONFIG_HAVE_ARCH_KASAN=y
 CONFIG_HAVE_ARCH_KASAN_SW_TAGS=y
+CONFIG_HAVE_ARCH_KASAN_HW_TAGS=y
 CONFIG_CC_HAS_KASAN_GENERIC=y
 CONFIG_CC_HAS_WORKING_NOSANITIZE_ADDRESS=y
 # CONFIG_KASAN is not set
+CONFIG_HAVE_ARCH_KFENCE=y
+# CONFIG_KFENCE is not set
 # end of Memory Debugging

 # CONFIG_DEBUG_SHIRQ is not set
@@ -2814,6 +2909,7 @@ CONFIG_LOCK_DEBUGGING_SUPPORT=y
 # CONFIG_CSD_LOCK_WAIT_DEBUG is not set
 # end of Lock Debugging (spinlocks, mutexes, etc...)

+# CONFIG_DEBUG_IRQFLAGS is not set
 CONFIG_STACKTRACE=y
 # CONFIG_WARN_ALL_UNSEEDED_RANDOM is not set
 # CONFIG_DEBUG_KOBJECT is not set
@@ -2856,7 +2952,6 @@ CONFIG_HAVE_C_RECORDMCOUNT=y
 CONFIG_TRACING_SUPPORT=y
 # CONFIG_FTRACE is not set
 # CONFIG_SAMPLES is not set
-CONFIG_ARCH_HAS_DEVMEM_IS_ALLOWED=y
 CONFIG_STRICT_DEVMEM=y
 # CONFIG_IO_STRICT_DEVMEM is not set

--- a/resources/linux-config-x86_64
+++ b/resources/linux-config-x86_64
@@ -1,12 +1,13 @@
 #
 # Automatically generated file; DO NOT EDIT.
-# Linux/x86 5.10.0 Kernel Configuration
+# Linux/x86 5.12.0 Kernel Configuration
 #
 CONFIG_CC_VERSION_TEXT="gcc (GCC) 10.2.1 20201125 (Red Hat 10.2.1-9)"
 CONFIG_CC_IS_GCC=y
 CONFIG_GCC_VERSION=100201
-CONFIG_LD_VERSION=234000000
 CONFIG_CLANG_VERSION=0
+CONFIG_LD_IS_BFD=y
+CONFIG_LD_VERSION=23500
 CONFIG_LLD_VERSION=0
 CONFIG_CC_CAN_LINK=y
 CONFIG_CC_CAN_LINK_STATIC=y
@@ -235,6 +236,7 @@ CONFIG_ARCH_WANT_DEFAULT_BPF_JIT=y
 # CONFIG_BPF_PRELOAD is not set
 CONFIG_USERFAULTFD=y
 CONFIG_ARCH_HAS_MEMBARRIER_SYNC_CORE=y
+# CONFIG_KCMP is not set
 CONFIG_RSEQ=y
 # CONFIG_DEBUG_RSEQ is not set
 # CONFIG_EMBEDDED is not set
@@ -250,7 +252,6 @@ CONFIG_PERF_EVENTS=y

 CONFIG_VM_EVENT_COUNTERS=y
 CONFIG_SLUB_DEBUG=y
-# CONFIG_SLUB_MEMCG_SYSFS_ON is not set
 # CONFIG_COMPAT_BRK is not set
 # CONFIG_SLAB is not set
 CONFIG_SLUB=y
@@ -289,7 +290,6 @@ CONFIG_ARCH_SUSPEND_POSSIBLE=y
 CONFIG_ARCH_WANT_GENERAL_HUGETLB=y
 CONFIG_ZONE_DMA32=y
 CONFIG_AUDIT_ARCH=y
-CONFIG_ARCH_SUPPORTS_DEBUG_PAGEALLOC=y
 CONFIG_X86_64_SMP=y
 CONFIG_ARCH_SUPPORTS_UPROBES=y
 CONFIG_FIX_EARLYCON_MEM=y
@@ -412,6 +412,7 @@ CONFIG_X86_SMAP=y
 CONFIG_X86_INTEL_TSX_MODE_OFF=y
 # CONFIG_X86_INTEL_TSX_MODE_ON is not set
 # CONFIG_X86_INTEL_TSX_MODE_AUTO is not set
+# CONFIG_X86_SGX is not set
 CONFIG_EFI=y
 CONFIG_EFI_STUB=y
 # CONFIG_EFI_MIXED is not set
@@ -478,6 +479,7 @@ CONFIG_ARCH_MIGHT_HAVE_ACPI_PDC=y
 CONFIG_ACPI_SYSTEM_POWER_STATES_SUPPORT=y
 # CONFIG_ACPI_DEBUGGER is not set
 CONFIG_ACPI_SPCR_TABLE=y
+# CONFIG_ACPI_FPDT is not set
 CONFIG_ACPI_LPIT=y
 CONFIG_ACPI_SLEEP=y
 CONFIG_ACPI_REV_OVERRIDE_POSSIBLE=y
@@ -519,7 +521,6 @@ CONFIG_HAVE_ACPI_APEI_NMI=y
 # CONFIG_ACPI_CONFIGFS is not set
 # CONFIG_PMIC_OPREGION is not set
 CONFIG_X86_PM_TIMER=y
-# CONFIG_SFI is not set

 #
 # CPU Frequency scaling
@@ -643,6 +644,7 @@ CONFIG_KVM=y
 # CONFIG_KVM_WERROR is not set
 CONFIG_KVM_INTEL=y
 # CONFIG_KVM_AMD is not set
+# CONFIG_KVM_XEN is not set
 CONFIG_AS_AVX512=y
 CONFIG_AS_SHA1_NI=y
 CONFIG_AS_SHA256_NI=y
@@ -655,8 +657,6 @@ CONFIG_CRASH_CORE=y
 CONFIG_KEXEC_CORE=y
 CONFIG_HOTPLUG_SMT=y
 CONFIG_GENERIC_ENTRY=y
-CONFIG_HAVE_OPROFILE=y
-CONFIG_OPROFILE_NMI_TIMER=y
 CONFIG_JUMP_LABEL=y
 # CONFIG_STATIC_KEYS_SELFTEST is not set
 # CONFIG_STATIC_CALL_SELFTEST is not set
@@ -701,14 +701,20 @@ CONFIG_HAVE_ARCH_SECCOMP=y
 CONFIG_HAVE_ARCH_SECCOMP_FILTER=y
 CONFIG_SECCOMP=y
 CONFIG_SECCOMP_FILTER=y
+# CONFIG_SECCOMP_CACHE_DEBUG is not set
 CONFIG_HAVE_ARCH_STACKLEAK=y
 CONFIG_HAVE_STACKPROTECTOR=y
 CONFIG_STACKPROTECTOR=y
 CONFIG_STACKPROTECTOR_STRONG=y
+CONFIG_ARCH_SUPPORTS_LTO_CLANG=y
+CONFIG_ARCH_SUPPORTS_LTO_CLANG_THIN=y
+CONFIG_LTO_NONE=y
 CONFIG_HAVE_ARCH_WITHIN_STACK_FRAMES=y
 CONFIG_HAVE_CONTEXT_TRACKING=y
+CONFIG_HAVE_CONTEXT_TRACKING_OFFSTACK=y
 CONFIG_HAVE_VIRT_CPU_ACCOUNTING_GEN=y
 CONFIG_HAVE_IRQ_TIME_ACCOUNTING=y
+CONFIG_HAVE_MOVE_PUD=y
 CONFIG_HAVE_MOVE_PMD=y
 CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE=y
 CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD=y
@@ -717,6 +723,8 @@ CONFIG_ARCH_WANT_HUGE_PMD_SHARE=y
 CONFIG_HAVE_ARCH_SOFT_DIRTY=y
 CONFIG_HAVE_MOD_ARCH_SPECIFIC=y
 CONFIG_MODULES_USE_ELF_RELA=y
+CONFIG_HAVE_IRQ_EXIT_ON_IRQ_STACK=y
+CONFIG_HAVE_SOFTIRQ_ON_OWN_STACK=y
 CONFIG_ARCH_HAS_ELF_RANDOMIZE=y
 CONFIG_HAVE_ARCH_MMAP_RND_BITS=y
 CONFIG_HAVE_EXIT_THREAD=y
@@ -735,7 +743,10 @@ CONFIG_ARCH_USE_MEMREMAP_PROT=y
 CONFIG_ARCH_HAS_MEM_ENCRYPT=y
 CONFIG_HAVE_STATIC_CALL=y
 CONFIG_HAVE_STATIC_CALL_INLINE=y
+CONFIG_HAVE_PREEMPT_DYNAMIC=y
 CONFIG_ARCH_WANT_LD_ORPHAN_WARN=y
+CONFIG_ARCH_SUPPORTS_DEBUG_PAGEALLOC=y
+CONFIG_ARCH_HAS_ELFCORE_COMPAT=y

 #
 # GCOV-based kernel profiling
@@ -897,7 +908,7 @@ CONFIG_ZONE_DEVICE=y
 CONFIG_DEV_PAGEMAP_OPS=y
 # CONFIG_DEVICE_PRIVATE is not set
 CONFIG_PERCPU_STATS=y
-# CONFIG_GUP_BENCHMARK is not set
+# CONFIG_GUP_TEST is not set
 # CONFIG_READ_ONLY_THP_FOR_FS is not set
 CONFIG_ARCH_HAS_PTE_SPECIAL=y
 # end of Memory Management options
@@ -993,6 +1004,7 @@ CONFIG_VIRTIO_VSOCKETS_COMMON=y
 # CONFIG_NET_NCSI is not set
 CONFIG_RPS=y
 CONFIG_RFS_ACCEL=y
+CONFIG_SOCK_RX_QUEUE_MAPPING=y
 CONFIG_XPS=y
 CONFIG_CGROUP_NET_PRIO=y
 CONFIG_CGROUP_NET_CLASSID=y
@@ -1015,7 +1027,6 @@ CONFIG_NET_FLOW_LIMIT=y
 # CONFIG_AF_KCM is not set
 CONFIG_STREAM_PARSER=y
 # CONFIG_WIRELESS is not set
-# CONFIG_WIMAX is not set
 # CONFIG_RFKILL is not set
 # CONFIG_NET_9P is not set
 # CONFIG_CAIF is not set
@@ -1051,7 +1062,6 @@ CONFIG_PCIEASPM_DEFAULT=y
 CONFIG_PCIE_PME=y
 # CONFIG_PCIE_DPC is not set
 # CONFIG_PCIE_PTM is not set
-# CONFIG_PCIE_BW is not set
 CONFIG_PCI_MSI=y
 CONFIG_PCI_MSI_IRQ_DOMAIN=y
 CONFIG_PCI_QUIRKS=y
@@ -1109,6 +1119,7 @@ CONFIG_HOTPLUG_PCI_ACPI=y
 # CONFIG_PCI_SW_SWITCHTEC is not set
 # end of PCI switch controller drivers

+# CONFIG_CXL_BUS is not set
 # CONFIG_PCCARD is not set
 # CONFIG_RAPIDIO is not set

@@ -1166,6 +1177,9 @@ CONFIG_BLK_DEV=y
 CONFIG_BLK_DEV_NULL_BLK=y
 # CONFIG_BLK_DEV_PCIESSD_MTIP32XX is not set
 CONFIG_ZRAM=y
+CONFIG_ZRAM_DEF_COMP_LZORLE=y
+# CONFIG_ZRAM_DEF_COMP_LZO is not set
+CONFIG_ZRAM_DEF_COMP="lzo-rle"
 # CONFIG_ZRAM_WRITEBACK is not set
 # CONFIG_ZRAM_MEMORY_TRACKING is not set
 # CONFIG_BLK_DEV_UMEM is not set
@@ -1174,7 +1188,6 @@ CONFIG_BLK_DEV_LOOP_MIN_COUNT=8
 CONFIG_BLK_DEV_CRYPTOLOOP=y
 # CONFIG_BLK_DEV_DRBD is not set
 # CONFIG_BLK_DEV_NBD is not set
-# CONFIG_BLK_DEV_SKD is not set
 # CONFIG_BLK_DEV_SX8 is not set
 CONFIG_BLK_DEV_RAM=y
 CONFIG_BLK_DEV_RAM_COUNT=16
@@ -1230,6 +1243,7 @@ CONFIG_VIRTIO_BLK=y
 # CONFIG_VMWARE_VMCI is not set
 # CONFIG_GENWQE is not set
 # CONFIG_ECHO is not set
+# CONFIG_BCM_VK is not set
 # CONFIG_MISC_ALCOR_PCI is not set
 # CONFIG_MISC_RTSX_PCI is not set
 # CONFIG_HABANA_AI is not set
@@ -1285,6 +1299,7 @@ CONFIG_VIRTIO_NET=y
 #
 # Distributed Switch Architecture drivers
 #
+# CONFIG_NET_DSA_MV88E6XXX_PTP is not set
 # end of Distributed Switch Architecture drivers

 # CONFIG_ETHERNET is not set
@@ -1306,10 +1321,6 @@ CONFIG_VIRTIO_NET=y
 # Host-side USB support is needed for USB Network Adapter support
 #
 # CONFIG_WLAN is not set
-
-#
-# Enable WiMAX (Networking options) to see the WiMAX drivers
-#
 # CONFIG_WAN is not set
 # CONFIG_VMXNET3 is not set
 # CONFIG_FUJITSU_ES is not set
@@ -1323,7 +1334,6 @@ CONFIG_NET_FAILOVER=y
 #
 CONFIG_INPUT=y
 CONFIG_INPUT_FF_MEMLESS=y
-CONFIG_INPUT_POLLDEV=y
 CONFIG_INPUT_SPARSEKMAP=y
 # CONFIG_INPUT_MATRIXKMAP is not set

@@ -1418,6 +1428,7 @@ CONFIG_SERIAL_CORE_CONSOLE=y
 # CONFIG_SERIAL_JSM is not set
 # CONFIG_SERIAL_LANTIQ is not set
 # CONFIG_SERIAL_SCCNXP is not set
+# CONFIG_SERIAL_BCM63XX is not set
 # CONFIG_SERIAL_ALTERA_JTAGUART is not set
 # CONFIG_SERIAL_ALTERA_UART is not set
 CONFIG_SERIAL_ARC=y
@@ -1500,6 +1511,7 @@ CONFIG_PTP_1588_CLOCK=y
 #
 CONFIG_PTP_1588_CLOCK_KVM=y
 # CONFIG_PTP_1588_CLOCK_VMW is not set
+# CONFIG_PTP_1588_CLOCK_OCP is not set
 # end of PTP clock support

 # CONFIG_PINCTRL is not set
@@ -1526,6 +1538,8 @@ CONFIG_THERMAL_GOV_USER_SPACE=y
 # Intel thermal drivers
 #
 # CONFIG_INTEL_POWERCLAMP is not set
+CONFIG_X86_THERMAL_VECTOR=y
+# CONFIG_X86_PKG_TEMP_THERMAL is not set
 # CONFIG_INTEL_SOC_DTS_THERMAL is not set

 #
@@ -1614,6 +1628,7 @@ CONFIG_BCMA_POSSIBLE=y
 # CONFIG_LPC_SCH is not set
 # CONFIG_MFD_INTEL_LPSS_ACPI is not set
 # CONFIG_MFD_INTEL_LPSS_PCI is not set
+# CONFIG_MFD_INTEL_PMT is not set
 # CONFIG_MFD_JANZ_CMODIO is not set
 # CONFIG_MFD_KEMPLD is not set
 # CONFIG_MFD_MT6397 is not set
@@ -1787,6 +1802,7 @@ CONFIG_HID_REDRAGON=y
 # CONFIG_HID_PETALYNX is not set
 # CONFIG_HID_PICOLCD is not set
 # CONFIG_HID_PLANTRONICS is not set
+# CONFIG_HID_PLAYSTATION is not set
 # CONFIG_HID_PRIMAX is not set
 # CONFIG_HID_SAITEK is not set
 # CONFIG_HID_SAMSUNG is not set
@@ -1813,6 +1829,12 @@ CONFIG_HID_REDRAGON=y
 #
 # CONFIG_INTEL_ISH_HID is not set
 # end of Intel ISH HID support
+
+#
+# AMD SFH HID Support
+#
+# CONFIG_AMD_SFH_HID is not set
+# end of AMD SFH HID Support
 # end of HID support

 CONFIG_USB_OHCI_LITTLE_ENDIAN=y
@@ -1852,6 +1874,7 @@ CONFIG_DW_DMAC_CORE=y
 # CONFIG_DW_EDMA_PCIE is not set
 CONFIG_HSU_DMA=y
 # CONFIG_SF_PDMA is not set
+# CONFIG_INTEL_LDMA is not set

 #
 # DMA Clients
@@ -1864,6 +1887,7 @@ CONFIG_HSU_DMA=y
 #
 # CONFIG_SYNC_FILE is not set
 # CONFIG_DMABUF_MOVE_NOTIFY is not set
+# CONFIG_DMABUF_DEBUG is not set
 # CONFIG_DMABUF_HEAPS is not set
 # end of DMABUF options

@@ -1891,6 +1915,7 @@ CONFIG_VFIO_PCI_INTX=y
 CONFIG_IRQ_BYPASS_MANAGER=y
 # CONFIG_VIRT_DRIVERS is not set
 CONFIG_VIRTIO=y
+CONFIG_VIRTIO_PCI_LIB=y
 CONFIG_VIRTIO_MENU=y
 CONFIG_VIRTIO_PCI=y
 CONFIG_VIRTIO_PCI_LEGACY=y
@@ -1915,10 +1940,12 @@ CONFIG_VIRTIO_MMIO_CMDLINE_DEVICES=y
 CONFIG_PMC_ATOM=y
 # CONFIG_CHROME_PLATFORMS is not set
 # CONFIG_MELLANOX_PLATFORM is not set
+# CONFIG_SURFACE_PLATFORMS is not set
 CONFIG_HAVE_CLK=y
 CONFIG_CLKDEV_LOOKUP=y
 CONFIG_HAVE_CLK_PREPARE=y
 CONFIG_COMMON_CLK=y
+# CONFIG_XILINX_VCU is not set
 # CONFIG_HWSPINLOCK is not set

 #
@@ -1976,11 +2003,6 @@ CONFIG_ACPI_VIOT=y
 #
 # end of Amlogic SoC drivers

-#
-# Aspeed SoC drivers
-#
-# end of Aspeed SoC drivers
-
 #
 # Broadcom SoC drivers
 #
@@ -1996,6 +2018,11 @@ CONFIG_ACPI_VIOT=y
 #
 # end of i.MX SoC drivers

+#
+# Enable LiteX SoC Builder specific drivers
+#
+# end of Enable LiteX SoC Builder specific drivers
+
 #
 # Qualcomm SoC drivers
 #
@@ -2006,7 +2033,6 @@ CONFIG_ACPI_VIOT=y
 #
 # Xilinx SoC drivers
 #
-# CONFIG_XILINX_VCU is not set
 # end of Xilinx SoC drivers
 # end of SOC (System On Chip) specific Drivers

@@ -2315,8 +2341,8 @@ CONFIG_CRYPTO_NULL2=y
 # CONFIG_CRYPTO_PCRYPT is not set
 CONFIG_CRYPTO_CRYPTD=y
 # CONFIG_CRYPTO_AUTHENC is not set
+# CONFIG_CRYPTO_TEST is not set
 CONFIG_CRYPTO_SIMD=y
-CONFIG_CRYPTO_GLUE_HELPER_X86=y

 #
 # Public-key cryptography
@@ -2385,10 +2411,7 @@ CONFIG_CRYPTO_POLY1305=y
 # CONFIG_CRYPTO_MD4 is not set
 CONFIG_CRYPTO_MD5=y
 # CONFIG_CRYPTO_MICHAEL_MIC is not set
-# CONFIG_CRYPTO_RMD128 is not set
 # CONFIG_CRYPTO_RMD160 is not set
-# CONFIG_CRYPTO_RMD256 is not set
-# CONFIG_CRYPTO_RMD320 is not set
 # CONFIG_CRYPTO_SHA1 is not set
 # CONFIG_CRYPTO_SHA1_SSSE3 is not set
 # CONFIG_CRYPTO_SHA256_SSSE3 is not set
@@ -2398,7 +2421,6 @@ CONFIG_CRYPTO_SHA512=y
 # CONFIG_CRYPTO_SHA3 is not set
 # CONFIG_CRYPTO_SM3 is not set
 # CONFIG_CRYPTO_STREEBOG is not set
-# CONFIG_CRYPTO_TGR192 is not set
 # CONFIG_CRYPTO_WP512 is not set
 # CONFIG_CRYPTO_GHASH_CLMUL_NI_INTEL is not set

@@ -2421,7 +2443,6 @@ CONFIG_CRYPTO_AES_NI_INTEL=y
 CONFIG_CRYPTO_DES=y
 # CONFIG_CRYPTO_DES3_EDE_X86_64 is not set
 # CONFIG_CRYPTO_FCRYPT is not set
-# CONFIG_CRYPTO_SALSA20 is not set
 # CONFIG_CRYPTO_CHACHA20 is not set
 # CONFIG_CRYPTO_CHACHA20_X86_64 is not set
 # CONFIG_CRYPTO_SERPENT is not set
@@ -2557,6 +2578,7 @@ CONFIG_ARCH_HAS_FORCE_DMA_UNENCRYPTED=y
 CONFIG_SWIOTLB=y
 CONFIG_DMA_COHERENT_POOL=y
 # CONFIG_DMA_API_DEBUG is not set
+# CONFIG_DMA_MAP_BENCHMARK is not set
 CONFIG_SGL_ALLOC=y
 CONFIG_CPU_RMAP=y
 CONFIG_DQL=y
@@ -2602,7 +2624,6 @@ CONFIG_SYMBOLIC_ERRNAME=y
 # Compile-time checks and compiler options
 #
 # CONFIG_DEBUG_INFO is not set
-# CONFIG_ENABLE_MUST_CHECK is not set
 CONFIG_FRAME_WARN=2048
 CONFIG_STRIP_ASM_SYMS=y
 # CONFIG_READABLE_ASM is not set
@@ -2662,11 +2683,15 @@ CONFIG_ARCH_HAS_DEBUG_VIRTUAL=y
 # CONFIG_DEBUG_VIRTUAL is not set
 CONFIG_DEBUG_MEMORY_INIT=y
 # CONFIG_DEBUG_PER_CPU_MAPS is not set
+CONFIG_ARCH_SUPPORTS_KMAP_LOCAL_FORCE_MAP=y
+# CONFIG_DEBUG_KMAP_LOCAL_FORCE_MAP is not set
 CONFIG_HAVE_ARCH_KASAN=y
 CONFIG_HAVE_ARCH_KASAN_VMALLOC=y
 CONFIG_CC_HAS_KASAN_GENERIC=y
 CONFIG_CC_HAS_WORKING_NOSANITIZE_ADDRESS=y
 # CONFIG_KASAN is not set
+CONFIG_HAVE_ARCH_KFENCE=y
+# CONFIG_KFENCE is not set
 # end of Memory Debugging

 # CONFIG_DEBUG_SHIRQ is not set
@@ -2714,6 +2739,7 @@ CONFIG_LOCK_DEBUGGING_SUPPORT=y
 # CONFIG_CSD_LOCK_WAIT_DEBUG is not set
 # end of Lock Debugging (spinlocks, mutexes, etc...)

+# CONFIG_DEBUG_IRQFLAGS is not set
 CONFIG_STACKTRACE=y
 # CONFIG_WARN_ALL_UNSEEDED_RANDOM is not set
 # CONFIG_DEBUG_KOBJECT is not set
@@ -2751,9 +2777,11 @@ CONFIG_HAVE_FUNCTION_GRAPH_TRACER=y
 CONFIG_HAVE_DYNAMIC_FTRACE=y
 CONFIG_HAVE_DYNAMIC_FTRACE_WITH_REGS=y
 CONFIG_HAVE_DYNAMIC_FTRACE_WITH_DIRECT_CALLS=y
+CONFIG_HAVE_DYNAMIC_FTRACE_WITH_ARGS=y
 CONFIG_HAVE_FTRACE_MCOUNT_RECORD=y
 CONFIG_HAVE_SYSCALL_TRACEPOINTS=y
 CONFIG_HAVE_FENTRY=y
+CONFIG_HAVE_OBJTOOL_MCOUNT=y
 CONFIG_HAVE_C_RECORDMCOUNT=y
 CONFIG_TRACING_SUPPORT=y
 # CONFIG_FTRACE is not set
--- a/scripts/dev_cli.sh
+++ b/scripts/dev_cli.sh
@@ -182,6 +182,7 @@ cmd_help() {
    echo "        --cargo               Run the cargo tests."
    echo "        --integration         Run the integration tests."
    echo "        --integration-sgx     Run the SGX integration tests."
+    echo "        --integration-vfio    Run the VFIO integration tests."
    echo "        --integration-windows Run the Windows guest integration tests."
    echo "        --libc                Select the C library Cloud Hypervisor will be built against. Default is gnu"
    echo "        --volumes             Hash separated volumes to be exported. Example --volumes /mnt:/mnt#/myvol:/myvol"
@@ -298,6 +299,7 @@ cmd_tests() {
    cargo=false
    integration=false
    integration_sgx=false
+    integration_vfio=false
    integration_windows=false
    libc="gnu"
    arg_vols=""
@@ -310,6 +312,7 @@ cmd_tests() {
            "--cargo")               { cargo=true; } ;;
            "--integration")         { integration=true; } ;;
            "--integration-sgx")     { integration_sgx=true; } ;;
+            "--integration-vfio")    { integration_vfio=true; } ;;
            "--integration-windows") { integration_windows=true; } ;;
            "--libc")
                shift
@@ -419,6 +422,25 @@ cmd_tests() {
 	       ./scripts/run_integration_tests_sgx.sh "$@" || fix_dir_perms $? || exit $?
    fi

+    if [ "$integration_vfio" = true ] ;  then
+	say "Running VFIO integration tests for $target..."
+	$DOCKER_RUNTIME run \
+	       --workdir "$CTR_CLH_ROOT_DIR" \
+	       --rm \
+	       --privileged \
+	       --security-opt seccomp=unconfined \
+	       --ipc=host \
+	       --net="$CTR_CLH_NET" \
+	       --mount type=tmpfs,destination=/tmp \
+	       --volume /dev:/dev \
+	       --volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR" $exported_volumes \
+	       --volume "$CLH_INTEGRATION_WORKLOADS:$CTR_CLH_INTEGRATION_WORKLOADS" \
+	       --env USER="root" \
+	       --env CH_LIBC="${libc}" \
+	       "$CTR_IMAGE" \
+	       ./scripts/run_integration_tests_vfio.sh "$@" || fix_dir_perms $? || exit $?
+    fi
+
    if [ "$integration_windows" = true ] ;  then
 	say "Running Windows integration tests for $target..."
 	$DOCKER_RUNTIME run \
--- a/scripts/run_integration_tests_aarch64.sh
+++ b/scripts/run_integration_tests_aarch64.sh
@@ -107,7 +107,7 @@ update_workloads() {
    }

    SRCDIR=$PWD
-    LINUX_CUSTOM_BRANCH="ch-5.10.6"
+    LINUX_CUSTOM_BRANCH="ch-5.12"

    # Check whether the local HEAD commit same as the remote HEAD or not. Remove the folder if they are different.
    if [ -d "$LINUX_CUSTOM_DIR" ]; then
@@ -203,11 +203,6 @@ if [ $RES -ne 0 ]; then
    exit 1
 fi

-# Create tap interface without multiple queues support for vhost_user_net test.
-sudo ip tuntap add name vunet-tap0 mode tap
-# Create tap interface with multiple queues support for vhost_user_net test.
-sudo ip tuntap add name vunet-tap1 mode tap multi_queue
-
 BUILD_TARGET="aarch64-unknown-linux-${CH_LIBC}"
 CFLAGS=""
 TARGET_CC=""
@@ -231,8 +226,4 @@ export RUST_BACKTRACE=1
 time cargo test $features_test "tests::parallel::$test_filter"
 RES=$?

-# Tear vhost_user_net test network down
-sudo ip link del vunet-tap0
-sudo ip link del vunet-tap1
-
 exit $RES
--- a/scripts/run_integration_tests_vfio.sh
+++ b/scripts/run_integration_tests_vfio.sh
@@ -0,0 +1,28 @@
+#!/bin/bash
+set -x
+
+source $HOME/.cargo/env
+source $(dirname "$0")/test-util.sh
+
+process_common_args "$@"
+# For now these values are deafult for kvm
+features_build=""
+features_test="--features integration_tests"
+
+BUILD_TARGET="$(uname -m)-unknown-linux-${CH_LIBC}"
+CFLAGS=""
+TARGET_CC=""
+if [[ "${BUILD_TARGET}" == "x86_64-unknown-linux-musl" ]]; then
+TARGET_CC="musl-gcc"
+CFLAGS="-I /usr/include/x86_64-linux-musl/ -idirafter /usr/include/"
+fi
+
+cargo build --all --release $features_build --target $BUILD_TARGET
+strip target/$BUILD_TARGET/release/cloud-hypervisor
+
+export RUST_BACKTRACE=1
+
+time cargo test $features_test "tests::vfio::$test_filter"
+RES=$?
+
+exit $RES
--- a/scripts/run_integration_tests_windows.sh
+++ b/scripts/run_integration_tests_windows.sh
@@ -9,6 +9,12 @@ process_common_args "$@"
 features_build=""
 features_test="--features integration_tests"

+if [ "$hypervisor" = "mshv" ] ;  then
+    features_build="--no-default-features --features mshv,common"
+    features_test="--no-default-features --features mshv,common,integration_tests"
+fi
+WIN_IMAGE_FILE="/root/workloads/windows-server-2019.raw"
+OVMF_FW_FILE="/root/workloads/OVMF-4b47d0c6c8.fd"
 BUILD_TARGET="$(uname -m)-unknown-linux-${CH_LIBC}"
 CFLAGS=""
 TARGET_CC=""
@@ -17,9 +23,15 @@ TARGET_CC="musl-gcc"
 CFLAGS="-I /usr/include/x86_64-linux-musl/ -idirafter /usr/include/"
 fi

+# Check if the images are present
+if [[ ! -f ${WIN_IMAGE_FILE} || ! -f ${OVMF_FW_FILE} ]]; then
+    echo "Windows image/firmware not present in the host"
+    exit 1
+fi
+
 # Use device mapper to create a snapshot of the Windows image
-img_blk_size=$(du -b -B 512 /root/workloads/windows-server-2019.raw | awk '{print $1;}')
-loop_device=$(losetup --find --show --read-only /root/workloads/windows-server-2019.raw)
+img_blk_size=$(du -b -B 512 ${WIN_IMAGE_FILE} | awk '{print $1;}')
+loop_device=$(losetup --find --show --read-only ${WIN_IMAGE_FILE})
 dmsetup create windows-base --table "0 $img_blk_size linear $loop_device 0"
 dmsetup mknodes
 dmsetup create windows-snapshot-base --table "0 $img_blk_size snapshot-origin /dev/mapper/windows-base"
--- a/scripts/run_integration_tests_x86_64.sh
+++ b/scripts/run_integration_tests_x86_64.sh
@@ -43,7 +43,7 @@ if [ ! -f "$BIONIC_OS_RAW_IMAGE" ]; then
 fi


-FOCAL_OS_IMAGE_NAME="focal-server-cloudimg-amd64-custom-20210106-1.qcow2"
+FOCAL_OS_IMAGE_NAME="focal-server-cloudimg-amd64-custom-20210407-0.qcow2"
 FOCAL_OS_IMAGE_URL="https://cloudhypervisorstorage.blob.core.windows.net/images/$FOCAL_OS_IMAGE_NAME"
 FOCAL_OS_IMAGE="$WORKLOADS_DIR/$FOCAL_OS_IMAGE_NAME"
 if [ ! -f "$FOCAL_OS_IMAGE" ]; then
@@ -52,7 +52,7 @@ if [ ! -f "$FOCAL_OS_IMAGE" ]; then
    popd
 fi

-FOCAL_OS_RAW_IMAGE_NAME="focal-server-cloudimg-amd64-custom-20210106-1.raw"
+FOCAL_OS_RAW_IMAGE_NAME="focal-server-cloudimg-amd64-custom-20210407-0.raw"
 FOCAL_OS_RAW_IMAGE="$WORKLOADS_DIR/$FOCAL_OS_RAW_IMAGE_NAME"
 if [ ! -f "$FOCAL_OS_RAW_IMAGE" ]; then
    pushd $WORKLOADS_DIR
@@ -96,36 +96,26 @@ popd

 # Build custom kernel based on virtio-pmem and virtio-fs upstream patches
 VMLINUX_IMAGE="$WORKLOADS_DIR/vmlinux"
-VMLINUX_PVH_IMAGE="$WORKLOADS_DIR/vmlinux.pvh"
 BZIMAGE_IMAGE="$WORKLOADS_DIR/bzImage"

 LINUX_CUSTOM_DIR="$WORKLOADS_DIR/linux-custom"

-if [ ! -f "$VMLINUX_IMAGE" ] || [ ! -f "$VMLINUX_PVH_IMAGE" ]; then
+if [ ! -f "$VMLINUX_IMAGE" ]; then
    SRCDIR=$PWD
    pushd $WORKLOADS_DIR
-    time git clone --depth 1 "https://github.com/cloud-hypervisor/linux.git" -b "ch-5.10.6" $LINUX_CUSTOM_DIR
+    time git clone --depth 1 "https://github.com/cloud-hypervisor/linux.git" -b "ch-5.12" $LINUX_CUSTOM_DIR
    cp $SRCDIR/resources/linux-config-x86_64 $LINUX_CUSTOM_DIR/.config
    popd
 fi

 if [ ! -f "$VMLINUX_IMAGE" ]; then
    pushd $LINUX_CUSTOM_DIR
-    scripts/config --disable "CONFIG_PVH"
    time make bzImage -j `nproc`
    cp vmlinux $VMLINUX_IMAGE || exit 1
    cp arch/x86/boot/bzImage $BZIMAGE_IMAGE || exit 1
    popd
 fi

-if [ ! -f "$VMLINUX_PVH_IMAGE" ]; then
-    pushd $LINUX_CUSTOM_DIR
-    scripts/config --enable "CONFIG_PVH"
-    time make bzImage -j `nproc`
-    cp vmlinux $VMLINUX_PVH_IMAGE || exit 1
-    popd
-fi
-
 if [ -d "$LINUX_CUSTOM_DIR" ]; then
    rm -rf $LINUX_CUSTOM_DIR
 fi
@@ -187,33 +177,6 @@ cp $FOCAL_OS_IMAGE $VFIO_DIR
 cp $FW $VFIO_DIR
 cp $VMLINUX_IMAGE $VFIO_DIR || exit 1

-# VFIO test network setup.
-# We reserve a different IP class for it: 172.18.0.0/24.
-sudo ip link add name vfio-br0 type bridge
-sudo ip link set vfio-br0 up
-sudo ip addr add 172.18.0.1/24 dev vfio-br0
-
-sudo ip tuntap add vfio-tap0 mode tap
-sudo ip link set vfio-tap0 master vfio-br0
-sudo ip link set vfio-tap0 up
-
-sudo ip tuntap add vfio-tap1 mode tap
-sudo ip link set vfio-tap1 master vfio-br0
-sudo ip link set vfio-tap1 up
-
-sudo ip tuntap add vfio-tap2 mode tap
-sudo ip link set vfio-tap2 master vfio-br0
-sudo ip link set vfio-tap2 up
-
-sudo ip tuntap add vfio-tap3 mode tap
-sudo ip link set vfio-tap3 master vfio-br0
-sudo ip link set vfio-tap3 up
-
-# Create tap interface without multiple queues support for vhost_user_net test.
-sudo ip tuntap add name vunet-tap0 mode tap
-# Create tap interface with multiple queues support for vhost_user_net test.
-sudo ip tuntap add name vunet-tap1 mode tap multi_queue
-
 BUILD_TARGET="$(uname -m)-unknown-linux-${CH_LIBC}"
 CFLAGS=""
 TARGET_CC=""
@@ -253,15 +216,4 @@ if [ $RES -eq 0 ]; then
    RES=$?
 fi

-# Tear VFIO test network down
-sudo ip link del vfio-br0
-sudo ip link del vfio-tap0
-sudo ip link del vfio-tap1
-sudo ip link del vfio-tap2
-sudo ip link del vfio-tap3
-
-# Tear vhost_user_net test network down
-sudo ip link del vunet-tap0
-sudo ip link del vunet-tap1
-
 exit $RES
--- a/scripts/sha1sums-x86_64
+++ b/scripts/sha1sums-x86_64
@@ -1,5 +1,5 @@
 27f3b17962ace69b51f0ddc2012095e3109e6ed8 bionic-server-cloudimg-amd64.qcow2
 8db9cc58b01452ce2d06c313177e6e74d8582d93 bionic-server-cloudimg-amd64.raw
 d4a44acc6014d5f83dea1c625c43d677a95fa75f alpine-minirootfs-x86_64.tar.gz
-4ee774e9e10f3cd985203c3ea20b487c285dfa1d focal-server-cloudimg-amd64-custom-20210106-1.qcow2
-7cd439f9dfe63b5f354011ea0e893256a8111174 focal-server-cloudimg-amd64-custom-20210106-1.raw
+f2f88ab4c08e36a4b6747a1aac4f94065952cadd focal-server-cloudimg-amd64-custom-20210407-0.qcow2
+6c5770ebd34c0dd53987bc9deae43690ef02ffc5 focal-server-cloudimg-amd64-custom-20210407-0.raw
--- a/Show More
+++ b/Show More