build: Release v26.0

Signed-off-by: Rob Bradford <robert.bradford@intel.com>
vmm: x86_64: Rename load_firmware() to reflect its purpose
2026-08-05 02:19:16 +00:00 · 2022-08-17 16:49:03 +01:00 · 2022-08-17 09:50:42 +01:00 · 2022-08-17 09:50:42 +01:00 · 2022-08-17 09:50:42 +01:00 · 2022-08-17 09:50:42 +01:00
176 changed files with 10407 additions and 6680 deletions
--- a/.github/workflows/build.yaml
+++ b/.github/workflows/build.yaml
@@ -29,14 +29,9 @@ jobs:
      - name: Install Rust toolchain (${{ matrix.rust }})
        uses: actions-rs/toolchain@v1
        with:
-            toolchain: ${{ matrix.rust }}
-            target: ${{ matrix.target }}
-            override: true
-
-      - name: Debug Check (default features)
-        run: |
-          git rev-list origin/main..$GITHUB_SHA | xargs -t -I % sh -c 'git checkout %; cargo check --tests --all --target=${{ matrix.target }}'
-          git checkout $GITHUB_SHA
+          toolchain: ${{ matrix.rust }}
+          target: ${{ matrix.target }}
+          override: true

      - name: Build (default features)
        run: cargo rustc --locked --bin cloud-hypervisor -- -D warnings
@@ -53,9 +48,15 @@ jobs:
      - name: Build (default features + gdb)
        run: cargo rustc --locked --bin cloud-hypervisor --features "gdb" -- -D warnings

+      - name: Build (default features + guest_debug)
+        run: cargo rustc --locked --bin cloud-hypervisor --features "guest_debug" -- -D warnings
+
      - name: Build (common + mshv)
        run: cargo rustc --locked --bin cloud-hypervisor --no-default-features --features "common,mshv"  -- -D warnings

+      - name: Build (common + mshv + kvm)
+        run: cargo rustc --locked --bin cloud-hypervisor --no-default-features --features "common,mshv,kvm"  -- -D warnings
+
      - name: Release Build (default features)
        run: cargo build --locked --all --release --target=${{ matrix.target }}

--- a/.github/workflows/quality-aarch64.yaml
+++ b/.github/workflows/quality-aarch64.yaml
@@ -25,10 +25,16 @@ jobs:
      - name: Install Rust toolchain (${{ matrix.rust }})
        uses: actions-rs/toolchain@v1
        with:
-            toolchain: ${{ matrix.rust }}
-            target: ${{ matrix.target }}
-            override: true
-            components: rustfmt, clippy
+          toolchain: ${{ matrix.rust }}
+          target: ${{ matrix.target }}
+          override: true
+          components: rustfmt, clippy
+
+      - name: Debug Check (default features)
+        run: |
+          git rev-list origin/main..$GITHUB_SHA | xargs -t -I % sh -c 'git checkout %; cargo check --tests --all --target=${{ matrix.target }}'
+          git checkout $GITHUB_SHA
+
      - name: Formatting (rustfmt)
        run: cargo fmt -- --check

--- a/.github/workflows/quality.yaml
+++ b/.github/workflows/quality.yaml
@@ -30,6 +30,11 @@ jobs:
          override: true
          components: rustfmt, clippy

+      - name: Debug Check (default features)
+        run: |
+          git rev-list origin/main..$GITHUB_SHA | xargs -t -I % sh -c 'git checkout %; cargo check --tests --all --target=${{ matrix.target }}'
+          git checkout $GITHUB_SHA
+
      - name: Formatting (rustfmt)
        run: cargo fmt -- --check

@@ -45,8 +50,14 @@ jobs:
      - name: Clippy (default features + gdb)
        run: cargo clippy --locked --all --all-targets --tests --features "gdb" -- -D warnings

+      - name: Clippy (default features + guest_debug)
+        run: cargo clippy --locked --all --all-targets --tests --features "guest_debug" -- -D warnings
+
      - name: Clippy (common + mshv)
        run: cargo clippy --locked --all --all-targets --no-default-features --tests --features "common,mshv" -- -D warnings

+      - name: Clippy (common + mshv + kvm)
+        run: cargo clippy --locked --all --all-targets --no-default-features --tests --features "common,mshv,kvm" -- -D warnings
+
      - name: Check build did not modify any files
        run: test -z "$(git status --porcelain)"
--- a/.github/workflows/release.yaml
+++ b/.github/workflows/release.yaml
@@ -16,23 +16,23 @@ jobs:
      - name: Install Rust toolchain (x86_64-unknown-linux-gnu)
        uses: actions-rs/toolchain@v1
        with:
-            toolchain: "1.60"
+            toolchain: "1.62"
            target: x86_64-unknown-linux-gnu
      - name: Install Rust toolchain (x86_64-unknown-linux-musl)
        uses: actions-rs/toolchain@v1
        with:
-            toolchain: "1.60"
+            toolchain: "1.62"
            target: x86_64-unknown-linux-musl
      - name: Build
        uses: actions-rs/cargo@v1
        with:
-            toolchain: "1.60"
+            toolchain: "1.62"
            command: build
            args: --all --release --target=x86_64-unknown-linux-gnu
      - name: Static Build
        uses: actions-rs/cargo@v1
        with:
-            toolchain: "1.60"
+            toolchain: "1.62"
            command: build
            args: --all --release --target=x86_64-unknown-linux-musl
      - name: Strip cloud-hypervisor binaries
@@ -40,7 +40,7 @@ jobs:
      - name: Install Rust toolchain (aarch64-unknown-linux-musl)
        uses: actions-rs/toolchain@v1
        with:
-            toolchain: "1.60"
+            toolchain: "1.62"
            target: aarch64-unknown-linux-musl
            override: true
      - name: Static Build (AArch64)
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -17,6 +17,23 @@ We follow the [Rust Style](https://github.com/rust-dev-tools/fmt-rfcs/blob/maste
 convention and enforce it through the Continuous Integration (CI) process calling into `rustfmt`
 for each submitted Pull Request (PR).

+## Basic Checks
+
+Please consider creating the following hook as `.git/hooks/pre-commit` in order
+to ensure basic correctness of your code. You can extend this further if you
+have specific features that you regularly develop against.
+
+```sh
+#!/bin/sh
+
+cargo fmt -- --check || exit 1
+cargo check --locked --all --all-targets --tests || exit 1
+cargo clippy --locked --all --all-targets --tests -- -D warnings || exit 1
+```
+
+You will need to `chmod +x .git/hooks/pre-commit` to have it run on every
+commit you make.
+
 ## Certificate of Origin

 In order to get a clear contribution chain of trust we use the [signed-off-by language](https://01.org/community/signed-process)
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -20,9 +20,9 @@ dependencies = [

 [[package]]
 name = "anyhow"
-version = "1.0.57"
+version = "1.0.62"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "08f9b8508dccb7687a1d6c4ce66b2b0ecef467c94667de27d8d7fe1f8d2a9cdc"
+checksum = "1485d4d2cc45e7b201ee3767015c96faa5904387c9d87c6efdd0fb511f12d305"

 [[package]]
 name = "api_client"
@@ -33,15 +33,14 @@ dependencies = [

 [[package]]
 name = "arc-swap"
-version = "1.5.0"
+version = "1.5.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c5d78ce20460b82d3fa150275ed9d55e21064fc7951177baacf86a145c4a4b1f"
+checksum = "983cd8b9d4b02a6dc6ffa557262eb5858a27a0038ffffe21a0f133eaa819a164"

 [[package]]
 name = "arch"
 version = "0.1.0"
 dependencies = [
- "acpi_tables",
 "anyhow",
 "byteorder",
 "fdt",
@@ -51,6 +50,7 @@ dependencies = [
 "log",
 "serde",
 "thiserror",
+ "uuid",
 "versionize",
 "versionize_derive",
 "vm-fdt",
@@ -130,15 +130,15 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"

 [[package]]
 name = "clap"
-version = "3.1.18"
+version = "3.2.17"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d2dbdf4bdacb33466e854ce889eee8dfd5729abf7ccd7664d0a2d60cd384440b"
+checksum = "29e724a68d9319343bb3328c9cc2dfde263f4b3142ee1059a9980580171c954b"
 dependencies = [
 "atty",
 "bitflags",
 "clap_lex",
 "indexmap",
- "lazy_static",
+ "once_cell",
 "strsim",
 "termcolor",
 "terminal_size",
@@ -147,16 +147,16 @@ dependencies = [

 [[package]]
 name = "clap_lex"
-version = "0.2.0"
+version = "0.2.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a37c35f1112dad5e6e0b1adaff798507497a18fceeb30cceb3bae7d1427b9213"
+checksum = "2850f2f5a82cbf437dd5af4d49848fbdfc27c157c3d010345776f952765261c5"
 dependencies = [
 "os_str_bytes",
 ]

 [[package]]
 name = "cloud-hypervisor"
-version = "24.0.0"
+version = "26.0.0"
 dependencies = [
 "anyhow",
 "api_client",
@@ -165,10 +165,10 @@ dependencies = [
 "epoll",
 "event_monitor",
 "hypervisor",
- "lazy_static",
 "libc",
 "log",
 "net_util",
+ "once_cell",
 "option_parser",
 "seccompiler",
 "serde_json",
@@ -196,6 +196,41 @@ version = "1.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "55626594feae15d266d52440b26ff77de0e22230cf0c113abe619084c1ddc910"

+[[package]]
+name = "darling"
+version = "0.14.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4529658bdda7fd6769b8614be250cdcfc3aeb0ee72fe66f9e41e5e5eb73eac02"
+dependencies = [
+ "darling_core",
+ "darling_macro",
+]
+
+[[package]]
+name = "darling_core"
+version = "0.14.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "649c91bc01e8b1eac09fb91e8dbc7d517684ca6be8ebc75bb9cafc894f9fdb6f"
+dependencies = [
+ "fnv",
+ "ident_case",
+ "proc-macro2",
+ "quote",
+ "strsim",
+ "syn",
+]
+
+[[package]]
+name = "darling_macro"
+version = "0.14.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ddfc69c5bfcbd2fc09a0f38451d2daf0e372e367986a83906d1b0dbc88134fb5"
+dependencies = [
+ "darling_core",
+ "quote",
+ "syn",
+]
+
 [[package]]
 name = "devices"
 version = "0.1.0"
@@ -205,7 +240,7 @@ dependencies = [
 "arch",
 "bitflags",
 "byteorder",
- "epoll",
+ "hypervisor",
 "libc",
 "log",
 "versionize",
@@ -270,15 +305,21 @@ dependencies = [

 [[package]]
 name = "fdt"
-version = "0.1.3"
+version = "0.1.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b643857cf70949306b81d7e92cb9d47add673868edac9863c4a49c42feaf3f1e"
+checksum = "964f5becd44d069dca0beea2b4bc05639ae7bf3b3f5369c295aff360bb57cca2"
+
+[[package]]
+name = "fnv"
+version = "1.0.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"

 [[package]]
 name = "gdbstub"
-version = "0.6.1"
+version = "0.6.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9fa2ca5d6b045de372cef3991f873389b421b4cabcfbe52f7787fae8b8b37906"
+checksum = "1c1f9371c87c11642ee94dcf92cb48b1484ba250b8e8bff3df71c28651f3f4e7"
 dependencies = [
 "bitflags",
 "cfg-if",
@@ -290,9 +331,9 @@ dependencies = [

 [[package]]
 name = "gdbstub_arch"
-version = "0.2.2"
+version = "0.2.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "51dc4b5718ac76d21e8605c0966dd32d80273b89b11e6cfef467b04e45934d37"
+checksum = "c24f469ba9556c5a063d6df35a8a338025fccf96ecae44f330a156b686f7a268"
 dependencies = [
 "gdbstub",
 "num-traits",
@@ -300,9 +341,9 @@ dependencies = [

 [[package]]
 name = "getrandom"
-version = "0.2.6"
+version = "0.2.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9be70c98951c83b8d2f8f60d7065fa6d5146873094452a1008da8c2f1e4205ad"
+checksum = "4eb1a864a501629691edf6c15a593b7a51eebaa1e8468e9ddc623de7c9b58ec6"
 dependencies = [
 "cfg-if",
 "libc",
@@ -317,9 +358,9 @@ checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574"

 [[package]]
 name = "hashbrown"
-version = "0.11.2"
+version = "0.12.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e"
+checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888"

 [[package]]
 name = "hermit-abi"
@@ -341,8 +382,8 @@ name = "hypervisor"
 version = "0.1.0"
 dependencies = [
 "anyhow",
+ "byteorder",
 "env_logger",
- "epoll",
 "iced-x86",
 "kvm-bindings",
 "kvm-ioctls",
@@ -351,8 +392,9 @@ dependencies = [
 "mshv-bindings",
 "mshv-ioctls",
 "serde",
- "serde_json",
+ "serde_with",
 "thiserror",
+ "vfio-ioctls",
 "vm-memory",
 "vmm-sys-util",
 ]
@@ -368,10 +410,16 @@ dependencies = [
 ]

 [[package]]
-name = "indexmap"
-version = "1.8.1"
+name = "ident_case"
+version = "1.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0f647032dfaa1f8b6dc29bd3edb7bbef4861b8b8007ebb118d6db284fd59f6ee"
+checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39"
+
+[[package]]
+name = "indexmap"
+version = "1.9.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "10a35a97730320ffe8e2d410b5d3b69279b98d2c14bdb8b70ea89ecf7888d41e"
 dependencies = [
 "autocfg",
 "hashbrown",
@@ -388,23 +436,14 @@ dependencies = [

 [[package]]
 name = "io-uring"
-version = "0.5.2"
+version = "0.5.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8d75829ed9377bab6c90039fe47b9d84caceb4b5063266142e21bcce6550cda8"
+checksum = "427496d698429d86ff947537dff68facb83fe753b7abea4fab4ac1c4b7a4b94b"
 dependencies = [
 "bitflags",
 "libc",
 ]

-[[package]]
-name = "ipnetwork"
-version = "0.18.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4088d739b183546b239688ddbc79891831df421773df95e236daf7867866d355"
-dependencies = [
- "serde",
-]
-
 [[package]]
 name = "ipnetwork"
 version = "0.19.0"
@@ -416,9 +455,9 @@ dependencies = [

 [[package]]
 name = "itoa"
-version = "1.0.2"
+version = "1.0.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "112c678d4050afce233f4f2852bb2eb519230b3cf12f33585275537d7e41578d"
+checksum = "6c8af84674fe1f223a982c933a0ee1086ac4d4052aa0fb8060c12c6ad838e754"

 [[package]]
 name = "kvm-bindings"
@@ -433,7 +472,7 @@ dependencies = [
 [[package]]
 name = "kvm-ioctls"
 version = "0.11.0"
-source = "git+https://github.com/rust-vmm/kvm-ioctls?branch=main#1e03e29cdfbb0cb108a98de7a78045a5a517f18e"
+source = "git+https://github.com/rust-vmm/kvm-ioctls?branch=main#ccf0bda07485b2433616de998c000aa5f04ce806"
 dependencies = [
 "kvm-bindings",
 "libc",
@@ -448,9 +487,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"

 [[package]]
 name = "libc"
-version = "0.2.126"
+version = "0.2.131"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "349d5a591cd28b49e1d1037471617a32ddcda5731b99419008085f72d5a53836"
+checksum = "04c3b4822ccebfa39c02fc03d1534441b22ead323fa0f48bb7ddd8e6ba076a40"

 [[package]]
 name = "libssh2-sys"
@@ -468,9 +507,9 @@ dependencies = [

 [[package]]
 name = "libz-sys"
-version = "1.1.6"
+version = "1.1.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "92e7e15d7610cce1d9752e137625f14e61a28cd45929b6e12e47b50fe154ee2e"
+checksum = "9702761c3935f8cc2f101793272e202c72b99da8f4224a19ddcf1279a6450bbf"
 dependencies = [
 "cc",
 "libc",
@@ -521,7 +560,7 @@ checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d"
 [[package]]
 name = "micro_http"
 version = "0.1.0"
-source = "git+https://github.com/firecracker-microvm/micro-http?branch=main#a730d86940081ad044cdfbc1285c1db6d3048392"
+source = "git+https://github.com/firecracker-microvm/micro-http?branch=main#863b0370ba7e57f7df5b908ada9e5b44809ccae9"
 dependencies = [
 "libc",
 "vmm-sys-util",
@@ -530,7 +569,7 @@ dependencies = [
 [[package]]
 name = "mshv-bindings"
 version = "0.1.0"
-source = "git+https://github.com/rust-vmm/mshv?branch=main#67e13faa36989a9226ad21c9ff3947e1d5738a54"
+source = "git+https://github.com/rust-vmm/mshv?branch=main#a4c9916124d771fb9d0ccd97423ffb72876baa83"
 dependencies = [
 "libc",
 "serde",
@@ -542,7 +581,7 @@ dependencies = [
 [[package]]
 name = "mshv-ioctls"
 version = "0.1.0"
-source = "git+https://github.com/rust-vmm/mshv?branch=main#67e13faa36989a9226ad21c9ff3947e1d5738a54"
+source = "git+https://github.com/rust-vmm/mshv?branch=main#a4c9916124d771fb9d0ccd97423ffb72876baa83"
 dependencies = [
 "libc",
 "mshv-bindings",
@@ -562,14 +601,16 @@ version = "0.1.0"
 dependencies = [
 "epoll",
 "getrandom",
- "lazy_static",
 "libc",
 "log",
 "net_gen",
+ "once_cell",
 "pnet",
+ "pnet_datalink",
 "rate_limiter",
 "serde",
 "serde_json",
+ "thiserror",
 "versionize",
 "versionize_derive",
 "virtio-bindings",
@@ -595,19 +636,25 @@ dependencies = [
 ]

 [[package]]
-name = "openssl-src"
-version = "111.20.0+1.1.1o"
+name = "once_cell"
+version = "1.13.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "92892c4f87d56e376e469ace79f1128fdaded07646ddf73aa0be4706ff712dec"
+checksum = "074864da206b4973b84eb91683020dbefd6a8c3f0f38e054d93954e891935e4e"
+
+[[package]]
+name = "openssl-src"
+version = "111.22.0+1.1.1q"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8f31f0d509d1c1ae9cada2f9539ff8f37933831fd5098879e482aa687d659853"
 dependencies = [
 "cc",
 ]

 [[package]]
 name = "openssl-sys"
-version = "0.9.73"
+version = "0.9.75"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9d5fd19fb3e0a8191c1e34935718976a3e70c112ab9a24af6d7cadccd9d90bc0"
+checksum = "e5f9bd0c2710541a3cda73d6f9ac4f1b240de4ae261065d309dbe73d9dceb42f"
 dependencies = [
 "autocfg",
 "cc",
@@ -623,9 +670,9 @@ version = "0.1.0"

 [[package]]
 name = "os_str_bytes"
-version = "6.0.1"
+version = "6.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "029d8d0b2f198229de29dca79676f2738ff952edf3fde542eb8bf94d8c21b435"
+checksum = "9ff7415e9ae3fff1225851df9e0d9e4e5479f947619774677a63572e55e80eff"

 [[package]]
 name = "parking_lot"
@@ -654,9 +701,9 @@ dependencies = [

 [[package]]
 name = "paste"
-version = "1.0.7"
+version = "1.0.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0c520e05135d6e763148b6426a837e239041653ba7becd2e538c076c738025fc"
+checksum = "9423e2b32f7a043629287a536f21951e8c6a82482d0acb1eeebfc90bc2225b22"

 [[package]]
 name = "pci"
@@ -702,34 +749,32 @@ checksum = "1df8c4ec4b0627e53bdf214615ad287367e482558cf84b109250b37464dc03ae"

 [[package]]
 name = "pnet"
-version = "0.30.0"
+version = "0.31.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d5cc57672f576f6b95370277fb445738d4887195c6cf4192bdf4f44697e2389b"
+checksum = "0caaf5b11fd907ff15cf14a4477bfabca4b37ab9e447a4f8dead969a59cdafad"
 dependencies = [
- "ipnetwork 0.19.0",
 "pnet_base",
 "pnet_datalink",
 "pnet_packet",
- "pnet_sys",
 "pnet_transport",
 ]

 [[package]]
 name = "pnet_base"
-version = "0.30.0"
+version = "0.31.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2e88341c6c842f89bdc7287f7b1e26b6fa64fa11c7ea3756971e6f18cd2510c4"
+checksum = "f9d3a993d49e5fd5d4d854d6999d4addca1f72d86c65adf224a36757161c02b6"
 dependencies = [
 "no-std-net",
 ]

 [[package]]
 name = "pnet_datalink"
-version = "0.30.0"
+version = "0.31.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bc6e55d71c51db73372db35bc54f43abd8460adff1c3a9b717804ca6416d5df2"
+checksum = "e466faf03a98ad27f6e15cd27a2b7cc89e73e640a43527742977bc503c37f8aa"
 dependencies = [
- "ipnetwork 0.18.0",
+ "ipnetwork",
 "libc",
 "pnet_base",
 "pnet_sys",
@@ -738,9 +783,9 @@ dependencies = [

 [[package]]
 name = "pnet_macros"
-version = "0.30.0"
+version = "0.31.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ebfcdc9c072966723026b3596a1f655fb8bbfe0142f9770f8d481aee4459d6b9"
+checksum = "48dd52a5211fac27e7acb14cfc9f30ae16ae0e956b7b779c8214c74559cef4c3"
 dependencies = [
 "proc-macro2",
 "quote",
@@ -750,18 +795,18 @@ dependencies = [

 [[package]]
 name = "pnet_macros_support"
-version = "0.30.0"
+version = "0.31.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bba532f5a4b320c029d89e612671fb621851b3b07e972c53850d34130033a5cd"
+checksum = "89de095dc7739349559913aed1ef6a11e73ceade4897dadc77c5e09de6740750"
 dependencies = [
 "pnet_base",
 ]

 [[package]]
 name = "pnet_packet"
-version = "0.30.0"
+version = "0.31.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7009716ac86091c1b6e2cdec95a2b028c880f516054c1ec11edd02f9f463cbde"
+checksum = "bc3b5111e697c39c8b9795b9fdccbc301ab696699e88b9ea5a4e4628978f495f"
 dependencies = [
 "glob",
 "pnet_base",
@@ -771,9 +816,9 @@ dependencies = [

 [[package]]
 name = "pnet_sys"
-version = "0.30.0"
+version = "0.31.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8e2a05efbc55c22f664c0ea475fbc4ffc4d09346aff9068438279d7e3d431f6f"
+checksum = "328e231f0add6d247d82421bf3790b4b33b39c8930637f428eef24c4c6a90805"
 dependencies = [
 "libc",
 "winapi",
@@ -781,9 +826,9 @@ dependencies = [

 [[package]]
 name = "pnet_transport"
-version = "0.30.0"
+version = "0.31.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "75b8ff06f37863f7590183f7044ab2e8d4dae991ecea0c791e3c6dd61ed2913d"
+checksum = "ff597185e6f1f5671b3122e4dba892a1c73e17c17e723d7669bd9299cbe7f124"
 dependencies = [
 "libc",
 "pnet_base",
@@ -793,9 +838,9 @@ dependencies = [

 [[package]]
 name = "proc-macro2"
-version = "1.0.39"
+version = "1.0.43"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c54b25569025b7fc9651de43004ae593a75ad88543b17178aa5e1b9c4f15f56f"
+checksum = "0a2ca2c61bc9f3d74d2886294ab7b9853abd9c1ad903a3ac7815c58989bb7bab"
 dependencies = [
 "unicode-ident",
 ]
@@ -813,9 +858,9 @@ dependencies = [

 [[package]]
 name = "quote"
-version = "1.0.18"
+version = "1.0.21"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a1feb54ed693b93a84e14094943b84b7c4eae204c512b7ccb95ab0c66d278ad1"
+checksum = "bbe448f377a7d6961e30f5955f9b8d106c3f5e449d493ee1b125c1d43c2b5179"
 dependencies = [
 "proc-macro2",
 ]
@@ -831,9 +876,9 @@ dependencies = [

 [[package]]
 name = "redox_syscall"
-version = "0.2.13"
+version = "0.2.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "62f25bc4c7e55e0b0b7a1d43fb893f4fa1361d0abe38b9ce4f323c2adfe6ef42"
+checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a"
 dependencies = [
 "bitflags",
 ]
@@ -851,9 +896,9 @@ dependencies = [

 [[package]]
 name = "regex"
-version = "1.5.6"
+version = "1.6.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d83f127d94bdbcda4c8cc2e50f6f84f4b611f69c902699ca385a39c3a75f9ff1"
+checksum = "4c4eb3267174b8c6c2f654116623910a0fef09c4753f8dd83db29c48a0df988b"
 dependencies = [
 "aho-corasick",
 "memchr",
@@ -862,15 +907,15 @@ dependencies = [

 [[package]]
 name = "regex-syntax"
-version = "0.6.26"
+version = "0.6.27"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "49b3de9ec5dc0a3417da371aab17d729997c15010e7fd24ff707773a33bddb64"
+checksum = "a3f87b73ce11b1619a3c6332f45341e0047173771e8b8b73f87bfeefb7b56244"

 [[package]]
 name = "remain"
-version = "0.2.3"
+version = "0.2.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0c35270ea384ac1762895831cc8acb96f171468e52cec82ed9186f9416209fa4"
+checksum = "06df529c0d271b27ac4a2c260f5ce2914b60bd44702cecec7b9f271adbdde23b"
 dependencies = [
 "proc-macro2",
 "quote",
@@ -888,9 +933,9 @@ dependencies = [

 [[package]]
 name = "ryu"
-version = "1.0.10"
+version = "1.0.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f3f6f92acf49d1b98f7a81226834412ada05458b7364277387724a237f062695"
+checksum = "4501abdff3ae82a1c1b477a17252eb69cee9e66eb915c1abaa4f44d873df9f09"

 [[package]]
 name = "scopeguard"
@@ -909,24 +954,24 @@ dependencies = [

 [[package]]
 name = "semver"
-version = "1.0.9"
+version = "1.0.13"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8cb243bdfdb5936c8dc3c45762a19d12ab4550cdc753bc247637d4ec35a040fd"
+checksum = "93f6841e709003d68bb2deee8c343572bf446003ec20a583e76f7b15cebf3711"

 [[package]]
 name = "serde"
-version = "1.0.137"
+version = "1.0.143"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "61ea8d54c77f8315140a05f4c7237403bf38b72704d031543aa1d16abbf517d1"
+checksum = "53e8e5d5b70924f74ff5c6d64d9a5acd91422117c60f48c4e07855238a254553"
 dependencies = [
 "serde_derive",
 ]

 [[package]]
 name = "serde_derive"
-version = "1.0.137"
+version = "1.0.143"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1f26faba0c3959972377d3b2d306ee9f71faee9714294e41bb777f83f88578be"
+checksum = "d3d8e8de557aee63c26b85b947f5e59b690d0454c753f3adeb5cd7835ab88391"
 dependencies = [
 "proc-macro2",
 "quote",
@@ -935,15 +980,37 @@ dependencies = [

 [[package]]
 name = "serde_json"
-version = "1.0.81"
+version = "1.0.83"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9b7ce2b32a1aed03c558dc61a5cd328f15aff2dbc17daad8fb8af04d2100e15c"
+checksum = "38dd04e3c8279e75b31ef29dbdceebfe5ad89f4d0937213c53f7d49d01b3d5a7"
 dependencies = [
 "itoa",
 "ryu",
 "serde",
 ]

+[[package]]
+name = "serde_with"
+version = "2.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "89df7a26519371a3cce44fbb914c2819c84d9b897890987fa3ab096491cc0ea8"
+dependencies = [
+ "serde",
+ "serde_with_macros",
+]
+
+[[package]]
+name = "serde_with_macros"
+version = "2.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "de337f322382fcdfbb21a014f7c224ee041a23785651db67b9827403178f698f"
+dependencies = [
+ "darling",
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
 [[package]]
 name = "signal-hook"
 version = "0.3.14"
@@ -965,9 +1032,9 @@ dependencies = [

 [[package]]
 name = "smallvec"
-version = "1.8.0"
+version = "1.9.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f2dd574626839106c320a323308629dcb1acfc96e32a8cba364ddc61ac23ee83"
+checksum = "2fd0db749597d91ff862fd1d55ea87f7855a744a8425a64695b6fca237d1dad1"

 [[package]]
 name = "ssh2"
@@ -995,9 +1062,9 @@ checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623"

 [[package]]
 name = "syn"
-version = "1.0.95"
+version = "1.0.99"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fbaf6116ab8924f39d52792136fb74fd60a80194cf1b1c6ffa6453eef1c3f942"
+checksum = "58dbef6ec655055e20b86b15a8cc6d439cca19b667537ac6a1369572d151ab13"
 dependencies = [
 "proc-macro2",
 "quote",
@@ -1041,8 +1108,8 @@ version = "0.1.0"
 dependencies = [
 "dirs",
 "epoll",
- "lazy_static",
 "libc",
+ "once_cell",
 "ssh2",
 "vmm-sys-util",
 "wait-timeout",
@@ -1059,18 +1126,18 @@ dependencies = [

 [[package]]
 name = "thiserror"
-version = "1.0.31"
+version = "1.0.32"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bd829fe32373d27f76265620b5309d0340cb8550f523c1dda251d6298069069a"
+checksum = "f5f6586b7f764adc0231f4c79be7b920e766bb2f3e51b3661cdb263828f19994"
 dependencies = [
 "thiserror-impl",
 ]

 [[package]]
 name = "thiserror-impl"
-version = "1.0.31"
+version = "1.0.32"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0396bc89e626244658bef819e22d0cc459e795a5ebe878e6ec336d1674a8d79a"
+checksum = "12bafc5b54507e0149cdf1b145a5d80ab80a90bcd9275df43d4fff68460f6c21"
 dependencies = [
 "proc-macro2",
 "quote",
@@ -1079,9 +1146,9 @@ dependencies = [

 [[package]]
 name = "unicode-ident"
-version = "1.0.0"
+version = "1.0.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d22af068fba1eb5edcb4aea19d382b2a3deb4c8f9d475c589b6ada9e0fd493ee"
+checksum = "c4f5b37a154999a8f3f98cc23a628d850e154479cd94decf3414696e12e31aaf"

 [[package]]
 name = "unicode-xid"
@@ -1091,9 +1158,9 @@ checksum = "957e51f3646910546462e67d5f7599b9e4fb8acdd304b087a6494730f9eebf04"

 [[package]]
 name = "uuid"
-version = "1.0.0"
+version = "1.1.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8cfcd319456c4d6ea10087ed423473267e1a071f3bc0aa89f80d60997843c6f0"
+checksum = "dd6469f4314d5f1ffec476e05f17cc9a78bc7a27a6a857842170bdf8d6f98d2f"
 dependencies = [
 "getrandom",
 ]
@@ -1143,7 +1210,7 @@ dependencies = [
 [[package]]
 name = "vfio-ioctls"
 version = "0.1.0"
-source = "git+https://github.com/rust-vmm/vfio?branch=main#f75a77c1ab6349c105bc1462a65508726b4c2e0f"
+source = "git+https://github.com/rust-vmm/vfio?branch=main#c52e88ac551183bd3784745a7143db45abdb05db"
 dependencies = [
 "byteorder",
 "kvm-bindings",
@@ -1162,7 +1229,6 @@ dependencies = [
 name = "vfio_user"
 version = "0.1.0"
 dependencies = [
- "anyhow",
 "libc",
 "log",
 "serde",
@@ -1185,7 +1251,6 @@ dependencies = [
 "remain",
 "thiserror",
 "uuid",
- "vmm-sys-util",
 ]

 [[package]]
@@ -1202,8 +1267,9 @@ dependencies = [

 [[package]]
 name = "vhost-user-backend"
-version = "0.3.0"
-source = "git+https://github.com/rust-vmm/vhost-user-backend?rev=14f58eda14076e973704d4f904850be1146fbb05#14f58eda14076e973704d4f904850be1146fbb05"
+version = "0.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "558ac5ca9569fb03f518b2bdd17606809ffdc894b619b92d30df6c40c33d15f3"
 dependencies = [
 "libc",
 "log",
@@ -1229,6 +1295,7 @@ dependencies = [
 "vhost",
 "vhost-user-backend",
 "virtio-bindings",
+ "virtio-queue",
 "vm-memory",
 "vmm-sys-util",
 ]
@@ -1293,11 +1360,12 @@ dependencies = [

 [[package]]
 name = "virtio-queue"
-version = "0.3.0"
+version = "0.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "88f2d73c184c18f8acc32dab77fcb6e3af92d53262538d3a68aa474810d6863c"
+checksum = "b4f59652909f276e6edd8bf36e9f106480b2202f5f046717b3de14f1b4072a28"
 dependencies = [
 "log",
+ "virtio-bindings",
 "vm-memory",
 "vmm-sys-util",
 ]
@@ -1318,7 +1386,6 @@ dependencies = [
 "anyhow",
 "hypervisor",
 "serde",
- "serde_json",
 "thiserror",
 "vfio-ioctls",
 "vm-memory",
@@ -1359,7 +1426,6 @@ name = "vm-virtio"
 version = "0.1.0"
 dependencies = [
 "log",
- "virtio-bindings",
 "virtio-queue",
 "vm-memory",
 ]
@@ -1381,12 +1447,12 @@ dependencies = [
 "gdbstub",
 "gdbstub_arch",
 "hypervisor",
- "lazy_static",
 "libc",
 "linux-loader",
 "log",
 "micro_http",
 "net_util",
+ "once_cell",
 "option_parser",
 "pci",
 "qcow",
@@ -1413,9 +1479,9 @@ dependencies = [

 [[package]]
 name = "vmm-sys-util"
-version = "0.9.0"
+version = "0.10.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "733537bded03aaa93543f785ae997727b30d1d9f4a03b7861d23290474242e11"
+checksum = "08604d7be03eb26e33b3cee3ed4aef2bf550b305d1cca60e84da5d28d3790b62"
 dependencies = [
 "bitflags",
 "libc",
@@ -1434,9 +1500,9 @@ dependencies = [

 [[package]]
 name = "wasi"
-version = "0.10.2+wasi-snapshot-preview1"
+version = "0.11.0+wasi-snapshot-preview1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6"
+checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"

 [[package]]
 name = "winapi"
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "cloud-hypervisor"
-version = "24.0.0"
+version = "26.0.0"
 authors = ["The Cloud Hypervisor Authors"]
 edition = "2021"
 default-run = "cloud-hypervisor"
@@ -16,25 +16,25 @@ rust-version = "1.60"
 lto = true

 [dependencies]
-anyhow = "1.0.57"
+anyhow = "1.0.62"
 api_client = { path = "api_client" }
-clap = { version = "3.1.18", features = ["wrap_help","cargo"] }
+clap = { version = "3.2.17", features = ["wrap_help","cargo"] }
 epoll = "4.3.1"
 event_monitor = { path = "event_monitor" }
 hypervisor = { path = "hypervisor" }
-libc = "0.2.126"
+libc = "0.2.131"
 log = { version = "0.4.17", features = ["std"] }
 option_parser = { path = "option_parser" }
 seccompiler = "0.2.0"
-serde_json = "1.0.81"
+serde_json = "1.0.83"
 signal-hook = "0.3.14"
-thiserror = "1.0.31"
+thiserror = "1.0.32"
 vmm = { path = "vmm" }
-vmm-sys-util = "0.9.0"
+vmm-sys-util = "0.10.0"
 vm-memory = "0.8.0"

 [build-dependencies]
-clap = { version = "3.1.18", features = ["cargo"] }
+clap = { version = "3.2.17", features = ["cargo"] }

 # List of patched crates
 [patch.crates-io]
@@ -44,9 +44,9 @@ versionize_derive = { git = "https://github.com/cloud-hypervisor/versionize_deri

 [dev-dependencies]
 dirs = "4.0.0"
-lazy_static= "1.4.0"
 net_util = { path = "net_util" }
-serde_json = "1.0.81"
+once_cell = "1.13.1"
+serde_json = "1.0.83"
 test_infra = { path = "test_infra" }
 wait-timeout = "0.2.0"

@@ -58,6 +58,7 @@ amx = ["vmm/amx"]
 cmos = ["vmm/cmos"]
 fwdebug = ["vmm/fwdebug"]
 gdb = ["vmm/gdb"]
+guest_debug = ["vmm/guest_debug"]
 kvm = ["vmm/kvm"]
 mshv = ["vmm/mshv"]
 tdx = ["vmm/tdx"]
--- a/192
+++ b/192
@@ -23,6 +23,19 @@ pipeline{
 						}
 					}
 				}
+				stage ('Check for fuzzer files only changes') {
+					when {
+						expression {
+							return fuzzFileOnly()
+						}
+					}
+					steps {
+						script {
+							runWorkers = false
+							echo "Fuzzer cargo files only changes, no need to run the CI"
+						}
+					}
+				}
 				stage ('Check for RFC/WIP builds') {
 					when {
  						changeRequest comparator: 'REGEXP', title: '.*(rfc|RFC|wip|WIP).*'
@@ -43,7 +56,7 @@ pipeline{
 		stage ('Build') {
 			parallel {
 				stage ('Worker build') {
-					agent { node { label 'focal' } }
+					agent { node { label 'jammy' } }
 					when {
 						beforeAgent true
 						expression {
@@ -90,6 +103,9 @@ pipeline{
 							return runWorkers
 						}
 					}
+					environment {
+        					AZURE_CONNECTION_STRING = credentials('46b4e7d6-315f-4cc1-8333-b58780863b9b')
+					}
 					stages {
 						stage ('Checkout') {
 							steps {
@@ -110,6 +126,42 @@ pipeline{
 								sh "scripts/dev_cli.sh tests --integration --libc musl"
 							}
 						}
+						stage ('Install azure-cli') {
+							steps {
+								installAzureCli("bionic", "arm64")
+							}
+						}
+						stage ('Download Windows image') {
+							steps {
+								sh '''#!/bin/bash -x
+									IMG_BASENAME=windows-11-iot-enterprise-aarch64.raw
+									IMG_PATH=$HOME/workloads/$IMG_BASENAME
+									IMG_GZ_PATH=$HOME/workloads/$IMG_BASENAME.gz
+									IMG_GZ_BLOB_NAME=windows-11-iot-enterprise-aarch64-9-min.raw.gz
+									cp "scripts/$IMG_BASENAME.sha1" "$HOME/workloads/"
+									pushd "$HOME/workloads"
+									if sha1sum "$IMG_BASENAME.sha1" --check; then
+										exit
+									fi
+									popd
+									mkdir -p "$HOME/workloads"
+									az storage blob download \
+										--container-name private-images \
+										--file "$IMG_GZ_PATH" \
+										--name "$IMG_GZ_BLOB_NAME" \
+										--connection-string "$AZURE_CONNECTION_STRING"
+									gzip -d $IMG_GZ_PATH
+								'''
+							}
+						}
+						stage ('Run Windows guest integration tests') {
+							options {
+								timeout(time: 1, unit: 'HOURS')
+							}
+							steps {
+								sh "scripts/dev_cli.sh tests --integration-windows --libc musl"
+							}
+						}
 					}
 					post {
 						always {
@@ -119,7 +171,7 @@ pipeline{
 					}
 				}
 				stage ('Worker build (musl)') {
-					agent { node { label 'focal' } }
+					agent { node { label 'jammy' } }
 					when {
 						beforeAgent true
 						expression {
@@ -153,8 +205,90 @@ pipeline{
 						}
 					}
 				}
+				stage ('Worker build SGX') {
+					agent { node { label 'bionic-sgx' } }
+					when {
+						beforeAgent true
+						allOf {
+							branch 'main'
+							expression {
+								return runWorkers
+							}
+						}
+					}
+					stages {
+						stage ('Checkout') {
+							steps {
+								checkout scm
+							}
+						}
+						stage ('Run SGX integration tests') {
+							options {
+								timeout(time: 1, unit: 'HOURS')
+							}
+							steps {
+								sh "scripts/dev_cli.sh tests --integration-sgx"
+							}
+						}
+						stage ('Run SGX integration tests for musl') {
+							options {
+								timeout(time: 1, unit: 'HOURS')
+							}
+							steps {
+								sh "scripts/dev_cli.sh tests --integration-sgx --libc musl"
+							}
+						}
+					}
+					post {
+						always {
+							sh "sudo chown -R jenkins.jenkins ${WORKSPACE}"
+							deleteDir()
+						}
+					}
+				}
+				stage ('Worker build VFIO') {
+					agent { node { label 'bionic-vfio' } }
+					when {
+						beforeAgent true
+						allOf {
+							branch 'main'
+							expression {
+								return runWorkers
+							}
+						}
+					}
+					stages {
+						stage ('Checkout') {
+							steps {
+								checkout scm
+							}
+						}
+						stage ('Run VFIO integration tests') {
+							options {
+								timeout(time: 1, unit: 'HOURS')
+							}
+							steps {
+								sh "scripts/dev_cli.sh tests --integration-vfio"
+							}
+						}
+						stage ('Run VFIO integration tests for musl') {
+							options {
+								timeout(time: 1, unit: 'HOURS')
+							}
+							steps {
+								sh "scripts/dev_cli.sh tests --integration-vfio --libc musl"
+							}
+						}
+					}
+					post {
+						always {
+							sh "sudo chown -R jenkins.jenkins ${WORKSPACE}"
+							deleteDir()
+						}
+					}
+				}
 				stage ('Worker build - Windows guest') {
-					agent { node { label 'focal' } }
+					agent { node { label 'jammy' } }
 					when {
 						beforeAgent true
 						expression {
@@ -172,7 +306,7 @@ pipeline{
 						}
 						stage ('Install azure-cli') {
 							steps {
-								installAzureCli()
+								installAzureCli("jammy", "amd64")
 							}
 						}
 						stage ('Download assets') {
@@ -200,7 +334,7 @@ pipeline{
 					}
 				}
 				stage ('Worker build - Live Migration') {
-					agent { node { label 'focal-small' } }
+					agent { node { label 'jammy' } }
 					when {
 						beforeAgent true
 						expression {
@@ -233,6 +367,39 @@ pipeline{
 						}
 					}
 				}
+				stage ('Worker build - Metrics') {
+					agent { node { label 'focal-metrics' } }
+					when {
+						branch 'main'
+						beforeAgent true
+						expression {
+							return runWorkers
+						}
+					}
+					environment {
+						METRICS_PUBLISH_KEY = credentials('52e0945f-ce7a-43d1-87af-67d1d87cc40f')
+					}
+					stages {
+						stage ('Checkout') {
+							steps {
+								checkout scm
+							}
+						}
+						stage ('Run metrics tests') {
+							options {
+								timeout(time: 1, unit: 'HOURS')
+							}
+							steps {
+								sh 'scripts/dev_cli.sh tests --metrics -- -- --report-file /root/workloads/metrics.json'
+							}
+						}
+						stage ('Upload metrics report') {
+							steps {
+								sh 'curl -X PUT https://cloud-hypervisor-metrics.azurewebsites.net/api/publishmetrics -H "x-functions-key: $METRICS_PUBLISH_KEY" -T ~/workloads/metrics.json'
+							}
+						}
+					}
+				}
 			}
 		}
 	}
@@ -269,10 +436,10 @@ def cancelPreviousBuilds() {
 		}
 }

-def installAzureCli() {
+def installAzureCli(distro, arch) {
 	sh "sudo apt install -y ca-certificates curl apt-transport-https lsb-release gnupg"
 	sh "curl -sL https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor | sudo tee /etc/apt/trusted.gpg.d/microsoft.gpg > /dev/null"
-	sh "echo \"deb [arch=amd64] https://packages.microsoft.com/repos/azure-cli/ focal main\" | sudo tee /etc/apt/sources.list.d/azure-cli.list"
+	sh "echo \"deb [arch=${arch}] https://packages.microsoft.com/repos/azure-cli/ ${distro} main\" | sudo tee /etc/apt/sources.list.d/azure-cli.list"
 	sh "sudo apt update"
 	sh "sudo apt install -y azure-cli"
 }
@@ -287,3 +454,14 @@ def boolean docsFileOnly() {
        script: "git diff --name-only origin/${env.CHANGE_TARGET}... | grep -v '\\.md'"
    ) != 0
 }
+
+def boolean fuzzFileOnly() {
+    if (env.CHANGE_TARGET == null) {
+        return false;
+    }
+
+    return sh(
+        returnStatus: true,
+        script: "git diff --name-only origin/${env.CHANGE_TARGET}... | grep -v -E 'fuzz/'"
+    ) != 0
+}
--- a/README.md
+++ b/README.md
@@ -88,8 +88,8 @@ Hypervisor. Here, all the steps are based on Ubuntu, for other Linux
 distributions please replace the package manager and package name.

 ```shell
-# Install build-essential, git, and qemu-img
-$ sudo apt install git build-essential qemu-img
+# Install build-essential, git, and qemu-utils
+$ sudo apt install git build-essential qemu-utils
 # Install rust tool chain
 $ curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh
 # If you want to build statically linked binary please add musl target
@@ -166,7 +166,7 @@ cloud image. Here we will use a Ubuntu image:
 $ pushd $CLOUDH
 $ wget https://cloud-images.ubuntu.com/focal/current/focal-server-cloudimg-amd64.img
 $ qemu-img convert -p -f qcow2 -O raw focal-server-cloudimg-amd64.img focal-server-cloudimg-amd64.raw
-$ wget https://github.com/cloud-hypervisor/rust-hypervisor-firmware/releases/download/0.4.0/hypervisor-fw
+$ wget https://github.com/cloud-hypervisor/rust-hypervisor-firmware/releases/download/0.4.1/hypervisor-fw
 $ popd
 ```

--- a/acpi_tables/src/aml.rs
+++ b/acpi_tables/src/aml.rs
@@ -68,7 +68,7 @@ impl Aml for Path {
        };

        for part in self.name_parts.clone().iter_mut() {
-            bytes.extend_from_slice(&part.to_vec());
+            bytes.extend_from_slice(part.as_ref());
        }
    }
 }
--- a/acpi_tables/src/sdt.rs
+++ b/acpi_tables/src/sdt.rs
@@ -4,6 +4,7 @@
 //

 #[repr(packed)]
+#[derive(Clone, Copy)]
 pub struct GenericAddress {
    pub address_space_id: u8,
    pub register_bit_width: u8,
--- a/api_client/Cargo.toml
+++ b/api_client/Cargo.toml
@@ -5,4 +5,4 @@ authors = ["The Cloud Hypervisor Authors"]
 edition = "2021"

 [dependencies]
-vmm-sys-util = "0.9.0"
+vmm-sys-util = "0.10.0"
--- a/api_client/src/lib.rs
+++ b/api_client/src/lib.rs
@@ -101,6 +101,10 @@ fn parse_http_response(socket: &mut dyn Read) -> Result<Option<String>, Error> {
    loop {
        let mut bytes = vec![0; 256];
        let count = socket.read(&mut bytes).map_err(Error::Socket)?;
+        // If the return value is 0, the peer has performed an orderly shutdown.
+        if count == 0 {
+            break;
+        }
        res.push_str(std::str::from_utf8(&bytes[0..count]).unwrap());

        // End of headers
@@ -127,7 +131,7 @@ fn parse_http_response(socket: &mut dyn Read) -> Result<Option<String>, Error> {
            }
        }
    }
-    let body_string = content_length.and(Some(String::from(&res[body_offset.unwrap()..])));
+    let body_string = content_length.and(body_offset.map(|o| String::from(&res[o..])));
    let status_code = get_status_code(&res)?;

    if status_code.is_server_error() {
--- a/arch/Cargo.toml
+++ b/arch/Cargo.toml
@@ -9,21 +9,21 @@ default = []
 tdx = []

 [dependencies]
-acpi_tables = { path = "../acpi_tables" }
-anyhow = "1.0.57"
+anyhow = "1.0.62"
 byteorder = "1.4.3"
 hypervisor = { path = "../hypervisor" }
-libc = "0.2.126"
+libc = "0.2.131"
 linux-loader = { version = "0.4.0", features = ["elf", "bzimage", "pe"] }
 log = "0.4.17"
-serde = { version = "1.0.137", features = ["rc", "derive"] }
-thiserror = "1.0.31"
+serde = { version = "1.0.143", features = ["rc", "derive"] }
+thiserror = "1.0.32"
+uuid = "1.1.2"
 versionize = "0.1.6"
 versionize_derive = "0.1.4"
 vm-memory = { version = "0.8.0", features = ["backend-mmap", "backend-bitmap"] }
 vm-migration = { path = "../vm-migration" }
-vmm-sys-util = { version = "0.9.0", features = ["with-serde"] }
+vmm-sys-util = { version = "0.10.0", features = ["with-serde"] }

 [target.'cfg(target_arch = "aarch64")'.dependencies]
-fdt_parser = { version = "0.1.3", package = 'fdt'}
+fdt_parser = { version = "0.1.4", package = "fdt" }
 vm-fdt = { git = "https://github.com/rust-vmm/vm-fdt", branch = "main" }
--- a/arch/src/aarch64/fdt.rs
+++ b/arch/src/aarch64/fdt.rs
@@ -8,17 +8,18 @@

 use crate::{NumaNodes, PciSpaceInfo};
 use byteorder::{BigEndian, ByteOrder};
+use hypervisor::arch::aarch64::gic::Vgic;
 use std::cmp;
 use std::collections::HashMap;
 use std::ffi::CStr;
 use std::fmt::Debug;
 use std::result;
 use std::str;
+use std::sync::{Arc, Mutex};

 use super::super::DeviceType;
 use super::super::GuestMemoryMmap;
 use super::super::InitramfsConfig;
-use super::gic::GicDevice;
 use super::layout::{
    IRQ_BASE, MEM_32BIT_DEVICES_SIZE, MEM_32BIT_DEVICES_START, MEM_PCI_IO_SIZE, MEM_PCI_IO_START,
    PCI_HIGH_BASE, PCI_MMIO_CONFIG_SIZE_PER_SEGMENT,
@@ -90,7 +91,7 @@ pub fn create_fdt<T: DeviceInfoForFdt + Clone + Debug, S: ::std::hash::BuildHash
    vcpu_mpidr: Vec<u64>,
    vcpu_topology: Option<(u8, u8, u8)>,
    device_info: &HashMap<(DeviceType, String), T, S>,
-    gic_device: &dyn GicDevice,
+    gic_device: &Arc<Mutex<dyn Vgic>>,
    initrd: &Option<InitramfsConfig>,
    pci_space_info: &[PciSpaceInfo],
    numa_nodes: &NumaNodes,
@@ -315,18 +316,18 @@ fn create_chosen_node(
    Ok(())
 }

-fn create_gic_node(fdt: &mut FdtWriter, gic_device: &dyn GicDevice) -> FdtWriterResult<()> {
-    let gic_reg_prop = gic_device.device_properties();
+fn create_gic_node(fdt: &mut FdtWriter, gic_device: &Arc<Mutex<dyn Vgic>>) -> FdtWriterResult<()> {
+    let gic_reg_prop = gic_device.lock().unwrap().device_properties();

    let intc_node = fdt.begin_node("intc")?;

-    fdt.property_string("compatible", gic_device.fdt_compatibility())?;
+    fdt.property_string("compatible", gic_device.lock().unwrap().fdt_compatibility())?;
    fdt.property_null("interrupt-controller")?;
    // "interrupt-cells" field specifies the number of cells needed to encode an
    // interrupt source. The type shall be a <u32> and the value shall be 3 if no PPI affinity description
    // is required.
    fdt.property_u32("#interrupt-cells", 3)?;
-    fdt.property_array_u64("reg", gic_reg_prop)?;
+    fdt.property_array_u64("reg", &gic_reg_prop)?;
    fdt.property_u32("phandle", GIC_PHANDLE)?;
    fdt.property_u32("#address-cells", 2)?;
    fdt.property_u32("#size-cells", 2)?;
@@ -334,18 +335,18 @@ fn create_gic_node(fdt: &mut FdtWriter, gic_device: &dyn GicDevice) -> FdtWriter

    let gic_intr_prop = [
        GIC_FDT_IRQ_TYPE_PPI,
-        gic_device.fdt_maint_irq(),
+        gic_device.lock().unwrap().fdt_maint_irq(),
        IRQ_TYPE_LEVEL_HI,
    ];
    fdt.property_array_u32("interrupts", &gic_intr_prop)?;

-    if gic_device.msi_compatible() {
+    if gic_device.lock().unwrap().msi_compatible() {
        let msic_node = fdt.begin_node("msic")?;
-        fdt.property_string("compatible", gic_device.msi_compatibility())?;
+        fdt.property_string("compatible", gic_device.lock().unwrap().msi_compatibility())?;
        fdt.property_null("msi-controller")?;
        fdt.property_u32("phandle", MSI_PHANDLE)?;
-        let msi_reg_prop = gic_device.msi_properties();
-        fdt.property_array_u64("reg", msi_reg_prop)?;
+        let msi_reg_prop = gic_device.lock().unwrap().msi_properties();
+        fdt.property_array_u64("reg", &msi_reg_prop)?;
        fdt.end_node(msic_node)?;
    }

--- a/arch/src/aarch64/gic/gicv3.rs
+++ b/arch/src/aarch64/gic/gicv3.rs
@@ -1,262 +0,0 @@
-// Copyright 2021 Arm Limited (or its affiliates). All rights reserved.
-// Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
-// SPDX-License-Identifier: Apache-2.0
-//
-// This file implements the GicV3 device.
-
-pub mod kvm {
-    use crate::aarch64::gic::dist_regs::{get_dist_regs, read_ctlr, set_dist_regs, write_ctlr};
-    use crate::aarch64::gic::icc_regs::{get_icc_regs, set_icc_regs};
-    use crate::aarch64::gic::kvm::{save_pending_tables, KvmGicDevice};
-    use crate::aarch64::gic::redist_regs::{
-        construct_gicr_typers, get_redist_regs, set_redist_regs,
-    };
-    use crate::aarch64::gic::GicDevice;
-    use crate::layout;
-    use anyhow::anyhow;
-    use hypervisor::kvm::kvm_bindings;
-    use hypervisor::CpuState;
-    use std::any::Any;
-    use std::convert::TryInto;
-    use std::sync::Arc;
-    use std::{boxed::Box, result};
-    use versionize::{VersionMap, Versionize, VersionizeResult};
-    use versionize_derive::Versionize;
-    use vm_memory::Address;
-    use vm_migration::{
-        Migratable, MigratableError, Pausable, Snapshot, Snapshottable, Transportable,
-        VersionMapped,
-    };
-
-    /// Errors thrown while saving/restoring the GICv3.
-    #[derive(Debug)]
-    pub enum Error {
-        /// Error in saving RDIST pending tables into guest RAM.
-        SavePendingTables(crate::aarch64::gic::Error),
-        /// Error in saving GIC distributor registers.
-        SaveDistributorRegisters(crate::aarch64::gic::Error),
-        /// Error in restoring GIC distributor registers.
-        RestoreDistributorRegisters(crate::aarch64::gic::Error),
-        /// Error in saving GIC distributor control registers.
-        SaveDistributorCtrlRegisters(crate::aarch64::gic::Error),
-        /// Error in restoring GIC distributor control registers.
-        RestoreDistributorCtrlRegisters(crate::aarch64::gic::Error),
-        /// Error in saving GIC redistributor registers.
-        SaveRedistributorRegisters(crate::aarch64::gic::Error),
-        /// Error in restoring GIC redistributor registers.
-        RestoreRedistributorRegisters(crate::aarch64::gic::Error),
-        /// Error in saving GIC CPU interface registers.
-        SaveIccRegisters(crate::aarch64::gic::Error),
-        /// Error in restoring GIC CPU interface registers.
-        RestoreIccRegisters(crate::aarch64::gic::Error),
-    }
-
-    type Result<T> = result::Result<T, Error>;
-
-    pub struct KvmGicV3 {
-        /// The hypervisor agnostic device for the GicV3
-        device: Arc<dyn hypervisor::Device>,
-
-        /// Vector holding values of GICR_TYPER for each vCPU
-        gicr_typers: Vec<u64>,
-
-        /// GIC device properties, to be used for setting up the fdt entry
-        properties: [u64; 4],
-
-        /// Number of CPUs handled by the device
-        vcpu_count: u64,
-    }
-
-    #[derive(Versionize)]
-    pub struct Gicv3State {
-        dist: Vec<u32>,
-        rdist: Vec<u32>,
-        icc: Vec<u32>,
-        // special register that enables interrupts and affinity routing
-        gicd_ctlr: u32,
-    }
-
-    impl VersionMapped for Gicv3State {}
-
-    impl KvmGicV3 {
-        // Device trees specific constants
-        pub const ARCH_GIC_V3_MAINT_IRQ: u32 = 9;
-
-        /// Get the address of the GIC distributor.
-        pub fn get_dist_addr() -> u64 {
-            layout::GIC_V3_DIST_START.raw_value()
-        }
-
-        /// Get the size of the GIC distributor.
-        pub fn get_dist_size() -> u64 {
-            layout::GIC_V3_DIST_SIZE
-        }
-
-        /// Get the address of the GIC redistributors.
-        pub fn get_redists_addr(vcpu_count: u64) -> u64 {
-            KvmGicV3::get_dist_addr() - KvmGicV3::get_redists_size(vcpu_count)
-        }
-
-        /// Get the size of the GIC redistributors.
-        pub fn get_redists_size(vcpu_count: u64) -> u64 {
-            vcpu_count * layout::GIC_V3_REDIST_SIZE
-        }
-
-        /// Save the state of GIC.
-        fn state(&self, gicr_typers: &[u64]) -> Result<Gicv3State> {
-            let gicd_ctlr =
-                read_ctlr(self.device()).map_err(Error::SaveDistributorCtrlRegisters)?;
-
-            let dist_state =
-                get_dist_regs(self.device()).map_err(Error::SaveDistributorRegisters)?;
-
-            let rdist_state = get_redist_regs(self.device(), gicr_typers)
-                .map_err(Error::SaveRedistributorRegisters)?;
-
-            let icc_state =
-                get_icc_regs(self.device(), gicr_typers).map_err(Error::SaveIccRegisters)?;
-
-            Ok(Gicv3State {
-                dist: dist_state,
-                rdist: rdist_state,
-                icc: icc_state,
-                gicd_ctlr,
-            })
-        }
-
-        /// Restore the state of GIC.
-        fn set_state(&mut self, gicr_typers: &[u64], state: &Gicv3State) -> Result<()> {
-            write_ctlr(self.device(), state.gicd_ctlr)
-                .map_err(Error::RestoreDistributorCtrlRegisters)?;
-
-            set_dist_regs(self.device(), &state.dist)
-                .map_err(Error::RestoreDistributorRegisters)?;
-
-            set_redist_regs(self.device(), gicr_typers, &state.rdist)
-                .map_err(Error::RestoreRedistributorRegisters)?;
-
-            set_icc_regs(self.device(), gicr_typers, &state.icc)
-                .map_err(Error::RestoreIccRegisters)?;
-
-            Ok(())
-        }
-    }
-
-    impl GicDevice for KvmGicV3 {
-        fn device(&self) -> &Arc<dyn hypervisor::Device> {
-            &self.device
-        }
-
-        fn fdt_compatibility(&self) -> &str {
-            "arm,gic-v3"
-        }
-
-        fn fdt_maint_irq(&self) -> u32 {
-            KvmGicV3::ARCH_GIC_V3_MAINT_IRQ
-        }
-
-        fn device_properties(&self) -> &[u64] {
-            &self.properties
-        }
-
-        fn vcpu_count(&self) -> u64 {
-            self.vcpu_count
-        }
-
-        fn set_its_device(&mut self, _its_device: Option<Arc<dyn hypervisor::Device>>) {}
-
-        fn set_gicr_typers(&mut self, vcpu_states: &[CpuState]) {
-            let gicr_typers = construct_gicr_typers(vcpu_states);
-            self.gicr_typers = gicr_typers;
-        }
-
-        fn as_any_concrete_mut(&mut self) -> &mut dyn Any {
-            self
-        }
-    }
-
-    impl KvmGicDevice for KvmGicV3 {
-        fn version() -> u32 {
-            kvm_bindings::kvm_device_type_KVM_DEV_TYPE_ARM_VGIC_V3
-        }
-
-        fn create_device(
-            device: Arc<dyn hypervisor::Device>,
-            vcpu_count: u64,
-        ) -> Box<dyn GicDevice> {
-            Box::new(KvmGicV3 {
-                device,
-                gicr_typers: vec![0; vcpu_count.try_into().unwrap()],
-                properties: [
-                    KvmGicV3::get_dist_addr(),
-                    KvmGicV3::get_dist_size(),
-                    KvmGicV3::get_redists_addr(vcpu_count),
-                    KvmGicV3::get_redists_size(vcpu_count),
-                ],
-                vcpu_count,
-            })
-        }
-
-        fn init_device_attributes(
-            _vm: &Arc<dyn hypervisor::Vm>,
-            gic_device: &mut dyn GicDevice,
-        ) -> crate::aarch64::gic::Result<()> {
-            /* Setting up the distributor attribute.
-             We are placing the GIC below 1GB so we need to substract the size of the distributor.
-            */
-            Self::set_device_attribute(
-                gic_device.device(),
-                kvm_bindings::KVM_DEV_ARM_VGIC_GRP_ADDR,
-                u64::from(kvm_bindings::KVM_VGIC_V3_ADDR_TYPE_DIST),
-                &KvmGicV3::get_dist_addr() as *const u64 as u64,
-                0,
-            )?;
-
-            /* Setting up the redistributors' attribute.
-            We are calculating here the start of the redistributors address. We have one per CPU.
-            */
-            Self::set_device_attribute(
-                gic_device.device(),
-                kvm_bindings::KVM_DEV_ARM_VGIC_GRP_ADDR,
-                u64::from(kvm_bindings::KVM_VGIC_V3_ADDR_TYPE_REDIST),
-                &KvmGicV3::get_redists_addr(gic_device.vcpu_count()) as *const u64 as u64,
-                0,
-            )?;
-
-            Ok(())
-        }
-    }
-
-    pub const GIC_V3_SNAPSHOT_ID: &str = "gic-v3";
-    impl Snapshottable for KvmGicV3 {
-        fn id(&self) -> String {
-            GIC_V3_SNAPSHOT_ID.to_string()
-        }
-
-        fn snapshot(&mut self) -> std::result::Result<Snapshot, MigratableError> {
-            let gicr_typers = self.gicr_typers.clone();
-            Snapshot::new_from_versioned_state(&self.id(), &self.state(&gicr_typers).unwrap())
-        }
-
-        fn restore(&mut self, snapshot: Snapshot) -> std::result::Result<(), MigratableError> {
-            let gicr_typers = self.gicr_typers.clone();
-            self.set_state(&gicr_typers, &snapshot.to_versioned_state(&self.id())?)
-                .map_err(|e| {
-                    MigratableError::Restore(anyhow!("Could not restore GICv3 state {:?}", e))
-                })
-        }
-    }
-
-    impl Pausable for KvmGicV3 {
-        fn pause(&mut self) -> std::result::Result<(), MigratableError> {
-            // Flush redistributors pending tables to guest RAM.
-            save_pending_tables(self.device()).map_err(|e| {
-                MigratableError::Pause(anyhow!("Could not save GICv3 GIC pending tables {:?}", e))
-            })?;
-
-            Ok(())
-        }
-    }
-    impl Transportable for KvmGicV3 {}
-    impl Migratable for KvmGicV3 {}
-}
--- a/arch/src/aarch64/gic/gicv3_its.rs
+++ b/arch/src/aarch64/gic/gicv3_its.rs
@@ -1,514 +0,0 @@
-// Copyright 2021 Arm Limited (or its affiliates). All rights reserved.
-// SPDX-License-Identifier: Apache-2.0
-//
-// This file implements the GicV3 device with ITS (Virtual Interrupt Translation Service).
-
-pub mod kvm {
-    use crate::aarch64::gic::dist_regs::{get_dist_regs, read_ctlr, set_dist_regs, write_ctlr};
-    use crate::aarch64::gic::icc_regs::{get_icc_regs, set_icc_regs};
-    use crate::aarch64::gic::redist_regs::{
-        construct_gicr_typers, get_redist_regs, set_redist_regs,
-    };
-
-    use crate::aarch64::gic::gicv3::kvm::KvmGicV3;
-    use crate::aarch64::gic::kvm::{save_pending_tables, KvmGicDevice};
-    use crate::aarch64::gic::GicDevice;
-    use crate::layout;
-    use anyhow::anyhow;
-    use hypervisor::kvm::kvm_bindings;
-    use hypervisor::CpuState;
-    use std::any::Any;
-    use std::convert::TryInto;
-    use std::sync::Arc;
-    use std::{boxed::Box, result};
-    use versionize::{VersionMap, Versionize, VersionizeResult};
-    use versionize_derive::Versionize;
-    use vm_migration::{
-        Migratable, MigratableError, Pausable, Snapshot, Snapshottable, Transportable,
-        VersionMapped,
-    };
-
-    const GITS_CTLR: u32 = 0x0000;
-    const GITS_IIDR: u32 = 0x0004;
-    const GITS_CBASER: u32 = 0x0080;
-    const GITS_CWRITER: u32 = 0x0088;
-    const GITS_CREADR: u32 = 0x0090;
-    const GITS_BASER: u32 = 0x0100;
-
-    /// Errors thrown while saving/restoring the GICv3ITS.
-    #[derive(Debug)]
-    pub enum Error {
-        /// Error in saving RDIST pending tables into guest RAM.
-        SavePendingTables(crate::aarch64::gic::Error),
-        /// Error in saving GIC distributor registers.
-        SaveDistributorRegisters(crate::aarch64::gic::Error),
-        /// Error in restoring GIC distributor registers.
-        RestoreDistributorRegisters(crate::aarch64::gic::Error),
-        /// Error in saving GIC distributor control registers.
-        SaveDistributorCtrlRegisters(crate::aarch64::gic::Error),
-        /// Error in restoring GIC distributor control registers.
-        RestoreDistributorCtrlRegisters(crate::aarch64::gic::Error),
-        /// Error in saving GIC redistributor registers.
-        SaveRedistributorRegisters(crate::aarch64::gic::Error),
-        /// Error in restoring GIC redistributor registers.
-        RestoreRedistributorRegisters(crate::aarch64::gic::Error),
-        /// Error in saving GIC CPU interface registers.
-        SaveIccRegisters(crate::aarch64::gic::Error),
-        /// Error in restoring GIC CPU interface registers.
-        RestoreIccRegisters(crate::aarch64::gic::Error),
-        /// Error in saving GICv3ITS IIDR register.
-        SaveITSIIDR(crate::aarch64::gic::Error),
-        /// Error in restoring GICv3ITS IIDR register.
-        RestoreITSIIDR(crate::aarch64::gic::Error),
-        /// Error in saving GICv3ITS CBASER register.
-        SaveITSCBASER(crate::aarch64::gic::Error),
-        /// Error in restoring GICv3ITS CBASER register.
-        RestoreITSCBASER(crate::aarch64::gic::Error),
-        /// Error in saving GICv3ITS CREADR register.
-        SaveITSCREADR(crate::aarch64::gic::Error),
-        /// Error in restoring GICv3ITS CREADR register.
-        RestoreITSCREADR(crate::aarch64::gic::Error),
-        /// Error in saving GICv3ITS CWRITER register.
-        SaveITSCWRITER(crate::aarch64::gic::Error),
-        /// Error in restoring GICv3ITS CWRITER register.
-        RestoreITSCWRITER(crate::aarch64::gic::Error),
-        /// Error in saving GICv3ITS BASER register.
-        SaveITSBASER(crate::aarch64::gic::Error),
-        /// Error in restoring GICv3ITS BASER register.
-        RestoreITSBASER(crate::aarch64::gic::Error),
-        /// Error in saving GICv3ITS CTLR register.
-        SaveITSCTLR(crate::aarch64::gic::Error),
-        /// Error in restoring GICv3ITS CTLR register.
-        RestoreITSCTLR(crate::aarch64::gic::Error),
-        /// Error in saving GICv3ITS restore tables.
-        SaveITSTables(crate::aarch64::gic::Error),
-        /// Error in restoring GICv3ITS restore tables.
-        RestoreITSTables(crate::aarch64::gic::Error),
-    }
-
-    type Result<T> = result::Result<T, Error>;
-
-    /// Access an ITS device attribute.
-    ///
-    /// This is a helper function to get/set the ITS device attribute depending
-    /// the bool parameter `set` provided.
-    pub fn gicv3_its_attr_access(
-        its_device: &Arc<dyn hypervisor::Device>,
-        group: u32,
-        attr: u32,
-        val: &u64,
-        set: bool,
-    ) -> crate::aarch64::gic::Result<()> {
-        let mut gicv3_its_attr = kvm_bindings::kvm_device_attr {
-            group,
-            attr: attr as u64,
-            addr: val as *const u64 as u64,
-            flags: 0,
-        };
-        if set {
-            its_device
-                .set_device_attr(&gicv3_its_attr)
-                .map_err(crate::aarch64::gic::Error::SetDeviceAttribute)?;
-        } else {
-            its_device
-                .get_device_attr(&mut gicv3_its_attr)
-                .map_err(crate::aarch64::gic::Error::GetDeviceAttribute)?;
-        }
-        Ok(())
-    }
-
-    /// Function that saves/restores ITS tables into guest RAM.
-    ///
-    /// The tables get flushed to guest RAM whenever the VM gets stopped.
-    pub fn gicv3_its_tables_access(
-        its_device: &Arc<dyn hypervisor::Device>,
-        save: bool,
-    ) -> crate::aarch64::gic::Result<()> {
-        let attr = if save {
-            u64::from(kvm_bindings::KVM_DEV_ARM_ITS_SAVE_TABLES)
-        } else {
-            u64::from(kvm_bindings::KVM_DEV_ARM_ITS_RESTORE_TABLES)
-        };
-
-        let init_gic_attr = kvm_bindings::kvm_device_attr {
-            group: kvm_bindings::KVM_DEV_ARM_VGIC_GRP_CTRL,
-            attr,
-            addr: 0,
-            flags: 0,
-        };
-        its_device
-            .set_device_attr(&init_gic_attr)
-            .map_err(crate::aarch64::gic::Error::SetDeviceAttribute)
-    }
-
-    pub struct KvmGicV3Its {
-        /// The hypervisor agnostic device for the GicV3
-        device: Arc<dyn hypervisor::Device>,
-
-        /// The hypervisor agnostic device for the Its device
-        its_device: Option<Arc<dyn hypervisor::Device>>,
-
-        /// Vector holding values of GICR_TYPER for each vCPU
-        gicr_typers: Vec<u64>,
-
-        /// GIC device properties, to be used for setting up the fdt entry
-        gic_properties: [u64; 4],
-
-        /// MSI device properties, to be used for setting up the fdt entry
-        msi_properties: [u64; 2],
-
-        /// Number of CPUs handled by the device
-        vcpu_count: u64,
-    }
-
-    #[derive(Versionize)]
-    pub struct Gicv3ItsState {
-        dist: Vec<u32>,
-        rdist: Vec<u32>,
-        icc: Vec<u32>,
-        // special register that enables interrupts and affinity routing
-        gicd_ctlr: u32,
-        its_ctlr: u64,
-        its_iidr: u64,
-        its_cbaser: u64,
-        its_cwriter: u64,
-        its_creadr: u64,
-        its_baser: [u64; 8],
-    }
-
-    impl VersionMapped for Gicv3ItsState {}
-
-    impl KvmGicV3Its {
-        fn get_msi_size() -> u64 {
-            layout::GIC_V3_ITS_SIZE
-        }
-
-        fn get_msi_addr(vcpu_count: u64) -> u64 {
-            KvmGicV3::get_redists_addr(vcpu_count) - KvmGicV3Its::get_msi_size()
-        }
-
-        /// Save the state of GICv3ITS.
-        fn state(&self, gicr_typers: &[u64]) -> Result<Gicv3ItsState> {
-            let gicd_ctlr =
-                read_ctlr(self.device()).map_err(Error::SaveDistributorCtrlRegisters)?;
-
-            let dist_state =
-                get_dist_regs(self.device()).map_err(Error::SaveDistributorRegisters)?;
-
-            let rdist_state = get_redist_regs(self.device(), gicr_typers)
-                .map_err(Error::SaveRedistributorRegisters)?;
-
-            let icc_state =
-                get_icc_regs(self.device(), gicr_typers).map_err(Error::SaveIccRegisters)?;
-
-            let its_baser_state: [u64; 8] = [0; 8];
-            for i in 0..8 {
-                gicv3_its_attr_access(
-                    self.its_device().unwrap(),
-                    kvm_bindings::KVM_DEV_ARM_VGIC_GRP_ITS_REGS,
-                    GITS_BASER + i * 8,
-                    &its_baser_state[i as usize],
-                    false,
-                )
-                .map_err(Error::SaveITSBASER)?;
-            }
-
-            let its_ctlr_state: u64 = 0;
-            gicv3_its_attr_access(
-                self.its_device().unwrap(),
-                kvm_bindings::KVM_DEV_ARM_VGIC_GRP_ITS_REGS,
-                GITS_CTLR,
-                &its_ctlr_state,
-                false,
-            )
-            .map_err(Error::SaveITSCTLR)?;
-
-            let its_cbaser_state: u64 = 0;
-            gicv3_its_attr_access(
-                self.its_device().unwrap(),
-                kvm_bindings::KVM_DEV_ARM_VGIC_GRP_ITS_REGS,
-                GITS_CBASER,
-                &its_cbaser_state,
-                false,
-            )
-            .map_err(Error::SaveITSCBASER)?;
-
-            let its_creadr_state: u64 = 0;
-            gicv3_its_attr_access(
-                self.its_device().unwrap(),
-                kvm_bindings::KVM_DEV_ARM_VGIC_GRP_ITS_REGS,
-                GITS_CREADR,
-                &its_creadr_state,
-                false,
-            )
-            .map_err(Error::SaveITSCREADR)?;
-
-            let its_cwriter_state: u64 = 0;
-            gicv3_its_attr_access(
-                self.its_device().unwrap(),
-                kvm_bindings::KVM_DEV_ARM_VGIC_GRP_ITS_REGS,
-                GITS_CWRITER,
-                &its_cwriter_state,
-                false,
-            )
-            .map_err(Error::SaveITSCWRITER)?;
-
-            let its_iidr_state: u64 = 0;
-            gicv3_its_attr_access(
-                self.its_device().unwrap(),
-                kvm_bindings::KVM_DEV_ARM_VGIC_GRP_ITS_REGS,
-                GITS_IIDR,
-                &its_iidr_state,
-                false,
-            )
-            .map_err(Error::SaveITSIIDR)?;
-
-            Ok(Gicv3ItsState {
-                dist: dist_state,
-                rdist: rdist_state,
-                icc: icc_state,
-                gicd_ctlr,
-                its_ctlr: its_ctlr_state,
-                its_iidr: its_iidr_state,
-                its_cbaser: its_cbaser_state,
-                its_cwriter: its_cwriter_state,
-                its_creadr: its_creadr_state,
-                its_baser: its_baser_state,
-            })
-        }
-
-        /// Restore the state of GICv3ITS.
-        fn set_state(&mut self, gicr_typers: &[u64], state: &Gicv3ItsState) -> Result<()> {
-            write_ctlr(self.device(), state.gicd_ctlr)
-                .map_err(Error::RestoreDistributorCtrlRegisters)?;
-
-            set_dist_regs(self.device(), &state.dist)
-                .map_err(Error::RestoreDistributorRegisters)?;
-
-            set_redist_regs(self.device(), gicr_typers, &state.rdist)
-                .map_err(Error::RestoreRedistributorRegisters)?;
-
-            set_icc_regs(self.device(), gicr_typers, &state.icc)
-                .map_err(Error::RestoreIccRegisters)?;
-
-            //Restore GICv3ITS registers
-            gicv3_its_attr_access(
-                self.its_device().unwrap(),
-                kvm_bindings::KVM_DEV_ARM_VGIC_GRP_ITS_REGS,
-                GITS_IIDR,
-                &state.its_iidr,
-                true,
-            )
-            .map_err(Error::RestoreITSIIDR)?;
-
-            gicv3_its_attr_access(
-                self.its_device().unwrap(),
-                kvm_bindings::KVM_DEV_ARM_VGIC_GRP_ITS_REGS,
-                GITS_CBASER,
-                &state.its_cbaser,
-                true,
-            )
-            .map_err(Error::RestoreITSCBASER)?;
-
-            gicv3_its_attr_access(
-                self.its_device().unwrap(),
-                kvm_bindings::KVM_DEV_ARM_VGIC_GRP_ITS_REGS,
-                GITS_CREADR,
-                &state.its_creadr,
-                true,
-            )
-            .map_err(Error::RestoreITSCREADR)?;
-
-            gicv3_its_attr_access(
-                self.its_device().unwrap(),
-                kvm_bindings::KVM_DEV_ARM_VGIC_GRP_ITS_REGS,
-                GITS_CWRITER,
-                &state.its_cwriter,
-                true,
-            )
-            .map_err(Error::RestoreITSCWRITER)?;
-
-            for i in 0..8 {
-                gicv3_its_attr_access(
-                    self.its_device().unwrap(),
-                    kvm_bindings::KVM_DEV_ARM_VGIC_GRP_ITS_REGS,
-                    GITS_BASER + i * 8,
-                    &state.its_baser[i as usize],
-                    true,
-                )
-                .map_err(Error::RestoreITSBASER)?;
-            }
-
-            // Restore ITS tables
-            gicv3_its_tables_access(self.its_device().unwrap(), false)
-                .map_err(Error::RestoreITSTables)?;
-
-            gicv3_its_attr_access(
-                self.its_device().unwrap(),
-                kvm_bindings::KVM_DEV_ARM_VGIC_GRP_ITS_REGS,
-                GITS_CTLR,
-                &state.its_ctlr,
-                true,
-            )
-            .map_err(Error::RestoreITSCTLR)?;
-
-            Ok(())
-        }
-    }
-
-    impl GicDevice for KvmGicV3Its {
-        fn device(&self) -> &Arc<dyn hypervisor::Device> {
-            &self.device
-        }
-
-        fn its_device(&self) -> Option<&Arc<dyn hypervisor::Device>> {
-            self.its_device.as_ref()
-        }
-
-        fn fdt_compatibility(&self) -> &str {
-            "arm,gic-v3"
-        }
-
-        fn msi_compatible(&self) -> bool {
-            true
-        }
-
-        fn msi_compatibility(&self) -> &str {
-            "arm,gic-v3-its"
-        }
-
-        fn fdt_maint_irq(&self) -> u32 {
-            KvmGicV3::ARCH_GIC_V3_MAINT_IRQ
-        }
-
-        fn msi_properties(&self) -> &[u64] {
-            &self.msi_properties
-        }
-
-        fn device_properties(&self) -> &[u64] {
-            &self.gic_properties
-        }
-
-        fn vcpu_count(&self) -> u64 {
-            self.vcpu_count
-        }
-
-        fn set_its_device(&mut self, its_device: Option<Arc<dyn hypervisor::Device>>) {
-            self.its_device = its_device;
-        }
-
-        fn set_gicr_typers(&mut self, vcpu_states: &[CpuState]) {
-            let gicr_typers = construct_gicr_typers(vcpu_states);
-            self.gicr_typers = gicr_typers;
-        }
-
-        fn as_any_concrete_mut(&mut self) -> &mut dyn Any {
-            self
-        }
-    }
-
-    impl KvmGicDevice for KvmGicV3Its {
-        fn version() -> u32 {
-            KvmGicV3::version()
-        }
-
-        fn create_device(
-            device: Arc<dyn hypervisor::Device>,
-            vcpu_count: u64,
-        ) -> Box<dyn GicDevice> {
-            Box::new(KvmGicV3Its {
-                device,
-                its_device: None,
-                gicr_typers: vec![0; vcpu_count.try_into().unwrap()],
-                gic_properties: [
-                    KvmGicV3::get_dist_addr(),
-                    KvmGicV3::get_dist_size(),
-                    KvmGicV3::get_redists_addr(vcpu_count),
-                    KvmGicV3::get_redists_size(vcpu_count),
-                ],
-                msi_properties: [
-                    KvmGicV3Its::get_msi_addr(vcpu_count),
-                    KvmGicV3Its::get_msi_size(),
-                ],
-                vcpu_count,
-            })
-        }
-
-        fn init_device_attributes(
-            vm: &Arc<dyn hypervisor::Vm>,
-            gic_device: &mut dyn GicDevice,
-        ) -> crate::aarch64::gic::Result<()> {
-            KvmGicV3::init_device_attributes(vm, gic_device)?;
-
-            let mut its_device = kvm_bindings::kvm_create_device {
-                type_: kvm_bindings::kvm_device_type_KVM_DEV_TYPE_ARM_VGIC_ITS,
-                fd: 0,
-                flags: 0,
-            };
-
-            let its_fd = vm
-                .create_device(&mut its_device)
-                .map_err(crate::aarch64::gic::Error::CreateGic)?;
-
-            Self::set_device_attribute(
-                &its_fd,
-                kvm_bindings::KVM_DEV_ARM_VGIC_GRP_ADDR,
-                u64::from(kvm_bindings::KVM_VGIC_ITS_ADDR_TYPE),
-                &KvmGicV3Its::get_msi_addr(gic_device.vcpu_count()) as *const u64 as u64,
-                0,
-            )?;
-
-            Self::set_device_attribute(
-                &its_fd,
-                kvm_bindings::KVM_DEV_ARM_VGIC_GRP_CTRL,
-                u64::from(kvm_bindings::KVM_DEV_ARM_VGIC_CTRL_INIT),
-                0,
-                0,
-            )?;
-
-            gic_device.set_its_device(Some(its_fd));
-
-            Ok(())
-        }
-    }
-
-    pub const GIC_V3_ITS_SNAPSHOT_ID: &str = "gic-v3-its";
-    impl Snapshottable for KvmGicV3Its {
-        fn id(&self) -> String {
-            GIC_V3_ITS_SNAPSHOT_ID.to_string()
-        }
-
-        fn snapshot(&mut self) -> std::result::Result<Snapshot, MigratableError> {
-            let gicr_typers = self.gicr_typers.clone();
-            Snapshot::new_from_versioned_state(&self.id(), &self.state(&gicr_typers).unwrap())
-        }
-
-        fn restore(&mut self, snapshot: Snapshot) -> std::result::Result<(), MigratableError> {
-            let gicr_typers = self.gicr_typers.clone();
-            self.set_state(&gicr_typers, &snapshot.to_versioned_state(&self.id())?)
-                .map_err(|e| {
-                    MigratableError::Restore(anyhow!("Could not restore GICv3ITS state {:?}", e))
-                })
-        }
-    }
-
-    impl Pausable for KvmGicV3Its {
-        fn pause(&mut self) -> std::result::Result<(), MigratableError> {
-            // Flush redistributors pending tables to guest RAM.
-            save_pending_tables(self.device()).map_err(|e| {
-                MigratableError::Pause(anyhow!(
-                    "Could not save GICv3ITS GIC pending tables {:?}",
-                    e
-                ))
-            })?;
-            // Flush ITS tables to guest RAM.
-            gicv3_its_tables_access(self.its_device().unwrap(), true).map_err(|e| {
-                MigratableError::Pause(anyhow!("Could not save GICv3ITS ITS tables {:?}", e))
-            })?;
-
-            Ok(())
-        }
-    }
-    impl Transportable for KvmGicV3Its {}
-    impl Migratable for KvmGicV3Its {}
-}
--- a/arch/src/aarch64/gic/mod.rs
+++ b/arch/src/aarch64/gic/mod.rs
@@ -1,220 +0,0 @@
-// Copyright 2021 Arm Limited (or its affiliates). All rights reserved.
-// Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
-// SPDX-License-Identifier: Apache-2.0
-
-pub mod dist_regs;
-pub mod gicv3;
-pub mod gicv3_its;
-pub mod icc_regs;
-pub mod redist_regs;
-
-pub use self::dist_regs::{get_dist_regs, read_ctlr, set_dist_regs, write_ctlr};
-pub use self::icc_regs::{get_icc_regs, set_icc_regs};
-pub use self::redist_regs::{get_redist_regs, set_redist_regs};
-use hypervisor::CpuState;
-use std::any::Any;
-use std::result;
-use std::sync::Arc;
-
-/// Errors thrown while setting up the GIC.
-#[derive(Debug)]
-pub enum Error {
-    /// Error while calling KVM ioctl for setting up the global interrupt controller.
-    CreateGic(hypervisor::HypervisorVmError),
-    /// Error while setting device attributes for the GIC.
-    SetDeviceAttribute(hypervisor::HypervisorDeviceError),
-    /// Error while getting device attributes for the GIC.
-    GetDeviceAttribute(hypervisor::HypervisorDeviceError),
-}
-type Result<T> = result::Result<T, Error>;
-
-pub trait GicDevice: Send {
-    /// Returns the hypervisor agnostic Device of the GIC device
-    fn device(&self) -> &Arc<dyn hypervisor::Device>;
-
-    /// Returns the hypervisor agnostic Device of the ITS device
-    fn its_device(&self) -> Option<&Arc<dyn hypervisor::Device>> {
-        None
-    }
-
-    /// Returns the fdt compatibility property of the device
-    fn fdt_compatibility(&self) -> &str;
-
-    /// Returns the maint_irq fdt property of the device
-    fn fdt_maint_irq(&self) -> u32;
-
-    /// Returns an array with GIC device properties
-    fn device_properties(&self) -> &[u64];
-
-    /// Returns the number of vCPUs this GIC handles
-    fn vcpu_count(&self) -> u64;
-
-    /// Returns whether the GIC device is MSI compatible or not
-    fn msi_compatible(&self) -> bool {
-        false
-    }
-
-    /// Returns the MSI compatibility property of the device
-    fn msi_compatibility(&self) -> &str {
-        ""
-    }
-
-    /// Returns the MSI reg property of the device
-    fn msi_properties(&self) -> &[u64] {
-        &[]
-    }
-
-    fn set_its_device(&mut self, its_device: Option<Arc<dyn hypervisor::Device>>);
-
-    /// Get the values of GICR_TYPER for each vCPU.
-    fn set_gicr_typers(&mut self, vcpu_states: &[CpuState]);
-
-    /// Downcast the trait object to its concrete type.
-    fn as_any_concrete_mut(&mut self) -> &mut dyn Any;
-}
-
-pub mod kvm {
-    use super::GicDevice;
-    use super::Result;
-    use crate::aarch64::gic::gicv3_its::kvm::KvmGicV3Its;
-    use crate::layout;
-    use hypervisor::kvm::kvm_bindings;
-    use std::boxed::Box;
-    use std::sync::Arc;
-
-    /// Trait for GIC devices.
-    pub trait KvmGicDevice: Send + Sync + GicDevice {
-        /// Returns the GIC version of the device
-        fn version() -> u32;
-
-        /// Create the GIC device object
-        fn create_device(
-            device: Arc<dyn hypervisor::Device>,
-            vcpu_count: u64,
-        ) -> Box<dyn GicDevice>;
-
-        /// Setup the device-specific attributes
-        fn init_device_attributes(
-            vm: &Arc<dyn hypervisor::Vm>,
-            gic_device: &mut dyn GicDevice,
-        ) -> Result<()>;
-
-        /// Initialize a GIC device
-        fn init_device(vm: &Arc<dyn hypervisor::Vm>) -> Result<Arc<dyn hypervisor::Device>> {
-            let mut gic_device = kvm_bindings::kvm_create_device {
-                type_: Self::version(),
-                fd: 0,
-                flags: 0,
-            };
-
-            vm.create_device(&mut gic_device)
-                .map_err(super::Error::CreateGic)
-        }
-
-        /// Set a GIC device attribute
-        fn set_device_attribute(
-            device: &Arc<dyn hypervisor::Device>,
-            group: u32,
-            attr: u64,
-            addr: u64,
-            flags: u32,
-        ) -> Result<()> {
-            let attr = kvm_bindings::kvm_device_attr {
-                flags,
-                group,
-                attr,
-                addr,
-            };
-            device
-                .set_device_attr(&attr)
-                .map_err(super::Error::SetDeviceAttribute)?;
-
-            Ok(())
-        }
-
-        /// Get a GIC device attribute
-        fn get_device_attribute(
-            device: &Arc<dyn hypervisor::Device>,
-            group: u32,
-            attr: u64,
-            addr: u64,
-            flags: u32,
-        ) -> Result<()> {
-            let mut attr = kvm_bindings::kvm_device_attr {
-                flags,
-                group,
-                attr,
-                addr,
-            };
-            device
-                .get_device_attr(&mut attr)
-                .map_err(super::Error::GetDeviceAttribute)?;
-
-            Ok(())
-        }
-
-        /// Finalize the setup of a GIC device
-        fn finalize_device(gic_device: &dyn GicDevice) -> Result<()> {
-            /* We need to tell the kernel how many irqs to support with this vgic.
-             * See the `layout` module for details.
-             */
-            let nr_irqs: u32 = layout::IRQ_NUM;
-            let nr_irqs_ptr = &nr_irqs as *const u32;
-            Self::set_device_attribute(
-                gic_device.device(),
-                kvm_bindings::KVM_DEV_ARM_VGIC_GRP_NR_IRQS,
-                0,
-                nr_irqs_ptr as u64,
-                0,
-            )?;
-
-            /* Finalize the GIC.
-             * See https://code.woboq.org/linux/linux/virt/kvm/arm/vgic/vgic-kvm-device.c.html#211.
-             */
-            Self::set_device_attribute(
-                gic_device.device(),
-                kvm_bindings::KVM_DEV_ARM_VGIC_GRP_CTRL,
-                u64::from(kvm_bindings::KVM_DEV_ARM_VGIC_CTRL_INIT),
-                0,
-                0,
-            )?;
-
-            Ok(())
-        }
-
-        /// Method to initialize the GIC device
-        #[allow(clippy::new_ret_no_self)]
-        fn new(vm: &Arc<dyn hypervisor::Vm>, vcpu_count: u64) -> Result<Box<dyn GicDevice>> {
-            let vgic_fd = Self::init_device(vm)?;
-
-            let mut device = Self::create_device(vgic_fd, vcpu_count);
-
-            Self::init_device_attributes(vm, &mut *device)?;
-
-            Self::finalize_device(&*device)?;
-
-            Ok(device)
-        }
-    }
-
-    /// Create a GICv3-ITS device.
-    ///
-    pub fn create_gic(vm: &Arc<dyn hypervisor::Vm>, vcpu_count: u64) -> Result<Box<dyn GicDevice>> {
-        debug!("creating a GICv3-ITS");
-        KvmGicV3Its::new(vm, vcpu_count)
-    }
-
-    /// Function that saves RDIST pending tables into guest RAM.
-    ///
-    /// The tables get flushed to guest RAM whenever the VM gets stopped.
-    pub fn save_pending_tables(gic: &Arc<dyn hypervisor::Device>) -> Result<()> {
-        let init_gic_attr = kvm_bindings::kvm_device_attr {
-            group: kvm_bindings::KVM_DEV_ARM_VGIC_GRP_CTRL,
-            attr: u64::from(kvm_bindings::KVM_DEV_ARM_VGIC_SAVE_PENDING_TABLES),
-            addr: 0,
-            flags: 0,
-        };
-        gic.set_device_attr(&init_gic_attr)
-            .map_err(super::Error::SetDeviceAttribute)
-    }
-}
--- a/arch/src/aarch64/mod.rs
+++ b/arch/src/aarch64/mod.rs
@@ -4,23 +4,21 @@

 /// Module for the flattened device tree.
 pub mod fdt;
-/// Module for the global interrupt controller configuration.
-pub mod gic;
 /// Layout for this aarch64 system.
 pub mod layout;
-/// Logic for configuring aarch64 registers.
+/// Module for system registers definition
 pub mod regs;
 /// Module for loading UEFI binary.
 pub mod uefi;

 pub use self::fdt::DeviceInfoForFdt;
 use crate::{DeviceType, GuestMemoryMmap, NumaNodes, PciSpaceInfo, RegionType};
-use gic::GicDevice;
+use hypervisor::arch::aarch64::gic::Vgic;
 use log::{log_enabled, Level};
 use std::collections::HashMap;
 use std::convert::TryInto;
 use std::fmt::Debug;
-use std::sync::Arc;
+use std::sync::{Arc, Mutex};
 use vm_memory::{Address, GuestAddress, GuestMemory, GuestUsize};

 /// Errors thrown while configuring aarch64 system.
@@ -33,13 +31,13 @@ pub enum Error {
    WriteFdtToMemory(fdt::Error),

    /// Failed to create a GIC.
-    SetupGic(gic::Error),
+    SetupGic,

    /// Failed to compute the initramfs address.
    InitramfsAddress,

    /// Error configuring the general purpose registers
-    RegsConfiguration(regs::Error),
+    RegsConfiguration(hypervisor::HypervisorCpuError),

    /// Error configuring the MPIDR register
    VcpuRegMpidr(hypervisor::HypervisorCpuError),
@@ -69,11 +67,17 @@ pub fn configure_vcpu(
    kernel_entry_point: Option<EntryPoint>,
 ) -> super::Result<u64> {
    if let Some(kernel_entry_point) = kernel_entry_point {
-        regs::setup_regs(vcpu, id, kernel_entry_point.entry_addr.raw_value())
-            .map_err(Error::RegsConfiguration)?;
+        vcpu.setup_regs(
+            id,
+            kernel_entry_point.entry_addr.raw_value(),
+            super::layout::FDT_START.raw_value(),
+        )
+        .map_err(Error::RegsConfiguration)?;
    }

-    let mpidr = vcpu.read_mpidr().map_err(Error::VcpuRegMpidr)?;
+    let mpidr = vcpu
+        .get_sys_reg(regs::MPIDR_EL1)
+        .map_err(Error::VcpuRegMpidr)?;
    Ok(mpidr)
 }

@@ -143,7 +147,7 @@ pub fn configure_system<T: DeviceInfoForFdt + Clone + Debug, S: ::std::hash::Bui
    initrd: &Option<super::InitramfsConfig>,
    pci_space_info: &[PciSpaceInfo],
    virtio_iommu_bdf: Option<u32>,
-    gic_device: &dyn GicDevice,
+    gic_device: &Arc<Mutex<dyn Vgic>>,
    numa_nodes: &NumaNodes,
    pmu_supported: bool,
 ) -> super::Result<()> {
--- a/arch/src/aarch64/regs.rs
+++ b/arch/src/aarch64/regs.rs
@@ -1,75 +1,41 @@
-// Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// Copyright 2022 Arm Limited (or its affiliates). All rights reserved.
 // SPDX-License-Identifier: Apache-2.0
-//
-// Portions Copyright 2017 The Chromium OS Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the THIRD-PARTY file.

-use hypervisor::kvm::kvm_bindings::{
-    kvm_regs, user_pt_regs, KVM_REG_ARM64, KVM_REG_ARM_CORE, KVM_REG_SIZE_U64,
-};
-use hypervisor::{arm64_core_reg_id, offset__of};
-use std::sync::Arc;
-use std::{mem, result};
-use vm_memory::Address;
-
-/// Errors thrown while setting aarch64 registers.
-#[derive(Debug)]
-pub enum Error {
-    /// Failed to set core register (PC, PSTATE or general purpose ones).
-    SetCoreRegister(hypervisor::HypervisorCpuError),
-    /// Failed to get a system register.
-    GetSysRegister(hypervisor::HypervisorCpuError),
-}
-type Result<T> = result::Result<T, Error>;
-
-#[allow(non_upper_case_globals)]
-// PSR (Processor State Register) bits.
-// Taken from arch/arm64/include/uapi/asm/ptrace.h.
-const PSR_MODE_EL1h: u64 = 0x0000_0005;
-const PSR_F_BIT: u64 = 0x0000_0040;
-const PSR_I_BIT: u64 = 0x0000_0080;
-const PSR_A_BIT: u64 = 0x0000_0100;
-const PSR_D_BIT: u64 = 0x0000_0200;
-// Taken from arch/arm64/kvm/inject_fault.c.
-const PSTATE_FAULT_BITS_64: u64 = PSR_MODE_EL1h | PSR_A_BIT | PSR_F_BIT | PSR_I_BIT | PSR_D_BIT;
-
-/// Configure core registers for a given CPU.
 ///
-/// # Arguments
+/// AArch64 system register encoding:
+/// See https://developer.arm.com/documentation/ddi0487 (chapter D12)
 ///
-/// * `vcpu` - Structure for the VCPU that holds the VCPU's fd.
-/// * `cpu_id` - Index of current vcpu.
-/// * `boot_ip` - Starting instruction pointer.
-/// * `mem` - Reserved DRAM for current VM.
-pub fn setup_regs(vcpu: &Arc<dyn hypervisor::Vcpu>, cpu_id: u8, boot_ip: u64) -> Result<()> {
-    let kreg_off = offset__of!(kvm_regs, regs);
+///   31      22  21 20 19 18 16 15 12 11  8 7   5 4  0
+///  +----------+---+-----+-----+-----+-----+-----+----+
+///  |1101010100| L | op0 | op1 | CRn | CRm | op2 | Rt |
+///  +----------+---+-----+-----+-----+-----+-----+----+
+///
+/// Notes:
+/// - L and Rt are reserved as implementation defined fields, ignored.
+///
+const SYSREG_HEAD: u32 = 0b1101010100u32 << 22;
+const SYSREG_OP0_SHIFT: u32 = 19;
+const SYSREG_OP0_MASK: u32 = 0b11u32 << 19;
+const SYSREG_OP1_SHIFT: u32 = 16;
+const SYSREG_OP1_MASK: u32 = 0b111u32 << 16;
+const SYSREG_CRN_SHIFT: u32 = 12;
+const SYSREG_CRN_MASK: u32 = 0b1111u32 << 12;
+const SYSREG_CRM_SHIFT: u32 = 8;
+const SYSREG_CRM_MASK: u32 = 0b1111u32 << 8;
+const SYSREG_OP2_SHIFT: u32 = 5;
+const SYSREG_OP2_MASK: u32 = 0b111u32 << 5;

-    // Get the register index of the PSTATE (Processor State) register.
-    let pstate = offset__of!(user_pt_regs, pstate) + kreg_off;
-    vcpu.set_reg(
-        arm64_core_reg_id!(KVM_REG_SIZE_U64, pstate),
-        PSTATE_FAULT_BITS_64,
-    )
-    .map_err(Error::SetCoreRegister)?;
-
-    // Other vCPUs are powered off initially awaiting PSCI wakeup.
-    if cpu_id == 0 {
-        // Setting the PC (Processor Counter) to the current program address (kernel address).
-        let pc = offset__of!(user_pt_regs, pc) + kreg_off;
-        vcpu.set_reg(arm64_core_reg_id!(KVM_REG_SIZE_U64, pc), boot_ip as u64)
-            .map_err(Error::SetCoreRegister)?;
-
-        // Last mandatory thing to set -> the address pointing to the FDT (also called DTB).
-        // "The device tree blob (dtb) must be placed on an 8-byte boundary and must
-        // not exceed 2 megabytes in size." -> https://www.kernel.org/doc/Documentation/arm64/booting.txt.
-        // We are choosing to place it the end of DRAM. See `get_fdt_addr`.
-        let regs0 = offset__of!(user_pt_regs, regs) + kreg_off;
-        vcpu.set_reg(
-            arm64_core_reg_id!(KVM_REG_SIZE_U64, regs0),
-            super::layout::FDT_START.raw_value(),
-        )
-        .map_err(Error::SetCoreRegister)?;
-    }
-    Ok(())
+/// Define the ID of system registers
+#[macro_export]
+macro_rules! arm64_sys_reg {
+    ($name: tt, $op0: tt, $op1: tt, $crn: tt, $crm: tt, $op2: tt) => {
+        pub const $name: u32 = SYSREG_HEAD
+            | ((($op0 as u32) << SYSREG_OP0_SHIFT) & SYSREG_OP0_MASK as u32)
+            | ((($op1 as u32) << SYSREG_OP1_SHIFT) & SYSREG_OP1_MASK as u32)
+            | ((($crn as u32) << SYSREG_CRN_SHIFT) & SYSREG_CRN_MASK as u32)
+            | ((($crm as u32) << SYSREG_CRM_SHIFT) & SYSREG_CRM_MASK as u32)
+            | ((($op2 as u32) << SYSREG_OP2_SHIFT) & SYSREG_OP2_MASK as u32);
+    };
 }
+
+arm64_sys_reg!(MPIDR_EL1, 3, 0, 0, 0, 5);
--- a/arch/src/lib.rs
+++ b/arch/src/lib.rs
@@ -54,7 +54,7 @@ pub enum Error {
 pub type Result<T> = result::Result<T, Error>;

 /// Type for memory region types.
-#[derive(Clone, Copy, PartialEq, Debug, Serialize, Deserialize, Versionize)]
+#[derive(Clone, Copy, PartialEq, Eq, Debug, Serialize, Deserialize, Versionize)]
 pub enum RegionType {
    /// RAM type
    Ram,
--- a/arch/src/x86_64/interrupts.rs
+++ b/arch/src/x86_64/interrupts.rs
@@ -5,10 +5,6 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE-BSD-3-Clause file.

-use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt};
-use hypervisor::x86_64::LapicState;
-use std::io::Cursor;
-use std::mem;
 use std::result;
 use std::sync::Arc;

@@ -20,32 +16,6 @@ pub const APIC_LVT1: usize = 0x360;
 pub const APIC_MODE_NMI: u32 = 0x4;
 pub const APIC_MODE_EXTINT: u32 = 0x7;

-pub fn get_klapic_reg(klapic: &LapicState, reg_offset: usize) -> u32 {
-    let sliceu8 = unsafe {
-        // This array is only accessed as parts of a u32 word, so interpret it as a u8 array.
-        // Cursors are only readable on arrays of u8, not i8(c_char).
-        mem::transmute::<&[i8], &[u8]>(&klapic.regs[reg_offset..])
-    };
-    let mut reader = Cursor::new(sliceu8);
-    // Following call can't fail if the offsets defined above are correct.
-    reader
-        .read_u32::<LittleEndian>()
-        .expect("Failed to read klapic register")
-}
-
-pub fn set_klapic_reg(klapic: &mut LapicState, reg_offset: usize, value: u32) {
-    let sliceu8 = unsafe {
-        // This array is only accessed as parts of a u32 word, so interpret it as a u8 array.
-        // Cursors are only readable on arrays of u8, not i8(c_char).
-        mem::transmute::<&mut [i8], &mut [u8]>(&mut klapic.regs[reg_offset..])
-    };
-    let mut writer = Cursor::new(sliceu8);
-    // Following call can't fail if the offsets defined above are correct.
-    writer
-        .write_u32::<LittleEndian>(value)
-        .expect("Failed to write klapic register")
-}
-
 pub fn set_apic_delivery_mode(reg: u32, mode: u32) -> u32 {
    ((reg) & !0x700) | ((mode) << 8)
 }
@@ -57,42 +27,13 @@ pub fn set_apic_delivery_mode(reg: u32, mode: u32) -> u32 {
 pub fn set_lint(vcpu: &Arc<dyn hypervisor::Vcpu>) -> Result<()> {
    let mut klapic = vcpu.get_lapic()?;

-    let lvt_lint0 = get_klapic_reg(&klapic, APIC_LVT0);
-    set_klapic_reg(
-        &mut klapic,
+    let lvt_lint0 = klapic.get_klapic_reg(APIC_LVT0);
+    klapic.set_klapic_reg(
        APIC_LVT0,
        set_apic_delivery_mode(lvt_lint0, APIC_MODE_EXTINT),
    );
-    let lvt_lint1 = get_klapic_reg(&klapic, APIC_LVT1);
-    set_klapic_reg(
-        &mut klapic,
-        APIC_LVT1,
-        set_apic_delivery_mode(lvt_lint1, APIC_MODE_NMI),
-    );
+    let lvt_lint1 = klapic.get_klapic_reg(APIC_LVT1);
+    klapic.set_klapic_reg(APIC_LVT1, set_apic_delivery_mode(lvt_lint1, APIC_MODE_NMI));

    vcpu.set_lapic(&klapic)
 }
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    const KVM_APIC_REG_SIZE: usize = 0x400;
-
-    #[test]
-    fn test_set_and_get_klapic_reg() {
-        let reg_offset = 0x340;
-        let mut klapic = LapicState::default();
-        set_klapic_reg(&mut klapic, reg_offset, 3);
-        let value = get_klapic_reg(&klapic, reg_offset);
-        assert_eq!(value, 3);
-    }
-
-    #[test]
-    #[should_panic]
-    fn test_set_and_get_klapic_out_of_bounds() {
-        let reg_offset = KVM_APIC_REG_SIZE + 10;
-        let mut klapic = LapicState::default();
-        set_klapic_reg(&mut klapic, reg_offset, 3);
-    }
-}
--- a/arch/src/x86_64/mod.rs
+++ b/arch/src/x86_64/mod.rs
@@ -15,7 +15,7 @@ pub mod regs;
 use crate::GuestMemoryMmap;
 use crate::InitramfsConfig;
 use crate::RegionType;
-use hypervisor::x86_64::{CpuId, CpuIdEntry, CPUID_FLAG_VALID_INDEX};
+use hypervisor::arch::x86::{CpuIdEntry, CPUID_FLAG_VALID_INDEX};
 use hypervisor::HypervisorError;
 use linux_loader::loader::bootparam::boot_params;
 use linux_loader::loader::elf::start_info::{
@@ -202,7 +202,7 @@ impl From<Error> for super::Error {
    }
 }

-#[allow(dead_code, clippy::upper_case_acronyms)]
+#[allow(clippy::upper_case_acronyms)]
 #[derive(Copy, Clone, Debug)]
 pub enum CpuidReg {
    EAX,
@@ -223,16 +223,14 @@ pub struct CpuidPatch {

 impl CpuidPatch {
    pub fn set_cpuid_reg(
-        cpuid: &mut CpuId,
+        cpuid: &mut Vec<CpuIdEntry>,
        function: u32,
        index: Option<u32>,
        reg: CpuidReg,
        value: u32,
    ) {
-        let entries = cpuid.as_mut_slice();
-
        let mut entry_found = false;
-        for entry in entries.iter_mut() {
+        for entry in cpuid.iter_mut() {
            if entry.function == function && (index == None || index.unwrap() == entry.index) {
                entry_found = true;
                match reg {
@@ -279,16 +277,12 @@ impl CpuidPatch {
                }
            }

-            if let Err(e) = cpuid.push(entry) {
-                error!("Failed adding new CPUID entry: {:?}", e);
-            }
+            cpuid.push(entry);
        }
    }

-    pub fn patch_cpuid(cpuid: &mut CpuId, patches: Vec<CpuidPatch>) {
-        let entries = cpuid.as_mut_slice();
-
-        for entry in entries.iter_mut() {
+    pub fn patch_cpuid(cpuid: &mut [CpuIdEntry], patches: Vec<CpuidPatch>) {
+        for entry in cpuid {
            for patch in patches.iter() {
                if entry.function == patch.function && entry.index == patch.index {
                    if let Some(flags_bit) = patch.flags_bit {
@@ -312,16 +306,15 @@ impl CpuidPatch {
    }

    pub fn is_feature_enabled(
-        cpuid: &CpuId,
+        cpuid: &[CpuIdEntry],
        function: u32,
        index: u32,
        reg: CpuidReg,
        feature_bit: usize,
    ) -> bool {
-        let entries = cpuid.as_slice();
        let mask = 1 << feature_bit;

-        for entry in entries.iter() {
+        for entry in cpuid {
            if entry.function == function && entry.index == index {
                let reg_val = match reg {
                    CpuidReg::EAX => entry.eax,
@@ -470,12 +463,12 @@ impl CpuidFeatureEntry {
    }

    fn get_features_from_cpuid(
-        cpuid: &CpuId,
+        cpuid: &[CpuIdEntry],
        feature_entry_list: &[CpuidFeatureEntry],
    ) -> Vec<u32> {
        let mut features = vec![0; feature_entry_list.len()];
        for (i, feature_entry) in feature_entry_list.iter().enumerate() {
-            for cpuid_entry in cpuid.as_slice().iter() {
+            for cpuid_entry in cpuid {
                if cpuid_entry.function == feature_entry.function
                    && cpuid_entry.index == feature_entry.index
                {
@@ -505,8 +498,8 @@ impl CpuidFeatureEntry {
    // The function returns `Error` (a.k.a. "incompatible"), when the CPUID features from `src_vm_cpuid`
    // is not a subset of those of the `dest_vm_cpuid`.
    pub fn check_cpuid_compatibility(
-        src_vm_cpuid: &CpuId,
-        dest_vm_cpuid: &CpuId,
+        src_vm_cpuid: &[CpuIdEntry],
+        dest_vm_cpuid: &[CpuIdEntry],
    ) -> Result<(), Error> {
        let feature_entry_list = &Self::checked_feature_entry_list();
        let src_vm_features = Self::get_features_from_cpuid(src_vm_cpuid, feature_entry_list);
@@ -558,7 +551,7 @@ pub fn generate_common_cpuid(
    phys_bits: u8,
    kvm_hyperv: bool,
    #[cfg(feature = "tdx")] tdx_enabled: bool,
-) -> super::Result<CpuId> {
+) -> super::Result<Vec<CpuIdEntry>> {
    let cpuid_patches = vec![
        // Patch tsc deadline timer bit
        CpuidPatch {
@@ -669,16 +662,14 @@ pub fn generate_common_cpuid(
    for i in 0x8000_0002..=0x8000_0004 {
        cpuid.retain(|c| c.function != i);
        let leaf = unsafe { std::arch::x86_64::__cpuid(i) };
-        cpuid
-            .push(CpuIdEntry {
-                function: i,
-                eax: leaf.eax,
-                ebx: leaf.ebx,
-                ecx: leaf.ecx,
-                edx: leaf.edx,
-                ..Default::default()
-            })
-            .map_err(Error::CpuidIdentification)?;
+        cpuid.push(CpuIdEntry {
+            function: i,
+            eax: leaf.eax,
+            ebx: leaf.ebx,
+            ecx: leaf.ecx,
+            edx: leaf.edx,
+            ..Default::default()
+        });
    }

    if kvm_hyperv {
@@ -687,56 +678,44 @@ pub fn generate_common_cpuid(
        cpuid.retain(|c| c.function != 0x4000_0001);
        // See "Hypervisor Top Level Functional Specification" for details
        // Compliance with "Hv#1" requires leaves up to 0x4000_000a
-        cpuid
-            .push(CpuIdEntry {
-                function: 0x40000000,
-                eax: 0x4000000a, // Maximum cpuid leaf
-                ebx: 0x756e694c, // "Linu"
-                ecx: 0x564b2078, // "x KV"
-                edx: 0x7648204d, // "M Hv"
-                ..Default::default()
-            })
-            .map_err(Error::CpuidKvmHyperV)?;
-        cpuid
-            .push(CpuIdEntry {
-                function: 0x40000001,
-                eax: 0x31237648, // "Hv#1"
-                ..Default::default()
-            })
-            .map_err(Error::CpuidKvmHyperV)?;
-        cpuid
-            .push(CpuIdEntry {
-                function: 0x40000002,
-                eax: 0x3839,  // "Build number"
-                ebx: 0xa0000, // "Version"
-                ..Default::default()
-            })
-            .map_err(Error::CpuidKvmHyperV)?;
-        cpuid
-            .push(CpuIdEntry {
-                function: 0x4000_0003,
-                eax: 1 << 1 // AccessPartitionReferenceCounter
+        cpuid.push(CpuIdEntry {
+            function: 0x40000000,
+            eax: 0x4000000a, // Maximum cpuid leaf
+            ebx: 0x756e694c, // "Linu"
+            ecx: 0x564b2078, // "x KV"
+            edx: 0x7648204d, // "M Hv"
+            ..Default::default()
+        });
+        cpuid.push(CpuIdEntry {
+            function: 0x40000001,
+            eax: 0x31237648, // "Hv#1"
+            ..Default::default()
+        });
+        cpuid.push(CpuIdEntry {
+            function: 0x40000002,
+            eax: 0x3839,  // "Build number"
+            ebx: 0xa0000, // "Version"
+            ..Default::default()
+        });
+        cpuid.push(CpuIdEntry {
+            function: 0x4000_0003,
+            eax: 1 << 1 // AccessPartitionReferenceCounter
                   | 1 << 2 // AccessSynicRegs
                   | 1 << 3 // AccessSyntheticTimerRegs
                   | 1 << 9, // AccessPartitionReferenceTsc
-                edx: 1 << 3, // CPU dynamic partitioning
-                ..Default::default()
-            })
-            .map_err(Error::CpuidKvmHyperV)?;
-        cpuid
-            .push(CpuIdEntry {
-                function: 0x4000_0004,
-                eax: 1 << 5, // Recommend relaxed timing
-                ..Default::default()
-            })
-            .map_err(Error::CpuidKvmHyperV)?;
+            edx: 1 << 3, // CPU dynamic partitioning
+            ..Default::default()
+        });
+        cpuid.push(CpuIdEntry {
+            function: 0x4000_0004,
+            eax: 1 << 5, // Recommend relaxed timing
+            ..Default::default()
+        });
        for i in 0x4000_0005..=0x4000_000a {
-            cpuid
-                .push(CpuIdEntry {
-                    function: i,
-                    ..Default::default()
-                })
-                .map_err(Error::CpuidKvmHyperV)?;
+            cpuid.push(CpuIdEntry {
+                function: i,
+                ..Default::default()
+            });
        }
    }

@@ -748,7 +727,7 @@ pub fn configure_vcpu(
    id: u8,
    kernel_entry_point: Option<EntryPoint>,
    vm_memory: &GuestMemoryAtomic<GuestMemoryMmap>,
-    cpuid: CpuId,
+    cpuid: Vec<CpuIdEntry>,
    kvm_hyperv: bool,
 ) -> super::Result<()> {
    // Per vCPU CPUID changes; common are handled via generate_common_cpuid()
@@ -840,13 +819,16 @@ pub fn configure_system(
    rsdp_addr: Option<GuestAddress>,
    sgx_epc_region: Option<SgxEpcRegion>,
    serial_number: Option<&str>,
+    uuid: Option<&str>,
+    oem_strings: Option<&[&str]>,
 ) -> super::Result<()> {
    // Write EBDA address to location where ACPICA expects to find it
    guest_mem
        .write_obj((layout::EBDA_START.0 >> 4) as u16, layout::EBDA_POINTER)
        .map_err(Error::EbdaSetup)?;

-    let size = smbios::setup_smbios(guest_mem, serial_number).map_err(Error::SmbiosSetup)?;
+    let size = smbios::setup_smbios(guest_mem, serial_number, uuid, oem_strings)
+        .map_err(Error::SmbiosSetup)?;

    // Place the MP table after the SMIOS table aligned to 16 bytes
    let offset = GuestAddress(layout::SMBIOS_START).unchecked_add(size);
@@ -1056,7 +1038,7 @@ pub fn get_host_cpu_phys_bits() -> u8 {
 }

 fn update_cpuid_topology(
-    cpuid: &mut CpuId,
+    cpuid: &mut Vec<CpuIdEntry>,
    threads_per_core: u8,
    cores_per_die: u8,
    dies_per_package: u8,
@@ -1120,7 +1102,10 @@ fn update_cpuid_topology(

 // The goal is to update the CPUID sub-leaves to reflect the number of EPC
 // sections exposed to the guest.
-fn update_cpuid_sgx(cpuid: &mut CpuId, epc_sections: Vec<SgxEpcSection>) -> Result<(), Error> {
+fn update_cpuid_sgx(
+    cpuid: &mut Vec<CpuIdEntry>,
+    epc_sections: Vec<SgxEpcSection>,
+) -> Result<(), Error> {
    // Something's wrong if there's no EPC section.
    if epc_sections.is_empty() {
        return Err(Error::NoSgxEpcSection);
@@ -1196,6 +1181,8 @@ mod tests {
            Some(layout::RSDP_POINTER),
            None,
            None,
+            None,
+            None,
        );
        assert!(config_err.is_err());

@@ -1209,7 +1196,18 @@ mod tests {
            .collect();
        let gm = GuestMemoryMmap::from_ranges(&ram_regions).unwrap();

-        configure_system(&gm, GuestAddress(0), &None, no_vcpus, None, None, None).unwrap();
+        configure_system(
+            &gm,
+            GuestAddress(0),
+            &None,
+            no_vcpus,
+            None,
+            None,
+            None,
+            None,
+            None,
+        )
+        .unwrap();

        // Now assigning some memory that is equal to the start of the 32bit memory hole.
        let mem_size = 3328 << 20;
@@ -1220,9 +1218,31 @@ mod tests {
            .map(|r| (r.0, r.1))
            .collect();
        let gm = GuestMemoryMmap::from_ranges(&ram_regions).unwrap();
-        configure_system(&gm, GuestAddress(0), &None, no_vcpus, None, None, None).unwrap();
+        configure_system(
+            &gm,
+            GuestAddress(0),
+            &None,
+            no_vcpus,
+            None,
+            None,
+            None,
+            None,
+            None,
+        )
+        .unwrap();

-        configure_system(&gm, GuestAddress(0), &None, no_vcpus, None, None, None).unwrap();
+        configure_system(
+            &gm,
+            GuestAddress(0),
+            &None,
+            no_vcpus,
+            None,
+            None,
+            None,
+            None,
+            None,
+        )
+        .unwrap();

        // Now assigning some memory that falls after the 32bit memory hole.
        let mem_size = 3330 << 20;
@@ -1233,9 +1253,31 @@ mod tests {
            .map(|r| (r.0, r.1))
            .collect();
        let gm = GuestMemoryMmap::from_ranges(&ram_regions).unwrap();
-        configure_system(&gm, GuestAddress(0), &None, no_vcpus, None, None, None).unwrap();
+        configure_system(
+            &gm,
+            GuestAddress(0),
+            &None,
+            no_vcpus,
+            None,
+            None,
+            None,
+            None,
+            None,
+        )
+        .unwrap();

-        configure_system(&gm, GuestAddress(0), &None, no_vcpus, None, None, None).unwrap();
+        configure_system(
+            &gm,
+            GuestAddress(0),
+            &None,
+            no_vcpus,
+            None,
+            None,
+            None,
+            None,
+            None,
+        )
+        .unwrap();
    }

    #[test]
--- a/arch/src/x86_64/regs.rs
+++ b/arch/src/x86_64/regs.rs
@@ -10,7 +10,7 @@ use crate::layout::{BOOT_GDT_START, BOOT_IDT_START, PVH_INFO_START};
 use crate::GuestMemoryMmap;
 use hypervisor::arch::x86::gdt::{gdt_entry, segment_from_gdt};
 use hypervisor::arch::x86::regs::CR0_PE;
-use hypervisor::x86_64::{FpuState, SpecialRegisters, StandardRegisters};
+use hypervisor::arch::x86::{FpuState, SpecialRegisters, StandardRegisters};
 use std::sync::Arc;
 use std::{mem, result};
 use vm_memory::{Address, Bytes, GuestMemory, GuestMemoryError};
@@ -66,7 +66,7 @@ pub fn setup_fpu(vcpu: &Arc<dyn hypervisor::Vcpu>) -> Result<()> {
 ///
 /// * `vcpu` - Structure for the VCPU that holds the VCPU's fd.
 pub fn setup_msrs(vcpu: &Arc<dyn hypervisor::Vcpu>) -> Result<()> {
-    vcpu.set_msrs(&hypervisor::x86_64::boot_msr_entries())
+    vcpu.set_msrs(&vcpu.boot_msr_entries())
        .map_err(Error::SetModelSpecificRegisters)?;

    Ok(())
--- a/arch/src/x86_64/smbios.rs
+++ b/arch/src/x86_64/smbios.rs
@@ -12,10 +12,10 @@ use std::fmt::{self, Display};
 use std::mem;
 use std::result;
 use std::slice;
+use uuid::Uuid;
 use vm_memory::ByteValued;
 use vm_memory::{Address, Bytes, GuestAddress};

-#[allow(unused_variables)]
 #[derive(Debug)]
 pub enum Error {
    /// There was too little guest memory to store the entire SMBIOS table.
@@ -28,6 +28,8 @@ pub enum Error {
    WriteSmbiosEp,
    /// Failure to write additional data to memory
    WriteData,
+    /// Failure to parse uuid, uuid format may be error
+    ParseUuid(uuid::Error),
 }

 impl std::error::Error for Error {}
@@ -37,11 +39,16 @@ impl Display for Error {
        use self::Error::*;

        let description = match self {
-            NotEnoughMemory => "There was too little guest memory to store the SMBIOS table",
-            AddressOverflow => "The SMBIOS table has too little address space to be stored",
-            Clear => "Failure while zeroing out the memory for the SMBIOS table",
-            WriteSmbiosEp => "Failure to write SMBIOS entrypoint structure",
-            WriteData => "Failure to write additional data to memory",
+            NotEnoughMemory => {
+                "There was too little guest memory to store the SMBIOS table".to_string()
+            }
+            AddressOverflow => {
+                "The SMBIOS table has too little address space to be stored".to_string()
+            }
+            Clear => "Failure while zeroing out the memory for the SMBIOS table".to_string(),
+            WriteSmbiosEp => "Failure to write SMBIOS entrypoint structure".to_string(),
+            WriteData => "Failure to write additional data to memory".to_string(),
+            ParseUuid(e) => format!("Failure to parse uuid: {}", e),
        };

        write!(f, "SMBIOS error: {}", description)
@@ -54,6 +61,7 @@ pub type Result<T> = result::Result<T, Error>;
 const SM3_MAGIC_IDENT: &[u8; 5usize] = b"_SM3_";
 const BIOS_INFORMATION: u8 = 0;
 const SYSTEM_INFORMATION: u8 = 1;
+const OEM_STRINGS: u8 = 11;
 const END_OF_TABLE: u8 = 127;
 const PCI_SUPPORTED: u64 = 1 << 7;
 const IS_VIRTUAL_MACHINE: u8 = 1 << 4;
@@ -68,75 +76,81 @@ fn compute_checksum<T: Copy>(v: &T) -> u8 {
    (!checksum).wrapping_add(1)
 }

+#[repr(C)]
 #[repr(packed)]
-#[derive(Default, Copy)]
-pub struct Smbios30Entrypoint {
-    pub signature: [u8; 5usize],
-    pub checksum: u8,
-    pub length: u8,
-    pub majorver: u8,
-    pub minorver: u8,
-    pub docrev: u8,
-    pub revision: u8,
-    pub reserved: u8,
-    pub max_size: u32,
-    pub physptr: u64,
-}
-
-impl Clone for Smbios30Entrypoint {
-    fn clone(&self) -> Self {
-        *self
-    }
+#[derive(Default, Copy, Clone)]
+struct Smbios30Entrypoint {
+    signature: [u8; 5usize],
+    checksum: u8,
+    length: u8,
+    majorver: u8,
+    minorver: u8,
+    docrev: u8,
+    revision: u8,
+    reserved: u8,
+    max_size: u32,
+    physptr: u64,
 }

+#[repr(C)]
 #[repr(packed)]
-#[derive(Default, Copy)]
-pub struct SmbiosBiosInfo {
-    pub typ: u8,
-    pub length: u8,
-    pub handle: u16,
-    pub vendor: u8,
-    pub version: u8,
-    pub start_addr: u16,
-    pub release_date: u8,
-    pub rom_size: u8,
-    pub characteristics: u64,
-    pub characteristics_ext1: u8,
-    pub characteristics_ext2: u8,
-}
-
-impl Clone for SmbiosBiosInfo {
-    fn clone(&self) -> Self {
-        *self
-    }
+#[derive(Default, Copy, Clone)]
+struct SmbiosBiosInfo {
+    r#type: u8,
+    length: u8,
+    handle: u16,
+    vendor: u8,
+    version: u8,
+    start_addr: u16,
+    release_date: u8,
+    rom_size: u8,
+    characteristics: u64,
+    characteristics_ext1: u8,
+    characteristics_ext2: u8,
 }

+#[repr(C)]
 #[repr(packed)]
-#[derive(Default, Copy)]
-pub struct SmbiosSysInfo {
-    pub typ: u8,
-    pub length: u8,
-    pub handle: u16,
-    pub manufacturer: u8,
-    pub product_name: u8,
-    pub version: u8,
-    pub serial_number: u8,
-    pub uuid: [u8; 16usize],
-    pub wake_up_type: u8,
-    pub sku: u8,
-    pub family: u8,
+#[derive(Default, Copy, Clone)]
+struct SmbiosSysInfo {
+    r#type: u8,
+    length: u8,
+    handle: u16,
+    manufacturer: u8,
+    product_name: u8,
+    version: u8,
+    serial_number: u8,
+    uuid: [u8; 16usize],
+    wake_up_type: u8,
+    sku: u8,
+    family: u8,
 }

-impl Clone for SmbiosSysInfo {
-    fn clone(&self) -> Self {
-        *self
-    }
+#[repr(C)]
+#[repr(packed)]
+#[derive(Default, Copy, Clone)]
+struct SmbiosOemStrings {
+    r#type: u8,
+    length: u8,
+    handle: u16,
+    count: u8,
+}
+
+#[repr(C)]
+#[repr(packed)]
+#[derive(Default, Copy, Clone)]
+struct SmbiosEndOfTable {
+    r#type: u8,
+    length: u8,
+    handle: u16,
 }

 // SAFETY: These data structures only contain a series of integers
 unsafe impl ByteValued for Smbios30Entrypoint {}
 unsafe impl ByteValued for SmbiosBiosInfo {}
 unsafe impl ByteValued for SmbiosSysInfo {}
+unsafe impl ByteValued for SmbiosOemStrings {}
+unsafe impl ByteValued for SmbiosEndOfTable {}

 fn write_and_incr<T: ByteValued>(
    mem: &GuestMemoryMmap,
@@ -162,7 +176,12 @@ fn write_string(
    Ok(curptr)
 }

-pub fn setup_smbios(mem: &GuestMemoryMmap, serial_number: Option<&str>) -> Result<u64> {
+pub fn setup_smbios(
+    mem: &GuestMemoryMmap,
+    serial_number: Option<&str>,
+    uuid: Option<&str>,
+    oem_strings: Option<&[&str]>,
+) -> Result<u64> {
    let physptr = GuestAddress(SMBIOS_START)
        .checked_add(mem::size_of::<Smbios30Entrypoint>() as u64)
        .ok_or(Error::NotEnoughMemory)?;
@@ -172,7 +191,7 @@ pub fn setup_smbios(mem: &GuestMemoryMmap, serial_number: Option<&str>) -> Resul
    {
        handle += 1;
        let smbios_biosinfo = SmbiosBiosInfo {
-            typ: BIOS_INFORMATION,
+            r#type: BIOS_INFORMATION,
            length: mem::size_of::<SmbiosBiosInfo>() as u8,
            handle,
            vendor: 1,  // First string written in this section
@@ -189,13 +208,20 @@ pub fn setup_smbios(mem: &GuestMemoryMmap, serial_number: Option<&str>) -> Resul

    {
        handle += 1;
+
+        let uuid_number = uuid
+            .map(Uuid::parse_str)
+            .transpose()
+            .map_err(Error::ParseUuid)?
+            .unwrap_or(Uuid::nil());
        let smbios_sysinfo = SmbiosSysInfo {
-            typ: SYSTEM_INFORMATION,
+            r#type: SYSTEM_INFORMATION,
            length: mem::size_of::<SmbiosSysInfo>() as u8,
            handle,
            manufacturer: 1, // First string written in this section
            product_name: 2, // Second string written in this section
            serial_number: serial_number.map(|_| 3).unwrap_or_default(), // 3rd string
+            uuid: uuid_number.to_bytes_le(), // set uuid
            ..Default::default()
        };
        curptr = write_and_incr(mem, smbios_sysinfo, curptr)?;
@@ -207,15 +233,34 @@ pub fn setup_smbios(mem: &GuestMemoryMmap, serial_number: Option<&str>) -> Resul
        curptr = write_and_incr(mem, 0u8, curptr)?;
    }

+    if let Some(oem_strings) = oem_strings {
+        handle += 1;
+
+        let smbios_oemstrings = SmbiosOemStrings {
+            r#type: OEM_STRINGS,
+            length: mem::size_of::<SmbiosOemStrings>() as u8,
+            handle,
+            count: oem_strings.len() as u8,
+        };
+
+        curptr = write_and_incr(mem, smbios_oemstrings, curptr)?;
+
+        for s in oem_strings {
+            curptr = write_string(mem, s, curptr)?;
+        }
+
+        curptr = write_and_incr(mem, 0u8, curptr)?;
+    }
+
    {
        handle += 1;
-        let smbios_sysinfo = SmbiosSysInfo {
-            typ: END_OF_TABLE,
-            length: mem::size_of::<SmbiosSysInfo>() as u8,
+        let smbios_end = SmbiosEndOfTable {
+            r#type: END_OF_TABLE,
+            length: mem::size_of::<SmbiosEndOfTable>() as u8,
            handle,
-            ..Default::default()
        };
-        curptr = write_and_incr(mem, smbios_sysinfo, curptr)?;
+        curptr = write_and_incr(mem, smbios_end, curptr)?;
+        curptr = write_and_incr(mem, 0u8, curptr)?;
        curptr = write_and_incr(mem, 0u8, curptr)?;
    }

@@ -237,7 +282,7 @@ pub fn setup_smbios(mem: &GuestMemoryMmap, serial_number: Option<&str>) -> Resul
            .map_err(|_| Error::WriteSmbiosEp)?;
    }

-    Ok(curptr.unchecked_offset_from(physptr))
+    Ok(curptr.unchecked_offset_from(physptr) + std::mem::size_of::<Smbios30Entrypoint>() as u64)
 }

 #[cfg(test)]
@@ -267,7 +312,7 @@ mod tests {
    fn entrypoint_checksum() {
        let mem = GuestMemoryMmap::from_ranges(&[(GuestAddress(SMBIOS_START), 4096)]).unwrap();

-        setup_smbios(&mem, None).unwrap();
+        setup_smbios(&mem, None, None, None).unwrap();

        let smbios_ep: Smbios30Entrypoint = mem.read_obj(GuestAddress(SMBIOS_START)).unwrap();

--- a/block_util/Cargo.toml
+++ b/block_util/Cargo.toml
@@ -8,17 +8,17 @@ edition = "2021"
 default = []

 [dependencies]
-io-uring = "0.5.2"
-libc = "0.2.126"
+io-uring = "0.5.3"
+libc = "0.2.131"
 log = "0.4.17"
 qcow = { path = "../qcow" }
-thiserror = "1.0.31"
+thiserror = "1.0.32"
 versionize = "0.1.6"
 versionize_derive = "0.1.4"
 vhdx = { path = "../vhdx" }
 virtio-bindings = { version = "0.1.0", features = ["virtio-v5_0_0"] }
-virtio-queue = "0.3.0"
+virtio-queue = "0.5.0"
 vm-memory = { version = "0.8.0", features = ["backend-mmap", "backend-atomic", "backend-bitmap"] }
 vm-virtio = { path = "../vm-virtio" }
-vmm-sys-util = "0.9.0"
+vmm-sys-util = "0.10.0"

--- a/block_util/src/async_io.rs
+++ b/block_util/src/async_io.rs
@@ -8,7 +8,7 @@ use std::fs::File;
 use std::os::unix::io::AsRawFd;
 use thiserror::Error;
 use vmm_sys_util::eventfd::EventFd;
-use vmm_sys_util::{ioctl_expr, ioctl_io_nr, ioctl_ioc_nr};
+use vmm_sys_util::{ioctl_io_nr, ioctl_ioc_nr};

 #[derive(Error, Debug)]
 pub enum DiskFileError {
@@ -104,7 +104,7 @@ impl DiskTopology {

 pub type DiskFileResult<T> = std::result::Result<T, DiskFileError>;

-pub trait DiskFile: Send + Sync {
+pub trait DiskFile: Send {
    fn size(&mut self) -> DiskFileResult<u64>;
    fn new_async_io(&self, ring_depth: u32) -> DiskFileResult<Box<dyn AsyncIo>>;
    fn topology(&mut self) -> DiskTopology {
@@ -127,7 +127,7 @@ pub enum AsyncIoError {

 pub type AsyncIoResult<T> = std::result::Result<T, AsyncIoError>;

-pub trait AsyncIo: Send + Sync {
+pub trait AsyncIo: Send {
    fn notifier(&self) -> &EventFd;
    fn read_vectored(
        &mut self,
--- a/block_util/src/lib.rs
+++ b/block_util/src/lib.rs
@@ -32,6 +32,7 @@ use std::path::Path;
 use std::result;
 use std::sync::Arc;
 use std::sync::MutexGuard;
+use thiserror::Error;
 use versionize::{VersionMap, Versionize, VersionizeResult};
 use versionize_derive::Versionize;
 use virtio_bindings::bindings::virtio_blk::*;
@@ -48,25 +49,25 @@ type GuestMemoryMmap = vm_memory::GuestMemoryMmap<AtomicBitmap>;
 const SECTOR_SHIFT: u8 = 9;
 pub const SECTOR_SIZE: u64 = 0x01 << SECTOR_SHIFT;

-#[derive(Debug)]
+#[derive(Error, Debug)]
 pub enum Error {
-    /// Guest gave us bad memory addresses.
+    #[error("Guest gave us bad memory addresses.")]
    GuestMemory(GuestMemoryError),
-    /// Guest gave us offsets that would have overflowed a usize.
+    #[error("Guest gave us offsets that would have overflowed a usize.")]
    CheckedOffset(GuestAddress, usize),
-    /// Guest gave us a write only descriptor that protocol says to read from.
+    #[error("Guest gave us a write only descriptor that protocol says to read from.")]
    UnexpectedWriteOnlyDescriptor,
-    /// Guest gave us a read only descriptor that protocol says to write to.
+    #[error("Guest gave us a read only descriptor that protocol says to write to.")]
    UnexpectedReadOnlyDescriptor,
-    /// Guest gave us too few descriptors in a descriptor chain.
+    #[error("Guest gave us too few descriptors in a descriptor chain.")]
    DescriptorChainTooShort,
-    /// Guest gave us a descriptor that was too short to use.
+    #[error("Guest gave us a descriptor that was too short to use.")]
    DescriptorLengthTooSmall,
-    /// Getting a block's metadata fails for any reason.
+    #[error("Getting a block's metadata fails for any reason.")]
    GetFileMetadata,
-    /// The requested operation would cause a seek beyond disk end.
+    #[error("The requested operation would cause a seek beyond disk end.")]
    InvalidOffset,
-    /// The requested operation does not support multiple descriptors.
+    #[error("The requested operation does not support multiple descriptors.")]
    TooManyDescriptors,
 }

@@ -102,20 +103,31 @@ pub fn build_disk_image_id(disk_path: &Path) -> Vec<u8> {
    default_disk_image_id
 }

-#[derive(Debug)]
+#[derive(Error, Debug)]
 pub enum ExecuteError {
+    #[error("Bad request: {0}")]
    BadRequest(Error),
+    #[error("Falied to flush: {0}")]
    Flush(io::Error),
+    #[error("Failed to read: {0}")]
    Read(GuestMemoryError),
+    #[error("Failed to seek: {0}")]
    Seek(io::Error),
+    #[error("Failed to write: {0}")]
    Write(GuestMemoryError),
+    #[error("Unsupported request: {0}")]
    Unsupported(u32),
+    #[error("Failed to submit io uring: {0}")]
    SubmitIoUring(io::Error),
+    #[error("Failed to get guest address: {0}")]
    GetHostAddress(GuestMemoryError),
+    #[error("Failed to async read: {0}")]
    AsyncRead(AsyncIoError),
+    #[error("Failed to async write: {0}")]
    AsyncWrite(AsyncIoError),
+    #[error("failed to async flush: {0}")]
    AsyncFlush(AsyncIoError),
-    /// Failed allocating a temporary buffer.
+    #[error("Failed allocating a temporary buffer: {0}")]
    TemporaryBufferAllocation(io::Error),
 }

@@ -138,7 +150,7 @@ impl ExecuteError {
    }
 }

-#[derive(Clone, Copy, Debug, PartialEq)]
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
 pub enum RequestType {
    In,
    Out,
--- a/devices/Cargo.toml
+++ b/devices/Cargo.toml
@@ -6,19 +6,22 @@ edition = "2021"

 [dependencies]
 acpi_tables = { path = "../acpi_tables" }
-anyhow = "1.0.57"
+anyhow = "1.0.62"
 arch = { path = "../arch" }
 bitflags = "1.3.2"
 byteorder = "1.4.3"
-epoll = "4.3.1"
-libc = "0.2.126"
+hypervisor = { path = "../hypervisor" }
+libc = "0.2.131"
 log = "0.4.17"
 versionize = "0.1.6"
 versionize_derive = "0.1.4"
 vm-device = { path = "../vm-device" }
 vm-memory = "0.8.0"
 vm-migration = { path = "../vm-migration" }
-vmm-sys-util = "0.9.0"
+vmm-sys-util = "0.10.0"
+
+[target.'cfg(target_arch = "aarch64")'.dependencies]
+arch = { path = "../arch" }

 [features]
 default = []
--- a/devices/src/gic.rs
+++ b/devices/src/gic.rs
@@ -4,30 +4,34 @@

 use super::interrupt_controller::{Error, InterruptController};
 extern crate arch;
-use arch::aarch64::gic::GicDevice;
+use anyhow::anyhow;
+use arch::layout;
+use hypervisor::{arch::aarch64::gic::Vgic, CpuState};
 use std::result;
 use std::sync::{Arc, Mutex};
 use vm_device::interrupt::{
    InterruptIndex, InterruptManager, InterruptSourceConfig, InterruptSourceGroup,
    LegacyIrqSourceConfig, MsiIrqGroupConfig,
 };
+use vm_memory::Address;
+use vm_migration::{Migratable, MigratableError, Pausable, Snapshot, Snapshottable, Transportable};
 use vmm_sys_util::eventfd::EventFd;

 type Result<T> = result::Result<T, Error>;

-// Reserve 32 IRQs for legacy device.
-pub const IRQ_LEGACY_BASE: usize = arch::layout::IRQ_BASE as usize;
+// Reserve 32 IRQs for legacy devices.
+pub const IRQ_LEGACY_BASE: usize = layout::IRQ_BASE as usize;
 pub const IRQ_LEGACY_COUNT: usize = 32;

-// This Gic struct implements InterruptController to provide interrupt delivery service.
-// The Gic source files in arch/ folder maintain the Aarch64 specific Gic device.
-// The 2 Gic instances could be merged together.
-// Leave this refactoring to future. Two options may be considered:
-//   1. Move Gic*.rs from arch/ folder here.
-//   2. Move this file and ioapic.rs to arch/, as they are architecture specific.
+// Gic (Generic Interupt Controller) struct provides all the functionality of a
+// GIC device. It wraps a hypervisor-emulated GIC device (Vgic) provided by the
+// `hypervisor` crate.
+// Gic struct also implements InterruptController to provide interrupt delivery
+// service.
 pub struct Gic {
    interrupt_source_group: Arc<dyn InterruptSourceGroup>,
-    gic_device: Option<Arc<Mutex<Box<dyn GicDevice>>>>,
+    // The hypervisor agnostic virtual GIC
+    vgic: Option<Arc<Mutex<dyn Vgic>>>,
 }

 impl Gic {
@@ -44,16 +48,32 @@ impl Gic {

        Ok(Gic {
            interrupt_source_group,
-            gic_device: None,
+            vgic: None,
        })
    }

-    pub fn set_gic_device(&mut self, gic_device: Arc<Mutex<Box<dyn GicDevice>>>) {
-        self.gic_device = Some(gic_device);
+    pub fn create_vgic(
+        &mut self,
+        vm: &Arc<dyn hypervisor::Vm>,
+        vcpu_count: u64,
+    ) -> Result<Arc<Mutex<dyn Vgic>>> {
+        let vgic = vm
+            .create_vgic(
+                vcpu_count,
+                layout::GIC_V3_DIST_START.raw_value(),
+                layout::GIC_V3_DIST_SIZE,
+                layout::GIC_V3_REDIST_SIZE,
+                layout::GIC_V3_ITS_SIZE,
+                layout::IRQ_NUM,
+            )
+            .map_err(Error::CreateGic)?;
+        self.vgic = Some(vgic.clone());
+        Ok(vgic.clone())
    }

-    pub fn get_gic_device(&self) -> Option<&Arc<Mutex<Box<dyn GicDevice>>>> {
-        self.gic_device.as_ref()
+    pub fn set_gicr_typers(&mut self, vcpu_states: &[CpuState]) {
+        let vgic = self.vgic.as_ref().unwrap().clone();
+        vgic.lock().unwrap().set_gicr_typers(vcpu_states);
    }
 }

@@ -97,3 +117,43 @@ impl InterruptController for Gic {
        self.interrupt_source_group.notifier(irq as InterruptIndex)
    }
 }
+
+pub const GIC_V3_ITS_SNAPSHOT_ID: &str = "gic-v3-its";
+impl Snapshottable for Gic {
+    fn id(&self) -> String {
+        GIC_V3_ITS_SNAPSHOT_ID.to_string()
+    }
+
+    fn snapshot(&mut self) -> std::result::Result<Snapshot, MigratableError> {
+        let vgic = self.vgic.as_ref().unwrap().clone();
+        let state = vgic.lock().unwrap().state().unwrap();
+        Snapshot::new_from_state(&self.id(), &state)
+    }
+
+    fn restore(&mut self, snapshot: Snapshot) -> std::result::Result<(), MigratableError> {
+        let vgic = self.vgic.as_ref().unwrap().clone();
+        vgic.lock()
+            .unwrap()
+            .set_state(&snapshot.to_state(&self.id())?)
+            .map_err(|e| {
+                MigratableError::Restore(anyhow!("Could not restore GICv3ITS state {:?}", e))
+            })?;
+        Ok(())
+    }
+}
+
+impl Pausable for Gic {
+    fn pause(&mut self) -> std::result::Result<(), MigratableError> {
+        // Flush tables to guest RAM
+        let vgic = self.vgic.as_ref().unwrap().clone();
+        vgic.lock().unwrap().save_data_tables().map_err(|e| {
+            MigratableError::Pause(anyhow!(
+                "Could not save GICv3ITS GIC pending tables {:?}",
+                e
+            ))
+        })?;
+        Ok(())
+    }
+}
+impl Transportable for Gic {}
+impl Migratable for Gic {}
--- a/devices/src/interrupt_controller.rs
+++ b/devices/src/interrupt_controller.rs
@@ -24,6 +24,8 @@ pub enum Error {
    UpdateInterrupt(io::Error),
    /// Failed enabling the interrupt.
    EnableInterrupt(io::Error),
+    /// Failed creating GIC device.
+    CreateGic(hypervisor::HypervisorVmError),
 }

 type Result<T> = result::Result<T, Error>;
--- a/devices/src/legacy/uart_pl011.rs
+++ b/devices/src/legacy/uart_pl011.rs
@@ -372,7 +372,7 @@ impl BusDevice for Pl011 {
                UARTIFLS => self.ifl,
                UARTIMSC => self.int_enabled,
                UARTRIS => self.int_level,
-                UARTMIS => (self.int_level & self.int_enabled),
+                UARTMIS => self.int_level & self.int_enabled,
                UARTDMACR => self.dmacr,
                UARTDEBUG => self.debug,
                _ => {
--- a/docs/api.md
+++ b/docs/api.md
@@ -1,21 +1,21 @@
 - [Cloud Hypervisor API](#cloud-hypervisor-api)
-  * [External API](#external-api)
-    + [REST API](#rest-api)
-      - [Location and availability](#location-and-availability)
-      - [Endpoints](#endpoints)
-		* [Virtual Machine Manager (VMM) Actions](#virtual-machine-manager-vmm-actions)
-		* [Virtual Machine (VM) Actions](#virtual-machine-vm-actions)
-      - [REST API Examples](#rest-api-examples)
-        * [Create a Virtual Machine](#create-a-virtual-machine)
-	    * [Boot a Virtual Machine](#boot-a-virtual-machine)
-        * [Dump a Virtual Machine Information](#dump-a-virtual-machine-information)
-        * [Reboot a Virtual Machine](#reboot-a-virtual-machine)
-        * [Shut a Virtual Machine Down](#shut-a-virtual-machine-down)
-    + [Command Line Interface](#command-line-interface)
-    + [REST API and CLI Architectural Relationship](#rest-api-and-cli-architectural-relationship)
-  * [Internal API](#internal-api)
-    + [Goals and Design](#goals-and-design)
-  * [End to End Example](#end-to-end-example)
+  - [External API](#external-api)
+    - [REST API](#rest-api)
+    - [Location and availability](#location-and-availability)
+    - [Endpoints](#endpoints)
+      - [Virtual Machine Manager (VMM) Actions](#virtual-machine-manager-vmm-actions)
+      - [Virtual Machine (VM) Actions](#virtual-machine-vm-actions)
+    - [REST API Examples](#rest-api-examples)
+      - [Create a Virtual Machine](#create-a-virtual-machine)
+      - [Boot a Virtual Machine](#boot-a-virtual-machine)
+      - [Dump a Virtual Machine Information](#dump-a-virtual-machine-information)
+      - [Reboot a Virtual Machine](#reboot-a-virtual-machine)
+      - [Shut a Virtual Machine Down](#shut-a-virtual-machine-down)
+    - [Command Line Interface](#command-line-interface)
+    - [REST API and CLI Architectural Relationship](#rest-api-and-cli-architectural-relationship)
+  - [Internal API](#internal-api)
+    - [Goals and Design](#goals-and-design)
+  - [End to End Example](#end-to-end-example)

 # Cloud Hypervisor API

@@ -71,39 +71,40 @@ The Cloud Hypervisor API exposes the following actions through its endpoints:

 #### Virtual Machine Manager (VMM) Actions

-Action                              | Endpoint        | Request Body | Response Body              | Prerequisites
------------------------------------|-----------------|--------------|----------------------------|---------------------------
-Check for the REST API availability | `/vmm.ping`     | N/A          | `/schemas/VmmPingResponse` | N/A
-Shut the VMM down                   | `/vmm.shutdown` | N/A          | N/A                        | The VMM is running
+| Action                              | Endpoint        | Request Body | Response Body              | Prerequisites      |
+| ----------------------------------- | --------------- | ------------ | -------------------------- | ------------------ |
+| Check for the REST API availability | `/vmm.ping`     | N/A          | `/schemas/VmmPingResponse` | N/A                |
+| Shut the VMM down                   | `/vmm.shutdown` | N/A          | N/A                        | The VMM is running |

 #### Virtual Machine (VM) Actions

-Action                             | Endpoint             | Request Body              | Response Body            | Prerequisites
-----------------------------------|----------------------|---------------------------|--------------------------|---------------------------
-Create the VM                      | `/vm.create`         | `/schemas/VmConfig`       | N/A                      | The VM is not created yet
-Delete the VM                      | `/vm.delete`         | N/A                       | N/A                      | N/A
-Boot the VM                        | `/vm.boot`           | N/A                       | N/A                      | The VM is created but not booted
-Shut the VM down                   | `/vm.shutdown`       | N/A                       | N/A                      | The VM is booted
-Reboot the VM                      | `/vm.reboot`         | N/A                       | N/A                      | The VM is booted
-Trigger power button of the VM     | `/vm.power-button`   | N/A                       | N/A                      | The VM is booted
-Pause the VM                       | `/vm.pause`          | N/A                       | N/A                      | The VM is booted
-Resume the VM                      | `/vm.resume`         | N/A                       | N/A                      | The VM is paused
-Task a snapshot of the VM          | `/vm.snapshot`       | `/schemas/VmSnapshotConfig`| N/A                     | The VM is paused
-Restore the VM from a snapshot     | `/vm.restore`        | `/schemas/RestoreConfig`  | N/A                      | The VM is created but not booted
-Add/remove CPUs to/from the VM     | `/vm.resize`         | `/schemas/VmResize`       | N/A                      | The VM is booted
-Add/remove memory from the VM      | `/vm.resize`         | `/schemas/VmResize`       | N/A                      | The VM is booted
-Add/remove memory from a zone      | `/vm.resize-zone`    | `/schemas/VmResizeZone`   | N/A                      | The VM is booted
-Dump the VM information            | `/vm.info`           | N/A                       | `/schemas/VmInfo`        | The VM is created
-Add VFIO PCI device to the VM      | `/vm.add-device`     | `/schemas/VmAddDevice`    | `/schemas/PciDeviceInfo` | The VM is booted
-Add disk device to the VM          | `/vm.add-disk`       | `/schemas/DiskConfig`     | `/schemas/PciDeviceInfo` | The VM is booted
-Add fs device to the VM            | `/vm.add-fs`         | `/schemas/FsConfig`       | `/schemas/PciDeviceInfo` | The VM is booted
-Add pmem device to the VM          | `/vm.add-pmem`       | `/schemas/PmemConfig`     | `/schemas/PciDeviceInfo` | The VM is booted
-Add network device to the VM       | `/vm.add-net`        | `/schemas/NetConfig`      | `/schemas/PciDeviceInfo` | The VM is booted
-Add userspace PCI device to the VM | `/vm.add-user-device`| `/schemas/VmAddUserDevice`| `/schemas/PciDeviceInfo` | The VM is booted
-Add vdpa device to the VM          | `/vm.add-vdpa`       | `/schemas/VdpaConfig`     | `/schemas/PciDeviceInfo` | The VM is booted
-Add vsock device to the VM         | `/vm.add-vsock`      | `/schemas/VsockConfig`    | `/schemas/PciDeviceInfo` | The VM is booted
-Remove device from the VM          | `/vm.remove-device`  | `/schemas/VmRemoveDevice` | N/A                      | The VM is booted
-Dump the VM counters               | `/vm.counters`       | N/A                       | `/schemas/VmCounters`    | The VM is booted
+| Action                             | Endpoint              | Request Body                | Response Body            | Prerequisites                    |
+| ---------------------------------- | --------------------- | --------------------------- | ------------------------ | -------------------------------- |
+| Create the VM                      | `/vm.create`          | `/schemas/VmConfig`         | N/A                      | The VM is not created yet        |
+| Delete the VM                      | `/vm.delete`          | N/A                         | N/A                      | N/A                              |
+| Boot the VM                        | `/vm.boot`            | N/A                         | N/A                      | The VM is created but not booted |
+| Shut the VM down                   | `/vm.shutdown`        | N/A                         | N/A                      | The VM is booted                 |
+| Reboot the VM                      | `/vm.reboot`          | N/A                         | N/A                      | The VM is booted                 |
+| Trigger power button of the VM     | `/vm.power-button`    | N/A                         | N/A                      | The VM is booted                 |
+| Pause the VM                       | `/vm.pause`           | N/A                         | N/A                      | The VM is booted                 |
+| Resume the VM                      | `/vm.resume`          | N/A                         | N/A                      | The VM is paused                 |
+| Task a snapshot of the VM          | `/vm.snapshot`        | `/schemas/VmSnapshotConfig` | N/A                      | The VM is paused                 |
+| Perform a coredump of the VM       | `/vm.coredump`        | `/schemas/VmCoredumpData`   | N/A                      | The VM is paused                 |
+| Restore the VM from a snapshot     | `/vm.restore`         | `/schemas/RestoreConfig`    | N/A                      | The VM is created but not booted |
+| Add/remove CPUs to/from the VM     | `/vm.resize`          | `/schemas/VmResize`         | N/A                      | The VM is booted                 |
+| Add/remove memory from the VM      | `/vm.resize`          | `/schemas/VmResize`         | N/A                      | The VM is booted                 |
+| Add/remove memory from a zone      | `/vm.resize-zone`     | `/schemas/VmResizeZone`     | N/A                      | The VM is booted                 |
+| Dump the VM information            | `/vm.info`            | N/A                         | `/schemas/VmInfo`        | The VM is created                |
+| Add VFIO PCI device to the VM      | `/vm.add-device`      | `/schemas/VmAddDevice`      | `/schemas/PciDeviceInfo` | The VM is booted                 |
+| Add disk device to the VM          | `/vm.add-disk`        | `/schemas/DiskConfig`       | `/schemas/PciDeviceInfo` | The VM is booted                 |
+| Add fs device to the VM            | `/vm.add-fs`          | `/schemas/FsConfig`         | `/schemas/PciDeviceInfo` | The VM is booted                 |
+| Add pmem device to the VM          | `/vm.add-pmem`        | `/schemas/PmemConfig`       | `/schemas/PciDeviceInfo` | The VM is booted                 |
+| Add network device to the VM       | `/vm.add-net`         | `/schemas/NetConfig`        | `/schemas/PciDeviceInfo` | The VM is booted                 |
+| Add userspace PCI device to the VM | `/vm.add-user-device` | `/schemas/VmAddUserDevice`  | `/schemas/PciDeviceInfo` | The VM is booted                 |
+| Add vdpa device to the VM          | `/vm.add-vdpa`        | `/schemas/VdpaConfig`       | `/schemas/PciDeviceInfo` | The VM is booted                 |
+| Add vsock device to the VM         | `/vm.add-vsock`       | `/schemas/VsockConfig`      | `/schemas/PciDeviceInfo` | The VM is booted                 |
+| Remove device from the VM          | `/vm.remove-device`   | `/schemas/VmRemoveDevice`   | N/A                      | The VM is booted                 |
+| Dump the VM counters               | `/vm.counters`        | N/A                         | `/schemas/VmCounters`    | The VM is booted                 |

 ### REST API Examples

@@ -142,8 +143,7 @@ curl --unix-socket /tmp/cloud-hypervisor.sock -i \
     -H 'Content-Type: application/json'         \
     -d '{
         "cpus":{"boot_vcpus": 4, "max_vcpus": 4},
-         "kernel":{"path":"/opt/clh/kernel/vmlinux-virtio-fs-virtio-iommu"},
-         "cmdline":{"args":"console=ttyS0 console=hvc0 root=/dev/vda1 rw"},
+         "payload":{"kernel":"/opt/clh/kernel/vmlinux-virtio-fs-virtio-iommu", "cmdline":"console=ttyS0 console=hvc0 root=/dev/vda1 rw"},
         "disks":[{"path":"/opt/clh/images/focal-server-cloudimg-amd64.raw"}],
         "rng":{"src":"/dev/urandom"},
         "net":[{"ip":"192.168.10.10", "mask":"255.255.255.0", "mac":"12:34:56:78:90:01"}]
@@ -310,8 +310,7 @@ APIs work together, let's look at a complete VM creation flow, from the
 		-H 'Content-Type: application/json'         \
 		-d '{
 			"cpus":{"boot_vcpus": 4, "max_vcpus": 4},
-			"kernel":{"path":"/opt/clh/kernel/vmlinux-virtio-fs-virtio-iommu"},
-			"cmdline":{"args":"console=ttyS0 console=hvc0 root=/dev/vda1 rw"},
+			"payload":{"kernel":"/opt/clh/kernel/vmlinux-virtio-fs-virtio-iommu", "cmdline":"console=ttyS0 console=hvc0 root=/dev/vda1 rw"},
 			"disks":[{"path":"/opt/clh/images/focal-server-cloudimg-amd64.raw"}],
 			"rng":{"src":"/dev/urandom"},
 			"net":[{"ip":"192.168.10.10", "mask":"255.255.255.0", "mac":"12:34:56:78:90:01"}]
--- a/docs/custom-image.md
+++ b/docs/custom-image.md
@@ -59,6 +59,16 @@ mkdir -p /mnt
 sudo mount -o loop,offset=$((227328 * 512)) focal-server-cloudimg-amd64.raw /mnt
 ```

+### Set up DNS
+
+The next step describes changing the root directory to the rootfs contained by 
+the cloud image. For DNS to work in the root directory, you will need to first bind-mount 
+the host `/etc/resolv.conf` onto the mounted linux partition of the cloud image.
+
+```bash
+sudo mount -o bind /etc/resolv.conf /mnt/etc/resolv.conf
+```
+
 ### Change root directory

 Changing the root directory will allow us to install new packages to the rootfs
@@ -108,6 +118,7 @@ umount /dev/pts
 umount /proc
 history -c
 exit
+umount /mnt/etc/resolv.conf
 umount /mnt
 ```

--- a/docs/device_model.md
+++ b/docs/device_model.md
@@ -6,22 +6,22 @@ This document describes the device model supported by `cloud-hypervisor`.

 | Device | Build configurable | Enabled by default | Runtime configurable |
 | :----: | :----: | :----: | :----: |
-| Serial port | :negative_squared_cross_mark: | :negative_squared_cross_mark: | :heavy_check_mark: |
-| RTC/CMOS | :heavy_check_mark: | :heavy_check_mark: | :negative_squared_cross_mark: |
-| I/O APIC | :negative_squared_cross_mark: | :negative_squared_cross_mark: | :heavy_check_mark: |
-| i8042 shutdown/reboot | :negative_squared_cross_mark: | :negative_squared_cross_mark: | :negative_squared_cross_mark: |
-| ACPI shutdown/reboot | :negative_squared_cross_mark: | :heavy_check_mark: | :negative_squared_cross_mark: |
-| virtio-blk | :negative_squared_cross_mark: | :negative_squared_cross_mark: | :heavy_check_mark: |
-| virtio-console | :negative_squared_cross_mark: | :negative_squared_cross_mark: | :heavy_check_mark: |
-| virtio-iommu | :negative_squared_cross_mark: | :negative_squared_cross_mark: | :heavy_check_mark: |
-| virtio-net | :negative_squared_cross_mark: | :negative_squared_cross_mark: | :heavy_check_mark: |
-| virtio-pmem | :negative_squared_cross_mark: | :negative_squared_cross_mark: | :heavy_check_mark: |
-| virtio-rng | :negative_squared_cross_mark: | :negative_squared_cross_mark: | :heavy_check_mark: |
-| virtio-vsock | :negative_squared_cross_mark: | :negative_squared_cross_mark: | :heavy_check_mark: |
-| vhost-user-blk | :negative_squared_cross_mark: | :negative_squared_cross_mark: | :heavy_check_mark: |
-| vhost-user-fs | :negative_squared_cross_mark: | :negative_squared_cross_mark: | :heavy_check_mark: |
-| vhost-user-net | :negative_squared_cross_mark: | :negative_squared_cross_mark: | :heavy_check_mark: |
-| VFIO | :heavy_check_mark: | :negative_squared_cross_mark: | :heavy_check_mark: |
+| Serial port | :x: | :x: | :heavy_check_mark: |
+| RTC/CMOS | :heavy_check_mark: | :heavy_check_mark: | :x: |
+| I/O APIC | :x: | :x: | :heavy_check_mark: |
+| i8042 shutdown/reboot | :x: | :x: | :x: |
+| ACPI shutdown/reboot | :x: | :heavy_check_mark: | :x: |
+| virtio-blk | :x: | :x: | :heavy_check_mark: |
+| virtio-console | :x: | :x: | :heavy_check_mark: |
+| virtio-iommu | :x: | :x: | :heavy_check_mark: |
+| virtio-net | :x: | :x: | :heavy_check_mark: |
+| virtio-pmem | :x: | :x: | :heavy_check_mark: |
+| virtio-rng | :x: | :x: | :heavy_check_mark: |
+| virtio-vsock | :x: | :x: | :heavy_check_mark: |
+| vhost-user-blk | :x: | :x: | :heavy_check_mark: |
+| vhost-user-fs | :x: | :x: | :heavy_check_mark: |
+| vhost-user-net | :x: | :x: | :heavy_check_mark: |
+| VFIO | :heavy_check_mark: | :x: | :heavy_check_mark: |

 ## Legacy devices

--- a/docs/fs.md
+++ b/docs/fs.md
@@ -29,10 +29,11 @@ mkdir /tmp/shared_dir
 _Run virtiofsd_
 ```bash
 ./virtiofsd \
-    -d \
+    --log-level debug \
    --socket-path=/tmp/virtiofs \
    --shared-dir=/tmp/shared_dir \
-    --cache=never
+    --cache=never \
+    --thread-pool-size=$N
 ```

 The `cache=never` option is the default when using `virtiofsd` with
@@ -44,6 +45,10 @@ The `cache=always` option will allow the host page cache to be used, which can
 result in better performance for the guest's workload at the cost of increasing
 the footprint on host memory.

+The `thread-pool-size` option controls how many IO threads are spawned. For
+very fast storage like NVMe spawning enough worker threads is critical to
+getting an acceptable performance compared to native.
+
 ### Kernel support

 Modern Linux kernels (at least v5.10) have support for virtio-fs. Use of older
@@ -90,4 +95,4 @@ Cloud Hypervisor command line, which happens to be `myfs` in this example.
 ## DAX feature

 Given the DAX feature is not stable yet from a daemon standpoint, it is not
-available in Cloud Hypervisor.
+available in Cloud Hypervisor.
--- a/docs/intel_tdx.md
+++ b/docs/intel_tdx.md
@@ -103,8 +103,18 @@ option as well.
    --disk path=tdx_guest_img
 ```

-### Guest kernel disables serial ports
+### Guest kernel limitations
+
+#### Serial ports disabled

 The latest guest kernel that can be found in the latest image
 `td-guest-rhel8.5.raw` disabled the support for serial ports. This means adding
-`console=ttyS0` will have no effect and will not print any log from the guest.
+`console=ttyS0` will have no effect and will not print any log from the guest.
+
+#### PCI hotplug through ACPI
+
+Unless you run the guest kernel with the parameter `tdx_disable_filter`, ACPI
+devices responsible for handling PCI hotplug (PCI hotplug controller, PCI
+Express Bus and Generic Event Device) will not be allowed, therefore the
+corresponding drivers will not be loaded and the PCI hotplug feature will not
+be supported.
--- a/docs/profiling.md
+++ b/docs/profiling.md
@@ -43,3 +43,22 @@ $ perf report -g
 ```

 If profiling with a network device attached either the TAP device must be already created and configured or the profiling must be done as root so that the TAP device can be created.
+
+## Userspace only profiling with LBR
+
+The use of LBR (Last Branch Record; available since Haswell) offers lower
+overhead if only userspace profiling is required. This lower overhead can allow
+a higher frequency of sampling. This also removes the requirement to compile
+with custom `RUSTFLAGS` however debug symbols should still be included:
+
+e.g.
+
+```
+$ perf record --call-graph lbr --all-user --user-callchains -g target/release/cloud-hypervisor \
+        --kernel ~/src/linux/vmlinux \
+        --pmem file=~/workloads/focal.raw \
+        --cpus boot=1 --memory size=1G \
+        --cmdline "root=/dev/pmem0p1 console=ttyS0" \
+        --serial tty --console off \
+        --api-socket=/tmp/api1
+```
--- a/event_monitor/Cargo.toml
+++ b/event_monitor/Cargo.toml
@@ -5,6 +5,6 @@ authors = ["The Cloud Hypervisor Authors"]
 edition = "2021"

 [dependencies]
-libc = "0.2.126"
-serde = { version = "1.0.137", features = ["rc", "derive"] }
-serde_json = "1.0.81"
+libc = "0.2.131"
+serde = { version = "1.0.143", features = ["rc", "derive"] }
+serde_json = "1.0.83"
--- a/fuzz/Cargo.lock
+++ b/fuzz/Cargo.lock
@@ -11,9 +11,9 @@ dependencies = [

 [[package]]
 name = "anyhow"
-version = "1.0.57"
+version = "1.0.62"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "08f9b8508dccb7687a1d6c4ce66b2b0ecef467c94667de27d8d7fe1f8d2a9cdc"
+checksum = "1485d4d2cc45e7b201ee3767015c96faa5904387c9d87c6efdd0fb511f12d305"

 [[package]]
 name = "api_client"
@@ -24,21 +24,20 @@ dependencies = [

 [[package]]
 name = "arbitrary"
-version = "1.1.0"
+version = "1.1.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c38b6b6b79f671c25e1a3e785b7b82d7562ffc9cd3efdc98627e5668a2472490"
+checksum = "5a7924531f38b1970ff630f03eb20a2fde69db5c590c93b0f3482e95dcc5fd60"

 [[package]]
 name = "arc-swap"
-version = "1.5.0"
+version = "1.5.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c5d78ce20460b82d3fa150275ed9d55e21064fc7951177baacf86a145c4a4b1f"
+checksum = "983cd8b9d4b02a6dc6ffa557262eb5858a27a0038ffffe21a0f133eaa819a164"

 [[package]]
 name = "arch"
 version = "0.1.0"
 dependencies = [
- "acpi_tables",
 "anyhow",
 "byteorder",
 "fdt",
@@ -48,6 +47,7 @@ dependencies = [
 "log",
 "serde",
 "thiserror",
+ "uuid",
 "versionize",
 "versionize_derive",
 "vm-fdt",
@@ -127,15 +127,15 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"

 [[package]]
 name = "clap"
-version = "3.1.18"
+version = "3.2.17"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d2dbdf4bdacb33466e854ce889eee8dfd5729abf7ccd7664d0a2d60cd384440b"
+checksum = "29e724a68d9319343bb3328c9cc2dfde263f4b3142ee1059a9980580171c954b"
 dependencies = [
 "atty",
 "bitflags",
 "clap_lex",
 "indexmap",
- "lazy_static",
+ "once_cell",
 "strsim",
 "termcolor",
 "terminal_size",
@@ -144,16 +144,16 @@ dependencies = [

 [[package]]
 name = "clap_lex"
-version = "0.2.0"
+version = "0.2.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a37c35f1112dad5e6e0b1adaff798507497a18fceeb30cceb3bae7d1427b9213"
+checksum = "2850f2f5a82cbf437dd5af4d49848fbdfc27c157c3d010345776f952765261c5"
 dependencies = [
 "os_str_bytes",
 ]

 [[package]]
 name = "cloud-hypervisor"
-version = "23.0.0"
+version = "25.0.0"
 dependencies = [
 "anyhow",
 "api_client",
@@ -179,15 +179,21 @@ version = "0.0.0"
 dependencies = [
 "block_util",
 "cloud-hypervisor",
+ "devices",
+ "epoll",
 "libc",
 "libfuzzer-sys",
+ "micro_http",
+ "once_cell",
 "qcow",
 "seccompiler",
 "vhdx",
 "virtio-devices",
 "virtio-queue",
+ "vm-device",
 "vm-memory",
 "vm-virtio",
+ "vmm",
 "vmm-sys-util",
 ]

@@ -206,6 +212,41 @@ version = "1.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "55626594feae15d266d52440b26ff77de0e22230cf0c113abe619084c1ddc910"

+[[package]]
+name = "darling"
+version = "0.14.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4529658bdda7fd6769b8614be250cdcfc3aeb0ee72fe66f9e41e5e5eb73eac02"
+dependencies = [
+ "darling_core",
+ "darling_macro",
+]
+
+[[package]]
+name = "darling_core"
+version = "0.14.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "649c91bc01e8b1eac09fb91e8dbc7d517684ca6be8ebc75bb9cafc894f9fdb6f"
+dependencies = [
+ "fnv",
+ "ident_case",
+ "proc-macro2",
+ "quote",
+ "strsim",
+ "syn",
+]
+
+[[package]]
+name = "darling_macro"
+version = "0.14.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ddfc69c5bfcbd2fc09a0f38451d2daf0e372e367986a83906d1b0dbc88134fb5"
+dependencies = [
+ "darling_core",
+ "quote",
+ "syn",
+]
+
 [[package]]
 name = "devices"
 version = "0.1.0"
@@ -215,7 +256,7 @@ dependencies = [
 "arch",
 "bitflags",
 "byteorder",
- "epoll",
+ "hypervisor",
 "libc",
 "log",
 "versionize",
@@ -247,39 +288,21 @@ dependencies = [

 [[package]]
 name = "fdt"
-version = "0.1.3"
+version = "0.1.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b643857cf70949306b81d7e92cb9d47add673868edac9863c4a49c42feaf3f1e"
+checksum = "964f5becd44d069dca0beea2b4bc05639ae7bf3b3f5369c295aff360bb57cca2"

 [[package]]
-name = "gdbstub"
-version = "0.6.1"
+name = "fnv"
+version = "1.0.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9fa2ca5d6b045de372cef3991f873389b421b4cabcfbe52f7787fae8b8b37906"
-dependencies = [
- "bitflags",
- "cfg-if",
- "log",
- "managed",
- "num-traits",
- "paste",
-]
-
-[[package]]
-name = "gdbstub_arch"
-version = "0.2.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "51dc4b5718ac76d21e8605c0966dd32d80273b89b11e6cfef467b04e45934d37"
-dependencies = [
- "gdbstub",
- "num-traits",
-]
+checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"

 [[package]]
 name = "getrandom"
-version = "0.2.6"
+version = "0.2.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9be70c98951c83b8d2f8f60d7065fa6d5146873094452a1008da8c2f1e4205ad"
+checksum = "4eb1a864a501629691edf6c15a593b7a51eebaa1e8468e9ddc623de7c9b58ec6"
 dependencies = [
 "cfg-if",
 "libc",
@@ -288,9 +311,9 @@ dependencies = [

 [[package]]
 name = "hashbrown"
-version = "0.11.2"
+version = "0.12.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e"
+checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888"

 [[package]]
 name = "hermit-abi"
@@ -306,15 +329,16 @@ name = "hypervisor"
 version = "0.1.0"
 dependencies = [
 "anyhow",
- "epoll",
+ "byteorder",
 "iced-x86",
 "kvm-bindings",
 "kvm-ioctls",
 "libc",
 "log",
 "serde",
- "serde_json",
+ "serde_with",
 "thiserror",
+ "vfio-ioctls",
 "vm-memory",
 "vmm-sys-util",
 ]
@@ -330,10 +354,16 @@ dependencies = [
 ]

 [[package]]
-name = "indexmap"
-version = "1.8.1"
+name = "ident_case"
+version = "1.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0f647032dfaa1f8b6dc29bd3edb7bbef4861b8b8007ebb118d6db284fd59f6ee"
+checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39"
+
+[[package]]
+name = "indexmap"
+version = "1.9.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "10a35a97730320ffe8e2d410b5d3b69279b98d2c14bdb8b70ea89ecf7888d41e"
 dependencies = [
 "autocfg",
 "hashbrown",
@@ -341,9 +371,9 @@ dependencies = [

 [[package]]
 name = "io-uring"
-version = "0.5.2"
+version = "0.5.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8d75829ed9377bab6c90039fe47b9d84caceb4b5063266142e21bcce6550cda8"
+checksum = "427496d698429d86ff947537dff68facb83fe753b7abea4fab4ac1c4b7a4b94b"
 dependencies = [
 "bitflags",
 "libc",
@@ -351,9 +381,9 @@ dependencies = [

 [[package]]
 name = "itoa"
-version = "1.0.2"
+version = "1.0.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "112c678d4050afce233f4f2852bb2eb519230b3cf12f33585275537d7e41578d"
+checksum = "6c8af84674fe1f223a982c933a0ee1086ac4d4052aa0fb8060c12c6ad838e754"

 [[package]]
 name = "kvm-bindings"
@@ -383,9 +413,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"

 [[package]]
 name = "libc"
-version = "0.2.126"
+version = "0.2.131"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "349d5a591cd28b49e1d1037471617a32ddcda5731b99419008085f72d5a53836"
+checksum = "04c3b4822ccebfa39c02fc03d1534441b22ead323fa0f48bb7ddd8e6ba076a40"

 [[package]]
 name = "libfuzzer-sys"
@@ -416,16 +446,10 @@ dependencies = [
 "cfg-if",
 ]

-[[package]]
-name = "managed"
-version = "0.8.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0ca88d725a0a943b096803bd34e73a4437208b6077654cc4ecb2947a5f91618d"
-
 [[package]]
 name = "micro_http"
 version = "0.1.0"
-source = "git+https://github.com/firecracker-microvm/micro-http?branch=main#9517a300370a158a7af0996b7eebf040d171e1a4"
+source = "git+https://github.com/firecracker-microvm/micro-http?branch=main#863b0370ba7e57f7df5b908ada9e5b44809ccae9"
 dependencies = [
 "libc",
 "vmm-sys-util",
@@ -449,6 +473,7 @@ dependencies = [
 "net_gen",
 "rate_limiter",
 "serde",
+ "thiserror",
 "versionize",
 "versionize_derive",
 "virtio-bindings",
@@ -458,20 +483,11 @@ dependencies = [
 "vmm-sys-util",
 ]

-[[package]]
-name = "num-traits"
-version = "0.2.15"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd"
-dependencies = [
- "autocfg",
-]
-
 [[package]]
 name = "once_cell"
-version = "1.12.0"
+version = "1.13.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7709cef83f0c1f58f666e746a08b21e0085f7440fa6a29cc194d68aac97a4225"
+checksum = "074864da206b4973b84eb91683020dbefd6a8c3f0f38e054d93954e891935e4e"

 [[package]]
 name = "option_parser"
@@ -479,15 +495,9 @@ version = "0.1.0"

 [[package]]
 name = "os_str_bytes"
-version = "6.0.1"
+version = "6.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "029d8d0b2f198229de29dca79676f2738ff952edf3fde542eb8bf94d8c21b435"
-
-[[package]]
-name = "paste"
-version = "1.0.7"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0c520e05135d6e763148b6426a837e239041653ba7becd2e538c076c738025fc"
+checksum = "9ff7415e9ae3fff1225851df9e0d9e4e5479f947619774677a63572e55e80eff"

 [[package]]
 name = "pci"
@@ -514,9 +524,9 @@ dependencies = [

 [[package]]
 name = "proc-macro2"
-version = "1.0.39"
+version = "1.0.43"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c54b25569025b7fc9651de43004ae593a75ad88543b17178aa5e1b9c4f15f56f"
+checksum = "0a2ca2c61bc9f3d74d2886294ab7b9853abd9c1ad903a3ac7815c58989bb7bab"
 dependencies = [
 "unicode-ident",
 ]
@@ -534,9 +544,9 @@ dependencies = [

 [[package]]
 name = "quote"
-version = "1.0.18"
+version = "1.0.21"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a1feb54ed693b93a84e14094943b84b7c4eae204c512b7ccb95ab0c66d278ad1"
+checksum = "bbe448f377a7d6961e30f5955f9b8d106c3f5e449d493ee1b125c1d43c2b5179"
 dependencies = [
 "proc-macro2",
 ]
@@ -552,9 +562,9 @@ dependencies = [

 [[package]]
 name = "remain"
-version = "0.2.3"
+version = "0.2.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0c35270ea384ac1762895831cc8acb96f171468e52cec82ed9186f9416209fa4"
+checksum = "06df529c0d271b27ac4a2c260f5ce2914b60bd44702cecec7b9f271adbdde23b"
 dependencies = [
 "proc-macro2",
 "quote",
@@ -572,9 +582,9 @@ dependencies = [

 [[package]]
 name = "ryu"
-version = "1.0.10"
+version = "1.0.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f3f6f92acf49d1b98f7a81226834412ada05458b7364277387724a237f062695"
+checksum = "4501abdff3ae82a1c1b477a17252eb69cee9e66eb915c1abaa4f44d873df9f09"

 [[package]]
 name = "seccompiler"
@@ -587,24 +597,24 @@ dependencies = [

 [[package]]
 name = "semver"
-version = "1.0.9"
+version = "1.0.13"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8cb243bdfdb5936c8dc3c45762a19d12ab4550cdc753bc247637d4ec35a040fd"
+checksum = "93f6841e709003d68bb2deee8c343572bf446003ec20a583e76f7b15cebf3711"

 [[package]]
 name = "serde"
-version = "1.0.137"
+version = "1.0.143"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "61ea8d54c77f8315140a05f4c7237403bf38b72704d031543aa1d16abbf517d1"
+checksum = "53e8e5d5b70924f74ff5c6d64d9a5acd91422117c60f48c4e07855238a254553"
 dependencies = [
 "serde_derive",
 ]

 [[package]]
 name = "serde_derive"
-version = "1.0.137"
+version = "1.0.143"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1f26faba0c3959972377d3b2d306ee9f71faee9714294e41bb777f83f88578be"
+checksum = "d3d8e8de557aee63c26b85b947f5e59b690d0454c753f3adeb5cd7835ab88391"
 dependencies = [
 "proc-macro2",
 "quote",
@@ -613,15 +623,37 @@ dependencies = [

 [[package]]
 name = "serde_json"
-version = "1.0.81"
+version = "1.0.83"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9b7ce2b32a1aed03c558dc61a5cd328f15aff2dbc17daad8fb8af04d2100e15c"
+checksum = "38dd04e3c8279e75b31ef29dbdceebfe5ad89f4d0937213c53f7d49d01b3d5a7"
 dependencies = [
 "itoa",
 "ryu",
 "serde",
 ]

+[[package]]
+name = "serde_with"
+version = "2.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "89df7a26519371a3cce44fbb914c2819c84d9b897890987fa3ab096491cc0ea8"
+dependencies = [
+ "serde",
+ "serde_with_macros",
+]
+
+[[package]]
+name = "serde_with_macros"
+version = "2.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "de337f322382fcdfbb21a014f7c224ee041a23785651db67b9827403178f698f"
+dependencies = [
+ "darling",
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
 [[package]]
 name = "signal-hook"
 version = "0.3.14"
@@ -655,9 +687,9 @@ checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623"

 [[package]]
 name = "syn"
-version = "1.0.95"
+version = "1.0.99"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fbaf6116ab8924f39d52792136fb74fd60a80194cf1b1c6ffa6453eef1c3f942"
+checksum = "58dbef6ec655055e20b86b15a8cc6d439cca19b667537ac6a1369572d151ab13"
 dependencies = [
 "proc-macro2",
 "quote",
@@ -694,18 +726,18 @@ dependencies = [

 [[package]]
 name = "thiserror"
-version = "1.0.31"
+version = "1.0.32"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bd829fe32373d27f76265620b5309d0340cb8550f523c1dda251d6298069069a"
+checksum = "f5f6586b7f764adc0231f4c79be7b920e766bb2f3e51b3661cdb263828f19994"
 dependencies = [
 "thiserror-impl",
 ]

 [[package]]
 name = "thiserror-impl"
-version = "1.0.31"
+version = "1.0.32"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0396bc89e626244658bef819e22d0cc459e795a5ebe878e6ec336d1674a8d79a"
+checksum = "12bafc5b54507e0149cdf1b145a5d80ab80a90bcd9275df43d4fff68460f6c21"
 dependencies = [
 "proc-macro2",
 "quote",
@@ -714,15 +746,15 @@ dependencies = [

 [[package]]
 name = "unicode-ident"
-version = "1.0.0"
+version = "1.0.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d22af068fba1eb5edcb4aea19d382b2a3deb4c8f9d475c589b6ada9e0fd493ee"
+checksum = "c4f5b37a154999a8f3f98cc23a628d850e154479cd94decf3414696e12e31aaf"

 [[package]]
 name = "uuid"
-version = "1.0.0"
+version = "1.1.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8cfcd319456c4d6ea10087ed423473267e1a071f3bc0aa89f80d60997843c6f0"
+checksum = "dd6469f4314d5f1ffec476e05f17cc9a78bc7a27a6a857842170bdf8d6f98d2f"
 dependencies = [
 "getrandom",
 ]
@@ -766,7 +798,7 @@ dependencies = [
 [[package]]
 name = "vfio-ioctls"
 version = "0.1.0"
-source = "git+https://github.com/rust-vmm/vfio?branch=main#4630612f2ff300e78b6d55be324fb5481aa84661"
+source = "git+https://github.com/rust-vmm/vfio?branch=main#1ff2324e5cb0fccab825f9c96bb9387a1a01863f"
 dependencies = [
 "byteorder",
 "kvm-bindings",
@@ -783,7 +815,6 @@ dependencies = [
 name = "vfio_user"
 version = "0.1.0"
 dependencies = [
- "anyhow",
 "libc",
 "log",
 "serde",
@@ -806,7 +837,6 @@ dependencies = [
 "remain",
 "thiserror",
 "uuid",
- "vmm-sys-util",
 ]

 [[package]]
@@ -863,11 +893,12 @@ dependencies = [

 [[package]]
 name = "virtio-queue"
-version = "0.3.0"
+version = "0.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "88f2d73c184c18f8acc32dab77fcb6e3af92d53262538d3a68aa474810d6863c"
+checksum = "b4f59652909f276e6edd8bf36e9f106480b2202f5f046717b3de14f1b4072a28"
 dependencies = [
 "log",
+ "virtio-bindings",
 "vm-memory",
 "vmm-sys-util",
 ]
@@ -888,7 +919,6 @@ dependencies = [
 "anyhow",
 "hypervisor",
 "serde",
- "serde_json",
 "thiserror",
 "vfio-ioctls",
 "vm-memory",
@@ -929,7 +959,6 @@ name = "vm-virtio"
 version = "0.1.0"
 dependencies = [
 "log",
- "virtio-bindings",
 "virtio-queue",
 "vm-memory",
 ]
@@ -948,15 +977,13 @@ dependencies = [
 "devices",
 "epoll",
 "event_monitor",
- "gdbstub",
- "gdbstub_arch",
 "hypervisor",
- "lazy_static",
 "libc",
 "linux-loader",
 "log",
 "micro_http",
 "net_util",
+ "once_cell",
 "option_parser",
 "pci",
 "qcow",
@@ -983,9 +1010,9 @@ dependencies = [

 [[package]]
 name = "vmm-sys-util"
-version = "0.9.0"
+version = "0.10.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "733537bded03aaa93543f785ae997727b30d1d9f4a03b7861d23290474242e11"
+checksum = "08604d7be03eb26e33b3cee3ed4aef2bf550b305d1cca60e84da5d28d3790b62"
 dependencies = [
 "bitflags",
 "libc",
@@ -995,9 +1022,9 @@ dependencies = [

 [[package]]
 name = "wasi"
-version = "0.10.2+wasi-snapshot-preview1"
+version = "0.11.0+wasi-snapshot-preview1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6"
+checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"

 [[package]]
 name = "winapi"
--- a/fuzz/Cargo.toml
+++ b/fuzz/Cargo.toml
@@ -10,16 +10,22 @@ cargo-fuzz = true

 [dependencies]
 block_util = { path = "../block_util" }
-libc = "0.2.126"
+devices = { path = "../devices" }
+epoll = "4.3.1"
+libc = "0.2.131"
 libfuzzer-sys = "0.4.3"
+micro_http = { git = "https://github.com/firecracker-microvm/micro-http", branch = "main" }
+once_cell = "1.13.1"
 qcow = { path = "../qcow" }
 seccompiler = "0.2.0"
 vhdx = { path = "../vhdx" }
 virtio-devices = { path = "../virtio-devices" }
-virtio-queue = "0.3.0"
-vmm-sys-util = "0.9.0"
-vm-virtio = { path = "../vm-virtio" }
+virtio-queue = "0.5.0"
+vmm = { path = "../vmm" }
+vmm-sys-util = "0.10.0"
 vm-memory = "0.8.0"
+vm-device = { path = "../vm-device" }
+vm-virtio = { path = "../vm-virtio" }

 [dependencies.cloud-hypervisor]
 path = ".."
@@ -33,6 +39,24 @@ versionize_derive = { git = "https://github.com/cloud-hypervisor/versionize_deri
 [workspace]
 members = ["."]

+[[bin]]
+name = "block"
+path = "fuzz_targets/block.rs"
+test = false
+doc = false
+
+[[bin]]
+name = "cmos"
+path = "fuzz_targets/cmos.rs"
+test = false
+doc = false
+
+[[bin]]
+name = "http_api"
+path = "fuzz_targets/http_api.rs"
+test = false
+doc = false
+
 [[bin]]
 name = "qcow"
 path = "fuzz_targets/qcow.rs"
@@ -40,8 +64,8 @@ test = false
 doc = false

 [[bin]]
-name = "block"
-path = "fuzz_targets/block.rs"
+name = "serial"
+path = "fuzz_targets/serial.rs"
 test = false
 doc = false

--- a/fuzz/fuzz_targets/block.rs
+++ b/fuzz/fuzz_targets/block.rs
@@ -9,84 +9,39 @@ use libfuzzer_sys::fuzz_target;
 use seccompiler::SeccompAction;
 use std::ffi;
 use std::fs::File;
-use std::io::{self, Cursor, Read, Seek, SeekFrom};
-use std::mem::size_of;
+use std::io;
 use std::os::unix::io::{AsRawFd, FromRawFd, RawFd};
 use std::path::PathBuf;
 use std::sync::Arc;
 use virtio_devices::{Block, VirtioDevice, VirtioInterrupt, VirtioInterruptType};
-use virtio_queue::{Queue, QueueState};
+use virtio_queue::{Queue, QueueT};
 use vm_memory::{bitmap::AtomicBitmap, Bytes, GuestAddress, GuestMemoryAtomic};
+use vmm::{EpollContext, EpollDispatch};
 use vmm_sys_util::eventfd::{EventFd, EFD_NONBLOCK};

 type GuestMemoryMmap = vm_memory::GuestMemoryMmap<AtomicBitmap>;

 const MEM_SIZE: u64 = 256 * 1024 * 1024;
-const DESC_SIZE: u64 = 16; // Bytes in one virtio descriptor.
 const QUEUE_SIZE: u16 = 16; // Max entries in the queue.
-const CMD_SIZE: usize = 16; // Bytes in the command.

 fuzz_target!(|bytes| {
-    let size_u64 = size_of::<u64>();
+    if bytes.len() as u64 > MEM_SIZE {
+        return;
+    }
+
    let mem = GuestMemoryMmap::from_ranges(&[(GuestAddress(0), MEM_SIZE as usize)]).unwrap();
-
-    // The fuzz data is interpreted as:
-    // starting index 8 bytes
-    // command location 8 bytes
-    // command 16 bytes
-    // descriptors circular buffer 16 bytes * 3
-    if bytes.len() < 4 * size_u64 {
-        // Need an index to start.
-        return;
-    }
-
-    let mut data_image = Cursor::new(bytes);
-
-    let first_index = read_u64(&mut data_image);
-    if first_index > MEM_SIZE / DESC_SIZE {
-        return;
-    }
-    let first_offset = first_index * DESC_SIZE;
-    if first_offset as usize + size_u64 > bytes.len() {
-        return;
-    }
-
-    let command_addr = read_u64(&mut data_image);
-    if command_addr > MEM_SIZE - CMD_SIZE as u64 {
-        return;
-    }
-    if mem
-        .write_slice(
-            &bytes[2 * size_u64..(2 * size_u64) + CMD_SIZE],
-            GuestAddress(command_addr as u64),
-        )
-        .is_err()
-    {
-        return;
-    }
-
-    data_image.seek(SeekFrom::Start(first_offset)).unwrap();
-    let desc_table = read_u64(&mut data_image);
-
-    if mem
-        .write_slice(&bytes[32..], GuestAddress(desc_table as u64))
-        .is_err()
-    {
+    if mem.write_slice(bytes, GuestAddress(0 as u64)).is_err() {
        return;
    }

    let guest_memory = GuestMemoryAtomic::new(mem);

-    let mut q = Queue::<
-        GuestMemoryAtomic<GuestMemoryMmap>,
-        QueueState,
-    >::new(guest_memory.clone(), QUEUE_SIZE);
-    q.state.ready = true;
-    q.state.size = QUEUE_SIZE / 2;
+    let mut q = Queue::new(QUEUE_SIZE).unwrap();
+    q.set_ready(true);
+    q.set_size(QUEUE_SIZE / 2);

-    let queue_evts: Vec<EventFd> = vec![EventFd::new(0).unwrap()];
-    let queue_fd = queue_evts[0].as_raw_fd();
-    let queue_evt = unsafe { EventFd::from_raw_fd(libc::dup(queue_fd)) };
+    let evt = EventFd::new(0).unwrap();
+    let queue_evt = unsafe { EventFd::from_raw_fd(libc::dup(evt.as_raw_fd())) };

    let shm = memfd_create(&ffi::CString::new("fuzz").unwrap(), 0).unwrap();
    let disk_file: File = unsafe { File::from_raw_fd(shm) };
@@ -110,19 +65,16 @@ fuzz_target!(|bytes| {
        .activate(
            guest_memory,
            Arc::new(NoopVirtioInterrupt {}),
-            vec![q],
-            queue_evts,
+            vec![(0, q, evt)],
        )
        .ok();

    queue_evt.write(77).unwrap(); // Rings the doorbell, any byte will do.
-});

-fn read_u64<T: Read>(readable: &mut T) -> u64 {
-    let mut buf = [0u8; size_of::<u64>()];
-    readable.read_exact(&mut buf[..]).unwrap();
-    u64::from_le_bytes(buf)
-}
+    wait_queue_event_processed(queue_evt);
+
+    block.reset(); // Ensure the virtio-block device thread is killed and joined
+});

 fn memfd_create(name: &ffi::CStr, flags: u32) -> Result<RawFd, io::Error> {
    let res = unsafe { libc::syscall(libc::SYS_memfd_create, name.as_ptr(), flags) };
@@ -137,10 +89,33 @@ fn memfd_create(name: &ffi::CStr, flags: u32) -> Result<RawFd, io::Error> {
 pub struct NoopVirtioInterrupt {}

 impl VirtioInterrupt for NoopVirtioInterrupt {
-    fn trigger(
-        &self,
-        _int_type: VirtioInterruptType,
-    ) -> std::result::Result<(), std::io::Error> {
+    fn trigger(&self, _int_type: VirtioInterruptType) -> std::result::Result<(), std::io::Error> {
        Ok(())
    }
 }
+
+fn wait_queue_event_processed(queue_evt: EventFd) {
+    let mut epoll = EpollContext::new().unwrap();
+    epoll.add_event(&queue_evt, EpollDispatch::Api).unwrap();
+
+    let epoll_fd = epoll.as_raw_fd();
+    let mut events = vec![epoll::Event::new(epoll::Events::empty(), 0); 1];
+
+    loop {
+        let num_events = match epoll::wait(epoll_fd, 0, &mut events[..]) {
+            Ok(res) => res,
+            Err(e) => {
+                if e.kind() == io::ErrorKind::Interrupted {
+                    continue;
+                }
+
+                println!("Warning: Unexpected epoll::wait() error!");
+                break;
+            }
+        };
+
+        if num_events == 0 {
+            break;
+        }
+    }
+}
--- a/fuzz/fuzz_targets/cmos.rs
+++ b/fuzz/fuzz_targets/cmos.rs
@@ -0,0 +1,48 @@
+// Copyright © 2022 Intel Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+
+#![no_main]
+use devices::legacy::Cmos;
+use libc::EFD_NONBLOCK;
+use libfuzzer_sys::fuzz_target;
+use vm_device::BusDevice;
+use vmm_sys_util::eventfd::EventFd;
+
+fuzz_target!(|bytes| {
+    // Need at least 16 bytes for the test
+    if bytes.len() < 16 {
+        return;
+    }
+
+    let mut below_4g = [0u8; 8];
+    let mut above_4g = [0u8; 8];
+
+    below_4g.copy_from_slice(&bytes[0..8]);
+    above_4g.copy_from_slice(&bytes[8..16]);
+
+    let mut cmos = Cmos::new(
+        u64::from_le_bytes(below_4g),
+        u64::from_le_bytes(above_4g),
+        EventFd::new(EFD_NONBLOCK).unwrap(),
+    );
+
+    let mut i = 16;
+    while i < bytes.len() {
+        let read = bytes.get(i).unwrap_or(&0) % 2 == 0;
+        i += 1;
+
+        if read {
+            let offset = (bytes.get(i).unwrap_or(&0) % 2) as u64;
+            i += 1;
+            let mut out_bytes = vec![0];
+            cmos.read(0, offset, &mut out_bytes);
+        } else {
+            let offset = (bytes.get(i).unwrap_or(&0) % 2) as u64;
+            i += 1;
+            let data = vec![*bytes.get(i).unwrap_or(&0)];
+            i += 1;
+            cmos.write(0, offset, &data);
+        }
+    }
+});
--- a/fuzz/fuzz_targets/http_api.rs
+++ b/fuzz/fuzz_targets/http_api.rs
@@ -0,0 +1,191 @@
+// Copyright © 2022 Intel Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+
+#![no_main]
+use libfuzzer_sys::fuzz_target;
+use micro_http::Request;
+use once_cell::sync::Lazy;
+use std::os::unix::io::AsRawFd;
+use std::sync::mpsc::{channel, Receiver};
+use std::thread;
+use vmm::api::{http::*, ApiRequest, ApiResponsePayload};
+use vmm::{EpollContext, EpollDispatch};
+use vmm_sys_util::eventfd::EventFd;
+
+// Need to be ordered for test case reproducibility
+static ROUTES: Lazy<Vec<&Box<dyn EndpointHandler + Sync + Send>>> =
+    Lazy::new(|| HTTP_ROUTES.routes.values().collect());
+
+fuzz_target!(|bytes| {
+    if bytes.len() < 2 {
+        return;
+    }
+
+    let route = ROUTES[bytes[0] as usize % ROUTES.len()];
+    if let Some(request) = generate_request(&bytes[1..]) {
+        let exit_evt = EventFd::new(libc::EFD_NONBLOCK).unwrap();
+        let api_evt = EventFd::new(libc::EFD_NONBLOCK).unwrap();
+        let (api_sender, api_receiver) = channel();
+
+        let http_receiver_thread = {
+            let exit_evt = exit_evt.try_clone().unwrap();
+            let api_evt = api_evt.try_clone().unwrap();
+            thread::Builder::new()
+                .name("http_receiver".to_string())
+                .spawn(move || {
+                    http_receiver_stub(exit_evt, api_evt, api_receiver);
+                })
+                .unwrap()
+        };
+
+        route.handle_request(&request, api_evt, api_sender);
+        exit_evt.write(1).ok();
+        http_receiver_thread.join().unwrap();
+    };
+});
+
+fn generate_request(bytes: &[u8]) -> Option<Request> {
+    let req_method = match bytes[0] % 5 {
+        0 => "GET",
+        1 => "PUT",
+        2 => "PATCH",
+        3 => "POST",
+        _ => "INVALID",
+    };
+    let request_line = format!("{} http://localhost/home HTTP/1.1\r\n", req_method);
+
+    let req_body = &bytes[1..];
+    let request = if req_body.len() > 0 {
+        [
+            format!("{}Content-Length: {}\r\n", request_line, req_body.len()).as_bytes(),
+            req_body,
+        ]
+        .concat()
+    } else {
+        format!("{}\r\n", request_line).as_bytes().to_vec()
+    };
+
+    Request::try_from(&request, None).ok()
+}
+
+fn http_receiver_stub(exit_evt: EventFd, api_evt: EventFd, api_receiver: Receiver<ApiRequest>) {
+    let mut epoll = EpollContext::new().unwrap();
+    epoll.add_event(&exit_evt, EpollDispatch::Exit).unwrap();
+    epoll.add_event(&api_evt, EpollDispatch::Api).unwrap();
+
+    let epoll_fd = epoll.as_raw_fd();
+    let mut events = vec![epoll::Event::new(epoll::Events::empty(), 0); 2];
+    let num_events;
+    loop {
+        num_events = match epoll::wait(epoll_fd, -1, &mut events[..]) {
+            Ok(num_events) => num_events,
+            Err(e) => match e.raw_os_error() {
+                Some(libc::EAGAIN) | Some(libc::EINTR) => continue,
+                _ => panic!("Unexpected epoll::wait error!"),
+            },
+        };
+
+        break;
+    }
+
+    for event in events.iter().take(num_events) {
+        let dispatch_event: EpollDispatch = event.data.into();
+        match dispatch_event {
+            EpollDispatch::Exit => {
+                break;
+            }
+            EpollDispatch::Api => {
+                for _ in 0..api_evt.read().unwrap() {
+                    let api_request = api_receiver.recv().unwrap();
+                    match api_request {
+                        ApiRequest::VmCreate(_, sender) => {
+                            sender.send(Ok(ApiResponsePayload::Empty)).unwrap();
+                        }
+                        ApiRequest::VmDelete(sender) => {
+                            sender.send(Ok(ApiResponsePayload::Empty)).unwrap();
+                        }
+                        ApiRequest::VmBoot(sender) => {
+                            sender.send(Ok(ApiResponsePayload::Empty)).unwrap();
+                        }
+                        ApiRequest::VmShutdown(sender) => {
+                            sender.send(Ok(ApiResponsePayload::Empty)).unwrap();
+                        }
+                        ApiRequest::VmReboot(sender) => {
+                            sender.send(Ok(ApiResponsePayload::Empty)).unwrap();
+                        }
+                        ApiRequest::VmInfo(sender) => {
+                            sender.send(Ok(ApiResponsePayload::Empty)).unwrap();
+                        }
+                        ApiRequest::VmmPing(sender) => {
+                            sender.send(Ok(ApiResponsePayload::Empty)).unwrap();
+                        }
+                        ApiRequest::VmPause(sender) => {
+                            sender.send(Ok(ApiResponsePayload::Empty)).unwrap();
+                        }
+                        ApiRequest::VmResume(sender) => {
+                            sender.send(Ok(ApiResponsePayload::Empty)).unwrap();
+                        }
+                        ApiRequest::VmSnapshot(_, sender) => {
+                            sender.send(Ok(ApiResponsePayload::Empty)).unwrap();
+                        }
+                        ApiRequest::VmRestore(_, sender) => {
+                            sender.send(Ok(ApiResponsePayload::Empty)).unwrap();
+                        }
+                        ApiRequest::VmmShutdown(sender) => {
+                            sender.send(Ok(ApiResponsePayload::Empty)).unwrap();
+                        }
+                        ApiRequest::VmResize(_, sender) => {
+                            sender.send(Ok(ApiResponsePayload::Empty)).unwrap();
+                        }
+                        ApiRequest::VmResizeZone(_, sender) => {
+                            sender.send(Ok(ApiResponsePayload::Empty)).unwrap();
+                        }
+                        ApiRequest::VmAddDevice(_, sender) => {
+                            sender.send(Ok(ApiResponsePayload::Empty)).unwrap();
+                        }
+                        ApiRequest::VmAddUserDevice(_, sender) => {
+                            sender.send(Ok(ApiResponsePayload::Empty)).unwrap();
+                        }
+                        ApiRequest::VmRemoveDevice(_, sender) => {
+                            sender.send(Ok(ApiResponsePayload::Empty)).unwrap();
+                        }
+                        ApiRequest::VmAddDisk(_, sender) => {
+                            sender.send(Ok(ApiResponsePayload::Empty)).unwrap();
+                        }
+                        ApiRequest::VmAddFs(_, sender) => {
+                            sender.send(Ok(ApiResponsePayload::Empty)).unwrap();
+                        }
+                        ApiRequest::VmAddPmem(_, sender) => {
+                            sender.send(Ok(ApiResponsePayload::Empty)).unwrap();
+                        }
+                        ApiRequest::VmAddNet(_, sender) => {
+                            sender.send(Ok(ApiResponsePayload::Empty)).unwrap();
+                        }
+                        ApiRequest::VmAddVdpa(_, sender) => {
+                            sender.send(Ok(ApiResponsePayload::Empty)).unwrap();
+                        }
+                        ApiRequest::VmAddVsock(_, sender) => {
+                            sender.send(Ok(ApiResponsePayload::Empty)).unwrap();
+                        }
+                        ApiRequest::VmCounters(sender) => {
+                            sender.send(Ok(ApiResponsePayload::Empty)).unwrap();
+                        }
+                        ApiRequest::VmReceiveMigration(_, sender) => {
+                            sender.send(Ok(ApiResponsePayload::Empty)).unwrap();
+                        }
+                        ApiRequest::VmSendMigration(_, sender) => {
+                            sender.send(Ok(ApiResponsePayload::Empty)).unwrap();
+                        }
+                        ApiRequest::VmPowerButton(sender) => {
+                            sender.send(Ok(ApiResponsePayload::Empty)).unwrap();
+                        }
+                    }
+                }
+            }
+            _ => {
+                panic!("Unexpected Epoll event");
+            }
+        }
+    }
+}
--- a/fuzz/fuzz_targets/serial.rs
+++ b/fuzz/fuzz_targets/serial.rs
@@ -0,0 +1,73 @@
+// Copyright © 2022 Intel Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+
+#![no_main]
+use devices::legacy::Serial;
+use libc::EFD_NONBLOCK;
+use libfuzzer_sys::fuzz_target;
+use std::sync::Arc;
+use vm_device::interrupt::{InterruptIndex, InterruptSourceConfig, InterruptSourceGroup};
+use vm_device::BusDevice;
+use vmm_sys_util::eventfd::EventFd;
+
+fuzz_target!(|bytes| {
+    let mut serial = Serial::new_sink(
+        "serial".into(),
+        Arc::new(TestInterrupt::new(EventFd::new(EFD_NONBLOCK).unwrap())),
+    );
+
+    let mut i = 0;
+    while i < bytes.len() {
+        let choice = bytes.get(i).unwrap_or(&0) % 3;
+        i += 1;
+
+        match choice {
+            0 => {
+                let offset = (bytes.get(i).unwrap_or(&0) % 8) as u64;
+                i += 1;
+                let mut out_bytes = vec![0];
+                serial.read(0, offset, &mut out_bytes);
+            }
+            1 => {
+                let offset = (bytes.get(i).unwrap_or(&0) % 8) as u64;
+                i += 1;
+                let data = vec![*bytes.get(i).unwrap_or(&0)];
+                i += 1;
+                serial.write(0, offset, &data);
+            }
+            _ => {
+                let data = vec![*bytes.get(i).unwrap_or(&0)];
+                i += 1;
+                serial.queue_input_bytes(&data).ok();
+            }
+        }
+    }
+});
+
+struct TestInterrupt {
+    event_fd: EventFd,
+}
+
+impl InterruptSourceGroup for TestInterrupt {
+    fn trigger(&self, _index: InterruptIndex) -> Result<(), std::io::Error> {
+        self.event_fd.write(1)
+    }
+    fn update(
+        &self,
+        _index: InterruptIndex,
+        _config: InterruptSourceConfig,
+        _masked: bool,
+    ) -> Result<(), std::io::Error> {
+        Ok(())
+    }
+    fn notifier(&self, _index: InterruptIndex) -> Option<EventFd> {
+        Some(self.event_fd.try_clone().unwrap())
+    }
+}
+
+impl TestInterrupt {
+    fn new(event_fd: EventFd) -> Self {
+        TestInterrupt { event_fd }
+    }
+}
--- a/hypervisor/Cargo.toml
+++ b/hypervisor/Cargo.toml
@@ -11,19 +11,20 @@ mshv = ["mshv-ioctls", "mshv-bindings"]
 tdx = []

 [dependencies]
-anyhow = "1.0.57"
-epoll = "4.3.1"
-thiserror = "1.0.31"
-libc = "0.2.126"
+anyhow = "1.0.62"
+byteorder = "1.4.3"
+thiserror = "1.0.32"
+libc = "0.2.131"
 log = "0.4.17"
 kvm-ioctls = { version = "0.11.0", optional = true }
 kvm-bindings = { git = "https://github.com/cloud-hypervisor/kvm-bindings", branch = "ch-v0.5.0-tdx", features = ["with-serde", "fam-wrappers"], optional  = true }
 mshv-bindings = { git = "https://github.com/rust-vmm/mshv", branch = "main", features = ["with-serde", "fam-wrappers"], optional  = true }
 mshv-ioctls = { git = "https://github.com/rust-vmm/mshv", branch = "main", optional  = true}
-serde = { version = "1.0.137", features = ["rc", "derive"] }
-serde_json = "1.0.81"
+serde = { version = "1.0.143", features = ["rc", "derive"] }
+serde_with = { version = "2.0.0", default-features = false, features = ["macros"] }
+vfio-ioctls = { git = "https://github.com/rust-vmm/vfio", branch = "main", default-features = false }
 vm-memory = { version = "0.8.0", features = ["backend-mmap", "backend-atomic"] }
-vmm-sys-util = { version = "0.9.0", features = ["with-serde"] }
+vmm-sys-util = { version = "0.10.0", features = ["with-serde"] }

 [target.'cfg(target_arch = "x86_64")'.dependencies.iced-x86]
 version = "1.17.0"
--- a/hypervisor/src/arch/aarch64/gic.rs
+++ b/hypervisor/src/arch/aarch64/gic.rs
@@ -0,0 +1,56 @@
+// Copyright 2022 Arm Limited (or its affiliates). All rights reserved.
+
+use crate::{CpuState, GicState, HypervisorDeviceError, HypervisorVmError};
+use std::any::Any;
+use std::result;
+
+/// Errors thrown while setting up the VGIC.
+#[derive(Debug)]
+pub enum Error {
+    /// Error while calling KVM ioctl for setting up the global interrupt controller.
+    CreateGic(HypervisorVmError),
+    /// Error while setting device attributes for the GIC.
+    SetDeviceAttribute(HypervisorDeviceError),
+    /// Error while getting device attributes for the GIC.
+    GetDeviceAttribute(HypervisorDeviceError),
+}
+pub type Result<T> = result::Result<T, Error>;
+
+/// Hypervisor agnostic interface for a virtualized GIC
+pub trait Vgic: Send + Sync {
+    /// Returns the fdt compatibility property of the device
+    fn fdt_compatibility(&self) -> &str;
+
+    /// Returns the maint_irq fdt property of the device
+    fn fdt_maint_irq(&self) -> u32;
+
+    /// Returns an array with GIC device properties
+    fn device_properties(&self) -> [u64; 4];
+
+    /// Returns the number of vCPUs this GIC handles
+    fn vcpu_count(&self) -> u64;
+
+    /// Returns whether the GIC device is MSI compatible or not
+    fn msi_compatible(&self) -> bool;
+
+    /// Returns the MSI compatibility property of the device
+    fn msi_compatibility(&self) -> &str;
+
+    /// Returns the MSI reg property of the device
+    fn msi_properties(&self) -> [u64; 2];
+
+    /// Get the values of GICR_TYPER for each vCPU.
+    fn set_gicr_typers(&mut self, vcpu_states: &[CpuState]);
+
+    /// Downcast the trait object to its concrete type.
+    fn as_any_concrete_mut(&mut self) -> &mut dyn Any;
+
+    /// Save the state of GICv3ITS.
+    fn state(&self) -> Result<GicState>;
+
+    /// Restore the state of GICv3ITS.
+    fn set_state(&mut self, state: &GicState) -> Result<()>;
+
+    /// Saves GIC internal data tables into RAM.
+    fn save_data_tables(&self) -> Result<()>;
+}
--- a/hypervisor/src/arch/aarch64/mod.rs
+++ b/hypervisor/src/arch/aarch64/mod.rs
@@ -0,0 +1,3 @@
+// Copyright 2022 Arm Limited (or its affiliates). All rights reserved.
+
+pub mod gic;
--- a/hypervisor/src/arch/emulator/mod.rs
+++ b/hypervisor/src/arch/emulator/mod.rs
@@ -99,7 +99,7 @@ pub enum EmulationError<T: Debug> {
 /// A CpuState is an architecture specific type, representing a CPU state.
 /// The emulator and its instruction handlers modify a given CPU state and
 /// eventually ask the platform to commit it back through `set_cpu_state`.
-pub trait PlatformEmulator: Send + Sync {
+pub trait PlatformEmulator {
    type CpuState: Clone;

    /// Read guest memory into a u8 slice.
--- a/hypervisor/src/arch/mod.rs
+++ b/hypervisor/src/arch/mod.rs
@@ -10,9 +10,14 @@
 //
 // Copyright © 2020, Microsoft Corporation
 //
+// Copyright 2022 Arm Limited (or its affiliates). All rights reserved.
+//

 pub mod emulator;

 #[cfg(target_arch = "x86_64")]
 #[macro_use]
 pub mod x86;
+
+#[cfg(target_arch = "aarch64")]
+pub mod aarch64;
--- a/hypervisor/src/arch/x86/emulator/instructions/mod.rs
+++ b/hypervisor/src/arch/x86/emulator/instructions/mod.rs
@@ -28,14 +28,11 @@ fn get_op<T: CpuStateManager>(
        )));
    }

-    match op_size {
-        1 | 2 | 4 | 8 => {}
-        _ => {
-            return Err(PlatformError::InvalidOperand(anyhow!(
-                "Invalid operand size {:?}",
-                op_size
-            )))
-        }
+    if !matches!(op_size, 1 | 2 | 4 | 8) {
+        return Err(PlatformError::InvalidOperand(anyhow!(
+            "Invalid operand size {:?}",
+            op_size
+        )));
    }

    let value = match insn
@@ -81,14 +78,11 @@ fn set_op<T: CpuStateManager>(
        )));
    }

-    match op_size {
-        1 | 2 | 4 | 8 => {}
-        _ => {
-            return Err(PlatformError::InvalidOperand(anyhow!(
-                "Invalid operand size {:?}",
-                op_size
-            )))
-        }
+    if !matches!(op_size, 1 | 2 | 4 | 8) {
+        return Err(PlatformError::InvalidOperand(anyhow!(
+            "Invalid operand size {:?}",
+            op_size
+        )));
    }

    match insn
--- a/hypervisor/src/arch/x86/emulator/mod.rs
+++ b/hypervisor/src/arch/x86/emulator/mod.rs
@@ -7,8 +7,10 @@
 use crate::arch::emulator::{EmulationError, EmulationResult, PlatformEmulator, PlatformError};
 use crate::arch::x86::emulator::instructions::*;
 use crate::arch::x86::regs::{CR0_PE, EFER_LMA};
-use crate::arch::x86::{segment_type_expand_down, segment_type_ro, Exception, SegmentRegisterOps};
-use crate::x86_64::{SegmentRegister, SpecialRegisters, StandardRegisters};
+use crate::arch::x86::{
+    segment_type_expand_down, segment_type_ro, Exception, SegmentRegister, SpecialRegisters,
+    StandardRegisters,
+};
 use anyhow::Context;
 use iced_x86::*;

@@ -16,7 +18,7 @@ use iced_x86::*;
 mod instructions;

 /// x86 CPU modes
-#[derive(Debug, PartialEq)]
+#[derive(Debug, PartialEq, Eq)]
 pub enum CpuMode {
    /// Real mode
    Real,
--- a/hypervisor/src/arch/x86/gdt.rs
+++ b/hypervisor/src/arch/x86/gdt.rs
@@ -8,7 +8,7 @@
 // found in the LICENSE-BSD-3-Clause file.

 // For GDT details see arch/x86/include/asm/segment.h
-use crate::x86_64::SegmentRegister;
+use crate::arch::x86::SegmentRegister;

 /// Constructor for a conventional segment GDT (or LDT) entry. Derived from the kernel's segment.h.
 pub fn gdt_entry(flags: u16, base: u32, limit: u32) -> u64 {
@@ -51,7 +51,7 @@ fn get_limit(entry: u64) -> u32 {
    // Perform manual limit scaling if G flag is set
    match get_g(entry) {
        0 => limit,
-        _ => ((limit << 12) | 0xFFF), // G flag is either 0 or 1
+        _ => (limit << 12) | 0xFFF, // G flag is either 0 or 1
    }
 }

@@ -106,7 +106,6 @@ pub fn segment_from_gdt(entry: u64, table_index: u8) -> SegmentRegister {
        l: get_l(entry),
        g: get_g(entry),
        avl: get_avl(entry),
-        padding: 0,
        unusable: match get_p(entry) {
            0 => 1,
            _ => 0,
--- a/hypervisor/src/arch/x86/mod.rs
+++ b/hypervisor/src/arch/x86/mod.rs
@@ -13,7 +13,6 @@

 pub mod emulator;
 pub mod gdt;
-#[allow(non_upper_case_globals)]
 #[allow(non_camel_case_types)]
 #[allow(non_snake_case)]
 #[allow(non_upper_case_globals)]
@@ -27,7 +26,7 @@ pub const MTRR_MEM_TYPE_WB: u64 = 0x6;
 pub const NUM_IOAPIC_PINS: usize = 24;

 // X86 Exceptions
-#[allow(dead_code, clippy::upper_case_acronyms)]
+#[allow(clippy::upper_case_acronyms)]
 #[derive(Clone, Debug)]
 pub enum Exception {
    DE = 0,  // Divide Error
@@ -53,40 +52,86 @@ pub enum Exception {

 pub mod regs;

-// Abstracted segment register ops.
-// Each x86 hypervisor should implement those.
-pub trait SegmentRegisterOps {
-    // Segment type
-    fn segment_type(&self) -> u8;
-    fn set_segment_type(&mut self, val: u8);
+#[derive(Debug, Default, Copy, Clone, PartialEq, Eq)]
+#[cfg_attr(feature = "with-serde", derive(Deserialize, Serialize))]
+pub struct SegmentRegister {
+    pub base: u64,
+    pub limit: u32,
+    pub selector: u16,
+    pub type_: u8,
+    pub present: u8,
+    pub dpl: u8,
+    pub db: u8,
+    pub s: u8,
+    pub l: u8,
+    pub g: u8,
+    pub avl: u8,
+    pub unusable: u8,
+}

-    // Descriptor Privilege Level (DPL)
-    fn dpl(&self) -> u8;
-    fn set_dpl(&mut self, val: u8);
+impl SegmentRegister {
+    pub fn segment_type(&self) -> u8 {
+        self.type_
+    }
+    pub fn set_segment_type(&mut self, val: u8) {
+        self.type_ = val;
+    }

-    // Granularity
-    fn granularity(&self) -> u8;
-    fn set_granularity(&mut self, val: u8);
+    pub fn dpl(&self) -> u8 {
+        self.dpl
+    }

-    // Memory Presence
-    fn present(&self) -> u8;
-    fn set_present(&mut self, val: u8);
+    pub fn set_dpl(&mut self, val: u8) {
+        self.dpl = val;
+    }

-    // Long mode
-    fn long(&self) -> u8;
-    fn set_long(&mut self, val: u8);
+    pub fn present(&self) -> u8 {
+        self.present
+    }

-    // Available for system use (AVL)
-    fn avl(&self) -> u8;
-    fn set_avl(&mut self, val: u8);
+    pub fn set_present(&mut self, val: u8) {
+        self.present = val;
+    }

-    // Descriptor type (System or code/data)
-    fn desc_type(&self) -> u8;
-    fn set_desc_type(&mut self, val: u8);
+    pub fn long(&self) -> u8 {
+        self.l
+    }

-    // D/B
-    fn db(&self) -> u8;
-    fn set_db(&mut self, val: u8);
+    pub fn set_long(&mut self, val: u8) {
+        self.l = val;
+    }
+
+    pub fn avl(&self) -> u8 {
+        self.avl
+    }
+
+    pub fn set_avl(&mut self, val: u8) {
+        self.avl = val;
+    }
+
+    pub fn desc_type(&self) -> u8 {
+        self.s
+    }
+
+    pub fn set_desc_type(&mut self, val: u8) {
+        self.s = val;
+    }
+
+    pub fn granularity(&self) -> u8 {
+        self.g
+    }
+
+    pub fn set_granularity(&mut self, val: u8) {
+        self.g = val;
+    }
+
+    pub fn db(&self) -> u8 {
+        self.db
+    }
+
+    pub fn set_db(&mut self, val: u8) {
+        self.db = val;
+    }
 }

 // Code segment
@@ -115,7 +160,6 @@ macro_rules! msr {
        MsrEntry {
            index: $msr,
            data: 0x0,
-            ..Default::default()
        }
    };
 }
@@ -125,7 +169,143 @@ macro_rules! msr_data {
        MsrEntry {
            index: $msr,
            data: $data,
-            ..Default::default()
        }
    };
 }
+
+#[derive(Debug, Default, Copy, Clone, PartialEq, Eq)]
+#[cfg_attr(feature = "with-serde", derive(Deserialize, Serialize))]
+pub struct StandardRegisters {
+    pub rax: u64,
+    pub rbx: u64,
+    pub rcx: u64,
+    pub rdx: u64,
+    pub rsi: u64,
+    pub rdi: u64,
+    pub rsp: u64,
+    pub rbp: u64,
+    pub r8: u64,
+    pub r9: u64,
+    pub r10: u64,
+    pub r11: u64,
+    pub r12: u64,
+    pub r13: u64,
+    pub r14: u64,
+    pub r15: u64,
+    pub rip: u64,
+    pub rflags: u64,
+}
+
+#[derive(Debug, Default, Copy, Clone, PartialEq, Eq)]
+#[cfg_attr(feature = "with-serde", derive(Deserialize, Serialize))]
+pub struct DescriptorTable {
+    pub base: u64,
+    pub limit: u16,
+}
+
+#[derive(Debug, Default, Copy, Clone, PartialEq, Eq)]
+#[cfg_attr(feature = "with-serde", derive(Deserialize, Serialize))]
+pub struct SpecialRegisters {
+    pub cs: SegmentRegister,
+    pub ds: SegmentRegister,
+    pub es: SegmentRegister,
+    pub fs: SegmentRegister,
+    pub gs: SegmentRegister,
+    pub ss: SegmentRegister,
+    pub tr: SegmentRegister,
+    pub ldt: SegmentRegister,
+    pub gdt: DescriptorTable,
+    pub idt: DescriptorTable,
+    pub cr0: u64,
+    pub cr2: u64,
+    pub cr3: u64,
+    pub cr4: u64,
+    pub cr8: u64,
+    pub efer: u64,
+    pub apic_base: u64,
+    pub interrupt_bitmap: [u64; 4usize],
+}
+
+#[derive(Debug, Default, Copy, Clone, PartialEq, Eq, serde::Deserialize, serde::Serialize)]
+pub struct CpuIdEntry {
+    pub function: u32,
+    pub index: u32,
+    pub flags: u32,
+    pub eax: u32,
+    pub ebx: u32,
+    pub ecx: u32,
+    pub edx: u32,
+}
+
+pub const CPUID_FLAG_VALID_INDEX: u32 = 1;
+
+#[derive(Default, Debug, Clone, serde::Deserialize, serde::Serialize)]
+pub struct FpuState {
+    pub fpr: [[u8; 16usize]; 8usize],
+    pub fcw: u16,
+    pub fsw: u16,
+    pub ftwx: u8,
+    pub last_opcode: u16,
+    pub last_ip: u64,
+    pub last_dp: u64,
+    pub xmm: [[u8; 16usize]; 16usize],
+    pub mxcsr: u32,
+}
+
+#[serde_with::serde_as]
+#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
+pub struct LapicState {
+    #[serde_as(as = "[_; 1024usize]")]
+    pub(crate) regs: [::std::os::raw::c_char; 1024usize],
+}
+
+impl Default for LapicState {
+    fn default() -> Self {
+        // SAFETY: this is plain old data structure
+        unsafe { ::std::mem::zeroed() }
+    }
+}
+
+impl LapicState {
+    pub fn get_klapic_reg(&self, reg_offset: usize) -> u32 {
+        use byteorder::{LittleEndian, ReadBytesExt};
+        use std::io::Cursor;
+        use std::mem;
+
+        let sliceu8 = unsafe {
+            // This array is only accessed as parts of a u32 word, so interpret it as a u8 array.
+            // Cursors are only readable on arrays of u8, not i8(c_char).
+            mem::transmute::<&[i8], &[u8]>(&self.regs[reg_offset..])
+        };
+
+        let mut reader = Cursor::new(sliceu8);
+        // Following call can't fail if the offsets defined above are correct.
+        reader
+            .read_u32::<LittleEndian>()
+            .expect("Failed to read klapic register")
+    }
+
+    pub fn set_klapic_reg(&mut self, reg_offset: usize, value: u32) {
+        use byteorder::{LittleEndian, WriteBytesExt};
+        use std::io::Cursor;
+        use std::mem;
+
+        let sliceu8 = unsafe {
+            // This array is only accessed as parts of a u32 word, so interpret it as a u8 array.
+            // Cursors are only readable on arrays of u8, not i8(c_char).
+            mem::transmute::<&mut [i8], &mut [u8]>(&mut self.regs[reg_offset..])
+        };
+
+        let mut writer = Cursor::new(sliceu8);
+        // Following call can't fail if the offsets defined above are correct.
+        writer
+            .write_u32::<LittleEndian>(value)
+            .expect("Failed to write klapic register")
+    }
+}
+
+#[derive(Debug, Default, Copy, Clone, PartialEq, Eq, serde::Deserialize, serde::Serialize)]
+pub struct MsrEntry {
+    pub index: u32,
+    pub data: u64,
+}
--- a/hypervisor/src/cpu.rs
+++ b/hypervisor/src/cpu.rs
@@ -9,28 +9,17 @@
 //

 #[cfg(target_arch = "aarch64")]
-use crate::aarch64::VcpuInit;
-#[cfg(target_arch = "aarch64")]
-use crate::aarch64::{RegList, Register, StandardRegisters};
+use crate::aarch64::{RegList, StandardRegisters, VcpuInit};
+#[cfg(target_arch = "x86_64")]
+use crate::arch::x86::{
+    CpuIdEntry, FpuState, LapicState, MsrEntry, SpecialRegisters, StandardRegisters,
+};
 #[cfg(feature = "tdx")]
 use crate::kvm::{TdxExitDetails, TdxExitStatus};
-#[cfg(all(feature = "mshv", target_arch = "x86_64"))]
-use crate::x86_64::SuspendRegisters;
-#[cfg(target_arch = "x86_64")]
-use crate::x86_64::Xsave;
-#[cfg(target_arch = "x86_64")]
-use crate::x86_64::{CpuId, LapicState};
-#[cfg(target_arch = "x86_64")]
-use crate::x86_64::{
-    ExtendedControlRegisters, FpuState, MsrEntries, SpecialRegisters, StandardRegisters, VcpuEvents,
-};
 use crate::CpuState;
-#[cfg(target_arch = "aarch64")]
-use crate::DeviceAttr;
-#[cfg(feature = "kvm")]
 use crate::MpState;
 use thiserror::Error;
-#[cfg(all(feature = "kvm", target_arch = "x86_64"))]
+#[cfg(target_arch = "x86_64")]
 use vm_memory::GuestAddress;

 #[derive(Error, Debug)]
@@ -48,11 +37,6 @@ pub enum HypervisorCpuError {
    #[error("Failed to get standard registers: {0}")]
    GetStandardRegs(#[source] anyhow::Error),
    ///
-    /// Getting suspend registers error
-    ///
-    #[error("Failed to get suspend registers: {0}")]
-    GetSuspendRegs(#[source] anyhow::Error),
-    ///
    /// Setting special register error
    ///
    #[error("Failed to set special registers: {0}")]
@@ -248,6 +232,12 @@ pub enum HypervisorCpuError {
    #[cfg(feature = "tdx")]
    #[error("Unknown TDX VM call")]
    UnknownTdxVmCall,
+    #[cfg(target_arch = "aarch64")]
+    ///
+    /// Failed to intialize PMU
+    ///
+    #[error("Failed to initialize PMU")]
+    InitializePmu,
 }

 #[derive(Debug)]
@@ -278,22 +268,10 @@ pub type Result<T> = anyhow::Result<T, HypervisorCpuError>;
 /// Trait to represent a generic Vcpu
 ///
 pub trait Vcpu: Send + Sync {
-    #[cfg(target_arch = "x86_64")]
    ///
    /// Returns the vCPU general purpose registers.
    ///
    fn get_regs(&self) -> Result<StandardRegisters>;
-    #[cfg(target_arch = "aarch64")]
-    ///
-    /// Sets vcpu attribute
-    ///
-    fn set_vcpu_attr(&self, attr: &DeviceAttr) -> Result<()>;
-    #[cfg(target_arch = "aarch64")]
-    ///
-    /// Check if vcpu has attribute.
-    ///
-    fn has_vcpu_attr(&self, attr: &DeviceAttr) -> Result<()>;
-    #[cfg(target_arch = "x86_64")]
    ///
    /// Sets the vCPU general purpose registers.
    ///
@@ -322,7 +300,7 @@ pub trait Vcpu: Send + Sync {
    ///
    /// X86 specific call to setup the CPUID registers.
    ///
-    fn set_cpuid2(&self, cpuid: &CpuId) -> Result<()>;
+    fn set_cpuid2(&self, cpuid: &[CpuIdEntry]) -> Result<()>;
    #[cfg(target_arch = "x86_64")]
    ///
    /// X86 specific call to enable HyperV SynIC
@@ -332,7 +310,7 @@ pub trait Vcpu: Send + Sync {
    ///
    /// X86 specific call to retrieve the CPUID registers.
    ///
-    fn get_cpuid2(&self, num_entries: usize) -> Result<CpuId>;
+    fn get_cpuid2(&self, num_entries: usize) -> Result<Vec<CpuIdEntry>>;
    #[cfg(target_arch = "x86_64")]
    ///
    /// Returns the state of the LAPIC (Local Advanced Programmable Interrupt Controller).
@@ -347,111 +325,66 @@ pub trait Vcpu: Send + Sync {
    ///
    /// Returns the model-specific registers (MSR) for this vCPU.
    ///
-    fn get_msrs(&self, msrs: &mut MsrEntries) -> Result<usize>;
+    fn get_msrs(&self, msrs: &mut Vec<MsrEntry>) -> Result<usize>;
    #[cfg(target_arch = "x86_64")]
    ///
    /// Setup the model-specific registers (MSR) for this vCPU.
    ///
-    fn set_msrs(&self, msrs: &MsrEntries) -> Result<usize>;
-    #[cfg(feature = "kvm")]
+    fn set_msrs(&self, msrs: &[MsrEntry]) -> Result<usize>;
    ///
    /// Returns the vcpu's current "multiprocessing state".
    ///
    fn get_mp_state(&self) -> Result<MpState>;
-    #[cfg(feature = "kvm")]
    ///
    /// Sets the vcpu's current "multiprocessing state".
    ///
    fn set_mp_state(&self, mp_state: MpState) -> Result<()>;
    #[cfg(target_arch = "x86_64")]
    ///
-    /// X86 specific call that returns the vcpu's current "xsave struct".
-    ///
-    fn get_xsave(&self) -> Result<Xsave>;
-    #[cfg(target_arch = "x86_64")]
-    ///
-    /// X86 specific call that sets the vcpu's current "xsave struct".
-    ///
-    fn set_xsave(&self, xsave: &Xsave) -> Result<()>;
-    #[cfg(target_arch = "x86_64")]
-    ///
-    /// X86 specific call that returns the vcpu's current "xcrs".
-    ///
-    fn get_xcrs(&self) -> Result<ExtendedControlRegisters>;
-    #[cfg(target_arch = "x86_64")]
-    ///
-    /// X86 specific call that sets the vcpu's current "xcrs".
-    ///
-    fn set_xcrs(&self, xcrs: &ExtendedControlRegisters) -> Result<()>;
-    #[cfg(target_arch = "x86_64")]
-    ///
-    /// Returns currently pending exceptions, interrupts, and NMIs as well as related
-    /// states of the vcpu.
-    ///
-    fn get_vcpu_events(&self) -> Result<VcpuEvents>;
-    #[cfg(target_arch = "x86_64")]
-    ///
-    /// Sets pending exceptions, interrupts, and NMIs as well as related states
-    /// of the vcpu.
-    ///
-    fn set_vcpu_events(&self, events: &VcpuEvents) -> Result<()>;
-    #[cfg(all(feature = "kvm", target_arch = "x86_64"))]
-    ///
    /// Let the guest know that it has been paused, which prevents from
    /// potential soft lockups when being resumed.
    ///
-    fn notify_guest_clock_paused(&self) -> Result<()>;
-    #[cfg(all(feature = "kvm", target_arch = "x86_64"))]
+    fn notify_guest_clock_paused(&self) -> Result<()> {
+        Ok(())
+    }
+    #[cfg(target_arch = "x86_64")]
    ///
    /// Sets debug registers to set hardware breakpoints and/or enable single step.
    ///
-    fn set_guest_debug(&self, addrs: &[GuestAddress], singlestep: bool) -> Result<()>;
+    fn set_guest_debug(&self, _addrs: &[GuestAddress], _singlestep: bool) -> Result<()> {
+        Err(HypervisorCpuError::SetDebugRegs(anyhow!("unimplemented")))
+    }
    ///
    /// Sets the type of CPU to be exposed to the guest and optional features.
    ///
-    #[cfg(any(target_arch = "arm", target_arch = "aarch64"))]
+    #[cfg(target_arch = "aarch64")]
    fn vcpu_init(&self, kvi: &VcpuInit) -> Result<()>;
    ///
-    /// Sets the value of one register for this vCPU.
-    ///
-    #[cfg(any(target_arch = "arm", target_arch = "aarch64"))]
-    fn set_reg(&self, reg_id: u64, data: u64) -> Result<()>;
-    ///
-    /// Sets the value of one register for this vCPU.
-    ///
-    #[cfg(any(target_arch = "arm", target_arch = "aarch64"))]
-    fn get_reg(&self, reg_id: u64) -> Result<u64>;
-    ///
    /// Gets a list of the guest registers that are supported for the
    /// KVM_GET_ONE_REG/KVM_SET_ONE_REG calls.
    ///
-    #[cfg(any(target_arch = "arm", target_arch = "aarch64"))]
+    #[cfg(target_arch = "aarch64")]
    fn get_reg_list(&self, reg_list: &mut RegList) -> Result<()>;
    ///
-    /// Save the state of the core registers.
+    /// Gets the value of a system register
    ///
-    #[cfg(any(target_arch = "arm", target_arch = "aarch64"))]
-    fn core_registers(&self, state: &mut StandardRegisters) -> Result<()>;
+    #[cfg(target_arch = "aarch64")]
+    fn get_sys_reg(&self, sys_reg: u32) -> Result<u64>;
    ///
-    /// Restore the state of the core registers.
+    /// Configure core registers for a given CPU.
    ///
-    #[cfg(any(target_arch = "arm", target_arch = "aarch64"))]
-    fn set_core_registers(&self, state: &StandardRegisters) -> Result<()>;
+    #[cfg(target_arch = "aarch64")]
+    fn setup_regs(&self, cpu_id: u8, boot_ip: u64, fdt_start: u64) -> Result<()>;
    ///
-    /// Save the state of the system registers.
+    /// Check if the CPU supports PMU
    ///
-    #[cfg(any(target_arch = "arm", target_arch = "aarch64"))]
-    fn system_registers(&self, state: &mut Vec<Register>) -> Result<()>;
+    #[cfg(target_arch = "aarch64")]
+    fn has_pmu_support(&self) -> bool;
    ///
-    /// Restore the state of the system registers.
+    /// Initialize PMU
    ///
-    #[cfg(any(target_arch = "arm", target_arch = "aarch64"))]
-    fn set_system_registers(&self, state: &[Register]) -> Result<()>;
-    ///
-    /// Read the MPIDR - Multiprocessor Affinity Register.
-    ///
-    #[cfg(any(target_arch = "arm", target_arch = "aarch64"))]
-    fn read_mpidr(&self) -> Result<u64>;
+    #[cfg(target_arch = "aarch64")]
+    fn init_pmu(&self, irq: u32) -> Result<()>;
    ///
    /// Retrieve the vCPU state.
    /// This function is necessary to snapshot the VM
@@ -476,16 +409,10 @@ pub trait Vcpu: Send + Sync {
    ///
    #[cfg(feature = "tdx")]
    fn tdx_init(&self, hob_address: u64) -> Result<()>;
-    #[cfg(all(feature = "mshv", target_arch = "x86_64"))]
-    ///
-    /// Return suspend registers(explicit and intercept suspend registers)
-    ///
-    fn get_suspend_regs(&self) -> Result<SuspendRegisters>;
-    #[cfg(feature = "kvm")]
    ///
    /// Set the "immediate_exit" state
    ///
-    fn set_immediate_exit(&self, exit: bool);
+    fn set_immediate_exit(&self, _exit: bool) {}
    #[cfg(feature = "tdx")]
    ///
    /// Returns the details about TDX exit reason
@@ -496,4 +423,9 @@ pub trait Vcpu: Send + Sync {
    /// Set the status code for TDX exit
    ///
    fn set_tdx_status(&mut self, status: TdxExitStatus);
+    #[cfg(target_arch = "x86_64")]
+    ///
+    /// Return the list of initial MSR entries for a VCPU
+    ///
+    fn boot_msr_entries(&self) -> Vec<MsrEntry>;
 }
--- a/hypervisor/src/device.rs
+++ b/hypervisor/src/device.rs
@@ -9,8 +9,6 @@
 // Copyright 2020, ARM Limited
 //

-use crate::DeviceAttr;
-use std::os::unix::io::AsRawFd;
 use thiserror::Error;

 #[derive(Error, Debug)]
@@ -28,20 +26,3 @@ pub enum HypervisorDeviceError {
    #[error("Failed to get device attribute: {0}")]
    GetDeviceAttribute(#[source] anyhow::Error),
 }
-
-///
-/// Result type for returning from a function
-///
-pub type Result<T> = std::result::Result<T, HypervisorDeviceError>;
-
-///
-/// Trait to represent a device
-///
-/// This crate provides a hypervisor-agnostic interfaces for device
-///
-pub trait Device: Send + Sync + AsRawFd {
-    /// Set device attribute.
-    fn set_device_attr(&self, attr: &DeviceAttr) -> Result<()>;
-    /// Get device attribute.
-    fn get_device_attr(&self, attr: &mut DeviceAttr) -> Result<()>;
-}
--- a/hypervisor/src/hypervisor.rs
+++ b/hypervisor/src/hypervisor.rs
@@ -7,13 +7,12 @@
 // Copyright 2018-2019 CrowdStrike, Inc.
 //
 //
+#[cfg(target_arch = "x86_64")]
+use crate::arch::x86::CpuIdEntry;
 #[cfg(feature = "tdx")]
 use crate::kvm::TdxCapabilities;
 use crate::vm::Vm;
-#[cfg(target_arch = "x86_64")]
-use crate::x86_64::CpuId;
-#[cfg(target_arch = "x86_64")]
-use crate::x86_64::MsrList;
+use crate::HypervisorType;
 use std::sync::Arc;
 use thiserror::Error;

@@ -21,6 +20,11 @@ use thiserror::Error;
 ///
 ///
 pub enum HypervisorError {
+    ///
+    /// Hypervisor availability check error
+    ///
+    #[error("Failed to check availability of the hypervisor: {0}")]
+    HypervisorAvailableCheck(#[source] anyhow::Error),
    ///
    /// hypervisor creation error
    ///
@@ -66,6 +70,11 @@ pub enum HypervisorError {
    ///
    #[error("Failed to retrieve TDX capabilities:{0}")]
    TdxCapabilities(#[source] anyhow::Error),
+    ///
+    /// Failed to set partition property
+    ///
+    #[error("Failed to set partition property:{0}")]
+    SetPartitionProperty(#[source] anyhow::Error),
 }

 ///
@@ -79,6 +88,10 @@ pub type Result<T> = std::result::Result<T, HypervisorError>;
 /// This crate provides a hypervisor-agnostic interfaces
 ///
 pub trait Hypervisor: Send + Sync {
+    ///
+    /// Returns the type of the hypervisor
+    ///
+    fn hypervisor_type(&self) -> HypervisorType;
    ///
    /// Create a Vm using the underlying hypervisor
    /// Return a hypervisor-agnostic Vm trait object
@@ -95,18 +108,13 @@ pub trait Hypervisor: Send + Sync {
    ///
    /// Get the supported CpuID
    ///
-    fn get_cpuid(&self) -> Result<CpuId>;
+    fn get_cpuid(&self) -> Result<Vec<CpuIdEntry>>;
    ///
    /// Check particular extensions if any
    ///
    fn check_required_extensions(&self) -> Result<()> {
        Ok(())
    }
-    #[cfg(target_arch = "x86_64")]
-    ///
-    /// Retrieve the list of MSRs supported by the hypervisor.
-    ///
-    fn get_msr_list(&self) -> Result<MsrList>;
    #[cfg(target_arch = "aarch64")]
    ///
    /// Retrieve AArch64 host maximum IPA size supported by KVM.
--- a/hypervisor/src/kvm/aarch64/gic/dist_regs.rs
+++ b/hypervisor/src/kvm/aarch64/gic/dist_regs.rs
@@ -1,12 +1,12 @@
 // Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.
 // SPDX-License-Identifier: Apache-2.0

-use super::{Error, Result};
-use crate::layout::IRQ_BASE;
-use hypervisor::kvm::kvm_bindings::{
+use crate::arch::aarch64::gic::{Error, Result};
+use crate::device::HypervisorDeviceError;
+use crate::kvm::kvm_bindings::{
    kvm_device_attr, KVM_DEV_ARM_VGIC_GRP_DIST_REGS, KVM_DEV_ARM_VGIC_GRP_NR_IRQS,
 };
-use std::sync::Arc;
+use kvm_ioctls::DeviceFd;

 /*
 Distributor registers as detailed at page 456 from
@@ -77,12 +77,7 @@ static VGIC_DIST_REGS: &[DistReg] = &[
    VGIC_DIST_REG!(GICD_IPRIORITYR, 8, 0),
 ];

-fn dist_attr_access(
-    gic: &Arc<dyn hypervisor::Device>,
-    offset: u32,
-    val: &u32,
-    set: bool,
-) -> Result<()> {
+fn dist_attr_access(gic: &DeviceFd, offset: u32, val: &u32, set: bool) -> Result<()> {
    let mut gic_dist_attr = kvm_device_attr {
        group: KVM_DEV_ARM_VGIC_GRP_DIST_REGS,
        attr: offset as u64,
@@ -90,28 +85,30 @@ fn dist_attr_access(
        flags: 0,
    };
    if set {
-        gic.set_device_attr(&gic_dist_attr)
-            .map_err(Error::SetDeviceAttribute)?;
+        gic.set_device_attr(&gic_dist_attr).map_err(|e| {
+            Error::SetDeviceAttribute(HypervisorDeviceError::SetDeviceAttribute(e.into()))
+        })?;
    } else {
-        gic.get_device_attr(&mut gic_dist_attr)
-            .map_err(Error::GetDeviceAttribute)?;
+        gic.get_device_attr(&mut gic_dist_attr).map_err(|e| {
+            Error::GetDeviceAttribute(HypervisorDeviceError::GetDeviceAttribute(e.into()))
+        })?;
    }
    Ok(())
 }

 /// Get the distributor control register.
-pub fn read_ctlr(gic: &Arc<dyn hypervisor::Device>) -> Result<u32> {
+pub fn read_ctlr(gic: &DeviceFd) -> Result<u32> {
    let val: u32 = 0;
    dist_attr_access(gic, GICD_CTLR, &val, false)?;
    Ok(val)
 }

 /// Set the distributor control register.
-pub fn write_ctlr(gic: &Arc<dyn hypervisor::Device>, val: u32) -> Result<()> {
+pub fn write_ctlr(gic: &DeviceFd, val: u32) -> Result<()> {
    dist_attr_access(gic, GICD_CTLR, &val, true)
 }

-fn get_interrupts_num(gic: &Arc<dyn hypervisor::Device>) -> Result<u32> {
+fn get_interrupts_num(gic: &DeviceFd) -> Result<u32> {
    let num_irq = 0;

    let mut nr_irqs_attr = kvm_device_attr {
@@ -120,12 +117,18 @@ fn get_interrupts_num(gic: &Arc<dyn hypervisor::Device>) -> Result<u32> {
        addr: &num_irq as *const u32 as u64,
        flags: 0,
    };
-    gic.get_device_attr(&mut nr_irqs_attr)
-        .map_err(Error::GetDeviceAttribute)?;
+    gic.get_device_attr(&mut nr_irqs_attr).map_err(|e| {
+        Error::GetDeviceAttribute(HypervisorDeviceError::GetDeviceAttribute(e.into()))
+    })?;
    Ok(num_irq)
 }

-fn compute_reg_len(gic: &Arc<dyn hypervisor::Device>, reg: &DistReg, base: u32) -> Result<u32> {
+fn compute_reg_len(gic: &DeviceFd, reg: &DistReg, base: u32) -> Result<u32> {
+    // FIXME:
+    // Redefine some GIC constants to avoid the dependency on `layout` crate.
+    // This is temporary solution, will be fixed in future refactoring.
+    const LAYOUT_IRQ_BASE: u32 = 32;
+
    let mut end = base;
    let num_irq = get_interrupts_num(gic)?;
    if reg.length > 0 {
@@ -138,8 +141,8 @@ fn compute_reg_len(gic: &Arc<dyn hypervisor::Device>, reg: &DistReg, base: u32)
        // This is the type of register that takes into account the number of interrupts
        // that the model has. It is also the type of register where
        // a register relates to multiple interrupts.
-        end = base + (reg.bpi as u32 * (num_irq - IRQ_BASE) / 8);
-        if reg.bpi as u32 * (num_irq - IRQ_BASE) % 8 > 0 {
+        end = base + (reg.bpi as u32 * (num_irq - LAYOUT_IRQ_BASE) / 8);
+        if reg.bpi as u32 * (num_irq - LAYOUT_IRQ_BASE) % 8 > 0 {
            end += REG_SIZE as u32;
        }
    }
@@ -147,7 +150,7 @@ fn compute_reg_len(gic: &Arc<dyn hypervisor::Device>, reg: &DistReg, base: u32)
 }

 /// Set distributor registers of the GIC.
-pub fn set_dist_regs(gic: &Arc<dyn hypervisor::Device>, state: &[u32]) -> Result<()> {
+pub fn set_dist_regs(gic: &DeviceFd, state: &[u32]) -> Result<()> {
    let mut idx = 0;

    for dreg in VGIC_DIST_REGS {
@@ -164,7 +167,7 @@ pub fn set_dist_regs(gic: &Arc<dyn hypervisor::Device>, state: &[u32]) -> Result
    Ok(())
 }
 /// Get distributor registers of the GIC.
-pub fn get_dist_regs(gic: &Arc<dyn hypervisor::Device>) -> Result<Vec<u32>> {
+pub fn get_dist_regs(gic: &DeviceFd) -> Result<Vec<u32>> {
    let mut state = Vec::new();

    for dreg in VGIC_DIST_REGS {
--- a/hypervisor/src/kvm/aarch64/gic/icc_regs.rs
+++ b/hypervisor/src/kvm/aarch64/gic/icc_regs.rs
@@ -1,14 +1,16 @@
+// Copyright 2022 Arm Limited (or its affiliates). All rights reserved.
 // Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.
 // SPDX-License-Identifier: Apache-2.0

-use super::{Error, Result};
-use hypervisor::kvm::kvm_bindings::{
+use crate::arch::aarch64::gic::{Error, Result};
+use crate::device::HypervisorDeviceError;
+use crate::kvm::kvm_bindings::{
    kvm_device_attr, KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS, KVM_REG_ARM64_SYSREG_CRM_MASK,
    KVM_REG_ARM64_SYSREG_CRM_SHIFT, KVM_REG_ARM64_SYSREG_CRN_MASK, KVM_REG_ARM64_SYSREG_CRN_SHIFT,
    KVM_REG_ARM64_SYSREG_OP0_MASK, KVM_REG_ARM64_SYSREG_OP0_SHIFT, KVM_REG_ARM64_SYSREG_OP1_MASK,
    KVM_REG_ARM64_SYSREG_OP1_SHIFT, KVM_REG_ARM64_SYSREG_OP2_MASK, KVM_REG_ARM64_SYSREG_OP2_SHIFT,
 };
-use std::sync::Arc;
+use kvm_ioctls::DeviceFd;

 const KVM_DEV_ARM_VGIC_V3_MPIDR_SHIFT: u32 = 32;
 const KVM_DEV_ARM_VGIC_V3_MPIDR_MASK: u64 = 0xffffffff << KVM_DEV_ARM_VGIC_V3_MPIDR_SHIFT as u64;
@@ -77,13 +79,7 @@ static VGIC_ICC_REGS: &[u64] = &[
    SYS_ICC_AP1R3_EL1,
 ];

-fn icc_attr_access(
-    gic: &Arc<dyn hypervisor::Device>,
-    offset: u64,
-    typer: u64,
-    val: &u32,
-    set: bool,
-) -> Result<()> {
+fn icc_attr_access(gic: &DeviceFd, offset: u64, typer: u64, val: &u32, set: bool) -> Result<()> {
    let mut gic_icc_attr = kvm_device_attr {
        group: KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS,
        attr: ((typer & KVM_DEV_ARM_VGIC_V3_MPIDR_MASK) | offset), // this needs the mpidr
@@ -91,17 +87,19 @@ fn icc_attr_access(
        flags: 0,
    };
    if set {
-        gic.set_device_attr(&gic_icc_attr)
-            .map_err(Error::SetDeviceAttribute)?;
+        gic.set_device_attr(&gic_icc_attr).map_err(|e| {
+            Error::SetDeviceAttribute(HypervisorDeviceError::SetDeviceAttribute(e.into()))
+        })?;
    } else {
-        gic.get_device_attr(&mut gic_icc_attr)
-            .map_err(Error::GetDeviceAttribute)?;
+        gic.get_device_attr(&mut gic_icc_attr).map_err(|e| {
+            Error::GetDeviceAttribute(HypervisorDeviceError::GetDeviceAttribute(e.into()))
+        })?;
    }
    Ok(())
 }

 /// Get ICC registers.
-pub fn get_icc_regs(gic: &Arc<dyn hypervisor::Device>, gicr_typer: &[u64]) -> Result<Vec<u32>> {
+pub fn get_icc_regs(gic: &DeviceFd, gicr_typer: &[u64]) -> Result<Vec<u32>> {
    let mut state: Vec<u32> = Vec::new();
    // We need this for the ICC_AP<m>R<n>_EL1 registers.
    let mut num_priority_bits = 0;
@@ -154,11 +152,7 @@ pub fn get_icc_regs(gic: &Arc<dyn hypervisor::Device>, gicr_typer: &[u64]) -> Re
 }

 /// Set ICC registers.
-pub fn set_icc_regs(
-    gic: &Arc<dyn hypervisor::Device>,
-    gicr_typer: &[u64],
-    state: &[u32],
-) -> Result<()> {
+pub fn set_icc_regs(gic: &DeviceFd, gicr_typer: &[u64], state: &[u32]) -> Result<()> {
    let mut num_priority_bits = 0;
    let mut idx = 0;
    for ix in gicr_typer {
--- a/hypervisor/src/kvm/aarch64/gic/mod.rs
+++ b/hypervisor/src/kvm/aarch64/gic/mod.rs
@@ -0,0 +1,617 @@
+// Copyright 2022 Arm Limited (or its affiliates). All rights reserved.
+
+mod dist_regs;
+mod icc_regs;
+mod redist_regs;
+
+use crate::arch::aarch64::gic::{Error, Result, Vgic};
+use crate::device::HypervisorDeviceError;
+use crate::kvm::{kvm_bindings, KvmVm};
+use crate::{CpuState, Vm};
+use dist_regs::{get_dist_regs, read_ctlr, set_dist_regs, write_ctlr};
+use icc_regs::{get_icc_regs, set_icc_regs};
+use kvm_ioctls::DeviceFd;
+use redist_regs::{construct_gicr_typers, get_redist_regs, set_redist_regs};
+use serde::{Deserialize, Serialize};
+use std::any::Any;
+use std::convert::TryInto;
+
+const GITS_CTLR: u32 = 0x0000;
+const GITS_IIDR: u32 = 0x0004;
+const GITS_CBASER: u32 = 0x0080;
+const GITS_CWRITER: u32 = 0x0088;
+const GITS_CREADR: u32 = 0x0090;
+const GITS_BASER: u32 = 0x0100;
+
+/// Access an ITS device attribute.
+///
+/// This is a helper function to get/set the ITS device attribute depending
+/// the bool parameter `set` provided.
+pub fn gicv3_its_attr_access(
+    its_device: &DeviceFd,
+    group: u32,
+    attr: u32,
+    val: &u64,
+    set: bool,
+) -> Result<()> {
+    let mut gicv3_its_attr = kvm_bindings::kvm_device_attr {
+        group,
+        attr: attr as u64,
+        addr: val as *const u64 as u64,
+        flags: 0,
+    };
+    if set {
+        its_device.set_device_attr(&gicv3_its_attr).map_err(|e| {
+            Error::SetDeviceAttribute(HypervisorDeviceError::SetDeviceAttribute(e.into()))
+        })
+    } else {
+        its_device
+            .get_device_attr(&mut gicv3_its_attr)
+            .map_err(|e| {
+                Error::GetDeviceAttribute(HypervisorDeviceError::GetDeviceAttribute(e.into()))
+            })
+    }
+}
+
+/// Function that saves/restores ITS tables into guest RAM.
+///
+/// The tables get flushed to guest RAM whenever the VM gets stopped.
+pub fn gicv3_its_tables_access(its_device: &DeviceFd, save: bool) -> Result<()> {
+    let attr = if save {
+        u64::from(kvm_bindings::KVM_DEV_ARM_ITS_SAVE_TABLES)
+    } else {
+        u64::from(kvm_bindings::KVM_DEV_ARM_ITS_RESTORE_TABLES)
+    };
+
+    let init_gic_attr = kvm_bindings::kvm_device_attr {
+        group: kvm_bindings::KVM_DEV_ARM_VGIC_GRP_CTRL,
+        attr,
+        addr: 0,
+        flags: 0,
+    };
+    its_device
+        .set_device_attr(&init_gic_attr)
+        .map_err(|e| Error::SetDeviceAttribute(HypervisorDeviceError::SetDeviceAttribute(e.into())))
+}
+
+pub struct KvmGicV3Its {
+    /// The KVM device for the GicV3
+    device: DeviceFd,
+
+    /// The KVM device for the Its device
+    its_device: Option<DeviceFd>,
+
+    /// Vector holding values of GICR_TYPER for each vCPU
+    gicr_typers: Vec<u64>,
+
+    /// GIC distributor address
+    dist_addr: u64,
+
+    /// GIC distributor size
+    dist_size: u64,
+
+    /// GIC distributors address
+    redists_addr: u64,
+
+    /// GIC distributors size
+    redists_size: u64,
+
+    /// GIC MSI address
+    msi_addr: u64,
+
+    /// GIC MSI size
+    msi_size: u64,
+
+    /// Number of CPUs handled by the device
+    vcpu_count: u64,
+}
+
+#[derive(Clone, Default, Serialize, Deserialize)]
+pub struct Gicv3ItsState {
+    dist: Vec<u32>,
+    rdist: Vec<u32>,
+    icc: Vec<u32>,
+    // special register that enables interrupts and affinity routing
+    gicd_ctlr: u32,
+    its_ctlr: u64,
+    its_iidr: u64,
+    its_cbaser: u64,
+    its_cwriter: u64,
+    its_creadr: u64,
+    its_baser: [u64; 8],
+}
+
+impl KvmGicV3Its {
+    /// Device trees specific constants
+    pub const ARCH_GIC_V3_MAINT_IRQ: u32 = 9;
+
+    /// Returns the GIC version of the device
+    fn version() -> u32 {
+        kvm_bindings::kvm_device_type_KVM_DEV_TYPE_ARM_VGIC_V3
+    }
+
+    /// Setup the device-specific attributes
+    fn init_device_attributes(&mut self, vm: &KvmVm, nr_irqs: u32) -> Result<()> {
+        // GicV3 part attributes
+        /* Setting up the distributor attribute.
+         We are placing the GIC below 1GB so we need to substract the size of the distributor.
+        */
+        Self::set_device_attribute(
+            &self.device,
+            kvm_bindings::KVM_DEV_ARM_VGIC_GRP_ADDR,
+            u64::from(kvm_bindings::KVM_VGIC_V3_ADDR_TYPE_DIST),
+            &self.dist_addr as *const u64 as u64,
+            0,
+        )?;
+
+        /* Setting up the redistributors' attribute.
+        We are calculating here the start of the redistributors address. We have one per CPU.
+        */
+        Self::set_device_attribute(
+            &self.device,
+            kvm_bindings::KVM_DEV_ARM_VGIC_GRP_ADDR,
+            u64::from(kvm_bindings::KVM_VGIC_V3_ADDR_TYPE_REDIST),
+            &self.redists_addr as *const u64 as u64,
+            0,
+        )?;
+
+        // ITS part attributes
+        let mut its_device = kvm_bindings::kvm_create_device {
+            type_: kvm_bindings::kvm_device_type_KVM_DEV_TYPE_ARM_VGIC_ITS,
+            fd: 0,
+            flags: 0,
+        };
+
+        let its_fd = vm
+            .create_device(&mut its_device)
+            .map_err(Error::CreateGic)?;
+
+        // We know vm is KvmVm
+        let its_fd = its_fd.to_kvm().unwrap();
+
+        Self::set_device_attribute(
+            &its_fd,
+            kvm_bindings::KVM_DEV_ARM_VGIC_GRP_ADDR,
+            u64::from(kvm_bindings::KVM_VGIC_ITS_ADDR_TYPE),
+            &self.msi_addr as *const u64 as u64,
+            0,
+        )?;
+
+        Self::set_device_attribute(
+            &its_fd,
+            kvm_bindings::KVM_DEV_ARM_VGIC_GRP_CTRL,
+            u64::from(kvm_bindings::KVM_DEV_ARM_VGIC_CTRL_INIT),
+            0,
+            0,
+        )?;
+
+        self.its_device = Some(its_fd);
+
+        /* We need to tell the kernel how many irqs to support with this vgic.
+         * See the `layout` module for details.
+         */
+        let nr_irqs_ptr = &nr_irqs as *const u32;
+        Self::set_device_attribute(
+            &self.device,
+            kvm_bindings::KVM_DEV_ARM_VGIC_GRP_NR_IRQS,
+            0,
+            nr_irqs_ptr as u64,
+            0,
+        )?;
+
+        /* Finalize the GIC.
+         * See https://code.woboq.org/linux/linux/virt/kvm/arm/vgic/vgic-kvm-device.c.html#211.
+         */
+        Self::set_device_attribute(
+            &self.device,
+            kvm_bindings::KVM_DEV_ARM_VGIC_GRP_CTRL,
+            u64::from(kvm_bindings::KVM_DEV_ARM_VGIC_CTRL_INIT),
+            0,
+            0,
+        )
+    }
+
+    /// Create a KVM Vgic device
+    fn create_device(vm: &KvmVm) -> Result<DeviceFd> {
+        let mut gic_device = kvm_bindings::kvm_create_device {
+            type_: Self::version(),
+            fd: 0,
+            flags: 0,
+        };
+
+        let device_fd = vm
+            .create_device(&mut gic_device)
+            .map_err(Error::CreateGic)?;
+
+        // We know for sure this is a KVM fd
+        Ok(device_fd.to_kvm().unwrap())
+    }
+
+    /// Set a GIC device attribute
+    fn set_device_attribute(
+        device: &DeviceFd,
+        group: u32,
+        attr: u64,
+        addr: u64,
+        flags: u32,
+    ) -> Result<()> {
+        let attr = kvm_bindings::kvm_device_attr {
+            flags,
+            group,
+            attr,
+            addr,
+        };
+        device.set_device_attr(&attr).map_err(|e| {
+            Error::SetDeviceAttribute(HypervisorDeviceError::SetDeviceAttribute(e.into()))
+        })
+    }
+
+    /// Method to initialize the GIC device
+    #[allow(clippy::new_ret_no_self)]
+    pub fn new(
+        vm: &dyn Vm,
+        vcpu_count: u64,
+        dist_addr: u64,
+        dist_size: u64,
+        redist_size: u64,
+        msi_size: u64,
+        nr_irqs: u32,
+    ) -> Result<KvmGicV3Its> {
+        // This is inside KVM module
+        let vm = vm.as_any().downcast_ref::<KvmVm>().expect("Wrong VM type?");
+
+        let vgic = Self::create_device(vm)?;
+        let redists_size: u64 = redist_size * vcpu_count;
+        let redists_addr: u64 = dist_addr - redists_size;
+        let msi_addr: u64 = redists_addr - msi_size;
+
+        let mut gic_device = KvmGicV3Its {
+            device: vgic,
+            its_device: None,
+            gicr_typers: vec![0; vcpu_count.try_into().unwrap()],
+            dist_addr,
+            dist_size,
+            redists_addr,
+            redists_size,
+            msi_addr,
+            msi_size,
+            vcpu_count,
+        };
+
+        gic_device.init_device_attributes(vm, nr_irqs)?;
+
+        Ok(gic_device)
+    }
+}
+
+impl Vgic for KvmGicV3Its {
+    fn fdt_compatibility(&self) -> &str {
+        "arm,gic-v3"
+    }
+
+    fn msi_compatible(&self) -> bool {
+        true
+    }
+
+    fn msi_compatibility(&self) -> &str {
+        "arm,gic-v3-its"
+    }
+
+    fn fdt_maint_irq(&self) -> u32 {
+        KvmGicV3Its::ARCH_GIC_V3_MAINT_IRQ
+    }
+
+    fn vcpu_count(&self) -> u64 {
+        self.vcpu_count
+    }
+
+    fn device_properties(&self) -> [u64; 4] {
+        [
+            self.dist_addr,
+            self.dist_size,
+            self.redists_addr,
+            self.redists_size,
+        ]
+    }
+
+    fn msi_properties(&self) -> [u64; 2] {
+        [self.msi_addr, self.msi_size]
+    }
+
+    fn set_gicr_typers(&mut self, vcpu_states: &[CpuState]) {
+        let gicr_typers = construct_gicr_typers(vcpu_states);
+        self.gicr_typers = gicr_typers;
+    }
+
+    fn as_any_concrete_mut(&mut self) -> &mut dyn Any {
+        self
+    }
+
+    /// Save the state of GICv3ITS.
+    fn state(&self) -> Result<Gicv3ItsState> {
+        let gicr_typers = self.gicr_typers.clone();
+
+        let gicd_ctlr = read_ctlr(&self.device)?;
+
+        let dist_state = get_dist_regs(&self.device)?;
+
+        let rdist_state = get_redist_regs(&self.device, &gicr_typers)?;
+
+        let icc_state = get_icc_regs(&self.device, &gicr_typers)?;
+
+        let its_baser_state: [u64; 8] = [0; 8];
+        for i in 0..8 {
+            gicv3_its_attr_access(
+                self.its_device.as_ref().unwrap(),
+                kvm_bindings::KVM_DEV_ARM_VGIC_GRP_ITS_REGS,
+                GITS_BASER + i * 8,
+                &its_baser_state[i as usize],
+                false,
+            )?;
+        }
+
+        let its_ctlr_state: u64 = 0;
+        gicv3_its_attr_access(
+            self.its_device.as_ref().unwrap(),
+            kvm_bindings::KVM_DEV_ARM_VGIC_GRP_ITS_REGS,
+            GITS_CTLR,
+            &its_ctlr_state,
+            false,
+        )?;
+
+        let its_cbaser_state: u64 = 0;
+        gicv3_its_attr_access(
+            self.its_device.as_ref().unwrap(),
+            kvm_bindings::KVM_DEV_ARM_VGIC_GRP_ITS_REGS,
+            GITS_CBASER,
+            &its_cbaser_state,
+            false,
+        )?;
+
+        let its_creadr_state: u64 = 0;
+        gicv3_its_attr_access(
+            self.its_device.as_ref().unwrap(),
+            kvm_bindings::KVM_DEV_ARM_VGIC_GRP_ITS_REGS,
+            GITS_CREADR,
+            &its_creadr_state,
+            false,
+        )?;
+
+        let its_cwriter_state: u64 = 0;
+        gicv3_its_attr_access(
+            self.its_device.as_ref().unwrap(),
+            kvm_bindings::KVM_DEV_ARM_VGIC_GRP_ITS_REGS,
+            GITS_CWRITER,
+            &its_cwriter_state,
+            false,
+        )?;
+
+        let its_iidr_state: u64 = 0;
+        gicv3_its_attr_access(
+            self.its_device.as_ref().unwrap(),
+            kvm_bindings::KVM_DEV_ARM_VGIC_GRP_ITS_REGS,
+            GITS_IIDR,
+            &its_iidr_state,
+            false,
+        )?;
+
+        Ok(Gicv3ItsState {
+            dist: dist_state,
+            rdist: rdist_state,
+            icc: icc_state,
+            gicd_ctlr,
+            its_ctlr: its_ctlr_state,
+            its_iidr: its_iidr_state,
+            its_cbaser: its_cbaser_state,
+            its_cwriter: its_cwriter_state,
+            its_creadr: its_creadr_state,
+            its_baser: its_baser_state,
+        })
+    }
+
+    /// Restore the state of GICv3ITS.
+    fn set_state(&mut self, state: &Gicv3ItsState) -> Result<()> {
+        let gicr_typers = self.gicr_typers.clone();
+
+        write_ctlr(&self.device, state.gicd_ctlr)?;
+
+        set_dist_regs(&self.device, &state.dist)?;
+
+        set_redist_regs(&self.device, &gicr_typers, &state.rdist)?;
+
+        set_icc_regs(&self.device, &gicr_typers, &state.icc)?;
+
+        //Restore GICv3ITS registers
+        gicv3_its_attr_access(
+            self.its_device.as_ref().unwrap(),
+            kvm_bindings::KVM_DEV_ARM_VGIC_GRP_ITS_REGS,
+            GITS_IIDR,
+            &state.its_iidr,
+            true,
+        )?;
+
+        gicv3_its_attr_access(
+            self.its_device.as_ref().unwrap(),
+            kvm_bindings::KVM_DEV_ARM_VGIC_GRP_ITS_REGS,
+            GITS_CBASER,
+            &state.its_cbaser,
+            true,
+        )?;
+
+        gicv3_its_attr_access(
+            self.its_device.as_ref().unwrap(),
+            kvm_bindings::KVM_DEV_ARM_VGIC_GRP_ITS_REGS,
+            GITS_CREADR,
+            &state.its_creadr,
+            true,
+        )?;
+
+        gicv3_its_attr_access(
+            self.its_device.as_ref().unwrap(),
+            kvm_bindings::KVM_DEV_ARM_VGIC_GRP_ITS_REGS,
+            GITS_CWRITER,
+            &state.its_cwriter,
+            true,
+        )?;
+
+        for i in 0..8 {
+            gicv3_its_attr_access(
+                self.its_device.as_ref().unwrap(),
+                kvm_bindings::KVM_DEV_ARM_VGIC_GRP_ITS_REGS,
+                GITS_BASER + i * 8,
+                &state.its_baser[i as usize],
+                true,
+            )?;
+        }
+
+        // Restore ITS tables
+        gicv3_its_tables_access(self.its_device.as_ref().unwrap(), false)?;
+
+        gicv3_its_attr_access(
+            self.its_device.as_ref().unwrap(),
+            kvm_bindings::KVM_DEV_ARM_VGIC_GRP_ITS_REGS,
+            GITS_CTLR,
+            &state.its_ctlr,
+            true,
+        )
+    }
+
+    /// Saves GIC internal data tables into RAM, including:
+    /// - RDIST pending tables
+    /// - ITS tables into guest RAM.
+    fn save_data_tables(&self) -> Result<()> {
+        // Flash RDIST pending tables
+        let init_gic_attr = kvm_bindings::kvm_device_attr {
+            group: kvm_bindings::KVM_DEV_ARM_VGIC_GRP_CTRL,
+            attr: u64::from(kvm_bindings::KVM_DEV_ARM_VGIC_SAVE_PENDING_TABLES),
+            addr: 0,
+            flags: 0,
+        };
+        self.device.set_device_attr(&init_gic_attr).map_err(|e| {
+            Error::SetDeviceAttribute(HypervisorDeviceError::SetDeviceAttribute(e.into()))
+        })?;
+        // Flush ITS tables to guest RAM.
+        gicv3_its_tables_access(self.its_device.as_ref().unwrap(), true)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use crate::aarch64::gic::{
+        get_dist_regs, get_icc_regs, get_redist_regs, set_dist_regs, set_icc_regs, set_redist_regs,
+    };
+    use crate::kvm::KvmGicV3Its;
+
+    #[test]
+    fn test_create_gic() {
+        let hv = crate::new().unwrap();
+        let vm = hv.create_vm().unwrap();
+
+        assert!(KvmGicV3Its::new(
+            &*vm,
+            1,
+            0x0900_0000 - 0x01_0000,
+            0x01_0000,
+            0x02_0000,
+            0x02_0000,
+            256
+        )
+        .is_ok());
+    }
+
+    #[test]
+    fn test_get_set_dist_regs() {
+        let hv = crate::new().unwrap();
+        let vm = hv.create_vm().unwrap();
+        let _ = vm.create_vcpu(0, None).unwrap();
+        let gic = KvmGicV3Its::new(
+            &*vm,
+            1,
+            0x0900_0000 - 0x01_0000,
+            0x01_0000,
+            0x02_0000,
+            0x02_0000,
+            256,
+        )
+        .expect("Cannot create gic");
+
+        let res = get_dist_regs(&gic.device);
+        assert!(res.is_ok());
+        let state = res.unwrap();
+        assert_eq!(state.len(), 568);
+
+        let res = set_dist_regs(&gic.device, &state);
+        assert!(res.is_ok());
+    }
+
+    #[test]
+    fn test_get_set_redist_regs() {
+        let hv = crate::new().unwrap();
+        let vm = hv.create_vm().unwrap();
+        let _ = vm.create_vcpu(0, None).unwrap();
+        let gic = KvmGicV3Its::new(
+            &*vm,
+            1,
+            0x0900_0000 - 0x01_0000,
+            0x01_0000,
+            0x02_0000,
+            0x02_0000,
+            256,
+        )
+        .expect("Cannot create gic");
+
+        let gicr_typer = vec![123];
+        let res = get_redist_regs(&gic.device, &gicr_typer);
+        assert!(res.is_ok());
+        let state = res.unwrap();
+        println!("{}", state.len());
+        assert!(state.len() == 24);
+
+        assert!(set_redist_regs(&gic.device, &gicr_typer, &state).is_ok());
+    }
+
+    #[test]
+    fn test_get_set_icc_regs() {
+        let hv = crate::new().unwrap();
+        let vm = hv.create_vm().unwrap();
+        let _ = vm.create_vcpu(0, None).unwrap();
+        let gic = KvmGicV3Its::new(
+            &*vm,
+            1,
+            0x0900_0000 - 0x01_0000,
+            0x01_0000,
+            0x02_0000,
+            0x02_0000,
+            256,
+        )
+        .expect("Cannot create gic");
+
+        let gicr_typer = vec![123];
+        let res = get_icc_regs(&gic.device, &gicr_typer);
+        assert!(res.is_ok());
+        let state = res.unwrap();
+        println!("{}", state.len());
+        assert!(state.len() == 9);
+
+        assert!(set_icc_regs(&gic.device, &gicr_typer, &state).is_ok());
+    }
+
+    #[test]
+    fn test_save_data_tables() {
+        let hv = crate::new().unwrap();
+        let vm = hv.create_vm().unwrap();
+        let _ = vm.create_vcpu(0, None).unwrap();
+        let gic = vm
+            .create_vgic(
+                1,
+                0x0900_0000 - 0x01_0000,
+                0x01_0000,
+                0x02_0000,
+                0x02_0000,
+                256,
+            )
+            .expect("Cannot create gic");
+
+        assert!(gic.lock().unwrap().save_data_tables().is_ok());
+    }
+}
--- a/hypervisor/src/kvm/aarch64/gic/redist_regs.rs
+++ b/hypervisor/src/kvm/aarch64/gic/redist_regs.rs
@@ -1,10 +1,18 @@
+// Copyright 2022 Arm Limited (or its affiliates). All rights reserved.
 // Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.
 // SPDX-License-Identifier: Apache-2.0

-use super::{Error, Result};
-use hypervisor::kvm::kvm_bindings::{kvm_device_attr, KVM_DEV_ARM_VGIC_GRP_REDIST_REGS};
-use hypervisor::CpuState;
-use std::sync::Arc;
+use crate::arch::aarch64::gic::{Error, Result};
+use crate::device::HypervisorDeviceError;
+use crate::kvm::kvm_bindings::{
+    kvm_device_attr, KVM_DEV_ARM_VGIC_GRP_REDIST_REGS, KVM_REG_ARM64, KVM_REG_ARM64_SYSREG,
+    KVM_REG_ARM64_SYSREG_OP0_MASK, KVM_REG_ARM64_SYSREG_OP0_SHIFT, KVM_REG_ARM64_SYSREG_OP2_MASK,
+    KVM_REG_ARM64_SYSREG_OP2_SHIFT, KVM_REG_SIZE_U64,
+};
+use crate::kvm::Register;
+use crate::kvm::VcpuKvmState;
+use crate::CpuState;
+use kvm_ioctls::DeviceFd;

 // Relevant redistributor registers that we want to save/restore.
 const GICR_CTLR: u32 = 0x0000;
@@ -31,6 +39,12 @@ const GICR_ICFGR0: u32 = GICR_SGI_OFFSET + 0x0C00;
 const KVM_DEV_ARM_VGIC_V3_MPIDR_SHIFT: u32 = 32;
 const KVM_DEV_ARM_VGIC_V3_MPIDR_MASK: u64 = 0xffffffff << KVM_DEV_ARM_VGIC_V3_MPIDR_SHIFT as u64;

+const KVM_ARM64_SYSREG_MPIDR_EL1: u64 = KVM_REG_ARM64 as u64
+    | KVM_REG_SIZE_U64 as u64
+    | KVM_REG_ARM64_SYSREG as u64
+    | (((3_u64) << KVM_REG_ARM64_SYSREG_OP0_SHIFT) & KVM_REG_ARM64_SYSREG_OP0_MASK as u64)
+    | (((5_u64) << KVM_REG_ARM64_SYSREG_OP2_SHIFT) & KVM_REG_ARM64_SYSREG_OP2_MASK as u64);
+
 /// This is how we represent the registers of a distributor.
 /// It is relrvant their offset from the base address of the
 /// distributor.
@@ -82,31 +96,27 @@ static VGIC_SGI_REGS: &[RdistReg] = &[
    VGIC_RDIST_REG!(GICR_IPRIORITYR0, 32),
 ];

-fn redist_attr_access(
-    gic: &Arc<dyn hypervisor::Device>,
-    offset: u32,
-    typer: u64,
-    val: &u32,
-    set: bool,
-) -> Result<()> {
-    let mut gic_dist_attr = kvm_device_attr {
+fn redist_attr_access(gic: &DeviceFd, offset: u32, typer: u64, val: &u32, set: bool) -> Result<()> {
+    let mut gic_redist_attr = kvm_device_attr {
        group: KVM_DEV_ARM_VGIC_GRP_REDIST_REGS,
        attr: (typer & KVM_DEV_ARM_VGIC_V3_MPIDR_MASK) | (offset as u64), // this needs the mpidr
        addr: val as *const u32 as u64,
        flags: 0,
    };
    if set {
-        gic.set_device_attr(&gic_dist_attr)
-            .map_err(Error::SetDeviceAttribute)?;
+        gic.set_device_attr(&gic_redist_attr).map_err(|e| {
+            Error::SetDeviceAttribute(HypervisorDeviceError::SetDeviceAttribute(e.into()))
+        })?;
    } else {
-        gic.get_device_attr(&mut gic_dist_attr)
-            .map_err(Error::GetDeviceAttribute)?;
+        gic.get_device_attr(&mut gic_redist_attr).map_err(|e| {
+            Error::GetDeviceAttribute(HypervisorDeviceError::GetDeviceAttribute(e.into()))
+        })?;
    }
    Ok(())
 }

 fn access_redists_aux(
-    gic: &Arc<dyn hypervisor::Device>,
+    gic: &DeviceFd,
    gicr_typer: &[u64],
    state: &mut Vec<u32>,
    reg_list: &[RdistReg],
@@ -136,7 +146,7 @@ fn access_redists_aux(
 }

 /// Get redistributor registers.
-pub fn get_redist_regs(gic: &Arc<dyn hypervisor::Device>, gicr_typer: &[u64]) -> Result<Vec<u32>> {
+pub fn get_redist_regs(gic: &DeviceFd, gicr_typer: &[u64]) -> Result<Vec<u32>> {
    let mut state = Vec::new();
    let mut idx: usize = 0;
    access_redists_aux(
@@ -153,11 +163,7 @@ pub fn get_redist_regs(gic: &Arc<dyn hypervisor::Device>, gicr_typer: &[u64]) ->
 }

 /// Set redistributor registers.
-pub fn set_redist_regs(
-    gic: &Arc<dyn hypervisor::Device>,
-    gicr_typer: &[u64],
-    state: &[u32],
-) -> Result<()> {
+pub fn set_redist_regs(gic: &DeviceFd, gicr_typer: &[u64], state: &[u32]) -> Result<()> {
    let mut idx: usize = 0;
    let mut mut_state = state.to_owned();
    access_redists_aux(
@@ -193,6 +199,7 @@ pub fn construct_gicr_typers(vcpu_states: &[CpuState]) -> Vec<u64> {
     */
    let mut gicr_typers: Vec<u64> = Vec::new();
    for (index, state) in vcpu_states.iter().enumerate() {
+        let state: VcpuKvmState = state.clone().into();
        let last = {
            if index == vcpu_states.len() - 1 {
                1
@@ -200,8 +207,14 @@ pub fn construct_gicr_typers(vcpu_states: &[CpuState]) -> Vec<u64> {
                0
            }
        };
+        // state.sys_regs is a big collection of system registers, including MIPDR_EL1
+        let mpidr: Vec<Register> = state
+            .sys_regs
+            .into_iter()
+            .filter(|reg| reg.id == KVM_ARM64_SYSREG_MPIDR_EL1)
+            .collect();
        //calculate affinity
-        let mut cpu_affid = state.mpidr & 1095233437695;
+        let mut cpu_affid = mpidr[0].addr & 1095233437695;
        cpu_affid = ((cpu_affid & 0xFF00000000) >> 8) | (cpu_affid & 0xFFFFFF);
        gicr_typers.push((cpu_affid << 32) | (1 << 24) | (index as u64) << 8 | (last << 4));
    }
--- a/hypervisor/src/kvm/aarch64/mod.rs
+++ b/hypervisor/src/kvm/aarch64/mod.rs
@@ -8,17 +8,12 @@
 //
 //

-///
-/// Export generically-named wrappers of kvm-bindings for Unix-based platforms
-///
+pub mod gic;
+
 use crate::kvm::{KvmError, KvmResult};
 use kvm_bindings::{
-    kvm_mp_state, kvm_one_reg, kvm_regs, KVM_REG_ARM64, KVM_REG_ARM64_SYSREG,
-    KVM_REG_ARM64_SYSREG_CRM_MASK, KVM_REG_ARM64_SYSREG_CRM_SHIFT, KVM_REG_ARM64_SYSREG_CRN_MASK,
-    KVM_REG_ARM64_SYSREG_CRN_SHIFT, KVM_REG_ARM64_SYSREG_OP0_MASK, KVM_REG_ARM64_SYSREG_OP0_SHIFT,
-    KVM_REG_ARM64_SYSREG_OP1_MASK, KVM_REG_ARM64_SYSREG_OP1_SHIFT, KVM_REG_ARM64_SYSREG_OP2_MASK,
-    KVM_REG_ARM64_SYSREG_OP2_SHIFT, KVM_REG_ARM_COPROC_MASK, KVM_REG_ARM_CORE, KVM_REG_SIZE_MASK,
-    KVM_REG_SIZE_U32, KVM_REG_SIZE_U64,
+    kvm_mp_state, kvm_one_reg, kvm_regs, KVM_REG_ARM_COPROC_MASK, KVM_REG_ARM_CORE,
+    KVM_REG_SIZE_MASK, KVM_REG_SIZE_U32, KVM_REG_SIZE_U64,
 };
 pub use kvm_bindings::{
    kvm_one_reg as Register, kvm_regs as StandardRegisters, kvm_vcpu_init as VcpuInit, RegList,
@@ -86,32 +81,6 @@ macro_rules! arm64_core_reg_id {
    };
 }

-// This macro computes the ID of a specific ARM64 system register similar to how
-// the kernel C macro does.
-// https://elixir.bootlin.com/linux/v4.20.17/source/arch/arm64/include/uapi/asm/kvm.h#L203
-#[macro_export]
-macro_rules! arm64_sys_reg {
-    ($name: tt, $op0: tt, $op1: tt, $crn: tt, $crm: tt, $op2: tt) => {
-        pub const $name: u64 = KVM_REG_ARM64 as u64
-            | KVM_REG_SIZE_U64 as u64
-            | KVM_REG_ARM64_SYSREG as u64
-            | ((($op0 as u64) << KVM_REG_ARM64_SYSREG_OP0_SHIFT)
-                & KVM_REG_ARM64_SYSREG_OP0_MASK as u64)
-            | ((($op1 as u64) << KVM_REG_ARM64_SYSREG_OP1_SHIFT)
-                & KVM_REG_ARM64_SYSREG_OP1_MASK as u64)
-            | ((($crn as u64) << KVM_REG_ARM64_SYSREG_CRN_SHIFT)
-                & KVM_REG_ARM64_SYSREG_CRN_MASK as u64)
-            | ((($crm as u64) << KVM_REG_ARM64_SYSREG_CRM_SHIFT)
-                & KVM_REG_ARM64_SYSREG_CRM_MASK as u64)
-            | ((($op2 as u64) << KVM_REG_ARM64_SYSREG_OP2_SHIFT)
-                & KVM_REG_ARM64_SYSREG_OP2_MASK as u64);
-    };
-}
-
-// Constant imported from the Linux kernel:
-// https://elixir.bootlin.com/linux/v4.20.17/source/arch/arm64/include/asm/sysreg.h#L135
-arm64_sys_reg!(MPIDR_EL1, 3, 0, 0, 0, 5);
-
 /// Specifies whether a particular register is a system register or not.
 /// The kernel splits the registers on aarch64 in core registers and system registers.
 /// So, below we get the system registers by checking that they are not core registers.
@@ -150,8 +119,4 @@ pub struct VcpuKvmState {
    pub mp_state: kvm_mp_state,
    pub core_regs: kvm_regs,
    pub sys_regs: Vec<kvm_one_reg>,
-    // We will be using the mpidr for passing it to the VmState.
-    // The VmState will give this away for saving restoring the icc and redistributor
-    // registers.
-    pub mpidr: u64,
 }
--- a/hypervisor/src/kvm/mod.rs
+++ b/hypervisor/src/kvm/mod.rs
--- a/hypervisor/src/kvm/x86_64/mod.rs
+++ b/hypervisor/src/kvm/x86_64/mod.rs
@@ -8,7 +8,10 @@
 //
 //

-use crate::arch::x86::{msr_index, SegmentRegisterOps, MTRR_ENABLE, MTRR_MEM_TYPE_WB};
+use crate::arch::x86::{
+    CpuIdEntry, DescriptorTable, FpuState, LapicState, MsrEntry, SegmentRegister, SpecialRegisters,
+    StandardRegisters, CPUID_FLAG_VALID_INDEX,
+};
 use crate::kvm::{Cap, Kvm, KvmError, KvmResult};
 use serde::{Deserialize, Serialize};

@@ -16,101 +19,15 @@ use serde::{Deserialize, Serialize};
 /// Export generically-named wrappers of kvm-bindings for Unix-based platforms
 ///
 pub use {
-    kvm_bindings::kvm_cpuid_entry2 as CpuIdEntry, kvm_bindings::kvm_dtable as DescriptorTable,
-    kvm_bindings::kvm_fpu as FpuState, kvm_bindings::kvm_lapic_state as LapicState,
-    kvm_bindings::kvm_mp_state as MpState, kvm_bindings::kvm_msr_entry as MsrEntry,
-    kvm_bindings::kvm_regs as StandardRegisters, kvm_bindings::kvm_segment as SegmentRegister,
-    kvm_bindings::kvm_sregs as SpecialRegisters, kvm_bindings::kvm_vcpu_events as VcpuEvents,
+    kvm_bindings::kvm_cpuid_entry2, kvm_bindings::kvm_dtable, kvm_bindings::kvm_fpu,
+    kvm_bindings::kvm_lapic_state, kvm_bindings::kvm_mp_state as MpState,
+    kvm_bindings::kvm_msr_entry, kvm_bindings::kvm_regs, kvm_bindings::kvm_segment,
+    kvm_bindings::kvm_sregs, kvm_bindings::kvm_vcpu_events as VcpuEvents,
    kvm_bindings::kvm_xcrs as ExtendedControlRegisters, kvm_bindings::kvm_xsave as Xsave,
    kvm_bindings::CpuId, kvm_bindings::MsrList, kvm_bindings::Msrs as MsrEntries,
-    kvm_bindings::KVM_CPUID_FLAG_SIGNIFCANT_INDEX as CPUID_FLAG_VALID_INDEX,
+    kvm_bindings::KVM_CPUID_FLAG_SIGNIFCANT_INDEX,
 };

-impl SegmentRegisterOps for SegmentRegister {
-    fn segment_type(&self) -> u8 {
-        self.type_
-    }
-    fn set_segment_type(&mut self, val: u8) {
-        self.type_ = val;
-    }
-
-    fn dpl(&self) -> u8 {
-        self.dpl
-    }
-
-    fn set_dpl(&mut self, val: u8) {
-        self.dpl = val;
-    }
-
-    fn present(&self) -> u8 {
-        self.present
-    }
-
-    fn set_present(&mut self, val: u8) {
-        self.present = val;
-    }
-
-    fn long(&self) -> u8 {
-        self.l
-    }
-
-    fn set_long(&mut self, val: u8) {
-        self.l = val;
-    }
-
-    fn avl(&self) -> u8 {
-        self.avl
-    }
-
-    fn set_avl(&mut self, val: u8) {
-        self.avl = val;
-    }
-
-    fn desc_type(&self) -> u8 {
-        self.s
-    }
-
-    fn set_desc_type(&mut self, val: u8) {
-        self.s = val;
-    }
-
-    fn granularity(&self) -> u8 {
-        self.g
-    }
-
-    fn set_granularity(&mut self, val: u8) {
-        self.g = val;
-    }
-
-    fn db(&self) -> u8 {
-        self.db
-    }
-
-    fn set_db(&mut self, val: u8) {
-        self.db = val;
-    }
-}
-
-pub fn boot_msr_entries() -> MsrEntries {
-    MsrEntries::from_entries(&[
-        msr!(msr_index::MSR_IA32_SYSENTER_CS),
-        msr!(msr_index::MSR_IA32_SYSENTER_ESP),
-        msr!(msr_index::MSR_IA32_SYSENTER_EIP),
-        msr!(msr_index::MSR_STAR),
-        msr!(msr_index::MSR_CSTAR),
-        msr!(msr_index::MSR_LSTAR),
-        msr!(msr_index::MSR_KERNEL_GS_BASE),
-        msr!(msr_index::MSR_SYSCALL_MASK),
-        msr!(msr_index::MSR_IA32_TSC),
-        msr_data!(
-            msr_index::MSR_IA32_MISC_ENABLE,
-            msr_index::MSR_IA32_MISC_ENABLE_FAST_STRING as u64
-        ),
-        msr_data!(msr_index::MSR_MTRRdefType, MTRR_ENABLE | MTRR_MEM_TYPE_WB),
-    ])
-    .unwrap()
-}
-
 ///
 /// Check KVM extension for Linux
 ///
@@ -137,14 +54,275 @@ pub fn check_required_kvm_extensions(kvm: &Kvm) -> KvmResult<()> {
 }
 #[derive(Clone, Serialize, Deserialize)]
 pub struct VcpuKvmState {
-    pub cpuid: CpuId,
-    pub msrs: MsrEntries,
+    pub cpuid: Vec<CpuIdEntry>,
+    pub msrs: Vec<MsrEntry>,
    pub vcpu_events: VcpuEvents,
-    pub regs: StandardRegisters,
-    pub sregs: SpecialRegisters,
+    pub regs: kvm_regs,
+    pub sregs: kvm_sregs,
    pub fpu: FpuState,
    pub lapic_state: LapicState,
    pub xsave: Xsave,
    pub xcrs: ExtendedControlRegisters,
    pub mp_state: MpState,
 }
+
+impl From<StandardRegisters> for kvm_regs {
+    fn from(regs: StandardRegisters) -> Self {
+        Self {
+            rax: regs.rax,
+            rbx: regs.rbx,
+            rcx: regs.rcx,
+            rdx: regs.rdx,
+            rsi: regs.rsi,
+            rdi: regs.rdi,
+            rsp: regs.rsp,
+            rbp: regs.rbp,
+            r8: regs.r8,
+            r9: regs.r9,
+            r10: regs.r10,
+            r11: regs.r11,
+            r12: regs.r12,
+            r13: regs.r13,
+            r14: regs.r14,
+            r15: regs.r15,
+            rip: regs.rip,
+            rflags: regs.rflags,
+        }
+    }
+}
+
+impl From<kvm_regs> for StandardRegisters {
+    fn from(regs: kvm_regs) -> Self {
+        Self {
+            rax: regs.rax,
+            rbx: regs.rbx,
+            rcx: regs.rcx,
+            rdx: regs.rdx,
+            rsi: regs.rsi,
+            rdi: regs.rdi,
+            rsp: regs.rsp,
+            rbp: regs.rbp,
+            r8: regs.r8,
+            r9: regs.r9,
+            r10: regs.r10,
+            r11: regs.r11,
+            r12: regs.r12,
+            r13: regs.r13,
+            r14: regs.r14,
+            r15: regs.r15,
+            rip: regs.rip,
+            rflags: regs.rflags,
+        }
+    }
+}
+
+impl From<SegmentRegister> for kvm_segment {
+    fn from(s: SegmentRegister) -> Self {
+        Self {
+            base: s.base,
+            limit: s.limit,
+            selector: s.selector,
+            type_: s.type_,
+            present: s.present,
+            dpl: s.dpl,
+            db: s.db,
+            s: s.s,
+            l: s.l,
+            g: s.g,
+            avl: s.avl,
+            unusable: s.unusable,
+            ..Default::default()
+        }
+    }
+}
+
+impl From<kvm_segment> for SegmentRegister {
+    fn from(s: kvm_segment) -> Self {
+        Self {
+            base: s.base,
+            limit: s.limit,
+            selector: s.selector,
+            type_: s.type_,
+            present: s.present,
+            dpl: s.dpl,
+            db: s.db,
+            s: s.s,
+            l: s.l,
+            g: s.g,
+            avl: s.avl,
+            unusable: s.unusable,
+        }
+    }
+}
+
+impl From<DescriptorTable> for kvm_dtable {
+    fn from(dt: DescriptorTable) -> Self {
+        Self {
+            base: dt.base,
+            limit: dt.limit,
+            ..Default::default()
+        }
+    }
+}
+
+impl From<kvm_dtable> for DescriptorTable {
+    fn from(dt: kvm_dtable) -> Self {
+        Self {
+            base: dt.base,
+            limit: dt.limit,
+        }
+    }
+}
+
+impl From<SpecialRegisters> for kvm_sregs {
+    fn from(s: SpecialRegisters) -> Self {
+        Self {
+            cs: s.cs.into(),
+            ds: s.ds.into(),
+            es: s.es.into(),
+            fs: s.fs.into(),
+            gs: s.gs.into(),
+            ss: s.ss.into(),
+            tr: s.tr.into(),
+            ldt: s.ldt.into(),
+            gdt: s.gdt.into(),
+            idt: s.idt.into(),
+            cr0: s.cr0,
+            cr2: s.cr2,
+            cr3: s.cr3,
+            cr4: s.cr4,
+            cr8: s.cr8,
+            efer: s.efer,
+            apic_base: s.apic_base,
+            interrupt_bitmap: s.interrupt_bitmap,
+        }
+    }
+}
+
+impl From<kvm_sregs> for SpecialRegisters {
+    fn from(s: kvm_sregs) -> Self {
+        Self {
+            cs: s.cs.into(),
+            ds: s.ds.into(),
+            es: s.es.into(),
+            fs: s.fs.into(),
+            gs: s.gs.into(),
+            ss: s.ss.into(),
+            tr: s.tr.into(),
+            ldt: s.ldt.into(),
+            gdt: s.gdt.into(),
+            idt: s.idt.into(),
+            cr0: s.cr0,
+            cr2: s.cr2,
+            cr3: s.cr3,
+            cr4: s.cr4,
+            cr8: s.cr8,
+            efer: s.efer,
+            apic_base: s.apic_base,
+            interrupt_bitmap: s.interrupt_bitmap,
+        }
+    }
+}
+
+impl From<CpuIdEntry> for kvm_cpuid_entry2 {
+    fn from(e: CpuIdEntry) -> Self {
+        let flags = if e.flags & CPUID_FLAG_VALID_INDEX != 0 {
+            KVM_CPUID_FLAG_SIGNIFCANT_INDEX
+        } else {
+            0
+        };
+        Self {
+            function: e.function,
+            index: e.index,
+            flags,
+            eax: e.eax,
+            ebx: e.ebx,
+            ecx: e.ecx,
+            edx: e.edx,
+            ..Default::default()
+        }
+    }
+}
+
+impl From<kvm_cpuid_entry2> for CpuIdEntry {
+    fn from(e: kvm_cpuid_entry2) -> Self {
+        let flags = if e.flags & KVM_CPUID_FLAG_SIGNIFCANT_INDEX != 0 {
+            CPUID_FLAG_VALID_INDEX
+        } else {
+            0
+        };
+        Self {
+            function: e.function,
+            index: e.index,
+            flags,
+            eax: e.eax,
+            ebx: e.ebx,
+            ecx: e.ecx,
+            edx: e.edx,
+        }
+    }
+}
+
+impl From<kvm_fpu> for FpuState {
+    fn from(s: kvm_fpu) -> Self {
+        Self {
+            fpr: s.fpr,
+            fcw: s.fcw,
+            fsw: s.fsw,
+            ftwx: s.ftwx,
+            last_opcode: s.last_opcode,
+            last_ip: s.last_ip,
+            last_dp: s.last_dp,
+            xmm: s.xmm,
+            mxcsr: s.mxcsr,
+        }
+    }
+}
+
+impl From<FpuState> for kvm_fpu {
+    fn from(s: FpuState) -> Self {
+        Self {
+            fpr: s.fpr,
+            fcw: s.fcw,
+            fsw: s.fsw,
+            ftwx: s.ftwx,
+            last_opcode: s.last_opcode,
+            last_ip: s.last_ip,
+            last_dp: s.last_dp,
+            xmm: s.xmm,
+            mxcsr: s.mxcsr,
+            ..Default::default()
+        }
+    }
+}
+
+impl From<LapicState> for kvm_lapic_state {
+    fn from(s: LapicState) -> Self {
+        Self { regs: s.regs }
+    }
+}
+
+impl From<kvm_lapic_state> for LapicState {
+    fn from(s: kvm_lapic_state) -> Self {
+        Self { regs: s.regs }
+    }
+}
+
+impl From<kvm_msr_entry> for MsrEntry {
+    fn from(e: kvm_msr_entry) -> Self {
+        Self {
+            index: e.index,
+            data: e.data,
+        }
+    }
+}
+
+impl From<MsrEntry> for kvm_msr_entry {
+    fn from(e: MsrEntry) -> Self {
+        Self {
+            index: e.index,
+            data: e.data,
+            ..Default::default()
+        }
+    }
+}
--- a/hypervisor/src/lib.rs
+++ b/hypervisor/src/lib.rs
@@ -18,6 +18,8 @@
 //! - arm64
 //!

+#![allow(clippy::significant_drop_in_scrutinee)]
+
 #[macro_use]
 extern crate anyhow;
 #[cfg(target_arch = "x86_64")]
@@ -49,40 +51,36 @@ mod cpu;
 mod device;

 pub use cpu::{HypervisorCpuError, Vcpu, VmExit};
-pub use device::{Device, HypervisorDeviceError};
+pub use device::HypervisorDeviceError;
 pub use hypervisor::{Hypervisor, HypervisorError};
 #[cfg(all(feature = "kvm", target_arch = "aarch64"))]
-pub use kvm::aarch64;
-#[cfg(all(feature = "kvm", target_arch = "x86_64"))]
-pub use kvm::x86_64;
-// Aliased types exposed from both hypervisors
-#[cfg(feature = "kvm")]
-pub use kvm::{
-    ClockData, CpuState, CreateDevice, DeviceAttr, DeviceFd, IoEventAddress, IrqRoutingEntry,
-    MemoryRegion, MpState, VcpuEvents, VcpuExit, VmState,
-};
-#[cfg(all(feature = "mshv", target_arch = "x86_64"))]
-pub use mshv::x86_64;
-// Aliased types exposed from both hypervisors
-#[cfg(all(feature = "mshv", target_arch = "x86_64"))]
-pub use mshv::{
-    CpuState, CreateDevice, DeviceAttr, DeviceFd, IoEventAddress, IrqRoutingEntry, MemoryRegion,
-    MpState, VcpuEvents, VcpuExit, VmState,
-};
+pub use kvm::{aarch64, GicState};
 use std::sync::Arc;
 pub use vm::{
    DataMatch, HypervisorVmError, InterruptSourceConfig, LegacyIrqSourceConfig, MsiIrqSourceConfig,
    Vm, VmOps,
 };

+#[derive(Debug, Copy, Clone)]
+pub enum HypervisorType {
+    Kvm,
+    Mshv,
+}
+
 pub fn new() -> std::result::Result<Arc<dyn Hypervisor>, HypervisorError> {
    #[cfg(feature = "kvm")]
-    let hv = kvm::KvmHypervisor::new()?;
+    if kvm::KvmHypervisor::is_available()? {
+        return kvm::KvmHypervisor::new();
+    }

    #[cfg(feature = "mshv")]
-    let hv = mshv::MshvHypervisor::new()?;
+    if mshv::MshvHypervisor::is_available()? {
+        return mshv::MshvHypervisor::new();
+    }

-    Ok(Arc::new(hv))
+    Err(HypervisorError::HypervisorCreate(anyhow!(
+        "no supported hypervisor"
+    )))
 }

 // Returns a `Vec<T>` with a size in bytes at least as large as `size_in_bytes`.
@@ -114,3 +112,75 @@ pub fn vec_with_array_field<T: Default, F>(count: usize) -> Vec<T> {
    let vec_size_bytes = size_of::<T>() + element_space;
    vec_with_size_in_bytes(vec_size_bytes)
 }
+
+///
+/// User memory region structure
+///
+#[derive(Debug, Default, Eq, PartialEq)]
+pub struct UserMemoryRegion {
+    pub slot: u32,
+    pub guest_phys_addr: u64,
+    pub memory_size: u64,
+    pub userspace_addr: u64,
+    pub flags: u32,
+}
+
+///
+/// Flags for user memory region
+///
+pub const USER_MEMORY_REGION_READ: u32 = 1;
+pub const USER_MEMORY_REGION_WRITE: u32 = 1 << 1;
+pub const USER_MEMORY_REGION_EXECUTE: u32 = 1 << 2;
+pub const USER_MEMORY_REGION_LOG_DIRTY: u32 = 1 << 3;
+
+#[derive(Debug)]
+pub enum MpState {
+    #[cfg(feature = "kvm")]
+    Kvm(kvm_bindings::kvm_mp_state),
+    #[cfg(all(feature = "mshv", target_arch = "x86_64"))]
+    Mshv, /* MSHV does not supprt MpState yet */
+}
+
+#[derive(Debug, Clone, Copy)]
+pub enum IoEventAddress {
+    Pio(u64),
+    Mmio(u64),
+}
+
+#[derive(Clone, serde::Serialize, serde::Deserialize)]
+#[allow(clippy::large_enum_variant)]
+pub enum CpuState {
+    #[cfg(feature = "kvm")]
+    Kvm(kvm::VcpuKvmState),
+    #[cfg(all(feature = "mshv", target_arch = "x86_64"))]
+    Mshv(mshv::VcpuMshvState),
+}
+
+#[derive(Debug, Clone, Copy, serde::Serialize, serde::Deserialize)]
+#[cfg(target_arch = "x86_64")]
+pub enum ClockData {
+    #[cfg(feature = "kvm")]
+    Kvm(kvm_bindings::kvm_clock_data),
+    #[cfg(feature = "mshv")]
+    Mshv, /* MSHV does not supprt ClockData yet */
+}
+
+#[cfg(target_arch = "x86_64")]
+impl ClockData {
+    pub fn reset_flags(&mut self) {
+        match self {
+            #[cfg(feature = "kvm")]
+            ClockData::Kvm(s) => s.flags = 0,
+            #[allow(unreachable_patterns)]
+            _ => {}
+        }
+    }
+}
+
+#[derive(Copy, Clone)]
+pub enum IrqRoutingEntry {
+    #[cfg(feature = "kvm")]
+    Kvm(kvm_bindings::kvm_irq_routing_entry),
+    #[cfg(feature = "mshv")]
+    Mshv(mshv_bindings::mshv_msi_routing_entry),
+}
--- a/hypervisor/src/mshv/mod.rs
+++ b/hypervisor/src/mshv/mod.rs
@@ -12,26 +12,35 @@ use crate::cpu::Vcpu;
 use crate::hypervisor;
 use crate::vec_with_array_field;
 use crate::vm::{self, InterruptSourceConfig, VmOps};
+use crate::HypervisorType;
 pub use mshv_bindings::*;
-pub use mshv_ioctls::IoEventAddress;
 use mshv_ioctls::{set_registers_64, Mshv, NoDatamatch, VcpuFd, VmFd};
-use serde::{Deserialize, Serialize};
+use std::any::Any;
 use std::collections::HashMap;
 use std::sync::{Arc, RwLock};
+use vfio_ioctls::VfioDeviceFd;
 use vm::DataMatch;
 // x86_64 dependencies
 #[cfg(target_arch = "x86_64")]
 pub mod x86_64;
-use crate::device;
+use crate::{
+    ClockData, CpuState, IoEventAddress, IrqRoutingEntry, MpState, UserMemoryRegion,
+    USER_MEMORY_REGION_EXECUTE, USER_MEMORY_REGION_READ, USER_MEMORY_REGION_WRITE,
+};
 use vmm_sys_util::eventfd::EventFd;
 #[cfg(target_arch = "x86_64")]
-pub use x86_64::VcpuMshvState as CpuState;
+pub use x86_64::VcpuMshvState;
 #[cfg(target_arch = "x86_64")]
 pub use x86_64::*;

 #[cfg(target_arch = "x86_64")]
 use std::fs::File;
-use std::os::unix::io::{AsRawFd, RawFd};
+use std::os::unix::io::AsRawFd;
+
+#[cfg(target_arch = "x86_64")]
+use crate::arch::x86::{
+    CpuIdEntry, FpuState, LapicState, MsrEntry, SpecialRegisters, StandardRegisters,
+};

 const DIRTY_BITMAP_CLEAR_DIRTY: u64 = 0x4;
 const DIRTY_BITMAP_SET_DIRTY: u64 = 0x8;
@@ -41,18 +50,108 @@ const DIRTY_BITMAP_SET_DIRTY: u64 = 0x8;
 ///
 pub use {
    mshv_bindings::mshv_create_device as CreateDevice,
-    mshv_bindings::mshv_device_attr as DeviceAttr,
-    mshv_bindings::mshv_msi_routing_entry as IrqRoutingEntry, mshv_ioctls::DeviceFd,
+    mshv_bindings::mshv_device_attr as DeviceAttr, mshv_ioctls::DeviceFd,
 };

 pub const PAGE_SHIFT: usize = 12;

-#[derive(Debug, Default, Copy, Clone, Serialize, Deserialize)]
-pub struct HvState {
-    hypercall_page: u64,
+impl From<mshv_user_mem_region> for UserMemoryRegion {
+    fn from(region: mshv_user_mem_region) -> Self {
+        let mut flags: u32 = 0;
+        if region.flags & HV_MAP_GPA_READABLE != 0 {
+            flags |= USER_MEMORY_REGION_READ;
+        }
+        if region.flags & HV_MAP_GPA_WRITABLE != 0 {
+            flags |= USER_MEMORY_REGION_WRITE;
+        }
+        if region.flags & HV_MAP_GPA_EXECUTABLE != 0 {
+            flags |= USER_MEMORY_REGION_EXECUTE;
+        }
+
+        UserMemoryRegion {
+            guest_phys_addr: (region.guest_pfn << PAGE_SHIFT as u64)
+                + (region.userspace_addr & ((1 << PAGE_SHIFT) - 1)),
+            memory_size: region.size,
+            userspace_addr: region.userspace_addr,
+            flags,
+            ..Default::default()
+        }
+    }
 }

-pub use HvState as VmState;
+impl From<UserMemoryRegion> for mshv_user_mem_region {
+    fn from(region: UserMemoryRegion) -> Self {
+        let mut flags: u32 = 0;
+        if region.flags & USER_MEMORY_REGION_READ != 0 {
+            flags |= HV_MAP_GPA_READABLE;
+        }
+        if region.flags & USER_MEMORY_REGION_WRITE != 0 {
+            flags |= HV_MAP_GPA_WRITABLE;
+        }
+        if region.flags & USER_MEMORY_REGION_EXECUTE != 0 {
+            flags |= HV_MAP_GPA_EXECUTABLE;
+        }
+
+        mshv_user_mem_region {
+            guest_pfn: region.guest_phys_addr >> PAGE_SHIFT,
+            size: region.memory_size,
+            userspace_addr: region.userspace_addr,
+            flags,
+        }
+    }
+}
+
+impl From<mshv_ioctls::IoEventAddress> for IoEventAddress {
+    fn from(a: mshv_ioctls::IoEventAddress) -> Self {
+        match a {
+            mshv_ioctls::IoEventAddress::Pio(x) => Self::Pio(x),
+            mshv_ioctls::IoEventAddress::Mmio(x) => Self::Mmio(x),
+        }
+    }
+}
+
+impl From<IoEventAddress> for mshv_ioctls::IoEventAddress {
+    fn from(a: IoEventAddress) -> Self {
+        match a {
+            IoEventAddress::Pio(x) => Self::Pio(x),
+            IoEventAddress::Mmio(x) => Self::Mmio(x),
+        }
+    }
+}
+
+impl From<VcpuMshvState> for CpuState {
+    fn from(s: VcpuMshvState) -> Self {
+        CpuState::Mshv(s)
+    }
+}
+
+impl From<CpuState> for VcpuMshvState {
+    fn from(s: CpuState) -> Self {
+        match s {
+            CpuState::Mshv(s) => s,
+            /* Needed in case other hypervisors are enabled */
+            #[allow(unreachable_patterns)]
+            _ => panic!("CpuState is not valid"),
+        }
+    }
+}
+
+impl From<mshv_msi_routing_entry> for IrqRoutingEntry {
+    fn from(s: mshv_msi_routing_entry) -> Self {
+        IrqRoutingEntry::Mshv(s)
+    }
+}
+
+impl From<IrqRoutingEntry> for mshv_msi_routing_entry {
+    fn from(e: IrqRoutingEntry) -> Self {
+        match e {
+            IrqRoutingEntry::Mshv(e) => e,
+            /* Needed in case other hypervisors are enabled */
+            #[allow(unreachable_patterns)]
+            _ => panic!("IrqRoutingEntry is not valid"),
+        }
+    }
+}

 struct MshvDirtyLogSlot {
    guest_pfn: u64,
@@ -64,12 +163,35 @@ pub struct MshvHypervisor {
    mshv: Mshv,
 }

+impl MshvHypervisor {
+    #[cfg(target_arch = "x86_64")]
+    ///
+    /// Retrieve the list of MSRs supported by MSHV.
+    ///
+    fn get_msr_list(&self) -> hypervisor::Result<MsrList> {
+        self.mshv
+            .get_msr_index_list()
+            .map_err(|e| hypervisor::HypervisorError::GetMsrList(e.into()))
+    }
+}
+
 impl MshvHypervisor {
    /// Create a hypervisor based on Mshv
-    pub fn new() -> hypervisor::Result<MshvHypervisor> {
+    #[allow(clippy::new_ret_no_self)]
+    pub fn new() -> hypervisor::Result<Arc<dyn hypervisor::Hypervisor>> {
        let mshv_obj =
            Mshv::new().map_err(|e| hypervisor::HypervisorError::HypervisorCreate(e.into()))?;
-        Ok(MshvHypervisor { mshv: mshv_obj })
+        Ok(Arc::new(MshvHypervisor { mshv: mshv_obj }))
+    }
+    /// Check if the hypervisor is available
+    pub fn is_available() -> hypervisor::Result<bool> {
+        match std::fs::metadata("/dev/mshv") {
+            Ok(_) => Ok(true),
+            Err(err) if err.kind() == std::io::ErrorKind::NotFound => Ok(false),
+            Err(err) => Err(hypervisor::HypervisorError::HypervisorAvailableCheck(
+                err.into(),
+            )),
+        }
    }
 }
 /// Implementation of Hypervisor trait for Mshv
@@ -81,6 +203,12 @@ impl MshvHypervisor {
 /// let vm = hypervisor.create_vm().expect("new VM fd creation failed");
 ///
 impl hypervisor::Hypervisor for MshvHypervisor {
+    ///
+    /// Returns the type of the hypervisor
+    ///
+    fn hypervisor_type(&self) -> HypervisorType {
+        HypervisorType::Mshv
+    }
    /// Create a mshv vm object and return the object as Vm trait object
    /// Example
    /// # extern crate hypervisor;
@@ -108,49 +236,51 @@ impl hypervisor::Hypervisor for MshvHypervisor {
            break;
        }

+        // Default Microsoft Hypervisor behavior for unimplemented MSR is to
+        // send a fault to the guest if it tries to access it. It is possible
+        // to override this behavior with a more suitable option i.e., ignore
+        // writes from the guest and return zero in attempt to read unimplemented
+        // MSR.
+        fd.set_partition_property(
+            hv_partition_property_code_HV_PARTITION_PROPERTY_UNIMPLEMENTED_MSR_ACTION,
+            hv_unimplemented_msr_action_HV_UNIMPLEMENTED_MSR_ACTION_IGNORE_WRITE_READ_ZERO as u64,
+        )
+        .map_err(|e| hypervisor::HypervisorError::SetPartitionProperty(e.into()))?;
+
        let msr_list = self.get_msr_list()?;
        let num_msrs = msr_list.as_fam_struct_ref().nmsrs as usize;
-        let mut msrs = MsrEntries::new(num_msrs).unwrap();
+        let mut msrs: Vec<MsrEntry> = vec![
+            MsrEntry {
+                ..Default::default()
+            };
+            num_msrs
+        ];
        let indices = msr_list.as_slice();
-        let msr_entries = msrs.as_mut_slice();
        for (pos, index) in indices.iter().enumerate() {
-            msr_entries[pos].index = *index;
+            msrs[pos].index = *index;
        }
        let vm_fd = Arc::new(fd);

        Ok(Arc::new(MshvVm {
            fd: vm_fd,
            msrs,
-            hv_state: hv_state_init(),
-            vm_ops: None,
            dirty_log_slots: Arc::new(RwLock::new(HashMap::new())),
        }))
    }
    ///
    /// Get the supported CpuID
    ///
-    fn get_cpuid(&self) -> hypervisor::Result<CpuId> {
-        Ok(CpuId::new(1).unwrap())
-    }
-    #[cfg(target_arch = "x86_64")]
-    ///
-    /// Retrieve the list of MSRs supported by MSHV.
-    ///
-    fn get_msr_list(&self) -> hypervisor::Result<MsrList> {
-        self.mshv
-            .get_msr_index_list()
-            .map_err(|e| hypervisor::HypervisorError::GetMsrList(e.into()))
+    fn get_cpuid(&self) -> hypervisor::Result<Vec<CpuIdEntry>> {
+        Ok(Vec::new())
    }
 }

-#[allow(dead_code)]
 /// Vcpu struct for Microsoft Hypervisor
 pub struct MshvVcpu {
    fd: VcpuFd,
    vp_index: u8,
-    cpuid: CpuId,
-    msrs: MsrEntries,
-    hv_state: Arc<RwLock<HvState>>, // Mshv State
+    cpuid: Vec<CpuIdEntry>,
+    msrs: Vec<MsrEntry>,
    vm_ops: Option<Arc<dyn vm::VmOps>>,
 }

@@ -170,17 +300,20 @@ impl cpu::Vcpu for MshvVcpu {
    /// Returns the vCPU general purpose registers.
    ///
    fn get_regs(&self) -> cpu::Result<StandardRegisters> {
-        self.fd
+        Ok(self
+            .fd
            .get_regs()
-            .map_err(|e| cpu::HypervisorCpuError::GetStandardRegs(e.into()))
+            .map_err(|e| cpu::HypervisorCpuError::GetStandardRegs(e.into()))?
+            .into())
    }
    #[cfg(target_arch = "x86_64")]
    ///
    /// Sets the vCPU general purpose registers.
    ///
    fn set_regs(&self, regs: &StandardRegisters) -> cpu::Result<()> {
+        let regs = (*regs).into();
        self.fd
-            .set_regs(regs)
+            .set_regs(&regs)
            .map_err(|e| cpu::HypervisorCpuError::SetStandardRegs(e.into()))
    }
    #[cfg(target_arch = "x86_64")]
@@ -188,17 +321,20 @@ impl cpu::Vcpu for MshvVcpu {
    /// Returns the vCPU special registers.
    ///
    fn get_sregs(&self) -> cpu::Result<SpecialRegisters> {
-        self.fd
+        Ok(self
+            .fd
            .get_sregs()
-            .map_err(|e| cpu::HypervisorCpuError::GetSpecialRegs(e.into()))
+            .map_err(|e| cpu::HypervisorCpuError::GetSpecialRegs(e.into()))?
+            .into())
    }
    #[cfg(target_arch = "x86_64")]
    ///
    /// Sets the vCPU special registers.
    ///
    fn set_sregs(&self, sregs: &SpecialRegisters) -> cpu::Result<()> {
+        let sregs = (*sregs).into();
        self.fd
-            .set_sregs(sregs)
+            .set_sregs(&sregs)
            .map_err(|e| cpu::HypervisorCpuError::SetSpecialRegs(e.into()))
    }
    #[cfg(target_arch = "x86_64")]
@@ -206,17 +342,20 @@ impl cpu::Vcpu for MshvVcpu {
    /// Returns the floating point state (FPU) from the vCPU.
    ///
    fn get_fpu(&self) -> cpu::Result<FpuState> {
-        self.fd
+        Ok(self
+            .fd
            .get_fpu()
-            .map_err(|e| cpu::HypervisorCpuError::GetFloatingPointRegs(e.into()))
+            .map_err(|e| cpu::HypervisorCpuError::GetFloatingPointRegs(e.into()))?
+            .into())
    }
    #[cfg(target_arch = "x86_64")]
    ///
    /// Set the floating point state (FPU) of a vCPU.
    ///
    fn set_fpu(&self, fpu: &FpuState) -> cpu::Result<()> {
+        let fpu: mshv_bindings::FloatingPointUnit = (*fpu).clone().into();
        self.fd
-            .set_fpu(fpu)
+            .set_fpu(&fpu)
            .map_err(|e| cpu::HypervisorCpuError::SetFloatingPointRegs(e.into()))
    }

@@ -224,60 +363,36 @@ impl cpu::Vcpu for MshvVcpu {
    ///
    /// Returns the model-specific registers (MSR) for this vCPU.
    ///
-    fn get_msrs(&self, msrs: &mut MsrEntries) -> cpu::Result<usize> {
-        self.fd
-            .get_msrs(msrs)
-            .map_err(|e| cpu::HypervisorCpuError::GetMsrEntries(e.into()))
+    fn get_msrs(&self, msrs: &mut Vec<MsrEntry>) -> cpu::Result<usize> {
+        let mshv_msrs: Vec<msr_entry> = msrs.iter().map(|e| (*e).into()).collect();
+        let mut mshv_msrs = MsrEntries::from_entries(&mshv_msrs).unwrap();
+        let succ = self
+            .fd
+            .get_msrs(&mut mshv_msrs)
+            .map_err(|e| cpu::HypervisorCpuError::GetMsrEntries(e.into()))?;
+
+        msrs[..succ].copy_from_slice(
+            &mshv_msrs.as_slice()[..succ]
+                .iter()
+                .map(|e| (*e).into())
+                .collect::<Vec<MsrEntry>>(),
+        );
+
+        Ok(succ)
    }
    #[cfg(target_arch = "x86_64")]
    ///
    /// Setup the model-specific registers (MSR) for this vCPU.
    /// Returns the number of MSR entries actually written.
    ///
-    fn set_msrs(&self, msrs: &MsrEntries) -> cpu::Result<usize> {
+    fn set_msrs(&self, msrs: &[MsrEntry]) -> cpu::Result<usize> {
+        let mshv_msrs: Vec<msr_entry> = msrs.iter().map(|e| (*e).into()).collect();
+        let mshv_msrs = MsrEntries::from_entries(&mshv_msrs).unwrap();
        self.fd
-            .set_msrs(msrs)
+            .set_msrs(&mshv_msrs)
            .map_err(|e| cpu::HypervisorCpuError::SetMsrEntries(e.into()))
    }

-    #[cfg(target_arch = "x86_64")]
-    ///
-    /// X86 specific call that returns the vcpu's current "xcrs".
-    ///
-    fn get_xcrs(&self) -> cpu::Result<ExtendedControlRegisters> {
-        self.fd
-            .get_xcrs()
-            .map_err(|e| cpu::HypervisorCpuError::GetXcsr(e.into()))
-    }
-    #[cfg(target_arch = "x86_64")]
-    ///
-    /// X86 specific call that sets the vcpu's current "xcrs".
-    ///
-    fn set_xcrs(&self, xcrs: &ExtendedControlRegisters) -> cpu::Result<()> {
-        self.fd
-            .set_xcrs(xcrs)
-            .map_err(|e| cpu::HypervisorCpuError::SetXcsr(e.into()))
-    }
-    #[cfg(target_arch = "x86_64")]
-    ///
-    /// Returns currently pending exceptions, interrupts, and NMIs as well as related
-    /// states of the vcpu.
-    ///
-    fn get_vcpu_events(&self) -> cpu::Result<VcpuEvents> {
-        self.fd
-            .get_vcpu_events()
-            .map_err(|e| cpu::HypervisorCpuError::GetVcpuEvents(e.into()))
-    }
-    #[cfg(target_arch = "x86_64")]
-    ///
-    /// Sets pending exceptions, interrupts, and NMIs as well as related states
-    /// of the vcpu.
-    ///
-    fn set_vcpu_events(&self, events: &VcpuEvents) -> cpu::Result<()> {
-        self.fd
-            .set_vcpu_events(events)
-            .map_err(|e| cpu::HypervisorCpuError::SetVcpuEvents(e.into()))
-    }
    #[cfg(target_arch = "x86_64")]
    ///
    /// X86 specific call to enable HyperV SynIC
@@ -452,14 +567,14 @@ impl cpu::Vcpu for MshvVcpu {
    ///
    /// X86 specific call to setup the CPUID registers.
    ///
-    fn set_cpuid2(&self, _cpuid: &CpuId) -> cpu::Result<()> {
+    fn set_cpuid2(&self, _cpuid: &[CpuIdEntry]) -> cpu::Result<()> {
        Ok(())
    }
    #[cfg(target_arch = "x86_64")]
    ///
    /// X86 specific call to retrieve the CPUID registers.
    ///
-    fn get_cpuid2(&self, _num_entries: usize) -> cpu::Result<CpuId> {
+    fn get_cpuid2(&self, _num_entries: usize) -> cpu::Result<Vec<CpuIdEntry>> {
        Ok(self.cpuid.clone())
    }
    #[cfg(target_arch = "x86_64")]
@@ -467,45 +582,43 @@ impl cpu::Vcpu for MshvVcpu {
    /// Returns the state of the LAPIC (Local Advanced Programmable Interrupt Controller).
    ///
    fn get_lapic(&self) -> cpu::Result<LapicState> {
-        self.fd
+        Ok(self
+            .fd
            .get_lapic()
-            .map_err(|e| cpu::HypervisorCpuError::GetlapicState(e.into()))
+            .map_err(|e| cpu::HypervisorCpuError::GetlapicState(e.into()))?
+            .into())
    }
    #[cfg(target_arch = "x86_64")]
    ///
    /// Sets the state of the LAPIC (Local Advanced Programmable Interrupt Controller).
    ///
    fn set_lapic(&self, lapic: &LapicState) -> cpu::Result<()> {
+        let lapic: mshv_bindings::LapicState = (*lapic).clone().into();
        self.fd
-            .set_lapic(lapic)
+            .set_lapic(&lapic)
            .map_err(|e| cpu::HypervisorCpuError::SetLapicState(e.into()))
    }
-    #[cfg(target_arch = "x86_64")]
    ///
-    /// X86 specific call that returns the vcpu's current "xsave struct".
+    /// Returns the vcpu's current "multiprocessing state".
    ///
-    fn get_xsave(&self) -> cpu::Result<Xsave> {
-        self.fd
-            .get_xsave()
-            .map_err(|e| cpu::HypervisorCpuError::GetXsaveState(e.into()))
+    fn get_mp_state(&self) -> cpu::Result<MpState> {
+        Ok(MpState::Mshv)
    }
-    #[cfg(target_arch = "x86_64")]
    ///
-    /// X86 specific call that sets the vcpu's current "xsave struct".
+    /// Sets the vcpu's current "multiprocessing state".
    ///
-    fn set_xsave(&self, xsave: &Xsave) -> cpu::Result<()> {
-        self.fd
-            .set_xsave(xsave)
-            .map_err(|e| cpu::HypervisorCpuError::SetXsaveState(e.into()))
+    fn set_mp_state(&self, _mp_state: MpState) -> cpu::Result<()> {
+        Ok(())
    }
    ///
    /// Set CPU state
    ///
    fn set_state(&self, state: &CpuState) -> cpu::Result<()> {
+        let state: VcpuMshvState = state.clone().into();
        self.set_msrs(&state.msrs)?;
        self.set_vcpu_events(&state.vcpu_events)?;
-        self.set_regs(&state.regs)?;
-        self.set_sregs(&state.sregs)?;
+        self.set_regs(&state.regs.into())?;
+        self.set_sregs(&state.sregs.into())?;
        self.set_fpu(&state.fpu)?;
        self.set_xcrs(&state.xcrs)?;
        self.set_lapic(&state.lapic)?;
@@ -544,18 +657,19 @@ impl cpu::Vcpu for MshvVcpu {
            .get_debug_regs()
            .map_err(|e| cpu::HypervisorCpuError::GetDebugRegs(e.into()))?;

-        Ok(CpuState {
+        Ok(VcpuMshvState {
            msrs,
            vcpu_events,
-            regs,
-            sregs,
+            regs: regs.into(),
+            sregs: sregs.into(),
            fpu,
            xcrs,
            lapic,
            dbg,
            xsave,
            misc,
-        })
+        }
+        .into())
    }
    #[cfg(target_arch = "x86_64")]
    ///
@@ -575,42 +689,83 @@ impl cpu::Vcpu for MshvVcpu {
    }
    #[cfg(target_arch = "x86_64")]
    ///
-    /// X86 specific call that returns the vcpu's current "suspend registers".
+    /// Return the list of initial MSR entries for a VCPU
    ///
-    fn get_suspend_regs(&self) -> cpu::Result<SuspendRegisters> {
-        self.fd
-            .get_suspend_regs()
-            .map_err(|e| cpu::HypervisorCpuError::GetSuspendRegs(e.into()))
+    fn boot_msr_entries(&self) -> Vec<MsrEntry> {
+        use crate::arch::x86::{msr_index, MTRR_ENABLE, MTRR_MEM_TYPE_WB};
+
+        [
+            msr!(msr_index::MSR_IA32_SYSENTER_CS),
+            msr!(msr_index::MSR_IA32_SYSENTER_ESP),
+            msr!(msr_index::MSR_IA32_SYSENTER_EIP),
+            msr!(msr_index::MSR_STAR),
+            msr!(msr_index::MSR_CSTAR),
+            msr!(msr_index::MSR_LSTAR),
+            msr!(msr_index::MSR_KERNEL_GS_BASE),
+            msr!(msr_index::MSR_SYSCALL_MASK),
+            msr!(msr_index::MSR_IA32_TSC),
+            msr_data!(msr_index::MSR_MTRRdefType, MTRR_ENABLE | MTRR_MEM_TYPE_WB),
+        ]
+        .to_vec()
    }
 }

-/// Device struct for MSHV
-pub struct MshvDevice {
-    fd: DeviceFd,
-}
-
-impl device::Device for MshvDevice {
+impl MshvVcpu {
+    #[cfg(target_arch = "x86_64")]
    ///
-    /// Set device attribute
+    /// X86 specific call that returns the vcpu's current "xsave struct".
    ///
-    fn set_device_attr(&self, attr: &DeviceAttr) -> device::Result<()> {
+    fn get_xsave(&self) -> cpu::Result<Xsave> {
        self.fd
-            .set_device_attr(attr)
-            .map_err(|e| device::HypervisorDeviceError::SetDeviceAttribute(e.into()))
+            .get_xsave()
+            .map_err(|e| cpu::HypervisorCpuError::GetXsaveState(e.into()))
    }
+    #[cfg(target_arch = "x86_64")]
    ///
-    /// Get device attribute
+    /// X86 specific call that sets the vcpu's current "xsave struct".
    ///
-    fn get_device_attr(&self, attr: &mut DeviceAttr) -> device::Result<()> {
+    fn set_xsave(&self, xsave: &Xsave) -> cpu::Result<()> {
        self.fd
-            .get_device_attr(attr)
-            .map_err(|e| device::HypervisorDeviceError::GetDeviceAttribute(e.into()))
+            .set_xsave(xsave)
+            .map_err(|e| cpu::HypervisorCpuError::SetXsaveState(e.into()))
    }
-}
-
-impl AsRawFd for MshvDevice {
-    fn as_raw_fd(&self) -> RawFd {
-        self.fd.as_raw_fd()
+    #[cfg(target_arch = "x86_64")]
+    ///
+    /// X86 specific call that returns the vcpu's current "xcrs".
+    ///
+    fn get_xcrs(&self) -> cpu::Result<ExtendedControlRegisters> {
+        self.fd
+            .get_xcrs()
+            .map_err(|e| cpu::HypervisorCpuError::GetXcsr(e.into()))
+    }
+    #[cfg(target_arch = "x86_64")]
+    ///
+    /// X86 specific call that sets the vcpu's current "xcrs".
+    ///
+    fn set_xcrs(&self, xcrs: &ExtendedControlRegisters) -> cpu::Result<()> {
+        self.fd
+            .set_xcrs(xcrs)
+            .map_err(|e| cpu::HypervisorCpuError::SetXcsr(e.into()))
+    }
+    #[cfg(target_arch = "x86_64")]
+    ///
+    /// Returns currently pending exceptions, interrupts, and NMIs as well as related
+    /// states of the vcpu.
+    ///
+    fn get_vcpu_events(&self) -> cpu::Result<VcpuEvents> {
+        self.fd
+            .get_vcpu_events()
+            .map_err(|e| cpu::HypervisorCpuError::GetVcpuEvents(e.into()))
+    }
+    #[cfg(target_arch = "x86_64")]
+    ///
+    /// Sets pending exceptions, interrupts, and NMIs as well as related states
+    /// of the vcpu.
+    ///
+    fn set_vcpu_events(&self, events: &VcpuEvents) -> cpu::Result<()> {
+        self.fd
+            .set_vcpu_events(events)
+            .map_err(|e| cpu::HypervisorCpuError::SetVcpuEvents(e.into()))
    }
 }

@@ -739,19 +894,25 @@ impl<'a> PlatformEmulator for MshvEmulatorContext<'a> {
    }
 }

-#[allow(dead_code)]
 /// Wrapper over Mshv VM ioctls.
 pub struct MshvVm {
    fd: Arc<VmFd>,
-    msrs: MsrEntries,
-    // Hypervisor State
-    hv_state: Arc<RwLock<HvState>>,
-    vm_ops: Option<Arc<dyn vm::VmOps>>,
+    msrs: Vec<MsrEntry>,
    dirty_log_slots: Arc<RwLock<HashMap<u64, MshvDirtyLogSlot>>>,
 }

-fn hv_state_init() -> Arc<RwLock<HvState>> {
-    Arc::new(RwLock::new(HvState { hypercall_page: 0 }))
+impl MshvVm {
+    ///
+    /// Creates an in-kernel device.
+    ///
+    /// See the documentation for `MSHV_CREATE_DEVICE`.
+    fn create_device(&self, device: &mut CreateDevice) -> vm::Result<VfioDeviceFd> {
+        let device_fd = self
+            .fd
+            .create_device(device)
+            .map_err(|e| vm::HypervisorVmError::CreateDevice(e.into()))?;
+        Ok(VfioDeviceFd::new_from_mshv(device_fd))
+    }
 }

 ///
@@ -825,9 +986,8 @@ impl vm::Vm for MshvVm {
        let vcpu = MshvVcpu {
            fd: vcpu_fd,
            vp_index: id,
-            cpuid: CpuId::new(1).unwrap(),
+            cpuid: Vec::new(),
            msrs: self.msrs.clone(),
-            hv_state: self.hv_state.clone(),
            vm_ops,
        };
        Ok(Arc::new(vcpu))
@@ -846,6 +1006,7 @@ impl vm::Vm for MshvVm {
        addr: &IoEventAddress,
        datamatch: Option<DataMatch>,
    ) -> vm::Result<()> {
+        let addr = &mshv_ioctls::IoEventAddress::from(*addr);
        debug!(
            "register_ioevent fd {} addr {:x?} datamatch {:?}",
            fd.as_raw_fd(),
@@ -871,6 +1032,7 @@ impl vm::Vm for MshvVm {
    }
    /// Unregister an event from a certain address it has been previously registered to.
    fn unregister_ioevent(&self, fd: &EventFd, addr: &IoEventAddress) -> vm::Result<()> {
+        let addr = &mshv_ioctls::IoEventAddress::from(*addr);
        debug!("unregister_ioevent fd {} addr {:x?}", fd.as_raw_fd(), addr);

        self.fd
@@ -879,7 +1041,8 @@ impl vm::Vm for MshvVm {
    }

    /// Creates a guest physical memory region.
-    fn create_user_memory_region(&self, user_memory_region: MemoryRegion) -> vm::Result<()> {
+    fn create_user_memory_region(&self, user_memory_region: UserMemoryRegion) -> vm::Result<()> {
+        let user_memory_region: mshv_user_mem_region = user_memory_region.into();
        // No matter read only or not we keep track the slots.
        // For readonly hypervisor can enable the dirty bits,
        // but a VM exit happens before setting the dirty bits
@@ -898,7 +1061,8 @@ impl vm::Vm for MshvVm {
    }

    /// Removes a guest physical memory region.
-    fn remove_user_memory_region(&self, user_memory_region: MemoryRegion) -> vm::Result<()> {
+    fn remove_user_memory_region(&self, user_memory_region: UserMemoryRegion) -> vm::Result<()> {
+        let user_memory_region: mshv_user_mem_region = user_memory_region.into();
        // Remove the corresponding entry from "self.dirty_log_slots" if needed
        self.dirty_log_slots
            .write()
@@ -919,7 +1083,7 @@ impl vm::Vm for MshvVm {
        userspace_addr: u64,
        readonly: bool,
        _log_dirty_pages: bool,
-    ) -> MemoryRegion {
+    ) -> UserMemoryRegion {
        let mut flags = HV_MAP_GPA_READABLE | HV_MAP_GPA_EXECUTABLE;
        if !readonly {
            flags |= HV_MAP_GPA_WRITABLE;
@@ -931,22 +1095,10 @@ impl vm::Vm for MshvVm {
            size: memory_size,
            userspace_addr: userspace_addr as u64,
        }
+        .into()
    }

-    ///
-    /// Creates an in-kernel device.
-    ///
-    /// See the documentation for `MSHV_CREATE_DEVICE`.
-    fn create_device(&self, device: &mut CreateDevice) -> vm::Result<Arc<dyn device::Device>> {
-        let fd = self
-            .fd
-            .create_device(device)
-            .map_err(|e| vm::HypervisorVmError::CreateDevice(e.into()))?;
-        let device = MshvDevice { fd };
-        Ok(Arc::new(device))
-    }
-
-    fn create_passthrough_device(&self) -> vm::Result<Arc<dyn device::Device>> {
+    fn create_passthrough_device(&self) -> vm::Result<VfioDeviceFd> {
        let mut vfio_dev = mshv_create_device {
            type_: mshv_device_type_MSHV_DEV_TYPE_VFIO,
            fd: 0,
@@ -960,18 +1112,15 @@ impl vm::Vm for MshvVm {
    ///
    /// Constructs a routing entry
    ///
-    fn make_routing_entry(
-        &self,
-        gsi: u32,
-        config: &InterruptSourceConfig,
-    ) -> mshv_msi_routing_entry {
+    fn make_routing_entry(&self, gsi: u32, config: &InterruptSourceConfig) -> IrqRoutingEntry {
        match config {
            InterruptSourceConfig::MsiIrq(cfg) => mshv_msi_routing_entry {
                gsi,
                address_lo: cfg.low_addr,
                address_hi: cfg.high_addr,
                data: cfg.data,
-            },
+            }
+            .into(),
            _ => {
                unreachable!()
            }
@@ -983,13 +1132,22 @@ impl vm::Vm for MshvVm {
            vec_with_array_field::<mshv_msi_routing, mshv_msi_routing_entry>(entries.len());
        msi_routing[0].nr = entries.len() as u32;

+        let entries: Vec<mshv_msi_routing_entry> = entries
+            .iter()
+            .map(|entry| match entry {
+                IrqRoutingEntry::Mshv(e) => *e,
+                #[allow(unreachable_patterns)]
+                _ => panic!("IrqRoutingEntry type is wrong"),
+            })
+            .collect();
+
        // SAFETY: msi_routing initialized with entries.len() and now it is being turned into
        // entries_slice with entries.len() again. It is guaranteed to be large enough to hold
        // everything from entries.
        unsafe {
            let entries_slice: &mut [mshv_msi_routing_entry] =
                msi_routing[0].entries.as_mut_slice(entries.len());
-            entries_slice.copy_from_slice(entries);
+            entries_slice.copy_from_slice(&entries);
        }

        self.fd
@@ -997,19 +1155,6 @@ impl vm::Vm for MshvVm {
            .map_err(|e| vm::HypervisorVmError::SetGsiRouting(e.into()))
    }
    ///
-    /// Get the Vm state. Return VM specific data
-    ///
-    fn state(&self) -> vm::Result<VmState> {
-        Ok(*self.hv_state.read().unwrap())
-    }
-    ///
-    /// Set the VM state
-    ///
-    fn set_state(&self, state: VmState) -> vm::Result<()> {
-        self.hv_state.write().unwrap().hypercall_page = state.hypercall_page;
-        Ok(())
-    }
-    ///
    /// Start logging dirty pages
    ///
    fn start_dirty_log(&self) -> vm::Result<()> {
@@ -1047,4 +1192,18 @@ impl vm::Vm for MshvVm {
            )
            .map_err(|e| vm::HypervisorVmError::GetDirtyLog(e.into()))
    }
+    /// Retrieve guest clock.
+    #[cfg(target_arch = "x86_64")]
+    fn get_clock(&self) -> vm::Result<ClockData> {
+        Ok(ClockData::Mshv)
+    }
+    /// Set guest clock.
+    #[cfg(target_arch = "x86_64")]
+    fn set_clock(&self, _data: &ClockData) -> vm::Result<()> {
+        Ok(())
+    }
+    /// Downcast to the underlying MshvVm type
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
 }
--- a/hypervisor/src/mshv/x86_64/mod.rs
+++ b/hypervisor/src/mshv/x86_64/mod.rs
@@ -7,7 +7,10 @@
 // Copyright 2018-2019 CrowdStrike, Inc.
 //
 //
-use crate::arch::x86::{msr_index, SegmentRegisterOps, MTRR_ENABLE, MTRR_MEM_TYPE_WB};
+use crate::arch::x86::{
+    CpuIdEntry, DescriptorTable, FpuState, LapicState, MsrEntry, SegmentRegister, SpecialRegisters,
+    StandardRegisters,
+};
 use serde::{Deserialize, Serialize};
 use std::fmt;

@@ -15,25 +18,24 @@ use std::fmt;
 /// Export generically-named wrappers of mshv_bindings for Unix-based platforms
 ///
 pub use {
-    mshv_bindings::hv_cpuid_entry as CpuIdEntry,
-    mshv_bindings::mshv_user_mem_region as MemoryRegion, mshv_bindings::msr_entry as MsrEntry,
-    mshv_bindings::CpuId, mshv_bindings::DebugRegisters,
-    mshv_bindings::FloatingPointUnit as FpuState, mshv_bindings::LapicState,
+    mshv_bindings::hv_cpuid_entry, mshv_bindings::mshv_user_mem_region as MemoryRegion,
+    mshv_bindings::msr_entry, mshv_bindings::CpuId, mshv_bindings::DebugRegisters,
+    mshv_bindings::FloatingPointUnit, mshv_bindings::LapicState as MshvLapicState,
    mshv_bindings::MiscRegs as MiscRegisters, mshv_bindings::MsrList,
-    mshv_bindings::Msrs as MsrEntries, mshv_bindings::Msrs, mshv_bindings::SegmentRegister,
-    mshv_bindings::SpecialRegisters, mshv_bindings::StandardRegisters,
-    mshv_bindings::SuspendRegisters, mshv_bindings::VcpuEvents, mshv_bindings::XSave as Xsave,
+    mshv_bindings::Msrs as MsrEntries, mshv_bindings::Msrs,
+    mshv_bindings::SegmentRegister as MshvSegmentRegister,
+    mshv_bindings::SpecialRegisters as MshvSpecialRegisters,
+    mshv_bindings::StandardRegisters as MshvStandardRegisters, mshv_bindings::SuspendRegisters,
+    mshv_bindings::TableRegister, mshv_bindings::VcpuEvents, mshv_bindings::XSave as Xsave,
    mshv_bindings::Xcrs as ExtendedControlRegisters,
 };

-pub const CPUID_FLAG_VALID_INDEX: u32 = 0;
-
 #[derive(Clone, Serialize, Deserialize)]
 pub struct VcpuMshvState {
-    pub msrs: MsrEntries,
+    pub msrs: Vec<MsrEntry>,
    pub vcpu_events: VcpuEvents,
-    pub regs: StandardRegisters,
-    pub sregs: SpecialRegisters,
+    pub regs: MshvStandardRegisters,
+    pub sregs: MshvSpecialRegisters,
    pub fpu: FpuState,
    pub xcrs: ExtendedControlRegisters,
    pub lapic: LapicState,
@@ -44,10 +46,10 @@ pub struct VcpuMshvState {

 impl fmt::Display for VcpuMshvState {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        let expected_num_msrs = self.msrs.as_fam_struct_ref().nmsrs as usize;
+        let expected_num_msrs = self.msrs.len();
        let mut msr_entries = vec![vec![0; 2]; expected_num_msrs];

-        for (i, entry) in self.msrs.as_slice().iter().enumerate() {
+        for (i, entry) in self.msrs.iter().enumerate() {
            msr_entries[i][1] = entry.data;
            msr_entries[i][0] = entry.index as u64;
        }
@@ -66,87 +68,252 @@ impl fmt::Display for VcpuMshvState {
    }
 }

-pub struct IrqRouting {}
-pub enum VcpuExit {}
-pub struct MpState {}
-
-impl SegmentRegisterOps for SegmentRegister {
-    fn segment_type(&self) -> u8 {
-        self.type_
-    }
-    fn set_segment_type(&mut self, val: u8) {
-        self.type_ = val;
-    }
-
-    fn dpl(&self) -> u8 {
-        self.dpl
-    }
-
-    fn set_dpl(&mut self, val: u8) {
-        self.dpl = val;
-    }
-
-    fn present(&self) -> u8 {
-        self.present
-    }
-
-    fn set_present(&mut self, val: u8) {
-        self.present = val;
-    }
-
-    fn long(&self) -> u8 {
-        self.l
-    }
-
-    fn set_long(&mut self, val: u8) {
-        self.l = val;
-    }
-
-    fn avl(&self) -> u8 {
-        self.avl
-    }
-
-    fn set_avl(&mut self, val: u8) {
-        self.avl = val;
-    }
-
-    fn desc_type(&self) -> u8 {
-        self.s
-    }
-
-    fn set_desc_type(&mut self, val: u8) {
-        self.s = val;
-    }
-
-    fn granularity(&self) -> u8 {
-        self.g
-    }
-
-    fn set_granularity(&mut self, val: u8) {
-        self.g = val;
-    }
-
-    fn db(&self) -> u8 {
-        self.db
-    }
-
-    fn set_db(&mut self, val: u8) {
-        self.db = val;
+impl From<StandardRegisters> for MshvStandardRegisters {
+    fn from(regs: StandardRegisters) -> Self {
+        Self {
+            rax: regs.rax,
+            rbx: regs.rbx,
+            rcx: regs.rcx,
+            rdx: regs.rdx,
+            rsi: regs.rsi,
+            rdi: regs.rdi,
+            rsp: regs.rsp,
+            rbp: regs.rbp,
+            r8: regs.r8,
+            r9: regs.r9,
+            r10: regs.r10,
+            r11: regs.r11,
+            r12: regs.r12,
+            r13: regs.r13,
+            r14: regs.r14,
+            r15: regs.r15,
+            rip: regs.rip,
+            rflags: regs.rflags,
+        }
    }
 }

-pub fn boot_msr_entries() -> MsrEntries {
-    MsrEntries::from_entries(&[
-        msr!(msr_index::MSR_IA32_SYSENTER_CS),
-        msr!(msr_index::MSR_IA32_SYSENTER_ESP),
-        msr!(msr_index::MSR_IA32_SYSENTER_EIP),
-        msr!(msr_index::MSR_STAR),
-        msr!(msr_index::MSR_CSTAR),
-        msr!(msr_index::MSR_LSTAR),
-        msr!(msr_index::MSR_KERNEL_GS_BASE),
-        msr!(msr_index::MSR_SYSCALL_MASK),
-        msr!(msr_index::MSR_IA32_TSC),
-        msr_data!(msr_index::MSR_MTRRdefType, MTRR_ENABLE | MTRR_MEM_TYPE_WB),
-    ])
-    .unwrap()
+impl From<MshvStandardRegisters> for StandardRegisters {
+    fn from(regs: MshvStandardRegisters) -> Self {
+        Self {
+            rax: regs.rax,
+            rbx: regs.rbx,
+            rcx: regs.rcx,
+            rdx: regs.rdx,
+            rsi: regs.rsi,
+            rdi: regs.rdi,
+            rsp: regs.rsp,
+            rbp: regs.rbp,
+            r8: regs.r8,
+            r9: regs.r9,
+            r10: regs.r10,
+            r11: regs.r11,
+            r12: regs.r12,
+            r13: regs.r13,
+            r14: regs.r14,
+            r15: regs.r15,
+            rip: regs.rip,
+            rflags: regs.rflags,
+        }
+    }
+}
+
+impl From<SegmentRegister> for MshvSegmentRegister {
+    fn from(s: SegmentRegister) -> Self {
+        Self {
+            base: s.base,
+            limit: s.limit,
+            selector: s.selector,
+            type_: s.type_,
+            present: s.present,
+            dpl: s.dpl,
+            db: s.db,
+            s: s.s,
+            l: s.l,
+            g: s.g,
+            avl: s.avl,
+            unusable: s.unusable,
+            ..Default::default()
+        }
+    }
+}
+
+impl From<MshvSegmentRegister> for SegmentRegister {
+    fn from(s: MshvSegmentRegister) -> Self {
+        Self {
+            base: s.base,
+            limit: s.limit,
+            selector: s.selector,
+            type_: s.type_,
+            present: s.present,
+            dpl: s.dpl,
+            db: s.db,
+            s: s.s,
+            l: s.l,
+            g: s.g,
+            avl: s.avl,
+            unusable: s.unusable,
+        }
+    }
+}
+
+impl From<DescriptorTable> for TableRegister {
+    fn from(dt: DescriptorTable) -> Self {
+        Self {
+            base: dt.base,
+            limit: dt.limit,
+        }
+    }
+}
+
+impl From<TableRegister> for DescriptorTable {
+    fn from(dt: TableRegister) -> Self {
+        Self {
+            base: dt.base,
+            limit: dt.limit,
+        }
+    }
+}
+
+impl From<SpecialRegisters> for MshvSpecialRegisters {
+    fn from(s: SpecialRegisters) -> Self {
+        Self {
+            cs: s.cs.into(),
+            ds: s.ds.into(),
+            es: s.es.into(),
+            fs: s.fs.into(),
+            gs: s.gs.into(),
+            ss: s.ss.into(),
+            tr: s.tr.into(),
+            ldt: s.ldt.into(),
+            gdt: s.gdt.into(),
+            idt: s.idt.into(),
+            cr0: s.cr0,
+            cr2: s.cr2,
+            cr3: s.cr3,
+            cr4: s.cr4,
+            cr8: s.cr8,
+            efer: s.efer,
+            apic_base: s.apic_base,
+            interrupt_bitmap: s.interrupt_bitmap,
+        }
+    }
+}
+
+impl From<MshvSpecialRegisters> for SpecialRegisters {
+    fn from(s: MshvSpecialRegisters) -> Self {
+        Self {
+            cs: s.cs.into(),
+            ds: s.ds.into(),
+            es: s.es.into(),
+            fs: s.fs.into(),
+            gs: s.gs.into(),
+            ss: s.ss.into(),
+            tr: s.tr.into(),
+            ldt: s.ldt.into(),
+            gdt: s.gdt.into(),
+            idt: s.idt.into(),
+            cr0: s.cr0,
+            cr2: s.cr2,
+            cr3: s.cr3,
+            cr4: s.cr4,
+            cr8: s.cr8,
+            efer: s.efer,
+            apic_base: s.apic_base,
+            interrupt_bitmap: s.interrupt_bitmap,
+        }
+    }
+}
+
+impl From<CpuIdEntry> for hv_cpuid_entry {
+    fn from(e: CpuIdEntry) -> Self {
+        Self {
+            function: e.function,
+            index: e.index,
+            flags: e.flags,
+            eax: e.eax,
+            ebx: e.ebx,
+            ecx: e.ecx,
+            edx: e.edx,
+            ..Default::default()
+        }
+    }
+}
+
+impl From<hv_cpuid_entry> for CpuIdEntry {
+    fn from(e: hv_cpuid_entry) -> Self {
+        Self {
+            function: e.function,
+            index: e.index,
+            flags: e.flags,
+            eax: e.eax,
+            ebx: e.ebx,
+            ecx: e.ecx,
+            edx: e.edx,
+        }
+    }
+}
+
+impl From<FloatingPointUnit> for FpuState {
+    fn from(s: FloatingPointUnit) -> Self {
+        Self {
+            fpr: s.fpr,
+            fcw: s.fcw,
+            fsw: s.fsw,
+            ftwx: s.ftwx,
+            last_opcode: s.last_opcode,
+            last_ip: s.last_ip,
+            last_dp: s.last_dp,
+            xmm: s.xmm,
+            mxcsr: s.mxcsr,
+        }
+    }
+}
+
+impl From<FpuState> for FloatingPointUnit {
+    fn from(s: FpuState) -> Self {
+        Self {
+            fpr: s.fpr,
+            fcw: s.fcw,
+            fsw: s.fsw,
+            ftwx: s.ftwx,
+            last_opcode: s.last_opcode,
+            last_ip: s.last_ip,
+            last_dp: s.last_dp,
+            xmm: s.xmm,
+            mxcsr: s.mxcsr,
+            ..Default::default()
+        }
+    }
+}
+
+impl From<LapicState> for MshvLapicState {
+    fn from(s: LapicState) -> Self {
+        Self { regs: s.regs }
+    }
+}
+
+impl From<MshvLapicState> for LapicState {
+    fn from(s: MshvLapicState) -> Self {
+        Self { regs: s.regs }
+    }
+}
+
+impl From<msr_entry> for MsrEntry {
+    fn from(e: msr_entry) -> Self {
+        Self {
+            index: e.index,
+            data: e.data,
+        }
+    }
+}
+
+impl From<MsrEntry> for msr_entry {
+    fn from(e: MsrEntry) -> Self {
+        Self {
+            index: e.index,
+            data: e.data,
+            ..Default::default()
+        }
+    }
 }
--- a/hypervisor/src/vm.rs
+++ b/hypervisor/src/vm.rs
@@ -10,23 +10,21 @@

 #[cfg(target_arch = "aarch64")]
 use crate::aarch64::VcpuInit;
-use crate::cpu::Vcpu;
-use crate::device::Device;
-#[cfg(feature = "kvm")]
-use crate::kvm::KvmVmState as VmState;
-#[cfg(feature = "mshv")]
-use crate::mshv::HvState as VmState;
+#[cfg(target_arch = "aarch64")]
+use crate::arch::aarch64::gic::Vgic;
 #[cfg(feature = "tdx")]
-use crate::x86_64::CpuId;
-#[cfg(all(feature = "kvm", target_arch = "x86_64"))]
+use crate::arch::x86::CpuIdEntry;
+use crate::cpu::Vcpu;
+#[cfg(target_arch = "x86_64")]
 use crate::ClockData;
-use crate::CreateDevice;
-use crate::{IoEventAddress, IrqRoutingEntry, MemoryRegion};
-#[cfg(feature = "kvm")]
-use kvm_ioctls::Cap;
+use crate::UserMemoryRegion;
+use crate::{IoEventAddress, IrqRoutingEntry};
+use std::any::Any;
 #[cfg(target_arch = "x86_64")]
 use std::fs::File;
 use std::sync::Arc;
+#[cfg(target_arch = "aarch64")]
+use std::sync::Mutex;
 use thiserror::Error;
 use vmm_sys_util::eventfd::EventFd;

@@ -210,6 +208,11 @@ pub enum HypervisorVmError {
    ///
    #[error("Failed to initialize memory region TDX: {0}")]
    InitMemRegionTdx(#[source] std::io::Error),
+    ///
+    /// Create Vgic error
+    ///
+    #[error("Failed to create Vgic: {0}")]
+    CreateVgic(#[source] anyhow::Error),
 }
 ///
 /// Result type for returning from a function
@@ -254,7 +257,7 @@ pub enum InterruptSourceConfig {
 ///
 /// This crate provides a hypervisor-agnostic interfaces for Vm
 ///
-pub trait Vm: Send + Sync {
+pub trait Vm: Send + Sync + Any {
    #[cfg(target_arch = "x86_64")]
    /// Sets the address of the one-page region in the VM's address space.
    fn set_identity_map_address(&self, address: u64) -> Result<()>;
@@ -269,6 +272,17 @@ pub trait Vm: Send + Sync {
    fn unregister_irqfd(&self, fd: &EventFd, gsi: u32) -> Result<()>;
    /// Creates a new KVM vCPU file descriptor and maps the memory corresponding
    fn create_vcpu(&self, id: u8, vm_ops: Option<Arc<dyn VmOps>>) -> Result<Arc<dyn Vcpu>>;
+    #[cfg(target_arch = "aarch64")]
+    fn create_vgic(
+        &self,
+        vcpu_count: u64,
+        dist_addr: u64,
+        dist_size: u64,
+        redist_size: u64,
+        msi_size: u64,
+        nr_irqs: u32,
+    ) -> Result<Arc<Mutex<dyn Vgic>>>;
+
    /// Registers an event to be signaled whenever a certain address is written to.
    fn register_ioevent(
        &self,
@@ -291,15 +305,13 @@ pub trait Vm: Send + Sync {
        userspace_addr: u64,
        readonly: bool,
        log_dirty_pages: bool,
-    ) -> MemoryRegion;
+    ) -> UserMemoryRegion;
    /// Creates a guest physical memory slot.
-    fn create_user_memory_region(&self, user_memory_region: MemoryRegion) -> Result<()>;
+    fn create_user_memory_region(&self, user_memory_region: UserMemoryRegion) -> Result<()>;
    /// Removes a guest physical memory slot.
-    fn remove_user_memory_region(&self, user_memory_region: MemoryRegion) -> Result<()>;
-    /// Creates an emulated device in the kernel.
-    fn create_device(&self, device: &mut CreateDevice) -> Result<Arc<dyn Device>>;
+    fn remove_user_memory_region(&self, user_memory_region: UserMemoryRegion) -> Result<()>;
    /// Returns the preferred CPU target type which can be emulated by KVM on underlying host.
-    #[cfg(any(target_arch = "arm", target_arch = "aarch64"))]
+    #[cfg(target_arch = "aarch64")]
    fn get_preferred_target(&self, kvi: &mut VcpuInit) -> Result<()>;
    /// Enable split Irq capability
    #[cfg(target_arch = "x86_64")]
@@ -307,20 +319,13 @@ pub trait Vm: Send + Sync {
    #[cfg(target_arch = "x86_64")]
    fn enable_sgx_attribute(&self, file: File) -> Result<()>;
    /// Retrieve guest clock.
-    #[cfg(all(feature = "kvm", target_arch = "x86_64"))]
+    #[cfg(target_arch = "x86_64")]
    fn get_clock(&self) -> Result<ClockData>;
    /// Set guest clock.
-    #[cfg(all(feature = "kvm", target_arch = "x86_64"))]
+    #[cfg(target_arch = "x86_64")]
    fn set_clock(&self, data: &ClockData) -> Result<()>;
-    #[cfg(feature = "kvm")]
-    /// Checks if a particular `Cap` is available.
-    fn check_extension(&self, c: Cap) -> bool;
    /// Create a device that is used for passthrough
-    fn create_passthrough_device(&self) -> Result<Arc<dyn Device>>;
-    /// Get the Vm state. Return VM specific data
-    fn state(&self) -> Result<VmState>;
-    /// Set the VM state
-    fn set_state(&self, state: VmState) -> Result<()>;
+    fn create_passthrough_device(&self) -> Result<vfio_ioctls::VfioDeviceFd>;
    /// Start logging dirty pages
    fn start_dirty_log(&self) -> Result<()>;
    /// Stop logging dirty pages
@@ -329,7 +334,7 @@ pub trait Vm: Send + Sync {
    fn get_dirty_log(&self, slot: u32, base_gpa: u64, memory_size: u64) -> Result<Vec<u64>>;
    #[cfg(feature = "tdx")]
    /// Initalize TDX on this VM
-    fn tdx_init(&self, cpuid: &CpuId, max_vcpus: u32) -> Result<()>;
+    fn tdx_init(&self, cpuid: &[CpuIdEntry], max_vcpus: u32) -> Result<()>;
    #[cfg(feature = "tdx")]
    /// Finalize the configuration of TDX on this VM
    fn tdx_finalize(&self) -> Result<()>;
@@ -342,6 +347,8 @@ pub trait Vm: Send + Sync {
        size: u64,
        measure: bool,
    ) -> Result<()>;
+    /// Downcast to the underlying hypervisor VM type
+    fn as_any(&self) -> &dyn Any;
 }

 pub trait VmOps: Send + Sync {
--- a/net_gen/Cargo.toml
+++ b/net_gen/Cargo.toml
@@ -5,4 +5,4 @@ authors = ["The Chromium OS Authors"]
 edition = "2021"

 [dependencies]
-vmm-sys-util = "0.9.0"
+vmm-sys-util = "0.10.0"
--- a/net_util/Cargo.toml
+++ b/net_util/Cargo.toml
@@ -6,21 +6,23 @@ edition = "2021"

 [dependencies]
 epoll = "4.3.1"
-getrandom = "0.2"
-libc = "0.2.126"
+getrandom = "0.2.7"
+libc = "0.2.131"
 log = "0.4.17"
 net_gen = { path = "../net_gen" }
 rate_limiter = { path = "../rate_limiter" }
-serde = "1.0.137"
+serde = "1.0.143"
+thiserror = "1.0.32"
 versionize = "0.1.6"
 versionize_derive = "0.1.4"
 virtio-bindings = "0.1.0"
-virtio-queue = "0.3.0"
+virtio-queue = "0.5.0"
 vm-memory = { version = "0.8.0", features = ["backend-mmap", "backend-atomic", "backend-bitmap"] }
 vm-virtio = { path = "../vm-virtio" }
-vmm-sys-util = "0.9.0"
+vmm-sys-util = "0.10.0"

 [dev-dependencies]
-lazy_static = "1.4.0"
-pnet = "0.30.0"
-serde_json = "1.0.81"
+once_cell = "1.13.1"
+pnet = "0.31.0"
+pnet_datalink = "0.31.0"
+serde_json = "1.0.83"
--- a/net_util/src/ctrl_queue.rs
+++ b/net_util/src/ctrl_queue.rs
@@ -13,8 +13,8 @@ use virtio_bindings::bindings::virtio_net::{
    VIRTIO_NET_F_GUEST_ECN, VIRTIO_NET_F_GUEST_TSO4, VIRTIO_NET_F_GUEST_TSO6,
    VIRTIO_NET_F_GUEST_UFO, VIRTIO_NET_OK,
 };
-use virtio_queue::Queue;
-use vm_memory::{ByteValued, Bytes, GuestMemoryAtomic, GuestMemoryError};
+use virtio_queue::{Queue, QueueT};
+use vm_memory::{ByteValued, Bytes, GuestMemoryError};
 use vm_virtio::{AccessPlatform, Translatable};

 #[derive(Debug)]
@@ -58,98 +58,93 @@ impl CtrlQueue {

    pub fn process(
        &mut self,
-        queue: &mut Queue<GuestMemoryAtomic<GuestMemoryMmap>>,
+        mem: &GuestMemoryMmap,
+        queue: &mut Queue,
        access_platform: Option<&Arc<dyn AccessPlatform>>,
    ) -> Result<()> {
-        let mut used_desc_heads = Vec::new();
-        loop {
-            for mut desc_chain in queue.iter().map_err(Error::QueueIterator)? {
-                let ctrl_desc = desc_chain.next().ok_or(Error::NoControlHeaderDescriptor)?;
+        while let Some(mut desc_chain) = queue.pop_descriptor_chain(mem) {
+            let ctrl_desc = desc_chain.next().ok_or(Error::NoControlHeaderDescriptor)?;

-                let ctrl_hdr: ControlHeader = desc_chain
-                    .memory()
-                    .read_obj(
-                        ctrl_desc
-                            .addr()
-                            .translate_gva(access_platform, ctrl_desc.len() as usize),
-                    )
-                    .map_err(Error::GuestMemory)?;
-                let data_desc = desc_chain.next().ok_or(Error::NoDataDescriptor)?;
+            let ctrl_hdr: ControlHeader = desc_chain
+                .memory()
+                .read_obj(
+                    ctrl_desc
+                        .addr()
+                        .translate_gva(access_platform, ctrl_desc.len() as usize),
+                )
+                .map_err(Error::GuestMemory)?;
+            let data_desc = desc_chain.next().ok_or(Error::NoDataDescriptor)?;

-                let data_desc_addr = data_desc
-                    .addr()
-                    .translate_gva(access_platform, data_desc.len() as usize);
+            let data_desc_addr = data_desc
+                .addr()
+                .translate_gva(access_platform, data_desc.len() as usize);

-                let status_desc = desc_chain.next().ok_or(Error::NoStatusDescriptor)?;
+            let status_desc = desc_chain.next().ok_or(Error::NoStatusDescriptor)?;

-                let ok = match u32::from(ctrl_hdr.class) {
-                    VIRTIO_NET_CTRL_MQ => {
-                        let queue_pairs = desc_chain
-                            .memory()
-                            .read_obj::<u16>(data_desc_addr)
-                            .map_err(Error::GuestMemory)?;
-                        if u32::from(ctrl_hdr.cmd) != VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET {
-                            warn!("Unsupported command: {}", ctrl_hdr.cmd);
-                            false
-                        } else if (queue_pairs < VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MIN as u16)
-                            || (queue_pairs > VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX as u16)
-                        {
-                            warn!("Number of MQ pairs out of range: {}", queue_pairs);
-                            false
-                        } else {
-                            info!("Number of MQ pairs requested: {}", queue_pairs);
-                            true
-                        }
-                    }
-                    VIRTIO_NET_CTRL_GUEST_OFFLOADS => {
-                        let features = desc_chain
-                            .memory()
-                            .read_obj::<u64>(data_desc_addr)
-                            .map_err(Error::GuestMemory)?;
-                        if u32::from(ctrl_hdr.cmd) != VIRTIO_NET_CTRL_GUEST_OFFLOADS_SET {
-                            warn!("Unsupported command: {}", ctrl_hdr.cmd);
-                            false
-                        } else {
-                            let mut ok = true;
-                            for tap in self.taps.iter_mut() {
-                                info!("Reprogramming tap offload with features: {}", features);
-                                tap.set_offload(virtio_features_to_tap_offload(features))
-                                    .map_err(|e| {
-                                        error!("Error programming tap offload: {:?}", e);
-                                        ok = false
-                                    })
-                                    .ok();
-                            }
-                            ok
-                        }
-                    }
-                    _ => {
-                        warn!("Unsupported command {:?}", ctrl_hdr);
+            let ok = match u32::from(ctrl_hdr.class) {
+                VIRTIO_NET_CTRL_MQ => {
+                    let queue_pairs = desc_chain
+                        .memory()
+                        .read_obj::<u16>(data_desc_addr)
+                        .map_err(Error::GuestMemory)?;
+                    if u32::from(ctrl_hdr.cmd) != VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET {
+                        warn!("Unsupported command: {}", ctrl_hdr.cmd);
                        false
+                    } else if (queue_pairs < VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MIN as u16)
+                        || (queue_pairs > VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX as u16)
+                    {
+                        warn!("Number of MQ pairs out of range: {}", queue_pairs);
+                        false
+                    } else {
+                        info!("Number of MQ pairs requested: {}", queue_pairs);
+                        true
                    }
-                };
+                }
+                VIRTIO_NET_CTRL_GUEST_OFFLOADS => {
+                    let features = desc_chain
+                        .memory()
+                        .read_obj::<u64>(data_desc_addr)
+                        .map_err(Error::GuestMemory)?;
+                    if u32::from(ctrl_hdr.cmd) != VIRTIO_NET_CTRL_GUEST_OFFLOADS_SET {
+                        warn!("Unsupported command: {}", ctrl_hdr.cmd);
+                        false
+                    } else {
+                        let mut ok = true;
+                        for tap in self.taps.iter_mut() {
+                            info!("Reprogramming tap offload with features: {}", features);
+                            tap.set_offload(virtio_features_to_tap_offload(features))
+                                .map_err(|e| {
+                                    error!("Error programming tap offload: {:?}", e);
+                                    ok = false
+                                })
+                                .ok();
+                        }
+                        ok
+                    }
+                }
+                _ => {
+                    warn!("Unsupported command {:?}", ctrl_hdr);
+                    false
+                }
+            };

-                desc_chain
-                    .memory()
-                    .write_obj(
-                        if ok { VIRTIO_NET_OK } else { VIRTIO_NET_ERR } as u8,
-                        status_desc
-                            .addr()
-                            .translate_gva(access_platform, status_desc.len() as usize),
-                    )
-                    .map_err(Error::GuestMemory)?;
-                let len = ctrl_desc.len() + data_desc.len() + status_desc.len();
-                used_desc_heads.push((desc_chain.head_index(), len));
-            }
+            desc_chain
+                .memory()
+                .write_obj(
+                    if ok { VIRTIO_NET_OK } else { VIRTIO_NET_ERR } as u8,
+                    status_desc
+                        .addr()
+                        .translate_gva(access_platform, status_desc.len() as usize),
+                )
+                .map_err(Error::GuestMemory)?;
+            let len = ctrl_desc.len() + data_desc.len() + status_desc.len();

-            for (desc_index, len) in used_desc_heads.iter() {
-                queue
-                    .add_used(*desc_index, *len)
-                    .map_err(Error::QueueAddUsed)?;
-            }
+            queue
+                .add_used(desc_chain.memory(), desc_chain.head_index(), len)
+                .map_err(Error::QueueAddUsed)?;

            if !queue
-                .enable_notification()
+                .enable_notification(mem)
                .map_err(Error::QueueEnableNotification)?
            {
                break;
--- a/net_util/src/lib.rs
+++ b/net_util/src/lib.rs
@@ -5,11 +5,6 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the THIRD-PARTY file.

-// This is only used by the tests module from tap.rs, but we cannot use #[macro_use] unless the
-// reference to lazy_static is declared at the root level of the importing crate.
-#[cfg(test)]
-#[macro_use]
-extern crate lazy_static;
 #[macro_use]
 extern crate log;

@@ -23,6 +18,7 @@ use std::io::Error as IoError;
 use std::os::raw::c_uint;
 use std::os::unix::io::{FromRawFd, RawFd};
 use std::{io, mem, net};
+use thiserror::Error;
 use versionize::{VersionMap, Versionize, VersionizeResult};
 use versionize_derive::Versionize;
 use virtio_bindings::bindings::virtio_net::{
@@ -40,9 +36,9 @@ pub use open_tap::{open_tap, Error as OpenTapError};
 pub use queue_pair::{NetCounters, NetQueuePair, NetQueuePairError, RxVirtio, TxVirtio};
 pub use tap::{Error as TapError, Tap};

-#[derive(Debug)]
+#[derive(Error, Debug)]
 pub enum Error {
-    /// Failed to create a socket.
+    #[error("Failed to create a socket: {0}")]
    CreateSocket(IoError),
 }

--- a/net_util/src/mac.rs
+++ b/net_util/src/mac.rs
@@ -15,7 +15,7 @@ use serde::ser::{Serialize, Serializer};

 pub const MAC_ADDR_LEN: usize = 6;

-#[derive(Clone, Copy, Debug, PartialEq)]
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
 pub struct MacAddr {
    bytes: [u8; MAC_ADDR_LEN],
 }
--- a/net_util/src/open_tap.rs
+++ b/net_util/src/open_tap.rs
@@ -6,30 +6,31 @@ use super::{vnet_hdr_len, MacAddr, Tap, TapError};
 use std::net::Ipv4Addr;
 use std::path::Path;
 use std::{fs, io};
+use thiserror::Error;

-#[derive(Debug)]
+#[derive(Error, Debug)]
 pub enum Error {
-    /// Failed to convert an hexadecimal string into an integer.
+    #[error("Failed to convert an hexadecimal string into an integer: {0}")]
    ConvertHexStringToInt(std::num::ParseIntError),
-    /// Error related to the multiqueue support (no support TAP side).
+    #[error("Error related to the multiqueue support (no support TAP side).")]
    MultiQueueNoTapSupport,
-    /// Error related to the multiqueue support (no support device side).
+    #[error("Error related to the multiqueue support (no support device side).")]
    MultiQueueNoDeviceSupport,
-    /// Failed to read the TAP flags from sysfs.
+    #[error("Failed to read the TAP flags from sysfs: {0}")]
    ReadSysfsTunFlags(io::Error),
-    /// Open tap device failed.
+    #[error("Open tap device failed: {0}")]
    TapOpen(TapError),
-    /// Setting tap IP failed.
+    #[error("Setting tap IP failed: {0}")]
    TapSetIp(TapError),
-    /// Setting tap netmask failed.
+    #[error("Setting tap netmask failed: {0}")]
    TapSetNetmask(TapError),
-    /// Setting MAC address failed
+    #[error("Setting MAC address failed: {0}")]
    TapSetMac(TapError),
-    /// Getting MAC address failed
+    #[error("Getting MAC address failed: {0}")]
    TapGetMac(TapError),
-    /// Setting vnet header size failed.
+    #[error("Setting vnet header size failed: {0}")]
    TapSetVnetHdrSize(TapError),
-    /// Enabling tap interface failed.
+    #[error("Enabling tap interface failed: {0}")]
    TapEnable(TapError),
 }

--- a/net_util/src/queue_pair.rs
+++ b/net_util/src/queue_pair.rs
@@ -10,8 +10,9 @@ use std::num::Wrapping;
 use std::os::unix::io::{AsRawFd, RawFd};
 use std::sync::atomic::{AtomicU64, Ordering};
 use std::sync::Arc;
-use virtio_queue::Queue;
-use vm_memory::{Bytes, GuestMemory, GuestMemoryAtomic};
+use thiserror::Error;
+use virtio_queue::{Queue, QueueOwnedT, QueueT};
+use vm_memory::{Bytes, GuestMemory};
 use vm_virtio::{AccessPlatform, Translatable};

 #[derive(Clone)]
@@ -36,104 +37,95 @@ impl TxVirtio {

    pub fn process_desc_chain(
        &mut self,
+        mem: &GuestMemoryMmap,
        tap: &mut Tap,
-        queue: &mut Queue<GuestMemoryAtomic<GuestMemoryMmap>>,
+        queue: &mut Queue,
        rate_limiter: &mut Option<RateLimiter>,
        access_platform: Option<&Arc<dyn AccessPlatform>>,
    ) -> Result<bool, NetQueuePairError> {
        let mut retry_write = false;
        let mut rate_limit_reached = false;

-        loop {
-            let used_desc_head: (u16, u32);
-            let mut avail_iter = queue
-                .iter()
-                .map_err(NetQueuePairError::QueueIteratorFailed)?;
-
-            if let Some(mut desc_chain) = avail_iter.next() {
-                if rate_limit_reached {
-                    avail_iter.go_to_previous_position();
-                    break;
-                }
-
-                let mut next_desc = desc_chain.next();
-
-                let mut iovecs = Vec::new();
-                while let Some(desc) = next_desc {
-                    let desc_addr = desc
-                        .addr()
-                        .translate_gva(access_platform, desc.len() as usize);
-                    if !desc.is_write_only() && desc.len() > 0 {
-                        let buf = desc_chain
-                            .memory()
-                            .get_slice(desc_addr, desc.len() as usize)
-                            .map_err(NetQueuePairError::GuestMemory)?
-                            .as_ptr();
-                        let iovec = libc::iovec {
-                            iov_base: buf as *mut libc::c_void,
-                            iov_len: desc.len() as libc::size_t,
-                        };
-                        iovecs.push(iovec);
-                    } else {
-                        error!(
-                            "Invalid descriptor chain: address = 0x{:x} length = {} write_only = {}",
-                            desc_addr.0,
-                            desc.len(),
-                            desc.is_write_only()
-                        );
-                        return Err(NetQueuePairError::DescriptorChainInvalid);
-                    }
-                    next_desc = desc_chain.next();
-                }
-
-                let len = if !iovecs.is_empty() {
-                    let result = unsafe {
-                        libc::writev(
-                            tap.as_raw_fd() as libc::c_int,
-                            iovecs.as_ptr() as *const libc::iovec,
-                            iovecs.len() as libc::c_int,
-                        )
-                    };
-
-                    if result < 0 {
-                        let e = std::io::Error::last_os_error();
-
-                        /* EAGAIN */
-                        if e.kind() == std::io::ErrorKind::WouldBlock {
-                            avail_iter.go_to_previous_position();
-                            retry_write = true;
-                            break;
-                        }
-                        error!("net: tx: failed writing to tap: {}", e);
-                        return Err(NetQueuePairError::WriteTap(e));
-                    }
-
-                    self.counter_bytes += Wrapping(result as u64 - vnet_hdr_len() as u64);
-                    self.counter_frames += Wrapping(1);
-
-                    result as u32
-                } else {
-                    0
-                };
-
-                used_desc_head = (desc_chain.head_index(), len);
-
-                // For the sake of simplicity (similar to the RX rate limiting), we always
-                // let the 'last' descriptor chain go-through even if it was over the rate
-                // limit, and simply stop processing oncoming `avail_desc` if any.
-                if let Some(rate_limiter) = rate_limiter {
-                    rate_limit_reached = !rate_limiter.consume(1, TokenType::Ops)
-                        || !rate_limiter.consume(len as u64, TokenType::Bytes);
-                }
-            } else {
+        while let Some(mut desc_chain) = queue.pop_descriptor_chain(mem) {
+            if rate_limit_reached {
+                queue.go_to_previous_position();
                break;
            }

+            let mut next_desc = desc_chain.next();
+
+            let mut iovecs = Vec::new();
+            while let Some(desc) = next_desc {
+                let desc_addr = desc
+                    .addr()
+                    .translate_gva(access_platform, desc.len() as usize);
+                if !desc.is_write_only() && desc.len() > 0 {
+                    let buf = desc_chain
+                        .memory()
+                        .get_slice(desc_addr, desc.len() as usize)
+                        .map_err(NetQueuePairError::GuestMemory)?
+                        .as_ptr();
+                    let iovec = libc::iovec {
+                        iov_base: buf as *mut libc::c_void,
+                        iov_len: desc.len() as libc::size_t,
+                    };
+                    iovecs.push(iovec);
+                } else {
+                    error!(
+                        "Invalid descriptor chain: address = 0x{:x} length = {} write_only = {}",
+                        desc_addr.0,
+                        desc.len(),
+                        desc.is_write_only()
+                    );
+                    return Err(NetQueuePairError::DescriptorChainInvalid);
+                }
+                next_desc = desc_chain.next();
+            }
+
+            let len = if !iovecs.is_empty() {
+                let result = unsafe {
+                    libc::writev(
+                        tap.as_raw_fd() as libc::c_int,
+                        iovecs.as_ptr() as *const libc::iovec,
+                        iovecs.len() as libc::c_int,
+                    )
+                };
+
+                if result < 0 {
+                    let e = std::io::Error::last_os_error();
+
+                    /* EAGAIN */
+                    if e.kind() == std::io::ErrorKind::WouldBlock {
+                        queue.go_to_previous_position();
+                        retry_write = true;
+                        break;
+                    }
+                    error!("net: tx: failed writing to tap: {}", e);
+                    return Err(NetQueuePairError::WriteTap(e));
+                }
+
+                self.counter_bytes += Wrapping(result as u64 - vnet_hdr_len() as u64);
+                self.counter_frames += Wrapping(1);
+
+                result as u32
+            } else {
+                0
+            };
+
+            // For the sake of simplicity (similar to the RX rate limiting), we always
+            // let the 'last' descriptor chain go-through even if it was over the rate
+            // limit, and simply stop processing oncoming `avail_desc` if any.
+            if let Some(rate_limiter) = rate_limiter {
+                rate_limit_reached = !rate_limiter.consume(1, TokenType::Ops)
+                    || !rate_limiter.consume(len as u64, TokenType::Bytes);
+            }
+
            queue
-                .add_used(used_desc_head.0, used_desc_head.1)
+                .add_used(desc_chain.memory(), desc_chain.head_index(), len)
                .map_err(NetQueuePairError::QueueAddUsed)?;
+
            if !queue
-                .enable_notification()
+                .enable_notification(mem)
                .map_err(NetQueuePairError::QueueEnableNotification)?
            {
                break;
@@ -166,125 +158,116 @@ impl RxVirtio {

    pub fn process_desc_chain(
        &mut self,
+        mem: &GuestMemoryMmap,
        tap: &mut Tap,
-        queue: &mut Queue<GuestMemoryAtomic<GuestMemoryMmap>>,
+        queue: &mut Queue,
        rate_limiter: &mut Option<RateLimiter>,
        access_platform: Option<&Arc<dyn AccessPlatform>>,
    ) -> Result<bool, NetQueuePairError> {
        let mut exhausted_descs = true;
        let mut rate_limit_reached = false;

-        loop {
-            let used_desc_head: (u16, u32);
-            let mut avail_iter = queue
-                .iter()
-                .map_err(NetQueuePairError::QueueIteratorFailed)?;
-
-            if let Some(mut desc_chain) = avail_iter.next() {
-                if rate_limit_reached {
-                    exhausted_descs = false;
-                    avail_iter.go_to_previous_position();
-                    break;
-                }
-
-                let desc = desc_chain
-                    .next()
-                    .ok_or(NetQueuePairError::DescriptorChainTooShort)?;
-
-                let num_buffers_addr = desc_chain
-                    .memory()
-                    .checked_offset(
-                        desc.addr()
-                            .translate_gva(access_platform, desc.len() as usize),
-                        10,
-                    )
-                    .unwrap();
-                let mut next_desc = Some(desc);
-
-                let mut iovecs = Vec::new();
-                while let Some(desc) = next_desc {
-                    let desc_addr = desc
-                        .addr()
-                        .translate_gva(access_platform, desc.len() as usize);
-                    if desc.is_write_only() && desc.len() > 0 {
-                        let buf = desc_chain
-                            .memory()
-                            .get_slice(desc_addr, desc.len() as usize)
-                            .map_err(NetQueuePairError::GuestMemory)?
-                            .as_ptr();
-                        let iovec = libc::iovec {
-                            iov_base: buf as *mut libc::c_void,
-                            iov_len: desc.len() as libc::size_t,
-                        };
-                        iovecs.push(iovec);
-                    } else {
-                        error!(
-                            "Invalid descriptor chain: address = 0x{:x} length = {} write_only = {}",
-                            desc_addr.0,
-                            desc.len(),
-                            desc.is_write_only()
-                        );
-                        return Err(NetQueuePairError::DescriptorChainInvalid);
-                    }
-                    next_desc = desc_chain.next();
-                }
-
-                let len = if !iovecs.is_empty() {
-                    let result = unsafe {
-                        libc::readv(
-                            tap.as_raw_fd() as libc::c_int,
-                            iovecs.as_ptr() as *const libc::iovec,
-                            iovecs.len() as libc::c_int,
-                        )
-                    };
-                    if result < 0 {
-                        let e = std::io::Error::last_os_error();
-                        exhausted_descs = false;
-                        avail_iter.go_to_previous_position();
-
-                        /* EAGAIN */
-                        if e.kind() == std::io::ErrorKind::WouldBlock {
-                            break;
-                        }
-
-                        error!("net: rx: failed reading from tap: {}", e);
-                        return Err(NetQueuePairError::ReadTap(e));
-                    }
-
-                    // Write num_buffers to guest memory. We simply write 1 as we
-                    // never spread the frame over more than one descriptor chain.
-                    desc_chain
-                        .memory()
-                        .write_obj(1u16, num_buffers_addr)
-                        .map_err(NetQueuePairError::GuestMemory)?;
-
-                    self.counter_bytes += Wrapping(result as u64 - vnet_hdr_len() as u64);
-                    self.counter_frames += Wrapping(1);
-
-                    result as u32
-                } else {
-                    0
-                };
-
-                used_desc_head = (desc_chain.head_index(), len);
-
-                // For the sake of simplicity (keeping the handling of RX_QUEUE_EVENT and
-                // RX_TAP_EVENT totally asynchronous), we always let the 'last' descriptor
-                // chain go-through even if it was over the rate limit, and simply stop
-                // processing oncoming `avail_desc` if any.
-                if let Some(rate_limiter) = rate_limiter {
-                    rate_limit_reached = !rate_limiter.consume(1, TokenType::Ops)
-                        || !rate_limiter.consume(len as u64, TokenType::Bytes);
-                }
-            } else {
+        while let Some(mut desc_chain) = queue.pop_descriptor_chain(mem) {
+            if rate_limit_reached {
+                exhausted_descs = false;
+                queue.go_to_previous_position();
                break;
            }

+            let desc = desc_chain
+                .next()
+                .ok_or(NetQueuePairError::DescriptorChainTooShort)?;
+
+            let num_buffers_addr = desc_chain
+                .memory()
+                .checked_offset(
+                    desc.addr()
+                        .translate_gva(access_platform, desc.len() as usize),
+                    10,
+                )
+                .unwrap();
+            let mut next_desc = Some(desc);
+
+            let mut iovecs = Vec::new();
+            while let Some(desc) = next_desc {
+                let desc_addr = desc
+                    .addr()
+                    .translate_gva(access_platform, desc.len() as usize);
+                if desc.is_write_only() && desc.len() > 0 {
+                    let buf = desc_chain
+                        .memory()
+                        .get_slice(desc_addr, desc.len() as usize)
+                        .map_err(NetQueuePairError::GuestMemory)?
+                        .as_ptr();
+                    let iovec = libc::iovec {
+                        iov_base: buf as *mut libc::c_void,
+                        iov_len: desc.len() as libc::size_t,
+                    };
+                    iovecs.push(iovec);
+                } else {
+                    error!(
+                        "Invalid descriptor chain: address = 0x{:x} length = {} write_only = {}",
+                        desc_addr.0,
+                        desc.len(),
+                        desc.is_write_only()
+                    );
+                    return Err(NetQueuePairError::DescriptorChainInvalid);
+                }
+                next_desc = desc_chain.next();
+            }
+
+            let len = if !iovecs.is_empty() {
+                let result = unsafe {
+                    libc::readv(
+                        tap.as_raw_fd() as libc::c_int,
+                        iovecs.as_ptr() as *const libc::iovec,
+                        iovecs.len() as libc::c_int,
+                    )
+                };
+                if result < 0 {
+                    let e = std::io::Error::last_os_error();
+                    exhausted_descs = false;
+                    queue.go_to_previous_position();
+
+                    /* EAGAIN */
+                    if e.kind() == std::io::ErrorKind::WouldBlock {
+                        break;
+                    }
+
+                    error!("net: rx: failed reading from tap: {}", e);
+                    return Err(NetQueuePairError::ReadTap(e));
+                }
+
+                // Write num_buffers to guest memory. We simply write 1 as we
+                // never spread the frame over more than one descriptor chain.
+                desc_chain
+                    .memory()
+                    .write_obj(1u16, num_buffers_addr)
+                    .map_err(NetQueuePairError::GuestMemory)?;
+
+                self.counter_bytes += Wrapping(result as u64 - vnet_hdr_len() as u64);
+                self.counter_frames += Wrapping(1);
+
+                result as u32
+            } else {
+                0
+            };
+
+            // For the sake of simplicity (keeping the handling of RX_QUEUE_EVENT and
+            // RX_TAP_EVENT totally asynchronous), we always let the 'last' descriptor
+            // chain go-through even if it was over the rate limit, and simply stop
+            // processing oncoming `avail_desc` if any.
+            if let Some(rate_limiter) = rate_limiter {
+                rate_limit_reached = !rate_limiter.consume(1, TokenType::Ops)
+                    || !rate_limiter.consume(len as u64, TokenType::Bytes);
+            }
+
            queue
-                .add_used(used_desc_head.0, used_desc_head.1)
+                .add_used(desc_chain.memory(), desc_chain.head_index(), len)
                .map_err(NetQueuePairError::QueueAddUsed)?;
+
            if !queue
-                .enable_notification()
+                .enable_notification(mem)
                .map_err(NetQueuePairError::QueueEnableNotification)?
            {
                break;
@@ -303,31 +286,31 @@ pub struct NetCounters {
    pub rx_frames: Arc<AtomicU64>,
 }

-#[derive(Debug)]
+#[derive(Error, Debug)]
 pub enum NetQueuePairError {
-    /// No memory configured
+    #[error("No memory configured.")]
    NoMemoryConfigured,
-    /// Error registering listener
+    #[error("Error registering listener: {0}")]
    RegisterListener(io::Error),
-    /// Error unregistering listener
+    #[error("Error unregistering listener: {0}")]
    UnregisterListener(io::Error),
-    /// Error writing to the TAP device
+    #[error("Error writing to the TAP device: {0}")]
    WriteTap(io::Error),
-    /// Error reading from the TAP device
+    #[error("Error reading from the TAP device: {0}")]
    ReadTap(io::Error),
-    /// Error related to guest memory
+    #[error("Error related to guest memory: {0}")]
    GuestMemory(vm_memory::GuestMemoryError),
-    /// Returned an error while iterating through the queue
+    #[error("Returned an error while iterating through the queue: {0}")]
    QueueIteratorFailed(virtio_queue::Error),
-    /// Descriptor chain is too short
+    #[error("Descriptor chain is too short.")]
    DescriptorChainTooShort,
-    /// Descriptor chain does not contain valid descriptors
+    #[error("Descriptor chain does not contain valid descriptors.")]
    DescriptorChainInvalid,
-    /// Failed to determine if queue needed notification
+    #[error("Failed to determine if queue needed notification: {0}")]
    QueueNeedsNotification(virtio_queue::Error),
-    /// Failed to enable notification on the queue
+    #[error("Failed to enable notification on the queue: {0}")]
    QueueEnableNotification(virtio_queue::Error),
-    /// Failed to add used index to the queue
+    #[error("Failed to add used index to the queue: {0}")]
    QueueAddUsed(virtio_queue::Error),
 }

@@ -355,9 +338,11 @@ pub struct NetQueuePair {
 impl NetQueuePair {
    pub fn process_tx(
        &mut self,
-        queue: &mut Queue<GuestMemoryAtomic<GuestMemoryMmap>>,
+        mem: &GuestMemoryMmap,
+        queue: &mut Queue,
    ) -> Result<bool, NetQueuePairError> {
        let tx_tap_retry = self.tx.process_desc_chain(
+            mem,
            &mut self.tap,
            queue,
            &mut self.tx_rate_limiter,
@@ -397,15 +382,17 @@ impl NetQueuePair {
        self.tx.counter_frames = Wrapping(0);

        queue
-            .needs_notification()
+            .needs_notification(mem)
            .map_err(NetQueuePairError::QueueNeedsNotification)
    }

    pub fn process_rx(
        &mut self,
-        queue: &mut Queue<GuestMemoryAtomic<GuestMemoryMmap>>,
+        mem: &GuestMemoryMmap,
+        queue: &mut Queue,
    ) -> Result<bool, NetQueuePairError> {
        self.rx_desc_avail = !self.rx.process_desc_chain(
+            mem,
            &mut self.tap,
            queue,
            &mut self.rx_rate_limiter,
@@ -440,7 +427,7 @@ impl NetQueuePair {
        self.rx.counter_frames = Wrapping(0);

        queue
-            .needs_notification()
+            .needs_notification(mem)
            .map_err(NetQueuePairError::QueueNeedsNotification)
    }
 }
--- a/net_util/src/tap.rs
+++ b/net_util/src/tap.rs
@@ -15,24 +15,26 @@ use std::io::{Error as IoError, Read, Result as IoResult, Write};
 use std::net;
 use std::os::raw::*;
 use std::os::unix::io::{AsRawFd, FromRawFd, RawFd};
+use thiserror::Error;
 use vmm_sys_util::ioctl::{ioctl_with_mut_ref, ioctl_with_ref, ioctl_with_val};

-#[derive(Debug)]
+#[derive(Error, Debug)]
 pub enum Error {
-    /// Couldn't open /dev/net/tun.
+    #[error("Couldn't open /dev/net/tun: {0}")]
    OpenTun(IoError),
-    /// Unable to configure tap interface.
+    #[error("Unable to configure tap interface: {0}")]
    ConfigureTap(IoError),
-    /// Unable to retrieve features.
+    #[error("Unable to retrieve features: {0}")]
    GetFeatures(IoError),
-    /// Missing multiqueue support in the kernel.
+    #[error("Missing multiqueue support in the kernel.")]
    MultiQueueKernelSupport,
-    /// ioctl failed.
+    #[error("ioctl failed: {0}")]
    IoctlError(IoError),
-    /// Failed to create a socket.
+    #[error("Failed to create a socket: {0}")]
    NetUtil(NetUtilError),
+    #[error("Invalid interface name.")]
    InvalidIfname,
-    /// Error parsing MAC data
+    #[error("Error parsing MAC data: {0}")]
    MacParsing(IoError),
 }

@@ -321,15 +323,13 @@ impl Tap {

        // If TAP device is already up don't try and enable it
        let ifru_flags = unsafe { ifreq.ifr_ifru.ifru_flags };
-        if ifru_flags
-            & (net_gen::net_device_flags_IFF_UP | net_gen::net_device_flags_IFF_RUNNING) as i16
-            == (net_gen::net_device_flags_IFF_UP | net_gen::net_device_flags_IFF_RUNNING) as i16
+        if ifru_flags & net_gen::net_device_flags_IFF_UP as i16
+            == net_gen::net_device_flags_IFF_UP as i16
        {
            return Ok(());
        }

-        ifreq.ifr_ifru.ifru_flags =
-            (net_gen::net_device_flags_IFF_UP | net_gen::net_device_flags_IFF_RUNNING) as i16;
+        ifreq.ifr_ifru.ifru_flags = net_gen::net_device_flags_IFF_UP as i16;

        // ioctl is safe. Called with a valid sock fd, and we check the return.
        let ret =
@@ -395,38 +395,35 @@ impl AsRawFd for Tap {

 #[cfg(test)]
 mod tests {
-    extern crate pnet;
-
    use std::net::Ipv4Addr;
    use std::str;
    use std::sync::{mpsc, Mutex};
    use std::thread;
    use std::time::Duration;

-    use self::pnet::datalink::Channel::Ethernet;
-    use self::pnet::datalink::{self, DataLinkReceiver, DataLinkSender, NetworkInterface};
-    use self::pnet::packet::ethernet::{EtherTypes, EthernetPacket, MutableEthernetPacket};
-    use self::pnet::packet::ip::IpNextHeaderProtocols;
-    use self::pnet::packet::ipv4::{Ipv4Packet, MutableIpv4Packet};
-    use self::pnet::packet::udp::{MutableUdpPacket, UdpPacket};
-    use self::pnet::packet::{MutablePacket, Packet};
-    use self::pnet::util::MacAddr;
+    use once_cell::sync::Lazy;
+
+    use pnet::packet::ethernet::{EtherTypes, EthernetPacket, MutableEthernetPacket};
+    use pnet::packet::ip::IpNextHeaderProtocols;
+    use pnet::packet::ipv4::{Ipv4Packet, MutableIpv4Packet};
+    use pnet::packet::udp::{MutableUdpPacket, UdpPacket};
+    use pnet::packet::{MutablePacket, Packet};
+    use pnet::util::MacAddr;
+    use pnet_datalink::Channel::Ethernet;
+    use pnet_datalink::{DataLinkReceiver, DataLinkSender, NetworkInterface};

    use super::*;

    static DATA_STRING: &str = "test for tap";
    static SUBNET_MASK: &str = "255.255.255.0";

-    // We needed to have a mutex as a global variable, so we used the crate that provides the
-    // lazy_static! macro for testing. The main potential problem, caused by tests being run in
-    // parallel by cargo, is creating different TAPs and trying to associate the same address,
-    // so we hide the IP address &str behind this mutex, more as a convention to remember to lock
-    // it at the very beginning of each function susceptible to this issue. Another variant is
-    // to use a different IP address per function, but we must remember to pick an unique one
-    // each time.
-    lazy_static! {
-        static ref TAP_IP_LOCK: Mutex<&'static str> = Mutex::new("192.168.241.1");
-    }
+    // We needed to have a mutex as a global variable, so we used once_cell for testing. The main
+    // potential problem, caused by tests being run in parallel by cargo, is creating different
+    // TAPs and trying to associate the same address, so we hide the IP address &str behind this
+    // mutex, more as a convention to remember to lock it at the very beginning of each function
+    // susceptible to this issue. Another variant is to use a different IP address per function,
+    // but we must remember to pick an unique one each time.
+    static TAP_IP_LOCK: Lazy<Mutex<&'static str>> = Lazy::new(|| Mutex::new("192.168.241.1"));

    // Describes the outcomes we are currently interested in when parsing a packet (we use
    // an UDP packet for testing).
@@ -543,10 +540,10 @@ mod tests {
        let interface_name_matches = |iface: &NetworkInterface| iface.name == ifname;

        // Find the network interface with the provided name.
-        let interfaces = datalink::interfaces();
+        let interfaces = pnet_datalink::interfaces();
        let interface = interfaces.into_iter().find(interface_name_matches).unwrap();

-        if let Ok(Ethernet(tx, rx)) = datalink::channel(&interface, Default::default()) {
+        if let Ok(Ethernet(tx, rx)) = pnet_datalink::channel(&interface, Default::default()) {
            (interface.mac.unwrap(), tx, rx)
        } else {
            panic!("datalink channel error or unhandled channel type");
--- a/option_parser/src/lib.rs
+++ b/option_parser/src/lib.rs
@@ -31,35 +31,49 @@ impl fmt::Display for OptionParserError {
            OptionParserError::UnknownOption(s) => write!(f, "unknown option: {}", s),
            OptionParserError::InvalidSyntax(s) => write!(f, "invalid syntax:{}", s),
            OptionParserError::Conversion(field, value) => {
-                write!(f, "unable to parse {} for {}", value, field)
+                write!(f, "unable to convert {} for {}", value, field)
            }
        }
    }
 }
 type OptionParserResult<T> = std::result::Result<T, OptionParserError>;

-fn split_commas_outside_brackets(s: &str) -> OptionParserResult<Vec<String>> {
+fn split_commas(s: &str) -> OptionParserResult<Vec<String>> {
    let mut list: Vec<String> = Vec::new();
-    let mut opened_brackets: usize = 0;
-    for element in s.trim().split(',') {
-        if opened_brackets > 0 {
-            if let Some(last) = list.last_mut() {
-                *last = format!("{},{}", last, element);
-            } else {
-                return Err(OptionParserError::InvalidSyntax(s.to_owned()));
-            }
-        } else {
-            list.push(element.to_string());
-        }
+    let mut opened_brackets = 0;
+    let mut in_quotes = false;
+    let mut current = String::new();

-        opened_brackets += element.matches('[').count();
-        let closing_brackets = element.matches(']').count();
-        if closing_brackets > opened_brackets {
-            return Err(OptionParserError::InvalidSyntax(s.to_owned()));
-        } else {
-            opened_brackets -= closing_brackets;
+    for c in s.trim().chars() {
+        match c {
+            '[' => {
+                opened_brackets += 1;
+                current.push('[');
+            }
+            ']' => {
+                opened_brackets -= 1;
+                if opened_brackets < 0 {
+                    return Err(OptionParserError::InvalidSyntax(s.to_owned()));
+                }
+                current.push(']');
+            }
+            '"' => in_quotes = !in_quotes,
+            ',' => {
+                if opened_brackets > 0 || in_quotes {
+                    current.push(',')
+                } else {
+                    list.push(current);
+                    current = String::new();
+                }
+            }
+            c => current.push(c),
        }
    }
+    list.push(current);
+
+    if opened_brackets != 0 || in_quotes {
+        return Err(OptionParserError::InvalidSyntax(s.to_owned()));
+    }

    Ok(list)
 }
@@ -76,7 +90,7 @@ impl OptionParser {
            return Ok(());
        }

-        for option in split_commas_outside_brackets(input)?.iter() {
+        for option in split_commas(input)?.iter() {
            let parts: Vec<&str> = option.splitn(2, '=').collect();

            match self.options.get_mut(parts[0]) {
@@ -291,9 +305,13 @@ impl<S: FromStr, T: TupleValue> FromStr for Tuple<S, T> {
    fn from_str(s: &str) -> std::result::Result<Self, Self::Err> {
        let mut list: Vec<(S, T)> = Vec::new();

-        let tuples_list =
-            split_commas_outside_brackets(s.trim().trim_matches(|c| c == '[' || c == ']'))
-                .map_err(TupleError::SplitOutsideBrackets)?;
+        let body = s
+            .trim()
+            .strip_prefix('[')
+            .and_then(|s| s.strip_suffix(']'))
+            .ok_or_else(|| TupleError::InvalidValue(s.to_string()))?;
+
+        let tuples_list = split_commas(body).map_err(TupleError::SplitOutsideBrackets)?;
        for tuple in tuples_list.iter() {
            let items: Vec<&str> = tuple.split('@').collect();

@@ -334,3 +352,58 @@ impl FromStr for StringList {
        Ok(StringList(string_list))
    }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_option_parser() {
+        let mut parser = OptionParser::new();
+        parser
+            .add("size")
+            .add("mergeable")
+            .add("hotplug_method")
+            .add("hotplug_size")
+            .add("topology")
+            .add("cmdline");
+
+        assert!(parser.parse("size=128M,hanging_param").is_err());
+        assert!(parser.parse("size=128M,too_many_equals=foo=bar").is_err());
+        assert!(parser.parse("size=128M,file=/dev/shm").is_err());
+
+        assert!(parser.parse("size=128M").is_ok());
+        assert_eq!(parser.get("size"), Some("128M".to_owned()));
+        assert!(!parser.is_set("mergeable"));
+        assert!(parser.is_set("size"));
+
+        assert!(parser.parse("size=128M,mergeable=on").is_ok());
+        assert_eq!(parser.get("size"), Some("128M".to_owned()));
+        assert_eq!(parser.get("mergeable"), Some("on".to_owned()));
+
+        assert!(parser
+            .parse("size=128M,mergeable=on,topology=[1,2]")
+            .is_ok());
+        assert_eq!(parser.get("size"), Some("128M".to_owned()));
+        assert_eq!(parser.get("mergeable"), Some("on".to_owned()));
+        assert_eq!(parser.get("topology"), Some("[1,2]".to_owned()));
+
+        assert!(parser
+            .parse("size=128M,mergeable=on,topology=[[1,2],[3,4]]")
+            .is_ok());
+        assert_eq!(parser.get("size"), Some("128M".to_owned()));
+        assert_eq!(parser.get("mergeable"), Some("on".to_owned()));
+        assert_eq!(parser.get("topology"), Some("[[1,2],[3,4]]".to_owned()));
+
+        assert!(parser.parse("topology=[").is_err());
+        assert!(parser.parse("topology=[[[]]]]").is_err());
+
+        assert!(parser.parse("cmdline=\"console=ttyS0,9600n8\"").is_ok());
+        assert_eq!(
+            parser.get("cmdline"),
+            Some("console=ttyS0,9600n8".to_owned())
+        );
+        assert!(parser.parse("cmdline=\"").is_err());
+        assert!(parser.parse("cmdline=\"\"\"").is_err());
+    }
+}
--- a/pci/Cargo.toml
+++ b/pci/Cargo.toml
@@ -10,16 +10,16 @@ kvm = ["vfio-ioctls/kvm"]
 mshv = ["vfio-ioctls/mshv"]

 [dependencies]
-anyhow = "1.0.57"
+anyhow = "1.0.62"
 byteorder = "1.4.3"
 hypervisor = { path = "../hypervisor" }
 vfio-ioctls = { git = "https://github.com/rust-vmm/vfio", branch = "main", default-features = false }
 vfio_user = { path = "../vfio_user" }
-vmm-sys-util = "0.9.0"
-libc = "0.2.126"
+vmm-sys-util = "0.10.0"
+libc = "0.2.131"
 log = "0.4.17"
-serde = { version="1.0.137", features=["derive"] }
-thiserror = "1.0.31"
+serde = { version = "1.0.143", features = ["derive"] }
+thiserror = "1.0.32"
 versionize = "0.1.6"
 versionize_derive = "0.1.4"
 vm-allocator = { path = "../vm-allocator" }
--- a/pci/src/configuration.rs
+++ b/pci/src/configuration.rs
@@ -187,10 +187,10 @@ pub trait PciProgrammingInterface {
 }

 /// Types of PCI capabilities.
-#[derive(PartialEq, Copy, Clone)]
+#[derive(PartialEq, Eq, Copy, Clone)]
 #[allow(dead_code)]
 #[allow(non_camel_case_types)]
-#[repr(C)]
+#[repr(u8)]
 pub enum PciCapabilityId {
    ListId = 0,
    PowerManagement = 0x01,
@@ -244,6 +244,118 @@ impl From<u8> for PciCapabilityId {
    }
 }

+/// Types of PCI Express capabilities.
+#[derive(PartialEq, Eq, Copy, Clone, Debug)]
+#[allow(dead_code)]
+#[repr(u16)]
+pub enum PciExpressCapabilityId {
+    NullCapability = 0x0000,
+    AdvancedErrorReporting = 0x0001,
+    VirtualChannelMultiFunctionVirtualChannelNotPresent = 0x0002,
+    DeviceSerialNumber = 0x0003,
+    PowerBudgeting = 0x0004,
+    RootComplexLinkDeclaration = 0x0005,
+    RootComplexInternalLinkControl = 0x0006,
+    RootComplexEventCollectorEndpointAssociation = 0x0007,
+    MultiFunctionVirtualChannel = 0x0008,
+    VirtualChannelMultiFunctionVirtualChannelPresent = 0x0009,
+    RootComplexRegisterBlock = 0x000a,
+    VendorSpecificExtendedCapability = 0x000b,
+    ConfigurationAccessCorrelation = 0x000c,
+    AccessControlServices = 0x000d,
+    AlternativeRoutingIdentificationIntepretation = 0x000e,
+    AddressTranslationServices = 0x000f,
+    SingleRootIoVirtualization = 0x0010,
+    DeprecatedMultiRootIoVirtualzation = 0x0011,
+    Multicast = 0x0012,
+    PageRequestInterface = 0x0013,
+    ReservedForAmd = 0x0014,
+    ResizeableBar = 0x0015,
+    DynamicPowerAllocation = 0x0016,
+    ThpRequester = 0x0017,
+    LatencyToleranceReporting = 0x0018,
+    SecondaryPciExpress = 0x0019,
+    ProtocolMultiplexing = 0x001a,
+    ProcessAddressSpaceId = 0x001b,
+    LnRequestor = 0x001c,
+    DownstreamPortContainment = 0x001d,
+    L1PmSubstates = 0x001e,
+    PrecisionTimeMeasurement = 0x001f,
+    PciExpressOverMphy = 0x0020,
+    FRSQueueing = 0x0021,
+    ReadinessTimeReporting = 0x0022,
+    DesignatedVendorSpecificExtendedCapability = 0x0023,
+    VfResizeableBar = 0x0024,
+    DataLinkFeature = 0x0025,
+    PhysicalLayerSixteenGts = 0x0026,
+    LaneMargeningAtTheReceiver = 0x0027,
+    HierarchyId = 0x0028,
+    NativePcieEnclosureManagement = 0x0029,
+    PhysicalLayerThirtyTwoGts = 0x002a,
+    AlternateProtocol = 0x002b,
+    SystemFirmwareIntermediary = 0x002c,
+    ShadowFunctions = 0x002d,
+    DataObjectExchange = 0x002e,
+    Reserved = 0x002f,
+    ExtendedCapabilitiesAbsence = 0xffff,
+}
+
+impl From<u16> for PciExpressCapabilityId {
+    fn from(c: u16) -> Self {
+        match c {
+            0x0000 => PciExpressCapabilityId::NullCapability,
+            0x0001 => PciExpressCapabilityId::AdvancedErrorReporting,
+            0x0002 => PciExpressCapabilityId::VirtualChannelMultiFunctionVirtualChannelNotPresent,
+            0x0003 => PciExpressCapabilityId::DeviceSerialNumber,
+            0x0004 => PciExpressCapabilityId::PowerBudgeting,
+            0x0005 => PciExpressCapabilityId::RootComplexLinkDeclaration,
+            0x0006 => PciExpressCapabilityId::RootComplexInternalLinkControl,
+            0x0007 => PciExpressCapabilityId::RootComplexEventCollectorEndpointAssociation,
+            0x0008 => PciExpressCapabilityId::MultiFunctionVirtualChannel,
+            0x0009 => PciExpressCapabilityId::VirtualChannelMultiFunctionVirtualChannelPresent,
+            0x000a => PciExpressCapabilityId::RootComplexRegisterBlock,
+            0x000b => PciExpressCapabilityId::VendorSpecificExtendedCapability,
+            0x000c => PciExpressCapabilityId::ConfigurationAccessCorrelation,
+            0x000d => PciExpressCapabilityId::AccessControlServices,
+            0x000e => PciExpressCapabilityId::AlternativeRoutingIdentificationIntepretation,
+            0x000f => PciExpressCapabilityId::AddressTranslationServices,
+            0x0010 => PciExpressCapabilityId::SingleRootIoVirtualization,
+            0x0011 => PciExpressCapabilityId::DeprecatedMultiRootIoVirtualzation,
+            0x0012 => PciExpressCapabilityId::Multicast,
+            0x0013 => PciExpressCapabilityId::PageRequestInterface,
+            0x0014 => PciExpressCapabilityId::ReservedForAmd,
+            0x0015 => PciExpressCapabilityId::ResizeableBar,
+            0x0016 => PciExpressCapabilityId::DynamicPowerAllocation,
+            0x0017 => PciExpressCapabilityId::ThpRequester,
+            0x0018 => PciExpressCapabilityId::LatencyToleranceReporting,
+            0x0019 => PciExpressCapabilityId::SecondaryPciExpress,
+            0x001a => PciExpressCapabilityId::ProtocolMultiplexing,
+            0x001b => PciExpressCapabilityId::ProcessAddressSpaceId,
+            0x001c => PciExpressCapabilityId::LnRequestor,
+            0x001d => PciExpressCapabilityId::DownstreamPortContainment,
+            0x001e => PciExpressCapabilityId::L1PmSubstates,
+            0x001f => PciExpressCapabilityId::PrecisionTimeMeasurement,
+            0x0020 => PciExpressCapabilityId::PciExpressOverMphy,
+            0x0021 => PciExpressCapabilityId::FRSQueueing,
+            0x0022 => PciExpressCapabilityId::ReadinessTimeReporting,
+            0x0023 => PciExpressCapabilityId::DesignatedVendorSpecificExtendedCapability,
+            0x0024 => PciExpressCapabilityId::VfResizeableBar,
+            0x0025 => PciExpressCapabilityId::DataLinkFeature,
+            0x0026 => PciExpressCapabilityId::PhysicalLayerSixteenGts,
+            0x0027 => PciExpressCapabilityId::LaneMargeningAtTheReceiver,
+            0x0028 => PciExpressCapabilityId::HierarchyId,
+            0x0029 => PciExpressCapabilityId::NativePcieEnclosureManagement,
+            0x002a => PciExpressCapabilityId::PhysicalLayerThirtyTwoGts,
+            0x002b => PciExpressCapabilityId::AlternateProtocol,
+            0x002c => PciExpressCapabilityId::SystemFirmwareIntermediary,
+            0x002d => PciExpressCapabilityId::ShadowFunctions,
+            0x002e => PciExpressCapabilityId::DataObjectExchange,
+            0xffff => PciExpressCapabilityId::ExtendedCapabilitiesAbsence,
+            _ => PciExpressCapabilityId::Reserved,
+        }
+    }
+}
+
 /// A PCI capability list. Devices can optionally specify capabilities in their configuration space.
 pub trait PciCapability {
    fn bytes(&self) -> &[u8];
@@ -321,7 +433,7 @@ pub struct PciConfiguration {
 }

 /// See pci_regs.h in kernel
-#[derive(Copy, Clone, PartialEq, Versionize, Debug)]
+#[derive(Copy, Clone, PartialEq, Eq, Versionize, Debug)]
 pub enum PciBarRegionType {
    Memory32BitRegion = 0,
    IoRegion = 0x01,
@@ -999,6 +1111,12 @@ impl PciBarConfiguration {
        self
    }

+    #[must_use]
+    pub fn set_prefetchable(mut self, prefetchable: PciBarPrefetchable) -> Self {
+        self.prefetchable = prefetchable;
+        self
+    }
+
    pub fn idx(&self) -> usize {
        self.idx
    }
--- a/pci/src/lib.rs
+++ b/pci/src/lib.rs
@@ -17,7 +17,7 @@ mod vfio_user;
 pub use self::bus::{PciBus, PciConfigIo, PciConfigMmio, PciRoot, PciRootError};
 pub use self::configuration::{
    PciBarConfiguration, PciBarPrefetchable, PciBarRegionType, PciCapability, PciCapabilityId,
-    PciClassCode, PciConfiguration, PciHeaderType, PciMassStorageSubclass,
+    PciClassCode, PciConfiguration, PciExpressCapabilityId, PciHeaderType, PciMassStorageSubclass,
    PciNetworkControllerSubclass, PciProgrammingInterface, PciSerialBusSubClass, PciSubclass,
 };
 pub use self::device::{
@@ -52,7 +52,7 @@ pub const PCI_CONFIG_IO_PORT: u64 = 0xcf8;
 #[cfg(target_arch = "x86_64")]
 pub const PCI_CONFIG_IO_PORT_SIZE: u64 = 0x8;

-#[derive(Clone, Copy, PartialEq, PartialOrd)]
+#[derive(Clone, Copy, PartialEq, Eq, PartialOrd)]
 pub struct PciBdf(u32);

 struct PciBdfVisitor;
--- a/pci/src/msi.rs
+++ b/pci/src/msi.rs
@@ -179,7 +179,7 @@ struct MsiConfigState {
 impl VersionMapped for MsiConfigState {}

 pub struct MsiConfig {
-    cap: MsiCap,
+    pub cap: MsiCap,
    interrupt_source_group: Arc<dyn InterruptSourceGroup>,
 }

--- a/pci/src/msix.rs
+++ b/pci/src/msix.rs
@@ -441,7 +441,7 @@ impl Snapshottable for MsixConfig {

 #[allow(dead_code)]
 #[repr(packed)]
-#[derive(Clone, Copy, Default)]
+#[derive(Clone, Copy, Default, Versionize)]
 pub struct MsixCap {
    // Message Control Register
    //   10-0:  MSI-X Table size
--- a/pci/src/vfio.rs
+++ b/pci/src/vfio.rs
@@ -4,19 +4,23 @@
 //

 use crate::{
-    msi_num_enabled_vectors, BarReprogrammingParams, MsiConfig, MsixCap, MsixConfig,
-    PciBarConfiguration, PciBarRegionType, PciBdf, PciCapabilityId, PciClassCode, PciConfiguration,
-    PciDevice, PciDeviceError, PciHeaderType, PciSubclass, MSIX_TABLE_ENTRY_SIZE,
+    msi_num_enabled_vectors, BarReprogrammingParams, MsiCap, MsiConfig, MsixCap, MsixConfig,
+    PciBarConfiguration, PciBarPrefetchable, PciBarRegionType, PciBdf, PciCapabilityId,
+    PciClassCode, PciConfiguration, PciDevice, PciDeviceError, PciExpressCapabilityId,
+    PciHeaderType, PciSubclass, MSIX_TABLE_ENTRY_SIZE,
 };
+use anyhow::anyhow;
 use byteorder::{ByteOrder, LittleEndian};
 use hypervisor::HypervisorVmError;
 use std::any::Any;
-use std::collections::BTreeMap;
+use std::collections::{BTreeMap, HashMap};
 use std::io;
 use std::os::unix::io::AsRawFd;
 use std::ptr::null_mut;
 use std::sync::{Arc, Barrier, Mutex};
 use thiserror::Error;
+use versionize::{VersionMap, Versionize, VersionizeResult};
+use versionize_derive::Versionize;
 use vfio_bindings::bindings::vfio::*;
 use vfio_ioctls::{
    VfioContainer, VfioDevice, VfioIrq, VfioRegionInfoCap, VfioRegionSparseMmapArea,
@@ -27,6 +31,9 @@ use vm_device::interrupt::{
 };
 use vm_device::{BusDevice, Resource};
 use vm_memory::{Address, GuestAddress, GuestUsize};
+use vm_migration::{
+    Migratable, MigratableError, Pausable, Snapshot, Snapshottable, Transportable, VersionMapped,
+};
 use vmm_sys_util::eventfd::EventFd;

 #[derive(Debug, Error)]
@@ -71,11 +78,22 @@ enum InterruptUpdateAction {
    DisableMsix,
 }

+#[derive(Versionize)]
+struct IntxState {
+    enabled: bool,
+}
+
 pub(crate) struct VfioIntx {
    interrupt_source_group: Arc<dyn InterruptSourceGroup>,
    enabled: bool,
 }

+#[derive(Versionize)]
+struct MsiState {
+    cap: MsiCap,
+    cap_offset: u32,
+}
+
 pub(crate) struct VfioMsi {
    pub(crate) cfg: MsiConfig,
    cap_offset: u32,
@@ -102,6 +120,13 @@ impl VfioMsi {
    }
 }

+#[derive(Versionize)]
+struct MsixState {
+    cap: MsixCap,
+    cap_offset: u32,
+    bdf: u32,
+}
+
 pub(crate) struct VfioMsix {
    pub(crate) bar: MsixConfig,
    cap: MsixCap,
@@ -356,6 +381,20 @@ impl Vfio for VfioDeviceWrapper {
    }
 }

+#[derive(Versionize)]
+struct VfioCommonState {
+    intx_state: Option<IntxState>,
+    msi_state: Option<MsiState>,
+    msix_state: Option<MsixState>,
+}
+
+impl VersionMapped for VfioCommonState {}
+
+pub(crate) struct ConfigPatch {
+    mask: u32,
+    patch: u32,
+}
+
 pub(crate) struct VfioCommon {
    pub(crate) configuration: PciConfiguration,
    pub(crate) mmio_regions: Vec<MmioRegion>,
@@ -363,6 +402,7 @@ pub(crate) struct VfioCommon {
    pub(crate) msi_interrupt_manager: Arc<dyn InterruptManager<GroupConfig = MsiIrqGroupConfig>>,
    pub(crate) legacy_interrupt_group: Option<Arc<dyn InterruptSourceGroup>>,
    pub(crate) vfio_wrapper: Arc<dyn Vfio>,
+    pub(crate) patches: HashMap<usize, ConfigPatch>,
 }

 impl VfioCommon {
@@ -382,6 +422,7 @@ impl VfioCommon {
        while bar_id < VFIO_PCI_CONFIG_REGION_INDEX {
            let mut region_size: u64 = 0;
            let mut region_type = PciBarRegionType::Memory32BitRegion;
+            let mut prefetchable = PciBarPrefetchable::NotPrefetchable;
            let mut flags: u32 = 0;

            let mut restored_bar_addr = None;
@@ -434,6 +475,13 @@ impl VfioCommon {
                    false
                };

+                if matches!(
+                    flags & PCI_CONFIG_BAR_PREFETCHABLE,
+                    PCI_CONFIG_BAR_PREFETCHABLE
+                ) {
+                    prefetchable = PciBarPrefetchable::Prefetchable
+                };
+
                // To get size write all 1s
                self.vfio_wrapper
                    .write_config_dword(bar_offset, 0xffff_ffff);
@@ -533,7 +581,8 @@ impl VfioCommon {
                .set_index(bar_id as usize)
                .set_address(bar_addr.raw_value())
                .set_size(region_size)
-                .set_region_type(region_type);
+                .set_region_type(region_type)
+                .set_prefetchable(prefetchable);

            if bar_id == VFIO_PCI_ROM_REGION_INDEX {
                self.configuration
@@ -651,6 +700,9 @@ impl VfioCommon {
            .vfio_wrapper
            .read_config_byte(PCI_CONFIG_CAPABILITY_OFFSET);

+        let mut pci_express_cap_found = false;
+        let mut power_management_cap_found = false;
+
        while cap_next != 0 {
            let cap_id = self.vfio_wrapper.read_config_byte(cap_next.into());

@@ -676,11 +728,50 @@ impl VfioCommon {
                        }
                    }
                }
+                PciCapabilityId::PciExpress => pci_express_cap_found = true,
+                PciCapabilityId::PowerManagement => power_management_cap_found = true,
                _ => {}
            };

            cap_next = self.vfio_wrapper.read_config_byte((cap_next + 1).into());
        }
+
+        if pci_express_cap_found && power_management_cap_found {
+            self.parse_extended_capabilities();
+        }
+    }
+
+    fn parse_extended_capabilities(&mut self) {
+        let mut current_offset = PCI_CONFIG_EXTENDED_CAPABILITY_OFFSET;
+
+        loop {
+            let ext_cap_hdr = self.vfio_wrapper.read_config_dword(current_offset);
+
+            let cap_id: u16 = (ext_cap_hdr & 0xffff) as u16;
+            let cap_next: u16 = ((ext_cap_hdr >> 20) & 0xfff) as u16;
+
+            match PciExpressCapabilityId::from(cap_id) {
+                PciExpressCapabilityId::AlternativeRoutingIdentificationIntepretation
+                | PciExpressCapabilityId::ResizeableBar
+                | PciExpressCapabilityId::SingleRootIoVirtualization => {
+                    let reg_idx = (current_offset / 4) as usize;
+                    self.patches.insert(
+                        reg_idx,
+                        ConfigPatch {
+                            mask: 0x0000_ffff,
+                            patch: PciExpressCapabilityId::NullCapability as u32,
+                        },
+                    );
+                }
+                _ => {}
+            }
+
+            if cap_next == 0 {
+                break;
+            }
+
+            current_offset = cap_next.into();
+        }
    }

    pub(crate) fn enable_intx(&mut self) -> Result<(), VfioPciError> {
@@ -968,7 +1059,134 @@ impl VfioCommon {
        };

        // The config register read comes from the VFIO device itself.
-        self.vfio_wrapper.read_config_dword((reg_idx * 4) as u32) & mask
+        let mut value = self.vfio_wrapper.read_config_dword((reg_idx * 4) as u32) & mask;
+
+        if let Some(config_patch) = self.patches.get(&reg_idx) {
+            value = (value & !config_patch.mask) | config_patch.patch;
+        }
+
+        value
+    }
+
+    fn state(&self) -> VfioCommonState {
+        let intx_state = self.interrupt.intx.as_ref().map(|intx| IntxState {
+            enabled: intx.enabled,
+        });
+
+        let msi_state = self.interrupt.msi.as_ref().map(|msi| MsiState {
+            cap: msi.cfg.cap,
+            cap_offset: msi.cap_offset,
+        });
+
+        let msix_state = self.interrupt.msix.as_ref().map(|msix| MsixState {
+            cap: msix.cap,
+            cap_offset: msix.cap_offset,
+            bdf: msix.bar.devid,
+        });
+
+        VfioCommonState {
+            intx_state,
+            msi_state,
+            msix_state,
+        }
+    }
+
+    fn set_state(&mut self, state: &VfioCommonState) -> Result<(), VfioPciError> {
+        if let (Some(intx), Some(interrupt_source_group)) =
+            (&state.intx_state, self.legacy_interrupt_group.clone())
+        {
+            self.interrupt.intx = Some(VfioIntx {
+                interrupt_source_group,
+                enabled: false,
+            });
+
+            if intx.enabled {
+                self.enable_intx()?;
+            }
+        }
+
+        if let Some(msi) = &state.msi_state {
+            self.initialize_msi(msi.cap.msg_ctl, msi.cap_offset);
+        }
+
+        if let Some(msix) = &state.msix_state {
+            self.initialize_msix(msix.cap, msix.cap_offset, msix.bdf.into());
+        }
+
+        Ok(())
+    }
+}
+
+impl Pausable for VfioCommon {}
+
+impl Snapshottable for VfioCommon {
+    fn id(&self) -> String {
+        String::from("vfio_common")
+    }
+
+    fn snapshot(&mut self) -> std::result::Result<Snapshot, MigratableError> {
+        let mut vfio_common_snapshot =
+            Snapshot::new_from_versioned_state(&self.id(), &self.state())?;
+
+        // Snapshot PciConfiguration
+        vfio_common_snapshot.add_snapshot(self.configuration.snapshot()?);
+
+        // Snapshot MSI
+        if let Some(msi) = &mut self.interrupt.msi {
+            vfio_common_snapshot.add_snapshot(msi.cfg.snapshot()?);
+        }
+
+        // Snapshot MSI-X
+        if let Some(msix) = &mut self.interrupt.msix {
+            vfio_common_snapshot.add_snapshot(msix.bar.snapshot()?);
+        }
+
+        Ok(vfio_common_snapshot)
+    }
+
+    fn restore(&mut self, snapshot: Snapshot) -> std::result::Result<(), MigratableError> {
+        if let Some(vfio_common_section) = snapshot
+            .snapshot_data
+            .get(&format!("{}-section", self.id()))
+        {
+            // It has to be invoked first as we want Interrupt to be initialized
+            // correctly before we try to restore MSI and MSI-X configurations.
+            self.set_state(&vfio_common_section.to_versioned_state()?)
+                .map_err(|e| {
+                    MigratableError::Restore(anyhow!("Could not restore VFIO_COMMON state {:?}", e))
+                })?;
+
+            // Restore PciConfiguration
+            if let Some(pci_config_snapshot) = snapshot.snapshots.get(&self.configuration.id()) {
+                self.configuration.restore(*pci_config_snapshot.clone())?;
+            }
+
+            // Restore MSI
+            if let Some(msi) = &mut self.interrupt.msi {
+                if let Some(msi_snapshot) = snapshot.snapshots.get(&msi.cfg.id()) {
+                    msi.cfg.restore(*msi_snapshot.clone())?;
+                }
+                if msi.cfg.enabled() {
+                    self.enable_msi().unwrap();
+                }
+            }
+
+            // Restore MSI-X
+            if let Some(msix) = &mut self.interrupt.msix {
+                if let Some(msix_snapshot) = snapshot.snapshots.get(&msix.bar.id()) {
+                    msix.bar.restore(*msix_snapshot.clone())?;
+                }
+                if msix.bar.enabled() {
+                    self.enable_msix().unwrap();
+                }
+            }
+
+            return Ok(());
+        }
+
+        Err(MigratableError::Restore(anyhow!(
+            "Could not find VFIO_COMMON snapshot section"
+        )))
    }
 }

@@ -985,6 +1203,7 @@ pub struct VfioPciDevice {
    container: Arc<VfioContainer>,
    common: VfioCommon,
    iommu_attached: bool,
+    memory_slot: Arc<dyn Fn() -> u32 + Send + Sync>,
 }

 impl VfioPciDevice {
@@ -999,6 +1218,8 @@ impl VfioPciDevice {
        legacy_interrupt_group: Option<Arc<dyn InterruptSourceGroup>>,
        iommu_attached: bool,
        bdf: PciBdf,
+        restoring: bool,
+        memory_slot: Arc<dyn Fn() -> u32 + Send + Sync>,
    ) -> Result<Self, VfioPciError> {
        let device = Arc::new(device);
        device.reset();
@@ -1029,10 +1250,16 @@ impl VfioPciDevice {
            msi_interrupt_manager,
            legacy_interrupt_group,
            vfio_wrapper: Arc::new(vfio_wrapper) as Arc<dyn Vfio>,
+            patches: HashMap::new(),
        };

-        common.parse_capabilities(bdf);
-        common.initialize_legacy_interrupt()?;
+        // No need to parse capabilities from the device if on the restore path.
+        // The initialization will be performed later when restore() will be
+        // called.
+        if !restoring {
+            common.parse_capabilities(bdf);
+            common.initialize_legacy_interrupt()?;
+        }

        let vfio_pci_device = VfioPciDevice {
            id,
@@ -1041,6 +1268,7 @@ impl VfioPciDevice {
            container,
            common,
            iommu_attached,
+            memory_slot,
        };

        Ok(vfio_pci_device)
@@ -1150,14 +1378,7 @@ impl VfioPciDevice {
    /// * `vm` - The VM object. It is used to set the VFIO MMIO regions
    ///          as user memory regions.
    /// * `mem_slot` - The closure to return a memory slot.
-    pub fn map_mmio_regions<F>(
-        &mut self,
-        vm: &Arc<dyn hypervisor::Vm>,
-        mem_slot: F,
-    ) -> Result<(), VfioPciError>
-    where
-        F: Fn() -> u32,
-    {
+    pub fn map_mmio_regions(&mut self) -> Result<(), VfioPciError> {
        let fd = self.device.as_raw_fd();

        for region in self.common.mmio_regions.iter_mut() {
@@ -1223,7 +1444,7 @@ impl VfioPciDevice {
                    }

                    let user_memory_region = UserMemoryRegion {
-                        slot: mem_slot(),
+                        slot: (self.memory_slot)(),
                        start: region.start.0 + area.offset,
                        size: area.size,
                        host_addr: host_addr as u64,
@@ -1231,7 +1452,7 @@ impl VfioPciDevice {

                    region.user_memory_regions.push(user_memory_region);

-                    let mem_region = vm.make_user_memory_region(
+                    let mem_region = self.vm.make_user_memory_region(
                        user_memory_region.slot,
                        user_memory_region.start,
                        user_memory_region.size,
@@ -1240,7 +1461,8 @@ impl VfioPciDevice {
                        false,
                    );

-                    vm.create_user_memory_region(mem_region)
+                    self.vm
+                        .create_user_memory_region(mem_region)
                        .map_err(VfioPciError::CreateUserMemoryRegion)?;
                }
            }
@@ -1344,10 +1566,14 @@ impl BusDevice for VfioPciDevice {
 const PCI_CONFIG_BAR_OFFSET: u32 = 0x10;
 // Capability register offset in the PCI config space.
 const PCI_CONFIG_CAPABILITY_OFFSET: u32 = 0x34;
+// Extended capabilities register offset in the PCI config space.
+const PCI_CONFIG_EXTENDED_CAPABILITY_OFFSET: u32 = 0x100;
 // IO BAR when first BAR bit is 1.
 const PCI_CONFIG_IO_BAR: u32 = 0x1;
 // 64-bit memory bar flag.
 const PCI_CONFIG_MEMORY_BAR_64BIT: u32 = 0x4;
+// Prefetchable BAR bit
+const PCI_CONFIG_BAR_PREFETCHABLE: u32 = 0x8;
 // PCI config register size (4 bytes).
 const PCI_CONFIG_REGISTER_SIZE: usize = 4;
 // Number of BARs for a PCI device
@@ -1464,3 +1690,37 @@ impl PciDevice for VfioPciDevice {
        Some(self.id.clone())
    }
 }
+
+impl Pausable for VfioPciDevice {}
+
+impl Snapshottable for VfioPciDevice {
+    fn id(&self) -> String {
+        self.id.clone()
+    }
+
+    fn snapshot(&mut self) -> std::result::Result<Snapshot, MigratableError> {
+        let mut vfio_pci_dev_snapshot = Snapshot::new(&self.id);
+
+        // Snapshot VfioCommon
+        vfio_pci_dev_snapshot.add_snapshot(self.common.snapshot()?);
+
+        Ok(vfio_pci_dev_snapshot)
+    }
+
+    fn restore(&mut self, snapshot: Snapshot) -> std::result::Result<(), MigratableError> {
+        // Restore VfioCommon
+        if let Some(vfio_common_snapshot) = snapshot.snapshots.get(&self.common.id()) {
+            self.common.restore(*vfio_common_snapshot.clone())?;
+            self.map_mmio_regions().map_err(|e| {
+                MigratableError::Restore(anyhow!(
+                    "Could not map MMIO regions for VfioPciDevice on restore {:?}",
+                    e
+                ))
+            })?;
+        }
+
+        Ok(())
+    }
+}
+impl Transportable for VfioPciDevice {}
+impl Migratable for VfioPciDevice {}
--- a/pci/src/vfio_user.rs
+++ b/pci/src/vfio_user.rs
@@ -8,8 +8,10 @@ use crate::{BarReprogrammingParams, PciBarConfiguration, VfioPciError};
 use crate::{
    PciBdf, PciClassCode, PciConfiguration, PciDevice, PciDeviceError, PciHeaderType, PciSubclass,
 };
+use anyhow::anyhow;
 use hypervisor::HypervisorVmError;
 use std::any::Any;
+use std::collections::HashMap;
 use std::os::unix::prelude::AsRawFd;
 use std::ptr::null_mut;
 use std::sync::{Arc, Barrier, Mutex};
@@ -26,6 +28,7 @@ use vm_memory::bitmap::AtomicBitmap;
 use vm_memory::{
    Address, GuestAddress, GuestAddressSpace, GuestMemory, GuestMemoryRegion, GuestRegionMmap,
 };
+use vm_migration::{Migratable, MigratableError, Pausable, Snapshot, Snapshottable, Transportable};
 use vmm_sys_util::eventfd::EventFd;

 pub struct VfioUserPciDevice {
@@ -33,6 +36,7 @@ pub struct VfioUserPciDevice {
    vm: Arc<dyn hypervisor::Vm>,
    client: Arc<Mutex<Client>>,
    common: VfioCommon,
+    memory_slot: Arc<dyn Fn() -> u32 + Send + Sync>,
 }

 #[derive(Error, Debug)]
@@ -61,6 +65,7 @@ impl PciSubclass for PciVfioUserSubclass {
 }

 impl VfioUserPciDevice {
+    #[allow(clippy::too_many_arguments)]
    pub fn new(
        id: String,
        vm: &Arc<dyn hypervisor::Vm>,
@@ -68,6 +73,8 @@ impl VfioUserPciDevice {
        msi_interrupt_manager: Arc<dyn InterruptManager<GroupConfig = MsiIrqGroupConfig>>,
        legacy_interrupt_group: Option<Arc<dyn InterruptSourceGroup>>,
        bdf: PciBdf,
+        restoring: bool,
+        memory_slot: Arc<dyn Fn() -> u32 + Send + Sync>,
    ) -> Result<Self, VfioUserPciDeviceError> {
        // This is used for the BAR and capabilities only
        let configuration = PciConfiguration::new(
@@ -106,29 +113,29 @@ impl VfioUserPciDevice {
            msi_interrupt_manager,
            legacy_interrupt_group,
            vfio_wrapper: Arc::new(vfio_wrapper) as Arc<dyn Vfio>,
+            patches: HashMap::new(),
        };

-        common.parse_capabilities(bdf);
-        common
-            .initialize_legacy_interrupt()
-            .map_err(VfioUserPciDeviceError::InitializeLegacyInterrupts)?;
+        // No need to parse capabilities from the device if on the restore path.
+        // The initialization will be performed later when restore() will be
+        // called.
+        if !restoring {
+            common.parse_capabilities(bdf);
+            common
+                .initialize_legacy_interrupt()
+                .map_err(VfioUserPciDeviceError::InitializeLegacyInterrupts)?;
+        }

        Ok(Self {
            id,
            vm: vm.clone(),
            client,
            common,
+            memory_slot,
        })
    }

-    pub fn map_mmio_regions<F>(
-        &mut self,
-        vm: &Arc<dyn hypervisor::Vm>,
-        mem_slot: F,
-    ) -> Result<(), VfioUserPciDeviceError>
-    where
-        F: Fn() -> u32,
-    {
+    pub fn map_mmio_regions(&mut self) -> Result<(), VfioUserPciDeviceError> {
        for mmio_region in &mut self.common.mmio_regions {
            let region_flags = self
                .client
@@ -195,7 +202,7 @@ impl VfioUserPciDevice {
                    }

                    let user_memory_region = UserMemoryRegion {
-                        slot: mem_slot(),
+                        slot: (self.memory_slot)(),
                        start: mmio_region.start.0 + s.offset,
                        size: s.size,
                        host_addr: host_addr as u64,
@@ -203,7 +210,7 @@ impl VfioUserPciDevice {

                    mmio_region.user_memory_regions.push(user_memory_region);

-                    let mem_region = vm.make_user_memory_region(
+                    let mem_region = self.vm.make_user_memory_region(
                        user_memory_region.slot,
                        user_memory_region.start,
                        user_memory_region.size,
@@ -212,7 +219,8 @@ impl VfioUserPciDevice {
                        false,
                    );

-                    vm.create_user_memory_region(mem_region)
+                    self.vm
+                        .create_user_memory_region(mem_region)
                        .map_err(VfioUserPciDeviceError::MapRegionGuest)?;
                }
            }
@@ -551,6 +559,40 @@ impl Drop for VfioUserPciDevice {
    }
 }

+impl Pausable for VfioUserPciDevice {}
+
+impl Snapshottable for VfioUserPciDevice {
+    fn id(&self) -> String {
+        self.id.clone()
+    }
+
+    fn snapshot(&mut self) -> std::result::Result<Snapshot, MigratableError> {
+        let mut vfio_pci_dev_snapshot = Snapshot::new(&self.id);
+
+        // Snapshot VfioCommon
+        vfio_pci_dev_snapshot.add_snapshot(self.common.snapshot()?);
+
+        Ok(vfio_pci_dev_snapshot)
+    }
+
+    fn restore(&mut self, snapshot: Snapshot) -> std::result::Result<(), MigratableError> {
+        // Restore VfioCommon
+        if let Some(vfio_common_snapshot) = snapshot.snapshots.get(&self.common.id()) {
+            self.common.restore(*vfio_common_snapshot.clone())?;
+            self.map_mmio_regions().map_err(|e| {
+                MigratableError::Restore(anyhow!(
+                    "Could not map MMIO regions for VfioUserPciDevice on restore {:?}",
+                    e
+                ))
+            })?;
+        }
+
+        Ok(())
+    }
+}
+impl Transportable for VfioUserPciDevice {}
+impl Migratable for VfioUserPciDevice {}
+
 pub struct VfioUserDmaMapping<M: GuestAddressSpace> {
    client: Arc<Mutex<Client>>,
    memory: Arc<M>,
@@ -584,10 +626,10 @@ impl<M: GuestAddressSpace + Sync + Send> ExternalDmaMapping for VfioUserDmaMappi
                    )
                })
        } else {
-            return Err(std::io::Error::new(
+            Err(std::io::Error::new(
                std::io::ErrorKind::Other,
                format!("Region not found for 0x{:x}", gpa),
-            ));
+            ))
        }
    }

--- a/performance-metrics/Cargo.toml
+++ b/performance-metrics/Cargo.toml
@@ -6,13 +6,13 @@ edition = "2021"
 build = "build.rs"

 [dependencies]
-clap = { version = "3.1.18", features = ["wrap_help","cargo"] }
+clap = { version = "3.2.17", features = ["wrap_help","cargo"] }
 dirs = "4.0.0"
-serde = { version = "1.0.137", features = ["rc", "derive"] }
-serde_json = "1.0.81"
+serde = { version = "1.0.143", features = ["rc", "derive"] }
+serde_json = "1.0.83"
 test_infra = { path = "../test_infra" }
-thiserror = "1.0.31"
+thiserror = "1.0.32"
 wait-timeout = "0.2.0"

 [build-dependencies]
-clap = { version = "3.1.18", features = ["cargo"] }
+clap = { version = "3.2.17", features = ["cargo"] }
--- a/performance-metrics/src/main.rs
+++ b/performance-metrics/src/main.rs
@@ -53,18 +53,30 @@ impl Default for MetricsReport {
        let mut git_human_readable = "".to_string();
        if let Ok(git_out) = Command::new("git").args(&["describe", "--dirty"]).output() {
            if git_out.status.success() {
-                if let Ok(git_out_str) = String::from_utf8(git_out.stdout) {
-                    git_human_readable = git_out_str.trim().to_string();
-                }
+                git_human_readable = String::from_utf8(git_out.stdout)
+                    .unwrap()
+                    .trim()
+                    .to_string();
+            } else {
+                eprintln!(
+                    "Error generating human readable git reference: {}",
+                    String::from_utf8(git_out.stderr).unwrap()
+                );
            }
        }

        let mut git_revision = "".to_string();
        if let Ok(git_out) = Command::new("git").args(&["rev-parse", "HEAD"]).output() {
            if git_out.status.success() {
-                if let Ok(git_out_str) = String::from_utf8(git_out.stdout) {
-                    git_revision = git_out_str.trim().to_string();
-                }
+                git_revision = String::from_utf8(git_out.stdout)
+                    .unwrap()
+                    .trim()
+                    .to_string();
+            } else {
+                eprintln!(
+                    "Error generating git reference: {}",
+                    String::from_utf8(git_out.stderr).unwrap()
+                );
            }
        }

@@ -74,9 +86,15 @@ impl Default for MetricsReport {
            .output()
        {
            if git_out.status.success() {
-                if let Ok(git_out_str) = String::from_utf8(git_out.stdout) {
-                    git_commit_date = git_out_str.trim().to_string();
-                }
+                git_commit_date = String::from_utf8(git_out.stdout)
+                    .unwrap()
+                    .trim()
+                    .to_string();
+            } else {
+                eprintln!(
+                    "Error generating git commit date: {}",
+                    String::from_utf8(git_out.stderr).unwrap()
+                );
            }
        }

--- a/performance-metrics/src/performance_tests.rs
+++ b/performance-metrics/src/performance_tests.rs
@@ -312,7 +312,7 @@ fn measure_virtio_net_latency(guest: &Guest, test_timeout: u32) -> Result<Vec<f6
        .ssh_command(&format!("{} -s &> /dev/null &", ethr_remote_path))
        .map_err(InfraError::SshCommand)?;

-    thread::sleep(Duration::new(1, 0));
+    thread::sleep(Duration::new(10, 0));

    // Start the ethr client on the host
    let log_file = guest
--- a/qcow/Cargo.toml
+++ b/qcow/Cargo.toml
@@ -10,7 +10,7 @@ path = "src/qcow.rs"

 [dependencies]
 byteorder = "1.4.3"
-libc = "0.2.126"
+libc = "0.2.131"
 log = "0.4.17"
-remain = "0.2.3"
-vmm-sys-util = "0.9.0"
+remain = "0.2.4"
+vmm-sys-util = "0.10.0"
--- a/qcow/src/qcow.rs
+++ b/qcow/src/qcow.rs
@@ -1487,16 +1487,11 @@ impl Write for QcowFile {
            let offset = self.file_offset_write(curr_addr)?;
            let count = self.limit_range_cluster(curr_addr, write_count - nwritten);

-            if let Err(e) = self.raw_file.file_mut().seek(SeekFrom::Start(offset)) {
-                return Err(e);
-            }
-            if let Err(e) = self
+            self.raw_file.file_mut().seek(SeekFrom::Start(offset))?;
+            let count = self
                .raw_file
                .file_mut()
-                .write(&buf[nwritten..(nwritten + count)])
-            {
-                return Err(e);
-            }
+                .write(&buf[nwritten..(nwritten + count)])?;

            nwritten += count;
        }
--- a/rate_limiter/Cargo.toml
+++ b/rate_limiter/Cargo.toml
@@ -4,6 +4,6 @@ version = "0.1.0"
 edition = "2021"

 [dependencies]
-libc = "0.2.126"
+libc = "0.2.131"
 log = "0.4.17"
-vmm-sys-util = "0.9.0"
+vmm-sys-util = "0.10.0"
--- a/rate_limiter/src/lib.rs
+++ b/rate_limiter/src/lib.rs
@@ -91,7 +91,7 @@ pub enum BucketReduction {

 /// TokenBucket provides a lower level interface to rate limiting with a
 /// configurable capacity, refill-rate and initial burst.
-#[derive(Clone, Debug, PartialEq)]
+#[derive(Clone, Debug, PartialEq, Eq)]
 pub struct TokenBucket {
    // Bucket defining traits.
    size: u64,
--- a/release-notes.md
+++ b/release-notes.md
@@ -1,22 +1,35 @@
+- [v26.0](#v260)
+    - [SMBIOS Improvements via `--platform`](#smbios-improvements-via---platform)
+    - [Unified Binary MSHV and KVM Support](#unified-binary-mshv-and-kvm-support)
+    - [Notable Bug Fixes](#notable-bug-fixes)
+    - [Deprecations](#deprecations)
+    - [Removals](#removals)
+    - [Contributors](#contributors)
+- [v25.0](#v250)
+    - [`ch-remote` Improvements](#ch-remote-improvements)
+    - [VM "Coredump" Support](#vm-coredump-support)
+    - [Notable Bug Fixes](#notable-bug-fixes-1)
+    - [Removals](#removals-1)
+    - [Contributors](#contributors-1)
 - [v24.0](#v240)
    - [Bypass Mode for `virtio-iommu`](#bypass-mode-for-virtio-iommu)
    - [Ensure Identifiers Uniqueness](#ensure-identifiers-uniqueness)
    - [Sparse Mmap support](#sparse-mmap-support)
    - [Expose Platform Serial Number](#expose-platform-serial-number)
-    - [Notable Bug Fixes](#notable-bug-fixes)
+    - [Notable Bug Fixes](#notable-bug-fixes-2)
    - [Notable Improvements](#notable-improvements)
-    - [Deprecations](#deprecations)
+    - [Deprecations](#deprecations-1)
    - [New on the Website](#new-on-the-website)
-    - [Contributors](#contributors)
+    - [Contributors](#contributors-2)
 - [v23.1](#v231)
 - [v23.0](#v230)
    - [vDPA Support](#vdpa-support)
    - [Updated OS Support list](#updated-os-support-list)
    - [`AArch64` Memory Map Improvements](#aarch64-memory-map-improvements)
    - [`AMX` Support](#amx-support)
-    - [Notable Bug Fixes](#notable-bug-fixes-1)
-    - [Deprecations](#deprecations-1)
-    - [Contributors](#contributors-1)
+    - [Notable Bug Fixes](#notable-bug-fixes-3)
+    - [Deprecations](#deprecations-2)
+    - [Contributors](#contributors-3)
 - [v22.1](#v221)
 - [v22.0](#v220)
    - [GDB Debug Stub Support](#gdb-debug-stub-support)
@@ -27,13 +40,13 @@
    - [PMU Support for AArch64](#pmu-support-for-aarch64)
    - [Documentation Under CC-BY-4.0 License](#documentation-under-cc-by-40-license)
    - [Deprecation of "Classic" `virtiofsd`](#deprecation-of-classic-virtiofsd)
-    - [Notable Bug Fixes](#notable-bug-fixes-2)
-    - [Contributors](#contributors-2)
+    - [Notable Bug Fixes](#notable-bug-fixes-4)
+    - [Contributors](#contributors-4)
 - [v21.0](#v210)
    - [Efficient Local Live Migration (for Live Upgrade)](#efficient-local-live-migration-for-live-upgrade)
    - [Recommended Kernel is Now 5.15](#recommended-kernel-is-now-515)
-    - [Notable Bug fixes](#notable-bug-fixes-3)
-    - [Contributors](#contributors-3)
+    - [Notable Bug fixes](#notable-bug-fixes-5)
+    - [Contributors](#contributors-5)
 - [v20.2](#v202)
 - [v20.1](#v201)
 - [v20.0](#v200)
@@ -42,8 +55,8 @@
    - [Improved VFIO support](#improved-vfio-support)
    - [Safer code](#safer-code)
    - [Extended documentation](#extended-documentation)
-    - [Notable bug fixes](#notable-bug-fixes-4)
-    - [Contributors](#contributors-4)
+    - [Notable bug fixes](#notable-bug-fixes-6)
+    - [Contributors](#contributors-6)
 - [v19.0](#v190)
    - [Improved PTY handling for serial and `virtio-console`](#improved-pty-handling-for-serial-and-virtio-console)
    - [PCI boot time optimisations](#pci-boot-time-optimisations)
@@ -51,8 +64,8 @@
    - [Live migration enhancements](#live-migration-enhancements)
    - [`virtio-mem` support with `vfio-user`](#virtio-mem-support-with-vfio-user)
    - [AArch64 for `virtio-iommu`](#aarch64-for-virtio-iommu)
-    - [Notable bug fixes](#notable-bug-fixes-5)
-    - [Contributors](#contributors-5)
+    - [Notable bug fixes](#notable-bug-fixes-7)
+    - [Contributors](#contributors-7)
 - [v18.0](#v180)
    - [Experimental User Device (`vfio-user`) support](#experimental-user-device-vfio-user-support)
    - [Migration support for `vhost-user` devices](#migration-support-for-vhost-user-devices)
@@ -62,31 +75,31 @@
    - [Live migration on MSHV hypervisor](#live-migration-on-mshv-hypervisor)
    - [AArch64 CPU topology support](#aarch64-cpu-topology-support)
    - [Power button support on AArch64](#power-button-support-on-aarch64)
-    - [Notable bug fixes](#notable-bug-fixes-6)
-    - [Contributors](#contributors-6)
+    - [Notable bug fixes](#notable-bug-fixes-8)
+    - [Contributors](#contributors-8)
 - [v17.0](#v170)
    - [ARM64 NUMA support using ACPI](#arm64-numa-support-using-acpi)
    - [`Seccomp` support for MSHV backend](#seccomp-support-for-mshv-backend)
    - [Hotplug of `macvtap` devices](#hotplug-of-macvtap-devices)
    - [Improved SGX support](#improved-sgx-support)
    - [Inflight tracking for `vhost-user` devices](#inflight-tracking-for-vhost-user-devices)
-    - [Notable bug fixes](#notable-bug-fixes-7)
-    - [Contributors](#contributors-7)
+    - [Notable bug fixes](#notable-bug-fixes-9)
+    - [Contributors](#contributors-9)
 - [v16.0](#v160)
    - [Improved live migration support](#improved-live-migration-support)
    - [Improved `vhost-user` support](#improved-vhost-user-support)
    - [ARM64 ACPI and UEFI support](#arm64-acpi-and-uefi-support)
-    - [Notable bug fixes](#notable-bug-fixes-8)
+    - [Notable bug fixes](#notable-bug-fixes-10)
    - [Removed functionality](#removed-functionality)
-    - [Contributors](#contributors-8)
+    - [Contributors](#contributors-10)
 - [v15.0](#v150)
    - [Version numbering and stability guarantees](#version-numbering-and-stability-guarantees)
    - [Network device rate limiting](#network-device-rate-limiting)
    - [Support for runtime control of `virtio-net` guest offload](#support-for-runtime-control-of-virtio-net-guest-offload)
    - [`--api-socket` supports file descriptor parameter](#--api-socket-supports-file-descriptor-parameter)
    - [Bug fixes](#bug-fixes)
-    - [Deprecations](#deprecations-2)
-    - [Contributors](#contributors-9)
+    - [Deprecations](#deprecations-3)
+    - [Contributors](#contributors-11)
 - [v0.14.1](#v0141)
 - [v0.14.0](#v0140)
    - [Structured event monitoring](#structured-event-monitoring)
@@ -95,8 +108,8 @@
    - [Updated hotplug documentation](#updated-hotplug-documentation)
    - [PTY control for serial and `virtio-console`](#pty-control-for-serial-and-virtio-console)
    - [Block device rate limiting](#block-device-rate-limiting)
-    - [Deprecations](#deprecations-3)
-    - [Contributors](#contributors-10)
+    - [Deprecations](#deprecations-4)
+    - [Contributors](#contributors-12)
 - [v0.13.0](#v0130)
    - [Wider VFIO device support](#wider-vfio-device-support)
    - [Improved huge page support](#improved-huge-page-support)
@@ -104,13 +117,13 @@
    - [VHD disk image support](#vhd-disk-image-support)
    - [Improved Virtio device threading](#improved-virtio-device-threading)
    - [Clean shutdown support via synthetic power button](#clean-shutdown-support-via-synthetic-power-button)
-    - [Contributors](#contributors-11)
+    - [Contributors](#contributors-13)
 - [v0.12.0](#v0120)
    - [ARM64 enhancements](#arm64-enhancements)
    - [Removal of `vhost-user-net` and `vhost-user-block` self spawning](#removal-of-vhost-user-net-and-vhost-user-block-self-spawning)
    - [Migration of `vhost-user-fs` backend](#migration-of-vhost-user-fs-backend)
    - [Enhanced "info" API](#enhanced-info-api)
-    - [Contributors](#contributors-12)
+    - [Contributors](#contributors-14)
 - [v0.11.0](#v0110)
    - [`io_uring` support by default for `virtio-block`](#io_uring-support-by-default-for-virtio-block)
    - [Windows Guest Support](#windows-guest-support)
@@ -122,15 +135,15 @@
    - [Default Log Level Changed](#default-log-level-changed)
    - [New `--balloon` Parameter Added](#new---balloon-parameter-added)
    - [Experimental `virtio-watchdog` Support](#experimental-virtio-watchdog-support)
-    - [Notable Bug Fixes](#notable-bug-fixes-9)
-    - [Contributors](#contributors-13)
+    - [Notable Bug Fixes](#notable-bug-fixes-11)
+    - [Contributors](#contributors-15)
 - [v0.10.0](#v0100)
    - [`virtio-block` Support for Multiple Descriptors](#virtio-block-support-for-multiple-descriptors)
    - [Memory Zones](#memory-zones)
    - [`Seccomp` Sandbox Improvements](#seccomp-sandbox-improvements)
    - [Preliminary KVM HyperV Emulation Control](#preliminary-kvm-hyperv-emulation-control)
-    - [Notable Bug Fixes](#notable-bug-fixes-10)
-    - [Contributors](#contributors-14)
+    - [Notable Bug Fixes](#notable-bug-fixes-12)
+    - [Contributors](#contributors-16)
 - [v0.9.0](#v090)
    - [`io_uring` Based Block Device Support](#io_uring-based-block-device-support)
    - [Block and Network Device Statistics](#block-and-network-device-statistics)
@@ -143,17 +156,17 @@
    - [Enhancements to ARM64 Support](#enhancements-to-arm64-support)
    - [Intel SGX Support](#intel-sgx-support)
    - [`Seccomp` Sandbox Improvements](#seccomp-sandbox-improvements-1)
-    - [Notable Bug Fixes](#notable-bug-fixes-11)
-    - [Contributors](#contributors-15)
+    - [Notable Bug Fixes](#notable-bug-fixes-13)
+    - [Contributors](#contributors-17)
 - [v0.8.0](#v080)
    - [Experimental Snapshot and Restore Support](#experimental-snapshot-and-restore-support)
    - [Experimental ARM64 Support](#experimental-arm64-support)
    - [Support for Using 5-level Paging in Guests](#support-for-using-5-level-paging-in-guests)
    - [Virtio Device Interrupt Suppression for Network Devices](#virtio-device-interrupt-suppression-for-network-devices)
    - [`vhost_user_fs` Improvements](#vhost_user_fs-improvements)
-    - [Notable Bug Fixes](#notable-bug-fixes-12)
+    - [Notable Bug Fixes](#notable-bug-fixes-14)
    - [Command Line and API Changes](#command-line-and-api-changes)
-    - [Contributors](#contributors-16)
+    - [Contributors](#contributors-18)
 - [v0.7.0](#v070)
    - [Block, Network, Persistent Memory (PMEM), VirtioFS and Vsock hotplug](#block-network-persistent-memory-pmem-virtiofs-and-vsock-hotplug)
    - [Alternative `libc` Support](#alternative-libc-support)
@@ -163,14 +176,14 @@
    - [`Seccomp` Sandboxing](#seccomp-sandboxing)
    - [Updated Distribution Support](#updated-distribution-support)
    - [Command Line and API Changes](#command-line-and-api-changes-1)
-    - [Contributors](#contributors-17)
+    - [Contributors](#contributors-19)
 - [v0.6.0](#v060)
    - [Directly Assigned Devices Hotplug](#directly-assigned-devices-hotplug)
    - [Shared Filesystem Improvements](#shared-filesystem-improvements)
    - [Block and Networking IO Self Offloading](#block-and-networking-io-self-offloading)
    - [Command Line Interface](#command-line-interface)
    - [PVH Boot](#pvh-boot)
-    - [Contributors](#contributors-18)
+    - [Contributors](#contributors-20)
 - [v0.5.1](#v051)
 - [v0.5.0](#v050)
    - [Virtual Machine Dynamic Resizing](#virtual-machine-dynamic-resizing)
@@ -178,7 +191,7 @@
    - [New Interrupt Management Framework](#new-interrupt-management-framework)
    - [Development Tools](#development-tools)
    - [Kata Containers Integration](#kata-containers-integration)
-    - [Contributors](#contributors-19)
+    - [Contributors](#contributors-21)
 - [v0.4.0](#v040)
    - [Dynamic virtual CPUs addition](#dynamic-virtual-cpus-addition)
    - [Programmatic firmware tables generation](#programmatic-firmware-tables-generation)
@@ -187,7 +200,7 @@
    - [Userspace IOAPIC by default](#userspace-ioapic-by-default)
    - [PCI BAR reprogramming](#pci-bar-reprogramming)
    - [New `cloud-hypervisor` organization](#new-cloud-hypervisor-organization)
-    - [Contributors](#contributors-20)
+    - [Contributors](#contributors-22)
 - [v0.3.0](#v030)
    - [Block device offloading](#block-device-offloading)
    - [Network device backend](#network-device-backend)
@@ -214,6 +227,118 @@
    - [Unit testing](#unit-testing)
    - [Integration tests parallelization](#integration-tests-parallelization)

+# v26.0
+
+This release has been tracked through the [v26.0
+project](https://github.com/cloud-hypervisor/cloud-hypervisor/projects/30).
+
+### SMBIOS Improvements via `--platform`
+
+`--platform` and the appropriate API structure has gained support for supplying
+OEM strings (primarily used to communicate metadata to systemd in the guest)
+(#4319, #4446) and support for specifying the UUID (#4389.)
+
+### Unified Binary MSHV and KVM Support
+
+Support for both the MSHV and KVM hypervisors can be compiled into the same
+binary with the detection of the hypervisor to use made at runtime.
+
+### Notable Bug Fixes
+
+* The prefetchable flag is preserved on BARs for VFIO devices (#4353, #4454)
+* PCI Express capabilties for functionality we do not support are now filtered
+  out (#4456)
+* GDB breakpoint support is more reliable (#4354, #4363)
+* `SIGINT` and `SIGTERM` signals are now handled before the VM has booted
+  (#4269, #4293)
+* Multiple API event loop handling bug fixes (#4309, #4362)
+* Incorrect assumptions in virtio queue numbering were addressed, allowing
+  the`virtio-fs` driver in OVMF to be used (#4341, #4314)
+* VHDX file format header fix (#4291)
+* The same VFIO device cannot be added twice (#4453, #4463)
+* SMBIOS tables were being incorrectly generated (#4442)
+
+### Deprecations
+
+Deprecated features will be removed in a subsequent release and users should
+plan to use alternatives.
+
+* The top-level `kernel` and `initramfs` members on the `VmConfig` have been
+  moved inside a `PayloadConfig` as the `payload` member. The OpenAPI document
+  has been updated to reflect the change and the old API members continue to
+  function and are mapped to the new version. The expectation is that these old
+  versions will be removed in the v28.0 release.
+
+### Removals
+
+The following functionality has been removed:
+
+* The unused `poll_queue` parameter has been removed from `--disk` and
+  equivalent. This was residual from the the removal of the `vhost-user-block`
+  spawning feature (#4402.)
+
+### Contributors
+
+Many thanks to everyone who has contributed to our release:
+
+* Alyssa Ross <hi@alyssa.is>
+* Anatol Belski <ab@php.net>
+* Archana Shinde <archana.m.shinde@intel.com>
+* Bo Chen <chen.bo@intel.com>
+* lizhaoxin1 <Lxiaoyouling@163.com>
+* Maximilian Nitsch <maximilian.nitsch@d3tn.com>
+* Michael Zhao <michael.zhao@arm.com>
+* Rob Bradford <robert.bradford@intel.com>
+* Sebastien Boeuf <sebastien.boeuf@intel.com>
+* Steven Dake <sdake@lambdal.com>
+* Wei Liu <liuwe@microsoft.com>
+
+# v25.0
+
+This release has been tracked through the [v25.0
+project](https://github.com/cloud-hypervisor/cloud-hypervisor/projects/29).
+
+### `ch-remote` Improvements
+
+The `ch-remote` command has gained support for creating the VM from a JSON
+config and support for booting and deleting the VM from the VMM.
+
+### VM "Coredump" Support
+
+Under the `guest_debug` feature flag it is now possible to extract the memory
+of the guest for use in debugging with e.g. the `crash` utility. (#4012)
+
+### Notable Bug Fixes
+
+* Always restore console mode on exit (#4249, #4248)
+* Restore vCPUs in numerical order which fixes aarch64 snapshot/restore (#4244)
+* Don't try and configure `IFF_RUNNING` on TAP devices (#4279)
+* Propagate configured queue size through to vhost-user backend (#4286)
+* Always Program vCPU CPUID before running the vCPU to fix running on Linux
+  5.16 (#4156)
+* Enable ACPI MADT "Online Capable" flag for hotpluggable vCPUs to fix newer
+  Linux guest
+
+### Removals
+
+The following functionality has been removed:
+
+* The `mergeable` option from the `virtio-pmem` support has been removed
+  (#3968)
+* The `dax` option from the `virtio-fs` support has been removed (#3889)
+
+### Contributors
+
+Many thanks to everyone who has contributed to our release:
+
+* Dylan Bargatze <dbargatz@users.noreply.github.com>
+* Jinank Jain <jinankjain@microsoft.com>
+* Michael Zhao <michael.zhao@arm.com>
+* Rob Bradford <robert.bradford@intel.com>
+* Sebastien Boeuf <sebastien.boeuf@intel.com>
+* Wei Liu <liuwe@microsoft.com>
+* Yi Wang <wang.yi59@zte.com.cn>
+
 # v24.0

 This release has been tracked through the [v24.0
--- a/resources/Dockerfile
+++ b/resources/Dockerfile
@@ -4,7 +4,7 @@
 FROM ubuntu:20.04 as dev

 ARG TARGETARCH
-ARG RUST_TOOLCHAIN="1.60.0"
+ARG RUST_TOOLCHAIN="1.62.0"
 ARG CLH_SRC_DIR="/cloud-hypervisor"
 ARG CLH_BUILD_DIR="$CLH_SRC_DIR/build"
 ARG CARGO_REGISTRY_DIR="$CLH_BUILD_DIR/cargo_registry"
--- a/rpm/cloud-hypervisor.spec
+++ b/rpm/cloud-hypervisor.spec
@@ -11,7 +11,7 @@

 Name:           cloud-hypervisor
 Summary:        Cloud Hypervisor is an open source Virtual Machine Monitor (VMM) that runs on top of KVM.
-Version:        24.0
+Version:        26.0
 Release:        0%{?dist}
 License:        ASL 2.0 or BSD-3-clause
 Group:          Applications/System
@@ -145,6 +145,12 @@ rm -rf %{buildroot}


 %changelog
+*   Wed Aug 17 2022 Rob Bradford <robert.bradford@intel.com> 26.0-0
+-   Update to 26.0
+
+*   Thu Jul 07 2022 Rob Bradford <robert.bradford@intel.com> 25.0-0
+-   Update to 25.0
+
 *   Wed May 25 2022 Sebastien Boeuf <sebastien.boeuf@intel.com> 24.0-0
 -   Update to 24.0

--- a/scripts/common-aarch64.sh
+++ b/scripts/common-aarch64.sh
@@ -0,0 +1,49 @@
+#!/bin/bash
+
+export BUILD_TARGET=${BUILD_TARGET-aarch64-unknown-linux-gnu}
+
+WORKLOADS_DIR="$HOME/workloads"
+
+mkdir -p "$WORKLOADS_DIR"
+
+build_edk2() {
+    EDK2_BUILD_DIR="$WORKLOADS_DIR/edk2_build"
+    EDK2_REPO="https://github.com/tianocore/edk2.git"
+    EDK2_DIR="$EDK2_BUILD_DIR/edk2"
+    EDK2_PLAT_REPO="https://github.com/tianocore/edk2-platforms.git"
+    EDK2_PLAT_DIR="$EDK2_BUILD_DIR/edk2-platforms"
+    ACPICA_REPO="https://github.com/acpica/acpica.git"
+    ACPICA_DIR="$EDK2_BUILD_DIR/acpica"
+    export WORKSPACE="$EDK2_BUILD_DIR"
+    export PACKAGES_PATH="$EDK2_DIR:$EDK2_PLAT_DIR"
+    export IASL_PREFIX="$ACPICA_DIR/generate/unix/bin/"
+
+    if [ ! -d "$EDK2_BUILD_DIR" ]; then
+        mkdir -p "$EDK2_BUILD_DIR"
+    fi
+
+    # Prepare source code
+    checkout_repo "$EDK2_DIR" "$EDK2_REPO" master "46b4606ba23498d3d0e66b53e498eb3d5d592586"
+    pushd "$EDK2_DIR"
+    git submodule update --init
+    popd
+    checkout_repo "$EDK2_PLAT_DIR" "$EDK2_PLAT_REPO" master "8227e9e9f6a8aefbd772b40138f835121ccb2307"
+    checkout_repo "$ACPICA_DIR" "$ACPICA_REPO" master "b9c69f81a05c45611c91ea9cbce8756078d76233"
+
+    if [[ ! -f "$EDK2_DIR/.built" || \
+          ! -f "$EDK2_PLAT_DIR/.built" || \
+          ! -f "$ACPICA_DIR/.built" ]]; then
+        pushd "$EDK2_BUILD_DIR"
+        # Build
+        make -C acpica -j `nproc`
+        source edk2/edksetup.sh
+        make -C edk2/BaseTools -j `nproc`
+        build -a AARCH64 -t GCC5 -p ArmVirtPkg/ArmVirtCloudHv.dsc -b RELEASE -n 0
+        cp Build/ArmVirtCloudHv-AARCH64/RELEASE_GCC5/FV/CLOUDHV_EFI.fd "$WORKLOADS_DIR"
+        touch "$EDK2_DIR"/.built
+        touch "$EDK2_PLAT_DIR"/.built
+        touch "$ACPICA_DIR"/.built
+        popd
+    fi
+}
+
--- a/scripts/dev_cli.sh
+++ b/scripts/dev_cli.sh
@@ -7,7 +7,7 @@
 CLI_NAME="Cloud Hypervisor"

 CTR_IMAGE_TAG="cloudhypervisor/dev"
-CTR_IMAGE_VERSION="20220524-0"
+CTR_IMAGE_VERSION="20220705-0"
 CTR_IMAGE="${CTR_IMAGE_TAG}:${CTR_IMAGE_VERSION}"

 DOCKER_RUNTIME="docker"
@@ -471,7 +471,7 @@ cmd_tests() {
            --env USER="root" \
            --env CH_LIBC="${libc}" \
            "$CTR_IMAGE" \
-            ./scripts/run_integration_tests_windows.sh "$@" || fix_dir_perms $? || exit $?
+            ./scripts/run_integration_tests_windows_"$(uname -m)".sh "$@" || fix_dir_perms $? || exit $?
    fi

    if [ "$integration_live_migration" = true ]; then
--- a/scripts/prepare_vdpa.sh
+++ b/scripts/prepare_vdpa.sh
@@ -1,7 +1,7 @@
 #!/bin/bash
 set -x

-sudo apt install -y libncurses-dev gawk flex bison openssl libssl-dev dkms libelf-dev libudev-dev libpci-dev libiberty-dev autoconf git make dpkg-dev libmnl-dev pkg-config
+sudo apt install -y libncurses-dev gawk flex bison openssl libssl-dev dkms libelf-dev libudev-dev libpci-dev libiberty-dev autoconf git make dpkg-dev libmnl-dev pkg-config iproute2
 sudo sed -i -- 's/# deb-src/deb-src/g' /etc/apt/sources.list
 sudo apt update
 apt-get source linux-image-unsigned-`uname -r`
@@ -21,15 +21,6 @@ sudo modprobe vhost_vdpa
 sudo modprobe vdpa_sim
 sudo modprobe vdpa_sim_blk
 sudo modprobe vdpa_sim_net
-# After the Jenkins builder are moved to Ubuntu 22.04, manually building and
-# installing iproute2/vdpa won't be required.
-# Tracked by: https://github.com/cloud-hypervisor/cloud-hypervisor/issues/3862
-git clone https://github.com/shemminger/iproute2.git
-pushd iproute2
-./configure 
-make -j
-sudo make install
-popd
 # Create /dev/vhost-vdpa-0
 sudo vdpa dev add name vdpa-blk1 mgmtdev vdpasim_blk
 # Create /dev/vhost-vdpa-1
@@ -39,4 +30,4 @@ sudo vdpa dev add name vdpa-net1 mgmtdev vdpasim_net
 sudo chmod 660 /dev/vhost-vdpa-0
 sudo chmod 660 /dev/vhost-vdpa-1
 sudo chmod 660 /dev/vhost-vdpa-2
-vdpa dev show -jp
+vdpa dev show -jp
--- a/scripts/run_integration_tests_aarch64.sh
+++ b/scripts/run_integration_tests_aarch64.sh
@@ -3,55 +3,10 @@ set -x

 source $HOME/.cargo/env
 source $(dirname "$0")/test-util.sh
+source $(dirname "$0")/common-aarch64.sh

-export BUILD_TARGET=${BUILD_TARGET-aarch64-unknown-linux-gnu}
-
-WORKLOADS_DIR="$HOME/workloads"
 WORKLOADS_LOCK="$WORKLOADS_DIR/integration_test.lock"

-mkdir -p "$WORKLOADS_DIR"
-
-build_edk2() {
-    EDK2_BUILD_DIR="$WORKLOADS_DIR/edk2_build"
-    EDK2_REPO="https://github.com/tianocore/edk2.git"
-    EDK2_DIR="$EDK2_BUILD_DIR/edk2"
-    EDK2_PLAT_REPO="https://github.com/tianocore/edk2-platforms.git"
-    EDK2_PLAT_DIR="$EDK2_BUILD_DIR/edk2-platforms"
-    ACPICA_REPO="https://github.com/acpica/acpica.git"
-    ACPICA_DIR="$EDK2_BUILD_DIR/acpica"
-    export WORKSPACE="$EDK2_BUILD_DIR"
-    export PACKAGES_PATH="$EDK2_DIR:$EDK2_PLAT_DIR"
-    export IASL_PREFIX="$ACPICA_DIR/generate/unix/bin/"
-
-    if [ ! -d "$EDK2_BUILD_DIR" ]; then
-        mkdir -p "$EDK2_BUILD_DIR"
-    fi
-
-    # Prepare source code
-    checkout_repo "$EDK2_DIR" "$EDK2_REPO" master "46b4606ba23498d3d0e66b53e498eb3d5d592586"
-    pushd "$EDK2_DIR"
-    git submodule update --init
-    popd
-    checkout_repo "$EDK2_PLAT_DIR" "$EDK2_PLAT_REPO" master "8227e9e9f6a8aefbd772b40138f835121ccb2307"
-    checkout_repo "$ACPICA_DIR" "$ACPICA_REPO" master "b9c69f81a05c45611c91ea9cbce8756078d76233"
-
-    if [[ ! -f "$EDK2_DIR/.built" || \
-          ! -f "$EDK2_PLAT_DIR/.built" || \
-          ! -f "$ACPICA_DIR/.built" ]]; then
-        pushd "$EDK2_BUILD_DIR"
-        # Build
-        make -C acpica -j `nproc`
-        source edk2/edksetup.sh
-        make -C edk2/BaseTools -j `nproc`
-        build -a AARCH64 -t GCC5 -p ArmVirtPkg/ArmVirtCloudHv.dsc -b RELEASE -n 0
-        cp Build/ArmVirtCloudHv-AARCH64/RELEASE_GCC5/FV/CLOUDHV_EFI.fd "$WORKLOADS_DIR"
-        touch "$EDK2_DIR"/.built
-        touch "$EDK2_PLAT_DIR"/.built
-        touch "$ACPICA_DIR"/.built
-        popd
-    fi
-}
-
 build_spdk_nvme() {
    SPDK_DIR="$WORKLOADS_DIR/spdk"
    SPDK_REPO="https://github.com/spdk/spdk.git"
@@ -295,13 +250,13 @@ echo 6144 | sudo tee /proc/sys/vm/nr_hugepages
 sudo chmod a+rwX /dev/hugepages

 # Run all direct kernel boot (Device Tree) test cases in mod `parallel`
-time cargo test $features "parallel::$test_filter" --target $BUILD_TARGET -- ${test_binary_args[*]}
+time cargo test $features "common_parallel::$test_filter" --target $BUILD_TARGET -- ${test_binary_args[*]}
 RES=$?

 # Run some tests in sequence since the result could be affected by other tests
 # running in parallel.
 if [ $RES -eq 0 ]; then
-    time cargo test $features "sequential::$test_filter" --target $BUILD_TARGET -- --test-threads=1 ${test_binary_args[*]}
+    time cargo test $features "common_sequential::$test_filter" --target $BUILD_TARGET -- --test-threads=1 ${test_binary_args[*]}
    RES=$?
 else
    exit $RES
@@ -317,7 +272,14 @@ fi

 # Run all test cases related to live migration
 if [ $RES -eq 0 ]; then
-    time cargo test $features "live_migration::$test_filter" --target $BUILD_TARGET -- --test-threads=1 ${test_binary_args[*]}
+    time cargo test $features "live_migration_parallel::$test_filter" --target $BUILD_TARGET -- ${test_binary_args[*]}
+    RES=$?
+else
+    exit $RES
+fi
+
+if [ $RES -eq 0 ]; then
+    time cargo test $features "live_migration_sequential::$test_filter" --target $BUILD_TARGET -- --test-threads=1 ${test_binary_args[*]}
    RES=$?
 else
    exit $RES
--- a/scripts/run_integration_tests_live_migration.sh
+++ b/scripts/run_integration_tests_live_migration.sh
@@ -96,7 +96,15 @@ echo 6144 | sudo tee /proc/sys/vm/nr_hugepages
 sudo chmod a+rwX /dev/hugepages

 export RUST_BACKTRACE=1
-time cargo test $features "live_migration::$test_filter" -- --test-threads=1 ${test_binary_args[*]}
+time cargo test $features "live_migration_parallel::$test_filter" -- ${test_binary_args[*]}
 RES=$?

+# Run some tests in sequence since the result could be affected by other tests
+# running in parallel.
+if [ $RES -eq 0 ]; then
+    export RUST_BACKTRACE=1
+    time cargo test $features "live_migration_sequential::$test_filter" -- --test-threads=1 ${test_binary_args[*]}
+    RES=$?
+fi
+
 exit $RES
--- a/scripts/run_integration_tests_vfio.sh
+++ b/scripts/run_integration_tests_vfio.sh
@@ -8,6 +8,33 @@ process_common_args "$@"
 # For now these values are default for kvm
 features=""

+WORKLOADS_DIR="$HOME/workloads"
+FOCAL_OS_IMAGE_NAME="focal-server-cloudimg-amd64-custom-20210609-0.raw"
+FOCAL_OS_IMAGE="$WORKLOADS_DIR/$FOCAL_OS_IMAGE_NAME"
+FW="$WORKLOADS_DIR/hypervisor-fw"
+VMLINUX_IMAGE="$WORKLOADS_DIR/vmlinux"
+
+BLK_IMAGE="$WORKLOADS_DIR/blk.img"
+MNT_DIR="mount_image"
+rm -rf $BLK_IMAGE
+pushd $WORKLOADS_DIR
+fallocate -l 16M $BLK_IMAGE
+mkfs.ext4 -j $BLK_IMAGE
+mkdir $MNT_DIR
+sudo mount -t ext4 $BLK_IMAGE $MNT_DIR
+sudo bash -c "echo bar > $MNT_DIR/foo" || exit 1
+sudo umount $BLK_IMAGE
+rm -r $MNT_DIR
+popd
+
+VFIO_DIR="$WORKLOADS_DIR/vfio"
+VFIO_DISK_IMAGE="$WORKLOADS_DIR/vfio.img"
+rm -rf $VFIO_DIR $VFIO_DISK_IMAGE
+mkdir -p $VFIO_DIR
+cp $FOCAL_OS_IMAGE $VFIO_DIR
+cp $FW $VFIO_DIR
+cp $VMLINUX_IMAGE $VFIO_DIR || exit 1
+
 BUILD_TARGET="$(uname -m)-unknown-linux-${CH_LIBC}"
 CFLAGS=""
 TARGET_CC=""
@@ -18,10 +45,23 @@ fi

 cargo build --all --release $features --target $BUILD_TARGET
 strip target/$BUILD_TARGET/release/cloud-hypervisor
+strip target/$BUILD_TARGET/release/ch-remote
+
+# We always copy a fresh version of our binary for our L2 guest.
+cp target/$BUILD_TARGET/release/cloud-hypervisor $VFIO_DIR
+cp target/$BUILD_TARGET/release/ch-remote $VFIO_DIR
+
+# test_vfio rely on hugepages
+echo 6144 | sudo tee /proc/sys/vm/nr_hugepages
+sudo chmod a+rwX /dev/hugepages

 export RUST_BACKTRACE=1
-
-time cargo test $features "vfio::$test_filter" -- --test-threads=1 ${test_binary_args[*]}
+time cargo test $features "vfio::test_vfio" -- ${test_binary_args[*]}
 RES=$?

+if [ $RES -eq 0 ]; then
+	time cargo test $features "vfio::test_nvidia" -- --test-threads=1 ${test_binary_args[*]}
+	RES=$?
+fi
+
 exit $RES
--- a/Show More
+++ b/Show More