.github: Use 1.60 toolchain consistently in release process

Signed-off-by: Rob Bradford <robert.bradford@intel.com>
build: Release v24.0
2026-08-05 02:19:16 +00:00 · 2022-05-25 23:56:51 +02:00 · 2022-05-25 23:31:50 +02:00 · 2022-05-25 17:05:45 +01:00 · 2022-05-25 09:38:20 +01:00 · 2022-05-25 10:08:27 +02:00
106 changed files with 4330 additions and 2501 deletions
--- a/.github/workflows/build.yaml
+++ b/.github/workflows/build.yaml
@@ -13,7 +13,7 @@ jobs:
          - stable
          - beta
          - nightly
-          - 1.56
+          - "1.60"
        target:
          - x86_64-unknown-linux-gnu
          - x86_64-unknown-linux-musl
@@ -39,25 +39,25 @@ jobs:
          git checkout $GITHUB_SHA

      - name: Build (default features)
-        run: cargo rustc --bin cloud-hypervisor -- -D warnings
+        run: cargo rustc --locked --bin cloud-hypervisor -- -D warnings

      - name: Build (common + kvm)
-        run: cargo rustc --bin cloud-hypervisor --no-default-features --features "common,kvm"  -- -D warnings
+        run: cargo rustc --locked --bin cloud-hypervisor --no-default-features --features "common,kvm"  -- -D warnings

      - name: Build (default features + tdx)
-        run: cargo rustc --bin cloud-hypervisor --features "tdx" -- -D warnings
+        run: cargo rustc --locked --bin cloud-hypervisor --features "tdx" -- -D warnings

      - name: Build (default features + amx)
-        run: cargo rustc --bin cloud-hypervisor --features "amx" -- -D warnings
+        run: cargo rustc --locked --bin cloud-hypervisor --features "amx" -- -D warnings

      - name: Build (default features + gdb)
-        run: cargo rustc --bin cloud-hypervisor --features "gdb" -- -D warnings
+        run: cargo rustc --locked --bin cloud-hypervisor --features "gdb" -- -D warnings

      - name: Build (common + mshv)
-        run: cargo rustc --bin cloud-hypervisor --no-default-features --features "common,mshv"  -- -D warnings
+        run: cargo rustc --locked --bin cloud-hypervisor --no-default-features --features "common,mshv"  -- -D warnings

      - name: Release Build (default features)
-        run: cargo build --all --release --target=${{ matrix.target }}
+        run: cargo build --locked --all --release --target=${{ matrix.target }}

      - name: Check build did not modify any files
        run: test -z "$(git status --porcelain)"
--- a/.github/workflows/quality.yaml
+++ b/.github/workflows/quality.yaml
@@ -34,19 +34,19 @@ jobs:
        run: cargo fmt -- --check

      - name: Clippy (common + kvm)
-        run: cargo clippy --all --all-targets --no-default-features --tests --features "common,kvm" -- -D warnings
+        run: cargo clippy --locked --all --all-targets --no-default-features --tests --features "common,kvm" -- -D warnings

      - name: Clippy (default features)
-        run: cargo clippy --all --all-targets --tests -- -D warnings
+        run: cargo clippy --locked --all --all-targets --tests -- -D warnings

      - name: Clippy (default features + amx)
-        run: cargo clippy --all --all-targets --tests --features "amx" -- -D warnings
+        run: cargo clippy --locked --all --all-targets --tests --features "amx" -- -D warnings

      - name: Clippy (default features + gdb)
-        run: cargo clippy --all --all-targets --tests --features "gdb" -- -D warnings
+        run: cargo clippy --locked --all --all-targets --tests --features "gdb" -- -D warnings

      - name: Clippy (common + mshv)
-        run: cargo clippy --all --all-targets --no-default-features --tests --features "common,mshv" -- -D warnings
+        run: cargo clippy --locked --all --all-targets --no-default-features --tests --features "common,mshv" -- -D warnings

      - name: Check build did not modify any files
        run: test -z "$(git status --porcelain)"
--- a/.github/workflows/release.yaml
+++ b/.github/workflows/release.yaml
@@ -16,23 +16,23 @@ jobs:
      - name: Install Rust toolchain (x86_64-unknown-linux-gnu)
        uses: actions-rs/toolchain@v1
        with:
-            toolchain: 1.58
+            toolchain: "1.60"
            target: x86_64-unknown-linux-gnu
      - name: Install Rust toolchain (x86_64-unknown-linux-musl)
        uses: actions-rs/toolchain@v1
        with:
-            toolchain: 1.58
+            toolchain: "1.60"
            target: x86_64-unknown-linux-musl
      - name: Build
        uses: actions-rs/cargo@v1
        with:
-            toolchain: 1.58
+            toolchain: "1.60"
            command: build
            args: --all --release --target=x86_64-unknown-linux-gnu
      - name: Static Build
        uses: actions-rs/cargo@v1
        with:
-            toolchain: 1.58
+            toolchain: "1.60"
            command: build
            args: --all --release --target=x86_64-unknown-linux-musl
      - name: Strip cloud-hypervisor binaries
@@ -40,7 +40,7 @@ jobs:
      - name: Install Rust toolchain (aarch64-unknown-linux-musl)
        uses: actions-rs/toolchain@v1
        with:
-            toolchain: 1.58
+            toolchain: "1.60"
            target: aarch64-unknown-linux-musl
            override: true
      - name: Static Build (AArch64)
--- a/.gitignore
+++ b/.gitignore
@@ -4,3 +4,4 @@
 **/*.rs.bk
 **/Cargo.lock
 **/rusty-tags.vi
+/rpm/SOURCES
--- a/2
+++ b/2
@@ -0,0 +1,2 @@
+# Add the list of code owners here (using their GitHub username)
+* @cloud-hypervisor/cloud-hypervisor-reviewers
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -2,12 +2,14 @@

 Cloud Hypervisor is an open source project licensed under the [Apache v2
 License](https://opensource.org/licenses/Apache-2.0) and the [BSD 3
-Clause](https://opensource.org/licenses/BSD-3-Clause) license. Contributions
-can be made under either license or both. Individual files contain details of
-their licensing and changes to that file are under the same license unless the
-contribution changes the license of the file. When importing code from a third
-party project (e.g. Firecracker or CrosVM) please respect the license of those
-projects.
+Clause](https://opensource.org/licenses/BSD-3-Clause) license. Individual files
+contain details of their licensing and changes to that file are under the same
+license unless the contribution changes the license of the file. When importing
+code from a third party project (e.g. Firecracker or CrosVM) please respect the
+license of those projects.
+
+New code should be under the [Apache v2
+License](https://opensource.org/licenses/Apache-2.0).

 ## Coding Style

@@ -63,11 +65,9 @@ you want to merge your changes to `cloud-hypervisor`:
 2. Within your fork, create a branch for your contribution.
 3. [Create a pull request](https://help.github.com/articles/creating-a-pull-request-from-a-fork/)
   against the main branch of the Cloud Hypervisor repository.
-4. Add reviewers to your pull request and then work with your reviewers to address
-   any comments and obtain minimum of 2 [maintainers](MAINTAINERS.md) approvals.
-   To update your pull request amend existing commits whenever applicable and
+4. To update your pull request amend existing commits whenever applicable and
   then push the new changes to your pull request branch.
-5. Once the pull request is approved, one of the maintainers will merge it.
+5. Once the pull request is approved it can be integrated.

 ## Issue tracking

--- a/Cargo.lock
+++ b/Cargo.lock
@@ -20,9 +20,9 @@ dependencies = [

 [[package]]
 name = "anyhow"
-version = "1.0.56"
+version = "1.0.57"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4361135be9122e0870de935d7c439aef945b9f9ddd4199a553b5270b49c82a27"
+checksum = "08f9b8508dccb7687a1d6c4ce66b2b0ecef467c94667de27d8d7fe1f8d2a9cdc"

 [[package]]
 name = "api_client"
@@ -50,7 +50,6 @@ dependencies = [
 "linux-loader",
 "log",
 "serde",
- "serde_derive",
 "thiserror",
 "versionize",
 "versionize_derive",
@@ -131,24 +130,33 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"

 [[package]]
 name = "clap"
-version = "3.1.8"
+version = "3.1.18"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "71c47df61d9e16dc010b55dba1952a57d8c215dbb533fd13cdd13369aac73b1c"
+checksum = "d2dbdf4bdacb33466e854ce889eee8dfd5729abf7ccd7664d0a2d60cd384440b"
 dependencies = [
 "atty",
 "bitflags",
+ "clap_lex",
 "indexmap",
 "lazy_static",
- "os_str_bytes",
 "strsim",
 "termcolor",
 "terminal_size",
 "textwrap",
 ]

+[[package]]
+name = "clap_lex"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a37c35f1112dad5e6e0b1adaff798507497a18fceeb30cceb3bae7d1427b9213"
+dependencies = [
+ "os_str_bytes",
+]
+
 [[package]]
 name = "cloud-hypervisor"
-version = "23.0.0"
+version = "24.0.0"
 dependencies = [
 "anyhow",
 "api_client",
@@ -257,7 +265,6 @@ version = "0.1.0"
 dependencies = [
 "libc",
 "serde",
- "serde_derive",
 "serde_json",
 ]

@@ -344,7 +351,6 @@ dependencies = [
 "mshv-bindings",
 "mshv-ioctls",
 "serde",
- "serde_derive",
 "serde_json",
 "thiserror",
 "vm-memory",
@@ -400,10 +406,19 @@ dependencies = [
 ]

 [[package]]
-name = "itoa"
-version = "1.0.1"
+name = "ipnetwork"
+version = "0.19.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1aab8fc367588b89dcee83ab0fd66b72b50b72fa1904d7095045ace2b0c81c35"
+checksum = "1f84f1612606f3753f205a4e9a2efd6fe5b4c573a6269b2cc6c3003d44a0d127"
+dependencies = [
+ "serde",
+]
+
+[[package]]
+name = "itoa"
+version = "1.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "112c678d4050afce233f4f2852bb2eb519230b3cf12f33585275537d7e41578d"

 [[package]]
 name = "kvm-bindings"
@@ -433,9 +448,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"

 [[package]]
 name = "libc"
-version = "0.2.123"
+version = "0.2.126"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cb691a747a7ab48abc15c5b42066eaafde10dc427e3b6ee2a1cf43db04c763bd"
+checksum = "349d5a591cd28b49e1d1037471617a32ddcda5731b99419008085f72d5a53836"

 [[package]]
 name = "libssh2-sys"
@@ -453,9 +468,9 @@ dependencies = [

 [[package]]
 name = "libz-sys"
-version = "1.1.5"
+version = "1.1.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6f35facd4a5673cb5a48822be2be1d4236c1c99cb4113cab7061ac720d5bf859"
+checksum = "92e7e15d7610cce1d9752e137625f14e61a28cd45929b6e12e47b50fe154ee2e"
 dependencies = [
 "cc",
 "libc",
@@ -484,9 +499,9 @@ dependencies = [

 [[package]]
 name = "log"
-version = "0.4.16"
+version = "0.4.17"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6389c490849ff5bc16be905ae24bc913a9c8892e19b2341dbc175e14c341c2b8"
+checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e"
 dependencies = [
 "cfg-if",
 ]
@@ -499,9 +514,9 @@ checksum = "0ca88d725a0a943b096803bd34e73a4437208b6077654cc4ecb2947a5f91618d"

 [[package]]
 name = "memchr"
-version = "2.4.1"
+version = "2.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "308cc39be01b73d0d18f82a0e7b2a3df85245f84af96fdddc5d202d27e47b86a"
+checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d"

 [[package]]
 name = "micro_http"
@@ -515,7 +530,7 @@ dependencies = [
 [[package]]
 name = "mshv-bindings"
 version = "0.1.0"
-source = "git+https://github.com/rust-vmm/mshv?branch=main#75cf309d566c3d9ba91e81582a7864032ecc5bbb"
+source = "git+https://github.com/rust-vmm/mshv?branch=main#67e13faa36989a9226ad21c9ff3947e1d5738a54"
 dependencies = [
 "libc",
 "serde",
@@ -527,7 +542,7 @@ dependencies = [
 [[package]]
 name = "mshv-ioctls"
 version = "0.1.0"
-source = "git+https://github.com/rust-vmm/mshv?branch=main#75cf309d566c3d9ba91e81582a7864032ecc5bbb"
+source = "git+https://github.com/rust-vmm/mshv?branch=main#67e13faa36989a9226ad21c9ff3947e1d5738a54"
 dependencies = [
 "libc",
 "mshv-bindings",
@@ -565,28 +580,34 @@ dependencies = [
 ]

 [[package]]
-name = "num-traits"
-version = "0.2.14"
+name = "no-std-net"
+version = "0.6.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9a64b1ec5cda2586e284722486d802acf1f7dbdc623e2bfc57e65ca1cd099290"
+checksum = "43794a0ace135be66a25d3ae77d41b91615fb68ae937f904090203e81f755b65"
+
+[[package]]
+name = "num-traits"
+version = "0.2.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd"
 dependencies = [
 "autocfg",
 ]

 [[package]]
 name = "openssl-src"
-version = "111.18.0+1.1.1n"
+version = "111.20.0+1.1.1o"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7897a926e1e8d00219127dc020130eca4292e5ca666dd592480d72c3eca2ff6c"
+checksum = "92892c4f87d56e376e469ace79f1128fdaded07646ddf73aa0be4706ff712dec"
 dependencies = [
 "cc",
 ]

 [[package]]
 name = "openssl-sys"
-version = "0.9.72"
+version = "0.9.73"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7e46109c383602735fa0a2e48dd2b7c892b048e1bf69e5c3b1d804b7d9c203cb"
+checksum = "9d5fd19fb3e0a8191c1e34935718976a3e70c112ab9a24af6d7cadccd9d90bc0"
 dependencies = [
 "autocfg",
 "cc",
@@ -602,12 +623,9 @@ version = "0.1.0"

 [[package]]
 name = "os_str_bytes"
-version = "6.0.0"
+version = "6.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8e22443d1643a904602595ba1cd8f7d896afe56d26712531c5ff73a15b2fbf64"
-dependencies = [
- "memchr",
-]
+checksum = "029d8d0b2f198229de29dca79676f2738ff952edf3fde542eb8bf94d8c21b435"

 [[package]]
 name = "parking_lot"
@@ -650,7 +668,6 @@ dependencies = [
 "libc",
 "log",
 "serde",
- "serde_derive",
 "thiserror",
 "versionize",
 "versionize_derive",
@@ -671,7 +688,6 @@ dependencies = [
 "clap",
 "dirs",
 "serde",
- "serde_derive",
 "serde_json",
 "test_infra",
 "thiserror",
@@ -686,11 +702,11 @@ checksum = "1df8c4ec4b0627e53bdf214615ad287367e482558cf84b109250b37464dc03ae"

 [[package]]
 name = "pnet"
-version = "0.29.0"
+version = "0.30.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8750e073f82219c01e771133c64718d7685aef922da8a0d430a46aed05b6341a"
+checksum = "d5cc57672f576f6b95370277fb445738d4887195c6cf4192bdf4f44697e2389b"
 dependencies = [
- "ipnetwork",
+ "ipnetwork 0.19.0",
 "pnet_base",
 "pnet_datalink",
 "pnet_packet",
@@ -700,17 +716,20 @@ dependencies = [

 [[package]]
 name = "pnet_base"
-version = "0.29.0"
+version = "0.30.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8205fe084bd43a3af79b3155c19feddd62e733640498842e631a2ffe107d1538"
+checksum = "2e88341c6c842f89bdc7287f7b1e26b6fa64fa11c7ea3756971e6f18cd2510c4"
+dependencies = [
+ "no-std-net",
+]

 [[package]]
 name = "pnet_datalink"
-version = "0.29.0"
+version = "0.30.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6f85aef5e52e22ff06b1b11f2eb6d52959a9e0ecad3cb3f5cc2d78cadc077f0e"
+checksum = "bc6e55d71c51db73372db35bc54f43abd8460adff1c3a9b717804ca6416d5df2"
 dependencies = [
- "ipnetwork",
+ "ipnetwork 0.18.0",
 "libc",
 "pnet_base",
 "pnet_sys",
@@ -719,9 +738,9 @@ dependencies = [

 [[package]]
 name = "pnet_macros"
-version = "0.29.0"
+version = "0.30.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "98cc3af95fed6dc318dfede3e81320f96ad5e237c6f7c4688108b19c8e67432d"
+checksum = "ebfcdc9c072966723026b3596a1f655fb8bbfe0142f9770f8d481aee4459d6b9"
 dependencies = [
 "proc-macro2",
 "quote",
@@ -731,18 +750,18 @@ dependencies = [

 [[package]]
 name = "pnet_macros_support"
-version = "0.29.0"
+version = "0.30.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "feaba58ba96abb218ec584d6caf0d3ff48922df05dbbeb1560553c197091b29e"
+checksum = "bba532f5a4b320c029d89e612671fb621851b3b07e972c53850d34130033a5cd"
 dependencies = [
 "pnet_base",
 ]

 [[package]]
 name = "pnet_packet"
-version = "0.29.0"
+version = "0.30.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f246edaaf1aaf82072d4cd38ee18bcc5dfc0464093f9ca39e4ac5962d68cf9d4"
+checksum = "7009716ac86091c1b6e2cdec95a2b028c880f516054c1ec11edd02f9f463cbde"
 dependencies = [
 "glob",
 "pnet_base",
@@ -752,9 +771,9 @@ dependencies = [

 [[package]]
 name = "pnet_sys"
-version = "0.29.0"
+version = "0.30.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "028c87a5e3a48fc07df099a2025f2ef16add5993712e1494ba69a6707ee7ed06"
+checksum = "8e2a05efbc55c22f664c0ea475fbc4ffc4d09346aff9068438279d7e3d431f6f"
 dependencies = [
 "libc",
 "winapi",
@@ -762,9 +781,9 @@ dependencies = [

 [[package]]
 name = "pnet_transport"
-version = "0.29.0"
+version = "0.30.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "950f2a7961e19d22e19e84ff0a6e0955013185fe149673499662633d02b41b7a"
+checksum = "75b8ff06f37863f7590183f7044ab2e8d4dae991ecea0c791e3c6dd61ed2913d"
 dependencies = [
 "libc",
 "pnet_base",
@@ -774,11 +793,11 @@ dependencies = [

 [[package]]
 name = "proc-macro2"
-version = "1.0.37"
+version = "1.0.39"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ec757218438d5fda206afc041538b2f6d889286160d649a86a24d37e1235afd1"
+checksum = "c54b25569025b7fc9651de43004ae593a75ad88543b17178aa5e1b9c4f15f56f"
 dependencies = [
- "unicode-xid",
+ "unicode-ident",
 ]

 [[package]]
@@ -832,9 +851,9 @@ dependencies = [

 [[package]]
 name = "regex"
-version = "1.5.5"
+version = "1.5.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1a11647b6b25ff05a515cb92c365cec08801e83423a235b51e231e1808747286"
+checksum = "d83f127d94bdbcda4c8cc2e50f6f84f4b611f69c902699ca385a39c3a75f9ff1"
 dependencies = [
 "aho-corasick",
 "memchr",
@@ -843,15 +862,15 @@ dependencies = [

 [[package]]
 name = "regex-syntax"
-version = "0.6.25"
+version = "0.6.26"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f497285884f3fcff424ffc933e56d7cbca511def0c9831a7f9b5f6153e3cc89b"
+checksum = "49b3de9ec5dc0a3417da371aab17d729997c15010e7fd24ff707773a33bddb64"

 [[package]]
 name = "remain"
-version = "0.2.2"
+version = "0.2.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "70ba1e78fa68412cb93ef642fd4d20b9a941be49ee9333875ebaf13112673ea7"
+checksum = "0c35270ea384ac1762895831cc8acb96f171468e52cec82ed9186f9416209fa4"
 dependencies = [
 "proc-macro2",
 "quote",
@@ -869,9 +888,9 @@ dependencies = [

 [[package]]
 name = "ryu"
-version = "1.0.9"
+version = "1.0.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "73b4b750c782965c211b42f022f59af1fbceabdd026623714f104152f1ec149f"
+checksum = "f3f6f92acf49d1b98f7a81226834412ada05458b7364277387724a237f062695"

 [[package]]
 name = "scopeguard"
@@ -890,21 +909,24 @@ dependencies = [

 [[package]]
 name = "semver"
-version = "1.0.7"
+version = "1.0.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d65bd28f48be7196d222d95b9243287f48d27aca604e08497513019ff0502cc4"
+checksum = "8cb243bdfdb5936c8dc3c45762a19d12ab4550cdc753bc247637d4ec35a040fd"

 [[package]]
 name = "serde"
-version = "1.0.136"
+version = "1.0.137"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ce31e24b01e1e524df96f1c2fdd054405f8d7376249a5110886fb4b658484789"
+checksum = "61ea8d54c77f8315140a05f4c7237403bf38b72704d031543aa1d16abbf517d1"
+dependencies = [
+ "serde_derive",
+]

 [[package]]
 name = "serde_derive"
-version = "1.0.136"
+version = "1.0.137"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "08597e7152fcd306f41838ed3e37be9eaeed2b61c42e2117266a554fab4662f9"
+checksum = "1f26faba0c3959972377d3b2d306ee9f71faee9714294e41bb777f83f88578be"
 dependencies = [
 "proc-macro2",
 "quote",
@@ -913,9 +935,9 @@ dependencies = [

 [[package]]
 name = "serde_json"
-version = "1.0.79"
+version = "1.0.81"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8e8d9fa5c3b304765ce1fd9c4c8a3de2c8db365a5b91be52f186efc675681d95"
+checksum = "9b7ce2b32a1aed03c558dc61a5cd328f15aff2dbc17daad8fb8af04d2100e15c"
 dependencies = [
 "itoa",
 "ryu",
@@ -924,9 +946,9 @@ dependencies = [

 [[package]]
 name = "signal-hook"
-version = "0.3.13"
+version = "0.3.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "647c97df271007dcea485bb74ffdb57f2e683f1306c854f468a0c244badabf2d"
+checksum = "a253b5e89e2698464fc26b545c9edceb338e18a89effeeecfea192c3025be29d"
 dependencies = [
 "libc",
 "signal-hook-registry",
@@ -973,13 +995,13 @@ checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623"

 [[package]]
 name = "syn"
-version = "1.0.91"
+version = "1.0.95"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b683b2b825c8eef438b77c36a06dc262294da3d5a5813fac20da149241dcd44d"
+checksum = "fbaf6116ab8924f39d52792136fb74fd60a80194cf1b1c6ffa6453eef1c3f942"
 dependencies = [
 "proc-macro2",
 "quote",
- "unicode-xid",
+ "unicode-ident",
 ]

 [[package]]
@@ -1037,18 +1059,18 @@ dependencies = [

 [[package]]
 name = "thiserror"
-version = "1.0.30"
+version = "1.0.31"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "854babe52e4df1653706b98fcfc05843010039b406875930a70e4d9644e5c417"
+checksum = "bd829fe32373d27f76265620b5309d0340cb8550f523c1dda251d6298069069a"
 dependencies = [
 "thiserror-impl",
 ]

 [[package]]
 name = "thiserror-impl"
-version = "1.0.30"
+version = "1.0.31"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "aa32fd3f627f367fe16f893e2597ae3c05020f8bba2666a4e6ea73d377e5714b"
+checksum = "0396bc89e626244658bef819e22d0cc459e795a5ebe878e6ec336d1674a8d79a"
 dependencies = [
 "proc-macro2",
 "quote",
@@ -1056,16 +1078,22 @@ dependencies = [
 ]

 [[package]]
-name = "unicode-xid"
-version = "0.2.2"
+name = "unicode-ident"
+version = "1.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8ccb82d61f80a663efe1f787a51b16b5a51e3314d6ac365b08639f52387b33f3"
+checksum = "d22af068fba1eb5edcb4aea19d382b2a3deb4c8f9d475c589b6ada9e0fd493ee"
+
+[[package]]
+name = "unicode-xid"
+version = "0.2.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "957e51f3646910546462e67d5f7599b9e4fb8acdd304b087a6494730f9eebf04"

 [[package]]
 name = "uuid"
-version = "0.8.2"
+version = "1.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bc5cf98d8186244414c848017f0e2676b3fcb46807f6668a97dfe67359a3c4b7"
+checksum = "8cfcd319456c4d6ea10087ed423473267e1a071f3bc0aa89f80d60997843c6f0"
 dependencies = [
 "getrandom",
 ]
@@ -1175,8 +1203,7 @@ dependencies = [
 [[package]]
 name = "vhost-user-backend"
 version = "0.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1490f2028d4f119b2292efe218b5f8cfc6471f039b53b6a6eb5d9513e964facc"
+source = "git+https://github.com/rust-vmm/vhost-user-backend?rev=14f58eda14076e973704d4f904850be1146fbb05#14f58eda14076e973704d4f904850be1146fbb05"
 dependencies = [
 "libc",
 "log",
@@ -1249,7 +1276,6 @@ dependencies = [
 "rate_limiter",
 "seccompiler",
 "serde",
- "serde_derive",
 "serde_json",
 "thiserror",
 "versionize",
@@ -1267,9 +1293,9 @@ dependencies = [

 [[package]]
 name = "virtio-queue"
-version = "0.2.0"
+version = "0.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3785325315e6496fa88673842ee6cd198b9658e88e8b0e1ad48a5dc818b221dc"
+checksum = "88f2d73c184c18f8acc32dab77fcb6e3af92d53262538d3a68aa474810d6863c"
 dependencies = [
 "log",
 "vm-memory",
@@ -1290,8 +1316,8 @@ name = "vm-device"
 version = "0.1.0"
 dependencies = [
 "anyhow",
+ "hypervisor",
 "serde",
- "serde_derive",
 "serde_json",
 "thiserror",
 "vfio-ioctls",
@@ -1306,9 +1332,9 @@ source = "git+https://github.com/rust-vmm/vm-fdt?branch=main#ca35d96191f8232bd7a

 [[package]]
 name = "vm-memory"
-version = "0.7.0"
+version = "0.8.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "339d4349c126fdcd87e034631d7274370cf19eb0e87b33166bcd956589fc72c5"
+checksum = "767ed8aaebbff902e02e6d3749dc2baef55e46565f8a6414a065e5baee4b4a81"
 dependencies = [
 "arc-swap",
 "libc",
@@ -1321,7 +1347,6 @@ version = "0.1.0"
 dependencies = [
 "anyhow",
 "serde",
- "serde_derive",
 "serde_json",
 "thiserror",
 "versionize",
@@ -1367,7 +1392,6 @@ dependencies = [
 "qcow",
 "seccompiler",
 "serde",
- "serde_derive",
 "serde_json",
 "signal-hook",
 "thiserror",
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "cloud-hypervisor"
-version = "23.0.0"
+version = "24.0.0"
 authors = ["The Cloud Hypervisor Authors"]
 edition = "2021"
 default-run = "cloud-hypervisor"
@@ -10,31 +10,31 @@ description = "Open source Virtual Machine Monitor (VMM) that runs on top of KVM
 homepage = "https://github.com/cloud-hypervisor/cloud-hypervisor"
 # Minimum buildable version:
 # Keep in sync with version in .github/workflows/build.yaml
-rust-version = "1.56"
+rust-version = "1.60"

 [profile.release]
 lto = true

 [dependencies]
-anyhow = "1.0.56"
+anyhow = "1.0.57"
 api_client = { path = "api_client" }
-clap = { version = "3.1.8", features = ["wrap_help","cargo"] }
+clap = { version = "3.1.18", features = ["wrap_help","cargo"] }
 epoll = "4.3.1"
 event_monitor = { path = "event_monitor" }
 hypervisor = { path = "hypervisor" }
-libc = "0.2.123"
-log = { version = "0.4.16", features = ["std"] }
+libc = "0.2.126"
+log = { version = "0.4.17", features = ["std"] }
 option_parser = { path = "option_parser" }
 seccompiler = "0.2.0"
-serde_json = "1.0.79"
-signal-hook = "0.3.13"
-thiserror = "1.0.30"
+serde_json = "1.0.81"
+signal-hook = "0.3.14"
+thiserror = "1.0.31"
 vmm = { path = "vmm" }
 vmm-sys-util = "0.9.0"
-vm-memory = "0.7.0"
+vm-memory = "0.8.0"

 [build-dependencies]
-clap = { version = "3.1.8", features = ["cargo"] }
+clap = { version = "3.1.18", features = ["cargo"] }

 # List of patched crates
 [patch.crates-io]
@@ -46,7 +46,7 @@ versionize_derive = { git = "https://github.com/cloud-hypervisor/versionize_deri
 dirs = "4.0.0"
 lazy_static= "1.4.0"
 net_util = { path = "net_util" }
-serde_json = "1.0.79"
+serde_json = "1.0.81"
 test_infra = { path = "test_infra" }
 wait-timeout = "0.2.0"

--- a/115
+++ b/115
@@ -153,88 +153,6 @@ pipeline{
 						}
 					}
 				}
-				stage ('Worker build SGX') {
-					agent { node { label 'bionic-sgx' } }
-					when {
-						beforeAgent true
-						allOf {
-							branch 'main'
-							expression {
-								return runWorkers
-							}
-						}
-					}
-					stages {
-						stage ('Checkout') {
-							steps {
-								checkout scm
-							}
-						}
-						stage ('Run SGX integration tests') {
-							options {
-								timeout(time: 1, unit: 'HOURS')
-							}
-							steps {
-								sh "scripts/dev_cli.sh tests --integration-sgx"
-							}
-						}
-						stage ('Run SGX integration tests for musl') {
-							options {
-								timeout(time: 1, unit: 'HOURS')
-							}
-							steps {
-								sh "scripts/dev_cli.sh tests --integration-sgx --libc musl"
-							}
-						}
-					}
-					post {
-						always {
-							sh "sudo chown -R jenkins.jenkins ${WORKSPACE}"
-							deleteDir()
-						}
-					}
-				}
-				stage ('Worker build VFIO') {
-					agent { node { label 'bionic-vfio' } }
-					when {
-						beforeAgent true
-						allOf {
-							branch 'main'
-							expression {
-								return runWorkers
-							}
-						}
-					}
-					stages {
-						stage ('Checkout') {
-							steps {
-								checkout scm
-							}
-						}
-						stage ('Run VFIO integration tests') {
-							options {
-								timeout(time: 1, unit: 'HOURS')
-							}
-							steps {
-								sh "scripts/dev_cli.sh tests --integration-vfio"
-							}
-						}
-						stage ('Run VFIO integration tests for musl') {
-							options {
-								timeout(time: 1, unit: 'HOURS')
-							}
-							steps {
-								sh "scripts/dev_cli.sh tests --integration-vfio --libc musl"
-							}
-						}
-					}
-					post {
-						always {
-							sh "sudo chown -R jenkins.jenkins ${WORKSPACE}"
-							deleteDir()
-						}
-					}
-				}
 				stage ('Worker build - Windows guest') {
 					agent { node { label 'focal' } }
 					when {
@@ -315,39 +233,6 @@ pipeline{
 						}
 					}
 				}
-				stage ('Worker build - Metrics') {
-					agent { node { label 'focal-metrics' } }
-					when {
-						branch 'main'
-						beforeAgent true
-						expression {
-							return runWorkers
-						}
-					}
-					environment {
-						METRICS_PUBLISH_KEY = credentials('52e0945f-ce7a-43d1-87af-67d1d87cc40f')
-					}
-					stages {
-						stage ('Checkout') {
-							steps {
-								checkout scm
-							}
-						}
-						stage ('Run metrics tests') {
-							options {
-								timeout(time: 1, unit: 'HOURS')
-							}
-							steps {
-								sh 'scripts/dev_cli.sh tests --metrics -- -- --report-file /root/workloads/metrics.json'
-							}
-						}
-						stage ('Upload metrics report') {
-							steps {
-								sh 'curl -X PUT https://cloud-hypervisor-metrics.azurewebsites.net/api/publishmetrics -H "x-functions-key: $METRICS_PUBLISH_KEY" -T ~/workloads/metrics.json'
-							}
-						}
-					}
-				}
 			}
 		}
 	}
--- a/README.md
+++ b/README.md
@@ -88,12 +88,10 @@ Hypervisor. Here, all the steps are based on Ubuntu, for other Linux
 distributions please replace the package manager and package name.

 ```shell
-# Install git
-$ sudo apt install git
+# Install build-essential, git, and qemu-img
+$ sudo apt install git build-essential qemu-img
 # Install rust tool chain
 $ curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh
-# Install build-essential
-$ sudo apt install build-essential
 # If you want to build statically linked binary please add musl target
 $ rustup target add x86_64-unknown-linux-musl
 ```
@@ -168,7 +166,7 @@ cloud image. Here we will use a Ubuntu image:
 $ pushd $CLOUDH
 $ wget https://cloud-images.ubuntu.com/focal/current/focal-server-cloudimg-amd64.img
 $ qemu-img convert -p -f qcow2 -O raw focal-server-cloudimg-amd64.img focal-server-cloudimg-amd64.raw
-$ wget https://github.com/cloud-hypervisor/rust-hypervisor-firmware/releases/download/0.3.2/hypervisor-fw
+$ wget https://github.com/cloud-hypervisor/rust-hypervisor-firmware/releases/download/0.4.0/hypervisor-fw
 $ popd
 ```

@@ -291,6 +289,8 @@ As of 2022-04-05, the following cloud images are supported:
 Direct kernel boot to userspace should work with a rootfs from most
 distributions.

+Further details can be found in the [release documentation](docs/releases.md).
+
 ## Hot Plug

 Cloud Hypervisor supports hotplug of CPUs, passthrough devices (VFIO),
--- a/acpi_tables/Cargo.toml
+++ b/acpi_tables/Cargo.toml
@@ -5,4 +5,4 @@ authors = ["The Cloud Hypervisor Authors"]
 edition = "2021"

 [dependencies]
-vm-memory = "0.7.0"
+vm-memory = "0.8.0"
--- a/arch/Cargo.toml
+++ b/arch/Cargo.toml
@@ -10,18 +10,17 @@ tdx = []

 [dependencies]
 acpi_tables = { path = "../acpi_tables" }
-anyhow = "1.0.56"
+anyhow = "1.0.57"
 byteorder = "1.4.3"
 hypervisor = { path = "../hypervisor" }
-libc = "0.2.123"
+libc = "0.2.126"
 linux-loader = { version = "0.4.0", features = ["elf", "bzimage", "pe"] }
-log = "0.4.16"
-serde = { version = "1.0.136", features = ["rc"] }
-serde_derive = "1.0.136"
-thiserror = "1.0.30"
+log = "0.4.17"
+serde = { version = "1.0.137", features = ["rc", "derive"] }
+thiserror = "1.0.31"
 versionize = "0.1.6"
 versionize_derive = "0.1.4"
-vm-memory = { version = "0.7.0", features = ["backend-mmap", "backend-bitmap"] }
+vm-memory = { version = "0.8.0", features = ["backend-mmap", "backend-bitmap"] }
 vm-migration = { path = "../vm-migration" }
 vmm-sys-util = { version = "0.9.0", features = ["with-serde"] }

--- a/arch/src/aarch64/mod.rs
+++ b/arch/src/aarch64/mod.rs
@@ -50,7 +50,7 @@ pub enum Error {

 impl From<Error> for super::Error {
    fn from(e: Error) -> super::Error {
-        super::Error::AArch64Setup(e)
+        super::Error::PlatformSpecific(e)
    }
 }

@@ -64,16 +64,16 @@ pub struct EntryPoint {

 /// Configure the specified VCPU, and return its MPIDR.
 pub fn configure_vcpu(
-    fd: &Arc<dyn hypervisor::Vcpu>,
+    vcpu: &Arc<dyn hypervisor::Vcpu>,
    id: u8,
    kernel_entry_point: Option<EntryPoint>,
 ) -> super::Result<u64> {
    if let Some(kernel_entry_point) = kernel_entry_point {
-        regs::setup_regs(fd, id, kernel_entry_point.entry_addr.raw_value())
+        regs::setup_regs(vcpu, id, kernel_entry_point.entry_addr.raw_value())
            .map_err(Error::RegsConfiguration)?;
    }

-    let mpidr = fd.read_mpidr().map_err(Error::VcpuRegMpidr)?;
+    let mpidr = vcpu.read_mpidr().map_err(Error::VcpuRegMpidr)?;
    Ok(mpidr)
 }

@@ -185,10 +185,10 @@ pub fn initramfs_load_addr(
            if guest_mem.address_in_range(offset) {
                Ok(offset.raw_value())
            } else {
-                Err(super::Error::AArch64Setup(Error::InitramfsAddress))
+                Err(super::Error::PlatformSpecific(Error::InitramfsAddress))
            }
        }
-        None => Err(super::Error::AArch64Setup(Error::InitramfsAddress)),
+        None => Err(super::Error::PlatformSpecific(Error::InitramfsAddress)),
    }
 }

--- a/arch/src/lib.rs
+++ b/arch/src/lib.rs
@@ -9,15 +9,15 @@

 #[macro_use]
 extern crate log;
-#[macro_use]
-extern crate serde_derive;

 #[cfg(target_arch = "x86_64")]
 use crate::x86_64::SgxEpcSection;
+use serde::{Deserialize, Serialize};
 use std::collections::BTreeMap;
 use std::fmt;
 use std::result;
 use std::sync::Arc;
+use thiserror::Error;
 use versionize::{VersionMap, Versionize, VersionizeError, VersionizeResult};
 use versionize_derive::Versionize;
 use vm_migration::VersionMapped;
@@ -26,31 +26,27 @@ type GuestMemoryMmap = vm_memory::GuestMemoryMmap<vm_memory::bitmap::AtomicBitma
 type GuestRegionMmap = vm_memory::GuestRegionMmap<vm_memory::bitmap::AtomicBitmap>;

 /// Type for returning error code.
-#[derive(Debug)]
+#[derive(Debug, Error)]
 pub enum Error {
    #[cfg(target_arch = "x86_64")]
-    /// X86_64 specific error triggered during system configuration.
-    X86_64Setup(x86_64::Error),
+    #[error("Platform specific error (x86_64): {0:?}")]
+    PlatformSpecific(x86_64::Error),
    #[cfg(target_arch = "aarch64")]
-    /// AArch64 specific error triggered during system configuration.
-    AArch64Setup(aarch64::Error),
-    /// The zero page extends past the end of guest_mem.
-    ZeroPagePastRamEnd,
-    /// Error writing the zero page of guest memory.
-    ZeroPageSetup(vm_memory::GuestMemoryError),
-    /// The memory map table extends past the end of guest memory.
+    #[error("Platform specific error (aarch64): {0:?}")]
+    PlatformSpecific(aarch64::Error),
+    #[error("The memory map table extends past the end of guest memory")]
    MemmapTablePastRamEnd,
-    /// Error writing memory map table to guest memory.
+    #[error("Error writing memory map table to guest memory")]
    MemmapTableSetup,
-    /// The hvm_start_info structure extends past the end of guest memory.
+    #[error("The hvm_start_info structure extends past the end of guest memory")]
    StartInfoPastRamEnd,
-    /// Error writing hvm_start_info to guest memory.
+    #[error("Error writing hvm_start_info to guest memory")]
    StartInfoSetup,
-    /// Failed to compute initramfs address.
+    #[error("Failed to compute initramfs address")]
    InitramfsAddress,
-    /// Error writing module entry to guest memory.
-    ModlistSetup(vm_memory::GuestMemoryError),
-    /// RSDP Beyond Guest Memory
+    #[error("Error writing module entry to guest memory: {0}")]
+    ModlistSetup(#[source] vm_memory::GuestMemoryError),
+    #[error("RSDP extends past the end of guest memory")]
    RsdpPastRamEnd,
 }

--- a/arch/src/x86_64/mod.rs
+++ b/arch/src/x86_64/mod.rs
@@ -15,7 +15,8 @@ pub mod regs;
 use crate::GuestMemoryMmap;
 use crate::InitramfsConfig;
 use crate::RegionType;
-use hypervisor::{CpuId, CpuIdEntry, HypervisorError, CPUID_FLAG_VALID_INDEX};
+use hypervisor::x86_64::{CpuId, CpuIdEntry, CPUID_FLAG_VALID_INDEX};
+use hypervisor::HypervisorError;
 use linux_loader::loader::bootparam::boot_params;
 use linux_loader::loader::elf::start_info::{
    hvm_memmap_table_entry, hvm_modlist_entry, hvm_start_info,
@@ -197,7 +198,7 @@ pub enum Error {

 impl From<Error> for super::Error {
    fn from(e: Error) -> super::Error {
-        super::Error::X86_64Setup(e)
+        super::Error::PlatformSpecific(e)
    }
 }

@@ -743,7 +744,7 @@ pub fn generate_common_cpuid(
 }

 pub fn configure_vcpu(
-    fd: &Arc<dyn hypervisor::Vcpu>,
+    vcpu: &Arc<dyn hypervisor::Vcpu>,
    id: u8,
    kernel_entry_point: Option<EntryPoint>,
    vm_memory: &GuestMemoryAtomic<GuestMemoryMmap>,
@@ -755,23 +756,23 @@ pub fn configure_vcpu(
    CpuidPatch::set_cpuid_reg(&mut cpuid, 0xb, None, CpuidReg::EDX, u32::from(id));
    CpuidPatch::set_cpuid_reg(&mut cpuid, 0x1f, None, CpuidReg::EDX, u32::from(id));

-    fd.set_cpuid2(&cpuid)
+    vcpu.set_cpuid2(&cpuid)
        .map_err(|e| Error::SetSupportedCpusFailed(e.into()))?;

    if kvm_hyperv {
-        fd.enable_hyperv_synic().unwrap();
+        vcpu.enable_hyperv_synic().unwrap();
    }

-    regs::setup_msrs(fd).map_err(Error::MsrsConfiguration)?;
+    regs::setup_msrs(vcpu).map_err(Error::MsrsConfiguration)?;
    if let Some(kernel_entry_point) = kernel_entry_point {
        if let Some(entry_addr) = kernel_entry_point.entry_addr {
            // Safe to unwrap because this method is called after the VM is configured
-            regs::setup_regs(fd, entry_addr.raw_value()).map_err(Error::RegsConfiguration)?;
-            regs::setup_fpu(fd).map_err(Error::FpuConfiguration)?;
-            regs::setup_sregs(&vm_memory.memory(), fd).map_err(Error::SregsConfiguration)?;
+            regs::setup_regs(vcpu, entry_addr.raw_value()).map_err(Error::RegsConfiguration)?;
+            regs::setup_fpu(vcpu).map_err(Error::FpuConfiguration)?;
+            regs::setup_sregs(&vm_memory.memory(), vcpu).map_err(Error::SregsConfiguration)?;
        }
    }
-    interrupts::set_lint(fd).map_err(|e| Error::LocalIntConfiguration(e.into()))?;
+    interrupts::set_lint(vcpu).map_err(|e| Error::LocalIntConfiguration(e.into()))?;
    Ok(())
 }

@@ -838,13 +839,14 @@ pub fn configure_system(
    _num_cpus: u8,
    rsdp_addr: Option<GuestAddress>,
    sgx_epc_region: Option<SgxEpcRegion>,
+    serial_number: Option<&str>,
 ) -> super::Result<()> {
    // Write EBDA address to location where ACPICA expects to find it
    guest_mem
        .write_obj((layout::EBDA_START.0 >> 4) as u16, layout::EBDA_POINTER)
        .map_err(Error::EbdaSetup)?;

-    let size = smbios::setup_smbios(guest_mem).map_err(Error::SmbiosSetup)?;
+    let size = smbios::setup_smbios(guest_mem, serial_number).map_err(Error::SmbiosSetup)?;

    // Place the MP table after the SMIOS table aligned to 16 bytes
    let offset = GuestAddress(layout::SMBIOS_START).unchecked_add(size);
@@ -1193,6 +1195,7 @@ mod tests {
            1,
            Some(layout::RSDP_POINTER),
            None,
+            None,
        );
        assert!(config_err.is_err());

@@ -1206,7 +1209,7 @@ mod tests {
            .collect();
        let gm = GuestMemoryMmap::from_ranges(&ram_regions).unwrap();

-        configure_system(&gm, GuestAddress(0), &None, no_vcpus, None, None).unwrap();
+        configure_system(&gm, GuestAddress(0), &None, no_vcpus, None, None, None).unwrap();

        // Now assigning some memory that is equal to the start of the 32bit memory hole.
        let mem_size = 3328 << 20;
@@ -1217,9 +1220,9 @@ mod tests {
            .map(|r| (r.0, r.1))
            .collect();
        let gm = GuestMemoryMmap::from_ranges(&ram_regions).unwrap();
-        configure_system(&gm, GuestAddress(0), &None, no_vcpus, None, None).unwrap();
+        configure_system(&gm, GuestAddress(0), &None, no_vcpus, None, None, None).unwrap();

-        configure_system(&gm, GuestAddress(0), &None, no_vcpus, None, None).unwrap();
+        configure_system(&gm, GuestAddress(0), &None, no_vcpus, None, None, None).unwrap();

        // Now assigning some memory that falls after the 32bit memory hole.
        let mem_size = 3330 << 20;
@@ -1230,9 +1233,9 @@ mod tests {
            .map(|r| (r.0, r.1))
            .collect();
        let gm = GuestMemoryMmap::from_ranges(&ram_regions).unwrap();
-        configure_system(&gm, GuestAddress(0), &None, no_vcpus, None, None).unwrap();
+        configure_system(&gm, GuestAddress(0), &None, no_vcpus, None, None, None).unwrap();

-        configure_system(&gm, GuestAddress(0), &None, no_vcpus, None, None).unwrap();
+        configure_system(&gm, GuestAddress(0), &None, no_vcpus, None, None, None).unwrap();
    }

    #[test]
--- a/arch/src/x86_64/regs.rs
+++ b/arch/src/x86_64/regs.rs
@@ -9,7 +9,7 @@
 use crate::layout::{BOOT_GDT_START, BOOT_IDT_START, PVH_INFO_START};
 use crate::GuestMemoryMmap;
 use hypervisor::arch::x86::gdt::{gdt_entry, segment_from_gdt};
-use hypervisor::arch::x86::regs::*;
+use hypervisor::arch::x86::regs::CR0_PE;
 use hypervisor::x86_64::{FpuState, SpecialRegisters, StandardRegisters};
 use std::sync::Arc;
 use std::{mem, result};
--- a/arch/src/x86_64/smbios.rs
+++ b/arch/src/x86_64/smbios.rs
@@ -162,7 +162,7 @@ fn write_string(
    Ok(curptr)
 }

-pub fn setup_smbios(mem: &GuestMemoryMmap) -> Result<u64> {
+pub fn setup_smbios(mem: &GuestMemoryMmap, serial_number: Option<&str>) -> Result<u64> {
    let physptr = GuestAddress(SMBIOS_START)
        .checked_add(mem::size_of::<Smbios30Entrypoint>() as u64)
        .ok_or(Error::NotEnoughMemory)?;
@@ -195,11 +195,15 @@ pub fn setup_smbios(mem: &GuestMemoryMmap) -> Result<u64> {
            handle,
            manufacturer: 1, // First string written in this section
            product_name: 2, // Second string written in this section
+            serial_number: serial_number.map(|_| 3).unwrap_or_default(), // 3rd string
            ..Default::default()
        };
        curptr = write_and_incr(mem, smbios_sysinfo, curptr)?;
        curptr = write_string(mem, "Cloud Hypervisor", curptr)?;
        curptr = write_string(mem, "cloud-hypervisor", curptr)?;
+        if let Some(serial_number) = serial_number {
+            curptr = write_string(mem, serial_number, curptr)?;
+        }
        curptr = write_and_incr(mem, 0u8, curptr)?;
    }

@@ -263,7 +267,7 @@ mod tests {
    fn entrypoint_checksum() {
        let mem = GuestMemoryMmap::from_ranges(&[(GuestAddress(SMBIOS_START), 4096)]).unwrap();

-        setup_smbios(&mem).unwrap();
+        setup_smbios(&mem, None).unwrap();

        let smbios_ep: Smbios30Entrypoint = mem.read_obj(GuestAddress(SMBIOS_START)).unwrap();

--- a/block_util/Cargo.toml
+++ b/block_util/Cargo.toml
@@ -9,16 +9,16 @@ default = []

 [dependencies]
 io-uring = "0.5.2"
-libc = "0.2.123"
-log = "0.4.16"
+libc = "0.2.126"
+log = "0.4.17"
 qcow = { path = "../qcow" }
-thiserror = "1.0.30"
+thiserror = "1.0.31"
 versionize = "0.1.6"
 versionize_derive = "0.1.4"
 vhdx = { path = "../vhdx" }
 virtio-bindings = { version = "0.1.0", features = ["virtio-v5_0_0"] }
-virtio-queue = "0.2.0"
-vm-memory = { version = "0.7.0", features = ["backend-mmap", "backend-atomic", "backend-bitmap"] }
+virtio-queue = "0.3.0"
+vm-memory = { version = "0.8.0", features = ["backend-mmap", "backend-atomic", "backend-bitmap"] }
 vm-virtio = { path = "../vm-virtio" }
 vmm-sys-util = "0.9.0"

--- a/devices/Cargo.toml
+++ b/devices/Cargo.toml
@@ -6,17 +6,17 @@ edition = "2021"

 [dependencies]
 acpi_tables = { path = "../acpi_tables" }
-anyhow = "1.0.56"
+anyhow = "1.0.57"
 arch = { path = "../arch" }
 bitflags = "1.3.2"
 byteorder = "1.4.3"
 epoll = "4.3.1"
-libc = "0.2.123"
-log = "0.4.16"
+libc = "0.2.126"
+log = "0.4.17"
 versionize = "0.1.6"
 versionize_derive = "0.1.4"
 vm-device = { path = "../vm-device" }
-vm-memory = "0.7.0"
+vm-memory = "0.8.0"
 vm-migration = { path = "../vm-migration" }
 vmm-sys-util = "0.9.0"

--- a/devices/src/legacy/debug_port.rs
+++ b/devices/src/legacy/debug_port.rs
@@ -0,0 +1,86 @@
+// Copyright © 2022 Intel Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+use std::fmt;
+use std::time::Instant;
+use vm_device::BusDevice;
+
+/// Debug I/O port, see:
+/// https://www.intel.com/content/www/us/en/support/articles/000005500/boards-and-kits.html
+///
+/// Since we're not a physical platform, we can freely assign code ranges for
+/// debugging specific parts of our virtual platform.
+pub enum DebugIoPortRange {
+    Firmware,
+    Bootloader,
+    Kernel,
+    Userspace,
+    Custom,
+}
+
+#[cfg(target_arch = "x86_64")]
+const DEBUG_IOPORT_PREFIX: &str = "Debug I/O port";
+
+#[cfg(target_arch = "x86_64")]
+impl DebugIoPortRange {
+    fn from_u8(value: u8) -> DebugIoPortRange {
+        match value {
+            0x00..=0x1f => DebugIoPortRange::Firmware,
+            0x20..=0x3f => DebugIoPortRange::Bootloader,
+            0x40..=0x5f => DebugIoPortRange::Kernel,
+            0x60..=0x7f => DebugIoPortRange::Userspace,
+            _ => DebugIoPortRange::Custom,
+        }
+    }
+}
+
+#[cfg(target_arch = "x86_64")]
+impl fmt::Display for DebugIoPortRange {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        match self {
+            DebugIoPortRange::Firmware => write!(f, "{}: Firmware", DEBUG_IOPORT_PREFIX),
+            DebugIoPortRange::Bootloader => write!(f, "{}: Bootloader", DEBUG_IOPORT_PREFIX),
+            DebugIoPortRange::Kernel => write!(f, "{}: Kernel", DEBUG_IOPORT_PREFIX),
+            DebugIoPortRange::Userspace => write!(f, "{}: Userspace", DEBUG_IOPORT_PREFIX),
+            DebugIoPortRange::Custom => write!(f, "{}: Custom", DEBUG_IOPORT_PREFIX),
+        }
+    }
+}
+
+pub struct DebugPort {
+    timestamp: Instant,
+}
+
+impl DebugPort {
+    pub fn new(timestamp: Instant) -> Self {
+        Self { timestamp }
+    }
+}
+
+impl BusDevice for DebugPort {
+    fn read(&mut self, _base: u64, _offset: u64, _data: &mut [u8]) {
+        error!("Invalid read to debug port")
+    }
+
+    fn write(
+        &mut self,
+        _base: u64,
+        _offset: u64,
+        data: &[u8],
+    ) -> Option<std::sync::Arc<std::sync::Barrier>> {
+        let elapsed = self.timestamp.elapsed();
+
+        let code = data[0];
+        warn!(
+            "[{} code 0x{:x}] {}.{:>06} seconds",
+            DebugIoPortRange::from_u8(code),
+            code,
+            elapsed.as_secs(),
+            elapsed.as_micros()
+        );
+
+        None
+    }
+}
--- a/devices/src/legacy/mod.rs
+++ b/devices/src/legacy/mod.rs
@@ -6,6 +6,8 @@
 // found in the LICENSE-BSD-3-Clause file.

 mod cmos;
+#[cfg(target_arch = "x86_64")]
+mod debug_port;
 #[cfg(feature = "fwdebug")]
 mod fwdebug;
 #[cfg(target_arch = "aarch64")]
@@ -18,6 +20,8 @@ mod serial;
 mod uart_pl011;

 pub use self::cmos::Cmos;
+#[cfg(target_arch = "x86_64")]
+pub use self::debug_port::DebugPort;
 #[cfg(feature = "fwdebug")]
 pub use self::fwdebug::FwDebugDevice;
 pub use self::i8042::I8042Device;
--- a/devices/src/legacy/uart_pl011.rs
+++ b/devices/src/legacy/uart_pl011.rs
@@ -10,6 +10,7 @@ use crate::{read_le_u32, write_le_u32};
 use std::collections::VecDeque;
 use std::fmt;
 use std::sync::{Arc, Barrier};
+use std::time::Instant;
 use std::{io, result};
 use versionize::{VersionMap, Versionize, VersionizeResult};
 use versionize_derive::Versionize;
@@ -120,6 +121,7 @@ impl Pl011 {
        id: String,
        irq: Arc<dyn InterruptSourceGroup>,
        out: Option<Box<dyn io::Write + Send>>,
+        timestamp: Instant,
    ) -> Self {
        Self {
            id,
@@ -140,7 +142,7 @@ impl Pl011 {
            read_trigger: 1u32,
            irq,
            out,
-            timestamp: std::time::Instant::now(),
+            timestamp,
        }
    }

@@ -495,6 +497,7 @@ mod tests {
            String::from(SERIAL_NAME),
            Arc::new(TestInterrupt::new(intr_evt.try_clone().unwrap())),
            Some(Box::new(pl011_out.clone())),
+            Instant::now(),
        );

        pl011.write(0, UARTDR as u64, &[b'x', b'y']);
@@ -515,6 +518,7 @@ mod tests {
            String::from(SERIAL_NAME),
            Arc::new(TestInterrupt::new(intr_evt.try_clone().unwrap())),
            Some(Box::new(pl011_out)),
+            Instant::now(),
        );

        // write 1 to the interrupt event fd, so that read doesn't block in case the event fd
--- a/docs/io_throttling.md
+++ b/docs/io_throttling.md
@@ -24,8 +24,8 @@ bucket is unbounded in speed which allows for bursts bound in size by
 the amount of tokens available. Once the token bucket is empty,
 consumption speed is bound by the "refill-rate". Similarly, Cloud
 Hypervisor provides another three options for limiting I/O operations,
-i.e., `ops_size` (I/O operations), `bw_one_time_burst` (I/O operations),
-and `bw_refill_time` (ms).
+i.e., `ops_size` (I/O operations), `ops_one_time_burst` (I/O operations),
+and `ops_refill_time` (ms).

 One caveat in the I/O throttling is that every-time the bucket gets
 empty, it will stop I/O operations for a fixed amount of time
--- a/docs/releases.md
+++ b/docs/releases.md
@@ -0,0 +1,103 @@
+# Release Documentation
+
+## Abstract
+
+This document provides guidance to users, downstream maintainers and
+any other consumers of the Cloud Hypervisor project, this document
+describes the release process, release cadence, stability expectations and
+related topics.
+
+## Basic Terms
+
+### Stability
+
+For Cloud Hypervisor the following areas are subject to stability guarantees:
+
+- [REST API](api.md#rest-api)
+- [Command line options](api.md#command-line-interface)
+- [Device Model](device_model.md)
+- Device tree, device list, ACPI, Hyper-V enlightenments and any other
+  features exposed to guest
+- KVM compatibility
+- Rust edition compatibility
+
+This list is incomplete but this document serves as a best effort guide to stability
+across releases.
+
+### Experimental features
+
+Experimental features are under active development and no guarantees are made about their stability.
+
+List of experimental features:
+
+- TDX
+- vfio-user
+- vDPA
+
+### Security
+
+Security fixes should be included in a new point release.
+
+For security issues an advisory will be published via the GitHub security advisory process along with the release. Watching the project on GitHub will notify you of those issues.
+
+## Releases
+
+### Versioning
+
+The versioning scheme uses `MAJOR.POINT` pattern:
+
+- `MAJOR` can introduce incompatible changes along with support for new features. Changes to the [API](api.md#rest-api),
+  [CLI options](api.md#command-line-interface) and [device model](device_model.md)
+  require a notice at least 2 releases in advance for the actual change to take
+  place.
+- `POINT` contains bug fixes and/or security fixes.
+
+### Major Release Cadence
+
+Cloud Hypervisor is under active development. A new major release is issued approximately
+every 6 weeks. Point releases are issued on demand, when important bug fixes are in
+the queue. A major release would receive bug fixes for the next two cycles (~12 weeks)
+and then be considered EOL.
+
+```
+ - Active release support
+E - EOL
+
+        2021                2022                2023 
+         |    |    |    |    |    |    |    |    | 
+18.0     |    |    |  ++++++++E
+19.0     |    |    |    |++++++++E
+20.0     |    |    |    |   ++++++++E
+21.0     |    |    |    |    | ++++++++E
+22.0     |    |    |    |    |    +++++++++E
+23.0     |    |    |    |    |    |  +++++++++E
+
+```
+
+### Major Release Stability Considerations
+
+Snapshot/restore support is not compatible across `MAJOR` versions.
+Live migration support is not compatible across `MAJOR` versions.
+
+### LTS Release Cadence
+
+A regular release is promoted to LTS every 12 months. An LTS release is supported for 18 months. This gives a 6 months window for users to move to the new LTS.
+
+```
+ - Active release support
+E - EOL
+
+        2022                2023               2024                 2025                2026
+         |    |    |    |    |    |    |    |    |    |    |    |    |    |    |    |    |
+23.0     |    |+++++++++++++++++++++++++++++E
+43.0     |    |    |    |    |    |+++++++++++++++++++++++++++++E
+63.0     |    |    |    |    |    |    |    |    |    |+++++++++++++++++++++++++++++E
+
+```
+
+### LTS Stablity Considerations
+
+An LTS release is just a `MAJOR` release for which point releases are made for
+longer following the same rules for what can be backported to a `POINT` release.
+The focus lays on critical and security bug fixes which are pulled at the
+maintainer's discretion.
--- a/docs/vhost-user-net-testing.md
+++ b/docs/vhost-user-net-testing.md
@@ -75,8 +75,7 @@ Here is an example how to create a bridge and add two DPDK ports to it
 ovs-vsctl add-br ovsbr0 -- set bridge ovsbr0 datapath_type=netdev
 # create two DPDK ports and add them to the bridge
 ovs-vsctl add-port ovsbr0 vhost-user1 -- set Interface vhost-user1 type=dpdkvhostuser
-ovs-vsctl add-port ovsbr0 vhost-user2 -- set Interface vhost-user2
-type=dpdkvhostuser
+ovs-vsctl add-port ovsbr0 vhost-user2 -- set Interface vhost-user2 type=dpdkvhostuser
 # set the number of rx queues
 ovs-vsctl set Interface vhost-user1 options:n_rxq=2
 ovs-vsctl set Interface vhost-user2 options:n_rxq=2
--- a/event_monitor/Cargo.toml
+++ b/event_monitor/Cargo.toml
@@ -5,7 +5,6 @@ authors = ["The Cloud Hypervisor Authors"]
 edition = "2021"

 [dependencies]
-libc = "0.2.123"
-serde = { version = "1.0.136", features = ["rc"] }
-serde_derive = "1.0.136"
-serde_json = "1.0.79"
+libc = "0.2.126"
+serde = { version = "1.0.137", features = ["rc", "derive"] }
+serde_json = "1.0.81"
--- a/event_monitor/src/lib.rs
+++ b/event_monitor/src/lib.rs
@@ -3,12 +3,11 @@
 // SPDX-License-Identifier: Apache-2.0
 //

-#[macro_use]
-extern crate serde_derive;
-
+use serde::Serialize;
 use std::borrow::Cow;
 use std::collections::HashMap;
 use std::fs::File;
+use std::io::Write;
 use std::os::unix::io::AsRawFd;
 use std::time::{Duration, Instant};

@@ -50,6 +49,9 @@ pub fn event_log(source: &str, event: &str, properties: Option<&HashMap<Cow<str>
            properties,
        };
        serde_json::to_writer_pretty(file, &e).ok();
+
+        let mut file = file;
+        file.write_all(b"\n\n").ok();
    }
 }

--- a/fuzz/Cargo.lock
+++ b/fuzz/Cargo.lock
@@ -11,9 +11,9 @@ dependencies = [

 [[package]]
 name = "anyhow"
-version = "1.0.56"
+version = "1.0.57"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4361135be9122e0870de935d7c439aef945b9f9ddd4199a553b5270b49c82a27"
+checksum = "08f9b8508dccb7687a1d6c4ce66b2b0ecef467c94667de27d8d7fe1f8d2a9cdc"

 [[package]]
 name = "api_client"
@@ -47,7 +47,6 @@ dependencies = [
 "linux-loader",
 "log",
 "serde",
- "serde_derive",
 "thiserror",
 "versionize",
 "versionize_derive",
@@ -128,24 +127,33 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"

 [[package]]
 name = "clap"
-version = "3.1.8"
+version = "3.1.18"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "71c47df61d9e16dc010b55dba1952a57d8c215dbb533fd13cdd13369aac73b1c"
+checksum = "d2dbdf4bdacb33466e854ce889eee8dfd5729abf7ccd7664d0a2d60cd384440b"
 dependencies = [
 "atty",
 "bitflags",
+ "clap_lex",
 "indexmap",
 "lazy_static",
- "os_str_bytes",
 "strsim",
 "termcolor",
 "terminal_size",
 "textwrap",
 ]

+[[package]]
+name = "clap_lex"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a37c35f1112dad5e6e0b1adaff798507497a18fceeb30cceb3bae7d1427b9213"
+dependencies = [
+ "os_str_bytes",
+]
+
 [[package]]
 name = "cloud-hypervisor"
-version = "22.0.0"
+version = "23.0.0"
 dependencies = [
 "anyhow",
 "api_client",
@@ -234,7 +242,6 @@ version = "0.1.0"
 dependencies = [
 "libc",
 "serde",
- "serde_derive",
 "serde_json",
 ]

@@ -306,7 +313,6 @@ dependencies = [
 "libc",
 "log",
 "serde",
- "serde_derive",
 "serde_json",
 "thiserror",
 "vm-memory",
@@ -345,9 +351,9 @@ dependencies = [

 [[package]]
 name = "itoa"
-version = "1.0.1"
+version = "1.0.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1aab8fc367588b89dcee83ab0fd66b72b50b72fa1904d7095045ace2b0c81c35"
+checksum = "112c678d4050afce233f4f2852bb2eb519230b3cf12f33585275537d7e41578d"

 [[package]]
 name = "kvm-bindings"
@@ -377,9 +383,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"

 [[package]]
 name = "libc"
-version = "0.2.123"
+version = "0.2.126"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cb691a747a7ab48abc15c5b42066eaafde10dc427e3b6ee2a1cf43db04c763bd"
+checksum = "349d5a591cd28b49e1d1037471617a32ddcda5731b99419008085f72d5a53836"

 [[package]]
 name = "libfuzzer-sys"
@@ -403,9 +409,9 @@ dependencies = [

 [[package]]
 name = "log"
-version = "0.4.16"
+version = "0.4.17"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6389c490849ff5bc16be905ae24bc913a9c8892e19b2341dbc175e14c341c2b8"
+checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e"
 dependencies = [
 "cfg-if",
 ]
@@ -416,12 +422,6 @@ version = "0.8.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0ca88d725a0a943b096803bd34e73a4437208b6077654cc4ecb2947a5f91618d"

-[[package]]
-name = "memchr"
-version = "2.4.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "308cc39be01b73d0d18f82a0e7b2a3df85245f84af96fdddc5d202d27e47b86a"
-
 [[package]]
 name = "micro_http"
 version = "0.1.0"
@@ -460,18 +460,18 @@ dependencies = [

 [[package]]
 name = "num-traits"
-version = "0.2.14"
+version = "0.2.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9a64b1ec5cda2586e284722486d802acf1f7dbdc623e2bfc57e65ca1cd099290"
+checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd"
 dependencies = [
 "autocfg",
 ]

 [[package]]
 name = "once_cell"
-version = "1.10.0"
+version = "1.12.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "87f3e037eac156d1775da914196f0f37741a274155e34a0b7e427c35d2a2ecb9"
+checksum = "7709cef83f0c1f58f666e746a08b21e0085f7440fa6a29cc194d68aac97a4225"

 [[package]]
 name = "option_parser"
@@ -479,12 +479,9 @@ version = "0.1.0"

 [[package]]
 name = "os_str_bytes"
-version = "6.0.0"
+version = "6.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8e22443d1643a904602595ba1cd8f7d896afe56d26712531c5ff73a15b2fbf64"
-dependencies = [
- "memchr",
-]
+checksum = "029d8d0b2f198229de29dca79676f2738ff952edf3fde542eb8bf94d8c21b435"

 [[package]]
 name = "paste"
@@ -502,7 +499,6 @@ dependencies = [
 "libc",
 "log",
 "serde",
- "serde_derive",
 "thiserror",
 "versionize",
 "versionize_derive",
@@ -518,11 +514,11 @@ dependencies = [

 [[package]]
 name = "proc-macro2"
-version = "1.0.37"
+version = "1.0.39"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ec757218438d5fda206afc041538b2f6d889286160d649a86a24d37e1235afd1"
+checksum = "c54b25569025b7fc9651de43004ae593a75ad88543b17178aa5e1b9c4f15f56f"
 dependencies = [
- "unicode-xid",
+ "unicode-ident",
 ]

 [[package]]
@@ -556,9 +552,9 @@ dependencies = [

 [[package]]
 name = "remain"
-version = "0.2.2"
+version = "0.2.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "70ba1e78fa68412cb93ef642fd4d20b9a941be49ee9333875ebaf13112673ea7"
+checksum = "0c35270ea384ac1762895831cc8acb96f171468e52cec82ed9186f9416209fa4"
 dependencies = [
 "proc-macro2",
 "quote",
@@ -576,9 +572,9 @@ dependencies = [

 [[package]]
 name = "ryu"
-version = "1.0.9"
+version = "1.0.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "73b4b750c782965c211b42f022f59af1fbceabdd026623714f104152f1ec149f"
+checksum = "f3f6f92acf49d1b98f7a81226834412ada05458b7364277387724a237f062695"

 [[package]]
 name = "seccompiler"
@@ -591,21 +587,24 @@ dependencies = [

 [[package]]
 name = "semver"
-version = "1.0.7"
+version = "1.0.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d65bd28f48be7196d222d95b9243287f48d27aca604e08497513019ff0502cc4"
+checksum = "8cb243bdfdb5936c8dc3c45762a19d12ab4550cdc753bc247637d4ec35a040fd"

 [[package]]
 name = "serde"
-version = "1.0.136"
+version = "1.0.137"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ce31e24b01e1e524df96f1c2fdd054405f8d7376249a5110886fb4b658484789"
+checksum = "61ea8d54c77f8315140a05f4c7237403bf38b72704d031543aa1d16abbf517d1"
+dependencies = [
+ "serde_derive",
+]

 [[package]]
 name = "serde_derive"
-version = "1.0.136"
+version = "1.0.137"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "08597e7152fcd306f41838ed3e37be9eaeed2b61c42e2117266a554fab4662f9"
+checksum = "1f26faba0c3959972377d3b2d306ee9f71faee9714294e41bb777f83f88578be"
 dependencies = [
 "proc-macro2",
 "quote",
@@ -614,9 +613,9 @@ dependencies = [

 [[package]]
 name = "serde_json"
-version = "1.0.79"
+version = "1.0.81"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8e8d9fa5c3b304765ce1fd9c4c8a3de2c8db365a5b91be52f186efc675681d95"
+checksum = "9b7ce2b32a1aed03c558dc61a5cd328f15aff2dbc17daad8fb8af04d2100e15c"
 dependencies = [
 "itoa",
 "ryu",
@@ -625,9 +624,9 @@ dependencies = [

 [[package]]
 name = "signal-hook"
-version = "0.3.13"
+version = "0.3.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "647c97df271007dcea485bb74ffdb57f2e683f1306c854f468a0c244badabf2d"
+checksum = "a253b5e89e2698464fc26b545c9edceb338e18a89effeeecfea192c3025be29d"
 dependencies = [
 "libc",
 "signal-hook-registry",
@@ -656,13 +655,13 @@ checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623"

 [[package]]
 name = "syn"
-version = "1.0.91"
+version = "1.0.95"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b683b2b825c8eef438b77c36a06dc262294da3d5a5813fac20da149241dcd44d"
+checksum = "fbaf6116ab8924f39d52792136fb74fd60a80194cf1b1c6ffa6453eef1c3f942"
 dependencies = [
 "proc-macro2",
 "quote",
- "unicode-xid",
+ "unicode-ident",
 ]

 [[package]]
@@ -695,18 +694,18 @@ dependencies = [

 [[package]]
 name = "thiserror"
-version = "1.0.30"
+version = "1.0.31"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "854babe52e4df1653706b98fcfc05843010039b406875930a70e4d9644e5c417"
+checksum = "bd829fe32373d27f76265620b5309d0340cb8550f523c1dda251d6298069069a"
 dependencies = [
 "thiserror-impl",
 ]

 [[package]]
 name = "thiserror-impl"
-version = "1.0.30"
+version = "1.0.31"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "aa32fd3f627f367fe16f893e2597ae3c05020f8bba2666a4e6ea73d377e5714b"
+checksum = "0396bc89e626244658bef819e22d0cc459e795a5ebe878e6ec336d1674a8d79a"
 dependencies = [
 "proc-macro2",
 "quote",
@@ -714,16 +713,16 @@ dependencies = [
 ]

 [[package]]
-name = "unicode-xid"
-version = "0.2.2"
+name = "unicode-ident"
+version = "1.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8ccb82d61f80a663efe1f787a51b16b5a51e3314d6ac365b08639f52387b33f3"
+checksum = "d22af068fba1eb5edcb4aea19d382b2a3deb4c8f9d475c589b6ada9e0fd493ee"

 [[package]]
 name = "uuid"
-version = "0.8.2"
+version = "1.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bc5cf98d8186244414c848017f0e2676b3fcb46807f6668a97dfe67359a3c4b7"
+checksum = "8cfcd319456c4d6ea10087ed423473267e1a071f3bc0aa89f80d60997843c6f0"
 dependencies = [
 "getrandom",
 ]
@@ -847,7 +846,6 @@ dependencies = [
 "rate_limiter",
 "seccompiler",
 "serde",
- "serde_derive",
 "serde_json",
 "thiserror",
 "versionize",
@@ -865,9 +863,9 @@ dependencies = [

 [[package]]
 name = "virtio-queue"
-version = "0.2.0"
+version = "0.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3785325315e6496fa88673842ee6cd198b9658e88e8b0e1ad48a5dc818b221dc"
+checksum = "88f2d73c184c18f8acc32dab77fcb6e3af92d53262538d3a68aa474810d6863c"
 dependencies = [
 "log",
 "vm-memory",
@@ -888,8 +886,8 @@ name = "vm-device"
 version = "0.1.0"
 dependencies = [
 "anyhow",
+ "hypervisor",
 "serde",
- "serde_derive",
 "serde_json",
 "thiserror",
 "vfio-ioctls",
@@ -904,9 +902,9 @@ source = "git+https://github.com/rust-vmm/vm-fdt?branch=main#720e48e435b791ec6cb

 [[package]]
 name = "vm-memory"
-version = "0.7.0"
+version = "0.8.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "339d4349c126fdcd87e034631d7274370cf19eb0e87b33166bcd956589fc72c5"
+checksum = "767ed8aaebbff902e02e6d3749dc2baef55e46565f8a6414a065e5baee4b4a81"
 dependencies = [
 "arc-swap",
 "libc",
@@ -919,7 +917,6 @@ version = "0.1.0"
 dependencies = [
 "anyhow",
 "serde",
- "serde_derive",
 "serde_json",
 "thiserror",
 "versionize",
@@ -965,7 +962,6 @@ dependencies = [
 "qcow",
 "seccompiler",
 "serde",
- "serde_derive",
 "serde_json",
 "signal-hook",
 "thiserror",
--- a/fuzz/Cargo.toml
+++ b/fuzz/Cargo.toml
@@ -10,16 +10,16 @@ cargo-fuzz = true

 [dependencies]
 block_util = { path = "../block_util" }
-libc = "0.2.123"
+libc = "0.2.126"
 libfuzzer-sys = "0.4.3"
 qcow = { path = "../qcow" }
 seccompiler = "0.2.0"
 vhdx = { path = "../vhdx" }
 virtio-devices = { path = "../virtio-devices" }
-virtio-queue = "0.2.0"
+virtio-queue = "0.3.0"
 vmm-sys-util = "0.9.0"
 vm-virtio = { path = "../vm-virtio" }
-vm-memory = "0.7.0"
+vm-memory = "0.8.0"

 [dependencies.cloud-hypervisor]
 path = ".."
--- a/hypervisor/Cargo.toml
+++ b/hypervisor/Cargo.toml
@@ -11,19 +11,18 @@ mshv = ["mshv-ioctls", "mshv-bindings"]
 tdx = []

 [dependencies]
-anyhow = "1.0.56"
+anyhow = "1.0.57"
 epoll = "4.3.1"
-thiserror = "1.0.30"
-libc = "0.2.123"
-log = "0.4.16"
+thiserror = "1.0.31"
+libc = "0.2.126"
+log = "0.4.17"
 kvm-ioctls = { version = "0.11.0", optional = true }
 kvm-bindings = { git = "https://github.com/cloud-hypervisor/kvm-bindings", branch = "ch-v0.5.0-tdx", features = ["with-serde", "fam-wrappers"], optional  = true }
-mshv-bindings = {git = "https://github.com/rust-vmm/mshv", branch = "main", features = ["with-serde", "fam-wrappers"], optional  = true }
+mshv-bindings = { git = "https://github.com/rust-vmm/mshv", branch = "main", features = ["with-serde", "fam-wrappers"], optional  = true }
 mshv-ioctls = { git = "https://github.com/rust-vmm/mshv", branch = "main", optional  = true}
-serde = { version = "1.0.136", features = ["rc"] }
-serde_derive = "1.0.136"
-serde_json = "1.0.79"
-vm-memory = { version = "0.7.0", features = ["backend-mmap", "backend-atomic"] }
+serde = { version = "1.0.137", features = ["rc", "derive"] }
+serde_json = "1.0.81"
+vm-memory = { version = "0.8.0", features = ["backend-mmap", "backend-atomic"] }
 vmm-sys-util = { version = "0.9.0", features = ["with-serde"] }

 [target.'cfg(target_arch = "x86_64")'.dependencies.iced-x86]
--- a/hypervisor/src/arch/x86/emulator/mod.rs
+++ b/hypervisor/src/arch/x86/emulator/mod.rs
@@ -6,9 +6,8 @@

 use crate::arch::emulator::{EmulationError, EmulationResult, PlatformEmulator, PlatformError};
 use crate::arch::x86::emulator::instructions::*;
-use crate::arch::x86::regs::*;
-use crate::arch::x86::*;
-use crate::arch::x86::{Exception, SegmentRegisterOps};
+use crate::arch::x86::regs::{CR0_PE, EFER_LMA};
+use crate::arch::x86::{segment_type_expand_down, segment_type_ro, Exception, SegmentRegisterOps};
 use crate::x86_64::{SegmentRegister, SpecialRegisters, StandardRegisters};
 use anyhow::Context;
 use iced_x86::*;
--- a/hypervisor/src/cpu.rs
+++ b/hypervisor/src/cpu.rs
@@ -12,6 +12,12 @@
 use crate::aarch64::VcpuInit;
 #[cfg(target_arch = "aarch64")]
 use crate::aarch64::{RegList, Register, StandardRegisters};
+#[cfg(feature = "tdx")]
+use crate::kvm::{TdxExitDetails, TdxExitStatus};
+#[cfg(all(feature = "mshv", target_arch = "x86_64"))]
+use crate::x86_64::SuspendRegisters;
+#[cfg(target_arch = "x86_64")]
+use crate::x86_64::Xsave;
 #[cfg(target_arch = "x86_64")]
 use crate::x86_64::{CpuId, LapicState};
 #[cfg(target_arch = "x86_64")]
@@ -23,12 +29,6 @@ use crate::CpuState;
 use crate::DeviceAttr;
 #[cfg(feature = "kvm")]
 use crate::MpState;
-#[cfg(all(feature = "mshv", target_arch = "x86_64"))]
-use crate::SuspendRegisters;
-#[cfg(target_arch = "x86_64")]
-use crate::Xsave;
-#[cfg(feature = "tdx")]
-use crate::{TdxExitDetails, TdxExitStatus};
 use thiserror::Error;
 #[cfg(all(feature = "kvm", target_arch = "x86_64"))]
 use vm_memory::GuestAddress;
--- a/hypervisor/src/hypervisor.rs
+++ b/hypervisor/src/hypervisor.rs
@@ -7,13 +7,13 @@
 // Copyright 2018-2019 CrowdStrike, Inc.
 //
 //
+#[cfg(feature = "tdx")]
+use crate::kvm::TdxCapabilities;
 use crate::vm::Vm;
 #[cfg(target_arch = "x86_64")]
 use crate::x86_64::CpuId;
 #[cfg(target_arch = "x86_64")]
 use crate::x86_64::MsrList;
-#[cfg(feature = "tdx")]
-use crate::TdxCapabilities;
 use std::sync::Arc;
 use thiserror::Error;

--- a/hypervisor/src/kvm/aarch64/mod.rs
+++ b/hypervisor/src/kvm/aarch64/mod.rs
@@ -23,7 +23,7 @@ use kvm_bindings::{
 pub use kvm_bindings::{
    kvm_one_reg as Register, kvm_regs as StandardRegisters, kvm_vcpu_init as VcpuInit, RegList,
 };
-use serde_derive::{Deserialize, Serialize};
+use serde::{Deserialize, Serialize};
 pub use {kvm_ioctls::Cap, kvm_ioctls::Kvm};

 // This macro gets the offset of a structure (i.e `str`) member (i.e `field`) without having
--- a/hypervisor/src/kvm/mod.rs
+++ b/hypervisor/src/kvm/mod.rs
@@ -17,11 +17,11 @@ use crate::cpu;
 use crate::device;
 use crate::hypervisor;
 use crate::vec_with_array_field;
-use crate::vm::{self, VmmOps};
+use crate::vm::{self, InterruptSourceConfig, VmOps};
 #[cfg(target_arch = "aarch64")]
 use crate::{arm64_core_reg_id, offset__of};
 use kvm_ioctls::{NoDatamatch, VcpuFd, VmFd};
-use serde_derive::{Deserialize, Serialize};
+use serde::{Deserialize, Serialize};
 use std::collections::HashMap;
 #[cfg(target_arch = "aarch64")]
 use std::convert::TryInto;
@@ -234,7 +234,7 @@ impl vm::Vm for KvmVm {
    fn create_vcpu(
        &self,
        id: u8,
-        vmmops: Option<Arc<dyn VmmOps>>,
+        vm_ops: Option<Arc<dyn VmOps>>,
    ) -> vm::Result<Arc<dyn cpu::Vcpu>> {
        let vc = self
            .fd
@@ -244,7 +244,7 @@ impl vm::Vm for KvmVm {
            fd: vc,
            #[cfg(target_arch = "x86_64")]
            msrs: self.msrs.clone(),
-            vmmops,
+            vm_ops,
            #[cfg(target_arch = "x86_64")]
            hyperv_synic: AtomicBool::new(false),
        };
@@ -284,6 +284,64 @@ impl vm::Vm for KvmVm {
            .unregister_ioevent(fd, addr, NoDatamatch)
            .map_err(|e| vm::HypervisorVmError::UnregisterIoEvent(e.into()))
    }
+
+    ///
+    /// Constructs a routing entry
+    ///
+    fn make_routing_entry(
+        &self,
+        gsi: u32,
+        config: &InterruptSourceConfig,
+    ) -> kvm_irq_routing_entry {
+        match &config {
+            InterruptSourceConfig::MsiIrq(cfg) => {
+                let mut kvm_route = kvm_irq_routing_entry {
+                    gsi,
+                    type_: KVM_IRQ_ROUTING_MSI,
+                    ..Default::default()
+                };
+
+                kvm_route.u.msi.address_lo = cfg.low_addr;
+                kvm_route.u.msi.address_hi = cfg.high_addr;
+                kvm_route.u.msi.data = cfg.data;
+
+                if self.check_extension(crate::kvm::Cap::MsiDevid) {
+                    // On AArch64, there is limitation on the range of the 'devid',
+                    // it can not be greater than 65536 (the max of u16).
+                    //
+                    // BDF can not be used directly, because 'segment' is in high
+                    // 16 bits. The layout of the u32 BDF is:
+                    // |---- 16 bits ----|-- 8 bits --|-- 5 bits --|-- 3 bits --|
+                    // |      segment    |     bus    |   device   |  function  |
+                    //
+                    // Now that we support 1 bus only in a segment, we can build a
+                    // 'devid' by replacing the 'bus' bits with the low 8 bits of
+                    // 'segment' data.
+                    // This way we can resolve the range checking problem and give
+                    // different `devid` to all the devices. Limitation is that at
+                    // most 256 segments can be supported.
+                    //
+                    let modified_devid = (cfg.devid & 0x00ff_0000) >> 8 | cfg.devid & 0xff;
+
+                    kvm_route.flags = KVM_MSI_VALID_DEVID;
+                    kvm_route.u.msi.__bindgen_anon_1.devid = modified_devid;
+                }
+                kvm_route
+            }
+            InterruptSourceConfig::LegacyIrq(cfg) => {
+                let mut kvm_route = kvm_irq_routing_entry {
+                    gsi,
+                    type_: KVM_IRQ_ROUTING_IRQCHIP,
+                    ..Default::default()
+                };
+                kvm_route.u.irqchip.irqchip = cfg.irqchip;
+                kvm_route.u.irqchip.pin = cfg.pin;
+
+                kvm_route
+            }
+        }
+    }
+
    ///
    /// Sets the GSI routing table entries, overwriting any previously set
    /// entries, as per the `KVM_SET_GSI_ROUTING` ioctl.
@@ -816,7 +874,7 @@ pub struct KvmVcpu {
    fd: VcpuFd,
    #[cfg(target_arch = "x86_64")]
    msrs: MsrEntries,
-    vmmops: Option<Arc<dyn vm::VmmOps>>,
+    vm_ops: Option<Arc<dyn vm::VmOps>>,
    #[cfg(target_arch = "x86_64")]
    hyperv_synic: AtomicBool,
 }
@@ -1056,8 +1114,8 @@ impl cpu::Vcpu for KvmVcpu {
            Ok(run) => match run {
                #[cfg(target_arch = "x86_64")]
                VcpuExit::IoIn(addr, data) => {
-                    if let Some(vmmops) = &self.vmmops {
-                        return vmmops
+                    if let Some(vm_ops) = &self.vm_ops {
+                        return vm_ops
                            .pio_read(addr.into(), data)
                            .map(|_| cpu::VmExit::Ignore)
                            .map_err(|e| cpu::HypervisorCpuError::RunVcpu(e.into()));
@@ -1067,8 +1125,8 @@ impl cpu::Vcpu for KvmVcpu {
                }
                #[cfg(target_arch = "x86_64")]
                VcpuExit::IoOut(addr, data) => {
-                    if let Some(vmmops) = &self.vmmops {
-                        return vmmops
+                    if let Some(vm_ops) = &self.vm_ops {
+                        return vm_ops
                            .pio_write(addr.into(), data)
                            .map(|_| cpu::VmExit::Ignore)
                            .map_err(|e| cpu::HypervisorCpuError::RunVcpu(e.into()));
@@ -1100,8 +1158,8 @@ impl cpu::Vcpu for KvmVcpu {
                }

                VcpuExit::MmioRead(addr, data) => {
-                    if let Some(vmmops) = &self.vmmops {
-                        return vmmops
+                    if let Some(vm_ops) = &self.vm_ops {
+                        return vm_ops
                            .mmio_read(addr, data)
                            .map(|_| cpu::VmExit::Ignore)
                            .map_err(|e| cpu::HypervisorCpuError::RunVcpu(e.into()));
@@ -1110,8 +1168,8 @@ impl cpu::Vcpu for KvmVcpu {
                    Ok(cpu::VmExit::MmioRead(addr, data))
                }
                VcpuExit::MmioWrite(addr, data) => {
-                    if let Some(vmmops) = &self.vmmops {
-                        return vmmops
+                    if let Some(vm_ops) = &self.vm_ops {
+                        return vm_ops
                            .mmio_write(addr, data)
                            .map(|_| cpu::VmExit::Ignore)
                            .map_err(|e| cpu::HypervisorCpuError::RunVcpu(e.into()));
--- a/hypervisor/src/kvm/x86_64/mod.rs
+++ b/hypervisor/src/kvm/x86_64/mod.rs
@@ -10,7 +10,7 @@

 use crate::arch::x86::{msr_index, SegmentRegisterOps, MTRR_ENABLE, MTRR_MEM_TYPE_WB};
 use crate::kvm::{Cap, Kvm, KvmError, KvmResult};
-use serde_derive::{Deserialize, Serialize};
+use serde::{Deserialize, Serialize};

 ///
 /// Export generically-named wrappers of kvm-bindings for Unix-based platforms
--- a/hypervisor/src/lib.rs
+++ b/hypervisor/src/lib.rs
@@ -37,10 +37,10 @@ pub mod kvm;
 pub mod mshv;

 /// Hypevisor related module
-pub mod hypervisor;
+mod hypervisor;

 /// Vm related module
-pub mod vm;
+mod vm;

 /// CPU related module
 mod cpu;
@@ -48,18 +48,32 @@ mod cpu;
 /// Device related module
 mod device;

-pub use crate::hypervisor::{Hypervisor, HypervisorError};
 pub use cpu::{HypervisorCpuError, Vcpu, VmExit};
 pub use device::{Device, HypervisorDeviceError};
-#[cfg(feature = "tdx")]
-pub use kvm::TdxCapabilities;
+pub use hypervisor::{Hypervisor, HypervisorError};
+#[cfg(all(feature = "kvm", target_arch = "aarch64"))]
+pub use kvm::aarch64;
+#[cfg(all(feature = "kvm", target_arch = "x86_64"))]
+pub use kvm::x86_64;
+// Aliased types exposed from both hypervisors
 #[cfg(feature = "kvm")]
-pub use kvm::*;
+pub use kvm::{
+    ClockData, CpuState, CreateDevice, DeviceAttr, DeviceFd, IoEventAddress, IrqRoutingEntry,
+    MemoryRegion, MpState, VcpuEvents, VcpuExit, VmState,
+};
 #[cfg(all(feature = "mshv", target_arch = "x86_64"))]
-pub use mshv::*;
-pub use vm::{DataMatch, HypervisorVmError, Vm};
-
+pub use mshv::x86_64;
+// Aliased types exposed from both hypervisors
+#[cfg(all(feature = "mshv", target_arch = "x86_64"))]
+pub use mshv::{
+    CpuState, CreateDevice, DeviceAttr, DeviceFd, IoEventAddress, IrqRoutingEntry, MemoryRegion,
+    MpState, VcpuEvents, VcpuExit, VmState,
+};
 use std::sync::Arc;
+pub use vm::{
+    DataMatch, HypervisorVmError, InterruptSourceConfig, LegacyIrqSourceConfig, MsiIrqSourceConfig,
+    Vm, VmOps,
+};

 pub fn new() -> std::result::Result<Arc<dyn Hypervisor>, HypervisorError> {
    #[cfg(feature = "kvm")]
--- a/hypervisor/src/mshv/mod.rs
+++ b/hypervisor/src/mshv/mod.rs
@@ -11,11 +11,11 @@ use crate::cpu;
 use crate::cpu::Vcpu;
 use crate::hypervisor;
 use crate::vec_with_array_field;
-use crate::vm::{self, VmmOps};
+use crate::vm::{self, InterruptSourceConfig, VmOps};
 pub use mshv_bindings::*;
 pub use mshv_ioctls::IoEventAddress;
 use mshv_ioctls::{set_registers_64, Mshv, NoDatamatch, VcpuFd, VmFd};
-use serde_derive::{Deserialize, Serialize};
+use serde::{Deserialize, Serialize};
 use std::collections::HashMap;
 use std::sync::{Arc, RwLock};
 use vm::DataMatch;
@@ -122,7 +122,7 @@ impl hypervisor::Hypervisor for MshvHypervisor {
            fd: vm_fd,
            msrs,
            hv_state: hv_state_init(),
-            vmmops: None,
+            vm_ops: None,
            dirty_log_slots: Arc::new(RwLock::new(HashMap::new())),
        }))
    }
@@ -151,7 +151,7 @@ pub struct MshvVcpu {
    cpuid: CpuId,
    msrs: MsrEntries,
    hv_state: Arc<RwLock<HvState>>, // Mshv State
-    vmmops: Option<Arc<dyn vm::VmmOps>>,
+    vm_ops: Option<Arc<dyn vm::VmOps>>,
 }

 /// Implementation of Vcpu trait for Microsoft Hypervisor
@@ -355,14 +355,14 @@ impl cpu::Vcpu for MshvVcpu {

                    if is_write {
                        let data = (info.rax as u32).to_le_bytes();
-                        if let Some(vmmops) = &self.vmmops {
-                            vmmops
+                        if let Some(vm_ops) = &self.vm_ops {
+                            vm_ops
                                .pio_write(port.into(), &data[0..len])
                                .map_err(|e| cpu::HypervisorCpuError::RunVcpu(e.into()))?;
                        }
                    } else {
-                        if let Some(vmmops) = &self.vmmops {
-                            vmmops
+                        if let Some(vm_ops) = &self.vm_ops {
+                            vm_ops
                                .pio_read(port.into(), &mut data[0..len])
                                .map_err(|e| cpu::HypervisorCpuError::RunVcpu(e.into()))?;
                        }
@@ -655,9 +655,9 @@ impl<'a> PlatformEmulator for MshvEmulatorContext<'a> {
            gpa
        );

-        if let Some(vmmops) = &self.vcpu.vmmops {
-            if vmmops.guest_mem_read(gpa, data).is_err() {
-                vmmops
+        if let Some(vm_ops) = &self.vcpu.vm_ops {
+            if vm_ops.guest_mem_read(gpa, data).is_err() {
+                vm_ops
                    .mmio_read(gpa, data)
                    .map_err(|e| PlatformError::MemoryReadFailure(e.into()))?;
            }
@@ -675,9 +675,9 @@ impl<'a> PlatformEmulator for MshvEmulatorContext<'a> {
            gpa
        );

-        if let Some(vmmops) = &self.vcpu.vmmops {
-            if vmmops.guest_mem_write(gpa, data).is_err() {
-                vmmops
+        if let Some(vm_ops) = &self.vcpu.vm_ops {
+            if vm_ops.guest_mem_write(gpa, data).is_err() {
+                vm_ops
                    .mmio_write(gpa, data)
                    .map_err(|e| PlatformError::MemoryWriteFailure(e.into()))?;
            }
@@ -746,7 +746,7 @@ pub struct MshvVm {
    msrs: MsrEntries,
    // Hypervisor State
    hv_state: Arc<RwLock<HvState>>,
-    vmmops: Option<Arc<dyn vm::VmmOps>>,
+    vm_ops: Option<Arc<dyn vm::VmOps>>,
    dirty_log_slots: Arc<RwLock<HashMap<u64, MshvDirtyLogSlot>>>,
 }

@@ -816,7 +816,7 @@ impl vm::Vm for MshvVm {
    fn create_vcpu(
        &self,
        id: u8,
-        vmmops: Option<Arc<dyn VmmOps>>,
+        vm_ops: Option<Arc<dyn VmOps>>,
    ) -> vm::Result<Arc<dyn cpu::Vcpu>> {
        let vcpu_fd = self
            .fd
@@ -828,7 +828,7 @@ impl vm::Vm for MshvVm {
            cpuid: CpuId::new(1).unwrap(),
            msrs: self.msrs.clone(),
            hv_state: self.hv_state.clone(),
-            vmmops,
+            vm_ops,
        };
        Ok(Arc::new(vcpu))
    }
@@ -957,6 +957,27 @@ impl vm::Vm for MshvVm {
            .map_err(|e| vm::HypervisorVmError::CreatePassthroughDevice(e.into()))
    }

+    ///
+    /// Constructs a routing entry
+    ///
+    fn make_routing_entry(
+        &self,
+        gsi: u32,
+        config: &InterruptSourceConfig,
+    ) -> mshv_msi_routing_entry {
+        match config {
+            InterruptSourceConfig::MsiIrq(cfg) => mshv_msi_routing_entry {
+                gsi,
+                address_lo: cfg.low_addr,
+                address_hi: cfg.high_addr,
+                data: cfg.data,
+            },
+            _ => {
+                unreachable!()
+            }
+        }
+    }
+
    fn set_gsi_routing(&self, entries: &[IrqRoutingEntry]) -> vm::Result<()> {
        let mut msi_routing =
            vec_with_array_field::<mshv_msi_routing, mshv_msi_routing_entry>(entries.len());
@@ -1027,6 +1048,3 @@ impl vm::Vm for MshvVm {
            .map_err(|e| vm::HypervisorVmError::GetDirtyLog(e.into()))
    }
 }
-pub use hv_cpuid_entry as CpuIdEntry;
-
-pub const CPUID_FLAG_VALID_INDEX: u32 = 0;
--- a/hypervisor/src/mshv/x86_64/mod.rs
+++ b/hypervisor/src/mshv/x86_64/mod.rs
@@ -8,13 +8,14 @@
 //
 //
 use crate::arch::x86::{msr_index, SegmentRegisterOps, MTRR_ENABLE, MTRR_MEM_TYPE_WB};
-use serde_derive::{Deserialize, Serialize};
+use serde::{Deserialize, Serialize};
 use std::fmt;

 ///
 /// Export generically-named wrappers of mshv_bindings for Unix-based platforms
 ///
 pub use {
+    mshv_bindings::hv_cpuid_entry as CpuIdEntry,
    mshv_bindings::mshv_user_mem_region as MemoryRegion, mshv_bindings::msr_entry as MsrEntry,
    mshv_bindings::CpuId, mshv_bindings::DebugRegisters,
    mshv_bindings::FloatingPointUnit as FpuState, mshv_bindings::LapicState,
@@ -25,6 +26,8 @@ pub use {
    mshv_bindings::Xcrs as ExtendedControlRegisters,
 };

+pub const CPUID_FLAG_VALID_INDEX: u32 = 0;
+
 #[derive(Clone, Serialize, Deserialize)]
 pub struct VcpuMshvState {
    pub msrs: MsrEntries,
--- a/hypervisor/src/vm.rs
+++ b/hypervisor/src/vm.rs
@@ -12,15 +12,15 @@
 use crate::aarch64::VcpuInit;
 use crate::cpu::Vcpu;
 use crate::device::Device;
+#[cfg(feature = "kvm")]
+use crate::kvm::KvmVmState as VmState;
+#[cfg(feature = "mshv")]
+use crate::mshv::HvState as VmState;
 #[cfg(feature = "tdx")]
 use crate::x86_64::CpuId;
 #[cfg(all(feature = "kvm", target_arch = "x86_64"))]
 use crate::ClockData;
 use crate::CreateDevice;
-#[cfg(feature = "mshv")]
-use crate::HvState as VmState;
-#[cfg(feature = "kvm")]
-use crate::KvmVmState as VmState;
 use crate::{IoEventAddress, IrqRoutingEntry, MemoryRegion};
 #[cfg(feature = "kvm")]
 use kvm_ioctls::Cap;
@@ -216,6 +216,39 @@ pub enum HypervisorVmError {
 ///
 pub type Result<T> = std::result::Result<T, HypervisorVmError>;

+/// Configuration data for legacy interrupts.
+///
+/// On x86 platforms, legacy interrupts means those interrupts routed through PICs or IOAPICs.
+#[derive(Copy, Clone, Debug)]
+pub struct LegacyIrqSourceConfig {
+    pub irqchip: u32,
+    pub pin: u32,
+}
+
+/// Configuration data for MSI/MSI-X interrupts.
+///
+/// On x86 platforms, these interrupts are vectors delivered directly to the LAPIC.
+#[derive(Copy, Clone, Debug, Default)]
+pub struct MsiIrqSourceConfig {
+    /// High address to delivery message signaled interrupt.
+    pub high_addr: u32,
+    /// Low address to delivery message signaled interrupt.
+    pub low_addr: u32,
+    /// Data to write to delivery message signaled interrupt.
+    pub data: u32,
+    /// Unique ID of the device to delivery message signaled interrupt.
+    pub devid: u32,
+}
+
+/// Configuration data for an interrupt source.
+#[derive(Copy, Clone, Debug)]
+pub enum InterruptSourceConfig {
+    /// Configuration data for Legacy interrupts.
+    LegacyIrq(LegacyIrqSourceConfig),
+    /// Configuration data for PciMsi, PciMsix and generic MSI interrupts.
+    MsiIrq(MsiIrqSourceConfig),
+}
+
 ///
 /// Trait to represent a Vm
 ///
@@ -235,7 +268,7 @@ pub trait Vm: Send + Sync {
    /// Unregister an event that will, when signaled, trigger the `gsi` IRQ.
    fn unregister_irqfd(&self, fd: &EventFd, gsi: u32) -> Result<()>;
    /// Creates a new KVM vCPU file descriptor and maps the memory corresponding
-    fn create_vcpu(&self, id: u8, vmmops: Option<Arc<dyn VmmOps>>) -> Result<Arc<dyn Vcpu>>;
+    fn create_vcpu(&self, id: u8, vm_ops: Option<Arc<dyn VmOps>>) -> Result<Arc<dyn Vcpu>>;
    /// Registers an event to be signaled whenever a certain address is written to.
    fn register_ioevent(
        &self,
@@ -245,6 +278,8 @@ pub trait Vm: Send + Sync {
    ) -> Result<()>;
    /// Unregister an event from a certain address it has been previously registered to.
    fn unregister_ioevent(&self, fd: &EventFd, addr: &IoEventAddress) -> Result<()>;
+    // Construct a routing entry
+    fn make_routing_entry(&self, gsi: u32, config: &InterruptSourceConfig) -> IrqRoutingEntry;
    /// Sets the GSI routing table entries, overwriting any previously set
    fn set_gsi_routing(&self, entries: &[IrqRoutingEntry]) -> Result<()>;
    /// Creates a memory region structure that can be used with {create/remove}_user_memory_region
@@ -309,7 +344,7 @@ pub trait Vm: Send + Sync {
    ) -> Result<()>;
 }

-pub trait VmmOps: Send + Sync {
+pub trait VmOps: Send + Sync {
    fn guest_mem_write(&self, gpa: u64, buf: &[u8]) -> Result<usize>;
    fn guest_mem_read(&self, gpa: u64, buf: &mut [u8]) -> Result<usize>;
    fn mmio_read(&self, gpa: u64, data: &mut [u8]) -> Result<()>;
--- a/net_util/Cargo.toml
+++ b/net_util/Cargo.toml
@@ -7,20 +7,20 @@ edition = "2021"
 [dependencies]
 epoll = "4.3.1"
 getrandom = "0.2"
-libc = "0.2.123"
-log = "0.4.16"
+libc = "0.2.126"
+log = "0.4.17"
 net_gen = { path = "../net_gen" }
 rate_limiter = { path = "../rate_limiter" }
-serde = "1.0.136"
+serde = "1.0.137"
 versionize = "0.1.6"
 versionize_derive = "0.1.4"
 virtio-bindings = "0.1.0"
-virtio-queue = "0.2.0"
-vm-memory = { version = "0.7.0", features = ["backend-mmap", "backend-atomic", "backend-bitmap"] }
+virtio-queue = "0.3.0"
+vm-memory = { version = "0.8.0", features = ["backend-mmap", "backend-atomic", "backend-bitmap"] }
 vm-virtio = { path = "../vm-virtio" }
 vmm-sys-util = "0.9.0"

 [dev-dependencies]
 lazy_static = "1.4.0"
-pnet = "0.29.0"
-serde_json = "1.0.79"
+pnet = "0.30.0"
+serde_json = "1.0.81"
--- a/option_parser/src/lib.rs
+++ b/option_parser/src/lib.rs
@@ -77,7 +77,7 @@ impl OptionParser {
        }

        for option in split_commas_outside_brackets(input)?.iter() {
-            let parts: Vec<&str> = option.split('=').collect();
+            let parts: Vec<&str> = option.splitn(2, '=').collect();

            match self.options.get_mut(parts[0]) {
                None => return Err(OptionParserError::UnknownOption(parts[0].to_owned())),
--- a/pci/Cargo.toml
+++ b/pci/Cargo.toml
@@ -10,22 +10,21 @@ kvm = ["vfio-ioctls/kvm"]
 mshv = ["vfio-ioctls/mshv"]

 [dependencies]
-anyhow = "1.0.56"
+anyhow = "1.0.57"
 byteorder = "1.4.3"
 hypervisor = { path = "../hypervisor" }
 vfio-ioctls = { git = "https://github.com/rust-vmm/vfio", branch = "main", default-features = false }
 vfio_user = { path = "../vfio_user" }
 vmm-sys-util = "0.9.0"
-libc = "0.2.123"
-log = "0.4.16"
-serde = "1.0.136"
-serde_derive = "1.0.136"
-thiserror = "1.0.30"
+libc = "0.2.126"
+log = "0.4.17"
+serde = { version="1.0.137", features=["derive"] }
+thiserror = "1.0.31"
 versionize = "0.1.6"
 versionize_derive = "0.1.4"
 vm-allocator = { path = "../vm-allocator" }
 vm-device = { path = "../vm-device" }
-vm-memory = "0.7.0"
+vm-memory = "0.8.0"
 vm-migration = { path = "../vm-migration" }

 [dependencies.vfio-bindings]
--- a/pci/src/bus.rs
+++ b/pci/src/bus.rs
@@ -6,13 +6,13 @@ use crate::configuration::{
    PciBarRegionType, PciBridgeSubclass, PciClassCode, PciConfiguration, PciHeaderType,
 };
 use crate::device::{DeviceRelocation, Error as PciDeviceError, PciDevice};
+use crate::PciBarConfiguration;
 use byteorder::{ByteOrder, LittleEndian};
 use std::any::Any;
 use std::collections::HashMap;
 use std::ops::DerefMut;
 use std::sync::{Arc, Barrier, Mutex};
 use vm_device::{Bus, BusDevice};
-use vm_memory::{Address, GuestAddress, GuestUsize};

 const VENDOR_ID_INTEL: u16 = 0x8086;
 const DEVICE_ID_INTEL_VIRT_PCIE_HOST: u16 = 0x0d57;
@@ -88,6 +88,10 @@ impl PciDevice for PciRoot {
    fn as_any(&mut self) -> &mut dyn Any {
        self
    }
+
+    fn id(&self) -> Option<String> {
+        None
+    }
 }

 pub struct PciBus {
@@ -118,21 +122,21 @@ impl PciBus {
        dev: Arc<Mutex<dyn BusDevice>>,
        #[cfg(target_arch = "x86_64")] io_bus: &Bus,
        mmio_bus: &Bus,
-        bars: Vec<(GuestAddress, GuestUsize, PciBarRegionType)>,
+        bars: Vec<PciBarConfiguration>,
    ) -> Result<()> {
-        for (address, size, type_) in bars {
-            match type_ {
+        for bar in bars {
+            match bar.region_type() {
                PciBarRegionType::IoRegion => {
                    #[cfg(target_arch = "x86_64")]
                    io_bus
-                        .insert(dev.clone(), address.raw_value(), size)
+                        .insert(dev.clone(), bar.addr(), bar.size())
                        .map_err(PciRootError::PioInsert)?;
                    #[cfg(target_arch = "aarch64")]
                    error!("I/O region is not supported");
                }
                PciBarRegionType::Memory32BitRegion | PciBarRegionType::Memory64BitRegion => {
                    mmio_bus
-                        .insert(dev.clone(), address.raw_value(), size)
+                        .insert(dev.clone(), bar.addr(), bar.size())
                        .map_err(PciRootError::MmioInsert)?;
                }
            }
--- a/pci/src/configuration.rs
+++ b/pci/src/configuration.rs
@@ -9,6 +9,7 @@ use std::fmt::{self, Display};
 use std::sync::{Arc, Mutex};
 use versionize::{VersionMap, Versionize, VersionizeError, VersionizeResult};
 use versionize_derive::Versionize;
+use vm_device::PciBarType;
 use vm_migration::{MigratableError, Pausable, Snapshot, Snapshottable, VersionMapped};

 // The number of 32bit registers in the config space, 4096 bytes.
@@ -18,6 +19,7 @@ const STATUS_REG: usize = 1;
 const STATUS_REG_CAPABILITIES_USED_MASK: u32 = 0x0010_0000;
 const BAR0_REG: usize = 4;
 const ROM_BAR_REG: usize = 12;
+const ROM_BAR_IDX: usize = 6;
 const BAR_IO_ADDR_MASK: u32 = 0xffff_fffc;
 const BAR_MEM_ADDR_MASK: u32 = 0xffff_fff0;
 const ROM_BAR_ADDR_MASK: u32 = 0xffff_f800;
@@ -326,17 +328,48 @@ pub enum PciBarRegionType {
    Memory64BitRegion = 0x04,
 }

+impl From<PciBarType> for PciBarRegionType {
+    fn from(type_: PciBarType) -> Self {
+        match type_ {
+            PciBarType::Io => PciBarRegionType::IoRegion,
+            PciBarType::Mmio32 => PciBarRegionType::Memory32BitRegion,
+            PciBarType::Mmio64 => PciBarRegionType::Memory64BitRegion,
+        }
+    }
+}
+
+#[allow(clippy::from_over_into)]
+impl Into<PciBarType> for PciBarRegionType {
+    fn into(self) -> PciBarType {
+        match self {
+            PciBarRegionType::IoRegion => PciBarType::Io,
+            PciBarRegionType::Memory32BitRegion => PciBarType::Mmio32,
+            PciBarRegionType::Memory64BitRegion => PciBarType::Mmio64,
+        }
+    }
+}
+
 #[derive(Copy, Clone)]
 pub enum PciBarPrefetchable {
    NotPrefetchable = 0,
    Prefetchable = 0x08,
 }

+#[allow(clippy::from_over_into)]
+impl Into<bool> for PciBarPrefetchable {
+    fn into(self) -> bool {
+        match self {
+            PciBarPrefetchable::NotPrefetchable => false,
+            PciBarPrefetchable::Prefetchable => true,
+        }
+    }
+}
+
 #[derive(Copy, Clone)]
 pub struct PciBarConfiguration {
    addr: u64,
    size: u64,
-    reg_idx: usize,
+    idx: usize,
    region_type: PciBarRegionType,
    prefetchable: PciBarPrefetchable,
 }
@@ -556,22 +589,23 @@ impl PciConfiguration {

    /// Adds a region specified by `config`.  Configures the specified BAR(s) to
    /// report this region and size to the guest kernel.  Enforces a few constraints
-    /// (i.e, region size must be power of two, register not already used). Returns 'None' on
-    /// failure all, `Some(BarIndex)` on success.
-    pub fn add_pci_bar(&mut self, config: &PciBarConfiguration) -> Result<usize> {
-        if self.bars[config.reg_idx].used {
-            return Err(Error::BarInUse(config.reg_idx));
+    /// (i.e, region size must be power of two, register not already used).
+    pub fn add_pci_bar(&mut self, config: &PciBarConfiguration) -> Result<()> {
+        let bar_idx = config.idx;
+        let reg_idx = BAR0_REG + bar_idx;
+
+        if self.bars[bar_idx].used {
+            return Err(Error::BarInUse(bar_idx));
        }

        if config.size.count_ones() != 1 {
            return Err(Error::BarSizeInvalid(config.size));
        }

-        if config.reg_idx >= NUM_BAR_REGS {
-            return Err(Error::BarInvalid(config.reg_idx));
+        if bar_idx >= NUM_BAR_REGS {
+            return Err(Error::BarInvalid(bar_idx));
        }

-        let bar_idx = BAR0_REG + config.reg_idx;
        let end_addr = config
            .addr
            .checked_add(config.size - 1)
@@ -584,20 +618,20 @@ impl PciConfiguration {

                // Encode the BAR size as expected by the software running in
                // the guest.
-                self.bars[config.reg_idx].size =
+                self.bars[bar_idx].size =
                    encode_32_bits_bar_size(config.size as u32).ok_or(Error::Encode32BarSize)?;
            }
            PciBarRegionType::Memory64BitRegion => {
-                if config.reg_idx + 1 >= NUM_BAR_REGS {
-                    return Err(Error::BarInvalid64(config.reg_idx));
+                if bar_idx + 1 >= NUM_BAR_REGS {
+                    return Err(Error::BarInvalid64(bar_idx));
                }

                if end_addr > u64::max_value() {
                    return Err(Error::BarAddressInvalid(config.addr, config.size));
                }

-                if self.bars[config.reg_idx + 1].used {
-                    return Err(Error::BarInUse64(config.reg_idx));
+                if self.bars[bar_idx + 1].used {
+                    return Err(Error::BarInUse64(bar_idx));
                }

                // Encode the BAR size as expected by the software running in
@@ -605,12 +639,12 @@ impl PciConfiguration {
                let (bar_size_hi, bar_size_lo) =
                    encode_64_bits_bar_size(config.size).ok_or(Error::Encode64BarSize)?;

-                self.registers[bar_idx + 1] = (config.addr >> 32) as u32;
-                self.writable_bits[bar_idx + 1] = 0xffff_ffff;
-                self.bars[config.reg_idx + 1].addr = self.registers[bar_idx + 1];
-                self.bars[config.reg_idx].size = bar_size_lo;
-                self.bars[config.reg_idx + 1].size = bar_size_hi;
-                self.bars[config.reg_idx + 1].used = true;
+                self.registers[reg_idx + 1] = (config.addr >> 32) as u32;
+                self.writable_bits[reg_idx + 1] = 0xffff_ffff;
+                self.bars[bar_idx + 1].addr = self.registers[reg_idx + 1];
+                self.bars[bar_idx].size = bar_size_lo;
+                self.bars[bar_idx + 1].size = bar_size_hi;
+                self.bars[bar_idx + 1].used = true;
            }
        }

@@ -622,26 +656,30 @@ impl PciConfiguration {
            PciBarRegionType::IoRegion => (BAR_IO_ADDR_MASK, config.region_type as u32),
        };

-        self.registers[bar_idx] = ((config.addr as u32) & mask) | lower_bits;
-        self.writable_bits[bar_idx] = mask;
-        self.bars[config.reg_idx].addr = self.registers[bar_idx];
-        self.bars[config.reg_idx].used = true;
-        self.bars[config.reg_idx].r#type = Some(config.region_type);
-        Ok(config.reg_idx)
+        self.registers[reg_idx] = ((config.addr as u32) & mask) | lower_bits;
+        self.writable_bits[reg_idx] = mask;
+        self.bars[bar_idx].addr = self.registers[reg_idx];
+        self.bars[bar_idx].used = true;
+        self.bars[bar_idx].r#type = Some(config.region_type);
+
+        Ok(())
    }

    /// Adds rom expansion BAR.
-    pub fn add_pci_rom_bar(&mut self, config: &PciBarConfiguration, active: u32) -> Result<usize> {
+    pub fn add_pci_rom_bar(&mut self, config: &PciBarConfiguration, active: u32) -> Result<()> {
+        let bar_idx = config.idx;
+        let reg_idx = ROM_BAR_REG;
+
        if self.rom_bar_used {
-            return Err(Error::RomBarInUse(config.reg_idx));
+            return Err(Error::RomBarInUse(bar_idx));
        }

        if config.size.count_ones() != 1 {
            return Err(Error::RomBarSizeInvalid(config.size));
        }

-        if config.reg_idx != ROM_BAR_REG {
-            return Err(Error::RomBarInvalid(config.reg_idx));
+        if bar_idx != ROM_BAR_IDX {
+            return Err(Error::RomBarInvalid(bar_idx));
        }

        let end_addr = config
@@ -653,13 +691,14 @@ impl PciConfiguration {
            return Err(Error::RomBarAddressInvalid(config.addr, config.size));
        }

-        self.registers[config.reg_idx] = (config.addr as u32) | active;
-        self.writable_bits[config.reg_idx] = ROM_BAR_ADDR_MASK;
-        self.rom_bar_addr = self.registers[config.reg_idx];
+        self.registers[reg_idx] = (config.addr as u32) | active;
+        self.writable_bits[reg_idx] = ROM_BAR_ADDR_MASK;
+        self.rom_bar_addr = self.registers[reg_idx];
        self.rom_bar_size =
            encode_32_bits_bar_size(config.size as u32).ok_or(Error::Encode32BarSize)?;
        self.rom_bar_used = true;
-        Ok(config.reg_idx)
+
+        Ok(())
    }

    /// Returns the address of the given BAR region.
@@ -911,7 +950,7 @@ impl Snapshottable for PciConfiguration {
 impl Default for PciBarConfiguration {
    fn default() -> Self {
        PciBarConfiguration {
-            reg_idx: 0,
+            idx: 0,
            addr: 0,
            size: 0,
            region_type: PciBarRegionType::Memory64BitRegion,
@@ -922,13 +961,13 @@ impl Default for PciBarConfiguration {

 impl PciBarConfiguration {
    pub fn new(
-        reg_idx: usize,
+        idx: usize,
        size: u64,
        region_type: PciBarRegionType,
        prefetchable: PciBarPrefetchable,
    ) -> Self {
        PciBarConfiguration {
-            reg_idx,
+            idx,
            addr: 0,
            size,
            region_type,
@@ -937,8 +976,8 @@ impl PciBarConfiguration {
    }

    #[must_use]
-    pub fn set_register_index(mut self, reg_idx: usize) -> Self {
-        self.reg_idx = reg_idx;
+    pub fn set_index(mut self, idx: usize) -> Self {
+        self.idx = idx;
        self
    }

@@ -954,15 +993,31 @@ impl PciBarConfiguration {
        self
    }

-    pub fn get_size(&self) -> u64 {
-        self.size
-    }
-
    #[must_use]
    pub fn set_region_type(mut self, region_type: PciBarRegionType) -> Self {
        self.region_type = region_type;
        self
    }
+
+    pub fn idx(&self) -> usize {
+        self.idx
+    }
+
+    pub fn addr(&self) -> u64 {
+        self.addr
+    }
+
+    pub fn size(&self) -> u64 {
+        self.size
+    }
+
+    pub fn region_type(&self) -> PciBarRegionType {
+        self.region_type
+    }
+
+    pub fn prefetchable(&self) -> PciBarPrefetchable {
+        self.prefetchable
+    }
 }

 #[cfg(test)]
--- a/pci/src/device.rs
+++ b/pci/src/device.rs
@@ -3,13 +3,13 @@
 // found in the LICENSE-BSD-3-Clause file.

 use crate::configuration::{self, PciBarRegionType};
+use crate::PciBarConfiguration;
 use std::any::Any;
 use std::fmt::{self, Display};
 use std::sync::{Arc, Barrier, Mutex};
 use std::{self, io, result};
 use vm_allocator::{AddressAllocator, SystemAllocator};
-use vm_device::BusDevice;
-use vm_memory::{GuestAddress, GuestUsize};
+use vm_device::{BusDevice, Resource};

 #[derive(Debug)]
 pub enum Error {
@@ -19,6 +19,8 @@ pub enum Error {
    IoAllocationFailed(u64),
    /// Registering an IO BAR failed.
    IoRegistrationFailed(u64, configuration::Error),
+    /// Expected resource not found.
+    MissingResource,
 }
 pub type Result<T> = std::result::Result<T, Error>;

@@ -34,6 +36,7 @@ impl Display for Error {
            IoRegistrationFailed(addr, e) => {
                write!(f, "failed to register an IO BAR, addr={} err={}", addr, e)
            }
+            MissingResource => write!(f, "failed to find expected resource"),
        }
    }
 }
@@ -53,7 +56,8 @@ pub trait PciDevice: BusDevice {
        &mut self,
        _allocator: &Arc<Mutex<SystemAllocator>>,
        _mmio_allocator: &mut AddressAllocator,
-    ) -> Result<Vec<(GuestAddress, GuestUsize, PciBarRegionType)>> {
+        _resources: Option<Vec<Resource>>,
+    ) -> Result<Vec<PciBarConfiguration>> {
        Ok(Vec::new())
    }

@@ -103,6 +107,9 @@ pub trait PciDevice: BusDevice {
    /// Provides a mutable reference to the Any trait. This is useful to let
    /// the caller have access to the underlying type behind the trait.
    fn as_any(&mut self) -> &mut dyn Any;
+
+    /// Optionally returns a unique identifier.
+    fn id(&self) -> Option<String>;
 }

 /// This trait defines a set of functions which can be triggered whenever a
--- a/pci/src/msi.rs
+++ b/pci/src/msi.rs
@@ -3,11 +3,17 @@
 // SPDX-License-Identifier: Apache-2.0 OR BSD-3-Clause
 //

+use anyhow::anyhow;
 use byteorder::{ByteOrder, LittleEndian};
+use std::io;
 use std::sync::Arc;
+use thiserror::Error;
+use versionize::{VersionMap, Versionize, VersionizeResult};
+use versionize_derive::Versionize;
 use vm_device::interrupt::{
    InterruptIndex, InterruptSourceConfig, InterruptSourceGroup, MsiIrqSourceConfig,
 };
+use vm_migration::{MigratableError, Pausable, Snapshot, Snapshottable, VersionMapped};

 // MSI control masks
 const MSI_CTL_ENABLE: u16 = 0x1;
@@ -32,7 +38,15 @@ pub fn msi_num_enabled_vectors(msg_ctl: u16) -> usize {
    1 << field
 }

-#[derive(Clone, Copy, Default)]
+#[derive(Error, Debug)]
+enum Error {
+    #[error("Failed enabling the interrupt route: {0}")]
+    EnableInterruptRoute(io::Error),
+    #[error("Failed updating the interrupt route: {0}")]
+    UpdateInterruptRoute(io::Error),
+}
+
+#[derive(Clone, Copy, Default, Versionize)]
 pub struct MsiCap {
    // Message Control Register
    //   0:     MSI enable.
@@ -157,6 +171,13 @@ impl MsiCap {
    }
 }

+#[derive(Versionize)]
+struct MsiConfigState {
+    cap: MsiCap,
+}
+
+impl VersionMapped for MsiConfigState {}
+
 pub struct MsiConfig {
    cap: MsiCap,
    interrupt_source_group: Arc<dyn InterruptSourceGroup>,
@@ -175,6 +196,39 @@ impl MsiConfig {
        }
    }

+    fn state(&self) -> MsiConfigState {
+        MsiConfigState { cap: self.cap }
+    }
+
+    fn set_state(&mut self, state: &MsiConfigState) -> Result<(), Error> {
+        self.cap = state.cap;
+
+        if self.enabled() {
+            for idx in 0..self.num_enabled_vectors() {
+                let config = MsiIrqSourceConfig {
+                    high_addr: self.cap.msg_addr_hi,
+                    low_addr: self.cap.msg_addr_lo,
+                    data: self.cap.msg_data as u32,
+                    devid: 0,
+                };
+
+                self.interrupt_source_group
+                    .update(
+                        idx as InterruptIndex,
+                        InterruptSourceConfig::MsiIrq(config),
+                        self.cap.vector_masked(idx),
+                    )
+                    .map_err(Error::UpdateInterruptRoute)?;
+            }
+
+            self.interrupt_source_group
+                .enable()
+                .map_err(Error::EnableInterruptRoute)?;
+        }
+
+        Ok(())
+    }
+
    pub fn enabled(&self) -> bool {
        self.cap.enabled()
    }
@@ -222,3 +276,26 @@ impl MsiConfig {
        }
    }
 }
+
+impl Pausable for MsiConfig {}
+
+impl Snapshottable for MsiConfig {
+    fn id(&self) -> String {
+        String::from("msi_config")
+    }
+
+    fn snapshot(&mut self) -> std::result::Result<Snapshot, MigratableError> {
+        Snapshot::new_from_versioned_state(&self.id(), &self.state())
+    }
+
+    fn restore(&mut self, snapshot: Snapshot) -> std::result::Result<(), MigratableError> {
+        self.set_state(&snapshot.to_versioned_state(&self.id())?)
+            .map_err(|e| {
+                MigratableError::Restore(anyhow!(
+                    "Could not restore state for {}: {:?}",
+                    self.id(),
+                    e
+                ))
+            })
+    }
+}
--- a/pci/src/vfio.rs
+++ b/pci/src/vfio.rs
--- a/pci/src/vfio_user.rs
+++ b/pci/src/vfio_user.rs
@@ -3,8 +3,8 @@
 // SPDX-License-Identifier: Apache-2.0
 //

-use crate::vfio::{Interrupt, Vfio, VfioCommon, VfioError};
-use crate::{BarReprogrammingParams, PciBarRegionType, VfioPciError};
+use crate::vfio::{Interrupt, UserMemoryRegion, Vfio, VfioCommon, VfioError};
+use crate::{BarReprogrammingParams, PciBarConfiguration, VfioPciError};
 use crate::{
    PciBdf, PciClassCode, PciConfiguration, PciDevice, PciDeviceError, PciHeaderType, PciSubclass,
 };
@@ -21,18 +21,17 @@ use vfio_user::{Client, Error as VfioUserError};
 use vm_allocator::{AddressAllocator, SystemAllocator};
 use vm_device::dma_mapping::ExternalDmaMapping;
 use vm_device::interrupt::{InterruptManager, InterruptSourceGroup, MsiIrqGroupConfig};
-use vm_device::BusDevice;
+use vm_device::{BusDevice, Resource};
 use vm_memory::bitmap::AtomicBitmap;
 use vm_memory::{
    Address, GuestAddress, GuestAddressSpace, GuestMemory, GuestMemoryRegion, GuestRegionMmap,
-    GuestUsize,
 };
 use vmm_sys_util::eventfd::EventFd;

 pub struct VfioUserPciDevice {
+    id: String,
    vm: Arc<dyn hypervisor::Vm>,
    client: Arc<Mutex<Client>>,
-    vfio_wrapper: VfioUserClientWrapper,
    common: VfioCommon,
 }

@@ -63,9 +62,10 @@ impl PciSubclass for PciVfioUserSubclass {

 impl VfioUserPciDevice {
    pub fn new(
+        id: String,
        vm: &Arc<dyn hypervisor::Vm>,
        client: Arc<Mutex<Client>>,
-        msi_interrupt_manager: &Arc<dyn InterruptManager<GroupConfig = MsiIrqGroupConfig>>,
+        msi_interrupt_manager: Arc<dyn InterruptManager<GroupConfig = MsiIrqGroupConfig>>,
        legacy_interrupt_group: Option<Arc<dyn InterruptSourceGroup>>,
        bdf: PciBdf,
    ) -> Result<Self, VfioUserPciDeviceError> {
@@ -103,17 +103,20 @@ impl VfioUserPciDevice {
                msi: None,
                msix: None,
            },
+            msi_interrupt_manager,
+            legacy_interrupt_group,
+            vfio_wrapper: Arc::new(vfio_wrapper) as Arc<dyn Vfio>,
        };

-        common.parse_capabilities(msi_interrupt_manager, &vfio_wrapper, bdf);
+        common.parse_capabilities(bdf);
        common
-            .initialize_legacy_interrupt(legacy_interrupt_group, &vfio_wrapper)
+            .initialize_legacy_interrupt()
            .map_err(VfioUserPciDeviceError::InitializeLegacyInterrupts)?;

        Ok(Self {
+            id,
            vm: vm.clone(),
            client,
-            vfio_wrapper,
            common,
        })
    }
@@ -143,6 +146,15 @@ impl VfioUserPciDevice {
                .file_offset
                .clone();

+            let sparse_areas = self
+                .client
+                .lock()
+                .unwrap()
+                .region(mmio_region.index)
+                .unwrap()
+                .sparse_areas
+                .clone();
+
            if region_flags & VFIO_REGION_INFO_FLAG_MMAP != 0 {
                let mut prot = 0;
                if region_flags & VFIO_REGION_INFO_FLAG_READ != 0 {
@@ -152,41 +164,57 @@ impl VfioUserPciDevice {
                    prot |= libc::PROT_WRITE;
                }

-                let host_addr = unsafe {
-                    libc::mmap(
-                        null_mut(),
-                        mmio_region.length as usize,
-                        prot,
-                        libc::MAP_SHARED,
-                        file_offset.as_ref().unwrap().file().as_raw_fd(),
-                        file_offset.as_ref().unwrap().start() as libc::off_t,
-                    )
+                let mmaps = if sparse_areas.is_empty() {
+                    vec![vfio_region_sparse_mmap_area {
+                        offset: 0,
+                        size: mmio_region.length,
+                    }]
+                } else {
+                    sparse_areas
                };

-                if host_addr == libc::MAP_FAILED {
-                    error!(
-                        "Could not mmap regions, error:{}",
-                        std::io::Error::last_os_error()
+                for s in mmaps.iter() {
+                    let host_addr = unsafe {
+                        libc::mmap(
+                            null_mut(),
+                            s.size as usize,
+                            prot,
+                            libc::MAP_SHARED,
+                            file_offset.as_ref().unwrap().file().as_raw_fd(),
+                            file_offset.as_ref().unwrap().start() as libc::off_t
+                                + s.offset as libc::off_t,
+                        )
+                    };
+
+                    if host_addr == libc::MAP_FAILED {
+                        error!(
+                            "Could not mmap regions, error:{}",
+                            std::io::Error::last_os_error()
+                        );
+                        continue;
+                    }
+
+                    let user_memory_region = UserMemoryRegion {
+                        slot: mem_slot(),
+                        start: mmio_region.start.0 + s.offset,
+                        size: s.size,
+                        host_addr: host_addr as u64,
+                    };
+
+                    mmio_region.user_memory_regions.push(user_memory_region);
+
+                    let mem_region = vm.make_user_memory_region(
+                        user_memory_region.slot,
+                        user_memory_region.start,
+                        user_memory_region.size,
+                        user_memory_region.host_addr,
+                        false,
+                        false,
                    );
-                    continue;
+
+                    vm.create_user_memory_region(mem_region)
+                        .map_err(VfioUserPciDeviceError::MapRegionGuest)?;
                }
-
-                let slot = mem_slot();
-                let mem_region = vm.make_user_memory_region(
-                    slot,
-                    mmio_region.start.0,
-                    mmio_region.length as u64,
-                    host_addr as u64,
-                    false,
-                    false,
-                );
-
-                vm.create_user_memory_region(mem_region)
-                    .map_err(VfioUserPciDeviceError::MapRegionGuest)?;
-
-                mmio_region.mem_slot = Some(slot);
-                mmio_region.host_addr = Some(host_addr as u64);
-                mmio_region.mmap_size = Some(mmio_region.length as usize);
            }
        }

@@ -195,17 +223,13 @@ impl VfioUserPciDevice {

    pub fn unmap_mmio_regions(&mut self) {
        for mmio_region in self.common.mmio_regions.iter() {
-            if let (Some(host_addr), Some(mmap_size), Some(mem_slot)) = (
-                mmio_region.host_addr,
-                mmio_region.mmap_size,
-                mmio_region.mem_slot,
-            ) {
+            for user_memory_region in mmio_region.user_memory_regions.iter() {
                // Remove region
                let r = self.vm.make_user_memory_region(
-                    mem_slot,
-                    mmio_region.start.raw_value(),
-                    mmap_size as u64,
-                    host_addr as u64,
+                    user_memory_region.slot,
+                    user_memory_region.start,
+                    user_memory_region.size,
+                    user_memory_region.host_addr,
                    false,
                    false,
                );
@@ -214,7 +238,13 @@ impl VfioUserPciDevice {
                    error!("Could not remove the userspace memory region: {}", e);
                }

-                let ret = unsafe { libc::munmap(host_addr as *mut libc::c_void, mmap_size) };
+                // Remove mmaps
+                let ret = unsafe {
+                    libc::munmap(
+                        user_memory_region.host_addr as *mut libc::c_void,
+                        user_memory_region.size as usize,
+                    )
+                };
                if ret != 0 {
                    error!(
                        "Could not unmap region {}, error:{}",
@@ -392,9 +422,10 @@ impl PciDevice for VfioUserPciDevice {
        &mut self,
        allocator: &Arc<Mutex<SystemAllocator>>,
        mmio_allocator: &mut AddressAllocator,
-    ) -> Result<Vec<(GuestAddress, GuestUsize, PciBarRegionType)>, PciDeviceError> {
+        resources: Option<Vec<Resource>>,
+    ) -> Result<Vec<PciBarConfiguration>, PciDeviceError> {
        self.common
-            .allocate_bars(allocator, mmio_allocator, &self.vfio_wrapper)
+            .allocate_bars(allocator, mmio_allocator, resources)
    }

    fn free_bars(
@@ -425,22 +456,19 @@ impl PciDevice for VfioUserPciDevice {
        offset: u64,
        data: &[u8],
    ) -> Option<Arc<Barrier>> {
-        self.common
-            .write_config_register(reg_idx, offset, data, &self.vfio_wrapper)
+        self.common.write_config_register(reg_idx, offset, data)
    }

    fn read_config_register(&mut self, reg_idx: usize) -> u32 {
-        self.common
-            .read_config_register(reg_idx, &self.vfio_wrapper)
+        self.common.read_config_register(reg_idx)
    }

    fn read_bar(&mut self, base: u64, offset: u64, data: &mut [u8]) {
-        self.common.read_bar(base, offset, data, &self.vfio_wrapper)
+        self.common.read_bar(base, offset, data)
    }

    fn write_bar(&mut self, base: u64, offset: u64, data: &[u8]) -> Option<Arc<Barrier>> {
-        self.common
-            .write_bar(base, offset, data, &self.vfio_wrapper)
+        self.common.write_bar(base, offset, data)
    }

    fn move_bar(&mut self, old_base: u64, new_base: u64) -> Result<(), std::io::Error> {
@@ -449,35 +477,41 @@ impl PciDevice for VfioUserPciDevice {
            if mmio_region.start.raw_value() == old_base {
                mmio_region.start = GuestAddress(new_base);

-                if let Some(mem_slot) = mmio_region.mem_slot {
-                    if let Some(host_addr) = mmio_region.host_addr {
-                        // Remove original region
-                        let old_region = self.vm.make_user_memory_region(
-                            mem_slot,
-                            old_base,
-                            mmio_region.length as u64,
-                            host_addr as u64,
-                            false,
-                            false,
-                        );
+                for user_memory_region in mmio_region.user_memory_regions.iter_mut() {
+                    // Remove old region
+                    let old_region = self.vm.make_user_memory_region(
+                        user_memory_region.slot,
+                        user_memory_region.start,
+                        user_memory_region.size,
+                        user_memory_region.host_addr,
+                        false,
+                        false,
+                    );

-                        self.vm
-                            .remove_user_memory_region(old_region)
-                            .map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e))?;
+                    self.vm
+                        .remove_user_memory_region(old_region)
+                        .map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e))?;

-                        let new_region = self.vm.make_user_memory_region(
-                            mem_slot,
-                            new_base,
-                            mmio_region.length as u64,
-                            host_addr as u64,
-                            false,
-                            false,
-                        );
-
-                        self.vm
-                            .create_user_memory_region(new_region)
-                            .map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e))?;
+                    // Update the user memory region with the correct start address.
+                    if new_base > old_base {
+                        user_memory_region.start += new_base - old_base;
+                    } else {
+                        user_memory_region.start -= old_base - new_base;
                    }
+
+                    // Insert new region
+                    let new_region = self.vm.make_user_memory_region(
+                        user_memory_region.slot,
+                        user_memory_region.start,
+                        user_memory_region.size,
+                        user_memory_region.host_addr,
+                        false,
+                        false,
+                    );
+
+                    self.vm
+                        .create_user_memory_region(new_region)
+                        .map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e))?;
                }
                info!("Moved bar 0x{:x} -> 0x{:x}", old_base, new_base);
            }
@@ -485,6 +519,10 @@ impl PciDevice for VfioUserPciDevice {

        Ok(())
    }
+
+    fn id(&self) -> Option<String> {
+        Some(self.id.clone())
+    }
 }

 impl Drop for VfioUserPciDevice {
@@ -493,18 +531,18 @@ impl Drop for VfioUserPciDevice {

        if let Some(msix) = &self.common.interrupt.msix {
            if msix.bar.enabled() {
-                self.common.disable_msix(&self.vfio_wrapper);
+                self.common.disable_msix();
            }
        }

        if let Some(msi) = &self.common.interrupt.msi {
            if msi.cfg.enabled() {
-                self.common.disable_msi(&self.vfio_wrapper)
+                self.common.disable_msi()
            }
        }

        if self.common.interrupt.intx_in_use() {
-            self.common.disable_intx(&self.vfio_wrapper);
+            self.common.disable_intx();
        }

        if let Err(e) = self.client.lock().unwrap().shutdown() {
--- a/performance-metrics/Cargo.toml
+++ b/performance-metrics/Cargo.toml
@@ -6,14 +6,13 @@ edition = "2021"
 build = "build.rs"

 [dependencies]
-clap = { version = "3.1.8", features = ["wrap_help","cargo"] }
+clap = { version = "3.1.18", features = ["wrap_help","cargo"] }
 dirs = "4.0.0"
-serde = { version = "1.0.136", features = ["rc"] }
-serde_derive = "1.0.136"
-serde_json = "1.0.78"
+serde = { version = "1.0.137", features = ["rc", "derive"] }
+serde_json = "1.0.81"
 test_infra = { path = "../test_infra" }
-thiserror = "1.0.30"
+thiserror = "1.0.31"
 wait-timeout = "0.2.0"

 [build-dependencies]
-clap = { version = "3.1.8", features = ["cargo"] }
+clap = { version = "3.1.18", features = ["cargo"] }
--- a/performance-metrics/src/main.rs
+++ b/performance-metrics/src/main.rs
@@ -12,8 +12,14 @@ mod performance_tests;

 use clap::{Arg, Command as ClapCommand};
 use performance_tests::*;
-use serde_derive::{Deserialize, Serialize};
-use std::{env, fmt, process::Command, sync::mpsc::channel, thread, time::Duration};
+use serde::{Deserialize, Serialize};
+use std::{
+    env, fmt,
+    process::Command,
+    sync::{mpsc::channel, Arc},
+    thread,
+    time::Duration,
+};
 use thiserror::Error;

 #[derive(Error, Debug)]
@@ -84,6 +90,21 @@ impl Default for MetricsReport {
    }
 }

+#[derive(Default)]
+pub struct PerformanceTestOverrides {
+    test_iterations: Option<u32>,
+}
+
+impl fmt::Display for PerformanceTestOverrides {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        if let Some(test_iterations) = self.test_iterations {
+            write!(f, "test_iterations = {}", test_iterations)?;
+        }
+
+        Ok(())
+    }
+}
+
 pub struct PerformanceTestControl {
    test_timeout: u32,
    test_iterations: u32,
@@ -91,6 +112,7 @@ pub struct PerformanceTestControl {
    queue_size: Option<u32>,
    net_rx: Option<bool>,
    fio_ops: Option<FioOps>,
+    num_boot_vcpus: Option<u8>,
 }

 impl fmt::Display for PerformanceTestControl {
@@ -125,6 +147,7 @@ impl PerformanceTestControl {
            queue_size: None,
            net_rx: None,
            fio_ops: None,
+            num_boot_vcpus: Some(1),
        }
    }
 }
@@ -140,9 +163,12 @@ struct PerformanceTest {
 }

 impl PerformanceTest {
-    pub fn run(&self) -> PerformanceTestResult {
+    pub fn run(&self, overrides: &PerformanceTestOverrides) -> PerformanceTestResult {
        let mut metrics = Vec::new();
-        for _ in 0..self.control.test_iterations {
+        for _ in 0..overrides
+            .test_iterations
+            .unwrap_or(self.control.test_iterations)
+        {
            metrics.push((self.func_ptr)(&self.control));
        }

@@ -162,8 +188,9 @@ impl PerformanceTest {

    // Calculate the timeout for each test
    // Note: To cover the setup/cleanup time, 20s is added for each iteration of the test
-    pub fn calc_timeout(&self) -> u64 {
-        ((self.control.test_timeout + 20) * self.control.test_iterations) as u64
+    pub fn calc_timeout(&self, test_iterations: &Option<u32>) -> u64 {
+        ((self.control.test_timeout + 20) * test_iterations.unwrap_or(self.control.test_iterations))
+            as u64
    }
 }

@@ -216,7 +243,7 @@ mod adjuster {
    }
 }

-const TEST_LIST: [PerformanceTest; 15] = [
+const TEST_LIST: [PerformanceTest; 17] = [
    PerformanceTest {
        name: "boot_time_ms",
        func_ptr: performance_boot_time,
@@ -237,6 +264,28 @@ const TEST_LIST: [PerformanceTest; 15] = [
        },
        unit_adjuster: adjuster::s_to_ms,
    },
+    PerformanceTest {
+        name: "boot_time_16_vcpus_ms",
+        func_ptr: performance_boot_time,
+        control: PerformanceTestControl {
+            test_timeout: 2,
+            test_iterations: 10,
+            num_boot_vcpus: Some(16),
+            ..PerformanceTestControl::default()
+        },
+        unit_adjuster: adjuster::s_to_ms,
+    },
+    PerformanceTest {
+        name: "boot_time_16_vcpus_pmem_ms",
+        func_ptr: performance_boot_time_pmem,
+        control: PerformanceTestControl {
+            test_timeout: 2,
+            test_iterations: 10,
+            num_boot_vcpus: Some(16),
+            ..PerformanceTestControl::default()
+        },
+        unit_adjuster: adjuster::s_to_ms,
+    },
    PerformanceTest {
        name: "virtio_net_latency_us",
        func_ptr: performance_net_latency,
@@ -381,12 +430,20 @@ const TEST_LIST: [PerformanceTest; 15] = [
    },
 ];

-fn run_test_with_timeout(test: &'static PerformanceTest) -> Result<PerformanceTestResult, Error> {
+fn run_test_with_timeout(
+    test: &'static PerformanceTest,
+    overrides: &Arc<PerformanceTestOverrides>,
+) -> Result<PerformanceTestResult, Error> {
    let (sender, receiver) = channel::<Result<PerformanceTestResult, Error>>();
+    let test_iterations = overrides.test_iterations;
+    let overrides = overrides.clone();
    thread::spawn(move || {
-        println!("Test '{}' running .. ({})", test.name, test.control);
+        println!(
+            "Test '{}' running .. (control: {}, overrides: {})",
+            test.name, test.control, overrides
+        );

-        let output = match std::panic::catch_unwind(|| test.run()) {
+        let output = match std::panic::catch_unwind(|| test.run(&overrides)) {
            Ok(test_result) => {
                println!(
                    "Test '{}' .. ok: mean = {}, std_dev = {}",
@@ -401,7 +458,7 @@ fn run_test_with_timeout(test: &'static PerformanceTest) -> Result<PerformanceTe
    });

    // Todo: Need to cleanup/kill all hanging child processes
-    let test_timeout = test.calc_timeout();
+    let test_timeout = test.calc_timeout(&test_iterations);
    receiver
        .recv_timeout(Duration::from_secs(test_timeout))
        .map_err(|_| {
@@ -445,6 +502,12 @@ fn main() {
                .help("Report file. Standard error is used if not specified")
                .takes_value(true),
        )
+        .arg(
+            Arg::new("iterations")
+                .long("iterations")
+                .help("Override number of test iterations")
+                .takes_value(true),
+        )
        .get_matches();

    // It seems that the tool (ethr) used for testing the virtio-net latency
@@ -473,9 +536,17 @@ fn main() {

    init_tests();

+    let overrides = Arc::new(PerformanceTestOverrides {
+        test_iterations: cmd_arguments
+            .value_of("iterations")
+            .map(|s| s.parse())
+            .transpose()
+            .unwrap_or_default(),
+    });
+
    for test in test_list.iter() {
        if test_filter.is_empty() || test_filter.iter().any(|&s| test.name.contains(s)) {
-            match run_test_with_timeout(test) {
+            match run_test_with_timeout(test, &overrides) {
                Ok(r) => {
                    metrics_report.results.push(r);
                }
--- a/performance-metrics/src/performance_tests.rs
+++ b/performance-metrics/src/performance_tests.rs
@@ -238,6 +238,7 @@ pub fn performance_net_throughput(control: &PerformanceTestControl) -> f64 {
        .default_disks()
        .args(&["--net", net_params.as_str()])
        .capture_output()
+        .verbosity(VerbosityLevel::Warn)
        .set_print_cmd(false)
        .spawn()
        .unwrap();
@@ -367,6 +368,7 @@ pub fn performance_net_latency(control: &PerformanceTestControl) -> f64 {
        .default_disks()
        .args(&["--net", net_params.as_str()])
        .capture_output()
+        .verbosity(VerbosityLevel::Warn)
        .set_print_cmd(false)
        .spawn()
        .unwrap();
@@ -472,7 +474,12 @@ fn parse_boot_time_output(output: &[u8]) -> Result<f64, Error> {
 }

 fn measure_boot_time(cmd: &mut GuestCommand, test_timeout: u32) -> Result<f64, Error> {
-    let mut child = cmd.capture_output().set_print_cmd(false).spawn().unwrap();
+    let mut child = cmd
+        .capture_output()
+        .verbosity(VerbosityLevel::Warn)
+        .set_print_cmd(false)
+        .spawn()
+        .unwrap();

    thread::sleep(Duration::new(test_timeout as u64, 0));
    let _ = child.kill();
@@ -499,6 +506,10 @@ pub fn performance_boot_time(control: &PerformanceTestControl) -> f64 {
        let mut cmd = GuestCommand::new(&guest);

        let c = cmd
+            .args(&[
+                "--cpus",
+                &format!("boot={}", control.num_boot_vcpus.unwrap_or(1)),
+            ])
            .args(&["--memory", "size=1G"])
            .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()])
            .args(&["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE])
@@ -522,6 +533,10 @@ pub fn performance_boot_time_pmem(control: &PerformanceTestControl) -> f64 {
        let guest = performance_test_new_guest(Box::new(focal));
        let mut cmd = GuestCommand::new(&guest);
        let c = cmd
+            .args(&[
+                "--cpus",
+                &format!("boot={}", control.num_boot_vcpus.unwrap_or(1)),
+            ])
            .args(&["--memory", "size=1G,hugepages=on"])
            .args(&["--kernel", direct_kernel_boot_path().to_str().unwrap()])
            .args(&["--cmdline", "root=/dev/pmem0p1 console=ttyS0 quiet rw"])
@@ -655,6 +670,7 @@ pub fn performance_block_io(control: &PerformanceTestControl) -> f64 {
        .default_net()
        .args(&["--api-socket", &api_socket])
        .capture_output()
+        .verbosity(VerbosityLevel::Warn)
        .set_print_cmd(false)
        .spawn()
        .unwrap();
--- a/qcow/Cargo.toml
+++ b/qcow/Cargo.toml
@@ -10,7 +10,7 @@ path = "src/qcow.rs"

 [dependencies]
 byteorder = "1.4.3"
-libc = "0.2.123"
-log = "0.4.16"
-remain = "0.2.2"
+libc = "0.2.126"
+log = "0.4.17"
+remain = "0.2.3"
 vmm-sys-util = "0.9.0"
--- a/qcow/src/qcow.rs
+++ b/qcow/src/qcow.rs
@@ -516,6 +516,19 @@ impl QcowFile {

        let l2_entries = cluster_size / size_of::<u64>() as u64;

+        // Check for compressed blocks
+        for l2_addr_disk in l1_table.get_values() {
+            if *l2_addr_disk != 0 {
+                if let Err(e) = Self::read_l2_cluster(&mut raw_file, *l2_addr_disk) {
+                    if let Some(os_error) = e.raw_os_error() {
+                        if os_error == ENOTSUP {
+                            return Err(Error::CompressedBlocksNotSupported);
+                        }
+                    }
+                }
+            }
+        }
+
        let mut qcow = QcowFile {
            raw_file,
            header,
--- a/rate_limiter/Cargo.toml
+++ b/rate_limiter/Cargo.toml
@@ -4,6 +4,6 @@ version = "0.1.0"
 edition = "2021"

 [dependencies]
-libc = "0.2.123"
-log = "0.4.16"
+libc = "0.2.126"
+log = "0.4.17"
 vmm-sys-util = "0.9.0"
--- a/release-notes.md
+++ b/release-notes.md
@@ -1,11 +1,22 @@
+- [v24.0](#v240)
+    - [Bypass Mode for `virtio-iommu`](#bypass-mode-for-virtio-iommu)
+    - [Ensure Identifiers Uniqueness](#ensure-identifiers-uniqueness)
+    - [Sparse Mmap support](#sparse-mmap-support)
+    - [Expose Platform Serial Number](#expose-platform-serial-number)
+    - [Notable Bug Fixes](#notable-bug-fixes)
+    - [Notable Improvements](#notable-improvements)
+    - [Deprecations](#deprecations)
+    - [New on the Website](#new-on-the-website)
+    - [Contributors](#contributors)
+- [v23.1](#v231)
 - [v23.0](#v230)
    - [vDPA Support](#vdpa-support)
    - [Updated OS Support list](#updated-os-support-list)
    - [`AArch64` Memory Map Improvements](#aarch64-memory-map-improvements)
    - [`AMX` Support](#amx-support)
-    - [Notable Bug Fixes](#notable-bug-fixes)
-    - [Deprecations](#deprecations)
-    - [Contributors](#contributors)
+    - [Notable Bug Fixes](#notable-bug-fixes-1)
+    - [Deprecations](#deprecations-1)
+    - [Contributors](#contributors-1)
 - [v22.1](#v221)
 - [v22.0](#v220)
    - [GDB Debug Stub Support](#gdb-debug-stub-support)
@@ -16,13 +27,13 @@
    - [PMU Support for AArch64](#pmu-support-for-aarch64)
    - [Documentation Under CC-BY-4.0 License](#documentation-under-cc-by-40-license)
    - [Deprecation of "Classic" `virtiofsd`](#deprecation-of-classic-virtiofsd)
-    - [Notable Bug Fixes](#notable-bug-fixes-1)
-    - [Contributors](#contributors-1)
+    - [Notable Bug Fixes](#notable-bug-fixes-2)
+    - [Contributors](#contributors-2)
 - [v21.0](#v210)
    - [Efficient Local Live Migration (for Live Upgrade)](#efficient-local-live-migration-for-live-upgrade)
    - [Recommended Kernel is Now 5.15](#recommended-kernel-is-now-515)
-    - [Notable Bug fixes](#notable-bug-fixes-2)
-    - [Contributors](#contributors-2)
+    - [Notable Bug fixes](#notable-bug-fixes-3)
+    - [Contributors](#contributors-3)
 - [v20.2](#v202)
 - [v20.1](#v201)
 - [v20.0](#v200)
@@ -31,8 +42,8 @@
    - [Improved VFIO support](#improved-vfio-support)
    - [Safer code](#safer-code)
    - [Extended documentation](#extended-documentation)
-    - [Notable bug fixes](#notable-bug-fixes-3)
-    - [Contributors](#contributors-3)
+    - [Notable bug fixes](#notable-bug-fixes-4)
+    - [Contributors](#contributors-4)
 - [v19.0](#v190)
    - [Improved PTY handling for serial and `virtio-console`](#improved-pty-handling-for-serial-and-virtio-console)
    - [PCI boot time optimisations](#pci-boot-time-optimisations)
@@ -40,8 +51,8 @@
    - [Live migration enhancements](#live-migration-enhancements)
    - [`virtio-mem` support with `vfio-user`](#virtio-mem-support-with-vfio-user)
    - [AArch64 for `virtio-iommu`](#aarch64-for-virtio-iommu)
-    - [Notable bug fixes](#notable-bug-fixes-4)
-    - [Contributors](#contributors-4)
+    - [Notable bug fixes](#notable-bug-fixes-5)
+    - [Contributors](#contributors-5)
 - [v18.0](#v180)
    - [Experimental User Device (`vfio-user`) support](#experimental-user-device-vfio-user-support)
    - [Migration support for `vhost-user` devices](#migration-support-for-vhost-user-devices)
@@ -51,31 +62,31 @@
    - [Live migration on MSHV hypervisor](#live-migration-on-mshv-hypervisor)
    - [AArch64 CPU topology support](#aarch64-cpu-topology-support)
    - [Power button support on AArch64](#power-button-support-on-aarch64)
-    - [Notable bug fixes](#notable-bug-fixes-5)
-    - [Contributors](#contributors-5)
+    - [Notable bug fixes](#notable-bug-fixes-6)
+    - [Contributors](#contributors-6)
 - [v17.0](#v170)
    - [ARM64 NUMA support using ACPI](#arm64-numa-support-using-acpi)
    - [`Seccomp` support for MSHV backend](#seccomp-support-for-mshv-backend)
    - [Hotplug of `macvtap` devices](#hotplug-of-macvtap-devices)
    - [Improved SGX support](#improved-sgx-support)
    - [Inflight tracking for `vhost-user` devices](#inflight-tracking-for-vhost-user-devices)
-    - [Notable bug fixes](#notable-bug-fixes-6)
-    - [Contributors](#contributors-6)
+    - [Notable bug fixes](#notable-bug-fixes-7)
+    - [Contributors](#contributors-7)
 - [v16.0](#v160)
    - [Improved live migration support](#improved-live-migration-support)
    - [Improved `vhost-user` support](#improved-vhost-user-support)
    - [ARM64 ACPI and UEFI support](#arm64-acpi-and-uefi-support)
-    - [Notable bug fixes](#notable-bug-fixes-7)
+    - [Notable bug fixes](#notable-bug-fixes-8)
    - [Removed functionality](#removed-functionality)
-    - [Contributors](#contributors-7)
+    - [Contributors](#contributors-8)
 - [v15.0](#v150)
    - [Version numbering and stability guarantees](#version-numbering-and-stability-guarantees)
    - [Network device rate limiting](#network-device-rate-limiting)
    - [Support for runtime control of `virtio-net` guest offload](#support-for-runtime-control-of-virtio-net-guest-offload)
    - [`--api-socket` supports file descriptor parameter](#--api-socket-supports-file-descriptor-parameter)
    - [Bug fixes](#bug-fixes)
-    - [Deprecations](#deprecations-1)
-    - [Contributors](#contributors-8)
+    - [Deprecations](#deprecations-2)
+    - [Contributors](#contributors-9)
 - [v0.14.1](#v0141)
 - [v0.14.0](#v0140)
    - [Structured event monitoring](#structured-event-monitoring)
@@ -84,8 +95,8 @@
    - [Updated hotplug documentation](#updated-hotplug-documentation)
    - [PTY control for serial and `virtio-console`](#pty-control-for-serial-and-virtio-console)
    - [Block device rate limiting](#block-device-rate-limiting)
-    - [Deprecations](#deprecations-2)
-    - [Contributors](#contributors-9)
+    - [Deprecations](#deprecations-3)
+    - [Contributors](#contributors-10)
 - [v0.13.0](#v0130)
    - [Wider VFIO device support](#wider-vfio-device-support)
    - [Improved huge page support](#improved-huge-page-support)
@@ -93,13 +104,13 @@
    - [VHD disk image support](#vhd-disk-image-support)
    - [Improved Virtio device threading](#improved-virtio-device-threading)
    - [Clean shutdown support via synthetic power button](#clean-shutdown-support-via-synthetic-power-button)
-    - [Contributors](#contributors-10)
+    - [Contributors](#contributors-11)
 - [v0.12.0](#v0120)
    - [ARM64 enhancements](#arm64-enhancements)
    - [Removal of `vhost-user-net` and `vhost-user-block` self spawning](#removal-of-vhost-user-net-and-vhost-user-block-self-spawning)
    - [Migration of `vhost-user-fs` backend](#migration-of-vhost-user-fs-backend)
    - [Enhanced "info" API](#enhanced-info-api)
-    - [Contributors](#contributors-11)
+    - [Contributors](#contributors-12)
 - [v0.11.0](#v0110)
    - [`io_uring` support by default for `virtio-block`](#io_uring-support-by-default-for-virtio-block)
    - [Windows Guest Support](#windows-guest-support)
@@ -111,15 +122,15 @@
    - [Default Log Level Changed](#default-log-level-changed)
    - [New `--balloon` Parameter Added](#new---balloon-parameter-added)
    - [Experimental `virtio-watchdog` Support](#experimental-virtio-watchdog-support)
-    - [Notable Bug Fixes](#notable-bug-fixes-8)
-    - [Contributors](#contributors-12)
+    - [Notable Bug Fixes](#notable-bug-fixes-9)
+    - [Contributors](#contributors-13)
 - [v0.10.0](#v0100)
    - [`virtio-block` Support for Multiple Descriptors](#virtio-block-support-for-multiple-descriptors)
    - [Memory Zones](#memory-zones)
    - [`Seccomp` Sandbox Improvements](#seccomp-sandbox-improvements)
    - [Preliminary KVM HyperV Emulation Control](#preliminary-kvm-hyperv-emulation-control)
-    - [Notable Bug Fixes](#notable-bug-fixes-9)
-    - [Contributors](#contributors-13)
+    - [Notable Bug Fixes](#notable-bug-fixes-10)
+    - [Contributors](#contributors-14)
 - [v0.9.0](#v090)
    - [`io_uring` Based Block Device Support](#io_uring-based-block-device-support)
    - [Block and Network Device Statistics](#block-and-network-device-statistics)
@@ -132,17 +143,17 @@
    - [Enhancements to ARM64 Support](#enhancements-to-arm64-support)
    - [Intel SGX Support](#intel-sgx-support)
    - [`Seccomp` Sandbox Improvements](#seccomp-sandbox-improvements-1)
-    - [Notable Bug Fixes](#notable-bug-fixes-10)
-    - [Contributors](#contributors-14)
+    - [Notable Bug Fixes](#notable-bug-fixes-11)
+    - [Contributors](#contributors-15)
 - [v0.8.0](#v080)
    - [Experimental Snapshot and Restore Support](#experimental-snapshot-and-restore-support)
    - [Experimental ARM64 Support](#experimental-arm64-support)
    - [Support for Using 5-level Paging in Guests](#support-for-using-5-level-paging-in-guests)
    - [Virtio Device Interrupt Suppression for Network Devices](#virtio-device-interrupt-suppression-for-network-devices)
    - [`vhost_user_fs` Improvements](#vhost_user_fs-improvements)
-    - [Notable Bug Fixes](#notable-bug-fixes-11)
+    - [Notable Bug Fixes](#notable-bug-fixes-12)
    - [Command Line and API Changes](#command-line-and-api-changes)
-    - [Contributors](#contributors-15)
+    - [Contributors](#contributors-16)
 - [v0.7.0](#v070)
    - [Block, Network, Persistent Memory (PMEM), VirtioFS and Vsock hotplug](#block-network-persistent-memory-pmem-virtiofs-and-vsock-hotplug)
    - [Alternative `libc` Support](#alternative-libc-support)
@@ -152,14 +163,14 @@
    - [`Seccomp` Sandboxing](#seccomp-sandboxing)
    - [Updated Distribution Support](#updated-distribution-support)
    - [Command Line and API Changes](#command-line-and-api-changes-1)
-    - [Contributors](#contributors-16)
+    - [Contributors](#contributors-17)
 - [v0.6.0](#v060)
    - [Directly Assigned Devices Hotplug](#directly-assigned-devices-hotplug)
    - [Shared Filesystem Improvements](#shared-filesystem-improvements)
    - [Block and Networking IO Self Offloading](#block-and-networking-io-self-offloading)
    - [Command Line Interface](#command-line-interface)
    - [PVH Boot](#pvh-boot)
-    - [Contributors](#contributors-17)
+    - [Contributors](#contributors-18)
 - [v0.5.1](#v051)
 - [v0.5.0](#v050)
    - [Virtual Machine Dynamic Resizing](#virtual-machine-dynamic-resizing)
@@ -167,7 +178,7 @@
    - [New Interrupt Management Framework](#new-interrupt-management-framework)
    - [Development Tools](#development-tools)
    - [Kata Containers Integration](#kata-containers-integration)
-    - [Contributors](#contributors-18)
+    - [Contributors](#contributors-19)
 - [v0.4.0](#v040)
    - [Dynamic virtual CPUs addition](#dynamic-virtual-cpus-addition)
    - [Programmatic firmware tables generation](#programmatic-firmware-tables-generation)
@@ -176,7 +187,7 @@
    - [Userspace IOAPIC by default](#userspace-ioapic-by-default)
    - [PCI BAR reprogramming](#pci-bar-reprogramming)
    - [New `cloud-hypervisor` organization](#new-cloud-hypervisor-organization)
-    - [Contributors](#contributors-19)
+    - [Contributors](#contributors-20)
 - [v0.3.0](#v030)
    - [Block device offloading](#block-device-offloading)
    - [Network device backend](#network-device-backend)
@@ -203,6 +214,94 @@
    - [Unit testing](#unit-testing)
    - [Integration tests parallelization](#integration-tests-parallelization)

+# v24.0
+
+This release has been tracked through the [v24.0
+project](https://github.com/cloud-hypervisor/cloud-hypervisor/projects/28).
+
+### Bypass Mode for `virtio-iommu`
+
+`virtio-iommu` specification describes how a device can be attached by default
+to a bypass domain. This feature is particularly helpful for booting a VM with
+guest software which doesn't support `virtio-iommu` but still need to access
+the device. Now that Cloud Hypervisor supports this feature, it can boot a VM
+with Rust Hypervisor Firmware or OVMF even if the `virtio-block` device exposing
+the disk image is placed behind a virtual IOMMU.
+
+### Ensure Identifiers Uniqueness
+
+Multiple checks have been added to the code to prevent devices with identical
+identifiers from being created, and therefore avoid unexpected behaviors at boot
+or whenever a device was hot plugged into the VM.
+
+### Sparse Mmap support
+
+Sparse mmap support has been added to both VFIO and vfio-user devices. This
+allows the device regions that are not fully mappable to be partially mapped.
+And the more a device region can be mapped into the guest address space, the
+fewer VM exits will be generated when this device is accessed. This directly
+impacts the performance related to this device.
+
+### Expose Platform Serial Number
+
+A new `serial_number` option has been added to `--platform`, allowing a user to
+set a specific serial number for the platform. This number is exposed to the
+guest through the SMBIOS.
+
+### Notable Bug Fixes
+
+* Fix loading RAW firmware (#4072)
+* Reject compressed QCOW images (#4055)
+* Reject virtio-mem resize if device is not activated (#4003)
+* Fix potential mmap leaks from VFIO/vfio-user MMIO regions (#4069)
+* Fix algorithm finding HOB memory resources (#3983)
+
+### Notable Improvements
+
+* Refactor interrupt handling (#4083)
+* Load kernel asynchronously (#4022)
+* Only create ACPI memory manager DSDT when resizable (#4013)
+
+### Deprecations
+
+Deprecated features will be removed in a subsequent release and users should
+plan to use alternatives
+
+* The `mergeable` option from the `virtio-pmem` support has been deprecated
+  (#3968)
+* The `dax` option from the `virtio-fs` support has been deprecated (#3889)
+
+### New on the Website
+
+A new blog post [Achieving Bare Metal Performance Within a Virtual
+Machine](https://www.cloudhypervisor.org/blog/achieving-bare-metal-performance-within-a-virtual-machine)
+has been added to the Cloud Hypervisor website.
+
+### Contributors
+
+Many thanks to everyone who has contributed to our release:
+
+* Anatol Belski <anbelski@linux.microsoft.com>
+* Bo Chen <chen.bo@intel.com>
+* Fabiano Fidêncio <fabiano.fidencio@intel.com>
+* LiHui <andrewli@kubesphere.io>
+* Maksym Pavlenko <pavlenko.maksym@gmail.com>
+* Rob Bradford <robert.bradford@intel.com>
+* Sebastien Boeuf <sebastien.boeuf@intel.com>
+* Steven Dake <steven.dake@gmail.com>
+* Vincent Batts <vbatts@hashbangbash.com>
+* Wei Liu <liuwe@microsoft.com>
+
+# v23.1
+
+This is a bug fix release. The following issues have been addressed:
+
+* Add some missing seccomp rules
+* Remove `virtio-fs` filesystem entries from config on removal
+* Do not delete API socket on API server start (#4026)
+* Reject `virtio-mem` resize if the guest doesn't activate the device
+* Fix OpenAPI naming of I/O throttling knobs
+
 # v23.0

 This release has been tracked through the [v23.0
--- a/resources/Dockerfile
+++ b/resources/Dockerfile
@@ -4,7 +4,7 @@
 FROM ubuntu:20.04 as dev

 ARG TARGETARCH
-ARG RUST_TOOLCHAIN="1.59.0"
+ARG RUST_TOOLCHAIN="1.60.0"
 ARG CLH_SRC_DIR="/cloud-hypervisor"
 ARG CLH_BUILD_DIR="$CLH_SRC_DIR/build"
 ARG CARGO_REGISTRY_DIR="$CLH_BUILD_DIR/cargo_registry"
--- a/rpm/cloud-hypervisor.spec
+++ b/rpm/cloud-hypervisor.spec
@@ -1,25 +1,32 @@
-# This spec file assumes you're building on an environment where:
-# * You have access to the internet during the build
-# * You have rustup installed on your system
-# * You have both x86_64-unknown-linux-gnu and  x86_64-unknown-linux-musl
-#   targets installed.
-
+# If this flag is set to 1, rustup installation needs to exist on the system and
+# <arch>-unknown-linux-gnu target is required.
+# If this flag is set to 0, distro specific Rust packages will be pulled into the build environment.
 %define using_rustup 1
+# If this flag is set to 1, <arch>-unknown-linux-musl target is required.
 %define using_musl_libc 1
+# If this flag is set to 1, the vendored crates archive and cargo.toml need to be prepared and
+# offline build is implied. Attached script update_src can be used for the vendorization.
+# If this flag is set to 0, access to the internet is required during the build.
+%define using_vendored_crates 0

 Name:           cloud-hypervisor
 Summary:        Cloud Hypervisor is an open source Virtual Machine Monitor (VMM) that runs on top of KVM.
-Version:        23.0
+Version:        24.0
 Release:        0%{?dist}
 License:        ASL 2.0 or BSD-3-clause
 Group:          Applications/System
 Source0:        https://github.com/cloud-hypervisor/cloud-hypervisor/archive/v%{version}.tar.gz
-ExclusiveArch:  x86_64
+%if 0%{?using_vendored_crates}
+Source1:        vendor.tar.gz
+Source2:        config.toml
+%endif
+ExclusiveArch:  x86_64 aarch64

 BuildRequires:  gcc
 BuildRequires:  glibc-devel
 BuildRequires:  binutils
 BuildRequires:  git
+BuildRequires:  openssl-devel

 %if ! 0%{?using_rustup}
 BuildRequires:  rust
@@ -31,29 +38,51 @@ Requires: glibc
 Requires: libgcc
 Requires: libcap
 
+%ifarch x86_64
+%define rust_def_target x86_64-unknown-linux-gnu
+%if 0%{?using_musl_libc}
+%define rust_musl_target x86_64-unknown-linux-musl
+%endif
+%endif
+%ifarch aarch64
+%define rust_def_target aarch64-unknown-linux-gnu
+%if 0%{?using_musl_libc}
+%define rust_musl_target aarch64-unknown-linux-musl
+%endif
+%endif
+
+%if 0%{?using_vendored_crates}
+%define cargo_offline --offline
+%endif
+
 %description
 Cloud Hypervisor is an open source Virtual Machine Monitor (VMM) that runs on top of KVM. The project focuses on exclusively running modern, cloud workloads, on top of a limited set of hardware architectures and platforms. Cloud workloads refers to those that are usually run by customers inside a cloud provider. For our purposes this means modern Linux* distributions with most I/O handled by paravirtualised devices (i.e. virtio), no requirement for legacy devices and recent CPUs and KVM.

 %prep

 %setup -q
+%if 0%{?using_vendored_crates}
+tar xf %{SOURCE1}
+mkdir -p .cargo
+cp %{SOURCE2} .cargo/
+%endif

 %install
 rm -rf %{buildroot}
 install -d %{buildroot}%{_bindir}
-install -D -m755  ./target/x86_64-unknown-linux-gnu/release/cloud-hypervisor %{buildroot}%{_bindir}
-install -D -m755  ./target/x86_64-unknown-linux-gnu/release/ch-remote %{buildroot}%{_bindir}
+install -D -m755  ./target/%{rust_def_target}/release/cloud-hypervisor %{buildroot}%{_bindir}
+install -D -m755  ./target/%{rust_def_target}/release/ch-remote %{buildroot}%{_bindir}
 install -d %{buildroot}%{_libdir}
 install -d %{buildroot}%{_libdir}/cloud-hypervisor
-install -D -m755 target/x86_64-unknown-linux-gnu/release/vhost_user_block %{buildroot}%{_libdir}/cloud-hypervisor
-install -D -m755 target/x86_64-unknown-linux-gnu/release/vhost_user_net %{buildroot}%{_libdir}/cloud-hypervisor
+install -D -m755 target/%{rust_def_target}/release/vhost_user_block %{buildroot}%{_libdir}/cloud-hypervisor
+install -D -m755 target/%{rust_def_target}/release/vhost_user_net %{buildroot}%{_libdir}/cloud-hypervisor

 %if 0%{?using_musl_libc}
 install -d %{buildroot}%{_libdir}/cloud-hypervisor/static
-install -D -m755 target/x86_64-unknown-linux-musl/release/cloud-hypervisor %{buildroot}%{_libdir}/cloud-hypervisor/static
-install -D -m755 target/x86_64-unknown-linux-musl/release/vhost_user_block %{buildroot}%{_libdir}/cloud-hypervisor/static
-install -D -m755 target/x86_64-unknown-linux-musl/release/vhost_user_net %{buildroot}%{_libdir}/cloud-hypervisor/static
-install -D -m755 target/x86_64-unknown-linux-musl/release/ch-remote %{buildroot}%{_libdir}/cloud-hypervisor/static
+install -D -m755 target/%{rust_musl_target}/release/cloud-hypervisor %{buildroot}%{_libdir}/cloud-hypervisor/static
+install -D -m755 target/%{rust_musl_target}/release/vhost_user_block %{buildroot}%{_libdir}/cloud-hypervisor/static
+install -D -m755 target/%{rust_musl_target}/release/vhost_user_net %{buildroot}%{_libdir}/cloud-hypervisor/static
+install -D -m755 target/%{rust_musl_target}/release/ch-remote %{buildroot}%{_libdir}/cloud-hypervisor/static
 %endif


@@ -74,21 +103,25 @@ fi
 echo ${cargo_version}

 %if 0%{?using_rustup}
-rustup target list --installed | grep x86_64-unknown-linux-gnu
+rustup target list --installed | grep -e "%{rust_def_target}"
 if [[ $? -ne 0 ]]; then
-         echo "Target  x86_64-unknown-linux-gnu not found, please install(#rustup target add x86_64-unknown-linux-gnu). exiting"
+         echo "Target  %{rust_def_target} not found, please install(#rustup target add %{rust_def_target}). exiting"
 fi
 	%if 0%{?using_musl_libc}
-rustup target list --installed | grep x86_64-unknown-linux-musl
+rustup target list --installed | grep -e "%{rust_musl_target}"
 if [[ $? -ne 0 ]]; then
-         echo "Target  x86_64-unknown-linux-musl not found, please install(#rustup target add x86_64-unknown-linux-musl). exiting"
+         echo "Target  %{rust_musl_target} not found, please install(#rustup target add %{rust_musl_target}). exiting"
 fi
 	%endif
 %endif

-cargo build --release --target=x86_64-unknown-linux-gnu --all
+%if 0%{?using_vendored_crates}
+# For vendored build, prepend this so openssl-sys doesn't trigger full OpenSSL build
+export OPENSSL_NO_VENDOR=1
+%endif
+cargo build --release --target=%{rust_def_target} --all %{cargo_offline}
 %if 0%{?using_musl_libc}
-cargo build --release --target=x86_64-unknown-linux-musl --all
+cargo build --release --target=%{rust_musl_target} --all %{cargo_offline}
 %endif


@@ -112,6 +145,15 @@ rm -rf %{buildroot}


 %changelog
+*   Wed May 25 2022 Sebastien Boeuf <sebastien.boeuf@intel.com> 24.0-0
+-   Update to 24.0
+
+*   Tue May 18 2022 Anatol Belski <anbelski@linux.microsoft.com> - 23.1-0
+-   Update to 23.1
+-   Add support for aarch64 build
+-   Add offline build configuration using vendored crates
+-   Fix dependency for openssl-sys
+
 *   Thu Apr 13 2022 Rob Bradford <robert.bradford@intel.com> 23.0-0
 -   Update to 23.0

--- a/rpm/update_src
+++ b/rpm/update_src
@@ -0,0 +1,36 @@
+#!/bin/bash
+
+set -e
+
+CH_VER=$1
+CH_SPEC_DIR=$(pwd)
+CH_TMP_DIR=$CH_SPEC_DIR/ch_tmp
+CH_SRC_DIR=$CH_SPEC_DIR/SOURCES
+
+if [ -z $CH_VER ]; then
+	echo "Replace sources with the given <version>"
+	echo "Usage: $0 <version>"
+	exit 1
+fi
+
+rm -rf $CH_TMP_DIR
+if [ ! -d $CH_SRC_DIR ]; then
+	mkdir $CH_SRC_DIR
+fi
+rm -f $CH_SRC_DIR/v*.tar.gz $CH_SRC_DIR/config.toml $CH_SRC_DIR/vendor.tar.gz
+
+
+wget https://github.com/cloud-hypervisor/cloud-hypervisor/archive/v${CH_VER}.tar.gz -O $CH_SRC_DIR/v${CH_VER}.tar.gz
+mkdir $CH_TMP_DIR
+tar xf $CH_SRC_DIR/v${CH_VER}.tar.gz -C $CH_TMP_DIR --strip-components=1
+
+
+pushd $CH_TMP_DIR
+
+cargo vendor > $CH_SRC_DIR/config.toml
+tar czvf $CH_SRC_DIR/vendor.tar.gz vendor
+
+popd
+
+rm -rf $CH_TMP_DIR
+
--- a/scripts/dev_cli.sh
+++ b/scripts/dev_cli.sh
@@ -7,7 +7,7 @@
 CLI_NAME="Cloud Hypervisor"

 CTR_IMAGE_TAG="cloudhypervisor/dev"
-CTR_IMAGE_VERSION="20220405-0"
+CTR_IMAGE_VERSION="20220524-0"
 CTR_IMAGE="${CTR_IMAGE_TAG}:${CTR_IMAGE_VERSION}"

 DOCKER_RUNTIME="docker"
@@ -226,6 +226,11 @@ cmd_build() {
        } ;;
        "--debug") { build="debug"; } ;;
        "--release") { build="release"; } ;;
+        "--runtime") 
+	    shift
+	    DOCKER_RUNTIME="$1"
+	    export DOCKER_RUNTIME
+	    ;;
        "--libc")
            shift
            [[ "$1" =~ ^(musl|gnu)$ ]] ||
--- a/scripts/run_integration_tests_aarch64.sh
+++ b/scripts/run_integration_tests_aarch64.sh
@@ -192,6 +192,15 @@ update_workloads() {
    fi
    popd

+    # Download Cloud Hypervisor binary from its last stable release
+    LAST_RELEASE_VERSION="v23.0"
+    CH_RELEASE_URL="https://github.com/cloud-hypervisor/cloud-hypervisor/releases/download/$LAST_RELEASE_VERSION/cloud-hypervisor-static-aarch64"
+    CH_RELEASE_NAME="cloud-hypervisor-static-aarch64"
+    pushd $WORKLOADS_DIR
+    time wget --quiet $CH_RELEASE_URL -O "$CH_RELEASE_NAME" || exit 1
+    chmod +x $CH_RELEASE_NAME
+    popd
+
    # Build custom kernel for guest VMs
    build_custom_linux

--- a/scripts/run_integration_tests_live_migration.sh
+++ b/scripts/run_integration_tests_live_migration.sh
@@ -45,6 +45,15 @@ if [ $? -ne 0 ]; then
 fi
 popd

+# Download Cloud Hypervisor binary from its last stable release
+LAST_RELEASE_VERSION="v23.0"
+CH_RELEASE_URL="https://github.com/cloud-hypervisor/cloud-hypervisor/releases/download/$LAST_RELEASE_VERSION/cloud-hypervisor-static"
+CH_RELEASE_NAME="cloud-hypervisor-static"
+pushd $WORKLOADS_DIR
+time wget --quiet $CH_RELEASE_URL -O "$CH_RELEASE_NAME" || exit 1
+chmod +x $CH_RELEASE_NAME
+popd
+
 # Build custom kernel based on virtio-pmem and virtio-fs upstream patches
 VMLINUX_IMAGE="$WORKLOADS_DIR/vmlinux"

--- a/scripts/run_unit_tests.sh
+++ b/scripts/run_unit_tests.sh
@@ -11,6 +11,8 @@ cargo_args=("")
 if [[ $hypervisor = "mshv" ]]; then
    cargo_args+=("--no-default-features")
    cargo_args+=("--features common,$hypervisor")
+elif [[ $(uname -m) = "x86_64" ]]; then
+    cargo_args+=("--features tdx")
 fi

 if [[ "${BUILD_TARGET}" == "aarch64-unknown-linux-musl" ]]; then
--- a/src/main.rs
+++ b/src/main.rs
@@ -161,7 +161,7 @@ fn create_app<'a>(
            Arg::new("platform")
                .long("platform")
                .help(
-                    "num_pci_segments=<num pci segments>,iommu_segments=<list_of_segments>",
+                    "num_pci_segments=<num pci segments>,iommu_segments=<list_of_segments>,serial_number=<(DMI) device serial number>",
                )
                .takes_value(true)
                .group("vm-config"),
--- a/test_infra/Cargo.toml
+++ b/test_infra/Cargo.toml
@@ -8,7 +8,7 @@ edition = "2021"
 dirs = "4.0.0"
 epoll = "4.3.1"
 lazy_static = "1.4.0"
-libc = "0.2.123"
+libc = "0.2.126"
 ssh2 = { version = "0.9.1", features = ["vendored-openssl"]}
 vmm-sys-util = "0.9.0"
 wait-timeout = "0.2.0"
--- a/test_infra/src/lib.rs
+++ b/test_infra/src/lib.rs
@@ -9,6 +9,7 @@ extern crate lazy_static;
 use ssh2::Session;
 use std::env;
 use std::ffi::OsStr;
+use std::fmt::Debug;
 use std::fs;
 use std::io;
 use std::io::{Read, Write};
@@ -1159,7 +1160,12 @@ impl Guest {
            .unwrap();
    }

-    pub fn check_devices_common(&self, socket: Option<&String>, console_text: Option<&String>) {
+    pub fn check_devices_common(
+        &self,
+        socket: Option<&String>,
+        console_text: Option<&String>,
+        pmem_path: Option<&String>,
+    ) {
        // Check block devices are readable
        self.ssh_command("sudo dd if=/dev/vda of=/dev/null bs=1M iflag=direct count=1024")
            .unwrap();
@@ -1178,6 +1184,63 @@ impl Guest {
            self.ssh_command(&console_cmd).unwrap();
        }
        // The net device is 'automatically' exercised through the above 'ssh' commands
+
+        // Check if the pmem device is usable
+        if let Some(pmem_path) = pmem_path {
+            assert_eq!(
+                self.ssh_command(&format!("ls {}", pmem_path))
+                    .unwrap()
+                    .trim(),
+                pmem_path
+            );
+            assert_eq!(
+                self.ssh_command(&format!("sudo mount {} /mnt", pmem_path))
+                    .unwrap(),
+                ""
+            );
+            assert_eq!(self.ssh_command("ls /mnt").unwrap(), "lost+found\n");
+            self.ssh_command("echo test123 | sudo tee /mnt/test")
+                .unwrap();
+            assert_eq!(self.ssh_command("sudo umount /mnt").unwrap(), "");
+            assert_eq!(self.ssh_command("ls /mnt").unwrap(), "");
+
+            assert_eq!(
+                self.ssh_command(&format!("sudo mount {} /mnt", pmem_path))
+                    .unwrap(),
+                ""
+            );
+            assert_eq!(
+                self.ssh_command("sudo cat /mnt/test || true")
+                    .unwrap()
+                    .trim(),
+                "test123"
+            );
+            self.ssh_command("sudo rm /mnt/test").unwrap();
+            assert_eq!(self.ssh_command("sudo umount /mnt").unwrap(), "");
+        }
+    }
+}
+
+pub enum VerbosityLevel {
+    Warn,
+    Info,
+    Debug,
+}
+
+impl Default for VerbosityLevel {
+    fn default() -> Self {
+        Self::Warn
+    }
+}
+
+impl ToString for VerbosityLevel {
+    fn to_string(&self) -> String {
+        use VerbosityLevel::*;
+        match self {
+            Warn => "".to_string(),
+            Info => "-v".to_string(),
+            Debug => "-vv".to_string(),
+        }
    }
 }

@@ -1186,22 +1249,29 @@ pub struct GuestCommand<'a> {
    guest: &'a Guest,
    capture_output: bool,
    print_cmd: bool,
+    verbosity: VerbosityLevel,
 }

 impl<'a> GuestCommand<'a> {
    pub fn new(guest: &'a Guest) -> Self {
-        Self::new_with_binary_name(guest, "cloud-hypervisor")
+        Self::new_with_binary_path(guest, &clh_command("cloud-hypervisor"))
    }

-    pub fn new_with_binary_name(guest: &'a Guest, binary_name: &str) -> Self {
+    pub fn new_with_binary_path(guest: &'a Guest, binary_path: &str) -> Self {
        Self {
-            command: Command::new(clh_command(binary_name)),
+            command: Command::new(binary_path),
            guest,
            capture_output: false,
            print_cmd: true,
+            verbosity: VerbosityLevel::Info,
        }
    }

+    pub fn verbosity(&mut self, verbosity: VerbosityLevel) -> &mut Self {
+        self.verbosity = verbosity;
+        self
+    }
+
    pub fn capture_output(&mut self) -> &mut Self {
        self.capture_output = true;
        self
@@ -1213,6 +1283,17 @@ impl<'a> GuestCommand<'a> {
    }

    pub fn spawn(&mut self) -> io::Result<Child> {
+        use VerbosityLevel::*;
+        match &self.verbosity {
+            Warn => {}
+            Info => {
+                self.command.arg("-v");
+            }
+            Debug => {
+                self.command.arg("-vv");
+            }
+        };
+
        if self.print_cmd {
            println!(
                "\n\n==== Start cloud-hypervisor command-line ====\n\n\
@@ -1225,7 +1306,6 @@ impl<'a> GuestCommand<'a> {
        if self.capture_output {
            let child = self
                .command
-                .arg("-v")
                .stderr(Stdio::piped())
                .stdout(Stdio::piped())
                .spawn()
@@ -1245,7 +1325,7 @@ impl<'a> GuestCommand<'a> {
                ))
            }
        } else {
-            self.command.arg("-v").spawn()
+            self.command.spawn()
        }
    }

--- a/tests/integration.rs
+++ b/tests/integration.rs
--- a/vfio_user/Cargo.toml
+++ b/vfio_user/Cargo.toml
@@ -5,14 +5,14 @@ authors = ["The Cloud Hypervisor Authors"]
 edition = "2021"

 [dependencies]
-anyhow = "1.0.56"
-libc = "0.2.123"
-log = "0.4.16"
+anyhow = "1.0.57"
+libc = "0.2.126"
+log = "0.4.17"
 serde = {version = ">=1.0.27", features = ["rc"] }
 serde_derive = ">=1.0.27"
 serde_json = ">=1.0.9"
-thiserror = "1.0.30"
-vm-memory = { version = "0.7.0", features = ["backend-mmap", "backend-atomic"] }
+thiserror = "1.0.31"
+vm-memory = { version = "0.8.0", features = ["backend-mmap", "backend-atomic"] }
 vmm-sys-util = ">=0.3.1"

 [dependencies.vfio-bindings]
--- a/vfio_user/src/lib.rs
+++ b/vfio_user/src/lib.rs
@@ -4,7 +4,9 @@
 //

 use std::ffi::CString;
+use std::fs::File;
 use std::io::{IoSlice, Read, Write};
+use std::mem::size_of;
 use std::num::Wrapping;
 use std::os::unix::net::UnixStream;
 use std::os::unix::prelude::RawFd;
@@ -234,6 +236,7 @@ pub struct Region {
    pub index: u32,
    pub size: u64,
    pub file_offset: Option<FileOffset>,
+    pub sparse_areas: Vec<vfio_region_sparse_mmap_area>,
 }

 #[derive(Debug)]
@@ -294,7 +297,7 @@ impl Client {
                message_id: self.next_message_id.0,
                command: Command::Version,
                flags: HeaderFlags::Command as u32,
-                message_size: (std::mem::size_of::<Version>() + version_data.len() + 1) as u32,
+                message_size: (size_of::<Version>() + version_data.len() + 1) as u32,
                ..Default::default()
            },
            major: 0,
@@ -330,7 +333,7 @@ impl Client {

        let mut server_version_data = Vec::new();
        server_version_data.resize(
-            server_version.header.message_size as usize - std::mem::size_of::<Version>(),
+            server_version.header.message_size as usize - size_of::<Version>(),
            0,
        );
        self.stream
@@ -361,10 +364,10 @@ impl Client {
                message_id: self.next_message_id.0,
                command: Command::DmaMap,
                flags: HeaderFlags::Command as u32,
-                message_size: std::mem::size_of::<DmaMap>() as u32,
+                message_size: size_of::<DmaMap>() as u32,
                ..Default::default()
            },
-            argsz: (std::mem::size_of::<DmaMap>() - std::mem::size_of::<Header>()) as u32,
+            argsz: (size_of::<DmaMap>() - size_of::<Header>()) as u32,
            flags: DmaMapFlags::ReadWrite,
            offset,
            address,
@@ -391,10 +394,10 @@ impl Client {
                message_id: self.next_message_id.0,
                command: Command::DmaUnmap,
                flags: HeaderFlags::Command as u32,
-                message_size: std::mem::size_of::<DmaUnmap>() as u32,
+                message_size: size_of::<DmaUnmap>() as u32,
                ..Default::default()
            },
-            argsz: (std::mem::size_of::<DmaUnmap>() - std::mem::size_of::<Header>()) as u32,
+            argsz: (size_of::<DmaUnmap>() - size_of::<Header>()) as u32,
            flags: 0,
            address,
            size,
@@ -420,7 +423,7 @@ impl Client {
                message_id: self.next_message_id.0,
                command: Command::DeviceReset,
                flags: HeaderFlags::Command as u32,
-                message_size: std::mem::size_of::<DeviceReset>() as u32,
+                message_size: size_of::<DeviceReset>() as u32,
                ..Default::default()
            },
        };
@@ -445,10 +448,10 @@ impl Client {
                message_id: self.next_message_id.0,
                command: Command::DeviceGetInfo,
                flags: HeaderFlags::Command as u32,
-                message_size: std::mem::size_of::<DeviceGetInfo>() as u32,
+                message_size: size_of::<DeviceGetInfo>() as u32,
                ..Default::default()
            },
-            argsz: std::mem::size_of::<DeviceGetInfo>() as u32,
+            argsz: size_of::<DeviceGetInfo>() as u32,
            ..Default::default()
        };
        debug!("Command: {:?}", get_info);
@@ -474,20 +477,62 @@ impl Client {
        let num_regions = reply.num_regions;
        let mut regions = Vec::new();
        for index in 0..num_regions {
-            let get_region_info = DeviceGetRegionInfo {
-                header: Header {
-                    message_id: self.next_message_id.0,
-                    command: Command::DeviceGetRegionInfo,
-                    flags: HeaderFlags::Command as u32,
-                    message_size: std::mem::size_of::<DeviceGetRegionInfo>() as u32,
-                    ..Default::default()
-                },
-                region_info: vfio_region_info {
-                    argsz: 1024, // Arbitrary max size
-                    index,
-                    ..Default::default()
-                },
-            };
+            let (region_info, fd, sparse_areas) = self.get_region_info(index)?;
+            regions.push(Region {
+                flags: region_info.flags,
+                index: region_info.index,
+                size: region_info.size,
+                file_offset: fd.map(|fd| FileOffset::new(fd, region_info.offset)),
+                sparse_areas,
+            });
+        }
+
+        Ok(regions)
+    }
+
+    fn get_region_info(
+        &mut self,
+        index: u32,
+    ) -> Result<
+        (
+            vfio_region_info,
+            Option<File>,
+            Vec<vfio_region_sparse_mmap_area>,
+        ),
+        Error,
+    > {
+        // Retrieve the region info without capability
+        let mut get_region_info = DeviceGetRegionInfo {
+            header: Header {
+                message_id: self.next_message_id.0,
+                command: Command::DeviceGetRegionInfo,
+                flags: HeaderFlags::Command as u32,
+                message_size: std::mem::size_of::<DeviceGetRegionInfo>() as u32,
+                ..Default::default()
+            },
+            region_info: vfio_region_info {
+                argsz: size_of::<vfio_region_info>() as u32,
+                index,
+                ..Default::default()
+            },
+        };
+        debug!("Command: {:?}", get_region_info);
+        self.next_message_id += Wrapping(1);
+
+        self.stream
+            .write_all(get_region_info.as_slice())
+            .map_err(Error::StreamWrite)?;
+
+        let mut reply = DeviceGetRegionInfo::default();
+        let (_, fd) = self
+            .stream
+            .recv_with_fd(reply.as_mut_slice())
+            .map_err(Error::ReceiveWithFd)?;
+        debug!("Reply: {:?}", reply);
+
+        // Retrieve the region info again with capabilities if needed
+        if reply.region_info.argsz > std::mem::size_of::<vfio_region_info>() as u32 {
+            get_region_info.region_info.argsz = reply.region_info.argsz;
            debug!("Command: {:?}", get_region_info);
            self.next_message_id += Wrapping(1);

@@ -502,24 +547,94 @@ impl Client {
                .map_err(Error::ReceiveWithFd)?;
            debug!("Reply: {:?}", reply);

-            regions.push(Region {
-                flags: reply.region_info.flags,
-                index: reply.region_info.index,
-                size: reply.region_info.size,
-                file_offset: fd.map(|fd| FileOffset::new(fd, reply.region_info.offset)),
-            });
-
-            // TODO: Handle region with capabilities
-            let mut _cap_data = Vec::with_capacity(
-                reply.header.message_size as usize - std::mem::size_of::<DeviceGetRegionInfo>(),
+            let cap_size = reply.region_info.argsz - std::mem::size_of::<vfio_region_info>() as u32;
+            assert_eq!(
+                cap_size,
+                reply.header.message_size - size_of::<DeviceGetRegionInfo>() as u32
            );
-            _cap_data.resize(_cap_data.capacity(), 0u8);
+            let mut cap_data = Vec::with_capacity(cap_size as usize);
+            cap_data.resize(cap_data.capacity(), 0u8);
            self.stream
-                .read_exact(_cap_data.as_mut_slice())
+                .read_exact(cap_data.as_mut_slice())
                .map_err(Error::StreamRead)?;
+
+            let sparse_areas = Self::parse_region_caps(&cap_data, &reply.region_info)?;
+
+            Ok((reply.region_info, fd, sparse_areas))
+        } else {
+            Ok((reply.region_info, fd, Vec::new()))
+        }
+    }
+
+    fn parse_region_caps(
+        cap_data: &[u8],
+        region_info: &vfio_region_info,
+    ) -> Result<Vec<vfio_region_sparse_mmap_area>, Error> {
+        let mut sparse_areas: Vec<vfio_region_sparse_mmap_area> = Vec::new();
+
+        let cap_size = cap_data.len() as u32;
+        let cap_header_size = size_of::<vfio_info_cap_header>() as u32;
+        let mmap_cap_size = size_of::<vfio_region_info_cap_sparse_mmap>() as u32;
+        let mmap_area_size = size_of::<vfio_region_sparse_mmap_area>() as u32;
+
+        let cap_data_ptr = cap_data.as_ptr() as *const u8;
+        let mut region_info_offset = region_info.cap_offset;
+        while region_info_offset != 0 {
+            // calculate the offset from the begining of the cap_data based on the offset
+            // that is relative to the begining of the VFIO region info structure
+            let cap_offset = region_info_offset - size_of::<vfio_region_info>() as u32;
+            if cap_offset + cap_header_size > cap_size {
+                warn!(
+                    "Unexpected end of cap data: 'cap_offset + cap_header_size > cap_size' \
+                cap_offset = {}, cap_header_size = {}, cap_size = {}",
+                    cap_offset, cap_header_size, cap_size
+                );
+                break;
+            }
+
+            // Safe because the `cap_data_ptr` is valid and the `cap_offset` is checked above
+            let cap_ptr = unsafe { cap_data_ptr.offset(cap_offset as isize) };
+            let cap_header = unsafe { &*(cap_ptr as *const vfio_info_cap_header) };
+            match cap_header.id as u32 {
+                VFIO_REGION_INFO_CAP_SPARSE_MMAP => {
+                    if cap_offset + mmap_cap_size > cap_size {
+                        warn!(
+                            "Unexpected end of cap data: 'cap_offset + mmap_cap_size > cap_size' \
+                        cap_offset = {}, mmap_cap_size = {}, cap_size = {}",
+                            cap_offset, mmap_cap_size, cap_size
+                        );
+                        break;
+                    }
+                    // Safe because the `cap_ptr` is valid and its size is also checked above
+                    let sparse_mmap = unsafe {
+                        &*(cap_ptr as *mut u8 as *const vfio_region_info_cap_sparse_mmap)
+                    };
+
+                    let area_num = sparse_mmap.nr_areas;
+                    if cap_offset + mmap_cap_size + area_num * mmap_area_size > cap_size {
+                        warn!("Unexpected end of cap data: 'cap_offset + mmap_cap_size + area_num * mmap_area_size > cap_size' \
+                        cap_offset = {}, mmap_cap_size = {}, area_num = {}, mmap_area_size = {}, cap_size = {}",
+                        cap_offset, mmap_cap_size, area_num, mmap_area_size, cap_size);
+                        break;
+                    }
+                    // Safe because the `sparse_mmap` is valid and its size is also checked above
+                    let areas =
+                        unsafe { sparse_mmap.areas.as_slice(sparse_mmap.nr_areas as usize) };
+                    for area in areas.iter() {
+                        sparse_areas.push(*area);
+                    }
+                }
+                _ => {
+                    warn!(
+                        "Ignoring unsupported vfio region capability (id = '{}')",
+                        cap_header.id
+                    );
+                }
+            }
+            region_info_offset = cap_header.next;
        }

-        Ok(regions)
+        Ok(sparse_areas)
    }

    pub fn region_read(&mut self, region: u32, offset: u64, data: &mut [u8]) -> Result<(), Error> {
@@ -528,7 +643,7 @@ impl Client {
                message_id: self.next_message_id.0,
                command: Command::RegionRead,
                flags: HeaderFlags::Command as u32,
-                message_size: std::mem::size_of::<RegionAccess>() as u32,
+                message_size: size_of::<RegionAccess>() as u32,
                ..Default::default()
            },
            offset,
@@ -556,7 +671,7 @@ impl Client {
                message_id: self.next_message_id.0,
                command: Command::RegionWrite,
                flags: HeaderFlags::Command as u32,
-                message_size: (std::mem::size_of::<RegionAccess>() + data.len()) as u32,
+                message_size: (size_of::<RegionAccess>() + data.len()) as u32,
                ..Default::default()
            },
            offset,
@@ -588,10 +703,10 @@ impl Client {
                message_id: self.next_message_id.0,
                command: Command::GetIrqInfo,
                flags: HeaderFlags::Command as u32,
-                message_size: std::mem::size_of::<GetIrqInfo>() as u32,
+                message_size: size_of::<GetIrqInfo>() as u32,
                ..Default::default()
            },
-            argsz: (std::mem::size_of::<GetIrqInfo>() - std::mem::size_of::<Header>()) as u32,
+            argsz: (size_of::<GetIrqInfo>() - size_of::<Header>()) as u32,
            flags: 0,
            index,
            count: 0,
@@ -629,10 +744,10 @@ impl Client {
                message_id: self.next_message_id.0,
                command: Command::SetIrqs,
                flags: HeaderFlags::Command as u32,
-                message_size: std::mem::size_of::<SetIrqs>() as u32,
+                message_size: size_of::<SetIrqs>() as u32,
                ..Default::default()
            },
-            argsz: (std::mem::size_of::<SetIrqs>() - std::mem::size_of::<Header>()) as u32,
+            argsz: (size_of::<SetIrqs>() - size_of::<Header>()) as u32,
            flags,
            start,
            index,
--- a/vhdx/Cargo.toml
+++ b/vhdx/Cargo.toml
@@ -8,9 +8,9 @@ license = "Apache-2.0"
 [dependencies]
 byteorder = "1.4.3"
 crc32c = "0.6.3"
-libc = "0.2.123"
-log = "0.4.16"
-remain = "0.2.2"
+libc = "0.2.126"
+log = "0.4.17"
+remain = "0.2.3"
 thiserror = "1.0"
-uuid = { version = "0.8.2", features = ["v4"] }
+uuid = { version = "1.0.0", features = ["v4"] }
 vmm-sys-util = ">=0.3.1"
--- a/vhdx/src/lib.rs
+++ b/vhdx/src/lib.rs
@@ -3,7 +3,6 @@
 // SPDX-License-Identifier: Apache-2.0

 use byteorder::{BigEndian, ByteOrder};
-use std::result::Result;
 use uuid::Uuid;

 macro_rules! div_round_up {
@@ -18,7 +17,7 @@ mod vhdx_header;
 mod vhdx_io;
 mod vhdx_metadata;

-pub(crate) fn uuid_from_guid(buf: &[u8]) -> Result<Uuid, uuid::Error> {
+pub(crate) fn uuid_from_guid(buf: &[u8]) -> Uuid {
    // The first 3 fields of UUID are stored in Big Endian format, and
    // the last 8 bytes are stored as byte array. Therefore, we read the
    // first 3 fields in Big Endian format instead of Little Endian.
@@ -26,6 +25,6 @@ pub(crate) fn uuid_from_guid(buf: &[u8]) -> Result<Uuid, uuid::Error> {
        BigEndian::read_u32(&buf[0..4]),
        BigEndian::read_u16(&buf[4..6]),
        BigEndian::read_u16(&buf[6..8]),
-        &buf[8..16],
+        buf[8..16].try_into().unwrap(),
    )
 }
--- a/vhdx/src/vhdx_header.rs
+++ b/vhdx/src/vhdx_header.rs
@@ -339,7 +339,7 @@ impl RegionTableEntry {
    pub fn new(buffer: &[u8]) -> Result<RegionTableEntry> {
        let mut region_table_entry = unsafe { *(buffer.as_ptr() as *mut RegionTableEntry) };

-        let uuid = crate::uuid_from_guid(buffer).map_err(VhdxHeaderError::InvalidUuid)?;
+        let uuid = crate::uuid_from_guid(buffer);
        region_table_entry.guid = uuid;

        Ok(region_table_entry)
--- a/vhdx/src/vhdx_metadata.rs
+++ b/vhdx/src/vhdx_metadata.rs
@@ -303,7 +303,7 @@ impl MetadataTableEntry {
    fn new(buffer: &[u8]) -> Result<MetadataTableEntry> {
        let mut metadata_table_entry = unsafe { *(buffer.as_ptr() as *mut MetadataTableEntry) };

-        let uuid = crate::uuid_from_guid(buffer).map_err(VhdxMetadataError::InvalidUuid)?;
+        let uuid = crate::uuid_from_guid(buffer);
        metadata_table_entry.item_id = uuid;

        if metadata_table_entry.length > METADATA_LENGTH_MAX {
--- a/vhost_user_block/Cargo.toml
+++ b/vhost_user_block/Cargo.toml
@@ -6,18 +6,18 @@ edition = "2021"

 [dependencies]
 block_util = { path = "../block_util" }
-clap = { version = "3.1.8", features = ["wrap_help","cargo"] }
+clap = { version = "3.1.18", features = ["wrap_help","cargo"] }
 env_logger = "0.9.0"
 epoll = "4.3.1"
-libc = "0.2.123"
-log = "0.4.16"
+libc = "0.2.126"
+log = "0.4.17"
 option_parser = { path = "../option_parser" }
 qcow = { path = "../qcow" }
 vhost = { version = "0.4.0", features = ["vhost-user-slave"] }
-vhost-user-backend = "0.3.0"
+vhost-user-backend = { git = "https://github.com/rust-vmm/vhost-user-backend", rev = "14f58eda14076e973704d4f904850be1146fbb05" }
 virtio-bindings = "0.1.0"
-vm-memory = "0.7.0"
+vm-memory = "0.8.0"
 vmm-sys-util = "0.9.0"

 [build-dependencies]
-clap = { version = "3.1.8", features = ["cargo"] }
+clap = { version = "3.1.18", features = ["cargo"] }
--- a/vhost_user_net/Cargo.toml
+++ b/vhost_user_net/Cargo.toml
@@ -5,18 +5,18 @@ authors = ["The Cloud Hypervisor Authors"]
 edition = "2021"

 [dependencies]
-clap = { version = "3.1.8", features = ["wrap_help","cargo"] }
+clap = { version = "3.1.18", features = ["wrap_help","cargo"] }
 env_logger = "0.9.0"
 epoll = "4.3.1"
-libc = "0.2.123"
-log = "0.4.16"
+libc = "0.2.126"
+log = "0.4.17"
 net_util = { path = "../net_util" }
 option_parser = { path = "../option_parser" }
 vhost = { version = "0.4.0", features = ["vhost-user-slave"] }
-vhost-user-backend = "0.3.0"
+vhost-user-backend = { git = "https://github.com/rust-vmm/vhost-user-backend", rev = "14f58eda14076e973704d4f904850be1146fbb05" }
 virtio-bindings = "0.1.0"
-vm-memory = "0.7.0"
+vm-memory = "0.8.0"
 vmm-sys-util = "0.9.0"

 [build-dependencies]
-clap = { version = "3.1.8", features = ["cargo"] }
+clap = { version = "3.1.18", features = ["cargo"] }
--- a/virtio-devices/Cargo.toml
+++ b/virtio-devices/Cargo.toml
@@ -9,32 +9,31 @@ default = []
 mshv = []

 [dependencies]
-anyhow = "1.0.56"
+anyhow = "1.0.57"
 arc-swap = "1.5.0"
 block_util = { path = "../block_util" }
 byteorder = "1.4.3"
 epoll = "4.3.1"
 event_monitor = { path = "../event_monitor" }
 io-uring = "0.5.2"
-libc = "0.2.123"
-log = "0.4.16"
+libc = "0.2.126"
+log = "0.4.17"
 net_gen = { path = "../net_gen" }
 net_util = { path = "../net_util" }
 pci = { path = "../pci" }
 rate_limiter = { path = "../rate_limiter" }
 seccompiler = "0.2.0"
-serde = "1.0.136"
-serde_derive = "1.0.136"
-serde_json = "1.0.79"
-thiserror = "1.0.30"
+serde = { version="1.0.137", features=["derive"] }
+serde_json = "1.0.81"
+thiserror = "1.0.31"
 versionize = "0.1.6"
 versionize_derive = "0.1.4"
 vhost = { version = "0.4.0", features = ["vhost-user-master", "vhost-user-slave", "vhost-kern", "vhost-vdpa"] }
 virtio-bindings = { version = "0.1.0", features = ["virtio-v5_0_0"] }
-virtio-queue = "0.2.0"
+virtio-queue = "0.3.0"
 vm-allocator = { path = "../vm-allocator" }
 vm-device = { path = "../vm-device" }
-vm-memory = { version = "0.7.0", features = ["backend-mmap", "backend-atomic", "backend-bitmap"] }
+vm-memory = { version = "0.8.0", features = ["backend-mmap", "backend-atomic", "backend-bitmap"] }
 vm-migration = { path = "../vm-migration" }
 vm-virtio = { path = "../vm-virtio" }
 vmm-sys-util = "0.9.0"
--- a/virtio-devices/src/iommu.rs
+++ b/virtio-devices/src/iommu.rs
@@ -19,7 +19,7 @@ use std::mem::size_of;
 use std::ops::Bound::Included;
 use std::os::unix::io::AsRawFd;
 use std::result;
-use std::sync::atomic::AtomicBool;
+use std::sync::atomic::{AtomicBool, Ordering};
 use std::sync::{Arc, Barrier, Mutex, RwLock};
 use versionize::{VersionMap, Versionize, VersionizeResult};
 use versionize_derive::Versionize;
@@ -69,7 +69,6 @@ const VIRTIO_IOMMU_F_BYPASS: u32 = 3;
 const VIRTIO_IOMMU_F_PROBE: u32 = 4;
 #[allow(unused)]
 const VIRTIO_IOMMU_F_MMIO: u32 = 5;
-#[allow(unused)]
 const VIRTIO_IOMMU_F_BYPASS_CONFIG: u32 = 6;

 // Support 2MiB and 4KiB page sizes.
@@ -150,9 +149,12 @@ struct VirtioIommuReqTail {
 struct VirtioIommuReqAttach {
    domain: u32,
    endpoint: u32,
-    _reserved: [u8; 8],
+    flags: u32,
+    _reserved: [u8; 4],
 }

+const VIRTIO_IOMMU_ATTACH_F_BYPASS: u32 = 1;
+
 /// DETACH request
 #[derive(Copy, Clone, Debug, Default)]
 #[repr(packed)]
@@ -299,8 +301,16 @@ enum Error {
    InvalidDetachRequest,
    /// Guest sent us invalid MAP request.
    InvalidMapRequest,
+    /// Invalid to map because the domain is in bypass mode.
+    InvalidMapRequestBypassDomain,
+    /// Invalid to map because the domain is missing.
+    InvalidMapRequestMissingDomain,
    /// Guest sent us invalid UNMAP request.
    InvalidUnmapRequest,
+    /// Invalid to unmap because the domain is in bypass mode.
+    InvalidUnmapRequestBypassDomain,
+    /// Invalid to unmap because the domain is missing.
+    InvalidUnmapRequestMissingDomain,
    /// Guest sent us invalid PROBE request.
    InvalidProbeRequest,
    /// Failed to performing external mapping.
@@ -321,7 +331,19 @@ impl Display for Error {
            InvalidAttachRequest => write!(f, "invalid attach request"),
            InvalidDetachRequest => write!(f, "invalid detach request"),
            InvalidMapRequest => write!(f, "invalid map request"),
+            InvalidMapRequestBypassDomain => {
+                write!(f, "invalid map request because domain in bypass mode")
+            }
+            InvalidMapRequestMissingDomain => {
+                write!(f, "invalid map request because missing domain")
+            }
            InvalidUnmapRequest => write!(f, "invalid unmap request"),
+            InvalidUnmapRequestBypassDomain => {
+                write!(f, "invalid unmap request because domain in bypass mode")
+            }
+            InvalidUnmapRequestMissingDomain => {
+                write!(f, "invalid unmap request because missing domain")
+            }
            InvalidProbeRequest => write!(f, "invalid probe request"),
            UnexpectedReadOnlyDescriptor => write!(f, "unexpected read-only descriptor"),
            UnexpectedWriteOnlyDescriptor => write!(f, "unexpected write-only descriptor"),
@@ -346,7 +368,6 @@ impl Request {
        desc_chain: &mut DescriptorChain<GuestMemoryLoadGuard<GuestMemoryMmap>>,
        mapping: &Arc<IommuMapping>,
        ext_mapping: &BTreeMap<u32, Arc<dyn ExternalDmaMapping>>,
-        ext_domain_mapping: &mut BTreeMap<u32, Arc<dyn ExternalDmaMapping>>,
        msi_iova_space: (u64, u64),
    ) -> result::Result<usize, Error> {
        let desc = desc_chain
@@ -382,162 +403,230 @@ impl Request {

        // Create the reply
        let mut reply: Vec<u8> = Vec::new();
+        let mut status = VIRTIO_IOMMU_S_OK;
+        let mut hdr_len = 0;

-        let hdr_len = match req_head.type_ {
-            VIRTIO_IOMMU_T_ATTACH => {
-                if desc_size_left != size_of::<VirtioIommuReqAttach>() {
-                    return Err(Error::InvalidAttachRequest);
+        let result = (|| {
+            match req_head.type_ {
+                VIRTIO_IOMMU_T_ATTACH => {
+                    if desc_size_left != size_of::<VirtioIommuReqAttach>() {
+                        status = VIRTIO_IOMMU_S_INVAL;
+                        return Err(Error::InvalidAttachRequest);
+                    }
+
+                    let req: VirtioIommuReqAttach = desc_chain
+                        .memory()
+                        .read_obj(req_addr as GuestAddress)
+                        .map_err(Error::GuestMemory)?;
+                    debug!("Attach request {:?}", req);
+
+                    // Copy the value to use it as a proper reference.
+                    let domain_id = req.domain;
+                    let endpoint = req.endpoint;
+                    let bypass =
+                        (req.flags & VIRTIO_IOMMU_ATTACH_F_BYPASS) == VIRTIO_IOMMU_ATTACH_F_BYPASS;
+
+                    // Add endpoint associated with specific domain
+                    mapping
+                        .endpoints
+                        .write()
+                        .unwrap()
+                        .insert(endpoint, domain_id);
+
+                    // Add new domain with no mapping if the entry didn't exist yet
+                    let mut domains = mapping.domains.write().unwrap();
+                    let domain = Domain {
+                        mappings: BTreeMap::new(),
+                        bypass,
+                    };
+                    domains.entry(domain_id).or_insert_with(|| domain);
                }
+                VIRTIO_IOMMU_T_DETACH => {
+                    if desc_size_left != size_of::<VirtioIommuReqDetach>() {
+                        status = VIRTIO_IOMMU_S_INVAL;
+                        return Err(Error::InvalidDetachRequest);
+                    }

-                let req: VirtioIommuReqAttach = desc_chain
-                    .memory()
-                    .read_obj(req_addr as GuestAddress)
-                    .map_err(Error::GuestMemory)?;
-                debug!("Attach request {:?}", req);
+                    let req: VirtioIommuReqDetach = desc_chain
+                        .memory()
+                        .read_obj(req_addr as GuestAddress)
+                        .map_err(Error::GuestMemory)?;
+                    debug!("Detach request {:?}", req);

-                // Copy the value to use it as a proper reference.
-                let domain = req.domain;
-                let endpoint = req.endpoint;
+                    // Copy the value to use it as a proper reference.
+                    let domain_id = req.domain;
+                    let endpoint = req.endpoint;

-                // Add endpoint associated with specific domain
-                mapping.endpoints.write().unwrap().insert(endpoint, domain);
+                    // Remove endpoint associated with specific domain
+                    mapping.endpoints.write().unwrap().remove(&endpoint);

-                // If the endpoint is part of the list of devices with an
-                // external mapping, insert a new entry for the corresponding
-                // domain, with the same reference to the trait.
-                if let Some(map) = ext_mapping.get(&endpoint) {
-                    ext_domain_mapping.insert(domain, map.clone());
+                    // After all endpoints have been successfully detached from a
+                    // domain, the domain can be removed. This means we must remove
+                    // the mappings associated with this domain.
+                    if mapping
+                        .endpoints
+                        .write()
+                        .unwrap()
+                        .iter()
+                        .filter(|(_, &d)| d == domain_id)
+                        .count()
+                        == 0
+                    {
+                        mapping.domains.write().unwrap().remove(&domain_id);
+                    }
                }
+                VIRTIO_IOMMU_T_MAP => {
+                    if desc_size_left != size_of::<VirtioIommuReqMap>() {
+                        status = VIRTIO_IOMMU_S_INVAL;
+                        return Err(Error::InvalidMapRequest);
+                    }

-                // Add new domain with no mapping if the entry didn't exist yet
-                let mut mappings = mapping.mappings.write().unwrap();
-                mappings.entry(domain).or_insert_with(BTreeMap::new);
+                    let req: VirtioIommuReqMap = desc_chain
+                        .memory()
+                        .read_obj(req_addr as GuestAddress)
+                        .map_err(Error::GuestMemory)?;
+                    debug!("Map request {:?}", req);

-                0
+                    // Copy the value to use it as a proper reference.
+                    let domain_id = req.domain;
+
+                    if let Some(domain) = mapping.domains.read().unwrap().get(&domain_id) {
+                        if domain.bypass {
+                            status = VIRTIO_IOMMU_S_INVAL;
+                            return Err(Error::InvalidMapRequestBypassDomain);
+                        }
+                    } else {
+                        status = VIRTIO_IOMMU_S_INVAL;
+                        return Err(Error::InvalidMapRequestMissingDomain);
+                    }
+
+                    // Find the list of endpoints attached to the given domain.
+                    let endpoints: Vec<u32> = mapping
+                        .endpoints
+                        .write()
+                        .unwrap()
+                        .iter()
+                        .filter(|(_, &d)| d == domain_id)
+                        .map(|(&e, _)| e)
+                        .collect();
+
+                    // Trigger external mapping if necessary.
+                    for endpoint in endpoints {
+                        if let Some(ext_map) = ext_mapping.get(&endpoint) {
+                            let size = req.virt_end - req.virt_start + 1;
+                            ext_map
+                                .map(req.virt_start, req.phys_start, size)
+                                .map_err(Error::ExternalMapping)?;
+                        }
+                    }
+
+                    // Add new mapping associated with the domain
+                    mapping
+                        .domains
+                        .write()
+                        .unwrap()
+                        .get_mut(&domain_id)
+                        .unwrap()
+                        .mappings
+                        .insert(
+                            req.virt_start,
+                            Mapping {
+                                gpa: req.phys_start,
+                                size: req.virt_end - req.virt_start + 1,
+                            },
+                        );
+                }
+                VIRTIO_IOMMU_T_UNMAP => {
+                    if desc_size_left != size_of::<VirtioIommuReqUnmap>() {
+                        status = VIRTIO_IOMMU_S_INVAL;
+                        return Err(Error::InvalidUnmapRequest);
+                    }
+
+                    let req: VirtioIommuReqUnmap = desc_chain
+                        .memory()
+                        .read_obj(req_addr as GuestAddress)
+                        .map_err(Error::GuestMemory)?;
+                    debug!("Unmap request {:?}", req);
+
+                    // Copy the value to use it as a proper reference.
+                    let domain_id = req.domain;
+                    let virt_start = req.virt_start;
+
+                    if let Some(domain) = mapping.domains.read().unwrap().get(&domain_id) {
+                        if domain.bypass {
+                            status = VIRTIO_IOMMU_S_INVAL;
+                            return Err(Error::InvalidUnmapRequestBypassDomain);
+                        }
+                    } else {
+                        status = VIRTIO_IOMMU_S_INVAL;
+                        return Err(Error::InvalidUnmapRequestMissingDomain);
+                    }
+
+                    // Find the list of endpoints attached to the given domain.
+                    let endpoints: Vec<u32> = mapping
+                        .endpoints
+                        .write()
+                        .unwrap()
+                        .iter()
+                        .filter(|(_, &d)| d == domain_id)
+                        .map(|(&e, _)| e)
+                        .collect();
+
+                    // Trigger external unmapping if necessary.
+                    for endpoint in endpoints {
+                        if let Some(ext_map) = ext_mapping.get(&endpoint) {
+                            let size = req.virt_end - virt_start + 1;
+                            ext_map
+                                .unmap(virt_start, size)
+                                .map_err(Error::ExternalUnmapping)?;
+                        }
+                    }
+
+                    // Remove mapping associated with the domain
+                    mapping
+                        .domains
+                        .write()
+                        .unwrap()
+                        .get_mut(&domain_id)
+                        .unwrap()
+                        .mappings
+                        .remove(&virt_start);
+                }
+                VIRTIO_IOMMU_T_PROBE => {
+                    if desc_size_left != size_of::<VirtioIommuReqProbe>() {
+                        status = VIRTIO_IOMMU_S_INVAL;
+                        return Err(Error::InvalidProbeRequest);
+                    }
+
+                    let req: VirtioIommuReqProbe = desc_chain
+                        .memory()
+                        .read_obj(req_addr as GuestAddress)
+                        .map_err(Error::GuestMemory)?;
+                    debug!("Probe request {:?}", req);
+
+                    let probe_prop = VirtioIommuProbeProperty {
+                        type_: VIRTIO_IOMMU_PROBE_T_RESV_MEM,
+                        length: size_of::<VirtioIommuProbeResvMem>() as u16,
+                    };
+                    reply.extend_from_slice(probe_prop.as_slice());
+
+                    let resv_mem = VirtioIommuProbeResvMem {
+                        subtype: VIRTIO_IOMMU_RESV_MEM_T_MSI,
+                        start: msi_iova_start,
+                        end: msi_iova_end,
+                        ..Default::default()
+                    };
+                    reply.extend_from_slice(resv_mem.as_slice());
+
+                    hdr_len = PROBE_PROP_SIZE;
+                }
+                _ => {
+                    status = VIRTIO_IOMMU_S_INVAL;
+                    return Err(Error::InvalidRequest);
+                }
            }
-            VIRTIO_IOMMU_T_DETACH => {
-                if desc_size_left != size_of::<VirtioIommuReqDetach>() {
-                    return Err(Error::InvalidDetachRequest);
-                }
-
-                let req: VirtioIommuReqDetach = desc_chain
-                    .memory()
-                    .read_obj(req_addr as GuestAddress)
-                    .map_err(Error::GuestMemory)?;
-                debug!("Detach request {:?}", req);
-
-                // Copy the value to use it as a proper reference.
-                let domain = req.domain;
-                let endpoint = req.endpoint;
-
-                // If the endpoint is part of the list of devices with an
-                // external mapping, remove the entry for the corresponding
-                // domain.
-                if ext_mapping.contains_key(&endpoint) {
-                    ext_domain_mapping.remove(&domain);
-                }
-
-                // Remove endpoint associated with specific domain
-                mapping.endpoints.write().unwrap().remove(&endpoint);
-
-                0
-            }
-            VIRTIO_IOMMU_T_MAP => {
-                if desc_size_left != size_of::<VirtioIommuReqMap>() {
-                    return Err(Error::InvalidMapRequest);
-                }
-
-                let req: VirtioIommuReqMap = desc_chain
-                    .memory()
-                    .read_obj(req_addr as GuestAddress)
-                    .map_err(Error::GuestMemory)?;
-                debug!("Map request {:?}", req);
-
-                // Copy the value to use it as a proper reference.
-                let domain = req.domain;
-
-                // Trigger external mapping if necessary.
-                if let Some(ext_map) = ext_domain_mapping.get(&domain) {
-                    let size = req.virt_end - req.virt_start + 1;
-                    ext_map
-                        .map(req.virt_start, req.phys_start, size)
-                        .map_err(Error::ExternalMapping)?;
-                }
-
-                // Add new mapping associated with the domain
-                if let Some(entry) = mapping.mappings.write().unwrap().get_mut(&domain) {
-                    entry.insert(
-                        req.virt_start,
-                        Mapping {
-                            gpa: req.phys_start,
-                            size: req.virt_end - req.virt_start + 1,
-                        },
-                    );
-                } else {
-                    return Err(Error::InvalidMapRequest);
-                }
-
-                0
-            }
-            VIRTIO_IOMMU_T_UNMAP => {
-                if desc_size_left != size_of::<VirtioIommuReqUnmap>() {
-                    return Err(Error::InvalidUnmapRequest);
-                }
-
-                let req: VirtioIommuReqUnmap = desc_chain
-                    .memory()
-                    .read_obj(req_addr as GuestAddress)
-                    .map_err(Error::GuestMemory)?;
-                debug!("Unmap request {:?}", req);
-
-                // Copy the value to use it as a proper reference.
-                let domain = req.domain;
-                let virt_start = req.virt_start;
-
-                // Trigger external unmapping if necessary.
-                if let Some(ext_map) = ext_domain_mapping.get(&domain) {
-                    let size = req.virt_end - virt_start + 1;
-                    ext_map
-                        .unmap(virt_start, size)
-                        .map_err(Error::ExternalUnmapping)?;
-                }
-
-                // Add new mapping associated with the domain
-                if let Some(entry) = mapping.mappings.write().unwrap().get_mut(&domain) {
-                    entry.remove(&virt_start);
-                }
-
-                0
-            }
-            VIRTIO_IOMMU_T_PROBE => {
-                if desc_size_left != size_of::<VirtioIommuReqProbe>() {
-                    return Err(Error::InvalidProbeRequest);
-                }
-
-                let req: VirtioIommuReqProbe = desc_chain
-                    .memory()
-                    .read_obj(req_addr as GuestAddress)
-                    .map_err(Error::GuestMemory)?;
-                debug!("Probe request {:?}", req);
-
-                let probe_prop = VirtioIommuProbeProperty {
-                    type_: VIRTIO_IOMMU_PROBE_T_RESV_MEM,
-                    length: size_of::<VirtioIommuProbeResvMem>() as u16,
-                };
-                reply.extend_from_slice(probe_prop.as_slice());
-
-                let resv_mem = VirtioIommuProbeResvMem {
-                    subtype: VIRTIO_IOMMU_RESV_MEM_T_MSI,
-                    start: msi_iova_start,
-                    end: msi_iova_end,
-                    ..Default::default()
-                };
-                reply.extend_from_slice(resv_mem.as_slice());
-
-                PROBE_PROP_SIZE
-            }
-            _ => return Err(Error::InvalidRequest),
-        };
+            Ok(())
+        })();

        let status_desc = desc_chain.next().ok_or(Error::DescriptorChainTooShort)?;

@@ -551,16 +640,21 @@ impl Request {
        }

        let tail = VirtioIommuReqTail {
-            status: VIRTIO_IOMMU_S_OK,
+            status,
            ..Default::default()
        };
        reply.extend_from_slice(tail.as_slice());

+        // Make sure we return the result of the request to the guest before
+        // we return a potential error internally.
        desc_chain
            .memory()
            .write_slice(reply.as_slice(), status_desc.addr())
            .map_err(Error::GuestMemory)?;

+        // Return the error if the result was not Ok().
+        result?;
+
        Ok((hdr_len as usize) + size_of::<VirtioIommuReqTail>())
    }
 }
@@ -573,7 +667,6 @@ struct IommuEpollHandler {
    pause_evt: EventFd,
    mapping: Arc<IommuMapping>,
    ext_mapping: Arc<Mutex<BTreeMap<u32, Arc<dyn ExternalDmaMapping>>>>,
-    ext_domain_mapping: BTreeMap<u32, Arc<dyn ExternalDmaMapping>>,
    msi_iova_space: (u64, u64),
 }

@@ -586,7 +679,6 @@ impl IommuEpollHandler {
                &mut desc_chain,
                &self.mapping,
                &self.ext_mapping.lock().unwrap(),
-                &mut self.ext_domain_mapping,
                self.msi_iova_space,
            ) {
                Ok(len) => len as u32,
@@ -674,25 +766,43 @@ struct Mapping {
    size: u64,
 }

+#[derive(Clone, Debug)]
+struct Domain {
+    mappings: BTreeMap<u64, Mapping>,
+    bypass: bool,
+}
+
 #[derive(Debug)]
 pub struct IommuMapping {
    // Domain related to an endpoint.
    endpoints: Arc<RwLock<BTreeMap<u32, u32>>>,
-    // List of mappings per domain.
-    mappings: Arc<RwLock<BTreeMap<u32, BTreeMap<u64, Mapping>>>>,
+    // Information related to each domain.
+    domains: Arc<RwLock<BTreeMap<u32, Domain>>>,
+    // Global flag indicating if endpoints that are not attached to any domain
+    // are in bypass mode.
+    bypass: AtomicBool,
 }

 impl DmaRemapping for IommuMapping {
    fn translate_gva(&self, id: u32, addr: u64) -> std::result::Result<u64, std::io::Error> {
        debug!("Translate GVA addr 0x{:x}", addr);
-        if let Some(domain) = self.endpoints.read().unwrap().get(&id) {
-            if let Some(mapping) = self.mappings.read().unwrap().get(domain) {
+        if let Some(domain_id) = self.endpoints.read().unwrap().get(&id) {
+            if let Some(domain) = self.domains.read().unwrap().get(domain_id) {
+                // Directly return identity mapping in case the domain is in
+                // bypass mode.
+                if domain.bypass {
+                    return Ok(addr);
+                }
+
                let range_start = if VIRTIO_IOMMU_PAGE_SIZE_MASK > addr {
                    0
                } else {
                    addr - VIRTIO_IOMMU_PAGE_SIZE_MASK
                };
-                for (&key, &value) in mapping.range((Included(&range_start), Included(&addr))) {
+                for (&key, &value) in domain
+                    .mappings
+                    .range((Included(&range_start), Included(&addr)))
+                {
                    if addr >= key && addr < key + value.size {
                        let new_addr = addr - key + value.gpa;
                        debug!("Into GPA addr 0x{:x}", new_addr);
@@ -700,6 +810,8 @@ impl DmaRemapping for IommuMapping {
                    }
                }
            }
+        } else if self.bypass.load(Ordering::Acquire) {
+            return Ok(addr);
        }

        Err(io::Error::new(
@@ -710,9 +822,15 @@ impl DmaRemapping for IommuMapping {

    fn translate_gpa(&self, id: u32, addr: u64) -> std::result::Result<u64, std::io::Error> {
        debug!("Translate GPA addr 0x{:x}", addr);
-        if let Some(domain) = self.endpoints.read().unwrap().get(&id) {
-            if let Some(mapping) = self.mappings.read().unwrap().get(domain) {
-                for (&key, &value) in mapping.iter() {
+        if let Some(domain_id) = self.endpoints.read().unwrap().get(&id) {
+            if let Some(domain) = self.domains.read().unwrap().get(domain_id) {
+                // Directly return identity mapping in case the domain is in
+                // bypass mode.
+                if domain.bypass {
+                    return Ok(addr);
+                }
+
+                for (&key, &value) in domain.mappings.iter() {
                    if addr >= value.gpa && addr < value.gpa + value.size {
                        let new_addr = addr - value.gpa + key;
                        debug!("Into GVA addr 0x{:x}", new_addr);
@@ -720,6 +838,8 @@ impl DmaRemapping for IommuMapping {
                    }
                }
            }
+        } else if self.bypass.load(Ordering::Acquire) {
+            return Ok(addr);
        }

        Err(io::Error::new(
@@ -761,12 +881,15 @@ pub struct Iommu {
    msi_iova_space: (u64, u64),
 }

+type EndpointsState = Vec<(u32, u32)>;
+type DomainsState = Vec<(u32, (Vec<(u64, Mapping)>, bool))>;
+
 #[derive(Versionize)]
 struct IommuState {
    avail_features: u64,
    acked_features: u64,
-    endpoints: Vec<(u32, u32)>,
-    mappings: Vec<(u32, Vec<(u64, Mapping)>)>,
+    endpoints: EndpointsState,
+    domains: DomainsState,
 }

 impl VersionMapped for IommuState {}
@@ -786,7 +909,8 @@ impl Iommu {

        let mapping = Arc::new(IommuMapping {
            endpoints: Arc::new(RwLock::new(BTreeMap::new())),
-            mappings: Arc::new(RwLock::new(BTreeMap::new())),
+            domains: Arc::new(RwLock::new(BTreeMap::new())),
+            bypass: AtomicBool::new(true),
        });

        Ok((
@@ -797,7 +921,8 @@ impl Iommu {
                    queue_sizes: QUEUE_SIZES.to_vec(),
                    avail_features: 1u64 << VIRTIO_F_VERSION_1
                        | 1u64 << VIRTIO_IOMMU_F_MAP_UNMAP
-                        | 1u64 << VIRTIO_IOMMU_F_PROBE,
+                        | 1u64 << VIRTIO_IOMMU_F_PROBE
+                        | 1u64 << VIRTIO_IOMMU_F_BYPASS_CONFIG,
                    paused_sync: Some(Arc::new(Barrier::new(2))),
                    ..Default::default()
                },
@@ -824,14 +949,14 @@ impl Iommu {
                .clone()
                .into_iter()
                .collect(),
-            mappings: self
+            domains: self
                .mapping
-                .mappings
+                .domains
                .read()
                .unwrap()
                .clone()
                .into_iter()
-                .map(|(k, v)| (k, v.into_iter().collect()))
+                .map(|(k, v)| (k, (v.mappings.into_iter().collect(), v.bypass)))
                .collect(),
        }
    }
@@ -840,14 +965,36 @@ impl Iommu {
        self.common.avail_features = state.avail_features;
        self.common.acked_features = state.acked_features;
        *(self.mapping.endpoints.write().unwrap()) = state.endpoints.clone().into_iter().collect();
-        *(self.mapping.mappings.write().unwrap()) = state
-            .mappings
+        *(self.mapping.domains.write().unwrap()) = state
+            .domains
            .clone()
            .into_iter()
-            .map(|(k, v)| (k, v.into_iter().collect()))
+            .map(|(k, v)| {
+                (
+                    k,
+                    Domain {
+                        mappings: v.0.into_iter().collect(),
+                        bypass: v.1,
+                    },
+                )
+            })
            .collect();
    }

+    fn update_bypass(&mut self) {
+        // Use bypass from config if VIRTIO_IOMMU_F_BYPASS_CONFIG has been negotiated
+        if !self
+            .common
+            .feature_acked(VIRTIO_IOMMU_F_BYPASS_CONFIG.into())
+        {
+            return;
+        }
+
+        let bypass = self.config.bypass == 1;
+        info!("Updating bypass mode to {}", bypass);
+        self.mapping.bypass.store(bypass, Ordering::Release);
+    }
+
    pub fn add_external_mapping(&mut self, device_id: u32, mapping: Arc<dyn ExternalDmaMapping>) {
        self.ext_mapping.lock().unwrap().insert(device_id, mapping);
    }
@@ -883,6 +1030,24 @@ impl VirtioDevice for Iommu {
        self.read_config_from_slice(self.config.as_slice(), offset, data);
    }

+    fn write_config(&mut self, offset: u64, data: &[u8]) {
+        // The "bypass" field is the only mutable field
+        let bypass_offset =
+            (&self.config.bypass as *const _ as u64) - (&self.config as *const _ as u64);
+        if offset != bypass_offset || data.len() != std::mem::size_of_val(&self.config.bypass) {
+            error!(
+                "Attempt to write to read-only field: offset {:x} length {}",
+                offset,
+                data.len()
+            );
+            return;
+        }
+
+        self.config.bypass = data[0];
+
+        self.update_bypass();
+    }
+
    fn activate(
        &mut self,
        _mem: GuestMemoryAtomic<GuestMemoryMmap>,
@@ -900,7 +1065,6 @@ impl VirtioDevice for Iommu {
            pause_evt,
            mapping: self.mapping.clone(),
            ext_mapping: self.ext_mapping.clone(),
-            ext_domain_mapping: BTreeMap::new(),
            msi_iova_space: self.msi_iova_space,
        };

--- a/virtio-devices/src/lib.rs
+++ b/virtio-devices/src/lib.rs
@@ -14,9 +14,8 @@
 extern crate event_monitor;
 #[macro_use]
 extern crate log;
-#[macro_use]
-extern crate serde_derive;

+use serde::{Deserialize, Serialize};
 use std::convert::TryInto;
 use std::io;

--- a/virtio-devices/src/mem.rs
+++ b/virtio-devices/src/mem.rs
@@ -137,6 +137,8 @@ pub enum Error {
    DmaUnmap(std::io::Error),
    // Invalid DMA mapping handler
    InvalidDmaMappingHandler,
+    // Not activated by the guest
+    NotActivatedByGuest,
 }

 #[repr(C)]
@@ -327,6 +329,7 @@ impl Request {

 pub struct ResizeSender {
    hotplugged_size: Arc<AtomicU64>,
+    activated: Arc<AtomicBool>,
    tx: mpsc::Sender<Result<(), Error>>,
    evt: EventFd,
 }
@@ -345,6 +348,7 @@ impl Clone for ResizeSender {
    fn clone(&self) -> Self {
        ResizeSender {
            hotplugged_size: self.hotplugged_size.clone(),
+            activated: self.activated.clone(),
            tx: self.tx.clone(),
            evt: self
                .evt
@@ -356,6 +360,7 @@ impl Clone for ResizeSender {

 pub struct Resize {
    hotplugged_size: Arc<AtomicU64>,
+    activated: Arc<AtomicBool>,
    tx: mpsc::Sender<Result<(), Error>>,
    rx: mpsc::Receiver<Result<(), Error>>,
    evt: EventFd,
@@ -367,6 +372,7 @@ impl Resize {

        Ok(Resize {
            hotplugged_size: Arc::new(AtomicU64::new(hotplugged_size)),
+            activated: Arc::new(AtomicBool::default()),
            tx,
            rx,
            evt: EventFd::new(EFD_NONBLOCK)?,
@@ -376,12 +382,17 @@ impl Resize {
    pub fn new_resize_sender(&self) -> Result<ResizeSender, Error> {
        Ok(ResizeSender {
            hotplugged_size: self.hotplugged_size.clone(),
+            activated: self.activated.clone(),
            tx: self.tx.clone(),
            evt: self.evt.try_clone().map_err(Error::EventFdTryCloneFail)?,
        })
    }

    pub fn work(&self, desired_size: u64) -> Result<(), Error> {
+        if !self.activated.load(Ordering::Acquire) {
+            return Err(Error::NotActivatedByGuest);
+        }
+
        self.hotplugged_size.store(desired_size, Ordering::Release);
        self.evt.write(1).map_err(Error::EventFdWriteFail)?;
        self.rx.recv().map_err(Error::MpscRecvFail)?
@@ -1046,11 +1057,14 @@ impl VirtioDevice for Mem {
        )?;
        self.common.epoll_threads = Some(epoll_threads);

+        self.resize.activated.store(true, Ordering::Release);
+
        event!("virtio-device", "activated", "id", &self.id);
        Ok(())
    }

    fn reset(&mut self) -> Option<Arc<dyn VirtioInterrupt>> {
+        self.resize.activated.store(false, Ordering::Release);
        let result = self.common.reset();
        event!("virtio-device", "reset", "id", &self.id);
        result
--- a/virtio-devices/src/seccomp_filters.rs
+++ b/virtio-devices/src/seccomp_filters.rs
@@ -34,7 +34,6 @@ pub enum Thread {
 /// [`SeccompCondition`]: struct.SeccompCondition.html
 /// [`SeccompRule`]: struct.SeccompRule.html
 macro_rules! and {
-    ($($x:expr,)*) => (SeccompRule::new(vec![$($x),*]).unwrap());
    ($($x:expr),*) => (SeccompRule::new(vec![$($x),*]).unwrap())
 }

--- a/virtio-devices/src/transport/pci_device.rs
+++ b/virtio-devices/src/transport/pci_device.rs
@@ -35,8 +35,8 @@ use vm_device::dma_mapping::ExternalDmaMapping;
 use vm_device::interrupt::{
    InterruptIndex, InterruptManager, InterruptSourceGroup, MsiIrqGroupConfig,
 };
-use vm_device::BusDevice;
-use vm_memory::{Address, ByteValued, GuestAddress, GuestMemoryAtomic, GuestUsize, Le32};
+use vm_device::{BusDevice, Resource};
+use vm_memory::{Address, ByteValued, GuestAddress, GuestMemoryAtomic, Le32};
 use vm_migration::{
    Migratable, MigratableError, Pausable, Snapshot, Snapshottable, Transportable, VersionMapped,
 };
@@ -260,6 +260,8 @@ const MSIX_PBA_BAR_OFFSET: u64 = 0x48000;
 const MSIX_PBA_SIZE: u64 = 0x800;
 // The BAR size must be a power of 2.
 const CAPABILITY_BAR_SIZE: u64 = 0x80000;
+const VIRTIO_COMMON_BAR_INDEX: usize = 0;
+const VIRTIO_SHM_BAR_INDEX: usize = 2;

 const NOTIFY_OFF_MULTIPLIER: u32 = 4; // A dword per notification address.

@@ -318,7 +320,6 @@ pub struct VirtioPciDevice {

    // Settings PCI BAR
    settings_bar: u8,
-    settings_bar_addr: Option<GuestAddress>,

    // Whether to use 64-bit bar location or 32-bit
    use_64bit_bar: bool,
@@ -332,7 +333,7 @@ pub struct VirtioPciDevice {
    cap_pci_cfg_info: VirtioPciCfgCapInfo,

    // Details of bar regions to free
-    bar_regions: Vec<(GuestAddress, GuestUsize, PciBarRegionType)>,
+    bar_regions: Vec<PciBarConfiguration>,

    // EventFd to signal on to request activation
    activate_evt: EventFd,
@@ -452,7 +453,6 @@ impl VirtioPciDevice {
            queue_evts,
            memory: Some(memory),
            settings_bar: 0,
-            settings_bar_addr: None,
            use_64bit_bar,
            interrupt_source_group,
            cap_pci_cfg_info: VirtioPciCfgCapInfo::default(),
@@ -542,12 +542,6 @@ impl VirtioPciDevice {
        self.common_config.driver_status == DEVICE_INIT as u8
    }

-    // This function is used by the caller to provide the expected base address
-    // for the virtio-pci configuration BAR.
-    pub fn set_config_bar_addr(&mut self, bar_addr: u64) {
-        self.settings_bar_addr = Some(GuestAddress(bar_addr));
-    }
-
    pub fn config_bar_addr(&self) -> u64 {
        self.configuration.get_bar_addr(self.settings_bar as usize)
    }
@@ -678,7 +672,8 @@ impl VirtioPciDevice {
                let mem = self.memory.as_ref().unwrap().clone();
                let mut device = self.device.lock().unwrap();
                let mut queue_evts = Vec::new();
-                let mut queues = self.queues.clone();
+                let mut queues: Vec<Queue<GuestMemoryAtomic<GuestMemoryMmap>>> =
+                    self.queues.iter().map(vm_virtio::clone_queue).collect();
                queues.retain(|q| q.state.ready);
                for (i, queue) in queues.iter().enumerate() {
                    queue_evts.push(self.queue_evts[i].try_clone().unwrap());
@@ -847,24 +842,39 @@ impl PciDevice for VirtioPciDevice {
        &mut self,
        allocator: &Arc<Mutex<SystemAllocator>>,
        mmio_allocator: &mut AddressAllocator,
-    ) -> std::result::Result<Vec<(GuestAddress, GuestUsize, PciBarRegionType)>, PciDeviceError>
-    {
-        let mut ranges = Vec::new();
+        resources: Option<Vec<Resource>>,
+    ) -> std::result::Result<Vec<PciBarConfiguration>, PciDeviceError> {
+        let mut bars = Vec::new();
        let device_clone = self.device.clone();
        let device = device_clone.lock().unwrap();

+        let mut settings_bar_addr = None;
+        if let Some(resources) = &resources {
+            for resource in resources {
+                if let Resource::PciBar { index, base, .. } = resource {
+                    if *index == VIRTIO_COMMON_BAR_INDEX {
+                        settings_bar_addr = Some(GuestAddress(*base));
+                        break;
+                    }
+                }
+            }
+            // Error out if no resource was matching the BAR id.
+            if settings_bar_addr.is_none() {
+                return Err(PciDeviceError::MissingResource);
+            }
+        }
+
        // Allocate the virtio-pci capability BAR.
        // See http://docs.oasis-open.org/virtio/virtio/v1.0/cs04/virtio-v1.0-cs04.html#x1-740004
        let (virtio_pci_bar_addr, region_type) = if self.use_64bit_bar {
            let region_type = PciBarRegionType::Memory64BitRegion;
            let addr = mmio_allocator
                .allocate(
-                    self.settings_bar_addr,
+                    settings_bar_addr,
                    CAPABILITY_BAR_SIZE,
                    Some(CAPABILITY_BAR_SIZE),
                )
                .ok_or(PciDeviceError::IoAllocationFailed(CAPABILITY_BAR_SIZE))?;
-            ranges.push((addr, CAPABILITY_BAR_SIZE, region_type));
            (addr, region_type)
        } else {
            let region_type = PciBarRegionType::Memory32BitRegion;
@@ -872,50 +882,44 @@ impl PciDevice for VirtioPciDevice {
                .lock()
                .unwrap()
                .allocate_mmio_hole_addresses(
-                    self.settings_bar_addr,
+                    settings_bar_addr,
                    CAPABILITY_BAR_SIZE,
                    Some(CAPABILITY_BAR_SIZE),
                )
                .ok_or(PciDeviceError::IoAllocationFailed(CAPABILITY_BAR_SIZE))?;
-            ranges.push((addr, CAPABILITY_BAR_SIZE, region_type));
            (addr, region_type)
        };
-        self.bar_regions
-            .push((virtio_pci_bar_addr, CAPABILITY_BAR_SIZE, region_type));

-        let config = PciBarConfiguration::default()
-            .set_register_index(0)
+        let bar = PciBarConfiguration::default()
+            .set_index(VIRTIO_COMMON_BAR_INDEX)
            .set_address(virtio_pci_bar_addr.raw_value())
            .set_size(CAPABILITY_BAR_SIZE)
            .set_region_type(region_type);
-        let virtio_pci_bar =
-            self.configuration.add_pci_bar(&config).map_err(|e| {
-                PciDeviceError::IoRegistrationFailed(virtio_pci_bar_addr.raw_value(), e)
-            })? as u8;
+        self.configuration.add_pci_bar(&bar).map_err(|e| {
+            PciDeviceError::IoRegistrationFailed(virtio_pci_bar_addr.raw_value(), e)
+        })?;
+
+        bars.push(bar);

        // Once the BARs are allocated, the capabilities can be added to the PCI configuration.
-        self.add_pci_capabilities(virtio_pci_bar)?;
+        self.add_pci_capabilities(VIRTIO_COMMON_BAR_INDEX as u8)?;

        // Allocate a dedicated BAR if there are some shared memory regions.
        if let Some(shm_list) = device.get_shm_regions() {
-            let config = PciBarConfiguration::default()
-                .set_register_index(2)
+            let bar = PciBarConfiguration::default()
+                .set_index(VIRTIO_SHM_BAR_INDEX)
                .set_address(shm_list.addr.raw_value())
                .set_size(shm_list.len);
-            let virtio_pci_shm_bar =
-                self.configuration.add_pci_bar(&config).map_err(|e| {
-                    PciDeviceError::IoRegistrationFailed(shm_list.addr.raw_value(), e)
-                })? as u8;
+            self.configuration
+                .add_pci_bar(&bar)
+                .map_err(|e| PciDeviceError::IoRegistrationFailed(shm_list.addr.raw_value(), e))?;

-            let region_type = PciBarRegionType::Memory64BitRegion;
-            ranges.push((shm_list.addr, shm_list.len, region_type));
-            self.bar_regions
-                .push((shm_list.addr, shm_list.len, region_type));
+            bars.push(bar);

            for (idx, shm) in shm_list.region_list.iter().enumerate() {
                let shm_cap = VirtioPciCap64::new(
                    PciCapabilityType::SharedMemoryConfig,
-                    virtio_pci_shm_bar,
+                    VIRTIO_SHM_BAR_INDEX as u8,
                    idx as u8,
                    shm.offset,
                    shm.len,
@@ -926,7 +930,9 @@ impl PciDevice for VirtioPciDevice {
            }
        }

-        Ok(ranges)
+        self.bar_regions = bars.clone();
+
+        Ok(bars)
    }

    fn free_bars(
@@ -934,13 +940,13 @@ impl PciDevice for VirtioPciDevice {
        allocator: &mut SystemAllocator,
        mmio_allocator: &mut AddressAllocator,
    ) -> std::result::Result<(), PciDeviceError> {
-        for (addr, length, type_) in self.bar_regions.drain(..) {
-            match type_ {
+        for bar in self.bar_regions.drain(..) {
+            match bar.region_type() {
                PciBarRegionType::Memory32BitRegion => {
-                    allocator.free_mmio_hole_addresses(addr, length);
+                    allocator.free_mmio_hole_addresses(GuestAddress(bar.addr()), bar.size());
                }
                PciBarRegionType::Memory64BitRegion => {
-                    mmio_allocator.free(addr, length);
+                    mmio_allocator.free(GuestAddress(bar.addr()), bar.size());
                }
                _ => error!("Unexpected PCI bar type"),
            }
@@ -951,9 +957,9 @@ impl PciDevice for VirtioPciDevice {
    fn move_bar(&mut self, old_base: u64, new_base: u64) -> result::Result<(), std::io::Error> {
        // We only update our idea of the bar in order to support free_bars() above.
        // The majority of the reallocation is done inside DeviceManager.
-        for (addr, _, _) in self.bar_regions.iter_mut() {
-            if (*addr).0 == old_base {
-                *addr = GuestAddress(new_base);
+        for bar in self.bar_regions.iter_mut() {
+            if bar.addr() == old_base {
+                *bar = bar.set_address(new_base);
            }
        }

@@ -1085,6 +1091,10 @@ impl PciDevice for VirtioPciDevice {
    fn as_any(&mut self) -> &mut dyn Any {
        self
    }
+
+    fn id(&self) -> Option<String> {
+        Some(self.id.clone())
+    }
 }

 impl BusDevice for VirtioPciDevice {
--- a/virtio-devices/src/vhost_user/mod.rs
+++ b/virtio-devices/src/vhost_user/mod.rs
@@ -224,7 +224,7 @@ impl<S: VhostUserMasterReqHandler> VhostUserEpollHandler<S> {
        vhost_user
            .reinitialize_vhost_user(
                self.mem.memory().deref(),
-                self.queues.clone(),
+                self.queues.iter().map(vm_virtio::clone_queue).collect(),
                self.queue_evts
                    .iter()
                    .map(|q| q.try_clone().unwrap())
@@ -324,7 +324,7 @@ impl VhostUserCommon {
            .unwrap()
            .setup_vhost_user(
                &mem.memory(),
-                queues.clone(),
+                queues.iter().map(vm_virtio::clone_queue).collect(),
                queue_evts.iter().map(|q| q.try_clone().unwrap()).collect(),
                &interrupt_cb,
                acked_features,
--- a/vm-allocator/Cargo.toml
+++ b/vm-allocator/Cargo.toml
@@ -5,6 +5,6 @@ authors = ["The Chromium OS Authors"]
 edition = "2021"

 [dependencies]
-libc = "0.2.123"
-vm-memory = "0.7.0"
+libc = "0.2.126"
+vm-memory = "0.8.0"
 arch = { path = "../arch" }
--- a/vm-device/Cargo.toml
+++ b/vm-device/Cargo.toml
@@ -10,12 +10,12 @@ kvm = ["vfio-ioctls/kvm"]
 mshv = ["vfio-ioctls/mshv"]

 [dependencies]
-anyhow = "1.0.56"
-thiserror = "1.0.30"
-serde = { version = "1.0.136", features = ["rc"] }
-serde_derive = "1.0.136"
-serde_json = "1.0.79"
+anyhow = "1.0.57"
+hypervisor = { path = "../hypervisor" }
+thiserror = "1.0.31"
+serde = { version = "1.0.137", features = ["rc", "derive"] }
+serde_json = "1.0.81"
 vfio-ioctls = { git = "https://github.com/rust-vmm/vfio", branch = "main", default-features = false }
-vm-memory = { version = "0.7.0", features = ["backend-mmap"] }
+vm-memory = { version = "0.8.0", features = ["backend-mmap"] }
 vmm-sys-util = "0.9.0"

--- a/vm-device/src/interrupt/mod.rs
+++ b/vm-device/src/interrupt/mod.rs
@@ -57,6 +57,7 @@
 //! * The virtual device backend requests the interrupt manager to create an interrupt group
 //!   according to guest configuration information

+pub use hypervisor::{InterruptSourceConfig, LegacyIrqSourceConfig, MsiIrqSourceConfig};
 use std::sync::Arc;
 use vmm_sys_util::eventfd::EventFd;

@@ -66,39 +67,6 @@ pub type Result<T> = std::io::Result<T>;
 /// Data type to store an interrupt source identifier.
 pub type InterruptIndex = u32;

-/// Configuration data for legacy interrupts.
-///
-/// On x86 platforms, legacy interrupts means those interrupts routed through PICs or IOAPICs.
-#[derive(Copy, Clone, Debug)]
-pub struct LegacyIrqSourceConfig {
-    pub irqchip: u32,
-    pub pin: u32,
-}
-
-/// Configuration data for MSI/MSI-X interrupts.
-///
-/// On x86 platforms, these interrupts are vectors delivered directly to the LAPIC.
-#[derive(Copy, Clone, Debug, Default)]
-pub struct MsiIrqSourceConfig {
-    /// High address to delivery message signaled interrupt.
-    pub high_addr: u32,
-    /// Low address to delivery message signaled interrupt.
-    pub low_addr: u32,
-    /// Data to write to delivery message signaled interrupt.
-    pub data: u32,
-    /// Unique ID of the device to delivery message signaled interrupt.
-    pub devid: u32,
-}
-
-/// Configuration data for an interrupt source.
-#[derive(Copy, Clone, Debug)]
-pub enum InterruptSourceConfig {
-    /// Configuration data for Legacy interrupts.
-    LegacyIrq(LegacyIrqSourceConfig),
-    /// Configuration data for PciMsi, PciMsix and generic MSI interrupts.
-    MsiIrq(MsiIrqSourceConfig),
-}
-
 /// Configuration data for legacy, pin based interrupt groups.
 ///
 /// A legacy interrupt group only takes one irq number as its configuration.
--- a/vm-device/src/lib.rs
+++ b/vm-device/src/lib.rs
@@ -3,8 +3,7 @@
 // SPDX-License-Identifier: Apache-2.0
 //

-#[macro_use]
-extern crate serde_derive;
+use serde::{Deserialize, Serialize};

 mod bus;
 pub mod dma_mapping;
@@ -23,6 +22,13 @@ pub enum MsiIrqType {
    GenericMsi,
 }

+#[derive(Copy, Clone, PartialEq, Serialize, Deserialize, Debug)]
+pub enum PciBarType {
+    Io,
+    Mmio32,
+    Mmio64,
+}
+
 /// Enumeration for device resources.
 #[allow(missing_docs)]
 #[derive(Clone, Debug, Serialize, Deserialize)]
@@ -31,6 +37,14 @@ pub enum Resource {
    PioAddressRange { base: u16, size: u16 },
    /// Memory Mapped IO address range.
    MmioAddressRange { base: u64, size: u64 },
+    /// PCI BAR
+    PciBar {
+        index: usize,
+        base: u64,
+        size: u64,
+        type_: PciBarType,
+        prefetchable: bool,
+    },
    /// Legacy IRQ number.
    LegacyIrq(u32),
    /// Message Signaled Interrupt
--- a/vm-migration/Cargo.toml
+++ b/vm-migration/Cargo.toml
@@ -5,11 +5,10 @@ authors = ["The Cloud Hypervisor Authors"]
 edition = "2021"

 [dependencies]
-anyhow = "1.0.56"
-thiserror = "1.0.30"
-serde = { version = "1.0.136", features = ["rc"] }
-serde_derive = "1.0.136"
-serde_json = "1.0.79"
+anyhow = "1.0.57"
+thiserror = "1.0.31"
+serde = { version = "1.0.137", features = ["rc", "derive"] }
+serde_json = "1.0.81"
 versionize = "0.1.6"
 versionize_derive = "0.1.4"
-vm-memory = { version = "0.7.0", features = ["backend-mmap", "backend-atomic"] }
+vm-memory = { version = "0.8.0", features = ["backend-mmap", "backend-atomic"] }
--- a/vm-migration/src/lib.rs
+++ b/vm-migration/src/lib.rs
@@ -3,9 +3,6 @@
 // SPDX-License-Identifier: Apache-2.0 AND BSD-3-Clause
 //

-#[macro_use]
-extern crate serde_derive;
-
 use crate::protocol::MemoryRangeTable;
 use anyhow::anyhow;
 use serde::{Deserialize, Serialize};
@@ -15,7 +12,7 @@ use versionize::{VersionMap, Versionize};
 pub mod protocol;

 /// Global VMM version for versioning
-const MAJOR_VERSION: u16 = 23;
+const MAJOR_VERSION: u16 = 24;
 const MINOR_VERSION: u16 = 0;
 const VMM_VERSION: u16 = MAJOR_VERSION << 12 | MINOR_VERSION & 0b1111;

--- a/vm-migration/src/protocol.rs
+++ b/vm-migration/src/protocol.rs
@@ -4,6 +4,7 @@
 //

 use crate::{MigratableError, VersionMapped};
+use serde::{Deserialize, Serialize};
 use std::io::{Read, Write};
 use versionize::{VersionMap, Versionize, VersionizeResult};
 use versionize_derive::Versionize;
--- a/vm-virtio/Cargo.toml
+++ b/vm-virtio/Cargo.toml
@@ -8,7 +8,7 @@ edition = "2021"
 default = []

 [dependencies]
-log = "0.4.16"
+log = "0.4.17"
 virtio-bindings = { version = "0.1.0", features = ["virtio-v5_0_0"] }
-virtio-queue = "0.2.0"
-vm-memory = { version = "0.7.0", features = ["backend-mmap", "backend-atomic", "backend-bitmap"] }
+virtio-queue = "0.3.0"
+vm-memory = { version = "0.8.0", features = ["backend-mmap", "backend-atomic", "backend-bitmap"] }
--- a/vm-virtio/src/lib.rs
+++ b/vm-virtio/src/lib.rs
@@ -12,8 +12,10 @@

 use std::fmt::{self, Debug};
 use std::sync::Arc;
+use virtio_queue::Queue;
+use vm_memory::{bitmap::AtomicBitmap, GuestAddress, GuestMemoryAtomic};

-use vm_memory::GuestAddress;
+type GuestMemoryMmap = vm_memory::GuestMemoryMmap<AtomicBitmap>;

 pub mod queue;
 pub use queue::*;
@@ -121,3 +123,24 @@ impl Translatable for GuestAddress {
        }
    }
 }
+
+/// Helper for cloning a Queue since QueueState doesn't derive Clone
+pub fn clone_queue(
+    queue: &Queue<GuestMemoryAtomic<GuestMemoryMmap>>,
+) -> Queue<GuestMemoryAtomic<GuestMemoryMmap>> {
+    Queue::<GuestMemoryAtomic<GuestMemoryMmap>, virtio_queue::QueueState> {
+        mem: queue.mem.clone(),
+        state: virtio_queue::QueueState {
+            max_size: queue.state.max_size,
+            next_avail: queue.state.next_avail,
+            next_used: queue.state.next_used,
+            event_idx_enabled: queue.state.event_idx_enabled,
+            num_added: queue.state.num_added,
+            size: queue.state.size,
+            ready: queue.state.ready,
+            desc_table: queue.state.desc_table,
+            avail_ring: queue.state.avail_ring,
+            used_ring: queue.state.used_ring,
+        },
+    }
+}
--- a/vmm/Cargo.toml
+++ b/vmm/Cargo.toml
@@ -16,12 +16,12 @@ tdx = ["arch/tdx", "hypervisor/tdx"]

 [dependencies]
 acpi_tables = { path = "../acpi_tables" }
-anyhow = "1.0.56"
+anyhow = "1.0.57"
 arc-swap = "1.5.0"
 arch = { path = "../arch" }
 bitflags = "1.3.2"
 block_util = { path = "../block_util" }
-clap = "3.1.8"
+clap = "3.1.18"
 devices = { path = "../devices" }
 epoll = "4.3.1"
 event_monitor = { path = "../event_monitor" }
@@ -29,31 +29,30 @@ gdbstub = "0.6.1"
 gdbstub_arch = "0.2.2"
 hypervisor = { path = "../hypervisor" }
 lazy_static = "1.4.0"
-libc = "0.2.123"
+libc = "0.2.126"
 linux-loader = { version = "0.4.0", features = ["elf", "bzimage", "pe"] }
-log = "0.4.16"
+log = "0.4.17"
 micro_http = { git = "https://github.com/firecracker-microvm/micro-http", branch = "main" }
 net_util = { path = "../net_util" }
 option_parser = { path = "../option_parser" }
 pci = { path = "../pci" }
 qcow = { path = "../qcow" }
 seccompiler = "0.2.0"
-serde = { version = "1.0.136", features = ["rc"] }
-serde_derive = "1.0.136"
-serde_json = "1.0.79"
-signal-hook = "0.3.13"
-thiserror = "1.0.30"
-uuid = "0.8.2"
+serde = { version = "1.0.137", features = ["rc", "derive"] }
+serde_json = "1.0.81"
+signal-hook = "0.3.14"
+thiserror = "1.0.31"
+uuid = "1.0.0"
 versionize = "0.1.6"
 versionize_derive = "0.1.4"
 vfio-ioctls = { git = "https://github.com/rust-vmm/vfio", branch = "main", default-features = false }
 vfio_user = { path = "../vfio_user" }
 vhdx = { path = "../vhdx" }
 virtio-devices = { path = "../virtio-devices" }
-virtio-queue = "0.2.0"
+virtio-queue = "0.3.0"
 vm-allocator = { path = "../vm-allocator" }
 vm-device = { path = "../vm-device" }
-vm-memory = { version = "0.7.0", features = ["backend-mmap", "backend-atomic", "backend-bitmap"] }
+vm-memory = { version = "0.8.0", features = ["backend-mmap", "backend-atomic", "backend-bitmap"] }
 vm-migration = { path = "../vm-migration" }
 vm-virtio = { path = "../vm-virtio" }
 vmm-sys-util = { version = "0.9.0", features = ["with-serde"] }
--- a/vmm/src/api/http.rs
+++ b/vmm/src/api/http.rs
@@ -6,7 +6,7 @@
 use crate::api::http_endpoint::{VmActionHandler, VmCreate, VmInfo, VmmPing, VmmShutdown};
 use crate::api::{ApiError, ApiRequest, VmAction};
 use crate::seccomp_filters::{get_seccomp_filter, Thread};
-use crate::{Error, Result};
+use crate::{Error as VmmError, Result};
 use micro_http::{Body, HttpServer, MediaType, Method, Request, Response, StatusCode, Version};
 use seccompiler::{apply_filter, SeccompAction};
 use serde_json::Error as SerdeError;
@@ -36,89 +36,8 @@ pub enum HttpError {
    /// Internal Server Error
    InternalServerError,

-    /// Could not create a VM
-    VmCreate(ApiError),
-
-    /// Could not boot a VM
-    VmBoot(ApiError),
-
-    /// Could not delete a VM
-    VmDelete(ApiError),
-
-    /// Could not get the VM information
-    VmInfo(ApiError),
-
-    /// Could not pause the VM
-    VmPause(ApiError),
-
-    /// Could not pause the VM
-    VmResume(ApiError),
-
-    /// Could not shut a VM down
-    VmShutdown(ApiError),
-
-    /// Could not reboot a VM
-    VmReboot(ApiError),
-
-    /// Could not snapshot a VM
-    VmSnapshot(ApiError),
-
-    /// Could not restore a VM
-    VmRestore(ApiError),
-
-    /// Could not act on a VM
-    VmAction(ApiError),
-
-    /// Could not resize a VM
-    VmResize(ApiError),
-
-    /// Could not resize a memory zone
-    VmResizeZone(ApiError),
-
-    /// Could not add a device to a VM
-    VmAddDevice(ApiError),
-
-    /// Could not add a user device to the VM
-    VmAddUserDevice(ApiError),
-
-    /// Could not remove a device from a VM
-    VmRemoveDevice(ApiError),
-
-    /// Could not shut the VMM down
-    VmmShutdown(ApiError),
-
-    /// Could not handle VMM ping
-    VmmPing(ApiError),
-
-    /// Could not add a disk to a VM
-    VmAddDisk(ApiError),
-
-    /// Could not add a fs to a VM
-    VmAddFs(ApiError),
-
-    /// Could not add a pmem device to a VM
-    VmAddPmem(ApiError),
-
-    /// Could not add a network device to a VM
-    VmAddNet(ApiError),
-
-    /// Could not add a vDPA device to a VM
-    VmAddVdpa(ApiError),
-
-    /// Could not add a vsock device to a VM
-    VmAddVsock(ApiError),
-
-    /// Could not get counters from VM
-    VmCounters(ApiError),
-
-    /// Error setting up migration received
-    VmReceiveMigration(ApiError),
-
-    /// Error setting up migration sender
-    VmSendMigration(ApiError),
-
-    /// Error activating power button
-    VmPowerButton(ApiError),
+    /// Error from internal API
+    ApiError(ApiError),
 }

 impl From<serde_json::Error> for HttpError {
@@ -280,7 +199,7 @@ fn start_http_thread(
 ) -> Result<thread::JoinHandle<Result<()>>> {
    // Retrieve seccomp filter for API thread
    let api_seccomp_filter =
-        get_seccomp_filter(seccomp_action, Thread::Api).map_err(Error::CreateSeccompFilter)?;
+        get_seccomp_filter(seccomp_action, Thread::Api).map_err(VmmError::CreateSeccompFilter)?;

    thread::Builder::new()
        .name("http-server".to_string())
@@ -288,7 +207,7 @@ fn start_http_thread(
            // Apply seccomp filter for API thread.
            if !api_seccomp_filter.is_empty() {
                apply_filter(&api_seccomp_filter)
-                    .map_err(Error::ApplySeccompFilter)
+                    .map_err(VmmError::ApplySeccompFilter)
                    .map_err(|e| {
                        error!("Error applying seccomp filter: {:?}", e);
                        exit_evt.write(1).ok();
@@ -326,7 +245,7 @@ fn start_http_thread(

            Ok(())
        })
-        .map_err(Error::HttpThreadSpawn)
+        .map_err(VmmError::HttpThreadSpawn)
 }

 pub fn start_http_path_thread(
@@ -336,11 +255,10 @@ pub fn start_http_path_thread(
    seccomp_action: &SeccompAction,
    exit_evt: EventFd,
 ) -> Result<thread::JoinHandle<Result<()>>> {
-    std::fs::remove_file(path).unwrap_or_default();
    let socket_path = PathBuf::from(path);
-    let socket_fd = UnixListener::bind(socket_path).map_err(Error::CreateApiServerSocket)?;
+    let socket_fd = UnixListener::bind(socket_path).map_err(VmmError::CreateApiServerSocket)?;
    let server =
-        HttpServer::new_from_fd(socket_fd.into_raw_fd()).map_err(Error::CreateApiServer)?;
+        HttpServer::new_from_fd(socket_fd.into_raw_fd()).map_err(VmmError::CreateApiServer)?;
    start_http_thread(server, api_notifier, api_sender, seccomp_action, exit_evt)
 }

@@ -351,6 +269,6 @@ pub fn start_http_fd_thread(
    seccomp_action: &SeccompAction,
    exit_evt: EventFd,
 ) -> Result<thread::JoinHandle<Result<()>>> {
-    let server = HttpServer::new_from_fd(fd).map_err(Error::CreateApiServer)?;
+    let server = HttpServer::new_from_fd(fd).map_err(VmmError::CreateApiServer)?;
    start_http_thread(server, api_notifier, api_sender, seccomp_action, exit_evt)
 }
--- a/vmm/src/api/http_endpoint.rs
+++ b/vmm/src/api/http_endpoint.rs
@@ -43,7 +43,7 @@ impl EndpointHandler for VmCreate {

                        // Call vm_create()
                        match vm_create(api_notifier, api_sender, Arc::new(Mutex::new(vm_config)))
-                            .map_err(HttpError::VmCreate)
+                            .map_err(HttpError::ApiError)
                        {
                            Ok(_) => Response::new(Version::Http11, StatusCode::NoContent),
                            Err(e) => error_response(e, StatusCode::InternalServerError),
@@ -85,30 +85,22 @@ impl EndpointHandler for VmActionHandler {
                    api_notifier,
                    api_sender,
                    Arc::new(serde_json::from_slice(body.raw())?),
-                )
-                .map_err(HttpError::VmAddDevice),
-
+                ),
                AddDisk(_) => vm_add_disk(
                    api_notifier,
                    api_sender,
                    Arc::new(serde_json::from_slice(body.raw())?),
-                )
-                .map_err(HttpError::VmAddDisk),
-
+                ),
                AddFs(_) => vm_add_fs(
                    api_notifier,
                    api_sender,
                    Arc::new(serde_json::from_slice(body.raw())?),
-                )
-                .map_err(HttpError::VmAddFs),
-
+                ),
                AddPmem(_) => vm_add_pmem(
                    api_notifier,
                    api_sender,
                    Arc::new(serde_json::from_slice(body.raw())?),
-                )
-                .map_err(HttpError::VmAddPmem),
-
+                ),
                AddNet(_) => {
                    let mut net_cfg: NetConfig = serde_json::from_slice(body.raw())?;
                    // Update network config with optional files that might have
@@ -118,95 +110,73 @@ impl EndpointHandler for VmActionHandler {
                        net_cfg.fds = Some(fds);
                    }
                    vm_add_net(api_notifier, api_sender, Arc::new(net_cfg))
-                        .map_err(HttpError::VmAddNet)
                }
-
                AddVdpa(_) => vm_add_vdpa(
                    api_notifier,
                    api_sender,
                    Arc::new(serde_json::from_slice(body.raw())?),
-                )
-                .map_err(HttpError::VmAddVdpa),
-
+                ),
                AddVsock(_) => vm_add_vsock(
                    api_notifier,
                    api_sender,
                    Arc::new(serde_json::from_slice(body.raw())?),
-                )
-                .map_err(HttpError::VmAddVsock),
-
+                ),
                AddUserDevice(_) => vm_add_user_device(
                    api_notifier,
                    api_sender,
                    Arc::new(serde_json::from_slice(body.raw())?),
-                )
-                .map_err(HttpError::VmAddUserDevice),
-
+                ),
                RemoveDevice(_) => vm_remove_device(
                    api_notifier,
                    api_sender,
                    Arc::new(serde_json::from_slice(body.raw())?),
-                )
-                .map_err(HttpError::VmRemoveDevice),
-
+                ),
                Resize(_) => vm_resize(
                    api_notifier,
                    api_sender,
                    Arc::new(serde_json::from_slice(body.raw())?),
-                )
-                .map_err(HttpError::VmResize),
-
+                ),
                ResizeZone(_) => vm_resize_zone(
                    api_notifier,
                    api_sender,
                    Arc::new(serde_json::from_slice(body.raw())?),
-                )
-                .map_err(HttpError::VmResizeZone),
-
+                ),
                Restore(_) => vm_restore(
                    api_notifier,
                    api_sender,
                    Arc::new(serde_json::from_slice(body.raw())?),
-                )
-                .map_err(HttpError::VmRestore),
-
+                ),
                Snapshot(_) => vm_snapshot(
                    api_notifier,
                    api_sender,
                    Arc::new(serde_json::from_slice(body.raw())?),
-                )
-                .map_err(HttpError::VmSnapshot),
-
+                ),
                ReceiveMigration(_) => vm_receive_migration(
                    api_notifier,
                    api_sender,
                    Arc::new(serde_json::from_slice(body.raw())?),
-                )
-                .map_err(HttpError::VmReceiveMigration),
-
+                ),
                SendMigration(_) => vm_send_migration(
                    api_notifier,
                    api_sender,
                    Arc::new(serde_json::from_slice(body.raw())?),
-                )
-                .map_err(HttpError::VmSendMigration),
+                ),

-                _ => Err(HttpError::BadRequest),
+                _ => return Err(HttpError::BadRequest),
            }
        } else {
            match self.action {
-                Boot => vm_boot(api_notifier, api_sender).map_err(HttpError::VmBoot),
-                Delete => vm_delete(api_notifier, api_sender).map_err(HttpError::VmDelete),
-                Shutdown => vm_shutdown(api_notifier, api_sender).map_err(HttpError::VmShutdown),
-                Reboot => vm_reboot(api_notifier, api_sender).map_err(HttpError::VmReboot),
-                Pause => vm_pause(api_notifier, api_sender).map_err(HttpError::VmPause),
-                Resume => vm_resume(api_notifier, api_sender).map_err(HttpError::VmResume),
-                PowerButton => {
-                    vm_power_button(api_notifier, api_sender).map_err(HttpError::VmPowerButton)
-                }
-                _ => Err(HttpError::BadRequest),
+                Boot => vm_boot(api_notifier, api_sender),
+                Delete => vm_delete(api_notifier, api_sender),
+                Shutdown => vm_shutdown(api_notifier, api_sender),
+                Reboot => vm_reboot(api_notifier, api_sender),
+                Pause => vm_pause(api_notifier, api_sender),
+                Resume => vm_resume(api_notifier, api_sender),
+                PowerButton => vm_power_button(api_notifier, api_sender),
+                _ => return Err(HttpError::BadRequest),
            }
        }
+        .map_err(HttpError::ApiError)
    }

    fn get_handler(
@@ -217,7 +187,7 @@ impl EndpointHandler for VmActionHandler {
    ) -> std::result::Result<Option<Body>, HttpError> {
        use VmAction::*;
        match self.action {
-            Counters => vm_counters(api_notifier, api_sender).map_err(HttpError::VmCounters),
+            Counters => vm_counters(api_notifier, api_sender).map_err(HttpError::ApiError),
            _ => Err(HttpError::BadRequest),
        }
    }
@@ -234,7 +204,7 @@ impl EndpointHandler for VmInfo {
        api_sender: Sender<ApiRequest>,
    ) -> Response {
        match req.method() {
-            Method::Get => match vm_info(api_notifier, api_sender).map_err(HttpError::VmInfo) {
+            Method::Get => match vm_info(api_notifier, api_sender).map_err(HttpError::ApiError) {
                Ok(info) => {
                    let mut response = Response::new(Version::Http11, StatusCode::OK);
                    let info_serialized = serde_json::to_string(&info).unwrap();
@@ -260,7 +230,7 @@ impl EndpointHandler for VmmPing {
        api_sender: Sender<ApiRequest>,
    ) -> Response {
        match req.method() {
-            Method::Get => match vmm_ping(api_notifier, api_sender).map_err(HttpError::VmmPing) {
+            Method::Get => match vmm_ping(api_notifier, api_sender).map_err(HttpError::ApiError) {
                Ok(pong) => {
                    let mut response = Response::new(Version::Http11, StatusCode::OK);
                    let info_serialized = serde_json::to_string(&pong).unwrap();
@@ -288,7 +258,7 @@ impl EndpointHandler for VmmShutdown {
    ) -> Response {
        match req.method() {
            Method::Put => {
-                match vmm_shutdown(api_notifier, api_sender).map_err(HttpError::VmmShutdown) {
+                match vmm_shutdown(api_notifier, api_sender).map_err(HttpError::ApiError) {
                    Ok(_) => Response::new(Version::Http11, StatusCode::OK),
                    Err(e) => error_response(e, StatusCode::InternalServerError),
                }
--- a/vmm/src/api/mod.rs
+++ b/vmm/src/api/mod.rs
@@ -41,6 +41,7 @@ use crate::config::{
 use crate::device_tree::DeviceTree;
 use crate::vm::{Error as VmError, VmState};
 use micro_http::Body;
+use serde::{Deserialize, Serialize};
 use std::io;
 use std::sync::mpsc::{channel, RecvError, SendError, Sender};
 use std::sync::{Arc, Mutex};
--- a/vmm/src/api/openapi/cloud-hypervisor.yaml
+++ b/vmm/src/api/openapi/cloud-hypervisor.yaml
@@ -616,6 +616,8 @@ components:
          items:
            type: integer
            format: int16
+        serial_number:
+          type: string

    MemoryZoneConfig:
      required:
--- a/vmm/src/config.rs
+++ b/vmm/src/config.rs
@@ -8,14 +8,15 @@ use net_util::MacAddr;
 use option_parser::{
    ByteSized, IntegerList, OptionParser, OptionParserError, StringList, Toggle, Tuple,
 };
-use std::collections::HashMap;
+use serde::{Deserialize, Serialize};
+use std::collections::{BTreeSet, HashMap};
 use std::convert::From;
 use std::fmt;
 use std::net::Ipv4Addr;
 use std::path::PathBuf;
 use std::result;
 use std::str::FromStr;
-
+use thiserror::Error;
 use virtio_devices::{RateLimiterConfig, TokenBucketConfig};

 pub const DEFAULT_VCPUS: u8 = 1;
@@ -36,7 +37,7 @@ pub const DEFAULT_NUM_PCI_SEGMENTS: u16 = 1;
 const MAX_NUM_PCI_SEGMENTS: u16 = 16;

 /// Errors associated with VM configuration parameters.
-#[derive(Debug)]
+#[derive(Debug, Error)]
 pub enum Error {
    /// Filesystem tag is missing
    ParseFsTagMissing,
@@ -114,7 +115,7 @@ pub enum Error {
    ParseVdpaPathMissing,
 }

-#[derive(Debug, PartialEq)]
+#[derive(Debug, PartialEq, Error)]
 pub enum ValidationError {
    /// Both console and serial are tty.
    DoubleTtyMode,
@@ -169,7 +170,13 @@ pub enum ValidationError {
    /// On a IOMMU segment but not behind IOMMU
    OnIommuSegment(u16),
    // On a IOMMU segment but IOMMU not suported
-    IommuNotSupported(u16),
+    IommuNotSupportedOnSegment(u16),
+    // Identifier is not unique
+    IdentifierNotUnique(String),
+    /// Invalid identifier
+    InvalidIdentifier(String),
+    /// Placing the device behind a virtual IOMMU is not supported
+    IommuNotSupported,
 }

 type ValidationResult<T> = std::result::Result<T, ValidationError>;
@@ -249,13 +256,22 @@ impl fmt::Display for ValidationError {
                    pci_segment
                )
            }
-            IommuNotSupported(pci_segment) => {
+            IommuNotSupportedOnSegment(pci_segment) => {
                write!(
                    f,
-                    "Device is on an IOMMU PCI segment ({}) but does support being placed behind IOMMU",
+                    "Device is on an IOMMU PCI segment ({}) but does not support being placed behind IOMMU",
                    pci_segment
                )
            }
+            IdentifierNotUnique(s) => {
+                write!(f, "Identifier {} is not unique", s)
+            }
+            InvalidIdentifier(s) => {
+                write!(f, "Identifier {} is invalid", s)
+            }
+            IommuNotSupported => {
+                write!(f, "Device does not support being placed behind IOMMU")
+            }
        }
    }
 }
@@ -419,7 +435,7 @@ impl<'a> VmParams<'a> {
    }
 }

-#[derive(Clone, Debug, PartialEq, Deserialize, Serialize)]
+#[derive(Clone, Copy, Debug, PartialEq, Deserialize, Serialize)]
 pub enum HotplugMethod {
    Acpi,
    VirtioMem,
@@ -621,6 +637,8 @@ pub struct PlatformConfig {
    pub num_pci_segments: u16,
    #[serde(default)]
    pub iommu_segments: Option<Vec<u16>>,
+    #[serde(default)]
+    pub serial_number: Option<String>,
 }

 impl PlatformConfig {
@@ -628,6 +646,7 @@ impl PlatformConfig {
        let mut parser = OptionParser::new();
        parser.add("num_pci_segments");
        parser.add("iommu_segments");
+        parser.add("serial_number");
        parser.parse(platform).map_err(Error::ParsePlatform)?;

        let num_pci_segments: u16 = parser
@@ -638,9 +657,13 @@ impl PlatformConfig {
            .convert::<IntegerList>("iommu_segments")
            .map_err(Error::ParsePlatform)?
            .map(|v| v.0.iter().map(|e| *e as u16).collect());
+        let serial_number = parser
+            .convert("serial_number")
+            .map_err(Error::ParsePlatform)?;
        Ok(PlatformConfig {
            num_pci_segments,
            iommu_segments,
+            serial_number,
        })
    }

@@ -668,6 +691,7 @@ impl Default for PlatformConfig {
        PlatformConfig {
            num_pci_segments: DEFAULT_NUM_PCI_SEGMENTS,
            iommu_segments: None,
+            serial_number: None,
        }
    }
 }
@@ -1139,6 +1163,10 @@ impl DiskConfig {
            return Err(ValidationError::TooManyQueues);
        }

+        if self.vhost_user && self.iommu {
+            return Err(ValidationError::IommuNotSupported);
+        }
+
        if let Some(platform_config) = vm_config.platform.as_ref() {
            if self.pci_segment >= platform_config.num_pci_segments {
                return Err(ValidationError::InvalidPciSegment(self.pci_segment));
@@ -1435,6 +1463,10 @@ impl NetConfig {
            return Err(ValidationError::TooManyQueues);
        }

+        if self.vhost_user && self.iommu {
+            return Err(ValidationError::IommuNotSupported);
+        }
+
        if let Some(platform_config) = vm_config.platform.as_ref() {
            if self.pci_segment >= platform_config.num_pci_segments {
                return Err(ValidationError::InvalidPciSegment(self.pci_segment));
@@ -1664,7 +1696,9 @@ impl FsConfig {

            if let Some(iommu_segments) = platform_config.iommu_segments.as_ref() {
                if iommu_segments.contains(&self.pci_segment) {
-                    return Err(ValidationError::IommuNotSupported(self.pci_segment));
+                    return Err(ValidationError::IommuNotSupportedOnSegment(
+                        self.pci_segment,
+                    ));
                }
            }
        }
@@ -1953,7 +1987,9 @@ impl UserDeviceConfig {

            if let Some(iommu_segments) = platform_config.iommu_segments.as_ref() {
                if iommu_segments.contains(&self.pci_segment) {
-                    return Err(ValidationError::IommuNotSupported(self.pci_segment));
+                    return Err(ValidationError::IommuNotSupportedOnSegment(
+                        self.pci_segment,
+                    ));
                }
            }
        }
@@ -2315,7 +2351,29 @@ pub struct VmConfig {
 }

 impl VmConfig {
-    pub fn validate(&self) -> ValidationResult<()> {
+    fn validate_identifier(
+        id_list: &mut BTreeSet<String>,
+        id: &Option<String>,
+    ) -> ValidationResult<()> {
+        if let Some(id) = id.as_ref() {
+            if id.starts_with("__") {
+                return Err(ValidationError::InvalidIdentifier(id.clone()));
+            }
+
+            if !id_list.insert(id.clone()) {
+                return Err(ValidationError::IdentifierNotUnique(id.clone()));
+            }
+        }
+
+        Ok(())
+    }
+
+    // Also enables virtio-iommu if the config needs it
+    // Returns the list of unique identifiers provided through the
+    // configuration.
+    pub fn validate(&mut self) -> ValidationResult<BTreeSet<String>> {
+        let mut id_list = BTreeSet::new();
+
        #[cfg(not(feature = "tdx"))]
        self.kernel.as_ref().ok_or(ValidationError::KernelMissing)?;

@@ -2359,6 +2417,9 @@ impl VmConfig {
                    return Err(ValidationError::VhostUserMissingSocket);
                }
                disk.validate(self)?;
+                self.iommu |= disk.iommu;
+
+                Self::validate_identifier(&mut id_list, &disk.id)?;
            }
        }

@@ -2368,6 +2429,9 @@ impl VmConfig {
                    return Err(ValidationError::VhostUserRequiresSharedMemory);
                }
                net.validate(self)?;
+                self.iommu |= net.iommu;
+
+                Self::validate_identifier(&mut id_list, &net.id)?;
            }
        }

@@ -2377,15 +2441,23 @@ impl VmConfig {
            }
            for fs in fses {
                fs.validate(self)?;
+
+                Self::validate_identifier(&mut id_list, &fs.id)?;
            }
        }

        if let Some(pmems) = &self.pmem {
            for pmem in pmems {
                pmem.validate(self)?;
+                self.iommu |= pmem.iommu;
+
+                Self::validate_identifier(&mut id_list, &pmem.id)?;
            }
        }

+        self.iommu |= self.rng.iommu;
+        self.iommu |= self.console.iommu;
+
        if let Some(t) = &self.cpus.topology {
            if t.threads_per_core == 0
                || t.cores_per_die == 0
@@ -2425,12 +2497,17 @@ impl VmConfig {

            for user_device in user_devices {
                user_device.validate(self)?;
+
+                Self::validate_identifier(&mut id_list, &user_device.id)?;
            }
        }

        if let Some(vdpa_devices) = &self.vdpa {
            for vdpa_device in vdpa_devices {
                vdpa_device.validate(self)?;
+                self.iommu |= vdpa_device.iommu;
+
+                Self::validate_identifier(&mut id_list, &vdpa_device.id)?;
            }
        }

@@ -2454,11 +2531,17 @@ impl VmConfig {
        if let Some(devices) = &self.devices {
            for device in devices {
                device.validate(self)?;
+                self.iommu |= device.iommu;
+
+                Self::validate_identifier(&mut id_list, &device.id)?;
            }
        }

        if let Some(vsock) = &self.vsock {
            vsock.validate(self)?;
+            self.iommu |= vsock.iommu;
+
+            Self::validate_identifier(&mut id_list, &vsock.id)?;
        }

        if let Some(numa) = &self.numa {
@@ -2479,22 +2562,37 @@ impl VmConfig {
            }
        }

-        self.platform.as_ref().map(|p| p.validate()).transpose()?;
+        if let Some(zones) = &self.memory.zones {
+            for zone in zones.iter() {
+                let id = zone.id.clone();
+                Self::validate_identifier(&mut id_list, &Some(id))?;
+            }
+        }

-        Ok(())
+        #[cfg(target_arch = "x86_64")]
+        if let Some(sgx_epcs) = &self.sgx_epc {
+            for sgx_epc in sgx_epcs.iter() {
+                let id = sgx_epc.id.clone();
+                Self::validate_identifier(&mut id_list, &Some(id))?;
+            }
+        }
+
+        self.platform.as_ref().map(|p| p.validate()).transpose()?;
+        self.iommu |= self
+            .platform
+            .as_ref()
+            .map(|p| p.iommu_segments.is_some())
+            .unwrap_or_default();
+
+        Ok(id_list)
    }

    pub fn parse(vm_params: VmParams) -> Result<Self> {
-        let mut iommu = false;
-
        let mut disks: Option<Vec<DiskConfig>> = None;
        if let Some(disk_list) = &vm_params.disks {
            let mut disk_config_list = Vec::new();
            for item in disk_list.iter() {
                let disk_config = DiskConfig::parse(item)?;
-                if disk_config.iommu {
-                    iommu = true;
-                }
                disk_config_list.push(disk_config);
            }
            disks = Some(disk_config_list);
@@ -2505,18 +2603,12 @@ impl VmConfig {
            let mut net_config_list = Vec::new();
            for item in net_list.iter() {
                let net_config = NetConfig::parse(item)?;
-                if net_config.iommu {
-                    iommu = true;
-                }
                net_config_list.push(net_config);
            }
            net = Some(net_config_list);
        }

        let rng = RngConfig::parse(vm_params.rng)?;
-        if rng.iommu {
-            iommu = true;
-        }

        let mut balloon: Option<BalloonConfig> = None;
        if let Some(balloon_params) = &vm_params.balloon {
@@ -2537,18 +2629,12 @@ impl VmConfig {
            let mut pmem_config_list = Vec::new();
            for item in pmem_list.iter() {
                let pmem_config = PmemConfig::parse(item)?;
-                if pmem_config.iommu {
-                    iommu = true;
-                }
                pmem_config_list.push(pmem_config);
            }
            pmem = Some(pmem_config_list);
        }

        let console = ConsoleConfig::parse(vm_params.console)?;
-        if console.iommu {
-            iommu = true;
-        }
        let serial = ConsoleConfig::parse(vm_params.serial)?;

        let mut devices: Option<Vec<DeviceConfig>> = None;
@@ -2556,9 +2642,6 @@ impl VmConfig {
            let mut device_config_list = Vec::new();
            for item in device_list.iter() {
                let device_config = DeviceConfig::parse(item)?;
-                if device_config.iommu {
-                    iommu = true;
-                }
                device_config_list.push(device_config);
            }
            devices = Some(device_config_list);
@@ -2579,9 +2662,6 @@ impl VmConfig {
            let mut vdpa_config_list = Vec::new();
            for item in vdpa_list.iter() {
                let vdpa_config = VdpaConfig::parse(item)?;
-                if vdpa_config.iommu {
-                    iommu = true;
-                }
                vdpa_config_list.push(vdpa_config);
            }
            vdpa = Some(vdpa_config_list);
@@ -2590,18 +2670,10 @@ impl VmConfig {
        let mut vsock: Option<VsockConfig> = None;
        if let Some(vs) = &vm_params.vsock {
            let vsock_config = VsockConfig::parse(vs)?;
-            if vsock_config.iommu {
-                iommu = true;
-            }
            vsock = Some(vsock_config);
        }

        let platform = vm_params.platform.map(PlatformConfig::parse).transpose()?;
-        if let Some(platform_config) = platform.as_ref() {
-            if platform_config.iommu_segments.is_some() {
-                iommu = true;
-            }
-        }

        #[cfg(target_arch = "x86_64")]
        let mut sgx_epc: Option<Vec<SgxEpcConfig>> = None;
@@ -2647,7 +2719,7 @@ impl VmConfig {
        #[cfg(feature = "gdb")]
        let gdb = vm_params.gdb;

-        let config = VmConfig {
+        let mut config = VmConfig {
            cpus: CpusConfig::parse(vm_params.cpus)?,
            memory: MemoryConfig::parse(vm_params.memory, vm_params.memory_zones)?,
            kernel,
@@ -2665,7 +2737,7 @@ impl VmConfig {
            user_devices,
            vdpa,
            vsock,
-            iommu,
+            iommu: false, // updated in VmConfig::validate()
            #[cfg(target_arch = "x86_64")]
            sgx_epc,
            numa,
@@ -3261,7 +3333,7 @@ mod tests {

    #[test]
    fn test_config_validation() {
-        let valid_config = VmConfig {
+        let mut valid_config = VmConfig {
            cpus: CpusConfig {
                boot_vcpus: 1,
                max_vcpus: 1,
@@ -3498,6 +3570,7 @@ mod tests {
        still_valid_config.platform = Some(PlatformConfig {
            num_pci_segments: 16,
            iommu_segments: Some(vec![1, 2, 3]),
+            ..Default::default()
        });
        assert!(still_valid_config.validate().is_ok());

@@ -3505,6 +3578,7 @@ mod tests {
        invalid_config.platform = Some(PlatformConfig {
            num_pci_segments: 16,
            iommu_segments: Some(vec![17, 18]),
+            ..Default::default()
        });
        assert_eq!(
            invalid_config.validate(),
@@ -3515,6 +3589,7 @@ mod tests {
        still_valid_config.platform = Some(PlatformConfig {
            num_pci_segments: 16,
            iommu_segments: Some(vec![1, 2, 3]),
+            ..Default::default()
        });
        still_valid_config.disks = Some(vec![DiskConfig {
            iommu: true,
@@ -3527,6 +3602,7 @@ mod tests {
        still_valid_config.platform = Some(PlatformConfig {
            num_pci_segments: 16,
            iommu_segments: Some(vec![1, 2, 3]),
+            ..Default::default()
        });
        still_valid_config.net = Some(vec![NetConfig {
            iommu: true,
@@ -3539,6 +3615,7 @@ mod tests {
        still_valid_config.platform = Some(PlatformConfig {
            num_pci_segments: 16,
            iommu_segments: Some(vec![1, 2, 3]),
+            ..Default::default()
        });
        still_valid_config.pmem = Some(vec![PmemConfig {
            iommu: true,
@@ -3551,6 +3628,7 @@ mod tests {
        still_valid_config.platform = Some(PlatformConfig {
            num_pci_segments: 16,
            iommu_segments: Some(vec![1, 2, 3]),
+            ..Default::default()
        });
        still_valid_config.devices = Some(vec![DeviceConfig {
            iommu: true,
@@ -3563,6 +3641,7 @@ mod tests {
        still_valid_config.platform = Some(PlatformConfig {
            num_pci_segments: 16,
            iommu_segments: Some(vec![1, 2, 3]),
+            ..Default::default()
        });
        still_valid_config.vsock = Some(VsockConfig {
            iommu: true,
@@ -3575,6 +3654,7 @@ mod tests {
        invalid_config.platform = Some(PlatformConfig {
            num_pci_segments: 16,
            iommu_segments: Some(vec![1, 2, 3]),
+            ..Default::default()
        });
        invalid_config.disks = Some(vec![DiskConfig {
            iommu: false,
@@ -3590,6 +3670,7 @@ mod tests {
        invalid_config.platform = Some(PlatformConfig {
            num_pci_segments: 16,
            iommu_segments: Some(vec![1, 2, 3]),
+            ..Default::default()
        });
        invalid_config.net = Some(vec![NetConfig {
            iommu: false,
@@ -3605,6 +3686,7 @@ mod tests {
        invalid_config.platform = Some(PlatformConfig {
            num_pci_segments: 16,
            iommu_segments: Some(vec![1, 2, 3]),
+            ..Default::default()
        });
        invalid_config.pmem = Some(vec![PmemConfig {
            iommu: false,
@@ -3620,6 +3702,7 @@ mod tests {
        invalid_config.platform = Some(PlatformConfig {
            num_pci_segments: 16,
            iommu_segments: Some(vec![1, 2, 3]),
+            ..Default::default()
        });
        invalid_config.devices = Some(vec![DeviceConfig {
            iommu: false,
@@ -3635,6 +3718,7 @@ mod tests {
        invalid_config.platform = Some(PlatformConfig {
            num_pci_segments: 16,
            iommu_segments: Some(vec![1, 2, 3]),
+            ..Default::default()
        });
        invalid_config.vsock = Some(VsockConfig {
            iommu: false,
@@ -3651,6 +3735,7 @@ mod tests {
        invalid_config.platform = Some(PlatformConfig {
            num_pci_segments: 16,
            iommu_segments: Some(vec![1, 2, 3]),
+            ..Default::default()
        });
        invalid_config.user_devices = Some(vec![UserDeviceConfig {
            pci_segment: 1,
@@ -3658,13 +3743,14 @@ mod tests {
        }]);
        assert_eq!(
            invalid_config.validate(),
-            Err(ValidationError::IommuNotSupported(1))
+            Err(ValidationError::IommuNotSupportedOnSegment(1))
        );

        let mut invalid_config = valid_config.clone();
        invalid_config.platform = Some(PlatformConfig {
            num_pci_segments: 16,
            iommu_segments: Some(vec![1, 2, 3]),
+            ..Default::default()
        });
        invalid_config.vdpa = Some(vec![VdpaConfig {
            pci_segment: 1,
@@ -3680,6 +3766,7 @@ mod tests {
        invalid_config.platform = Some(PlatformConfig {
            num_pci_segments: 16,
            iommu_segments: Some(vec![1, 2, 3]),
+            ..Default::default()
        });
        invalid_config.fs = Some(vec![FsConfig {
            pci_segment: 1,
@@ -3687,7 +3774,7 @@ mod tests {
        }]);
        assert_eq!(
            invalid_config.validate(),
-            Err(ValidationError::IommuNotSupported(1))
+            Err(ValidationError::IommuNotSupportedOnSegment(1))
        );
    }
 }
--- a/vmm/src/cpu.rs
+++ b/vmm/src/cpu.rs
@@ -30,13 +30,13 @@ use devices::interrupt_controller::InterruptController;
 use gdbstub_arch::x86::reg::{X86SegmentRegs, X86_64CoreRegs};
 #[cfg(target_arch = "aarch64")]
 use hypervisor::kvm::kvm_bindings;
+#[cfg(feature = "tdx")]
+use hypervisor::kvm::{TdxExitDetails, TdxExitStatus};
+#[cfg(target_arch = "x86_64")]
+use hypervisor::x86_64::CpuId;
 #[cfg(all(target_arch = "x86_64", feature = "gdb"))]
 use hypervisor::x86_64::{SpecialRegisters, StandardRegisters};
-#[cfg(target_arch = "x86_64")]
-use hypervisor::CpuId;
-use hypervisor::{vm::VmmOps, CpuState, HypervisorCpuError, VmExit};
-#[cfg(feature = "tdx")]
-use hypervisor::{TdxExitDetails, TdxExitStatus};
+use hypervisor::{CpuState, HypervisorCpuError, VmExit, VmOps};
 use libc::{c_void, siginfo_t};
 use seccompiler::{apply_filter, SeccompAction};
 use std::collections::BTreeMap;
@@ -44,9 +44,11 @@ use std::os::unix::thread::JoinHandleExt;
 use std::sync::atomic::{AtomicBool, Ordering};
 use std::sync::{Arc, Barrier, Mutex};
 use std::{cmp, io, result, thread};
+use thiserror::Error;
 use vm_device::BusDevice;
-use vm_memory::GuestAddress;
-use vm_memory::GuestMemoryAtomic;
+#[cfg(feature = "gdb")]
+use vm_memory::{Bytes, GuestAddressSpace};
+use vm_memory::{GuestAddress, GuestMemoryAtomic};
 use vm_migration::{
    Migratable, MigratableError, Pausable, Snapshot, SnapshotDataSection, Snapshottable,
    Transportable,
@@ -56,75 +58,74 @@ use vmm_sys_util::signal::{register_signal_handler, SIGRTMIN};

 pub const CPU_MANAGER_ACPI_SIZE: usize = 0xc;

-#[derive(Debug)]
+#[derive(Debug, Error)]
 pub enum Error {
-    /// Cannot create the vCPU.
-    VcpuCreate(anyhow::Error),
+    #[error("Error creating vCPU: {0}")]
+    VcpuCreate(#[source] anyhow::Error),

-    /// Cannot run the VCPUs.
-    VcpuRun(anyhow::Error),
+    #[error("Error running bCPU: {0}")]
+    VcpuRun(#[source] anyhow::Error),

-    /// Cannot spawn a new vCPU thread.
-    VcpuSpawn(io::Error),
+    #[error("Error spawning vCPU thread: {0}")]
+    VcpuSpawn(#[source] io::Error),

-    /// Cannot generate common CPUID
-    CommonCpuId(arch::Error),
+    #[error("Error generating common CPUID: {0}")]
+    CommonCpuId(#[source] arch::Error),

-    /// Error configuring VCPU
-    VcpuConfiguration(arch::Error),
+    #[error("Error configuring vCPU: {0}")]
+    VcpuConfiguration(#[source] arch::Error),

    #[cfg(target_arch = "aarch64")]
-    /// Error fetching prefered target
-    VcpuArmPreferredTarget(hypervisor::HypervisorVmError),
+    #[error("Error fetching preferred target: {0}")]
+    VcpuArmPreferredTarget(#[source] hypervisor::HypervisorVmError),

    #[cfg(target_arch = "aarch64")]
-    /// Error doing vCPU init on Arm.
-    VcpuArmInit(hypervisor::HypervisorCpuError),
+    #[error("Error initialising vCPU: {0}")]
+    VcpuArmInit(#[source] hypervisor::HypervisorCpuError),

-    /// Failed to join on vCPU threads
+    #[error("Failed to join on vCPU threads: {0:?}")]
    ThreadCleanup(std::boxed::Box<dyn std::any::Any + std::marker::Send>),

-    /// Cannot add legacy device to Bus.
-    BusError(vm_device::BusError),
+    #[error("Error adding CpuManager to MMIO bus: {0}")]
+    BusError(#[source] vm_device::BusError),

-    /// Asking for more vCPUs that we can have
+    #[error("Requested vCPUs exceed maximum")]
    DesiredVCpuCountExceedsMax,

-    /// Cannot create seccomp filter
-    CreateSeccompFilter(seccompiler::Error),
+    #[error("Cannot create seccomp filter: {0}")]
+    CreateSeccompFilter(#[source] seccompiler::Error),

-    /// Cannot apply seccomp filter
-    ApplySeccompFilter(seccompiler::Error),
+    #[error("Cannot apply seccomp filter: {0}")]
+    ApplySeccompFilter(#[source] seccompiler::Error),

-    /// Error starting vCPU after restore
-    StartRestoreVcpu(anyhow::Error),
+    #[error("Error starting vCPU after restore: {0}")]
+    StartRestoreVcpu(#[source] anyhow::Error),

-    /// Error because an unexpected VmExit type was received.
+    #[error("Unexpected VmExit")]
    UnexpectedVmExit,

-    /// Failed to allocate MMIO address
+    #[error("Failed to allocate MMIO address for CpuManager")]
    AllocateMmmioAddress,

    #[cfg(feature = "tdx")]
-    InitializeTdx(hypervisor::HypervisorCpuError),
+    #[error("Error initializing TDX: {0}")]
+    InitializeTdx(#[source] hypervisor::HypervisorCpuError),

    #[cfg(target_arch = "aarch64")]
-    InitPmu(hypervisor::HypervisorCpuError),
-
-    /// Failed scheduling the thread on the expected CPU set.
-    ScheduleCpuSet,
+    #[error("Error initializing PMU: {0}")]
+    InitPmu(#[source] hypervisor::HypervisorCpuError),

    #[cfg(all(target_arch = "x86_64", feature = "gdb"))]
-    /// Error on debug related CPU ops.
-    CpuDebug(hypervisor::HypervisorCpuError),
+    #[error("Error during CPU debug: {0}")]
+    CpuDebug(#[source] hypervisor::HypervisorCpuError),

    #[cfg(all(target_arch = "x86_64", feature = "gdb"))]
-    /// Failed to translate guest virtual address.
-    TranslateVirtualAddress(hypervisor::HypervisorCpuError),
+    #[error("Error translating virtual address: {0}")]
+    TranslateVirtualAddress(#[source] hypervisor::HypervisorCpuError),

    #[cfg(all(feature = "amx", target_arch = "x86_64"))]
-    /// "Failed to setup AMX.
-    AmxEnable(anyhow::Error),
+    #[error("Error setting up AMX: {0}")]
+    AmxEnable(#[source] anyhow::Error),
 }
 pub type Result<T> = result::Result<T, Error>;

@@ -254,14 +255,14 @@ impl Vcpu {
    ///
    /// * `id` - Represents the CPU number between [0, max vcpus).
    /// * `vm` - The virtual machine this vcpu will get attached to.
-    /// * `vmmops` - Optional object for exit handling.
+    /// * `vm_ops` - Optional object for exit handling.
    pub fn new(
        id: u8,
        vm: &Arc<dyn hypervisor::Vm>,
-        vmmops: Option<Arc<dyn VmmOps>>,
+        vm_ops: Option<Arc<dyn VmOps>>,
    ) -> Result<Self> {
        let vcpu = vm
-            .create_vcpu(id, vmmops)
+            .create_vcpu(id, vm_ops)
            .map_err(|e| Error::VcpuCreate(e.into()))?;
        // Initially the cpuid per vCPU is the one supported by this VM.
        Ok(Vcpu {
@@ -411,7 +412,7 @@ pub struct CpuManager {
    selected_cpu: u8,
    vcpus: Vec<Arc<Mutex<Vcpu>>>,
    seccomp_action: SeccompAction,
-    vmmops: Arc<dyn VmmOps>,
+    vm_ops: Arc<dyn VmOps>,
    #[cfg_attr(target_arch = "aarch64", allow(dead_code))]
    acpi_address: Option<GuestAddress>,
    proximity_domain_per_cpu: BTreeMap<u8, u32>,
@@ -561,7 +562,7 @@ impl CpuManager {
        #[cfg(feature = "gdb")] vm_debug_evt: EventFd,
        hypervisor: Arc<dyn hypervisor::Hypervisor>,
        seccomp_action: SeccompAction,
-        vmmops: Arc<dyn VmmOps>,
+        vm_ops: Arc<dyn VmOps>,
        #[cfg(feature = "tdx")] tdx_enabled: bool,
        numa_nodes: &NumaNodes,
    ) -> Result<Arc<Mutex<CpuManager>>> {
@@ -683,7 +684,7 @@ impl CpuManager {
            selected_cpu: 0,
            vcpus: Vec::with_capacity(usize::from(config.max_vcpus)),
            seccomp_action,
-            vmmops,
+            vm_ops,
            acpi_address,
            proximity_domain_per_cpu,
            affinity,
@@ -712,7 +713,7 @@ impl CpuManager {
    ) -> Result<()> {
        info!("Creating vCPU: cpu_id = {}", cpu_id);

-        let mut vcpu = Vcpu::new(cpu_id, &self.vm, Some(self.vmmops.clone()))?;
+        let mut vcpu = Vcpu::new(cpu_id, &self.vm, Some(self.vm_ops.clone()))?;

        if let Some(snapshot) = snapshot {
            // AArch64 vCPUs should be initialized after created.
@@ -988,8 +989,8 @@ impl CpuManager {
                                    }
                                    #[cfg(feature = "tdx")]
                                    VmExit::Tdx => {
-                                        if let Some(vcpu_fd) = Arc::get_mut(&mut vcpu.vcpu) {
-                                            match vcpu_fd.get_tdx_exit_details() {
+                                        if let Some(vcpu) = Arc::get_mut(&mut vcpu.vcpu) {
+                                            match vcpu.get_tdx_exit_details() {
                                                Ok(details) => match details {
                                                    TdxExitDetails::GetQuote => warn!("TDG_VP_VMCALL_GET_QUOTE not supported"),
                                                    TdxExitDetails::SetupEventNotifyInterrupt => {
@@ -998,7 +999,7 @@ impl CpuManager {
                                                },
                                                Err(e) => error!("Unexpected TDX VMCALL: {}", e),
                                            }
-                                            vcpu_fd.set_tdx_status(TdxExitStatus::InvalidOperand);
+                                            vcpu.set_tdx_status(TdxExitStatus::InvalidOperand);
                                        } else {
                                            // We should never reach this code as
                                            // this means the design from the code
@@ -2046,10 +2047,11 @@ impl Debuggable for CpuManager {
            };
            let psize = arch::PAGE_SIZE as u64;
            let read_len = std::cmp::min(len as u64 - total_read, psize - (paddr & (psize - 1)));
-            self.vmmops
-                .guest_mem_read(
-                    paddr,
+            self.vm_memory
+                .memory()
+                .read(
                    &mut buf[total_read as usize..total_read as usize + read_len as usize],
+                    GuestAddress(paddr),
                )
                .map_err(DebuggableError::ReadMem)?;
            total_read += read_len;
@@ -2078,10 +2080,11 @@ impl Debuggable for CpuManager {
                data.len() as u64 - total_written,
                psize - (paddr & (psize - 1)),
            );
-            self.vmmops
-                .guest_mem_write(
-                    paddr,
+            self.vm_memory
+                .memory()
+                .write(
                    &data[total_written as usize..total_written as usize + write_len as usize],
+                    GuestAddress(paddr),
                )
                .map_err(DebuggableError::WriteMem)?;
            total_written += write_len;
--- a/vmm/src/device_manager.rs
+++ b/vmm/src/device_manager.rs
@@ -14,11 +14,8 @@ use crate::config::{
    VdpaConfig, VhostMode, VmConfig, VsockConfig,
 };
 use crate::device_tree::{DeviceNode, DeviceTree};
-#[cfg(feature = "kvm")]
-use crate::interrupt::kvm::KvmMsiInterruptManager as MsiInterruptManager;
-#[cfg(feature = "mshv")]
-use crate::interrupt::mshv::MshvMsiInterruptManager as MsiInterruptManager;
 use crate::interrupt::LegacyUserspaceInterruptManager;
+use crate::interrupt::MsiInterruptManager;
 use crate::memory_manager::MEMORY_MANAGER_ACPI_SIZE;
 use crate::memory_manager::{Error as MemoryManagerError, MemoryManager};
 use crate::pci_segment::PciSegment;
@@ -56,11 +53,7 @@ use devices::legacy::Serial;
 use devices::{
    interrupt_controller, interrupt_controller::InterruptController, AcpiNotificationFlags,
 };
-#[cfg(feature = "kvm")]
-use hypervisor::kvm_ioctls::*;
-use hypervisor::DeviceFd;
-#[cfg(feature = "mshv")]
-use hypervisor::IoEventAddress;
+use hypervisor::{DeviceFd, HypervisorVmError, IoEventAddress};
 use libc::{
    cfmakeraw, isatty, tcgetattr, tcsetattr, termios, MAP_NORESERVE, MAP_PRIVATE, MAP_SHARED,
    O_TMPFILE, PROT_READ, PROT_WRITE, TCSANOW,
@@ -72,7 +65,8 @@ use pci::{
    VfioUserPciDevice, VfioUserPciDeviceError,
 };
 use seccompiler::SeccompAction;
-use std::collections::HashMap;
+use serde::{Deserialize, Serialize};
+use std::collections::{BTreeSet, HashMap};
 use std::convert::TryInto;
 use std::fs::{read_link, File, OpenOptions};
 use std::io::{self, stdout, Seek, SeekFrom};
@@ -83,6 +77,7 @@ use std::os::unix::io::{AsRawFd, FromRawFd, RawFd};
 use std::path::PathBuf;
 use std::result;
 use std::sync::{Arc, Mutex};
+use std::time::Instant;
 use vfio_ioctls::{VfioContainer, VfioDevice};
 use virtio_devices::transport::VirtioPciDevice;
 use virtio_devices::transport::VirtioTransport;
@@ -115,30 +110,28 @@ use vmm_sys_util::eventfd::EventFd;
 #[cfg(target_arch = "aarch64")]
 const MMIO_LEN: u64 = 0x1000;

-const VFIO_DEVICE_NAME_PREFIX: &str = "_vfio";
-
-const VFIO_USER_DEVICE_NAME_PREFIX: &str = "_vfio_user";
-
+// Singleton devices / devices the user cannot name
 #[cfg(target_arch = "x86_64")]
-const IOAPIC_DEVICE_NAME: &str = "_ioapic";
-
-const SERIAL_DEVICE_NAME_PREFIX: &str = "_serial";
+const IOAPIC_DEVICE_NAME: &str = "__ioapic";
+const SERIAL_DEVICE_NAME: &str = "__serial";
 #[cfg(target_arch = "aarch64")]
-const GPIO_DEVICE_NAME_PREFIX: &str = "_gpio";
+const GPIO_DEVICE_NAME: &str = "__gpio";
+const RNG_DEVICE_NAME: &str = "__rng";
+const IOMMU_DEVICE_NAME: &str = "__iommu";
+const BALLOON_DEVICE_NAME: &str = "__balloon";
+const CONSOLE_DEVICE_NAME: &str = "__console";

-const CONSOLE_DEVICE_NAME: &str = "_console";
+// Devices that the user may name and for which we generate
+// identifiers if the user doesn't give one
 const DISK_DEVICE_NAME_PREFIX: &str = "_disk";
 const FS_DEVICE_NAME_PREFIX: &str = "_fs";
-const BALLOON_DEVICE_NAME: &str = "_balloon";
 const NET_DEVICE_NAME_PREFIX: &str = "_net";
 const PMEM_DEVICE_NAME_PREFIX: &str = "_pmem";
-const RNG_DEVICE_NAME: &str = "_rng";
 const VDPA_DEVICE_NAME_PREFIX: &str = "_vdpa";
 const VSOCK_DEVICE_NAME_PREFIX: &str = "_vsock";
-const WATCHDOG_DEVICE_NAME: &str = "_watchdog";
-
-const IOMMU_DEVICE_NAME: &str = "_iommu";
-
+const WATCHDOG_DEVICE_NAME: &str = "__watchdog";
+const VFIO_DEVICE_NAME_PREFIX: &str = "_vfio";
+const VFIO_USER_DEVICE_NAME_PREFIX: &str = "_vfio_user";
 const VIRTIO_PCI_DEVICE_NAME_PREFIX: &str = "_virtio-pci";

 /// Errors associated with device manager
@@ -399,9 +392,6 @@ pub enum DeviceManagerError {
    /// Resource was already found.
    ResourceAlreadyExists,

-    /// Expected resources for virtio-pci could not be found.
-    MissingVirtioPciResources,
-
    /// Expected resources for virtio-pmem could not be found.
    MissingVirtioPmemResources,

@@ -479,7 +469,13 @@ pub enum DeviceManagerError {
    InvalidIommuHotplug,

    /// Failed to create UEFI flash
-    CreateUefiFlash(hypervisor::vm::HypervisorVmError),
+    CreateUefiFlash(HypervisorVmError),
+
+    /// Invalid identifier as it is not unique.
+    IdentifierNotUnique(String),
+
+    /// Invalid identifier
+    InvalidIdentifier(String),
 }
 pub type DeviceManagerResult<T> = result::Result<T, DeviceManagerError>;

@@ -663,19 +659,13 @@ impl DeviceRelocation for AddressManager {
            }
        }

-        let any_dev = pci_dev.as_any();
-        if let Some(virtio_pci_dev) = any_dev.downcast_ref::<VirtioPciDevice>() {
-            // Update the device_tree resources associated with the device
-            if let Some(node) = self
-                .device_tree
-                .lock()
-                .unwrap()
-                .get_mut(&virtio_pci_dev.id())
-            {
+        // Update the device_tree resources associated with the device
+        if let Some(id) = pci_dev.id() {
+            if let Some(node) = self.device_tree.lock().unwrap().get_mut(&id) {
                let mut resource_updated = false;
                for resource in node.resources.iter_mut() {
-                    if let Resource::MmioAddressRange { base, .. } = resource {
-                        if *base == old_base {
+                    if let Resource::PciBar { base, type_, .. } = resource {
+                        if PciBarRegionType::from(*type_) == region_type && *base == old_base {
                            *base = new_base;
                            resource_updated = true;
                            break;
@@ -688,21 +678,20 @@ impl DeviceRelocation for AddressManager {
                        io::ErrorKind::Other,
                        format!(
                            "Couldn't find a resource with base 0x{:x} for device {}",
-                            old_base,
-                            virtio_pci_dev.id()
+                            old_base, id
                        ),
                    ));
                }
            } else {
                return Err(io::Error::new(
                    io::ErrorKind::Other,
-                    format!(
-                        "Couldn't find device {} from device tree",
-                        virtio_pci_dev.id()
-                    ),
+                    format!("Couldn't find device {} from device tree", id),
                ));
            }
+        }

+        let any_dev = pci_dev.as_any();
+        if let Some(virtio_pci_dev) = any_dev.downcast_ref::<VirtioPciDevice>() {
            let bar_addr = virtio_pci_dev.config_bar_addr();
            if bar_addr == new_base {
                for (event, addr) in virtio_pci_dev.ioeventfds(old_base) {
@@ -844,7 +833,8 @@ pub struct DeviceManager {
    #[cfg(target_arch = "aarch64")]
    interrupt_controller: Option<Arc<Mutex<gic::Gic>>>,

-    // Things to be added to the commandline (i.e. for virtio-mmio)
+    // Things to be added to the commandline (e.g. aarch64 early console)
+    #[cfg(target_arch = "aarch64")]
    cmdline_additions: Vec<String>,

    // ACPI GED notification device
@@ -943,6 +933,12 @@ pub struct DeviceManager {

    // io_uring availability if detected
    io_uring_supported: Option<bool>,
+
+    // List of unique identifiers provided at boot through the configuration.
+    boot_id_list: BTreeSet<String>,
+
+    // Start time of the VM
+    timestamp: Instant,
 }

 impl DeviceManager {
@@ -958,6 +954,8 @@ impl DeviceManager {
        activate_evt: &EventFd,
        force_iommu: bool,
        restoring: bool,
+        boot_id_list: BTreeSet<String>,
+        timestamp: Instant,
    ) -> DeviceManagerResult<Arc<Mutex<Self>>> {
        let device_tree = Arc::new(Mutex::new(DeviceTree::new()));

@@ -1039,8 +1037,8 @@ impl DeviceManager {
            address_manager: Arc::clone(&address_manager),
            console: Arc::new(Console::default()),
            interrupt_controller: None,
+            #[cfg(target_arch = "aarch64")]
            cmdline_additions: Vec::new(),
-
            ged_notification_device: None,
            config,
            memory_manager,
@@ -1061,15 +1059,12 @@ impl DeviceManager {
            #[cfg(target_arch = "aarch64")]
            id_to_dev_info: HashMap::new(),
            seccomp_action,
-
            numa_nodes,
            balloon: None,
            activate_evt: activate_evt
                .try_clone()
                .map_err(DeviceManagerError::EventFd)?,
-
            acpi_address,
-
            selected_segment: 0,
            serial_pty: None,
            serial_manager: None,
@@ -1083,6 +1078,8 @@ impl DeviceManager {
            force_iommu,
            restoring,
            io_uring_supported: None,
+            boot_id_list,
+            timestamp,
        };

        let device_manager = Arc::new(Mutex::new(device_manager));
@@ -1524,6 +1521,15 @@ impl DeviceManager {
                .map_err(DeviceManagerError::BusError)?;
        }

+        // 0x80 debug port
+        let debug_port = Arc::new(Mutex::new(devices::legacy::DebugPort::new(self.timestamp)));
+        self.bus_devices
+            .push(Arc::clone(&debug_port) as Arc<Mutex<dyn BusDevice>>);
+        self.address_manager
+            .io_bus
+            .insert(debug_port, 0x80, 0x1)
+            .map_err(DeviceManagerError::BusError)?;
+
        Ok(())
    }

@@ -1569,7 +1575,7 @@ impl DeviceManager {
        );

        // Add a GPIO device
-        let id = String::from(GPIO_DEVICE_NAME_PREFIX);
+        let id = String::from(GPIO_DEVICE_NAME);
        let gpio_irq = self
            .address_manager
            .allocator
@@ -1659,7 +1665,7 @@ impl DeviceManager {
        // Serial is tied to IRQ #4
        let serial_irq = 4;

-        let id = String::from(SERIAL_DEVICE_NAME_PREFIX);
+        let id = String::from(SERIAL_DEVICE_NAME);

        let interrupt_group = interrupt_manager
            .create_group(LegacyIrqGroupConfig {
@@ -1705,7 +1711,7 @@ impl DeviceManager {
        interrupt_manager: &Arc<dyn InterruptManager<GroupConfig = LegacyIrqGroupConfig>>,
        serial_writer: Option<Box<dyn io::Write + Send>>,
    ) -> DeviceManagerResult<Arc<Mutex<Pl011>>> {
-        let id = String::from(SERIAL_DEVICE_NAME_PREFIX);
+        let id = String::from(SERIAL_DEVICE_NAME);

        let serial_irq = self
            .address_manager
@@ -1725,6 +1731,7 @@ impl DeviceManager {
            id.clone(),
            interrupt_group,
            serial_writer,
+            self.timestamp,
        )));

        self.bus_devices
@@ -2032,14 +2039,14 @@ impl DeviceManager {

        info!("Creating virtio-block device: {:?}", disk_cfg);

-        if disk_cfg.vhost_user {
+        let (virtio_device, migratable_device) = if disk_cfg.vhost_user {
            let socket = disk_cfg.vhost_socket.as_ref().unwrap().clone();
            let vu_cfg = VhostUserConfig {
                socket,
                num_queues: disk_cfg.num_queues,
                queue_size: disk_cfg.queue_size,
            };
-            let vhost_user_block_device = Arc::new(Mutex::new(
+            let vhost_user_block = Arc::new(Mutex::new(
                match virtio_devices::vhost_user::Blk::new(
                    id.clone(),
                    vu_cfg,
@@ -2057,22 +2064,10 @@ impl DeviceManager {
                },
            ));

-            // Fill the device tree with a new node. In case of restore, we
-            // know there is nothing to do, so we can simply override the
-            // existing entry.
-            self.device_tree
-                .lock()
-                .unwrap()
-                .insert(id.clone(), device_node!(id, vhost_user_block_device));
-
-            Ok(MetaVirtioDevice {
-                virtio_device: Arc::clone(&vhost_user_block_device)
-                    as Arc<Mutex<dyn virtio_devices::VirtioDevice>>,
-                iommu: false,
-                id,
-                pci_segment: disk_cfg.pci_segment,
-                dma_handler: None,
-            })
+            (
+                Arc::clone(&vhost_user_block) as Arc<Mutex<dyn virtio_devices::VirtioDevice>>,
+                vhost_user_block as Arc<Mutex<dyn Migratable>>,
+            )
        } else {
            let mut options = OpenOptions::new();
            options.read(true);
@@ -2138,7 +2133,7 @@ impl DeviceManager {
                }
            };

-            let dev = Arc::new(Mutex::new(
+            let virtio_block = Arc::new(Mutex::new(
                virtio_devices::Block::new(
                    id.clone(),
                    image,
@@ -2160,25 +2155,27 @@ impl DeviceManager {
                .map_err(DeviceManagerError::CreateVirtioBlock)?,
            ));

-            let virtio_device = Arc::clone(&dev) as Arc<Mutex<dyn virtio_devices::VirtioDevice>>;
-            let migratable_device = dev as Arc<Mutex<dyn Migratable>>;
+            (
+                Arc::clone(&virtio_block) as Arc<Mutex<dyn virtio_devices::VirtioDevice>>,
+                virtio_block as Arc<Mutex<dyn Migratable>>,
+            )
+        };

-            // Fill the device tree with a new node. In case of restore, we
-            // know there is nothing to do, so we can simply override the
-            // existing entry.
-            self.device_tree
-                .lock()
-                .unwrap()
-                .insert(id.clone(), device_node!(id, migratable_device));
+        // Fill the device tree with a new node. In case of restore, we
+        // know there is nothing to do, so we can simply override the
+        // existing entry.
+        self.device_tree
+            .lock()
+            .unwrap()
+            .insert(id.clone(), device_node!(id, migratable_device));

-            Ok(MetaVirtioDevice {
-                virtio_device,
-                iommu: disk_cfg.iommu,
-                id,
-                pci_segment: disk_cfg.pci_segment,
-                dma_handler: None,
-            })
-        }
+        Ok(MetaVirtioDevice {
+            virtio_device,
+            iommu: disk_cfg.iommu,
+            id,
+            pci_segment: disk_cfg.pci_segment,
+            dma_handler: None,
+        })
    }

    fn make_virtio_block_devices(&mut self) -> DeviceManagerResult<Vec<MetaVirtioDevice>> {
@@ -2208,7 +2205,7 @@ impl DeviceManager {
        };
        info!("Creating virtio-net device: {:?}", net_cfg);

-        if net_cfg.vhost_user {
+        let (virtio_device, migratable_device) = if net_cfg.vhost_user {
            let socket = net_cfg.vhost_socket.as_ref().unwrap().clone();
            let vu_cfg = VhostUserConfig {
                socket,
@@ -2219,7 +2216,7 @@ impl DeviceManager {
                VhostMode::Client => false,
                VhostMode::Server => true,
            };
-            let vhost_user_net_device = Arc::new(Mutex::new(
+            let vhost_user_net = Arc::new(Mutex::new(
                match virtio_devices::vhost_user::Net::new(
                    id.clone(),
                    net_cfg.mac,
@@ -2239,24 +2236,12 @@ impl DeviceManager {
                },
            ));

-            // Fill the device tree with a new node. In case of restore, we
-            // know there is nothing to do, so we can simply override the
-            // existing entry.
-            self.device_tree
-                .lock()
-                .unwrap()
-                .insert(id.clone(), device_node!(id, vhost_user_net_device));
-
-            Ok(MetaVirtioDevice {
-                virtio_device: Arc::clone(&vhost_user_net_device)
-                    as Arc<Mutex<dyn virtio_devices::VirtioDevice>>,
-                iommu: net_cfg.iommu,
-                id,
-                pci_segment: net_cfg.pci_segment,
-                dma_handler: None,
-            })
+            (
+                Arc::clone(&vhost_user_net) as Arc<Mutex<dyn virtio_devices::VirtioDevice>>,
+                vhost_user_net as Arc<Mutex<dyn Migratable>>,
+            )
        } else {
-            let virtio_net_device = if let Some(ref tap_if_name) = net_cfg.tap {
+            let virtio_net = if let Some(ref tap_if_name) = net_cfg.tap {
                Arc::new(Mutex::new(
                    virtio_devices::Net::new(
                        id.clone(),
@@ -2314,23 +2299,27 @@ impl DeviceManager {
                ))
            };

-            // Fill the device tree with a new node. In case of restore, we
-            // know there is nothing to do, so we can simply override the
-            // existing entry.
-            self.device_tree
-                .lock()
-                .unwrap()
-                .insert(id.clone(), device_node!(id, virtio_net_device));
+            (
+                Arc::clone(&virtio_net) as Arc<Mutex<dyn virtio_devices::VirtioDevice>>,
+                virtio_net as Arc<Mutex<dyn Migratable>>,
+            )
+        };

-            Ok(MetaVirtioDevice {
-                virtio_device: Arc::clone(&virtio_net_device)
-                    as Arc<Mutex<dyn virtio_devices::VirtioDevice>>,
-                iommu: net_cfg.iommu,
-                id,
-                pci_segment: net_cfg.pci_segment,
-                dma_handler: None,
-            })
-        }
+        // Fill the device tree with a new node. In case of restore, we
+        // know there is nothing to do, so we can simply override the
+        // existing entry.
+        self.device_tree
+            .lock()
+            .unwrap()
+            .insert(id.clone(), device_node!(id, migratable_device));
+
+        Ok(MetaVirtioDevice {
+            virtio_device,
+            iommu: net_cfg.iommu,
+            id,
+            pci_segment: net_cfg.pci_segment,
+            dma_handler: None,
+        })
    }

    /// Add virto-net and vhost-user-net devices
@@ -3014,7 +3003,9 @@ impl DeviceManager {
            // Increment the counter.
            self.device_id_cnt += Wrapping(1);
            // Check if the name is already in use.
-            if !self.device_tree.lock().unwrap().contains_key(&name) {
+            if !self.boot_id_list.contains(&name)
+                && !self.device_tree.lock().unwrap().contains_key(&name)
+            {
                return Ok(name);
            }

@@ -3082,8 +3073,20 @@ impl DeviceManager {
        &mut self,
        device_cfg: &mut DeviceConfig,
    ) -> DeviceManagerResult<(PciBdf, String)> {
-        let pci_segment_id = device_cfg.pci_segment;
-        let pci_device_bdf = self.pci_segments[pci_segment_id as usize].next_device_bdf()?;
+        let vfio_name = if let Some(id) = &device_cfg.id {
+            if self.device_tree.lock().unwrap().contains_key(id) {
+                return Err(DeviceManagerError::DeviceIdAlreadyInUse);
+            }
+
+            id.clone()
+        } else {
+            let id = self.next_device_name(VFIO_DEVICE_NAME_PREFIX)?;
+            device_cfg.id = Some(id.clone());
+            id
+        };
+
+        let (pci_segment_id, pci_device_bdf, resources) =
+            self.pci_resources(&vfio_name, device_cfg.pci_segment)?;

        let mut needs_dma_mapping = false;

@@ -3175,35 +3178,25 @@ impl DeviceManager {
            };

        let vfio_pci_device = VfioPciDevice::new(
+            vfio_name.clone(),
            &self.address_manager.vm,
            vfio_device,
            vfio_container,
-            &self.msi_interrupt_manager,
+            self.msi_interrupt_manager.clone(),
            legacy_interrupt_group,
            device_cfg.iommu,
            pci_device_bdf,
        )
        .map_err(DeviceManagerError::VfioPciCreate)?;

-        let vfio_name = if let Some(id) = &device_cfg.id {
-            if self.device_tree.lock().unwrap().contains_key(id) {
-                return Err(DeviceManagerError::DeviceIdAlreadyInUse);
-            }
-
-            id.clone()
-        } else {
-            let id = self.next_device_name(VFIO_DEVICE_NAME_PREFIX)?;
-            device_cfg.id = Some(id.clone());
-            id
-        };
-
        let vfio_pci_device = Arc::new(Mutex::new(vfio_pci_device));

-        self.add_pci_device(
+        let new_resources = self.add_pci_device(
            vfio_pci_device.clone(),
            vfio_pci_device.clone(),
            pci_segment_id,
            pci_device_bdf,
+            resources,
        )?;

        vfio_pci_device
@@ -3216,13 +3209,8 @@ impl DeviceManager {

        let mut node = device_node!(vfio_name);

-        for region in vfio_pci_device.lock().unwrap().mmio_regions() {
-            node.resources.push(Resource::MmioAddressRange {
-                base: region.start.0,
-                size: region.length as u64,
-            });
-        }
-
+        // Update the device tree with correct resource information.
+        node.resources = new_resources;
        node.pci_bdf = Some(pci_device_bdf);
        node.pci_device_handle = Some(PciDeviceHandle::Vfio(vfio_pci_device));

@@ -3240,7 +3228,8 @@ impl DeviceManager {
        pci_device: Arc<Mutex<dyn PciDevice>>,
        segment_id: u16,
        bdf: PciBdf,
-    ) -> DeviceManagerResult<Vec<(GuestAddress, GuestUsize, PciBarRegionType)>> {
+        resources: Option<Vec<Resource>>,
+    ) -> DeviceManagerResult<Vec<Resource>> {
        let bars = pci_device
            .lock()
            .unwrap()
@@ -3250,6 +3239,7 @@ impl DeviceManager {
                    .allocator
                    .lock()
                    .unwrap(),
+                resources,
            )
            .map_err(DeviceManagerError::AllocateBars)?;

@@ -3274,7 +3264,18 @@ impl DeviceManager {
            )
            .map_err(DeviceManagerError::AddPciDevice)?;

-        Ok(bars)
+        let mut new_resources = Vec::new();
+        for bar in bars {
+            new_resources.push(Resource::PciBar {
+                index: bar.idx(),
+                base: bar.addr(),
+                size: bar.size(),
+                type_: bar.region_type().into(),
+                prefetchable: bar.prefetchable().into(),
+            });
+        }
+
+        Ok(new_resources)
    }

    fn add_vfio_devices(&mut self) -> DeviceManagerResult<Vec<PciBdf>> {
@@ -3300,8 +3301,20 @@ impl DeviceManager {
        &mut self,
        device_cfg: &mut UserDeviceConfig,
    ) -> DeviceManagerResult<(PciBdf, String)> {
-        let pci_segment_id = device_cfg.pci_segment;
-        let pci_device_bdf = self.pci_segments[pci_segment_id as usize].next_device_bdf()?;
+        let vfio_user_name = if let Some(id) = &device_cfg.id {
+            if self.device_tree.lock().unwrap().contains_key(id) {
+                return Err(DeviceManagerError::DeviceIdAlreadyInUse);
+            }
+
+            id.clone()
+        } else {
+            let id = self.next_device_name(VFIO_USER_DEVICE_NAME_PREFIX)?;
+            device_cfg.id = Some(id.clone());
+            id
+        };
+
+        let (pci_segment_id, pci_device_bdf, resources) =
+            self.pci_resources(&vfio_user_name, device_cfg.pci_segment)?;

        let legacy_interrupt_group =
            if let Some(legacy_interrupt_manager) = &self.legacy_interrupt_manager {
@@ -3324,20 +3337,15 @@ impl DeviceManager {
        ));

        let mut vfio_user_pci_device = VfioUserPciDevice::new(
+            vfio_user_name.clone(),
            &self.address_manager.vm,
            client.clone(),
-            &self.msi_interrupt_manager,
+            self.msi_interrupt_manager.clone(),
            legacy_interrupt_group,
            pci_device_bdf,
        )
        .map_err(DeviceManagerError::VfioUserCreate)?;

-        vfio_user_pci_device
-            .map_mmio_regions(&self.address_manager.vm, || {
-                self.memory_manager.lock().unwrap().allocate_memory_slot()
-            })
-            .map_err(DeviceManagerError::VfioUserMapRegion)?;
-
        let memory = self.memory_manager.lock().unwrap().guest_memory();
        let vfio_user_mapping = Arc::new(VfioUserDmaMapping::new(client, Arc::new(memory)));
        for virtio_mem_device in self.virtio_mem_devices.iter() {
@@ -3361,27 +3369,28 @@ impl DeviceManager {

        let vfio_user_pci_device = Arc::new(Mutex::new(vfio_user_pci_device));

-        let vfio_user_name = if let Some(id) = &device_cfg.id {
-            if self.device_tree.lock().unwrap().contains_key(id) {
-                return Err(DeviceManagerError::DeviceIdAlreadyInUse);
-            }
-
-            id.clone()
-        } else {
-            let id = self.next_device_name(VFIO_USER_DEVICE_NAME_PREFIX)?;
-            device_cfg.id = Some(id.clone());
-            id
-        };
-
-        self.add_pci_device(
+        let new_resources = self.add_pci_device(
            vfio_user_pci_device.clone(),
            vfio_user_pci_device.clone(),
            pci_segment_id,
            pci_device_bdf,
+            resources,
        )?;

+        // Note it is required to call 'add_pci_device()' in advance to have the list of
+        // mmio regions provisioned correctly
+        vfio_user_pci_device
+            .lock()
+            .unwrap()
+            .map_mmio_regions(&self.address_manager.vm, || {
+                self.memory_manager.lock().unwrap().allocate_memory_slot()
+            })
+            .map_err(DeviceManagerError::VfioUserMapRegion)?;
+
        let mut node = device_node!(vfio_user_name);

+        // Update the device tree with correct resource information.
+        node.resources = new_resources;
        node.pci_bdf = Some(pci_device_bdf);
        node.pci_device_handle = Some(PciDeviceHandle::VfioUser(vfio_user_pci_device));

@@ -3422,44 +3431,8 @@ impl DeviceManager {
        let mut node = device_node!(id);
        node.children = vec![virtio_device_id.clone()];

-        // Look for the id in the device tree. If it can be found, that means
-        // the device is being restored, otherwise it's created from scratch.
-        let (pci_segment_id, pci_device_bdf, config_bar_addr) = if let Some(node) =
-            self.device_tree.lock().unwrap().get(&id)
-        {
-            info!("Restoring virtio-pci {} resources", id);
-            let pci_device_bdf: PciBdf = node
-                .pci_bdf
-                .ok_or(DeviceManagerError::MissingDeviceNodePciBdf)?;
-            let pci_segment_id = pci_device_bdf.segment();
-
-            self.pci_segments[pci_segment_id as usize]
-                .pci_bus
-                .lock()
-                .unwrap()
-                .get_device_id(pci_device_bdf.device() as usize)
-                .map_err(DeviceManagerError::GetPciDeviceId)?;
-
-            if node.resources.is_empty() {
-                return Err(DeviceManagerError::MissingVirtioPciResources);
-            }
-
-            // We know the configuration BAR address is stored on the first
-            // resource in the list.
-            let config_bar_addr = match node.resources[0] {
-                Resource::MmioAddressRange { base, .. } => Some(base),
-                _ => {
-                    error!("Unexpected resource {:?} for {}", node.resources[0], id);
-                    return Err(DeviceManagerError::MissingVirtioPciResources);
-                }
-            };
-
-            (pci_segment_id, pci_device_bdf, config_bar_addr)
-        } else {
-            let pci_device_bdf = self.pci_segments[pci_segment_id as usize].next_device_bdf()?;
-
-            (pci_segment_id, pci_device_bdf, None)
-        };
+        let (pci_segment_id, pci_device_bdf, resources) =
+            self.pci_resources(&id, pci_segment_id)?;

        // Update the existing virtio node by setting the parent.
        if let Some(node) = self.device_tree.lock().unwrap().get_mut(&virtio_device_id) {
@@ -3529,38 +3502,34 @@ impl DeviceManager {
        }

        let device_type = virtio_device.lock().unwrap().device_type();
-        let mut virtio_pci_device = VirtioPciDevice::new(
-            id.clone(),
-            memory,
-            virtio_device,
-            msix_num,
-            access_platform,
-            &self.msi_interrupt_manager,
-            pci_device_bdf.into(),
-            self.activate_evt
-                .try_clone()
-                .map_err(DeviceManagerError::EventFd)?,
-            // All device types *except* virtio block devices should be allocated a 64-bit bar
-            // The block devices should be given a 32-bit BAR so that they are easily accessible
-            // to firmware without requiring excessive identity mapping.
-            // The exception being if not on the default PCI segment.
-            pci_segment_id > 0 || device_type != VirtioDeviceType::Block as u32,
-            dma_handler,
-        )
-        .map_err(DeviceManagerError::VirtioDevice)?;
+        let virtio_pci_device = Arc::new(Mutex::new(
+            VirtioPciDevice::new(
+                id.clone(),
+                memory,
+                virtio_device,
+                msix_num,
+                access_platform,
+                &self.msi_interrupt_manager,
+                pci_device_bdf.into(),
+                self.activate_evt
+                    .try_clone()
+                    .map_err(DeviceManagerError::EventFd)?,
+                // All device types *except* virtio block devices should be allocated a 64-bit bar
+                // The block devices should be given a 32-bit BAR so that they are easily accessible
+                // to firmware without requiring excessive identity mapping.
+                // The exception being if not on the default PCI segment.
+                pci_segment_id > 0 || device_type != VirtioDeviceType::Block as u32,
+                dma_handler,
+            )
+            .map_err(DeviceManagerError::VirtioDevice)?,
+        ));

-        // This is important as this will set the BAR address if it exists,
-        // which is mandatory on the restore path.
-        if let Some(addr) = config_bar_addr {
-            virtio_pci_device.set_config_bar_addr(addr);
-        }
-
-        let virtio_pci_device = Arc::new(Mutex::new(virtio_pci_device));
-        let bars = self.add_pci_device(
+        let new_resources = self.add_pci_device(
            virtio_pci_device.clone(),
            virtio_pci_device.clone(),
            pci_segment_id,
            pci_device_bdf,
+            resources,
        )?;

        let bar_addr = virtio_pci_device.lock().unwrap().config_bar_addr();
@@ -3573,12 +3542,7 @@ impl DeviceManager {
        }

        // Update the device tree with correct resource information.
-        for pci_bar in bars.iter() {
-            node.resources.push(Resource::MmioAddressRange {
-                base: pci_bar.0.raw_value(),
-                size: pci_bar.1 as u64,
-            });
-        }
+        node.resources = new_resources;
        node.migratable = Some(Arc::clone(&virtio_pci_device) as Arc<Mutex<dyn Migratable>>);
        node.pci_bdf = Some(pci_device_bdf);
        node.pci_device_handle = Some(PciDeviceHandle::Virtio(virtio_pci_device));
@@ -3587,6 +3551,38 @@ impl DeviceManager {
        Ok(pci_device_bdf)
    }

+    fn pci_resources(
+        &self,
+        id: &str,
+        pci_segment_id: u16,
+    ) -> DeviceManagerResult<(u16, PciBdf, Option<Vec<Resource>>)> {
+        // Look for the id in the device tree. If it can be found, that means
+        // the device is being restored, otherwise it's created from scratch.
+        Ok(
+            if let Some(node) = self.device_tree.lock().unwrap().get(id) {
+                info!("Restoring virtio-pci {} resources", id);
+                let pci_device_bdf: PciBdf = node
+                    .pci_bdf
+                    .ok_or(DeviceManagerError::MissingDeviceNodePciBdf)?;
+                let pci_segment_id = pci_device_bdf.segment();
+
+                self.pci_segments[pci_segment_id as usize]
+                    .pci_bus
+                    .lock()
+                    .unwrap()
+                    .get_device_id(pci_device_bdf.device() as usize)
+                    .map_err(DeviceManagerError::GetPciDeviceId)?;
+
+                (pci_segment_id, pci_device_bdf, Some(node.resources.clone()))
+            } else {
+                let pci_device_bdf =
+                    self.pci_segments[pci_segment_id as usize].next_device_bdf()?;
+
+                (pci_segment_id, pci_device_bdf, None)
+            },
+        )
+    }
+
    #[cfg(target_arch = "x86_64")]
    pub fn io_bus(&self) -> &Arc<Bus> {
        &self.address_manager.io_bus
@@ -3620,6 +3616,7 @@ impl DeviceManager {
        &self.console
    }

+    #[cfg(target_arch = "aarch64")]
    pub fn cmdline_additions(&self) -> &[String] {
        self.cmdline_additions.as_slice()
    }
@@ -3710,6 +3707,8 @@ impl DeviceManager {
        &mut self,
        device_cfg: &mut DeviceConfig,
    ) -> DeviceManagerResult<PciDeviceInfo> {
+        self.validate_identifier(&device_cfg.id)?;
+
        if device_cfg.iommu && !self.is_iommu_segment(device_cfg.pci_segment) {
            return Err(DeviceManagerError::InvalidIommuHotplug);
        }
@@ -3729,6 +3728,8 @@ impl DeviceManager {
        &mut self,
        device_cfg: &mut UserDeviceConfig,
    ) -> DeviceManagerResult<PciDeviceInfo> {
+        self.validate_identifier(&device_cfg.id)?;
+
        let (bdf, device_name) = self.add_vfio_user_device(device_cfg)?;

        // Update the PCIU bitmap
@@ -3820,6 +3821,22 @@ impl DeviceManager {
        let pci_device_node = device_tree
            .remove_node_by_pci_bdf(pci_device_bdf)
            .ok_or(DeviceManagerError::MissingPciDevice)?;
+
+        // For VFIO and vfio-user the PCI device id is the id.
+        // For virtio we overwrite it later as we want the id of the
+        // underlying device.
+        let mut id = pci_device_node.id;
+        let pci_device_handle = pci_device_node
+            .pci_device_handle
+            .ok_or(DeviceManagerError::MissingPciDevice)?;
+        if matches!(pci_device_handle, PciDeviceHandle::Virtio(_)) {
+            // The virtio-pci device has a single child
+            if !pci_device_node.children.is_empty() {
+                assert_eq!(pci_device_node.children.len(), 1);
+                let child_id = &pci_device_node.children[0];
+                id = child_id.clone();
+            }
+        }
        for child in pci_device_node.children.iter() {
            device_tree.remove(child);
        }
@@ -3831,9 +3848,6 @@ impl DeviceManager {
            }
        }

-        let pci_device_handle = pci_device_node
-            .pci_device_handle
-            .ok_or(DeviceManagerError::MissingPciDevice)?;
        let (pci_device, bus_device, virtio_device, remove_dma_handler) = match pci_device_handle {
            // No need to remove any virtio-mem mapping here as the container outlives all devices
            PciDeviceHandle::Vfio(vfio_pci_device) => (
@@ -3963,6 +3977,15 @@ impl DeviceManager {
                .retain(|handler| !Arc::ptr_eq(&handler.virtio_device, &virtio_device));
        }

+        event!(
+            "vm",
+            "device-removed",
+            "id",
+            &id,
+            "bdf",
+            pci_device_bdf.to_string()
+        );
+
        // At this point, the device has been removed from all the list and
        // buses where it was stored. At the end of this function, after
        // any_device, bus_device and pci_device are released, the actual
@@ -4016,6 +4039,8 @@ impl DeviceManager {
    }

    pub fn add_disk(&mut self, disk_cfg: &mut DiskConfig) -> DeviceManagerResult<PciDeviceInfo> {
+        self.validate_identifier(&disk_cfg.id)?;
+
        if disk_cfg.iommu && !self.is_iommu_segment(disk_cfg.pci_segment) {
            return Err(DeviceManagerError::InvalidIommuHotplug);
        }
@@ -4025,11 +4050,15 @@ impl DeviceManager {
    }

    pub fn add_fs(&mut self, fs_cfg: &mut FsConfig) -> DeviceManagerResult<PciDeviceInfo> {
+        self.validate_identifier(&fs_cfg.id)?;
+
        let device = self.make_virtio_fs_device(fs_cfg)?;
        self.hotplug_virtio_pci_device(device)
    }

    pub fn add_pmem(&mut self, pmem_cfg: &mut PmemConfig) -> DeviceManagerResult<PciDeviceInfo> {
+        self.validate_identifier(&pmem_cfg.id)?;
+
        if pmem_cfg.iommu && !self.is_iommu_segment(pmem_cfg.pci_segment) {
            return Err(DeviceManagerError::InvalidIommuHotplug);
        }
@@ -4039,6 +4068,8 @@ impl DeviceManager {
    }

    pub fn add_net(&mut self, net_cfg: &mut NetConfig) -> DeviceManagerResult<PciDeviceInfo> {
+        self.validate_identifier(&net_cfg.id)?;
+
        if net_cfg.iommu && !self.is_iommu_segment(net_cfg.pci_segment) {
            return Err(DeviceManagerError::InvalidIommuHotplug);
        }
@@ -4048,6 +4079,8 @@ impl DeviceManager {
    }

    pub fn add_vdpa(&mut self, vdpa_cfg: &mut VdpaConfig) -> DeviceManagerResult<PciDeviceInfo> {
+        self.validate_identifier(&vdpa_cfg.id)?;
+
        if vdpa_cfg.iommu && !self.is_iommu_segment(vdpa_cfg.pci_segment) {
            return Err(DeviceManagerError::InvalidIommuHotplug);
        }
@@ -4057,6 +4090,8 @@ impl DeviceManager {
    }

    pub fn add_vsock(&mut self, vsock_cfg: &mut VsockConfig) -> DeviceManagerResult<PciDeviceInfo> {
+        self.validate_identifier(&vsock_cfg.id)?;
+
        if vsock_cfg.iommu && !self.is_iommu_segment(vsock_cfg.pci_segment) {
            return Err(DeviceManagerError::InvalidIommuHotplug);
        }
@@ -4184,6 +4219,20 @@ impl DeviceManager {
    pub fn uefi_flash(&self) -> GuestMemoryAtomic<GuestMemoryMmap> {
        self.uefi_flash.as_ref().unwrap().clone()
    }
+
+    fn validate_identifier(&self, id: &Option<String>) -> DeviceManagerResult<()> {
+        if let Some(id) = id {
+            if id.starts_with("__") {
+                return Err(DeviceManagerError::InvalidIdentifier(id.clone()));
+            }
+
+            if self.device_tree.lock().unwrap().contains_key(id) {
+                return Err(DeviceManagerError::IdentifierNotUnique(id.clone()));
+            }
+        }
+
+        Ok(())
+    }
 }

 fn numa_node_id_from_memory_zone_id(numa_nodes: &NumaNodes, memory_zone_id: &str) -> Option<u32> {
--- a/vmm/src/device_tree.rs
+++ b/vmm/src/device_tree.rs
@@ -4,6 +4,7 @@

 use crate::device_manager::PciDeviceHandle;
 use pci::PciBdf;
+use serde::{Deserialize, Serialize};
 use std::collections::HashMap;
 use std::sync::{Arc, Mutex};
 use vm_device::Resource;
--- a/Show More
+++ b/Show More