build: Release v28.4 (bug fix release)

Signed-off-by: Bo Chen <chen.bo@intel.com>
build: Bump MSRV to 1.65
2026-08-05 02:19:16 +00:00 · 2024-01-19 16:04:30 -08:00 · 2024-01-19 09:09:22 -08:00 · 2024-01-19 09:09:22 -08:00 · 2024-01-19 09:09:22 -08:00 · 2024-01-19 09:09:22 -08:00
171 changed files with 3684 additions and 4527 deletions
--- a/.github/workflows/build.yaml
+++ b/.github/workflows/build.yaml
@@ -13,7 +13,7 @@ jobs:
          - stable
          - beta
          - nightly
-          - "1.60"
+          - "1.65"
        target:
          - x86_64-unknown-linux-gnu
          - x86_64-unknown-linux-musl
@@ -34,22 +34,22 @@ jobs:
          override: true

      - name: Build (default features)
-        run: cargo rustc --locked --bin cloud-hypervisor -- -D warnings -D clippy::undocumented_unsafe_blocks
+        run: cargo rustc --locked --bin cloud-hypervisor -- -D warnings

      - name: Build (kvm)
-        run: cargo rustc --locked --bin cloud-hypervisor --no-default-features --features "kvm"  -- -D warnings -D clippy::undocumented_unsafe_blocks
+        run: cargo rustc --locked --bin cloud-hypervisor --no-default-features --features "kvm"  -- -D warnings

      - name: Build (default features + tdx)
-        run: cargo rustc --locked --bin cloud-hypervisor --features "tdx" -- -D warnings -D clippy::undocumented_unsafe_blocks
+        run: cargo rustc --locked --bin cloud-hypervisor --features "tdx" -- -D warnings

      - name: Build (default features + guest_debug)
-        run: cargo rustc --locked --bin cloud-hypervisor --features "guest_debug" -- -D warnings -D clippy::undocumented_unsafe_blocks
+        run: cargo rustc --locked --bin cloud-hypervisor --features "guest_debug" -- -D warnings

      - name: Build (mshv)
-        run: cargo rustc --locked --bin cloud-hypervisor --no-default-features --features "mshv"  -- -D warnings -D clippy::undocumented_unsafe_blocks
+        run: cargo rustc --locked --bin cloud-hypervisor --no-default-features --features "mshv"  -- -D warnings

      - name: Build (mshv + kvm)
-        run: cargo rustc --locked --bin cloud-hypervisor --no-default-features --features "mshv,kvm"  -- -D warnings -D clippy::undocumented_unsafe_blocks
+        run: cargo rustc --locked --bin cloud-hypervisor --no-default-features --features "mshv,kvm"  -- -D warnings

      - name: Release Build (default features)
        run: cargo build --locked --all --release --target=${{ matrix.target }}
--- a/.github/workflows/quality.yaml
+++ b/.github/workflows/quality.yaml
@@ -13,24 +13,15 @@ jobs:
        rust:
          - stable
        target:
-          - aarch64-unknown-linux-gnu
-          - aarch64-unknown-linux-musl
          - x86_64-unknown-linux-gnu
-          - x86_64-unknown-linux-musl
-
+          - aarch64-unknown-linux-gnu
        experimental: [false]
        include:
-          - rust: beta
-            target: aarch64-unknown-linux-gnu
-            experimental: true
-          - rust: beta
-            target: aarch64-unknown-linux-musl
-            experimental: true
          - rust: beta
            target: x86_64-unknown-linux-gnu
            experimental: true
          - rust: beta
-            target: x86_64-unknown-linux-musl
+            target: aarch64-unknown-linux-gnu
            experimental: true
    steps:
      - name: Code checkout
@@ -62,28 +53,28 @@ jobs:
        with:
          use-cross: ${{ matrix.target != 'x86_64-unknown-linux-gnu' }}
          command: clippy
-          args: --target=${{ matrix.target }} --locked --all --all-targets --no-default-features --tests --features "kvm" -- -D warnings -D clippy::undocumented_unsafe_blocks
+          args: --locked --all --all-targets --no-default-features --tests --features "kvm" -- -D warnings

      - name: Clippy (default features)
        uses: actions-rs/cargo@v1
        with:
          use-cross: ${{ matrix.target != 'x86_64-unknown-linux-gnu' }}
          command: clippy
-          args: --target=${{ matrix.target }} --locked --all --all-targets --tests -- -D warnings -D clippy::undocumented_unsafe_blocks
+          args: --locked --all --all-targets --tests -- -D warnings

      - name: Clippy (default features + guest_debug)
        uses: actions-rs/cargo@v1
        with:
          use-cross: ${{ matrix.target != 'x86_64-unknown-linux-gnu' }}
          command: clippy
-          args: --target=${{ matrix.target }} --locked --all --all-targets --tests --features "guest_debug" -- -D warnings -D clippy::undocumented_unsafe_blocks
+          args: --locked --all --all-targets --tests --features "guest_debug" -- -D warnings

      - name: Clippy (default features + tracing)
        uses: actions-rs/cargo@v1
        with:
          use-cross: ${{ matrix.target != 'x86_64-unknown-linux-gnu' }}
          command: clippy
-          args: --target=${{ matrix.target }} --locked --all --all-targets --tests --features "tracing" -- -D warnings -D clippy::undocumented_unsafe_blocks
+          args: --locked --all --all-targets --tests --features "tracing" -- -D warnings

      - name: Clippy (mshv)
        if: ${{ matrix.target == 'x86_64-unknown-linux-gnu' }}
@@ -91,7 +82,7 @@ jobs:
        with:
          use-cross: ${{ matrix.target != 'x86_64-unknown-linux-gnu' }}
          command: clippy
-          args: --target=${{ matrix.target }} --locked --all --all-targets --no-default-features --tests --features "mshv" -- -D warnings -D clippy::undocumented_unsafe_blocks
+          args: --locked --all --all-targets --no-default-features --tests --features "mshv" -- -D warnings

      - name: Clippy (mshv + kvm)
        if: ${{ matrix.target == 'x86_64-unknown-linux-gnu' }}
@@ -99,7 +90,7 @@ jobs:
        with:
          use-cross: ${{ matrix.target != 'x86_64-unknown-linux-gnu' }}
          command: clippy
-          args: --target=${{ matrix.target }} --locked --all --all-targets --no-default-features --tests --features "mshv,kvm" -- -D warnings -D clippy::undocumented_unsafe_blocks
+          args: --locked --all --all-targets --no-default-features --tests --features "mshv,kvm" -- -D warnings

      - name: Clippy (kvm + tdx)
        if: ${{ matrix.target == 'x86_64-unknown-linux-gnu' }}
@@ -107,7 +98,7 @@ jobs:
        with:
          use-cross: ${{ matrix.target != 'x86_64-unknown-linux-gnu' }}
          command: clippy
-          args: --target=${{ matrix.target }} --locked --all --all-targets --no-default-features --tests --features "tdx,kvm" -- -D warnings -D clippy::undocumented_unsafe_blocks
+          args: --locked --all --all-targets --no-default-features --tests --features "tdx,kvm" -- -D warnings

      - name: Check build did not modify any files
        run: test -z "$(git status --porcelain)"
--- a/.github/workflows/release.yaml
+++ b/.github/workflows/release.yaml
@@ -3,6 +3,7 @@ on: [pull_request, create]

 jobs:
  release:
+    if: (github.event_name == 'create' && github.event.ref_type == 'tag') || github.event_name == 'pull_request'
    name: Release
    runs-on: ubuntu-latest
    steps:
@@ -15,29 +16,29 @@ jobs:
      - name: Install Rust toolchain (x86_64-unknown-linux-gnu)
        uses: actions-rs/toolchain@v1
        with:
-          toolchain: "1.62"
+          toolchain: "1.65"
          target: x86_64-unknown-linux-gnu
      - name: Install Rust toolchain (x86_64-unknown-linux-musl)
        uses: actions-rs/toolchain@v1
        with:
-          toolchain: "1.62"
+          toolchain: "1.65"
          target: x86_64-unknown-linux-musl
      - name: Build
        uses: actions-rs/cargo@v1
        with:
-          toolchain: "1.62"
-          command: build
-          args: --all --release --no-default-features --features "kvm,mshv" --target=x86_64-unknown-linux-gnu
+            toolchain: "1.65"
+            command: build
+            args: --all --release --target=x86_64-unknown-linux-gnu
      - name: Static Build
        uses: actions-rs/cargo@v1
        with:
-          toolchain: "1.62"
-          command: build
-          args: --all --release --no-default-features --features "kvm,mshv" --target=x86_64-unknown-linux-musl
+            toolchain: "1.65"
+            command: build
+            args: --all --release --target=x86_64-unknown-linux-musl
      - name: Install Rust toolchain (aarch64-unknown-linux-musl)
        uses: actions-rs/toolchain@v1
        with:
-          toolchain: "1.62"
+          toolchain: "1.65"
          target: aarch64-unknown-linux-musl
          override: true
      - name: Create Release
--- a/.gitignore
+++ b/.gitignore
@@ -5,4 +5,3 @@
 **/Cargo.lock
 **/rusty-tags.vi
 /rpm/SOURCES
-/.vscode
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -11,18 +11,18 @@ dependencies = [

 [[package]]
 name = "aho-corasick"
-version = "0.7.20"
+version = "1.1.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cc936419f96fa211c1b9166887b38e5e40b19958e5b895be7c1f93adec7071ac"
+checksum = "b2969dcb958b36655471fc61f7e416fa76033bdd4bfed0678d8fee1e2d07a1f0"
 dependencies = [
 "memchr",
 ]

 [[package]]
 name = "anyhow"
-version = "1.0.68"
+version = "1.0.66"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2cb2f989d18dd141ab8ae82f64d1a8cdd37e0840f73a406896cf5e99502fab61"
+checksum = "216261ddc8289130e551ddcd5ce8a064710c0d064a4d2895c67151c92b5443f6"

 [[package]]
 name = "api_client"
@@ -80,6 +80,12 @@ version = "1.3.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"

+[[package]]
+name = "bitflags"
+version = "2.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "327762f6e5a765692301e5bb513e0d9fef63be86bbc14528052b1cd3e6f03e07"
+
 [[package]]
 name = "block_util"
 version = "0.1.0"
@@ -88,7 +94,6 @@ dependencies = [
 "libc",
 "log",
 "qcow",
- "smallvec",
 "thiserror",
 "versionize",
 "versionize_derive",
@@ -108,9 +113,9 @@ checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610"

 [[package]]
 name = "cc"
-version = "1.0.78"
+version = "1.0.77"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a20104e2335ce8a659d6dd92a51a767a0c062599c73b343fd152cb401e828c3d"
+checksum = "e9f73505338f7d905b19d18738976aae232eb46b8efc15554ffc56deb5d9ebe4"

 [[package]]
 name = "cfg-if"
@@ -120,11 +125,11 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"

 [[package]]
 name = "clap"
-version = "4.0.32"
+version = "4.0.29"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a7db700bc935f9e43e88d00b0850dae18a63773cfbec6d8e070fccf7fef89a39"
+checksum = "4d63b9e9c07271b9957ad22c173bae2a4d9a81127680962039296abcd2f8251d"
 dependencies = [
- "bitflags",
+ "bitflags 1.3.2",
 "clap_lex",
 "is-terminal",
 "once_cell",
@@ -144,7 +149,7 @@ dependencies = [

 [[package]]
 name = "cloud-hypervisor"
-version = "29.0.0"
+version = "28.4.0"
 dependencies = [
 "anyhow",
 "api_client",
@@ -207,7 +212,7 @@ dependencies = [
 "proc-macro2",
 "quote",
 "strsim",
- "syn",
+ "syn 1.0.105",
 ]

 [[package]]
@@ -218,7 +223,7 @@ checksum = "7618812407e9402654622dd402b0a89dff9ba93badd6540781526117b92aab7e"
 dependencies = [
 "darling_core",
 "quote",
- "syn",
+ "syn 1.0.105",
 ]

 [[package]]
@@ -228,7 +233,7 @@ dependencies = [
 "acpi_tables",
 "anyhow",
 "arch",
- "bitflags",
+ "bitflags 2.4.1",
 "byteorder",
 "hypervisor",
 "libc",
@@ -283,7 +288,7 @@ version = "4.3.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "20df693c700404f7e19d4d6fae6b15215d2913c27955d2b9d6f2c0f537511cd0"
 dependencies = [
- "bitflags",
+ "bitflags 1.3.2",
 "libc",
 ]

@@ -313,6 +318,7 @@ name = "event_monitor"
 version = "0.1.0"
 dependencies = [
 "libc",
+ "once_cell",
 "serde",
 "serde_json",
 ]
@@ -335,7 +341,7 @@ version = "0.6.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "32c95766e0414f8bfc1d07055574c621b67739466d6ba516c4fef8e99d30d2e6"
 dependencies = [
- "bitflags",
+ "bitflags 1.3.2",
 "cfg-if",
 "log",
 "managed",
@@ -366,9 +372,9 @@ dependencies = [

 [[package]]
 name = "glob"
-version = "0.3.0"
+version = "0.3.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574"
+checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b"

 [[package]]
 name = "hermit-abi"
@@ -409,11 +415,12 @@ dependencies = [

 [[package]]
 name = "iced-x86"
-version = "1.18.0"
+version = "1.17.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1dd04b950d75b3498320253b17fb92745b2cc79ead8814aede2f7c1bab858bec"
+checksum = "158f5204401d08f91d19176112146d75e99b3cf745092e268fa7be33e09adcec"
 dependencies = [
 "lazy_static",
+ "static_assertions",
 ]

 [[package]]
@@ -443,28 +450,28 @@ dependencies = [

 [[package]]
 name = "io-uring"
-version = "0.5.11"
+version = "0.5.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c9ddf18110cedc09617f5a965bdb51adb0c0fb02c791e4ab22f7a21bb25269b0"
+checksum = "7ba34abb5175052fc1a2227a10d2275b7386c9990167de9786c0b88d8b062330"
 dependencies = [
- "bitflags",
+ "bitflags 1.3.2",
 "libc",
 ]

 [[package]]
 name = "ipnetwork"
-version = "0.19.0"
+version = "0.20.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1f84f1612606f3753f205a4e9a2efd6fe5b4c573a6269b2cc6c3003d44a0d127"
+checksum = "bf466541e9d546596ee94f9f69590f89473455f88372423e0008fc1a7daf100e"
 dependencies = [
 "serde",
 ]

 [[package]]
 name = "is-terminal"
-version = "0.4.2"
+version = "0.4.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "28dfb6c8100ccc63462345b67d1bbc3679177c75ee4bf59bf29c8b1d110b8189"
+checksum = "927609f78c2913a6f6ac3c27a4fe87f43e2a35367c0c4b0f8265e8f49a104330"
 dependencies = [
 "hermit-abi",
 "io-lifetimes",
@@ -474,9 +481,9 @@ dependencies = [

 [[package]]
 name = "itoa"
-version = "1.0.5"
+version = "1.0.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fad582f4b9e86b6caa621cabeb0963332d92eea04729ab12892c2533951e6440"
+checksum = "4217ad341ebadf8d8e724e264f13e593e0648f5b3e94b3896a5df283be015ecc"

 [[package]]
 name = "kvm-bindings"
@@ -490,9 +497,8 @@ dependencies = [

 [[package]]
 name = "kvm-ioctls"
-version = "0.12.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c3a321cabd827642499c77e27314f388dd83a717a5ca716b86476fb947f73ae4"
+version = "0.13.0"
+source = "git+https://github.com/rust-vmm/kvm-ioctls?branch=main#23a3bb045a467e60bb00328a0b13cea13b5815d0"
 dependencies = [
 "kvm-bindings",
 "libc",
@@ -507,9 +513,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"

 [[package]]
 name = "libc"
-version = "0.2.139"
+version = "0.2.152"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "201de327520df007757c1f0adce6e827fe8562fbc28bfd9c15571c66ca1f5f79"
+checksum = "13e3bf6590cbc649f4d1a3eefc9d5d6eb746f5200ffb04e5e142700b8faa56e7"

 [[package]]
 name = "libssh2-sys"
@@ -548,9 +554,9 @@ dependencies = [

 [[package]]
 name = "linux-raw-sys"
-version = "0.1.4"
+version = "0.1.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f051f77a7c8e6957c0696eac88f26b0117e54f52d3fc682ab19397a8812846a4"
+checksum = "8f9f08d8963a6c613f4b1a78f4f4a4dbfadf8e6545b2d72861731e4858b8b47f"

 [[package]]
 name = "lock_api"
@@ -579,9 +585,9 @@ checksum = "0ca88d725a0a943b096803bd34e73a4437208b6077654cc4ecb2947a5f91618d"

 [[package]]
 name = "memchr"
-version = "2.5.0"
+version = "2.7.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d"
+checksum = "523dc4f511e55ab87b694dc30d0f820d60906ef06413f93d4d7a1385599cc149"

 [[package]]
 name = "micro_http"
@@ -595,7 +601,7 @@ dependencies = [
 [[package]]
 name = "mshv-bindings"
 version = "0.1.1"
-source = "git+https://github.com/rust-vmm/mshv?branch=main#fb19bb30110c3a3268ad815897bba407fce1843c"
+source = "git+https://github.com/rust-vmm/mshv?branch=main#10d0c5208cc41a3958e0fd31fabf2c88b18a97b5"
 dependencies = [
 "libc",
 "serde",
@@ -607,7 +613,7 @@ dependencies = [
 [[package]]
 name = "mshv-ioctls"
 version = "0.1.1"
-source = "git+https://github.com/rust-vmm/mshv?branch=main#fb19bb30110c3a3268ad815897bba407fce1843c"
+source = "git+https://github.com/rust-vmm/mshv?branch=main#10d0c5208cc41a3958e0fd31fabf2c88b18a97b5"
 dependencies = [
 "libc",
 "mshv-bindings",
@@ -678,9 +684,9 @@ dependencies = [

 [[package]]
 name = "openssl-sys"
-version = "0.9.80"
+version = "0.9.79"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "23bbbf7854cd45b83958ebe919f0e8e516793727652e27fda10a8384cfc790b7"
+checksum = "5454462c0eced1e97f2ec09036abc8da362e66802f66fd20f86854d9d8cbcbc4"
 dependencies = [
 "autocfg",
 "cc",
@@ -713,9 +719,9 @@ dependencies = [

 [[package]]
 name = "parking_lot_core"
-version = "0.8.5"
+version = "0.8.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d76e8e1493bcac0d2766c42737f34458f1c8c50c0d23bcb24ea953affb273216"
+checksum = "60a2cfe6f0ad2bfc16aefa463b497d5c7a5ecd44a23efa72aa342d90177356dc"
 dependencies = [
 "cfg-if",
 "instant",
@@ -727,9 +733,9 @@ dependencies = [

 [[package]]
 name = "paste"
-version = "1.0.11"
+version = "1.0.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d01a5bd0424d00070b0098dd17ebca6f961a959dead1dbcbbbc1d1cd8d3deeba"
+checksum = "cf1c2c742266c2f1041c914ba65355a83ae8747b05f208319784083583494b4b"

 [[package]]
 name = "pci"
@@ -797,7 +803,7 @@ dependencies = [
 "phf_shared",
 "proc-macro2",
 "quote",
- "syn",
+ "syn 1.0.105",
 ]

 [[package]]
@@ -817,30 +823,32 @@ checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160"

 [[package]]
 name = "pnet"
-version = "0.31.0"
+version = "0.34.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0caaf5b11fd907ff15cf14a4477bfabca4b37ab9e447a4f8dead969a59cdafad"
+checksum = "130c5b738eeda2dc5796fe2671e49027e6935e817ab51b930a36ec9e6a206a64"
 dependencies = [
+ "ipnetwork",
 "pnet_base",
 "pnet_datalink",
 "pnet_packet",
+ "pnet_sys",
 "pnet_transport",
 ]

 [[package]]
 name = "pnet_base"
-version = "0.31.0"
+version = "0.34.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f9d3a993d49e5fd5d4d854d6999d4addca1f72d86c65adf224a36757161c02b6"
+checksum = "fe4cf6fb3ab38b68d01ab2aea03ed3d1132b4868fa4e06285f29f16da01c5f4c"
 dependencies = [
 "no-std-net",
 ]

 [[package]]
 name = "pnet_datalink"
-version = "0.31.0"
+version = "0.34.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e466faf03a98ad27f6e15cd27a2b7cc89e73e640a43527742977bc503c37f8aa"
+checksum = "ad5854abf0067ebbd3967f7d45ebc8976ff577ff0c7bd101c4973ae3c70f98fe"
 dependencies = [
 "ipnetwork",
 "libc",
@@ -851,30 +859,30 @@ dependencies = [

 [[package]]
 name = "pnet_macros"
-version = "0.31.0"
+version = "0.34.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "48dd52a5211fac27e7acb14cfc9f30ae16ae0e956b7b779c8214c74559cef4c3"
+checksum = "688b17499eee04a0408aca0aa5cba5fc86401d7216de8a63fdf7a4c227871804"
 dependencies = [
 "proc-macro2",
 "quote",
 "regex",
- "syn",
+ "syn 2.0.48",
 ]

 [[package]]
 name = "pnet_macros_support"
-version = "0.31.0"
+version = "0.34.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "89de095dc7739349559913aed1ef6a11e73ceade4897dadc77c5e09de6740750"
+checksum = "eea925b72f4bd37f8eab0f221bbe4c78b63498350c983ffa9dd4bcde7e030f56"
 dependencies = [
 "pnet_base",
 ]

 [[package]]
 name = "pnet_packet"
-version = "0.31.0"
+version = "0.34.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bc3b5111e697c39c8b9795b9fdccbc301ab696699e88b9ea5a4e4628978f495f"
+checksum = "a9a005825396b7fe7a38a8e288dbc342d5034dac80c15212436424fef8ea90ba"
 dependencies = [
 "glob",
 "pnet_base",
@@ -884,9 +892,9 @@ dependencies = [

 [[package]]
 name = "pnet_sys"
-version = "0.31.0"
+version = "0.34.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "328e231f0add6d247d82421bf3790b4b33b39c8930637f428eef24c4c6a90805"
+checksum = "417c0becd1b573f6d544f73671070b039051e5ad819cc64aa96377b536128d00"
 dependencies = [
 "libc",
 "winapi",
@@ -894,9 +902,9 @@ dependencies = [

 [[package]]
 name = "pnet_transport"
-version = "0.31.0"
+version = "0.34.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ff597185e6f1f5671b3122e4dba892a1c73e17c17e723d7669bd9299cbe7f124"
+checksum = "2637e14d7de974ee2f74393afccbc8704f3e54e6eb31488715e72481d1662cc3"
 dependencies = [
 "libc",
 "pnet_base",
@@ -906,9 +914,9 @@ dependencies = [

 [[package]]
 name = "proc-macro2"
-version = "1.0.47"
+version = "1.0.76"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5ea3d908b0e36316caf9e9e2c4625cdde190a7e6f440d794667ed17a1855e725"
+checksum = "95fc56cda0b5c3325f5fbbd7ff9fda9e02bb00bb3dac51252d2f1bfa1cb8cc8c"
 dependencies = [
 "unicode-ident",
 ]
@@ -926,9 +934,9 @@ dependencies = [

 [[package]]
 name = "quote"
-version = "1.0.23"
+version = "1.0.35"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8856d8364d252a14d474036ea1358d63c9e6965c8e5c1885c18f73d70bff9c7b"
+checksum = "291ec9ab5efd934aaf503a6466c5d5251535d108ee747472c3977cc5acc868ef"
 dependencies = [
 "proc-macro2",
 ]
@@ -963,7 +971,7 @@ version = "0.2.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a"
 dependencies = [
- "bitflags",
+ "bitflags 1.3.2",
 ]

 [[package]]
@@ -979,9 +987,21 @@ dependencies = [

 [[package]]
 name = "regex"
-version = "1.7.0"
+version = "1.10.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e076559ef8e241f2ae3479e36f97bd5741c0330689e217ad51ce2c76808b868a"
+checksum = "380b951a9c5e80ddfd6136919eef32310721aa4aacd4889a8d39124b026ab343"
+dependencies = [
+ "aho-corasick",
+ "memchr",
+ "regex-automata",
+ "regex-syntax",
+]
+
+[[package]]
+name = "regex-automata"
+version = "0.4.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5f804c7828047e88b2d32e2d7fe5a105da8ee3264f01902f796c8e067dc2483f"
 dependencies = [
 "aho-corasick",
 "memchr",
@@ -990,19 +1010,19 @@ dependencies = [

 [[package]]
 name = "regex-syntax"
-version = "0.6.28"
+version = "0.8.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "456c603be3e8d448b072f410900c09faf164fbce2d480456f50eea6e25f9c848"
+checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f"

 [[package]]
 name = "remain"
-version = "0.2.6"
+version = "0.2.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5704e2cda92fd54202f05430725317ba0ea7d0c96b246ca0a92e45177127ba3b"
+checksum = "1a81bc6a3aaeb180f767bd2bda8c03bac5b93b3b69a783c66f35d16a4df276cf"
 dependencies = [
 "proc-macro2",
 "quote",
- "syn",
+ "syn 1.0.105",
 ]

 [[package]]
@@ -1020,7 +1040,7 @@ version = "0.36.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a3807b5d10909833d3e9acd1eb5fb988f79376ff10fce42937de71a449c4c588"
 dependencies = [
- "bitflags",
+ "bitflags 1.3.2",
 "errno",
 "io-lifetimes",
 "libc",
@@ -1051,28 +1071,28 @@ dependencies = [

 [[package]]
 name = "semver"
-version = "1.0.16"
+version = "1.0.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "58bc9567378fc7690d6b2addae4e60ac2eeea07becb2c64b9f218b53865cba2a"
+checksum = "e25dfac463d778e353db5be2449d1cce89bd6fd23c9f1ea21310ce6e5a1b29c4"

 [[package]]
 name = "serde"
-version = "1.0.151"
+version = "1.0.150"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "97fed41fc1a24994d044e6db6935e69511a1153b52c15eb42493b26fa87feba0"
+checksum = "e326c9ec8042f1b5da33252c8a37e9ffbd2c9bef0155215b6e6c80c790e05f91"
 dependencies = [
 "serde_derive",
 ]

 [[package]]
 name = "serde_derive"
-version = "1.0.151"
+version = "1.0.150"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "255abe9a125a985c05190d687b320c12f9b1f0b99445e608c21ba0782c719ad8"
+checksum = "42a3df25b0713732468deadad63ab9da1f1fd75a48a15024b50363f128db627e"
 dependencies = [
 "proc-macro2",
 "quote",
- "syn",
+ "syn 1.0.105",
 ]

 [[package]]
@@ -1105,7 +1125,7 @@ dependencies = [
 "darling",
 "proc-macro2",
 "quote",
- "syn",
+ "syn 1.0.105",
 ]

 [[package]]
@@ -1149,12 +1169,18 @@ version = "0.9.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "269343e64430067a14937ae0e3c4ec604c178fb896dde0964b1acd22b3e2eeb1"
 dependencies = [
- "bitflags",
+ "bitflags 1.3.2",
 "libc",
 "libssh2-sys",
 "parking_lot",
 ]

+[[package]]
+name = "static_assertions"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f"
+
 [[package]]
 name = "strsim"
 version = "0.10.0"
@@ -1172,6 +1198,17 @@ dependencies = [
 "unicode-ident",
 ]

+[[package]]
+name = "syn"
+version = "2.0.48"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0f3531638e407dfc0814761abb7c00a5b54992b849452a0646b7f65c9f770f3f"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "unicode-ident",
+]
+
 [[package]]
 name = "termcolor"
 version = "1.1.3"
@@ -1208,22 +1245,22 @@ dependencies = [

 [[package]]
 name = "thiserror"
-version = "1.0.38"
+version = "1.0.37"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6a9cd18aa97d5c45c6603caea1da6628790b37f7a34b6ca89522331c5180fed0"
+checksum = "10deb33631e3c9018b9baf9dcbbc4f737320d2b576bac10f6aefa048fa407e3e"
 dependencies = [
 "thiserror-impl",
 ]

 [[package]]
 name = "thiserror-impl"
-version = "1.0.38"
+version = "1.0.37"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1fb327af4685e4d03fa8cbcf1716380da910eeb2bb8be417e7f9fd3fb164f36f"
+checksum = "982d17546b47146b28f7c22e3d08465f6b8903d0ea13c1660d9d84a6e7adcdbb"
 dependencies = [
 "proc-macro2",
 "quote",
- "syn",
+ "syn 1.0.105",
 ]

 [[package]]
@@ -1251,9 +1288,9 @@ dependencies = [

 [[package]]
 name = "unicode-ident"
-version = "1.0.6"
+version = "1.0.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "84a22b9f218b40614adcb3f4ff08b703773ad44fa9423e4e0d346d5db86e4ebc"
+checksum = "6ceab39d59e4c9499d4e5a8ee0e2735b891bb7308ac83dfb4e80cad195c9f6f3"

 [[package]]
 name = "uuid"
@@ -1282,7 +1319,7 @@ dependencies = [
 "quote",
 "serde",
 "serde_derive",
- "syn",
+ "syn 1.0.105",
 "versionize_derive",
 "vmm-sys-util",
 ]
@@ -1294,13 +1331,13 @@ source = "git+https://github.com/cloud-hypervisor/versionize_derive?branch=ch#ae
 dependencies = [
 "proc-macro2",
 "quote",
- "syn",
+ "syn 1.0.105",
 ]

 [[package]]
 name = "vfio-bindings"
 version = "0.4.0"
-source = "git+https://github.com/rust-vmm/vfio?branch=main#7ef33f6759ae6d37a54cd66f749aa47c1c81509c"
+source = "git+https://github.com/rust-vmm/vfio?branch=main#43439e056ddfa84a4f7906ee7f2f58be70505c08"
 dependencies = [
 "vmm-sys-util",
 ]
@@ -1308,7 +1345,7 @@ dependencies = [
 [[package]]
 name = "vfio-ioctls"
 version = "0.2.0"
-source = "git+https://github.com/rust-vmm/vfio?branch=main#7ef33f6759ae6d37a54cd66f749aa47c1c81509c"
+source = "git+https://github.com/rust-vmm/vfio?branch=main#43439e056ddfa84a4f7906ee7f2f58be70505c08"
 dependencies = [
 "byteorder",
 "kvm-bindings",
@@ -1357,7 +1394,7 @@ version = "0.6.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c9b791c5b0717a0558888a4cf7240cea836f39a99cb342e12ce633dcaa078072"
 dependencies = [
- "bitflags",
+ "bitflags 1.3.2",
 "libc",
 "vm-memory",
 "vmm-sys-util",
@@ -1537,7 +1574,7 @@ dependencies = [
 "anyhow",
 "arc-swap",
 "arch",
- "bitflags",
+ "bitflags 2.4.1",
 "block_util",
 "clap",
 "devices",
@@ -1584,7 +1621,7 @@ version = "0.11.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "cc06a16ee8ebf0d9269aed304030b0d20a866b8b3dd3d4ce532596ac567a0d24"
 dependencies = [
- "bitflags",
+ "bitflags 1.3.2",
 "libc",
 "serde",
 "serde_derive",
@@ -1711,5 +1748,5 @@ checksum = "6505e6815af7de1746a08f69c69606bb45695a17149517680f3b2149713b19a3"
 dependencies = [
 "proc-macro2",
 "quote",
- "syn",
+ "syn 1.0.105",
 ]
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "cloud-hypervisor"
-version = "29.0.0"
+version = "28.4.0"
 authors = ["The Cloud Hypervisor Authors"]
 edition = "2021"
 default-run = "cloud-hypervisor"
@@ -15,7 +15,7 @@ homepage = "https://github.com/cloud-hypervisor/cloud-hypervisor"
 # a.) A dependency requires it, 
 # b.) If we want to use a new feature and that MSRV is at least 6 months old,
 # c.) There is a security issue that is addressed by the toolchain update.
-rust-version = "1.60"
+rust-version = "1.65"

 [profile.release]
 lto = true
@@ -24,28 +24,32 @@ opt-level = "s"
 strip = true

 [dependencies]
-anyhow = "1.0.68"
+anyhow = "1.0.66"
 api_client = { path = "api_client" }
-clap = { version = "4.0.32", features = ["wrap_help","cargo","string"] }
+clap = { version = "4.0.29", features = ["wrap_help","cargo","string"] }
 epoll = "4.3.1"
 event_monitor = { path = "event_monitor" }
 hypervisor = { path = "hypervisor" }
-libc = "0.2.139"
+libc = "0.2.138"
 log = { version = "0.4.17", features = ["std"] }
 option_parser = { path = "option_parser" }
 seccompiler = "0.3.0"
 serde_json = "1.0.89"
 signal-hook = "0.3.14"
-thiserror = "1.0.38"
+thiserror = "1.0.37"
 tpm = { path = "tpm"}
 tracer = { path = "tracer" }
 vmm = { path = "vmm" }
 vmm-sys-util = "0.11.0"
 vm-memory = "0.10.0"

+[build-dependencies]
+clap = { version = "4.0.29", features = ["cargo"] }
+
 # List of patched crates
 [patch.crates-io]
 kvm-bindings = { git = "https://github.com/cloud-hypervisor/kvm-bindings", branch = "ch-v0.6.0-tdx" }
+kvm-ioctls = { git = "https://github.com/rust-vmm/kvm-ioctls", branch = "main" }
 versionize_derive = { git = "https://github.com/cloud-hypervisor/versionize_derive", branch = "ch" }

 [dev-dependencies]
--- a/153
+++ b/153
@@ -10,16 +10,29 @@ pipeline {
                        checkout scm
                    }
                }
-                stage('Check if worker build can be skipped') {
+                stage('Check for documentation only changes') {
                    when {
                        expression {
-                            return skipWorkerBuild()
+                            return docsFileOnly()
                        }
                    }
                    steps {
                        script {
                            runWorkers = false
-                            echo 'No changes requring a build'
+                            echo 'Documentation only changes, no need to run the CI'
+                        }
+                    }
+                }
+                stage('Check for fuzzer files only changes') {
+                    when {
+                        expression {
+                            return fuzzFileOnly()
+                        }
+                    }
+                    steps {
+                        script {
+                            runWorkers = false
+                            echo 'Fuzzer cargo files only changes, no need to run the CI'
                        }
                    }
                }
@@ -268,113 +281,6 @@ pipeline {
                        }
                    }
                }
-                stage('Worker build - Rate Limiter') {
-                    agent { node { label 'focal-metrics' } }
-                    when {
-                        branch 'main'
-                        beforeAgent true
-                        expression {
-                            return runWorkers
-                        }
-                    }
-                    stages {
-                        stage('Checkout') {
-                            steps {
-                                checkout scm
-                            }
-                        }
-                        stage('Run rate-limiter integration tests') {
-                            options {
-                                timeout(time: 10, unit: 'MINUTES')
-                            }
-                            steps {
-                                sh 'scripts/dev_cli.sh tests --integration-rate-limiter'
-                            }
-                        }
-                    }
-                }
-                stage('Worker build - SGX') {
-                    agent { node { label 'jammy-sgx' } }
-                    when {
-                        beforeAgent true
-                        allOf {
-                            branch 'main'
-                            expression {
-                                return runWorkers
-                            }
-                        }
-                    }
-                    stages {
-                        stage('Checkout') {
-                            steps {
-                                checkout scm
-                            }
-                        }
-                        stage('Run SGX integration tests') {
-                            options {
-                                timeout(time: 1, unit: 'HOURS')
-                            }
-                            steps {
-                                sh 'scripts/dev_cli.sh tests --integration-sgx'
-                            }
-                        }
-                        stage('Run SGX integration tests for musl') {
-                            options {
-                                timeout(time: 1, unit: 'HOURS')
-                            }
-                            steps {
-                                sh 'scripts/dev_cli.sh tests --integration-sgx --libc musl'
-                            }
-                        }
-                    }
-                    post {
-                        always {
-                            sh "sudo chown -R jenkins.jenkins ${WORKSPACE}"
-                            deleteDir()
-                        }
-                    }
-                }
-                stage('Worker build - VFIO') {
-                    agent { node { label 'jammy-vfio' } }
-                    when {
-                        beforeAgent true
-                        allOf {
-                            branch 'main'
-                            expression {
-                                return runWorkers
-                            }
-                        }
-                    }
-                    stages {
-                        stage('Checkout') {
-                            steps {
-                                checkout scm
-                            }
-                        }
-                        stage('Run VFIO integration tests') {
-                            options {
-                                timeout(time: 1, unit: 'HOURS')
-                            }
-                            steps {
-                                sh 'scripts/dev_cli.sh tests --integration-vfio'
-                            }
-                        }
-                        stage('Run VFIO integration tests for musl') {
-                            options {
-                                timeout(time: 1, unit: 'HOURS')
-                            }
-                            steps {
-                                sh 'scripts/dev_cli.sh tests --integration-vfio --libc musl'
-                            }
-                        }
-                    }
-                    post {
-                        always {
-                            sh "sudo chown -R jenkins.jenkins ${WORKSPACE}"
-                            deleteDir()
-                        }
-                    }
-                }
            }
        }
    }
@@ -419,31 +325,24 @@ def installAzureCli(distro, arch) {
    sh 'sudo apt install -y azure-cli'
 }

-def boolean skipWorkerBuild() {
+def boolean docsFileOnly() {
    if (env.CHANGE_TARGET == null) {
        return false
    }

-    if (sh(
+    return sh(
        returnStatus: true,
        script: "git diff --name-only origin/${env.CHANGE_TARGET}... | grep -v '\\.md'"
-    ) != 0) {
-        return true
+    ) != 0
+}
+
+def boolean fuzzFileOnly() {
+    if (env.CHANGE_TARGET == null) {
+        return false
    }

-    if (sh(
+    return sh(
        returnStatus: true,
        script: "git diff --name-only origin/${env.CHANGE_TARGET}... | grep -v -E 'fuzz/'"
-    ) != 0) {
-        return true
-    }
-
-    if (sh(
-        returnStatus: true,
-        script: "git diff --name-only origin/${env.CHANGE_TARGET}... | grep -v -E '.github/'"
-    ) != 0) {
-        return true
-    }
-
-    return false
+    ) != 0
 }
--- a/README.md
+++ b/README.md
@@ -69,12 +69,10 @@ Cloud Hypervisor supports `64-bit Linux` and Windows 10/Windows Server 2019.

 # 2. Getting Started

-The following sections describe how to build and run Cloud Hypervisor.
-
-## Prerequisites for AArch64
-
- AArch64 servers (recommended) or development boards equipped with the GICv3
-  interrupt controller.
+The following sections describe how to build and run Cloud Hypervisor on the
+`x86-64` platform. For getting started on the `AArch64` platform, please refer
+to the
+[AArch64 documentation](docs/arm64.md).

 ## Host OS

@@ -107,10 +105,13 @@ do not wish to use the pre-built binaries.

 ## Booting Linux

-Cloud Hypervisor supports direct kernel boot (the x86-64 kernel requires the kernel
-built with PVH support) or booting via a firmware (either [Rust Hypervisor
+The instructions below are for the `x86-64` platform. For `AArch64` please see
+the [AArch64 specific documentation](docs/arm64.md).
+
+Cloud Hypervisor supports direct kernel boot (if the kernel is built with PVH
+support) or booting via a firmware (either [Rust Hypervisor
 Firmware](https://github.com/cloud-hypervisor/rust-hypervisor-firmware) or an
-edk2 UEFI firmware called `CLOUDHV` / `CLOUDHV_EFI`.)
+edk2 UEFI firmware called `CLOUDHV`.)

 Binary builds of the firmware files are available for the latest release of
 [Rust Hyperivor
@@ -172,7 +173,7 @@ $ ./cloud-hypervisor \

 #### Building your Kernel

-Cloud Hypervisor also supports direct kernel boot. For x86-64, a `vmlinux` ELF kernel (compiled with PVH support) is needed. In order to support development there is a custom branch; however provided the required options are enabled any recent kernel will suffice.
+Cloud Hypervisor also supports direct kernel boot into a `vmlinux` ELF kernel (compiled with PVH support). In order to support development there is a custom branch; however provided the required options are enabled any recent kernel will suffice.

 To build the kernel:

@@ -180,23 +181,15 @@ To build the kernel:
 # Clone the Cloud Hypervisor Linux branch
 $ git clone --depth 1 https://github.com/cloud-hypervisor/linux.git -b ch-5.15.12 linux-cloud-hypervisor
 $ pushd linux-cloud-hypervisor
-# Use the x86-64 cloud-hypervisor kernel config to build your kernel for x86-64
+# Use the cloud-hypervisor kernel config to build your kernel
 $ wget https://raw.githubusercontent.com/cloud-hypervisor/cloud-hypervisor/main/resources/linux-config-x86_64
-# Use the AArch64 cloud-hypervisor kernel config to build your kernel for AArch64
-$ wget https://raw.githubusercontent.com/cloud-hypervisor/cloud-hypervisor/main/resources/linux-config-aarch64
-$ cp linux-config-x86_64 .config  # x86-64
-$ cp linux-config-aarch64 .config # AArch64
-# Do native build of the x86-64 kernel
+$ cp linux-config-x86_64 .config
 $ KCFLAGS="-Wa,-mx86-used-note=no" make bzImage -j `nproc`
-# Do native build of the AArch64 kernel
-$ make -j `nproc`
 $ popd
 ```

-For x86-64, the `vmlinux` kernel image will then be located at
+The `vmlinux` kernel image will then be located at
 `linux-cloud-hypervisor/arch/x86/boot/compressed/vmlinux.bin`.
-For AArch64, the `Image` kernel image will then be located at
-`linux-cloud-hypervisor/arch/arm64/boot/Image`.

 #### Disk image

@@ -204,10 +197,8 @@ For the disk image the same Ubuntu image as before can be used. This contains
 an `ext4` root filesystem.

 ```shell
-$ wget https://cloud-images.ubuntu.com/focal/current/focal-server-cloudimg-amd64.img # x86-64
-$ wget https://cloud-images.ubuntu.com/focal/current/focal-server-cloudimg-arm64.img # AArch64
-$ qemu-img convert -p -f qcow2 -O raw focal-server-cloudimg-amd64.img focal-server-cloudimg-amd64.raw # x86-64
-$ qemu-img convert -p -f qcow2 -O raw focal-server-cloudimg-arm64.img focal-server-cloudimg-arm64.raw # AArch64
+$ wget https://cloud-images.ubuntu.com/focal/current/focal-server-cloudimg-amd64.img
+$ qemu-img convert -p -f qcow2 -O raw focal-server-cloudimg-amd64.img focal-server-cloudimg-amd64.raw
 ```

 #### Booting the guest VM
@@ -215,8 +206,6 @@ $ qemu-img convert -p -f qcow2 -O raw focal-server-cloudimg-arm64.img focal-serv
 These sample commands boot the disk image using the custom kernel whilst also
 supplying the desired kernel command line.

- x86-64
-
 ```shell
 $ sudo setcap cap_net_admin+ep ./cloud-hypervisor
 $ ./create-cloud-init.sh
@@ -229,26 +218,9 @@ $ ./cloud-hypervisor \
 	--net "tap=,mac=,ip=,mask="
 ```

- AArch64
-
-```shell
-$ sudo setcap cap_net_admin+ep ./cloud-hypervisor
-$ ./create-cloud-init.sh
-$ ./cloud-hypervisor \
-	--kernel ./linux-cloud-hypervisor/arch/arm64/boot/Image \
-	--disk path=focal-server-cloudimg-arm64.raw path=/tmp/ubuntu-cloudinit.img \
-	--cmdline "console=hvc0 root=/dev/vda1 rw" \
-	--cpus boot=4 \
-	--memory size=1024M \
-	--net "tap=,mac=,ip=,mask="
-```
-
 If earlier kernel messages are required the serial console should be used instead of `virtio-console`.

- x86-64
-
-```shell
-$ ./cloud-hypervisor \
+```./cloud-hypervisor \
 	--kernel ./linux-cloud-hypervisor/arch/x86/boot/compressed/vmlinux.bin \
 	--console off \
 	--serial tty \
@@ -259,20 +231,6 @@ $ ./cloud-hypervisor \
 	--net "tap=,mac=,ip=,mask="
 ```

- AArch64
-
-```shell
-$ ./cloud-hypervisor \
-	--kernel ./linux-cloud-hypervisor/arch/arm64/boot/Image \
-	--console off \
-	--serial tty \
-	--disk path=focal-server-cloudimg-arm64.raw \
-	--cmdline "console=ttyAMA0 root=/dev/vda1 rw" \
-	--cpus boot=4 \
-	--memory size=1024M \
-	--net "tap=,mac=,ip=,mask="
-```
-
 # 3. Status

 Cloud Hypervisor is under active development. The following stability
@@ -296,11 +254,12 @@ Currently the following items are **not** guaranteed across updates:

 Further details can be found in the [release documentation](docs/releases.md).

-As of 2023-01-03, the following cloud images are supported:
+As of 2022-10-13, the following cloud images are supported:

- [Ubuntu Focal](https://cloud-images.ubuntu.com/focal/current/) (focal-server-cloudimg-{amd64,arm64}.img)
- [Ubuntu Jammy](https://cloud-images.ubuntu.com/jammy/current/) (jammy-server-cloudimg-{amd64,arm64}.img )
- [Fedora 36](https://fedora.mirrorservice.org/fedora/linux/releases/36/Cloud/) ([Fedora-Cloud-Base-36-1.5.x86_64.raw.xz](https://fedora.mirrorservice.org/fedora/linux/releases/36/Cloud/x86_64/images/) / [Fedora-Cloud-Base-36-1.5.aarch64.raw.xz](https://fedora.mirrorservice.org/fedora/linux/releases/36/Cloud/aarch64/images/))
+- [Ubuntu Bionic](https://cloud-images.ubuntu.com/bionic/current/) (bionic-server-cloudimg-amd64.img)
+- [Ubuntu Focal](https://cloud-images.ubuntu.com/focal/current/) (focal-server-cloudimg-amd64.img)
+- [Ubuntu Jammy](https://cloud-images.ubuntu.com/jammy/current/) (jammy-server-cloudimg-amd64.img )
+- [Fedora 36](https://fedora.mirrorservice.org/fedora/linux/releases/36/Cloud/x86_64/images/) (Fedora-Cloud-Base-36-1.5.x86_64.raw.xz)

 Direct kernel boot to userspace should work with a rootfs from most
 distributions although you may need to enable exotic filesystem types in the
--- a/acpi_tables/src/aml.rs
+++ b/acpi_tables/src/aml.rs
@@ -1083,7 +1083,7 @@ impl<'a, T> CreateField<'a, T> {
            buffer,
            offset,
            field,
-            phantom: PhantomData::default(),
+            phantom: PhantomData,
        }
    }
 }
--- a/acpi_tables/src/sdt.rs
+++ b/acpi_tables/src/sdt.rs
@@ -38,6 +38,7 @@ pub struct Sdt {
    data: Vec<u8>,
 }

+#[allow(clippy::len_without_is_empty)]
 impl Sdt {
    pub fn new(
        signature: [u8; 4],
@@ -97,7 +98,6 @@ impl Sdt {
    /// Write a value at the given offset
    pub fn write<T>(&mut self, offset: usize, value: T) {
        assert!((offset + (std::mem::size_of::<T>() - 1)) < self.data.len());
-        // SAFETY: The assertion above makes sure we don't do out of bounds write.
        unsafe {
            *(((self.data.as_mut_ptr() as usize) + offset) as *mut T) = value;
        }
@@ -123,10 +123,6 @@ impl Sdt {
    pub fn len(&self) -> usize {
        self.data.len()
    }
-
-    pub fn is_empty(&self) -> bool {
-        self.data.is_empty()
-    }
 }

 #[cfg(test)]
--- a/arch/Cargo.toml
+++ b/arch/Cargo.toml
@@ -9,14 +9,14 @@ default = []
 tdx = []

 [dependencies]
-anyhow = "1.0.68"
+anyhow = "1.0.66"
 byteorder = "1.4.3"
 hypervisor = { path = "../hypervisor" }
-libc = "0.2.139"
+libc = "0.2.138"
 linux-loader = { version = "0.8.1", features = ["elf", "bzimage", "pe"] }
 log = "0.4.17"
-serde = { version = "1.0.151", features = ["rc", "derive"] }
-thiserror = "1.0.38"
+serde = { version = "1.0.150", features = ["rc", "derive"] }
+thiserror = "1.0.37"
 uuid = "1.2.2"
 versionize = "0.1.9"
 versionize_derive = "0.1.4"
--- a/arch/src/aarch64/fdt.rs
+++ b/arch/src/aarch64/fdt.rs
@@ -259,7 +259,7 @@ fn create_memory_node(
        if last_addr < super::layout::MEM_32BIT_RESERVED_START.raw_value() {
            // Case 1: all RAM is under the hole
            let mem_size = last_addr - super::layout::RAM_START.raw_value() + 1;
-            let mem_reg_prop = [super::layout::RAM_START.raw_value(), mem_size];
+            let mem_reg_prop = [super::layout::RAM_START.raw_value() as u64, mem_size as u64];
            let memory_node = fdt.begin_node("memory")?;
            fdt.property_string("device_type", "memory")?;
            fdt.property_array_u64("reg", &mem_reg_prop)?;
@@ -269,7 +269,7 @@ fn create_memory_node(
            // Region 1: RAM before the hole
            let mem_size = super::layout::MEM_32BIT_RESERVED_START.raw_value()
                - super::layout::RAM_START.raw_value();
-            let mem_reg_prop = [super::layout::RAM_START.raw_value(), mem_size];
+            let mem_reg_prop = [super::layout::RAM_START.raw_value() as u64, mem_size as u64];
            let memory_node_name = format!("memory@{:x}", super::layout::RAM_START.raw_value());
            let memory_node = fdt.begin_node(&memory_node_name)?;
            fdt.property_string("device_type", "memory")?;
@@ -278,7 +278,10 @@ fn create_memory_node(

            // Region 2: RAM after the hole
            let mem_size = last_addr - super::layout::RAM_64BIT_START.raw_value() + 1;
-            let mem_reg_prop = [super::layout::RAM_64BIT_START.raw_value(), mem_size];
+            let mem_reg_prop = [
+                super::layout::RAM_64BIT_START.raw_value() as u64,
+                mem_size as u64,
+            ];
            let memory_node_name =
                format!("memory@{:x}", super::layout::RAM_64BIT_START.raw_value());
            let memory_node = fdt.begin_node(&memory_node_name)?;
@@ -300,7 +303,7 @@ fn create_chosen_node(
    fdt.property_string("bootargs", cmdline)?;

    if let Some(initrd_config) = initrd {
-        let initrd_start = initrd_config.address.raw_value();
+        let initrd_start = initrd_config.address.raw_value() as u64;
        let initrd_end = initrd_config.address.raw_value() + initrd_config.size as u64;
        fdt.property_u64("linux,initrd-start", initrd_start)?;
        fdt.property_u64("linux,initrd-end", initrd_end)?;
--- a/arch/src/aarch64/mod.rs
+++ b/arch/src/aarch64/mod.rs
@@ -19,7 +19,7 @@ use std::collections::HashMap;
 use std::convert::TryInto;
 use std::fmt::Debug;
 use std::sync::{Arc, Mutex};
-use vm_memory::{Address, GuestAddress, GuestMemory, GuestMemoryAtomic, GuestUsize};
+use vm_memory::{Address, GuestAddress, GuestMemory, GuestUsize};

 /// Errors thrown while configuring aarch64 system.
 #[derive(Debug)]
@@ -64,9 +64,9 @@ pub struct EntryPoint {
 pub fn configure_vcpu(
    vcpu: &Arc<dyn hypervisor::Vcpu>,
    id: u8,
-    boot_setup: Option<(EntryPoint, &GuestMemoryAtomic<GuestMemoryMmap>)>,
+    kernel_entry_point: Option<EntryPoint>,
 ) -> super::Result<u64> {
-    if let Some((kernel_entry_point, _guest_memory)) = boot_setup {
+    if let Some(kernel_entry_point) = kernel_entry_point {
        vcpu.setup_regs(
            id,
            kernel_entry_point.entry_addr.raw_value(),
@@ -108,7 +108,7 @@ pub fn arch_memory_regions(size: GuestUsize) -> Vec<(GuestAddress, usize, Region

    // RAM space
    // Case1: guest memory fits before the gap
-    if size <= ram_32bit_space_size {
+    if size as u64 <= ram_32bit_space_size {
        regions.push((layout::RAM_START, size as usize, RegionType::Ram));
    // Case2: guest memory extends beyond the gap
    } else {
@@ -231,7 +231,7 @@ mod tests {
            layout::MEM_32BIT_RESERVED_START.unchecked_offset_from(layout::RAM_START) as usize;
        assert_eq!(6, regions.len());
        assert_eq!(layout::RAM_START, regions[3].0);
-        assert_eq!(ram_32bit_space_size, regions[3].1);
+        assert_eq!(ram_32bit_space_size as usize, regions[3].1);
        assert_eq!(RegionType::Ram, regions[3].2);
        assert_eq!(RegionType::Reserved, regions[5].2);
        assert_eq!(RegionType::Ram, regions[4].2);
--- a/arch/src/lib.rs
+++ b/arch/src/lib.rs
@@ -99,7 +99,7 @@ pub use x86_64::{
 #[cfg(target_arch = "x86_64")]
 #[inline(always)]
 fn pagesize() -> usize {
-    // SAFETY: Trivially safe
+    // Trivially safe
    unsafe { libc::sysconf(libc::_SC_PAGESIZE) as usize }
 }

--- a/arch/src/x86_64/mod.rs
+++ b/arch/src/x86_64/mod.rs
@@ -16,7 +16,7 @@ use crate::GuestMemoryMmap;
 use crate::InitramfsConfig;
 use crate::RegionType;
 use hypervisor::arch::x86::{CpuIdEntry, CPUID_FLAG_VALID_INDEX};
-use hypervisor::HypervisorError;
+use hypervisor::{HypervisorCpuError, HypervisorError};
 use linux_loader::loader::bootparam::boot_params;
 use linux_loader::loader::elf::start_info::{
    hvm_memmap_table_entry, hvm_modlist_entry, hvm_start_info,
@@ -36,6 +36,7 @@ pub mod tdx;
 const TSC_DEADLINE_TIMER_ECX_BIT: u8 = 24; // tsc deadline timer ecx bit.
 const HYPERVISOR_ECX_BIT: u8 = 31; // Hypervisor ecx bit.
 const MTRR_EDX_BIT: u8 = 12; // Hypervisor ecx bit.
+const INVARIANT_TSC_EDX_BIT: u8 = 8; // Invariant TSC bit on 0x8000_0007 EDX

 // KVM feature bits
 const KVM_FEATURE_ASYNC_PF_INT_BIT: u8 = 14;
@@ -125,11 +126,9 @@ struct MemmapTableEntryWrapper(hvm_memmap_table_entry);
 #[derive(Copy, Clone, Default)]
 struct ModlistEntryWrapper(hvm_modlist_entry);

-// SAFETY: data structure only contain a series of integers
+// SAFETY: These data structures only contain a series of integers
 unsafe impl ByteValued for StartInfoWrapper {}
-// SAFETY: data structure only contain a series of integers
 unsafe impl ByteValued for MemmapTableEntryWrapper {}
-// SAFETY: data structure only contain a series of integers
 unsafe impl ByteValued for ModlistEntryWrapper {}

 // This is a workaround to the Rust enforcement specifying that any implementation of a foreign
@@ -193,6 +192,9 @@ pub enum Error {
    // Error writing EBDA address
    EbdaSetup(vm_memory::GuestMemoryError),

+    // Error getting CPU TSC frequency
+    GetTscFrequency(HypervisorCpuError),
+
    /// Error retrieving TDX capabilities through the hypervisor (kvm/mshv) API
    #[cfg(feature = "tdx")]
    TdxCapabilities(HypervisorError),
@@ -547,30 +549,13 @@ impl CpuidFeatureEntry {
 }

 pub fn generate_common_cpuid(
-    hypervisor: &Arc<dyn hypervisor::Hypervisor>,
+    hypervisor: Arc<dyn hypervisor::Hypervisor>,
    topology: Option<(u8, u8, u8)>,
    sgx_epc_sections: Option<Vec<SgxEpcSection>>,
    phys_bits: u8,
    kvm_hyperv: bool,
    #[cfg(feature = "tdx")] tdx_enabled: bool,
 ) -> super::Result<Vec<CpuIdEntry>> {
-    // SAFETY: cpuid called with valid leaves
-    if unsafe { x86_64::__cpuid(1) }.ecx & 1 << HYPERVISOR_ECX_BIT == 1 << HYPERVISOR_ECX_BIT {
-        // SAFETY: cpuid called with valid leaves
-        let hypervisor_cpuid = unsafe { x86_64::__cpuid(0x4000_0000) };
-
-        let mut identifier: [u8; 12] = [0; 12];
-        identifier[0..4].copy_from_slice(&hypervisor_cpuid.ebx.to_le_bytes()[..]);
-        identifier[4..8].copy_from_slice(&hypervisor_cpuid.ecx.to_le_bytes()[..]);
-        identifier[8..12].copy_from_slice(&hypervisor_cpuid.edx.to_le_bytes()[..]);
-
-        info!(
-            "Running under nested virtualisation. Hypervisor string: {}",
-            String::from_utf8_lossy(&identifier)
-        );
-    }
-
-    info!("Generating guest CPUID for with physical address size: {phys_bits}");
    let cpuid_patches = vec![
        // Patch tsc deadline timer bit
        CpuidPatch {
@@ -694,7 +679,6 @@ pub fn generate_common_cpuid(
    // Copy CPU identification string
    for i in 0x8000_0002..=0x8000_0004 {
        cpuid.retain(|c| c.function != i);
-        // SAFETY: call cpuid with valid leaves
        let leaf = unsafe { std::arch::x86_64::__cpuid(i) };
        cpuid.push(CpuIdEntry {
            function: i,
@@ -759,7 +743,8 @@ pub fn generate_common_cpuid(
 pub fn configure_vcpu(
    vcpu: &Arc<dyn hypervisor::Vcpu>,
    id: u8,
-    boot_setup: Option<(EntryPoint, &GuestMemoryAtomic<GuestMemoryMmap>)>,
+    kernel_entry_point: Option<EntryPoint>,
+    vm_memory: &GuestMemoryAtomic<GuestMemoryMmap>,
    cpuid: Vec<CpuIdEntry>,
    kvm_hyperv: bool,
 ) -> super::Result<()> {
@@ -768,6 +753,41 @@ pub fn configure_vcpu(
    CpuidPatch::set_cpuid_reg(&mut cpuid, 0xb, None, CpuidReg::EDX, u32::from(id));
    CpuidPatch::set_cpuid_reg(&mut cpuid, 0x1f, None, CpuidReg::EDX, u32::from(id));

+    // Set ApicId in cpuid for each vcpu
+    // SAFETY: get host cpuid when eax=1
+    let mut cpu_ebx = unsafe { core::arch::x86_64::__cpuid(1) }.ebx;
+    cpu_ebx &= 0xffffff;
+    cpu_ebx |= (id as u32) << 24;
+    CpuidPatch::set_cpuid_reg(&mut cpuid, 0x1, None, CpuidReg::EBX, cpu_ebx);
+
+    // The TSC frequency CPUID leaf should not be included when running with HyperV emulation
+    if !kvm_hyperv {
+        if let Some(tsc_khz) = vcpu.tsc_khz().map_err(Error::GetTscFrequency)? {
+            // Need to check that the TSC doesn't vary with dynamic frequency
+            // SAFETY: cpuid called with valid leaves
+            if unsafe { std::arch::x86_64::__cpuid(0x8000_0007) }.edx
+                & (1u32 << INVARIANT_TSC_EDX_BIT)
+                > 0
+            {
+                CpuidPatch::set_cpuid_reg(
+                    &mut cpuid,
+                    0x4000_0000,
+                    None,
+                    CpuidReg::EAX,
+                    0x4000_0010,
+                );
+                cpuid.retain(|c| c.function != 0x4000_0010);
+                cpuid.push(CpuIdEntry {
+                    function: 0x4000_0010,
+                    eax: tsc_khz,
+                    ebx: 1000000, /* LAPIC resolution of 1ns (freq: 1GHz) is hardcoded in KVM's
+                                   * APIC_BUS_CYCLE_NS */
+                    ..Default::default()
+                });
+            };
+        }
+    }
+
    vcpu.set_cpuid2(&cpuid)
        .map_err(|e| Error::SetSupportedCpusFailed(e.into()))?;

@@ -776,12 +796,12 @@ pub fn configure_vcpu(
    }

    regs::setup_msrs(vcpu).map_err(Error::MsrsConfiguration)?;
-    if let Some((kernel_entry_point, guest_memory)) = boot_setup {
+    if let Some(kernel_entry_point) = kernel_entry_point {
        if let Some(entry_addr) = kernel_entry_point.entry_addr {
            // Safe to unwrap because this method is called after the VM is configured
            regs::setup_regs(vcpu, entry_addr.raw_value()).map_err(Error::RegsConfiguration)?;
            regs::setup_fpu(vcpu).map_err(Error::FpuConfiguration)?;
-            regs::setup_sregs(&guest_memory.memory(), vcpu).map_err(Error::SregsConfiguration)?;
+            regs::setup_sregs(&vm_memory.memory(), vcpu).map_err(Error::SregsConfiguration)?;
        }
    }
    interrupts::set_lint(vcpu).map_err(|e| Error::LocalIntConfiguration(e.into()))?;
@@ -1044,7 +1064,6 @@ pub fn initramfs_load_addr(
 }

 pub fn get_host_cpu_phys_bits() -> u8 {
-    // SAFETY: call cpuid with valid leaves
    unsafe {
        let leaf = x86_64::__cpuid(0x8000_0000);

@@ -1155,7 +1174,6 @@ fn update_cpuid_sgx(

    // Get host CPUID for leaf 0x12, subleaf 0x2. This is to retrieve EPC
    // properties such as confidentiality and integrity.
-    // SAFETY: call cpuid with valid leaves
    let leaf = unsafe { std::arch::x86_64::__cpuid_count(0x12, 0x2) };

    for (i, epc_section) in epc_sections.iter().enumerate() {
--- a/arch/src/x86_64/mptable.rs
+++ b/arch/src/x86_64/mptable.rs
@@ -37,17 +37,11 @@ struct MpfIntelWrapper(mpspec::mpf_intel);

 // SAFETY: These `mpspec` wrapper types are only data, reading them from data is a safe initialization.
 unsafe impl ByteValued for MpcBusWrapper {}
-// SAFETY: see above
 unsafe impl ByteValued for MpcCpuWrapper {}
-// SAFETY: see above
 unsafe impl ByteValued for MpcIntsrcWrapper {}
-// SAFETY: see above
 unsafe impl ByteValued for MpcIoapicWrapper {}
-// SAFETY: see above
 unsafe impl ByteValued for MpcTableWrapper {}
-// SAFETY: see above
 unsafe impl ByteValued for MpcLintsrcWrapper {}
-// SAFETY: see above
 unsafe impl ByteValued for MpfIntelWrapper {}

 #[derive(Debug)]
@@ -101,7 +95,7 @@ const CPU_FEATURE_APIC: u32 = 0x200;
 const CPU_FEATURE_FPU: u32 = 0x001;

 fn compute_checksum<T: Copy>(v: &T) -> u8 {
-    // SAFETY: we are only reading the bytes within the size of the `T` reference `v`.
+    // Safe because we are only reading the bytes within the size of the `T` reference `v`.
    let v_slice = unsafe { slice::from_raw_parts(v as *const T as *const u8, mem::size_of::<T>()) };
    let mut checksum: u8 = 0;
    for i in v_slice.iter() {
--- a/arch/src/x86_64/smbios.rs
+++ b/arch/src/x86_64/smbios.rs
@@ -67,7 +67,7 @@ const PCI_SUPPORTED: u64 = 1 << 7;
 const IS_VIRTUAL_MACHINE: u8 = 1 << 4;

 fn compute_checksum<T: Copy>(v: &T) -> u8 {
-    // SAFETY: we are only reading the bytes within the size of the `T` reference `v`.
+    // Safe because we are only reading the bytes within the size of the `T` reference `v`.
    let v_slice = unsafe { slice::from_raw_parts(v as *const T as *const u8, mem::size_of::<T>()) };
    let mut checksum: u8 = 0;
    for i in v_slice.iter() {
@@ -145,15 +145,11 @@ struct SmbiosEndOfTable {
    handle: u16,
 }

-// SAFETY: data structure only contain a series of integers
+// SAFETY: These data structures only contain a series of integers
 unsafe impl ByteValued for Smbios30Entrypoint {}
-// SAFETY: data structure only contain a series of integers
 unsafe impl ByteValued for SmbiosBiosInfo {}
-// SAFETY: data structure only contain a series of integers
 unsafe impl ByteValued for SmbiosSysInfo {}
-// SAFETY: data structure only contain a series of integers
 unsafe impl ByteValued for SmbiosOemStrings {}
-// SAFETY: data structure only contain a series of integers
 unsafe impl ByteValued for SmbiosEndOfTable {}

 fn write_and_incr<T: ByteValued>(
--- a/arch/src/x86_64/tdx/mod.rs
+++ b/arch/src/x86_64/tdx/mod.rs
@@ -4,9 +4,7 @@
 use crate::GuestMemoryMmap;
 use std::fs::File;
 use std::io::{Read, Seek, SeekFrom};
-use std::str::FromStr;
 use thiserror::Error;
-use uuid::Uuid;
 use vm_memory::{ByteValued, Bytes, GuestAddress, GuestMemoryError};

 #[derive(Error, Debug)]
@@ -15,8 +13,6 @@ pub enum TdvfError {
    ReadDescriptor(#[source] std::io::Error),
    #[error("Failed read TDVF descriptor offset: {0}")]
    ReadDescriptorOffset(#[source] std::io::Error),
-    #[error("Failed read GUID table: {0}")]
-    ReadGuidTable(#[source] std::io::Error),
    #[error("Invalid descriptor signature")]
    InvalidDescriptorSignature,
    #[error("Invalid descriptor size")]
@@ -25,13 +21,8 @@ pub enum TdvfError {
    InvalidDescriptorVersion,
    #[error("Failed to write HOB details to guest memory: {0}")]
    GuestMemoryWriteHob(#[source] GuestMemoryError),
-    #[error("Failed to create Uuid: {0}")]
-    UuidCreation(#[source] uuid::Error),
 }

-const TABLE_FOOTER_GUID: &str = "96b582de-1fb2-45f7-baea-a366c55a082d";
-const TDVF_METADATA_OFFSET_GUID: &str = "e47a6535-984a-4798-865e-4685a7bf8ec2";
-
 // TDVF_DESCRIPTOR
 #[repr(packed)]
 #[derive(Default)]
@@ -55,7 +46,7 @@ pub struct TdvfSection {
 }

 #[repr(u32)]
-#[derive(Clone, Copy, Debug)]
+#[derive(Clone, Copy, Debug, Default)]
 pub enum TdvfSectionType {
    Bfv,
    Cfv,
@@ -64,81 +55,11 @@ pub enum TdvfSectionType {
    PermMem,
    Payload,
    PayloadParam,
+    #[default]
    Reserved = 0xffffffff,
 }

-impl Default for TdvfSectionType {
-    fn default() -> Self {
-        TdvfSectionType::Reserved
-    }
-}
-
-fn tdvf_descriptor_offset(file: &mut File) -> Result<(SeekFrom, bool), TdvfError> {
-    // Let's first try to identify the presence of the table footer GUID
-    file.seek(SeekFrom::End(-0x30))
-        .map_err(TdvfError::ReadGuidTable)?;
-    let mut table_footer_guid: [u8; 16] = [0; 16];
-    file.read_exact(&mut table_footer_guid)
-        .map_err(TdvfError::ReadGuidTable)?;
-    let uuid =
-        Uuid::from_slice_le(table_footer_guid.as_slice()).map_err(TdvfError::UuidCreation)?;
-    let expected_uuid = Uuid::from_str(TABLE_FOOTER_GUID).map_err(TdvfError::UuidCreation)?;
-    if uuid == expected_uuid {
-        // Retrieve the table size
-        file.seek(SeekFrom::End(-0x32))
-            .map_err(TdvfError::ReadGuidTable)?;
-        let mut table_size: [u8; 2] = [0; 2];
-        file.read_exact(&mut table_size)
-            .map_err(TdvfError::ReadGuidTable)?;
-        let table_size = u16::from_le_bytes(table_size) as usize;
-        let mut table: Vec<u8> = vec![0; table_size];
-
-        // Read the entire table
-        file.seek(SeekFrom::End(-(table_size as i64 + 0x20)))
-            .map_err(TdvfError::ReadGuidTable)?;
-        file.read_exact(table.as_mut_slice())
-            .map_err(TdvfError::ReadGuidTable)?;
-
-        // Let's start from the top and go backward down the table.
-        // We start after the footer GUID and the table length.
-        let mut offset = table_size - 18;
-
-        debug!("Parsing GUIDed structure");
-        while offset >= 18 {
-            let entry_uuid = Uuid::from_slice_le(&table[offset - 16..offset])
-                .map_err(TdvfError::UuidCreation)?;
-            let entry_size =
-                u16::from_le_bytes(table[offset - 18..offset - 16].try_into().unwrap()) as usize;
-            debug!(
-                "Entry GUID = {}, size = {}",
-                entry_uuid.hyphenated().to_string(),
-                entry_size
-            );
-
-            // Avoid going through an infinite loop if the entry size is 0
-            if entry_size == 0 {
-                break;
-            }
-
-            offset -= entry_size;
-
-            let expected_uuid =
-                Uuid::from_str(TDVF_METADATA_OFFSET_GUID).map_err(TdvfError::UuidCreation)?;
-            if entry_uuid == expected_uuid && entry_size == 22 {
-                return Ok((
-                    SeekFrom::End(
-                        -(u32::from_le_bytes(table[offset..offset + 4].try_into().unwrap()) as i64),
-                    ),
-                    true,
-                ));
-            }
-        }
-    }
-
-    // If we end up here, this means the firmware doesn't support the new way
-    // of exposing the TDVF descriptor offset through the table of GUIDs.
-    // That's why we fallback onto the deprecated method.
-
+pub fn parse_tdvf_sections(file: &mut File) -> Result<Vec<TdvfSection>, TdvfError> {
    // The 32-bit offset to the TDVF metadata is located 32 bytes from
    // the end of the file.
    // See "TDVF Metadata Pointer" in "TDX Virtual Firmware Design Guide
@@ -148,21 +69,13 @@ fn tdvf_descriptor_offset(file: &mut File) -> Result<(SeekFrom, bool), TdvfError
    let mut descriptor_offset: [u8; 4] = [0; 4];
    file.read_exact(&mut descriptor_offset)
        .map_err(TdvfError::ReadDescriptorOffset)?;
+    let descriptor_offset = u32::from_le_bytes(descriptor_offset) as u64;

-    Ok((
-        SeekFrom::Start(u32::from_le_bytes(descriptor_offset) as u64),
-        false,
-    ))
-}
-
-pub fn parse_tdvf_sections(file: &mut File) -> Result<(Vec<TdvfSection>, bool), TdvfError> {
-    let (descriptor_offset, guid_found) = tdvf_descriptor_offset(file)?;
-
-    file.seek(descriptor_offset)
+    file.seek(SeekFrom::Start(descriptor_offset))
        .map_err(TdvfError::ReadDescriptor)?;

    let mut descriptor: TdvfDescriptor = Default::default();
-    // SAFETY: we read exactly the size of the descriptor header
+    // Safe as we read exactly the size of the descriptor header
    file.read_exact(unsafe {
        std::slice::from_raw_parts_mut(
            &mut descriptor as *mut _ as *mut u8,
@@ -189,7 +102,7 @@ pub fn parse_tdvf_sections(file: &mut File) -> Result<(Vec<TdvfSection>, bool),
    let mut sections = Vec::new();
    sections.resize_with(descriptor.num_sections as usize, TdvfSection::default);

-    // SAFETY: we read exactly the advertised sections
+    // Safe as we read exactly the advertised sections
    file.read_exact(unsafe {
        std::slice::from_raw_parts_mut(
            sections.as_mut_ptr() as *mut u8,
@@ -198,25 +111,20 @@ pub fn parse_tdvf_sections(file: &mut File) -> Result<(Vec<TdvfSection>, bool),
    })
    .map_err(TdvfError::ReadDescriptor)?;

-    Ok((sections, guid_found))
+    Ok(sections)
 }

 #[repr(u16)]
-#[derive(Copy, Clone, Debug)]
+#[derive(Copy, Clone, Debug, Default)]
 enum HobType {
    Handoff = 0x1,
    ResourceDescriptor = 0x3,
    GuidExtension = 0x4,
+    #[default]
    Unused = 0xfffe,
    EndOfHobList = 0xffff,
 }

-impl Default for HobType {
-    fn default() -> Self {
-        HobType::Unused
-    }
-}
-
 #[repr(C)]
 #[derive(Copy, Clone, Default, Debug)]
 struct HobHeader {
@@ -266,19 +174,14 @@ struct HobGuidType {
 }

 #[repr(u32)]
-#[derive(Clone, Copy, Debug)]
+#[derive(Clone, Copy, Debug, Default)]
 pub enum PayloadImageType {
+    #[default]
    ExecutablePayload,
    BzImage,
    RawVmLinux,
 }

-impl Default for PayloadImageType {
-    fn default() -> Self {
-        PayloadImageType::ExecutablePayload
-    }
-}
-
 #[repr(C)]
 #[derive(Copy, Clone, Default, Debug)]
 pub struct PayloadInfo {
@@ -293,17 +196,12 @@ struct TdPayload {
    payload_info: PayloadInfo,
 }

-// SAFETY: data structure only contain a series of integers
+// SAFETY: These data structures only contain a series of integers
 unsafe impl ByteValued for HobHeader {}
-// SAFETY: data structure only contain a series of integers
 unsafe impl ByteValued for HobHandoffInfoTable {}
-// SAFETY: data structure only contain a series of integers
 unsafe impl ByteValued for HobResourceDescriptor {}
-// SAFETY: data structure only contain a series of integers
 unsafe impl ByteValued for HobGuidType {}
-// SAFETY: data structure only contain a series of integers
 unsafe impl ByteValued for PayloadInfo {}
-// SAFETY: data structure only contain a series of integers
 unsafe impl ByteValued for TdPayload {}

 pub struct TdHob {
@@ -399,19 +297,12 @@ impl TdHob {
        physical_start: u64,
        resource_length: u64,
        ram: bool,
-        guid_found: bool,
    ) -> Result<(), TdvfError> {
        self.add_resource(
            mem,
            physical_start,
            resource_length,
            if ram {
-                if guid_found {
-                    0x7 /* EFI_RESOURCE_MEMORY_UNACCEPTED */
-                } else {
-                    0 /* EFI_RESOURCE_SYSTEM_MEMORY */
-                }
-            } else if guid_found {
                0 /* EFI_RESOURCE_SYSTEM_MEMORY */
            } else {
                0x5 /*EFI_RESOURCE_MEMORY_RESERVED */
@@ -539,7 +430,7 @@ mod tests {
    #[ignore]
    fn test_parse_tdvf_sections() {
        let mut f = std::fs::File::open("tdvf.fd").unwrap();
-        let (sections, _) = parse_tdvf_sections(&mut f).unwrap();
+        let sections = parse_tdvf_sections(&mut f).unwrap();
        for section in sections {
            eprintln!("{section:x?}")
        }
--- a/block_util/Cargo.toml
+++ b/block_util/Cargo.toml
@@ -8,12 +8,11 @@ edition = "2021"
 default = []

 [dependencies]
-io-uring = "0.5.11"
-libc = "0.2.139"
+io-uring = "0.5.9"
+libc = "0.2.138"
 log = "0.4.17"
 qcow = { path = "../qcow" }
-smallvec = "1.10.0"
-thiserror = "1.0.38"
+thiserror = "1.0.37"
 versionize = "0.1.9"
 versionize_derive = "0.1.4"
 vhdx = { path = "../vhdx" }
--- a/block_util/src/async_io.rs
+++ b/block_util/src/async_io.rs
@@ -3,6 +3,7 @@
 // SPDX-License-Identifier: Apache-2.0 AND BSD-3-Clause

 use libc::{ioctl, S_IFBLK, S_IFMT};
+use std::convert::TryInto;
 use std::fs::File;
 use std::os::unix::io::AsRawFd;
 use thiserror::Error;
@@ -53,21 +54,19 @@ enum BlockSize {
 impl DiskTopology {
    fn is_block_device(f: &mut File) -> std::io::Result<bool> {
        let mut stat = std::mem::MaybeUninit::<libc::stat>::uninit();
-        // SAFETY: FFI call with a valid fd and buffer
        let ret = unsafe { libc::fstat(f.as_raw_fd(), stat.as_mut_ptr()) };
        if ret != 0 {
            return Err(std::io::Error::last_os_error());
        }

-        // SAFETY: stat is valid at this point
        let is_block = unsafe { (*stat.as_ptr()).st_mode & S_IFMT == S_IFBLK };
        Ok(is_block)
    }

    // libc::ioctl() takes different types on different architectures
+    #[allow(clippy::useless_conversion)]
    fn query_block_size(f: &mut File, block_size_type: BlockSize) -> std::io::Result<u64> {
        let mut block_size = 0;
-        // SAFETY: FFI call with correct arguments
        let ret = unsafe {
            ioctl(
                f.as_raw_fd(),
@@ -76,7 +75,9 @@ impl DiskTopology {
                    BlockSize::PhysicalBlock => BLKPBSZGET(),
                    BlockSize::MinimumIo => BLKIOMIN(),
                    BlockSize::OptimalIo => BLKIOOPT(),
-                } as _,
+                }
+                .try_into()
+                .unwrap(),
                &mut block_size,
            )
        };
@@ -131,15 +132,15 @@ pub trait AsyncIo: Send {
    fn read_vectored(
        &mut self,
        offset: libc::off_t,
-        iovecs: &[libc::iovec],
+        iovecs: Vec<libc::iovec>,
        user_data: u64,
    ) -> AsyncIoResult<()>;
    fn write_vectored(
        &mut self,
        offset: libc::off_t,
-        iovecs: &[libc::iovec],
+        iovecs: Vec<libc::iovec>,
        user_data: u64,
    ) -> AsyncIoResult<()>;
    fn fsync(&mut self, user_data: Option<u64>) -> AsyncIoResult<()>;
-    fn next_completed_request(&mut self) -> Option<(u64, i32)>;
+    fn complete(&mut self) -> Vec<(u64, i32)>;
 }
--- a/block_util/src/fixed_vhd_async.rs
+++ b/block_util/src/fixed_vhd_async.rs
@@ -64,7 +64,7 @@ impl AsyncIo for FixedVhdAsync {
    fn read_vectored(
        &mut self,
        offset: libc::off_t,
-        iovecs: &[libc::iovec],
+        iovecs: Vec<libc::iovec>,
        user_data: u64,
    ) -> AsyncIoResult<()> {
        if offset as u64 >= self.size {
@@ -83,7 +83,7 @@ impl AsyncIo for FixedVhdAsync {
    fn write_vectored(
        &mut self,
        offset: libc::off_t,
-        iovecs: &[libc::iovec],
+        iovecs: Vec<libc::iovec>,
        user_data: u64,
    ) -> AsyncIoResult<()> {
        if offset as u64 >= self.size {
@@ -104,7 +104,7 @@ impl AsyncIo for FixedVhdAsync {
        self.raw_file_async.fsync(user_data)
    }

-    fn next_completed_request(&mut self) -> Option<(u64, i32)> {
-        self.raw_file_async.next_completed_request()
+    fn complete(&mut self) -> Vec<(u64, i32)> {
+        self.raw_file_async.complete()
    }
 }
--- a/block_util/src/fixed_vhd_sync.rs
+++ b/block_util/src/fixed_vhd_sync.rs
@@ -62,7 +62,7 @@ impl AsyncIo for FixedVhdSync {
    fn read_vectored(
        &mut self,
        offset: libc::off_t,
-        iovecs: &[libc::iovec],
+        iovecs: Vec<libc::iovec>,
        user_data: u64,
    ) -> AsyncIoResult<()> {
        if offset as u64 >= self.size {
@@ -81,7 +81,7 @@ impl AsyncIo for FixedVhdSync {
    fn write_vectored(
        &mut self,
        offset: libc::off_t,
-        iovecs: &[libc::iovec],
+        iovecs: Vec<libc::iovec>,
        user_data: u64,
    ) -> AsyncIoResult<()> {
        if offset as u64 >= self.size {
@@ -101,7 +101,7 @@ impl AsyncIo for FixedVhdSync {
        self.raw_file_sync.fsync(user_data)
    }

-    fn next_completed_request(&mut self) -> Option<(u64, i32)> {
-        self.raw_file_sync.next_completed_request()
+    fn complete(&mut self) -> Vec<(u64, i32)> {
+        self.raw_file_sync.complete()
    }
 }
--- a/block_util/src/lib.rs
+++ b/block_util/src/lib.rs
@@ -22,10 +22,8 @@ pub mod vhdx_sync;

 use crate::async_io::{AsyncIo, AsyncIoError, AsyncIoResult};
 use io_uring::{opcode, IoUring, Probe};
-use smallvec::SmallVec;
 use std::alloc::{alloc_zeroed, dealloc, Layout};
 use std::cmp;
-use std::collections::VecDeque;
 use std::convert::TryInto;
 use std::fs::File;
 use std::io::{self, IoSlice, IoSliceMut, Read, Seek, SeekFrom, Write};
@@ -34,7 +32,6 @@ use std::path::Path;
 use std::result;
 use std::sync::Arc;
 use std::sync::MutexGuard;
-use std::time::Instant;
 use thiserror::Error;
 use versionize::{VersionMap, Versionize, VersionizeResult};
 use versionize_derive::Versionize;
@@ -198,11 +195,10 @@ pub struct AlignedOperation {
 pub struct Request {
    pub request_type: RequestType,
    pub sector: u64,
-    pub data_descriptors: SmallVec<[(GuestAddress, u32); 1]>,
+    pub data_descriptors: Vec<(GuestAddress, u32)>,
    pub status_addr: GuestAddress,
    pub writeback: bool,
    pub aligned_operations: Vec<AlignedOperation>,
-    pub start: Instant,
 }

 impl Request {
@@ -230,11 +226,10 @@ impl Request {
        let mut req = Request {
            request_type: request_type(desc_chain.memory(), hdr_desc_addr)?,
            sector: sector(desc_chain.memory(), hdr_desc_addr)?,
-            data_descriptors: SmallVec::with_capacity(1),
+            data_descriptors: Vec::new(),
            status_addr: GuestAddress(0),
            writeback: true,
            aligned_operations: Vec::new(),
-            start: Instant::now(),
        };

        let status_desc;
@@ -254,7 +249,6 @@ impl Request {
                return Err(Error::DescriptorChainTooShort);
            }
        } else {
-            req.data_descriptors.reserve_exact(1);
            while desc.has_next() {
                if desc.is_write_only() && req.request_type == RequestType::Out {
                    return Err(Error::UnexpectedWriteOnlyDescriptor);
@@ -359,8 +353,7 @@ impl Request {
        let request_type = self.request_type;
        let offset = (sector << SECTOR_SHIFT) as libc::off_t;

-        let mut iovecs: SmallVec<[libc::iovec; 1]> =
-            SmallVec::with_capacity(self.data_descriptors.len());
+        let mut iovecs = Vec::new();
        for (data_addr, data_len) in &self.data_descriptors {
            if *data_len == 0 {
                continue;
@@ -388,7 +381,7 @@ impl Request {
            let iov_base = if (origin_ptr as u64) % SECTOR_SIZE != 0 {
                let layout =
                    Layout::from_size_align(*data_len as usize, SECTOR_SIZE as usize).unwrap();
-                // SAFETY: layout has non-zero size
+                // Safe because layout has non-zero size
                let aligned_ptr = unsafe { alloc_zeroed(layout) };
                if aligned_ptr.is_null() {
                    return Err(ExecuteError::TemporaryBufferAllocation(
@@ -399,7 +392,7 @@ impl Request {
                // We need to perform the copy beforehand in case we're writing
                // data out.
                if request_type == RequestType::Out {
-                    // SAFETY: destination buffer has been allocated with
+                    // Safe because destination buffer has been allocated with
                    // the proper size.
                    unsafe {
                        std::ptr::copy(origin_ptr as *const u8, aligned_ptr, *data_len as usize)
@@ -437,12 +430,12 @@ impl Request {
                        .mark_dirty(0, *data_len as usize);
                }
                disk_image
-                    .read_vectored(offset, &iovecs, user_data)
+                    .read_vectored(offset, iovecs, user_data)
                    .map_err(ExecuteError::AsyncRead)?;
            }
            RequestType::Out => {
                disk_image
-                    .write_vectored(offset, &iovecs, user_data)
+                    .write_vectored(offset, iovecs, user_data)
                    .map_err(ExecuteError::AsyncWrite)?;
            }
            RequestType::Flush => {
@@ -474,7 +467,7 @@ impl Request {
            // We need to perform the copy after the data has been read inside
            // the aligned buffer in case we're reading data in.
            if self.request_type == RequestType::In {
-                // SAFETY: origin buffer has been allocated with the
+                // Safe because origin buffer has been allocated with the
                // proper size.
                unsafe {
                    std::ptr::copy(
@@ -486,7 +479,7 @@ impl Request {
            }

            // Free the temporary aligned buffer.
-            // SAFETY: aligned_ptr was allocated by alloc_zeroed with the same
+            // Safe because aligned_ptr was allocated by alloc_zeroed with the same
            // layout
            unsafe {
                dealloc(
@@ -535,9 +528,8 @@ pub struct VirtioBlockGeometry {
    pub sectors: u8,
 }

-// SAFETY: data structure only contain a series of integers
+// SAFETY: these data structures only contain a series of integers
 unsafe impl ByteValued for VirtioBlockConfig {}
-// SAFETY: data structure only contain a series of integers
 unsafe impl ByteValued for VirtioBlockGeometry {}

 /// Check if io_uring for block device can be used on the current system, as
@@ -596,15 +588,14 @@ where
    fn read_vectored_sync(
        &mut self,
        offset: libc::off_t,
-        iovecs: &[libc::iovec],
+        iovecs: Vec<libc::iovec>,
        user_data: u64,
        eventfd: &EventFd,
-        completion_list: &mut VecDeque<(u64, i32)>,
+        completion_list: &mut Vec<(u64, i32)>,
    ) -> AsyncIoResult<()> {
        // Convert libc::iovec into IoSliceMut
        let mut slices = Vec::new();
        for iovec in iovecs.iter() {
-            // SAFETY: on Linux IoSliceMut wraps around libc::iovec
            slices.push(IoSliceMut::new(unsafe { std::mem::transmute(*iovec) }));
        }

@@ -620,7 +611,7 @@ where
                .map_err(AsyncIoError::ReadVectored)?
        };

-        completion_list.push_back((user_data, result as i32));
+        completion_list.push((user_data, result as i32));
        eventfd.write(1).unwrap();

        Ok(())
@@ -629,15 +620,14 @@ where
    fn write_vectored_sync(
        &mut self,
        offset: libc::off_t,
-        iovecs: &[libc::iovec],
+        iovecs: Vec<libc::iovec>,
        user_data: u64,
        eventfd: &EventFd,
-        completion_list: &mut VecDeque<(u64, i32)>,
+        completion_list: &mut Vec<(u64, i32)>,
    ) -> AsyncIoResult<()> {
        // Convert libc::iovec into IoSlice
        let mut slices = Vec::new();
        for iovec in iovecs.iter() {
-            // SAFETY: on Linux IoSliceMut wraps around libc::iovec
            slices.push(IoSlice::new(unsafe { std::mem::transmute(*iovec) }));
        }

@@ -653,7 +643,7 @@ where
                .map_err(AsyncIoError::WriteVectored)?
        };

-        completion_list.push_back((user_data, result as i32));
+        completion_list.push((user_data, result as i32));
        eventfd.write(1).unwrap();

        Ok(())
@@ -663,7 +653,7 @@ where
        &mut self,
        user_data: Option<u64>,
        eventfd: &EventFd,
-        completion_list: &mut VecDeque<(u64, i32)>,
+        completion_list: &mut Vec<(u64, i32)>,
    ) -> AsyncIoResult<()> {
        let result: i32 = {
            let mut file = self.file();
@@ -675,7 +665,7 @@ where
        };

        if let Some(user_data) = user_data {
-            completion_list.push_back((user_data, result));
+            completion_list.push((user_data, result));
            eventfd.write(1).unwrap();
        }

--- a/block_util/src/qcow_sync.rs
+++ b/block_util/src/qcow_sync.rs
@@ -5,7 +5,6 @@
 use crate::async_io::{AsyncIo, AsyncIoResult, DiskFile, DiskFileError, DiskFileResult};
 use crate::AsyncAdaptor;
 use qcow::{QcowFile, RawFile, Result as QcowResult};
-use std::collections::VecDeque;
 use std::fs::File;
 use std::io::{Seek, SeekFrom};
 use std::sync::{Arc, Mutex, MutexGuard};
@@ -38,7 +37,7 @@ impl DiskFile for QcowDiskSync {
 pub struct QcowSync {
    qcow_file: Arc<Mutex<QcowFile>>,
    eventfd: EventFd,
-    completion_list: VecDeque<(u64, i32)>,
+    completion_list: Vec<(u64, i32)>,
 }

 impl QcowSync {
@@ -47,7 +46,7 @@ impl QcowSync {
            qcow_file,
            eventfd: EventFd::new(libc::EFD_NONBLOCK)
                .expect("Failed creating EventFd for QcowSync"),
-            completion_list: VecDeque::new(),
+            completion_list: Vec::new(),
        }
    }
 }
@@ -66,7 +65,7 @@ impl AsyncIo for QcowSync {
    fn read_vectored(
        &mut self,
        offset: libc::off_t,
-        iovecs: &[libc::iovec],
+        iovecs: Vec<libc::iovec>,
        user_data: u64,
    ) -> AsyncIoResult<()> {
        self.qcow_file.read_vectored_sync(
@@ -81,7 +80,7 @@ impl AsyncIo for QcowSync {
    fn write_vectored(
        &mut self,
        offset: libc::off_t,
-        iovecs: &[libc::iovec],
+        iovecs: Vec<libc::iovec>,
        user_data: u64,
    ) -> AsyncIoResult<()> {
        self.qcow_file.write_vectored_sync(
@@ -98,7 +97,7 @@ impl AsyncIo for QcowSync {
            .fsync_sync(user_data, &self.eventfd, &mut self.completion_list)
    }

-    fn next_completed_request(&mut self) -> Option<(u64, i32)> {
-        self.completion_list.pop_front()
+    fn complete(&mut self) -> Vec<(u64, i32)> {
+        self.completion_list.drain(..).collect()
    }
 }
--- a/block_util/src/raw_async.rs
+++ b/block_util/src/raw_async.rs
@@ -76,12 +76,12 @@ impl AsyncIo for RawFileAsync {
    fn read_vectored(
        &mut self,
        offset: libc::off_t,
-        iovecs: &[libc::iovec],
+        iovecs: Vec<libc::iovec>,
        user_data: u64,
    ) -> AsyncIoResult<()> {
        let (submitter, mut sq, _) = self.io_uring.split();

-        // SAFETY: we know the file descriptor is valid and we
+        // Safe because we know the file descriptor is valid and we
        // relied on vm-memory to provide the buffer address.
        let _ = unsafe {
            sq.push(
@@ -104,12 +104,12 @@ impl AsyncIo for RawFileAsync {
    fn write_vectored(
        &mut self,
        offset: libc::off_t,
-        iovecs: &[libc::iovec],
+        iovecs: Vec<libc::iovec>,
        user_data: u64,
    ) -> AsyncIoResult<()> {
        let (submitter, mut sq, _) = self.io_uring.split();

-        // SAFETY: we know the file descriptor is valid and we
+        // Safe because we know the file descriptor is valid and we
        // relied on vm-memory to provide the buffer address.
        let _ = unsafe {
            sq.push(
@@ -133,7 +133,7 @@ impl AsyncIo for RawFileAsync {
        if let Some(user_data) = user_data {
            let (submitter, mut sq, _) = self.io_uring.split();

-            // SAFETY: we know the file descriptor is valid.
+            // Safe because we know the file descriptor is valid.
            let _ = unsafe {
                sq.push(
                    &opcode::Fsync::new(types::Fd(self.fd))
@@ -148,17 +148,20 @@ impl AsyncIo for RawFileAsync {
            sq.sync();
            submitter.submit().map_err(AsyncIoError::Fsync)?;
        } else {
-            // SAFETY: FFI call with a valid fd
            unsafe { libc::fsync(self.fd) };
        }

        Ok(())
    }

-    fn next_completed_request(&mut self) -> Option<(u64, i32)> {
-        self.io_uring
-            .completion()
-            .next()
-            .map(|entry| (entry.user_data(), entry.result()))
+    fn complete(&mut self) -> Vec<(u64, i32)> {
+        let mut completion_list = Vec::new();
+
+        let cq = self.io_uring.completion();
+        for cq_entry in cq {
+            completion_list.push((cq_entry.user_data(), cq_entry.result()));
+        }
+
+        completion_list
    }
 }
--- a/block_util/src/raw_sync.rs
+++ b/block_util/src/raw_sync.rs
@@ -5,7 +5,6 @@
 use crate::async_io::{
    AsyncIo, AsyncIoError, AsyncIoResult, DiskFile, DiskFileError, DiskFileResult, DiskTopology,
 };
-use std::collections::VecDeque;
 use std::fs::File;
 use std::io::{Seek, SeekFrom};
 use std::os::unix::io::{AsRawFd, RawFd};
@@ -45,7 +44,7 @@ impl DiskFile for RawFileDiskSync {
 pub struct RawFileSync {
    fd: RawFd,
    eventfd: EventFd,
-    completion_list: VecDeque<(u64, i32)>,
+    completion_list: Vec<(u64, i32)>,
 }

 impl RawFileSync {
@@ -53,7 +52,7 @@ impl RawFileSync {
        RawFileSync {
            fd,
            eventfd: EventFd::new(libc::EFD_NONBLOCK).expect("Failed creating EventFd for RawFile"),
-            completion_list: VecDeque::new(),
+            completion_list: Vec::new(),
        }
    }
 }
@@ -66,14 +65,13 @@ impl AsyncIo for RawFileSync {
    fn read_vectored(
        &mut self,
        offset: libc::off_t,
-        iovecs: &[libc::iovec],
+        iovecs: Vec<libc::iovec>,
        user_data: u64,
    ) -> AsyncIoResult<()> {
-        // SAFETY: FFI call with valid arguments
        let result = unsafe {
            libc::preadv(
                self.fd as libc::c_int,
-                iovecs.as_ptr() as *const libc::iovec,
+                iovecs.as_ptr(),
                iovecs.len() as libc::c_int,
                offset,
            )
@@ -82,7 +80,7 @@ impl AsyncIo for RawFileSync {
            return Err(AsyncIoError::ReadVectored(std::io::Error::last_os_error()));
        }

-        self.completion_list.push_back((user_data, result as i32));
+        self.completion_list.push((user_data, result as i32));
        self.eventfd.write(1).unwrap();

        Ok(())
@@ -91,14 +89,13 @@ impl AsyncIo for RawFileSync {
    fn write_vectored(
        &mut self,
        offset: libc::off_t,
-        iovecs: &[libc::iovec],
+        iovecs: Vec<libc::iovec>,
        user_data: u64,
    ) -> AsyncIoResult<()> {
-        // SAFETY: FFI call with valid arguments
        let result = unsafe {
            libc::pwritev(
                self.fd as libc::c_int,
-                iovecs.as_ptr() as *const libc::iovec,
+                iovecs.as_ptr(),
                iovecs.len() as libc::c_int,
                offset,
            )
@@ -107,28 +104,27 @@ impl AsyncIo for RawFileSync {
            return Err(AsyncIoError::WriteVectored(std::io::Error::last_os_error()));
        }

-        self.completion_list.push_back((user_data, result as i32));
+        self.completion_list.push((user_data, result as i32));
        self.eventfd.write(1).unwrap();

        Ok(())
    }

    fn fsync(&mut self, user_data: Option<u64>) -> AsyncIoResult<()> {
-        // SAFETY: FFI call
        let result = unsafe { libc::fsync(self.fd as libc::c_int) };
        if result < 0 {
            return Err(AsyncIoError::Fsync(std::io::Error::last_os_error()));
        }

        if let Some(user_data) = user_data {
-            self.completion_list.push_back((user_data, result));
+            self.completion_list.push((user_data, result));
            self.eventfd.write(1).unwrap();
        }

        Ok(())
    }

-    fn next_completed_request(&mut self) -> Option<(u64, i32)> {
-        self.completion_list.pop_front()
+    fn complete(&mut self) -> Vec<(u64, i32)> {
+        self.completion_list.drain(..).collect()
    }
 }
--- a/block_util/src/vhdx_sync.rs
+++ b/block_util/src/vhdx_sync.rs
@@ -4,7 +4,6 @@

 use crate::async_io::{AsyncIo, AsyncIoResult, DiskFile, DiskFileError, DiskFileResult};
 use crate::AsyncAdaptor;
-use std::collections::VecDeque;
 use std::fs::File;
 use std::sync::{Arc, Mutex, MutexGuard};
 use vhdx::vhdx::{Result as VhdxResult, Vhdx};
@@ -38,7 +37,7 @@ impl DiskFile for VhdxDiskSync {
 pub struct VhdxSync {
    vhdx_file: Arc<Mutex<Vhdx>>,
    eventfd: EventFd,
-    completion_list: VecDeque<(u64, i32)>,
+    completion_list: Vec<(u64, i32)>,
 }

 impl VhdxSync {
@@ -46,7 +45,7 @@ impl VhdxSync {
        Ok(VhdxSync {
            vhdx_file,
            eventfd: EventFd::new(libc::EFD_NONBLOCK)?,
-            completion_list: VecDeque::new(),
+            completion_list: Vec::new(),
        })
    }
 }
@@ -65,7 +64,7 @@ impl AsyncIo for VhdxSync {
    fn read_vectored(
        &mut self,
        offset: libc::off_t,
-        iovecs: &[libc::iovec],
+        iovecs: Vec<libc::iovec>,
        user_data: u64,
    ) -> AsyncIoResult<()> {
        self.vhdx_file.read_vectored_sync(
@@ -80,7 +79,7 @@ impl AsyncIo for VhdxSync {
    fn write_vectored(
        &mut self,
        offset: libc::off_t,
-        iovecs: &[libc::iovec],
+        iovecs: Vec<libc::iovec>,
        user_data: u64,
    ) -> AsyncIoResult<()> {
        self.vhdx_file.write_vectored_sync(
@@ -97,7 +96,7 @@ impl AsyncIo for VhdxSync {
            .fsync_sync(user_data, &self.eventfd, &mut self.completion_list)
    }

-    fn next_completed_request(&mut self) -> Option<(u64, i32)> {
-        self.completion_list.pop_front()
+    fn complete(&mut self) -> Vec<(u64, i32)> {
+        self.completion_list.drain(..).collect()
    }
 }
--- a/build.rs
+++ b/build.rs
@@ -3,10 +3,13 @@
 // SPDX-License-Identifier: Apache-2.0
 //

+#[macro_use(crate_version)]
+extern crate clap;
+
 use std::process::Command;

 fn main() {
-    let mut version = "v".to_owned() + env!("CARGO_PKG_VERSION");
+    let mut version = "v".to_owned() + crate_version!();

    if let Ok(git_out) = Command::new("git").args(["describe", "--dirty"]).output() {
        if git_out.status.success() {
--- a/devices/Cargo.toml
+++ b/devices/Cargo.toml
@@ -6,15 +6,15 @@ edition = "2021"

 [dependencies]
 acpi_tables = { path = "../acpi_tables" }
-anyhow = "1.0.68"
+anyhow = "1.0.66"
 arch = { path = "../arch" }
-bitflags = "1.3.2"
+bitflags = "2.4.1"
 byteorder = "1.4.3"
 hypervisor = { path = "../hypervisor" }
-libc = "0.2.139"
+libc = "0.2.138"
 log = "0.4.17"
 phf = { version = "0.11.1", features = ["macros"] }
-thiserror = "1.0.38"
+thiserror = "1.0.37"
 tpm = { path = "../tpm" }
 versionize = "0.1.9"
 versionize_derive = "0.1.4"
--- a/devices/src/gic.rs
+++ b/devices/src/gic.rs
@@ -8,7 +8,7 @@ use anyhow::anyhow;
 use arch::layout;
 use hypervisor::{
    arch::aarch64::gic::{Vgic, VgicConfig},
-    CpuState, GicState,
+    CpuState,
 };
 use std::result;
 use std::sync::{Arc, Mutex};
@@ -25,7 +25,6 @@ type Result<T> = result::Result<T, Error>;
 // Reserve 32 IRQs for legacy devices.
 pub const IRQ_LEGACY_BASE: usize = layout::IRQ_BASE as usize;
 pub const IRQ_LEGACY_COUNT: usize = 32;
-pub const GIC_SNAPSHOT_ID: &str = "gic-v3-its";

 // Gic (Generic Interupt Controller) struct provides all the functionality of a
 // GIC device. It wraps a hypervisor-emulated GIC device (Vgic) provided by the
@@ -40,9 +39,8 @@ pub struct Gic {

 impl Gic {
    pub fn new(
-        vcpu_count: u8,
+        _vcpu_count: u8,
        interrupt_manager: Arc<dyn InterruptManager<GroupConfig = MsiIrqGroupConfig>>,
-        vm: Arc<dyn hypervisor::Vm>,
    ) -> Result<Gic> {
        let interrupt_source_group = interrupt_manager
            .create_group(MsiIrqGroupConfig {
@@ -51,34 +49,45 @@ impl Gic {
            })
            .map_err(Error::CreateInterruptSourceGroup)?;

-        let vgic = vm
-            .create_vgic(Gic::create_default_config(vcpu_count as u64))
-            .map_err(Error::CreateGic)?;
-
-        let gic = Gic {
+        Ok(Gic {
            interrupt_source_group,
-            vgic: Some(vgic),
-        };
-        gic.enable()?;
-
-        Ok(gic)
+            vgic: None,
+        })
    }

-    pub fn restore_vgic(
+    /// Default config implied by arch::layout
+    pub fn create_default_config(vcpu_count: u64) -> VgicConfig {
+        let redists_size = layout::GIC_V3_REDIST_SIZE * vcpu_count;
+        let redists_addr = layout::GIC_V3_DIST_START.raw_value() - redists_size;
+        VgicConfig {
+            vcpu_count,
+            dist_addr: layout::GIC_V3_DIST_START.raw_value(),
+            dist_size: layout::GIC_V3_DIST_SIZE,
+            redists_addr,
+            redists_size,
+            msi_addr: redists_addr - layout::GIC_V3_ITS_SIZE,
+            msi_size: layout::GIC_V3_ITS_SIZE,
+            nr_irqs: layout::IRQ_NUM,
+        }
+    }
+
+    pub fn create_vgic(
        &mut self,
-        state: Option<GicState>,
-        saved_vcpu_states: &[CpuState],
-    ) -> Result<()> {
-        self.set_gicr_typers(saved_vcpu_states);
-        self.vgic
-            .clone()
-            .unwrap()
-            .lock()
-            .unwrap()
-            .set_state(&state.unwrap())
-            .map_err(Error::RestoreGic)
+        vm: &Arc<dyn hypervisor::Vm>,
+        config: VgicConfig,
+    ) -> Result<Arc<Mutex<dyn Vgic>>> {
+        let vgic = vm.create_vgic(config).map_err(Error::CreateGic)?;
+        self.vgic = Some(vgic.clone());
+        Ok(vgic.clone())
    }

+    pub fn set_gicr_typers(&mut self, vcpu_states: &[CpuState]) {
+        let vgic = self.vgic.as_ref().unwrap().clone();
+        vgic.lock().unwrap().set_gicr_typers(vcpu_states);
+    }
+}
+
+impl InterruptController for Gic {
    fn enable(&self) -> Result<()> {
        // Set irqfd for legacy interrupts
        self.interrupt_source_group
@@ -104,33 +113,6 @@ impl Gic {
        Ok(())
    }

-    /// Default config implied by arch::layout
-    pub fn create_default_config(vcpu_count: u64) -> VgicConfig {
-        let redists_size = layout::GIC_V3_REDIST_SIZE * vcpu_count;
-        let redists_addr = layout::GIC_V3_DIST_START.raw_value() - redists_size;
-        VgicConfig {
-            vcpu_count,
-            dist_addr: layout::GIC_V3_DIST_START.raw_value(),
-            dist_size: layout::GIC_V3_DIST_SIZE,
-            redists_addr,
-            redists_size,
-            msi_addr: redists_addr - layout::GIC_V3_ITS_SIZE,
-            msi_size: layout::GIC_V3_ITS_SIZE,
-            nr_irqs: layout::IRQ_NUM,
-        }
-    }
-
-    pub fn get_vgic(&mut self) -> Result<Arc<Mutex<dyn Vgic>>> {
-        Ok(self.vgic.clone().unwrap())
-    }
-
-    pub fn set_gicr_typers(&mut self, vcpu_states: &[CpuState]) {
-        let vgic = self.vgic.as_ref().unwrap().clone();
-        vgic.lock().unwrap().set_gicr_typers(vcpu_states);
-    }
-}
-
-impl InterruptController for Gic {
    // This should be called anytime an interrupt needs to be injected into the
    // running guest.
    fn service_irq(&mut self, irq: usize) -> Result<()> {
@@ -146,15 +128,27 @@ impl InterruptController for Gic {
    }
 }

+pub const GIC_V3_ITS_SNAPSHOT_ID: &str = "gic-v3-its";
 impl Snapshottable for Gic {
    fn id(&self) -> String {
-        GIC_SNAPSHOT_ID.to_string()
+        GIC_V3_ITS_SNAPSHOT_ID.to_string()
    }

    fn snapshot(&mut self) -> std::result::Result<Snapshot, MigratableError> {
        let vgic = self.vgic.as_ref().unwrap().clone();
        let state = vgic.lock().unwrap().state().unwrap();
-        Snapshot::new_from_state(&state)
+        Snapshot::new_from_state(&self.id(), &state)
+    }
+
+    fn restore(&mut self, snapshot: Snapshot) -> std::result::Result<(), MigratableError> {
+        let vgic = self.vgic.as_ref().unwrap().clone();
+        vgic.lock()
+            .unwrap()
+            .set_state(&snapshot.to_state(&self.id())?)
+            .map_err(|e| {
+                MigratableError::Restore(anyhow!("Could not restore GICv3ITS state {:?}", e))
+            })?;
+        Ok(())
    }
 }

--- a/devices/src/interrupt_controller.rs
+++ b/devices/src/interrupt_controller.rs
@@ -24,12 +24,8 @@ pub enum Error {
    UpdateInterrupt(io::Error),
    /// Failed enabling the interrupt.
    EnableInterrupt(io::Error),
-    #[cfg(target_arch = "aarch64")]
    /// Failed creating GIC device.
    CreateGic(hypervisor::HypervisorVmError),
-    #[cfg(target_arch = "aarch64")]
-    /// Failed restoring GIC device.
-    RestoreGic(hypervisor::arch::aarch64::gic::Error),
 }

 type Result<T> = result::Result<T, Error>;
@@ -59,6 +55,8 @@ pub struct MsiMessage {
 // IOAPIC (X86) or GIC (Arm).
 pub trait InterruptController: Send {
    fn service_irq(&mut self, irq: usize) -> Result<()>;
+    #[cfg(target_arch = "aarch64")]
+    fn enable(&self) -> Result<()>;
    #[cfg(target_arch = "x86_64")]
    fn end_of_interrupt(&mut self, vec: u8);
    fn notifier(&self, irq: usize) -> Option<EventFd>;
--- a/devices/src/ioapic.rs
+++ b/devices/src/ioapic.rs
@@ -10,6 +10,7 @@
 // See https://pdos.csail.mit.edu/6.828/2016/readings/ia32/ioapic.pdf for a specification.

 use super::interrupt_controller::{Error, InterruptController};
+use anyhow::anyhow;
 use byteorder::{ByteOrder, LittleEndian};
 use std::result;
 use std::sync::{Arc, Barrier};
@@ -193,7 +194,6 @@ impl Ioapic {
        id: String,
        apic_address: GuestAddress,
        interrupt_manager: Arc<dyn InterruptManager<GroupConfig = MsiIrqGroupConfig>>,
-        state: Option<IoapicState>,
    ) -> Result<Ioapic> {
        let interrupt_source_group = interrupt_manager
            .create_group(MsiIrqGroupConfig {
@@ -202,47 +202,17 @@ impl Ioapic {
            })
            .map_err(Error::CreateInterruptSourceGroup)?;

-        let (id_reg, reg_sel, reg_entries, used_entries, apic_address) = if let Some(state) = &state
-        {
-            (
-                state.id_reg,
-                state.reg_sel,
-                state.reg_entries,
-                state.used_entries,
-                GuestAddress(state.apic_address),
-            )
-        } else {
-            (
-                0,
-                0,
-                [0x10000; NUM_IOAPIC_PINS],
-                [false; NUM_IOAPIC_PINS],
-                apic_address,
-            )
-        };
-
        // The IOAPIC is created with entries already masked. The guest will be
        // in charge of unmasking them if/when necessary.
-        let ioapic = Ioapic {
+        Ok(Ioapic {
            id,
-            id_reg,
-            reg_sel,
-            reg_entries,
-            used_entries,
+            id_reg: 0,
+            reg_sel: 0,
+            reg_entries: [0x10000; NUM_IOAPIC_PINS],
+            used_entries: [false; NUM_IOAPIC_PINS],
            apic_address,
            interrupt_source_group,
-        };
-
-        // When restoring the Ioapic, we must enable used entries.
-        if state.is_some() {
-            for (irq, entry) in ioapic.used_entries.iter().enumerate() {
-                if *entry {
-                    ioapic.update_entry(irq)?;
-                }
-            }
-        }
-
-        Ok(ioapic)
+        })
    }

    fn ioapic_write(&mut self, val: u32) {
@@ -329,6 +299,21 @@ impl Ioapic {
        }
    }

+    fn set_state(&mut self, state: &IoapicState) -> Result<()> {
+        self.id_reg = state.id_reg;
+        self.reg_sel = state.reg_sel;
+        self.reg_entries = state.reg_entries;
+        self.used_entries = state.used_entries;
+        self.apic_address = GuestAddress(state.apic_address);
+        for (irq, entry) in self.used_entries.iter().enumerate() {
+            if *entry {
+                self.update_entry(irq)?;
+            }
+        }
+
+        Ok(())
+    }
+
    fn update_entry(&self, irq: usize) -> Result<()> {
        let entry = self.reg_entries[irq];

@@ -438,7 +423,18 @@ impl Snapshottable for Ioapic {
    }

    fn snapshot(&mut self) -> std::result::Result<Snapshot, MigratableError> {
-        Snapshot::new_from_versioned_state(&self.state())
+        Snapshot::new_from_versioned_state(&self.id, &self.state())
+    }
+
+    fn restore(&mut self, snapshot: Snapshot) -> std::result::Result<(), MigratableError> {
+        self.set_state(&snapshot.to_versioned_state(&self.id)?)
+            .map_err(|e| {
+                MigratableError::Restore(anyhow!(
+                    "Could not restore state for {}: {:?}",
+                    self.id,
+                    e
+                ))
+            })
    }
 }

--- a/devices/src/legacy/cmos.rs
+++ b/devices/src/legacy/cmos.rs
@@ -97,7 +97,7 @@ impl BusDevice for Cmos {
                let day;
                let month;
                let year;
-                // SAFETY: The clock_gettime and gmtime_r calls are safe as long as the structs they are
+                // The clock_gettime and gmtime_r calls are safe as long as the structs they are
                // given are large enough, and neither of them fail. It is safe to zero initialize
                // the tm and timespec struct because it contains only plain data.
                let update_in_progress = unsafe {
--- a/devices/src/legacy/gpio_pl061.rs
+++ b/devices/src/legacy/gpio_pl061.rs
@@ -106,39 +106,18 @@ impl VersionMapped for GpioState {}

 impl Gpio {
    /// Constructs an PL061 GPIO device.
-    pub fn new(
-        id: String,
-        interrupt: Arc<dyn InterruptSourceGroup>,
-        state: Option<GpioState>,
-    ) -> Self {
-        let (data, old_in_data, dir, isense, ibe, iev, im, istate, afsel) =
-            if let Some(state) = state {
-                (
-                    state.data,
-                    state.old_in_data,
-                    state.dir,
-                    state.isense,
-                    state.ibe,
-                    state.iev,
-                    state.im,
-                    state.istate,
-                    state.afsel,
-                )
-            } else {
-                (0, 0, 0, 0, 0, 0, 0, 0, 0)
-            };
-
+    pub fn new(id: String, interrupt: Arc<dyn InterruptSourceGroup>) -> Self {
        Self {
            id,
-            data,
-            old_in_data,
-            dir,
-            isense,
-            ibe,
-            iev,
-            im,
-            istate,
-            afsel,
+            data: 0,
+            old_in_data: 0,
+            dir: 0,
+            isense: 0,
+            ibe: 0,
+            iev: 0,
+            im: 0,
+            istate: 0,
+            afsel: 0,
            interrupt,
        }
    }
@@ -157,12 +136,24 @@ impl Gpio {
        }
    }

+    fn set_state(&mut self, state: &GpioState) {
+        self.data = state.data;
+        self.old_in_data = state.old_in_data;
+        self.dir = state.dir;
+        self.isense = state.isense;
+        self.ibe = state.ibe;
+        self.iev = state.iev;
+        self.im = state.im;
+        self.istate = state.istate;
+        self.afsel = state.afsel;
+    }
+
    fn pl061_internal_update(&mut self) {
        // FIXME:
        //  Missing Output Interrupt Emulation.

        // Input Edging Interrupt Emulation.
-        let changed = (self.old_in_data ^ self.data) & !self.dir;
+        let changed = ((self.old_in_data ^ self.data) & !self.dir) as u32;
        if changed > 0 {
            self.old_in_data = self.data;
            for i in 0..N_GPIOS {
@@ -328,7 +319,12 @@ impl Snapshottable for Gpio {
    }

    fn snapshot(&mut self) -> std::result::Result<Snapshot, MigratableError> {
-        Snapshot::new_from_versioned_state(&self.state())
+        Snapshot::new_from_versioned_state(&self.id, &self.state())
+    }
+
+    fn restore(&mut self, snapshot: Snapshot) -> std::result::Result<(), MigratableError> {
+        self.set_state(&snapshot.to_versioned_state(&self.id)?);
+        Ok(())
    }
 }

@@ -382,7 +378,6 @@ mod tests {
        let mut gpio = Gpio::new(
            String::from(GPIO_NAME),
            Arc::new(TestInterrupt::new(intr_evt.try_clone().unwrap())),
-            None,
        );
        let mut data = [0; 4];

--- a/devices/src/legacy/rtc_pl031.rs
+++ b/devices/src/legacy/rtc_pl031.rs
@@ -61,10 +61,12 @@ pub enum ClockType {
    /// Equivalent to `libc::CLOCK_MONOTONIC`.
    Monotonic,
    /// Equivalent to `libc::CLOCK_REALTIME`.
+    #[allow(dead_code)]
    Real,
    /// Equivalent to `libc::CLOCK_PROCESS_CPUTIME_ID`.
    ProcessCpu,
    /// Equivalent to `libc::CLOCK_THREAD_CPUTIME_ID`.
+    #[allow(dead_code)]
    ThreadCpu,
 }

@@ -99,7 +101,7 @@ pub struct LocalTime {

 impl LocalTime {
    /// Returns the [LocalTime](struct.LocalTime.html) structure for the calling moment.
-    #[cfg(test)]
+    #[allow(dead_code)]
    pub fn now() -> LocalTime {
        let mut timespec = libc::timespec {
            tv_sec: 0,
@@ -119,7 +121,7 @@ impl LocalTime {
            tm_zone: std::ptr::null(),
        };

-        // SAFETY: the parameters are valid.
+        // Safe because the parameters are valid.
        unsafe {
            libc::clock_gettime(libc::CLOCK_REALTIME, &mut timespec);
            libc::localtime_r(&timespec.tv_sec, &mut tm);
@@ -171,6 +173,22 @@ impl Default for TimestampUs {
    }
 }

+/// Returns a timestamp in nanoseconds from a monotonic clock.
+///
+/// Uses `_rdstc` on `x86_64` and [`get_time`](fn.get_time.html) on other architectures.
+#[allow(dead_code)]
+pub fn timestamp_cycles() -> u64 {
+    #[cfg(target_arch = "x86_64")]
+    // Safe because there's nothing that can go wrong with this call.
+    unsafe {
+        std::arch::x86_64::_rdtsc() as u64
+    }
+    #[cfg(not(target_arch = "x86_64"))]
+    {
+        get_time(ClockType::Monotonic)
+    }
+}
+
 /// Returns a timestamp in nanoseconds based on the provided clock type.
 ///
 /// # Arguments
@@ -181,7 +199,7 @@ pub fn get_time(clock_type: ClockType) -> u64 {
        tv_sec: 0,
        tv_nsec: 0,
    };
-    // SAFETY: the parameters are valid.
+    // Safe because the parameters are valid.
    unsafe { libc::clock_gettime(clock_type.into(), &mut time_struct) };
    seconds_to_nanoseconds(time_struct.tv_sec).unwrap() as u64 + (time_struct.tv_nsec as u64)
 }
@@ -507,6 +525,7 @@ mod tests {
        ($test_name: ident, $write_fn_name: ident, $read_fn_name: ident, $is_be: expr, $data_type: ty) => {
            #[test]
            fn $test_name() {
+                #[allow(overflowing_literals)]
                let test_cases = [
                    (
                        0x0123_4567_89AB_CDEF as u64,
--- a/devices/src/legacy/serial.rs
+++ b/devices/src/legacy/serial.rs
@@ -63,6 +63,7 @@ pub struct Serial {
    id: String,
    interrupt_enable: u8,
    interrupt_identification: u8,
+    interrupt: Arc<dyn InterruptSourceGroup>,
    line_control: u8,
    line_status: u8,
    modem_control: u8,
@@ -70,7 +71,6 @@ pub struct Serial {
    scratch: u8,
    baud_divisor: u16,
    in_buffer: VecDeque<u8>,
-    interrupt: Arc<dyn InterruptSourceGroup>,
    out: Option<Box<dyn io::Write + Send>>,
 }

@@ -93,56 +93,19 @@ impl Serial {
        id: String,
        interrupt: Arc<dyn InterruptSourceGroup>,
        out: Option<Box<dyn io::Write + Send>>,
-        state: Option<SerialState>,
    ) -> Serial {
-        let (
-            interrupt_enable,
-            interrupt_identification,
-            line_control,
-            line_status,
-            modem_control,
-            modem_status,
-            scratch,
-            baud_divisor,
-            in_buffer,
-        ) = if let Some(state) = state {
-            (
-                state.interrupt_enable,
-                state.interrupt_identification,
-                state.line_control,
-                state.line_status,
-                state.modem_control,
-                state.modem_status,
-                state.scratch,
-                state.baud_divisor,
-                state.in_buffer.into(),
-            )
-        } else {
-            (
-                0,
-                DEFAULT_INTERRUPT_IDENTIFICATION,
-                DEFAULT_LINE_CONTROL,
-                DEFAULT_LINE_STATUS,
-                DEFAULT_MODEM_CONTROL,
-                DEFAULT_MODEM_STATUS,
-                0,
-                DEFAULT_BAUD_DIVISOR,
-                VecDeque::new(),
-            )
-        };
-
        Serial {
            id,
-            interrupt_enable,
-            interrupt_identification,
-            line_control,
-            line_status,
-            modem_control,
-            modem_status,
-            scratch,
-            baud_divisor,
-            in_buffer,
+            interrupt_enable: 0,
+            interrupt_identification: DEFAULT_INTERRUPT_IDENTIFICATION,
            interrupt,
+            line_control: DEFAULT_LINE_CONTROL,
+            line_status: DEFAULT_LINE_STATUS,
+            modem_control: DEFAULT_MODEM_CONTROL,
+            modem_status: DEFAULT_MODEM_STATUS,
+            scratch: 0,
+            baud_divisor: DEFAULT_BAUD_DIVISOR,
+            in_buffer: VecDeque::new(),
            out,
        }
    }
@@ -152,18 +115,13 @@ impl Serial {
        id: String,
        interrupt: Arc<dyn InterruptSourceGroup>,
        out: Box<dyn io::Write + Send>,
-        state: Option<SerialState>,
    ) -> Serial {
-        Self::new(id, interrupt, Some(out), state)
+        Self::new(id, interrupt, Some(out))
    }

    /// Constructs a Serial port with no connected output.
-    pub fn new_sink(
-        id: String,
-        interrupt: Arc<dyn InterruptSourceGroup>,
-        state: Option<SerialState>,
-    ) -> Serial {
-        Self::new(id, interrupt, None, state)
+    pub fn new_sink(id: String, interrupt: Arc<dyn InterruptSourceGroup>) -> Serial {
+        Self::new(id, interrupt, None)
    }

    pub fn set_out(&mut self, out: Box<dyn io::Write + Send>) {
@@ -284,6 +242,18 @@ impl Serial {
            in_buffer: self.in_buffer.clone().into(),
        }
    }
+
+    fn set_state(&mut self, state: &SerialState) {
+        self.interrupt_enable = state.interrupt_enable;
+        self.interrupt_identification = state.interrupt_identification;
+        self.line_control = state.line_control;
+        self.line_status = state.line_status;
+        self.modem_control = state.modem_control;
+        self.modem_status = state.modem_status;
+        self.scratch = state.scratch;
+        self.baud_divisor = state.baud_divisor;
+        self.in_buffer = state.in_buffer.clone().into();
+    }
 }

 impl BusDevice for Serial {
@@ -334,7 +304,12 @@ impl Snapshottable for Serial {
    }

    fn snapshot(&mut self) -> std::result::Result<Snapshot, MigratableError> {
-        Snapshot::new_from_versioned_state(&self.state())
+        Snapshot::new_from_versioned_state(&self.id, &self.state())
+    }
+
+    fn restore(&mut self, snapshot: Snapshot) -> std::result::Result<(), MigratableError> {
+        self.set_state(&snapshot.to_versioned_state(&self.id)?);
+        Ok(())
    }
 }

@@ -409,7 +384,6 @@ mod tests {
            String::from(SERIAL_NAME),
            Arc::new(TestInterrupt::new(intr_evt.try_clone().unwrap())),
            Box::new(serial_out.clone()),
-            None,
        );

        serial.write(0, DATA as u64, &[b'x', b'y']);
@@ -430,7 +404,6 @@ mod tests {
            String::from(SERIAL_NAME),
            Arc::new(TestInterrupt::new(intr_evt.try_clone().unwrap())),
            Box::new(serial_out),
-            None,
        );

        // write 1 to the interrupt event fd, so that read doesn't block in case the event fd
@@ -467,7 +440,6 @@ mod tests {
        let mut serial = Serial::new_sink(
            String::from(SERIAL_NAME),
            Arc::new(TestInterrupt::new(intr_evt.try_clone().unwrap())),
-            None,
        );

        // write 1 to the interrupt event fd, so that read doesn't block in case the event fd
@@ -490,7 +462,6 @@ mod tests {
        let mut serial = Serial::new_sink(
            String::from(SERIAL_NAME),
            Arc::new(TestInterrupt::new(intr_evt.try_clone().unwrap())),
-            None,
        );

        serial.write(0, LCR as u64, &[LCR_DLAB_BIT]);
@@ -512,7 +483,6 @@ mod tests {
        let mut serial = Serial::new_sink(
            String::from(SERIAL_NAME),
            Arc::new(TestInterrupt::new(intr_evt.try_clone().unwrap())),
-            None,
        );

        serial.write(0, MCR as u64, &[MCR_LOOP_BIT]);
@@ -539,7 +509,6 @@ mod tests {
        let mut serial = Serial::new_sink(
            String::from(SERIAL_NAME),
            Arc::new(TestInterrupt::new(intr_evt.try_clone().unwrap())),
-            None,
        );

        serial.write(0, SCR as u64, &[0x12]);
--- a/devices/src/legacy/uart_pl011.rs
+++ b/devices/src/legacy/uart_pl011.rs
@@ -122,79 +122,24 @@ impl Pl011 {
        irq: Arc<dyn InterruptSourceGroup>,
        out: Option<Box<dyn io::Write + Send>>,
        timestamp: Instant,
-        state: Option<Pl011State>,
    ) -> Self {
-        let (
-            flags,
-            lcr,
-            rsr,
-            cr,
-            dmacr,
-            debug,
-            int_enabled,
-            int_level,
-            read_fifo,
-            ilpr,
-            ibrd,
-            fbrd,
-            ifl,
-            read_count,
-            read_trigger,
-        ) = if let Some(state) = state {
-            (
-                state.flags,
-                state.lcr,
-                state.rsr,
-                state.cr,
-                state.dmacr,
-                state.debug,
-                state.int_enabled,
-                state.int_level,
-                state.read_fifo.into(),
-                state.ilpr,
-                state.ibrd,
-                state.fbrd,
-                state.ifl,
-                state.read_count,
-                state.read_trigger,
-            )
-        } else {
-            (
-                0x90,
-                0,
-                0,
-                0x300,
-                0,
-                0,
-                0,
-                0,
-                VecDeque::new(),
-                0,
-                0,
-                0,
-                0x12,
-                0,
-                1,
-            )
-        };
-
        Self {
            id,
-            flags,
-            lcr,
-            rsr,
-            cr,
-            dmacr,
-            debug,
-            int_enabled,
-            int_level,
-            read_fifo,
-            ilpr,
-            ibrd,
-            fbrd,
-            ifl,
-            read_count,
-            read_trigger,
+            flags: 0x90u32,
+            lcr: 0u32,
+            rsr: 0u32,
+            cr: 0x300u32,
+            dmacr: 0u32,
+            debug: 0u32,
+            int_enabled: 0u32,
+            int_level: 0u32,
+            read_fifo: VecDeque::new(),
+            ilpr: 0u32,
+            ibrd: 0u32,
+            fbrd: 0u32,
+            ifl: 0x12u32,
+            read_count: 0u32,
+            read_trigger: 1u32,
            irq,
            out,
            timestamp,
@@ -225,6 +170,24 @@ impl Pl011 {
        }
    }

+    fn set_state(&mut self, state: &Pl011State) {
+        self.flags = state.flags;
+        self.lcr = state.lcr;
+        self.rsr = state.rsr;
+        self.cr = state.cr;
+        self.dmacr = state.dmacr;
+        self.debug = state.debug;
+        self.int_enabled = state.int_enabled;
+        self.int_level = state.int_level;
+        self.read_fifo = state.read_fifo.clone().into();
+        self.ilpr = state.ilpr;
+        self.ibrd = state.ibrd;
+        self.fbrd = state.fbrd;
+        self.ifl = state.ifl;
+        self.read_count = state.read_count;
+        self.read_trigger = state.read_trigger;
+    }
+
    /// Queues raw bytes for the guest to read and signals the interrupt
    pub fn queue_input_bytes(&mut self, c: &[u8]) -> vmm_sys_util::errno::Result<()> {
        self.read_fifo.extend(c);
@@ -454,7 +417,12 @@ impl Snapshottable for Pl011 {
    }

    fn snapshot(&mut self) -> std::result::Result<Snapshot, MigratableError> {
-        Snapshot::new_from_versioned_state(&self.state())
+        Snapshot::new_from_versioned_state(&self.id, &self.state())
+    }
+
+    fn restore(&mut self, snapshot: Snapshot) -> std::result::Result<(), MigratableError> {
+        self.set_state(&snapshot.to_versioned_state(&self.id)?);
+        Ok(())
    }
 }

@@ -530,13 +498,12 @@ mod tests {
            Arc::new(TestInterrupt::new(intr_evt.try_clone().unwrap())),
            Some(Box::new(pl011_out.clone())),
            Instant::now(),
-            None,
        );

-        pl011.write(0, UARTDR, &[b'x', b'y']);
-        pl011.write(0, UARTDR, &[b'a']);
-        pl011.write(0, UARTDR, &[b'b']);
-        pl011.write(0, UARTDR, &[b'c']);
+        pl011.write(0, UARTDR as u64, &[b'x', b'y']);
+        pl011.write(0, UARTDR as u64, &[b'a']);
+        pl011.write(0, UARTDR as u64, &[b'b']);
+        pl011.write(0, UARTDR as u64, &[b'c']);
        assert_eq!(
            pl011_out.buf.lock().unwrap().as_slice(),
            &[b'x', b'a', b'b', b'c']
@@ -552,7 +519,6 @@ mod tests {
            Arc::new(TestInterrupt::new(intr_evt.try_clone().unwrap())),
            Some(Box::new(pl011_out)),
            Instant::now(),
-            None,
        );

        // write 1 to the interrupt event fd, so that read doesn't block in case the event fd
@@ -563,11 +529,11 @@ mod tests {
        assert_eq!(intr_evt.read().unwrap(), 2);

        let mut data = [0u8];
-        pl011.read(0, UARTDR, &mut data);
+        pl011.read(0, UARTDR as u64, &mut data);
        assert_eq!(data[0], b'a');
-        pl011.read(0, UARTDR, &mut data);
+        pl011.read(0, UARTDR as u64, &mut data);
        assert_eq!(data[0], b'b');
-        pl011.read(0, UARTDR, &mut data);
+        pl011.read(0, UARTDR as u64, &mut data);
        assert_eq!(data[0], b'c');
    }
 }
--- a/devices/src/lib.rs
+++ b/devices/src/lib.rs
@@ -33,9 +33,11 @@ bitflags! {
    }
 }

+#[allow(unused_macros)]
 #[cfg(target_arch = "aarch64")]
 macro_rules! generate_read_fn {
    ($fn_name: ident, $data_type: ty, $byte_type: ty, $type_size: expr, $endian_type: ident) => {
+        #[allow(dead_code)]
        pub fn $fn_name(input: &[$byte_type]) -> $data_type {
            assert!($type_size == std::mem::size_of::<$data_type>());
            let mut array = [0u8; $type_size];
@@ -47,9 +49,11 @@ macro_rules! generate_read_fn {
    };
 }

+#[allow(unused_macros)]
 #[cfg(target_arch = "aarch64")]
 macro_rules! generate_write_fn {
    ($fn_name: ident, $data_type: ty, $byte_type: ty, $endian_type: ident) => {
+        #[allow(dead_code)]
        pub fn $fn_name(buf: &mut [$byte_type], n: $data_type) {
            for (byte, read) in buf
                .iter_mut()
--- a/docs/arm64.md
+++ b/docs/arm64.md
@@ -0,0 +1,152 @@
+# How to build and test Cloud Hypervisor on AArch64
+
+This document introduces how to build and test Cloud Hypervisor on AArch64.
+Currently, Cloud Hypervisor supports 2 methods of booting on AArch64: UEFI
+booting and direct-kernel booting. The document covers both methods.
+
+All the steps are based on Ubuntu. We use the Ubuntu cloud image for guest VM
+disk.
+
+## Hardware requirements
+
+- AArch64 servers (recommended) or development boards equipped with the GICv3
+interrupt controller.
+
+- On development boards that have constrained RAM resources, if the creation of
+a VM consumes a large portion of the free memory on the host, it may be required
+to enable swap. For example, this was required on a board with 3 GB of RAM
+booting a 2 GB VM at a point in time when 2.8 GB were free. Without enabling
+swap the `cloud-hypervisor` process was terminated by the OOM killer. In this
+situation memory was allocated for the virtual machine using memfd while the
+page cache was filled, leading to a situation where the kernel could not even
+drop caches. Making a small section of swap available (observably, 1 to 15 MB),
+this situation can be resolved and the resulting memory footprint of
+`cloud-hypervisor` is as expected.
+
+## Getting started
+
+We create a folder to build and run Cloud Hypervisor at `$HOME/cloud-hypervisor`
+
+```shell
+$ export CLOUDH=$HOME/cloud-hypervisor
+$ mkdir $CLOUDH
+```
+
+## Prerequisites
+
+You need to install some prerequisite packages to build and test Cloud Hypervisor.
+
+### Tools
+
+```bash
+# Install rust tool chain
+$ curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh
+# Install the tools used for building guest kernel, EDK2 and converting guest disk
+$ sudo apt-get update
+$ sudo apt-get install git build-essential m4 bison flex uuid-dev qemu-utils
+```
+
+### Building Cloud Hypervisor
+
+```bash
+$ pushd $CLOUDH
+$ git clone https://github.com/cloud-hypervisor/cloud-hypervisor.git
+$ cd cloud-hypervisor
+$ cargo build
+$ popd
+```
+
+### Disk image
+
+Download the Ubuntu cloud image and convert the image type.
+
+```bash
+$ pushd $CLOUDH
+$ wget https://cloud-images.ubuntu.com/focal/current/focal-server-cloudimg-arm64.img
+$ qemu-img convert -p -f qcow2 -O raw focal-server-cloudimg-arm64.img focal-server-cloudimg-arm64.raw
+$ popd
+```
+
+## UEFI booting
+
+This part introduces how to build EDK2 firmware and boot Cloud Hypervisor with it.
+
+### Building EDK2
+
+```bash
+$ pushd $CLOUDH
+
+# Clone source code repos
+$ git clone --depth 1 https://github.com/tianocore/edk2.git -b master
+$ cd edk2
+$ git submodule update --init
+$ cd ..
+$ git clone --depth 1 https://github.com/tianocore/edk2-platforms.git -b master
+$ git clone --depth 1 https://github.com/acpica/acpica.git -b master
+
+# Build tools
+$ export PACKAGES_PATH="$PWD/edk2:$PWD/edk2-platforms"
+$ export IASL_PREFIX="$PWD/acpica/generate/unix/bin/"
+$ make -C acpica
+$ cd edk2/
+$ . edksetup.sh
+$ cd ..
+$ make -C edk2/BaseTools
+
+# Build EDK2
+$ build -a AARCH64 -t GCC5 -p ArmVirtPkg/ArmVirtCloudHv.dsc -b RELEASE
+
+$ popd
+```
+
+If the build goes well, the EDK2 binary is available at
+`edk2/Build/ArmVirtCloudHv-AARCH64/RELEASE_GCC5/FV/CLOUDHV_EFI.fd`.
+
+### Booting the guest VM
+
+```bash
+$ pushd $CLOUDH
+$ sudo RUST_BACKTRACE=1 $CLOUDH/cloud-hypervisor/target/debug/cloud-hypervisor \
+           --api-socket /tmp/cloud-hypervisor.sock \
+           --kernel $CLOUDH/edk2/Build/ArmVirtCloudHv-AARCH64/RELEASE_GCC5/FV/CLOUDHV_EFI.fd \
+           --disk path=$CLOUDH/focal-server-cloudimg-arm64.raw \
+           --cpus boot=4 \
+           --memory size=4096M \
+           --net tap=,mac=12:34:56:78:90:01,ip=192.168.1.1,mask=255.255.255.0 \
+           --serial tty \
+           --console off
+$ popd
+```
+
+## Direct-kernel booting
+
+Alternativelly, you can build your own kernel for guest VM. This way, UEFI is
+not involved and ACPI cannot be enabled.
+
+### Building kernel
+
+```bash
+$ pushd $CLOUDH
+$ git clone --depth 1 "https://github.com/cloud-hypervisor/linux.git" -b ch-5.12
+$ cd linux
+$ cp $CLOUDH/cloud-hypervisor/resources/linux-config-aarch64 .config
+$ make -j `nproc`
+$ popd
+```
+
+### Booting the guest VM
+
+```bash
+$ pushd $CLOUDH
+$ sudo $CLOUDH/cloud-hypervisor/target/debug/cloud-hypervisor \
+           --api-socket /tmp/cloud-hypervisor.sock \
+           --kernel $CLOUDH/linux/arch/arm64/boot/Image \
+           --disk path=focal-server-cloudimg-arm64.raw \
+           --cmdline "keep_bootcon console=ttyAMA0 reboot=k panic=1 root=/dev/vda1 rw" \
+           --cpus boot=4 \
+           --memory size=4096M \
+           --net tap=,mac=12:34:56:78:90:01,ip=192.168.1.1,mask=255.255.255.0 \
+           --serial tty \
+           --console off
+$ popd
+```
--- a/docs/building.md
+++ b/docs/building.md
@@ -24,15 +24,12 @@ Hypervisor. Here, all the steps are based on Ubuntu, for other Linux
 distributions please replace the package manager and package name.

 ```shell
-# Install basic packages needed. For a package list targeting for more
-# functionalities for example the test, please see resources/Dockerfile.
-$ sudo apt-get update
-$ sudo apt install git build-essential m4 bison flex uuid-dev qemu-utils
+# Install build-essential, git, and qemu-utils
+$ sudo apt install git build-essential qemu-utils
 # Install rust tool chain
 $ curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh
 # If you want to build statically linked binary please add musl target
-$ rustup target add x86_64-unknown-linux-musl  # x86-64
-$ rustup target add aarch64-unknown-linux-musl # AArch64
+$ rustup target add x86_64-unknown-linux-musl
 ```

 ## Clone and build
@@ -49,8 +46,7 @@ $ cargo build --release
 $ sudo setcap cap_net_admin+ep ./target/release/cloud-hypervisor

 # If you want to build statically linked binary
-$ cargo build --release --target=x86_64-unknown-linux-musl --all  # x86-64
-$ cargo build --release --target=aarch64-unknown-linux-musl --all # AArch64
+$ cargo build --release --target=x86_64-unknown-linux-musl --all
 $ popd
 ```

--- a/docs/custom-image.md
+++ b/docs/custom-image.md
@@ -88,7 +88,7 @@ Ubuntu distributions.

 ```bash
 apt update
-apt install fio iperf iperf3 socat stress cpuid
+apt install fio iperf iperf3 socat stress
 ```

 ### Remove counterproductive packages
@@ -158,172 +158,3 @@ as we might need to update the direct kernel boot command line, replacing
 `/dev/vda1` with the appropriate partition number.

 Update all references to the previous image name to the new one.
-
-## NVIDIA image for VFIO baremetal CI
-
-Here we are going to describe how to create a cloud image that contains the
-necessary NVIDIA drivers for our VFIO baremetal CI.
-
-### Download base image
-
-We usually start from one of the custom cloud image we have previously created
-but we can use a stock cloud image as well.
-
-```bash
-wget https://cloud-hypervisor.azureedge.net/jammy-server-cloudimg-amd64-custom-20221118-1.raw
-mv jammy-server-cloudimg-amd64-custom-20221118-1.raw jammy-server-cloudimg-amd64-nvidia.raw
-```
-
-### Extend the image size
-
-The NVIDIA drivers consume lots of space, which is why we must resize the image
-before we proceed any further.
-
-```bash
-qemu-img resize jammy-server-cloudimg-amd64-nvidia.raw 5G
-```
-
-### Resize the partition
-
-We use `parted` for fixing the GPT after the image was resized, as well as for
-resizing the `Linux` partition.
-
-```bash
-sudo parted jammy-server-cloudimg-amd64-nvidia.raw
-
-(parted) print
-Warning: Not all of the space available to jammy-server-cloudimg-amd64-nvidia.raw
-appears to be used, you can fix the GPT to use all of the space (an extra 5873664
-blocks) or continue with the current setting?
-Fix/Ignore? Fix
-Model:  (file)
-Disk jammy-server-cloudimg-amd64-nvidia.raw: 5369MB
-Sector size (logical/physical): 512B/512B
-Partition Table: gpt
-Disk Flags:
-
-Number  Start   End     Size    File system  Name  Flags
-14      1049kB  5243kB  4194kB                     bios_grub
-15      5243kB  116MB   111MB   fat32              boot, esp
- 1      116MB   2361MB  2245MB  ext4
-
-(parted) resizepart 1 5369MB
-(parted) print
-Model:  (file)
-Disk jammy-server-cloudimg-amd64-nvidia.raw: 5369MB
-Sector size (logical/physical): 512B/512B
-Partition Table: gpt
-Disk Flags:
-
-Number  Start   End     Size    File system  Name  Flags
-14      1049kB  5243kB  4194kB                     bios_grub
-15      5243kB  116MB   111MB   fat32              boot, esp
- 1      116MB   5369MB  5252MB  ext4
-
-(parted) quit
-```
-
-### Create a macvtap interface
-
-Rely on the following [documentation](docs/macvtap-bridge.md) to set up a
-macvtap interface to provide your VM with proper connectivity.
-
-### Boot the image
-
-It is particularly important to boot with a `cloud-init` disk attached to the
-VM as it will automatically resize the Linux `ext4` filesystem based on the
-partition that we have previously resized.
-
-```bash
-./cloud-hypervisor \
-	--kernel hypervisor-fw  \
-	--disk path=focal-server-cloudimg-amd64-nvidia.raw path=/tmp/ubuntu-cloudinit.img \
-	--cpus boot=4 \
-	--memory size=4G \
-	--net fd=3,mac=$mac 3<>$"$tapdevice"
-```
-	
-### Bring up connectivity
-
-If your network has a DHCP server, run the following from your VM
-
-```bash
-sudo dhclient
-```
-
-But if that's not the case, let's give it an IP manually (the IP addresses
-depend on your actual network) and set the DNS server IP address as well.
-
-```bash
-sudo ip addr add 192.168.2.10/24 dev ens4
-sudo ip link set up dev ens4
-sudo ip route add default via 192.168.2.1
-sudo resolvectl dns ens4 8.8.8.8
-```
-
-#### Check connectivity and update the image
-
-```bash
-sudo apt update
-sudo apt upgrade
-```
-
-### Install NVIDIA drivers
-
-The following steps and commands are referenced from the
-[NVIDIA official documentation](https://docs.nvidia.com/datacenter/tesla/tesla-installation-notes/index.html#ubuntu-lts)
-about Tesla compute cards.
-
-```bash
-distribution=$(. /etc/os-release;echo $ID$VERSION_ID | sed -e 's/\.//g')
-wget https://developer.download.nvidia.com/compute/cuda/repos/$distribution/x86_64/cuda-keyring_1.0-1_all.deb
-sudo dpkg -i cuda-keyring_1.0-1_all.deb
-sudo apt-key del 7fa2af80
-sudo apt update
-sudo apt -y install cuda-drivers
-```
-
-### Check the `nvidia-smi` tool
-
-Quickly validate that you can find and run the `nvidia-smi` command from your
-VM. At this point it should fail given no NVIDIA card has been passed through
-the VM, therefore no NVIDIA driver is loaded.
-
-### Workaround LA57 reboot issue
-
-Add `reboot=a` to `GRUB_CMDLINE_LINUX` in `etc/default/grub` so that the VM
-will be booted with the ACPI reboot type. This resolves a reboot issue when
-running on 5-level paging systems.
-
-```bash
-sudo vim /etc/default/grub
-sudo update-grub
-sudo reboot
-```
-
-### Remove previous logins
-
-Since our integration tests rely on past logins to count the number of reboots,
-we must ensure to clear the list.
-
-```bash
->/var/log/lastlog
->/var/log/wtmp
->/var/log/btmp
-```
-
-### Clear history
-
-```
-history -c
-rm /home/cloud/.bash_history
-```
-
-### Reset cloud-init
-
-This is mandatory as we want `cloud-init` provisioning to work again when a new
-VM will be booted with this image.
-
-```
-sudo cloud-init clean
-```
--- a/docs/uefi.md
+++ b/docs/uefi.md
@@ -2,19 +2,17 @@

 Cloud Hypervisor supports UEFI boot through the utilization of the EDK II based UEFI firmware. 

-## Building UEFI Firmware for x86-64
+## Building UEFI Firmware

 To avoid any unnecessary issues, it is recommended to use Ubuntu 18.04 and its default toolset. Any other compatible Linux distribution is otherwise suitable, however it is suggested to use a temporary Docker container with Ubuntu 18.04 for a quick build on an existing Linux machine.

-Please note that nasm-2.15 is required for the build to succeed.
-
 The commands below will compile an OVMF firmware suitable for Cloud Hypervisor.

 ```shell
 sudo apt-get update
 sudo apt-get install uuid-dev nasm iasl build-essential python3-distutils git

-git clone https://github.com/tianocore/edk2
+git clone https://github.com/cloud-hypervisor/edk2 -b ch
 cd edk2
 . edksetup.sh
 git submodule update --init
@@ -29,40 +27,11 @@ build

 After the successful build, the resulting firmware binaries are available under `Build/CloudHvX64/DEBUG_GCC5/FV` underneath the edk2 checkout.

-## Building UEFI Firmware for AArch64
-
-```shell
-# On an AArch64 machine:
-$ sudo apt-get update
-$ sudo apt-get install uuid-dev nasm iasl build-essential python3-distutils git
-$ git clone --depth 1 https://github.com/tianocore/edk2.git -b master
-$ cd edk2
-$ git submodule update --init
-$ cd ..
-$ git clone --depth 1 https://github.com/tianocore/edk2-platforms.git -b master
-$ git clone --depth 1 https://github.com/acpica/acpica.git -b master
-
-# Build tools
-$ export PACKAGES_PATH="$PWD/edk2:$PWD/edk2-platforms"
-$ export IASL_PREFIX="$PWD/acpica/generate/unix/bin/"
-$ make -C acpica
-$ cd edk2/
-$ . edksetup.sh
-$ cd ..
-$ make -C edk2/BaseTools
-
-# Build EDK2
-$ build -a AARCH64 -t GCC5 -p ArmVirtPkg/ArmVirtCloudHv.dsc -b RELEASE
-```
-
-If the build goes well, the EDK2 binary is available at
-`edk2/Build/ArmVirtCloudHv-AARCH64/RELEASE_GCC5/FV/CLOUDHV_EFI.fd`.
-
 ## Using OVMF Binaries

 Any UEFI capable image can be booted using the Cloud Hypervisor specific firmware. Windows guests under Cloud Hypervisor only support UEFI boot, therefore OVMF is mandatory there.

-To make Cloud Hypervisor use UEFI boot, pass the `CLOUDHV.fd` (for x86-64) / `CLOUDHV_EFI.fd` (for AArch64) file path as an argument to the `--kernel` option. The firmware file will be opened in read only mode.
+To make Cloud Hypervisor use UEFI boot, pass the `CLOUDHV.fd` file path as an argument to the `--kernel` option. The firmware file will be opened in read only mode.

 # Links

--- a/event_monitor/Cargo.toml
+++ b/event_monitor/Cargo.toml
@@ -5,6 +5,7 @@ authors = ["The Cloud Hypervisor Authors"]
 edition = "2021"

 [dependencies]
-libc = "0.2.139"
-serde = { version = "1.0.151", features = ["rc", "derive"] }
+libc = "0.2.138"
+once_cell = "1.16.0"
+serde = { version = "1.0.150", features = ["rc", "derive"] }
 serde_json = "1.0.89"
--- a/event_monitor/src/lib.rs
+++ b/event_monitor/src/lib.rs
@@ -3,6 +3,7 @@
 // SPDX-License-Identifier: Apache-2.0
 //

+use once_cell::sync::OnceCell;
 use serde::Serialize;
 use std::borrow::Cow;
 use std::collections::HashMap;
@@ -11,15 +12,14 @@ use std::io::Write;
 use std::os::unix::io::AsRawFd;
 use std::time::{Duration, Instant};

-static mut MONITOR: Option<(File, Instant)> = None;
+static MONITOR: OnceCell<(File, Instant)> = OnceCell::new();

 /// This function must only be called once from the main process before any threads
 /// are created to avoid race conditions
 pub fn set_monitor(file: File) -> Result<(), std::io::Error> {
-    // SAFETY: there is only one caller of this function, so MONITOR is written to only once
-    assert!(unsafe { MONITOR.is_none() });
+    // There is only one caller of this function, so MONITOR is written to only once
+    assert!(MONITOR.get().is_none());
    let fd = file.as_raw_fd();
-    // SAFETY: FFI call to configure the fd
    let ret = unsafe {
        let mut flags = libc::fcntl(fd, libc::F_GETFL);
        flags |= libc::O_NONBLOCK;
@@ -28,10 +28,9 @@ pub fn set_monitor(file: File) -> Result<(), std::io::Error> {
    if ret < 0 {
        return Err(std::io::Error::last_os_error());
    }
-    // SAFETY: MONITOR is None. Nobody else can hold a reference to it.
-    unsafe {
-        MONITOR = Some((file, Instant::now()));
-    };
+
+    MONITOR.get_or_init(|| (file, Instant::now()));
+
    Ok(())
 }

@@ -44,8 +43,7 @@ struct Event<'a> {
 }

 pub fn event_log(source: &str, event: &str, properties: Option<&HashMap<Cow<str>, Cow<str>>>) {
-    // SAFETY: MONITOR is always in a valid state (None or Some).
-    if let Some((file, start)) = unsafe { MONITOR.as_ref() } {
+    if let Some((file, start)) = MONITOR.get().as_ref() {
        let e = Event {
            timestamp: start.elapsed(),
            source,
--- a/fuzz/Cargo.lock
+++ b/fuzz/Cargo.lock
@@ -11,9 +11,9 @@ dependencies = [

 [[package]]
 name = "anyhow"
-version = "1.0.68"
+version = "1.0.66"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2cb2f989d18dd141ab8ae82f64d1a8cdd37e0840f73a406896cf5e99502fab61"
+checksum = "216261ddc8289130e551ddcd5ce8a064710c0d064a4d2895c67151c92b5443f6"

 [[package]]
 name = "api_client"
@@ -71,6 +71,12 @@ version = "1.3.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"

+[[package]]
+name = "bitflags"
+version = "2.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "327762f6e5a765692301e5bb513e0d9fef63be86bbc14528052b1cd3e6f03e07"
+
 [[package]]
 name = "block_util"
 version = "0.1.0"
@@ -79,7 +85,6 @@ dependencies = [
 "libc",
 "log",
 "qcow",
- "smallvec",
 "thiserror",
 "versionize",
 "versionize_derive",
@@ -99,9 +104,9 @@ checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610"

 [[package]]
 name = "cc"
-version = "1.0.78"
+version = "1.0.77"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a20104e2335ce8a659d6dd92a51a767a0c062599c73b343fd152cb401e828c3d"
+checksum = "e9f73505338f7d905b19d18738976aae232eb46b8efc15554ffc56deb5d9ebe4"
 dependencies = [
 "jobserver",
 ]
@@ -114,11 +119,11 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"

 [[package]]
 name = "clap"
-version = "4.0.32"
+version = "4.0.29"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a7db700bc935f9e43e88d00b0850dae18a63773cfbec6d8e070fccf7fef89a39"
+checksum = "4d63b9e9c07271b9957ad22c173bae2a4d9a81127680962039296abcd2f8251d"
 dependencies = [
- "bitflags",
+ "bitflags 1.3.2",
 "clap_lex",
 "is-terminal",
 "once_cell",
@@ -138,7 +143,7 @@ dependencies = [

 [[package]]
 name = "cloud-hypervisor"
-version = "28.0.0"
+version = "28.3.0"
 dependencies = [
 "anyhow",
 "api_client",
@@ -170,9 +175,7 @@ dependencies = [
 "epoll",
 "libc",
 "libfuzzer-sys",
- "linux-loader",
 "micro_http",
- "net_util",
 "once_cell",
 "qcow",
 "seccompiler",
@@ -243,7 +246,7 @@ dependencies = [
 "acpi_tables",
 "anyhow",
 "arch",
- "bitflags",
+ "bitflags 2.4.1",
 "byteorder",
 "hypervisor",
 "libc",
@@ -265,7 +268,7 @@ version = "4.3.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "20df693c700404f7e19d4d6fae6b15215d2913c27955d2b9d6f2c0f537511cd0"
 dependencies = [
- "bitflags",
+ "bitflags 1.3.2",
 "libc",
 ]

@@ -295,6 +298,7 @@ name = "event_monitor"
 version = "0.1.0"
 dependencies = [
 "libc",
+ "once_cell",
 "serde",
 "serde_json",
 ]
@@ -352,11 +356,12 @@ dependencies = [

 [[package]]
 name = "iced-x86"
-version = "1.18.0"
+version = "1.17.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1dd04b950d75b3498320253b17fb92745b2cc79ead8814aede2f7c1bab858bec"
+checksum = "158f5204401d08f91d19176112146d75e99b3cf745092e268fa7be33e09adcec"
 dependencies = [
 "lazy_static",
+ "static_assertions",
 ]

 [[package]]
@@ -377,19 +382,19 @@ dependencies = [

 [[package]]
 name = "io-uring"
-version = "0.5.11"
+version = "0.5.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c9ddf18110cedc09617f5a965bdb51adb0c0fb02c791e4ab22f7a21bb25269b0"
+checksum = "7ba34abb5175052fc1a2227a10d2275b7386c9990167de9786c0b88d8b062330"
 dependencies = [
- "bitflags",
+ "bitflags 1.3.2",
 "libc",
 ]

 [[package]]
 name = "is-terminal"
-version = "0.4.2"
+version = "0.4.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "28dfb6c8100ccc63462345b67d1bbc3679177c75ee4bf59bf29c8b1d110b8189"
+checksum = "927609f78c2913a6f6ac3c27a4fe87f43e2a35367c0c4b0f8265e8f49a104330"
 dependencies = [
 "hermit-abi",
 "io-lifetimes",
@@ -399,9 +404,9 @@ dependencies = [

 [[package]]
 name = "itoa"
-version = "1.0.5"
+version = "1.0.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fad582f4b9e86b6caa621cabeb0963332d92eea04729ab12892c2533951e6440"
+checksum = "4217ad341ebadf8d8e724e264f13e593e0648f5b3e94b3896a5df283be015ecc"

 [[package]]
 name = "jobserver"
@@ -424,9 +429,9 @@ dependencies = [

 [[package]]
 name = "kvm-ioctls"
-version = "0.12.0"
+version = "0.13.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c3a321cabd827642499c77e27314f388dd83a717a5ca716b86476fb947f73ae4"
+checksum = "b8f8dc9c1896e5f144ec5d07169bc29f39a047686d29585a91f30489abfaeb6b"
 dependencies = [
 "kvm-bindings",
 "libc",
@@ -441,9 +446,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"

 [[package]]
 name = "libc"
-version = "0.2.139"
+version = "0.2.138"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "201de327520df007757c1f0adce6e827fe8562fbc28bfd9c15571c66ca1f5f79"
+checksum = "db6d7e329c562c5dfab7a46a2afabc8b987ab9a4834c9d1ca04dc54c1546cef8"

 [[package]]
 name = "libfuzzer-sys"
@@ -467,9 +472,9 @@ dependencies = [

 [[package]]
 name = "linux-raw-sys"
-version = "0.1.4"
+version = "0.1.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f051f77a7c8e6957c0696eac88f26b0117e54f52d3fc682ab19397a8812846a4"
+checksum = "8f9f08d8963a6c613f4b1a78f4f4a4dbfadf8e6545b2d72861731e4858b8b47f"

 [[package]]
 name = "log"
@@ -483,7 +488,7 @@ dependencies = [
 [[package]]
 name = "micro_http"
 version = "0.1.0"
-source = "git+https://github.com/firecracker-microvm/micro-http?branch=main#fbef706e28c7ca7cce8dfec7ca073f3b359879d3"
+source = "git+https://github.com/firecracker-microvm/micro-http?branch=main#4b18a043e997da5b5f679e3defc279fec908753e"
 dependencies = [
 "libc",
 "vmm-sys-util",
@@ -519,9 +524,9 @@ dependencies = [

 [[package]]
 name = "once_cell"
-version = "1.17.0"
+version = "1.16.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6f61fba1741ea2b3d6a1e3178721804bb716a68a6aeba1149b5d52e3d464ea66"
+checksum = "86f0b0d4bf799edbc74508c1e8bf170ff5f41238e5f8225603ca7caaae2b7860"

 [[package]]
 name = "option_parser"
@@ -600,9 +605,9 @@ dependencies = [

 [[package]]
 name = "proc-macro2"
-version = "1.0.49"
+version = "1.0.76"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "57a8eca9f9c4ffde41714334dee777596264c7825420f521abc92b5b5deb63a5"
+checksum = "95fc56cda0b5c3325f5fbbd7ff9fda9e02bb00bb3dac51252d2f1bfa1cb8cc8c"
 dependencies = [
 "unicode-ident",
 ]
@@ -653,9 +658,9 @@ dependencies = [

 [[package]]
 name = "remain"
-version = "0.2.6"
+version = "0.2.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5704e2cda92fd54202f05430725317ba0ea7d0c96b246ca0a92e45177127ba3b"
+checksum = "1a81bc6a3aaeb180f767bd2bda8c03bac5b93b3b69a783c66f35d16a4df276cf"
 dependencies = [
 "proc-macro2",
 "quote",
@@ -677,7 +682,7 @@ version = "0.36.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a3807b5d10909833d3e9acd1eb5fb988f79376ff10fce42937de71a449c4c588"
 dependencies = [
- "bitflags",
+ "bitflags 1.3.2",
 "errno",
 "io-lifetimes",
 "libc",
@@ -687,9 +692,9 @@ dependencies = [

 [[package]]
 name = "ryu"
-version = "1.0.12"
+version = "1.0.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7b4b9743ed687d4b4bcedf9ff5eaa7398495ae14e61cba0a295704edbc7decde"
+checksum = "4501abdff3ae82a1c1b477a17252eb69cee9e66eb915c1abaa4f44d873df9f09"

 [[package]]
 name = "seccompiler"
@@ -702,24 +707,24 @@ dependencies = [

 [[package]]
 name = "semver"
-version = "1.0.16"
+version = "1.0.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "58bc9567378fc7690d6b2addae4e60ac2eeea07becb2c64b9f218b53865cba2a"
+checksum = "e25dfac463d778e353db5be2449d1cce89bd6fd23c9f1ea21310ce6e5a1b29c4"

 [[package]]
 name = "serde"
-version = "1.0.152"
+version = "1.0.150"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bb7d1f0d3021d347a83e556fc4683dea2ea09d87bccdf88ff5c12545d89d5efb"
+checksum = "e326c9ec8042f1b5da33252c8a37e9ffbd2c9bef0155215b6e6c80c790e05f91"
 dependencies = [
 "serde_derive",
 ]

 [[package]]
 name = "serde_derive"
-version = "1.0.152"
+version = "1.0.150"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "af487d118eecd09402d70a5d72551860e788df87b464af30e5ea6a38c75c541e"
+checksum = "42a3df25b0713732468deadad63ab9da1f1fd75a48a15024b50363f128db627e"
 dependencies = [
 "proc-macro2",
 "quote",
@@ -728,9 +733,9 @@ dependencies = [

 [[package]]
 name = "serde_json"
-version = "1.0.91"
+version = "1.0.89"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "877c235533714907a8c2464236f5c4b2a17262ef1bd71f38f35ea592c8da6883"
+checksum = "020ff22c755c2ed3f8cf162dbb41a7268d934702f3ed3631656ea597e08fc3db"
 dependencies = [
 "itoa",
 "ryu",
@@ -749,9 +754,9 @@ dependencies = [

 [[package]]
 name = "serde_with_macros"
-version = "2.2.0"
+version = "2.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a1966009f3c05f095697c537312f5415d1e3ed31ce0a56942bac4c771c5c335e"
+checksum = "e3452b4c0f6c1e357f73fdb87cd1efabaa12acf328c7a528e252893baeb3f4aa"
 dependencies = [
 "darling",
 "proc-macro2",
@@ -789,10 +794,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7bd3e3206899af3f8b12af284fafc038cc1dc2b41d1b89dd17297221c5d225de"

 [[package]]
-name = "smallvec"
-version = "1.10.0"
+name = "static_assertions"
+version = "1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0"
+checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f"

 [[package]]
 name = "strsim"
@@ -802,9 +807,9 @@ checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623"

 [[package]]
 name = "syn"
-version = "1.0.107"
+version = "1.0.105"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1f4064b5b16e03ae50984a5a8ed5d4f8803e6bc1fd170a3cda91a1be4b18e3f5"
+checksum = "60b9b43d45702de4c839cb9b51d9f529c5dd26a4aff255b42b1ebc03e88ee908"
 dependencies = [
 "proc-macro2",
 "quote",
@@ -875,9 +880,9 @@ dependencies = [

 [[package]]
 name = "unicode-ident"
-version = "1.0.6"
+version = "1.0.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "84a22b9f218b40614adcb3f4ff08b703773ad44fa9423e4e0d346d5db86e4ebc"
+checksum = "6ceab39d59e4c9499d4e5a8ee0e2735b891bb7308ac83dfb4e80cad195c9f6f3"

 [[package]]
 name = "uuid"
@@ -918,7 +923,7 @@ dependencies = [
 [[package]]
 name = "vfio-bindings"
 version = "0.4.0"
-source = "git+https://github.com/rust-vmm/vfio?branch=main#7ef33f6759ae6d37a54cd66f749aa47c1c81509c"
+source = "git+https://github.com/rust-vmm/vfio?branch=main#43439e056ddfa84a4f7906ee7f2f58be70505c08"
 dependencies = [
 "vmm-sys-util",
 ]
@@ -926,7 +931,7 @@ dependencies = [
 [[package]]
 name = "vfio-ioctls"
 version = "0.2.0"
-source = "git+https://github.com/rust-vmm/vfio?branch=main#7ef33f6759ae6d37a54cd66f749aa47c1c81509c"
+source = "git+https://github.com/rust-vmm/vfio?branch=main#43439e056ddfa84a4f7906ee7f2f58be70505c08"
 dependencies = [
 "byteorder",
 "kvm-bindings",
@@ -973,7 +978,7 @@ version = "0.6.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c9b791c5b0717a0558888a4cf7240cea836f39a99cb342e12ce633dcaa078072"
 dependencies = [
- "bitflags",
+ "bitflags 1.3.2",
 "libc",
 "vm-memory",
 "vmm-sys-util",
@@ -1100,7 +1105,7 @@ dependencies = [
 "anyhow",
 "arc-swap",
 "arch",
- "bitflags",
+ "bitflags 2.4.1",
 "block_util",
 "clap",
 "devices",
@@ -1145,7 +1150,7 @@ version = "0.11.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "cc06a16ee8ebf0d9269aed304030b0d20a866b8b3dd3d4ce532596ac567a0d24"
 dependencies = [
- "bitflags",
+ "bitflags 1.3.2",
 "libc",
 "serde",
 "serde_derive",
--- a/fuzz/Cargo.toml
+++ b/fuzz/Cargo.toml
@@ -12,12 +12,10 @@ cargo-fuzz = true
 block_util = { path = "../block_util" }
 devices = { path = "../devices" }
 epoll = "4.3.1"
-libc = "0.2.138"
+libc = "0.2.135"
 libfuzzer-sys = "0.4.5"
-linux-loader = { version = "0.8.1", features = ["elf", "bzimage", "pe"] }
 micro_http = { git = "https://github.com/firecracker-microvm/micro-http", branch = "main" }
-net_util = { path = "../net_util" }
-once_cell = "1.17.0"
+once_cell = "1.16.0"
 qcow = { path = "../qcow" }
 seccompiler = "0.3.0"
 vhdx = { path = "../vhdx" }
@@ -76,30 +74,12 @@ path = "fuzz_targets/iommu.rs"
 test = false
 doc = false

-[[bin]]
-name = "linux_loader"
-path = "fuzz_targets/linux_loader.rs"
-test = false
-doc = false
-
-[[bin]]
-name = "linux_loader_cmdline"
-path = "fuzz_targets/linux_loader_cmdline.rs"
-test = false
-doc = false
-
 [[bin]]
 name = "mem"
 path = "fuzz_targets/mem.rs"
 test = false
 doc = false

-[[bin]]
-name = "net"
-path = "fuzz_targets/net.rs"
-test = false
-doc = false
-
 [[bin]]
 name = "pmem"
 path = "fuzz_targets/pmem.rs"
--- a/fuzz/fuzz_targets/linux_loader.rs
+++ b/fuzz/fuzz_targets/linux_loader.rs
@@ -1,50 +0,0 @@
-// Copyright 2018 The Chromium OS Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file.
-//
-// Copyright © 2022 Intel Corporation
-//
-// SPDX-License-Identifier: Apache-2.0 AND BSD-3-Clause
-
-#![no_main]
-
-use libfuzzer_sys::fuzz_target;
-use linux_loader::loader::KernelLoader;
-use std::ffi;
-use std::fs::File;
-use std::io;
-use std::io::{Seek, SeekFrom, Write};
-use std::os::unix::io::{FromRawFd, RawFd};
-use vm_memory::{bitmap::AtomicBitmap, GuestAddress};
-
-type GuestMemoryMmap = vm_memory::GuestMemoryMmap<AtomicBitmap>;
-
-const MEM_SIZE: usize = 256 * 1024 * 1024;
-// From 'arch::x86_64::layout::HIGH_RAM_START'
-const HIGH_RAM_START: GuestAddress = GuestAddress(0x100000);
-
-fuzz_target!(|bytes| {
-    let shm = memfd_create(&ffi::CString::new("fuzz_load_kernel").unwrap(), 0).unwrap();
-    let mut kernel_file: File = unsafe { File::from_raw_fd(shm) };
-    kernel_file.write_all(&bytes).unwrap();
-    kernel_file.seek(SeekFrom::Start(0)).unwrap();
-
-    let guest_memory = GuestMemoryMmap::from_ranges(&[(GuestAddress(0), MEM_SIZE)]).unwrap();
-    linux_loader::loader::elf::Elf::load(
-        &guest_memory,
-        None,
-        &mut kernel_file,
-        Some(HIGH_RAM_START),
-    )
-    .ok();
-});
-
-fn memfd_create(name: &ffi::CStr, flags: u32) -> Result<RawFd, io::Error> {
-    let res = unsafe { libc::syscall(libc::SYS_memfd_create, name.as_ptr(), flags) };
-
-    if res < 0 {
-        Err(io::Error::last_os_error())
-    } else {
-        Ok(res as RawFd)
-    }
-}
--- a/fuzz/fuzz_targets/linux_loader_cmdline.rs
+++ b/fuzz/fuzz_targets/linux_loader_cmdline.rs
@@ -1,34 +0,0 @@
-// Copyright 2018 The Chromium OS Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file.
-//
-// Copyright © 2022 Intel Corporation
-//
-// SPDX-License-Identifier: Apache-2.0 AND BSD-3-Clause
-
-#![no_main]
-
-use libfuzzer_sys::fuzz_target;
-use vm_memory::{bitmap::AtomicBitmap, GuestAddress};
-
-type GuestMemoryMmap = vm_memory::GuestMemoryMmap<AtomicBitmap>;
-
-const MEM_SIZE: usize = 256 * 1024 * 1024;
-// From 'arch::x86_64::layout::CMDLINE_START'
-const CMDLINE_START: GuestAddress = GuestAddress(0x20000);
-
-fuzz_target!(|bytes| {
-    let payload_config = vmm::config::PayloadConfig {
-        firmware: None,
-        kernel: None,
-        cmdline: Some(String::from_utf8_lossy(&bytes).to_string()),
-        initramfs: None,
-    };
-    let kernel_cmdline = match vmm::vm::Vm::generate_cmdline(&payload_config) {
-        Ok(cmdline) => cmdline,
-        _ => return,
-    };
-    let guest_memory = GuestMemoryMmap::from_ranges(&[(GuestAddress(0), MEM_SIZE)]).unwrap();
-
-    linux_loader::loader::load_cmdline(&guest_memory, CMDLINE_START, &kernel_cmdline).ok();
-});
--- a/fuzz/fuzz_targets/net.rs
+++ b/fuzz/fuzz_targets/net.rs
@@ -1,285 +0,0 @@
-// Copyright © 2022 Intel Corporation
-//
-// SPDX-License-Identifier: Apache-2.0
-
-#![no_main]
-
-use libfuzzer_sys::fuzz_target;
-use seccompiler::SeccompAction;
-use std::fs::File;
-use std::io::{Read, Write};
-use std::os::unix::io::{AsRawFd, FromRawFd};
-use std::sync::Arc;
-use virtio_devices::{VirtioDevice, VirtioInterrupt, VirtioInterruptType};
-use virtio_queue::{Queue, QueueT};
-use vm_memory::{bitmap::AtomicBitmap, Bytes, GuestAddress, GuestMemoryAtomic};
-use vmm::EpollContext;
-use vmm_sys_util::eventfd::{EventFd, EFD_NONBLOCK};
-
-type GuestMemoryMmap = vm_memory::GuestMemoryMmap<AtomicBitmap>;
-
-macro_rules! align {
-    ($n:expr, $align:expr) => {{
-        (($n + $align - 1) / $align) * $align
-    }};
-}
-
-const TAP_INPUT_SIZE: usize = 128;
-const QUEUE_DATA_SIZE: usize = 4;
-const MEM_SIZE: usize = 32 * 1024 * 1024;
-// Guest memory gap
-const GUEST_MEM_GAP: u64 = 1 * 1024 * 1024;
-// Guest physical address for the first virt queue
-const BASE_VIRT_QUEUE_ADDR: u64 = MEM_SIZE as u64 + GUEST_MEM_GAP;
-// Number of queues
-const QUEUE_NUM: usize = 2;
-// Max entries in the queue.
-const QUEUE_SIZE: u16 = 256;
-// Descriptor table alignment
-const DESC_TABLE_ALIGN_SIZE: u64 = 16;
-// Used ring alignment
-const USED_RING_ALIGN_SIZE: u64 = 4;
-// Descriptor table size
-const DESC_TABLE_SIZE: u64 = 16_u64 * QUEUE_SIZE as u64;
-// Available ring size
-const AVAIL_RING_SIZE: u64 = 6_u64 + 2 * QUEUE_SIZE as u64;
-// Padding size before used ring
-const PADDING_SIZE: u64 = align!(AVAIL_RING_SIZE, USED_RING_ALIGN_SIZE) - AVAIL_RING_SIZE;
-// Used ring size
-const USED_RING_SIZE: u64 = 6_u64 + 8 * QUEUE_SIZE as u64;
-// Virtio-queue size in bytes
-const QUEUE_BYTES_SIZE: usize = align!(
-    DESC_TABLE_SIZE + AVAIL_RING_SIZE + PADDING_SIZE + USED_RING_SIZE,
-    DESC_TABLE_ALIGN_SIZE
-) as usize;
-
-fuzz_target!(|bytes| {
-    if bytes.len() < TAP_INPUT_SIZE + (QUEUE_DATA_SIZE + QUEUE_BYTES_SIZE) * QUEUE_NUM
-        || bytes.len()
-            > TAP_INPUT_SIZE + (QUEUE_DATA_SIZE + QUEUE_BYTES_SIZE) * QUEUE_NUM + MEM_SIZE
-    {
-        return;
-    }
-
-    let (dummy_tap_frontend, dummy_tap_backend) = create_socketpair().unwrap();
-    let if_name = "fuzzer_tap_name".as_bytes().to_vec();
-    let tap = net_util::Tap::new_for_fuzzing(dummy_tap_frontend, if_name);
-
-    let mut net = virtio_devices::Net::new_with_tap(
-        "fuzzer_net".to_owned(),
-        vec![tap],
-        None,  // guest_mac
-        false, // iommu
-        QUEUE_NUM,
-        QUEUE_SIZE,
-        SeccompAction::Allow,
-        None,
-        EventFd::new(EFD_NONBLOCK).unwrap(),
-        None,
-        true,
-        true,
-        true,
-    )
-    .unwrap();
-
-    let tap_input_bytes = &bytes[..TAP_INPUT_SIZE];
-    let queue_data = &bytes[TAP_INPUT_SIZE..TAP_INPUT_SIZE + QUEUE_DATA_SIZE * QUEUE_NUM];
-    let queue_bytes = &bytes[TAP_INPUT_SIZE + QUEUE_DATA_SIZE * QUEUE_NUM
-        ..TAP_INPUT_SIZE + (QUEUE_DATA_SIZE + QUEUE_BYTES_SIZE) * QUEUE_NUM];
-    let mem_bytes = &bytes[TAP_INPUT_SIZE + (QUEUE_DATA_SIZE + QUEUE_BYTES_SIZE) * QUEUE_NUM..];
-
-    // Setup the virt queues with the input bytes
-    let mut queues = setup_virt_queues(
-        &[
-            &queue_data[..QUEUE_DATA_SIZE].try_into().unwrap(),
-            &queue_data[QUEUE_DATA_SIZE..QUEUE_DATA_SIZE * 2]
-                .try_into()
-                .unwrap(),
-        ],
-        BASE_VIRT_QUEUE_ADDR,
-    );
-
-    // Setup the guest memory with the input bytes
-    let mem = GuestMemoryMmap::from_ranges(&[
-        (GuestAddress(0), MEM_SIZE),
-        (GuestAddress(BASE_VIRT_QUEUE_ADDR), queue_bytes.len()),
-    ])
-    .unwrap();
-    if mem
-        .write_slice(queue_bytes, GuestAddress(BASE_VIRT_QUEUE_ADDR))
-        .is_err()
-    {
-        return;
-    }
-    if mem.write_slice(mem_bytes, GuestAddress(0 as u64)).is_err() {
-        return;
-    }
-    let guest_memory = GuestMemoryAtomic::new(mem);
-
-    let input_queue = queues.remove(0);
-    let input_evt = EventFd::new(0).unwrap();
-    let input_queue_evt = unsafe { EventFd::from_raw_fd(libc::dup(input_evt.as_raw_fd())) };
-    let output_queue = queues.remove(0);
-    let output_evt = EventFd::new(0).unwrap();
-    let output_queue_evt = unsafe { EventFd::from_raw_fd(libc::dup(output_evt.as_raw_fd())) };
-
-    // Start the thread of dummy tap backend to handle the rx and tx from the virtio-net
-    let exit_evt = EventFd::new(libc::EFD_NONBLOCK).unwrap();
-    let tap_backend_thread = {
-        let dummy_tap_backend = dummy_tap_backend.try_clone().unwrap();
-        let tap_input_bytes: [u8; TAP_INPUT_SIZE] = tap_input_bytes[..].try_into().unwrap();
-        let exit_evt = exit_evt.try_clone().unwrap();
-        std::thread::Builder::new()
-            .name("dummy_tap_backend".to_string())
-            .spawn(move || {
-                tap_backend_stub(dummy_tap_backend, &tap_input_bytes, exit_evt);
-            })
-            .unwrap()
-    };
-
-    // Kick the 'queue' events and endpoint event before activate the net device
-    input_queue_evt.write(1).unwrap();
-    output_queue_evt.write(1).unwrap();
-
-    net.activate(
-        guest_memory,
-        Arc::new(NoopVirtioInterrupt {}),
-        vec![(0, input_queue, input_evt), (1, output_queue, output_evt)],
-    )
-    .unwrap();
-
-    // Wait for the events to finish and net device worker thread to return
-    net.wait_for_epoll_threads();
-    // Terminate the thread for the dummy tap backend
-    exit_evt.write(1).ok();
-    tap_backend_thread.join().unwrap();
-});
-
-pub struct NoopVirtioInterrupt {}
-
-impl VirtioInterrupt for NoopVirtioInterrupt {
-    fn trigger(&self, _int_type: VirtioInterruptType) -> std::result::Result<(), std::io::Error> {
-        Ok(())
-    }
-}
-
-fn setup_virt_queues(bytes: &[&[u8; QUEUE_DATA_SIZE]], base_addr: u64) -> Vec<Queue> {
-    let mut queues = Vec::new();
-    for (i, b) in bytes.iter().enumerate() {
-        let mut q = Queue::new(QUEUE_SIZE).unwrap();
-
-        let desc_table_addr = base_addr + (QUEUE_BYTES_SIZE * i) as u64;
-        let avail_ring_addr = desc_table_addr + DESC_TABLE_SIZE;
-        let used_ring_addr = avail_ring_addr + PADDING_SIZE + AVAIL_RING_SIZE;
-        q.try_set_desc_table_address(GuestAddress(desc_table_addr))
-            .unwrap();
-        q.try_set_avail_ring_address(GuestAddress(avail_ring_addr))
-            .unwrap();
-        q.try_set_used_ring_address(GuestAddress(used_ring_addr))
-            .unwrap();
-
-        q.set_next_avail(b[0] as u16); // 'u8' is enough given the 'QUEUE_SIZE' is small
-        q.set_next_used(b[1] as u16);
-        q.set_event_idx(b[2] % 2 != 0);
-        q.set_size(b[3] as u16 % QUEUE_SIZE);
-
-        q.set_ready(true);
-        queues.push(q);
-    }
-
-    queues
-}
-
-fn create_socketpair() -> Result<(File, File), std::io::Error> {
-    let mut fds = [-1, -1];
-    unsafe {
-        let ret = libc::socketpair(
-            libc::AF_UNIX,
-            libc::SOCK_STREAM | libc::SOCK_NONBLOCK,
-            0,
-            fds.as_mut_ptr(),
-        );
-        if ret == -1 {
-            return Err(std::io::Error::last_os_error());
-        }
-    }
-
-    let socket1 = unsafe { File::from_raw_fd(fds[0]) };
-    let socket2 = unsafe { File::from_raw_fd(fds[1]) };
-    Ok((socket1, socket2))
-}
-
-enum EpollEvent {
-    Exit = 0,
-    Rx = 1,
-    Tx = 2,
-    Unknown,
-}
-
-impl From<u64> for EpollEvent {
-    fn from(v: u64) -> Self {
-        use EpollEvent::*;
-        match v {
-            0 => Exit,
-            1 => Rx,
-            2 => Tx,
-            _ => Unknown,
-        }
-    }
-}
-
-// Handle the rx and tx requests from the virtio-net device
-fn tap_backend_stub(
-    mut dummy_tap: File,
-    tap_input_bytes: &[u8; TAP_INPUT_SIZE],
-    exit_evt: EventFd,
-) {
-    let mut epoll = EpollContext::new().unwrap();
-    epoll
-        .add_event_custom(&exit_evt, EpollEvent::Exit as u64, epoll::Events::EPOLLIN)
-        .unwrap();
-    let dummy_tap_write = dummy_tap.try_clone().unwrap();
-    epoll
-        .add_event_custom(
-            &dummy_tap_write,
-            EpollEvent::Rx as u64,
-            epoll::Events::EPOLLOUT,
-        )
-        .unwrap();
-    epoll
-        .add_event_custom(&dummy_tap, EpollEvent::Tx as u64, epoll::Events::EPOLLIN)
-        .unwrap();
-
-    let epoll_fd = epoll.as_raw_fd();
-    let mut events = vec![epoll::Event::new(epoll::Events::empty(), 0); 3];
-    loop {
-        let num_events = match epoll::wait(epoll_fd, -1, &mut events[..]) {
-            Ok(num_events) => num_events,
-            Err(e) => match e.raw_os_error() {
-                Some(libc::EAGAIN) | Some(libc::EINTR) => continue,
-                _ => panic!("Unexpected epoll::wait error!"),
-            },
-        };
-
-        for event in events.iter().take(num_events) {
-            let dispatch_event: EpollEvent = event.data.into();
-            match dispatch_event {
-                EpollEvent::Exit => {
-                    return;
-                }
-                EpollEvent::Rx => {
-                    dummy_tap.write_all(tap_input_bytes).unwrap();
-                    break;
-                }
-                EpollEvent::Tx => {
-                    let mut buffer = Vec::new();
-                    dummy_tap.read_to_end(&mut buffer).ok();
-                    break;
-                }
-                _ => {
-                    panic!("Unexpected Epoll event");
-                }
-            }
-        }
-    }
-}
--- a/fuzz/fuzz_targets/serial.rs
+++ b/fuzz/fuzz_targets/serial.rs
@@ -15,7 +15,6 @@ fuzz_target!(|bytes| {
    let mut serial = Serial::new_sink(
        "serial".into(),
        Arc::new(TestInterrupt::new(EventFd::new(EFD_NONBLOCK).unwrap())),
-        None,
    );

    let mut i = 0;
--- a/hypervisor/Cargo.toml
+++ b/hypervisor/Cargo.toml
@@ -11,23 +11,23 @@ mshv = ["mshv-ioctls", "mshv-bindings"]
 tdx = []

 [dependencies]
-anyhow = "1.0.68"
+anyhow = "1.0.66"
 byteorder = "1.4.3"
-thiserror = "1.0.38"
-libc = "0.2.139"
+thiserror = "1.0.37"
+libc = "0.2.138"
 log = "0.4.17"
-kvm-ioctls = { version = "0.12.0", optional = true }
+kvm-ioctls = { version = "0.13.0", optional = true }
 kvm-bindings = { git = "https://github.com/cloud-hypervisor/kvm-bindings", branch = "ch-v0.6.0-tdx", features = ["with-serde", "fam-wrappers"], optional  = true }
 mshv-bindings = { git = "https://github.com/rust-vmm/mshv", branch = "main", features = ["with-serde", "fam-wrappers"], optional  = true }
 mshv-ioctls = { git = "https://github.com/rust-vmm/mshv", branch = "main", optional  = true}
-serde = { version = "1.0.151", features = ["rc", "derive"] }
+serde = { version = "1.0.150", features = ["rc", "derive"] }
 serde_with = { version = "2.1.0", default-features = false, features = ["macros"] }
 vfio-ioctls = { git = "https://github.com/rust-vmm/vfio", branch = "main", default-features = false }
 vm-memory = { version = "0.10.0", features = ["backend-mmap", "backend-atomic"] }
 vmm-sys-util = { version = "0.11.0", features = ["with-serde"] }

 [target.'cfg(target_arch = "x86_64")'.dependencies.iced-x86]
-version = "1.18.0"
+version = "1.17.0"
 default-features = false
 features = ["std", "decoder", "op_code_info", "instr_info", "fast_fmt"]

--- a/hypervisor/src/arch/x86/emulator/instructions/mod.rs
+++ b/hypervisor/src/arch/x86/emulator/instructions/mod.rs
@@ -18,8 +18,8 @@ fn get_op<T: CpuStateManager>(
    insn: &Instruction,
    op_index: u32,
    op_size: usize,
-    state: &mut T,
-    platform: &mut dyn PlatformEmulator<CpuState = T>,
+    state: &T,
+    platform: &dyn PlatformEmulator<CpuState = T>,
 ) -> Result<u64, PlatformError> {
    if insn.op_count() < op_index + 1 {
        return Err(PlatformError::InvalidOperand(anyhow!(
--- a/hypervisor/src/arch/x86/mod.rs
+++ b/hypervisor/src/arch/x86/mod.rs
@@ -272,7 +272,6 @@ impl LapicState {
        use std::io::Cursor;
        use std::mem;

-        // SAFETY: plain old data type
        let sliceu8 = unsafe {
            // This array is only accessed as parts of a u32 word, so interpret it as a u8 array.
            // Cursors are only readable on arrays of u8, not i8(c_char).
@@ -291,7 +290,6 @@ impl LapicState {
        use std::io::Cursor;
        use std::mem;

-        // SAFETY: plain old data type
        let sliceu8 = unsafe {
            // This array is only accessed as parts of a u32 word, so interpret it as a u8 array.
            // Cursors are only readable on arrays of u8, not i8(c_char).
--- a/hypervisor/src/cpu.rs
+++ b/hypervisor/src/cpu.rs
@@ -237,6 +237,12 @@ pub enum HypervisorCpuError {
    ///
    #[error("Failed to initialize PMU")]
    InitializePmu,
+    #[cfg(target_arch = "x86_64")]
+    ///
+    /// Error getting TSC frequency
+    ///
+    #[error("Failed to get TSC frequency: {0}")]
+    GetTscKhz(#[source] anyhow::Error),
 }

 #[derive(Debug)]
@@ -432,4 +438,11 @@ pub trait Vcpu: Send + Sync {
    /// Return the list of initial MSR entries for a VCPU
    ///
    fn boot_msr_entries(&self) -> Vec<MsrEntry>;
+    #[cfg(target_arch = "x86_64")]
+    ///
+    /// Get the frequency of the TSC if available
+    ///
+    fn tsc_khz(&self) -> Result<Option<u32>> {
+        Ok(None)
+    }
 }
--- a/hypervisor/src/kvm/aarch64/gic/redist_regs.rs
+++ b/hypervisor/src/kvm/aarch64/gic/redist_regs.rs
@@ -39,8 +39,8 @@ const GICR_ICFGR0: u32 = GICR_SGI_OFFSET + 0x0C00;
 const KVM_DEV_ARM_VGIC_V3_MPIDR_SHIFT: u32 = 32;
 const KVM_DEV_ARM_VGIC_V3_MPIDR_MASK: u64 = 0xffffffff << KVM_DEV_ARM_VGIC_V3_MPIDR_SHIFT as u64;

-const KVM_ARM64_SYSREG_MPIDR_EL1: u64 = KVM_REG_ARM64
-    | KVM_REG_SIZE_U64
+const KVM_ARM64_SYSREG_MPIDR_EL1: u64 = KVM_REG_ARM64 as u64
+    | KVM_REG_SIZE_U64 as u64
    | KVM_REG_ARM64_SYSREG as u64
    | (((3_u64) << KVM_REG_ARM64_SYSREG_OP0_SHIFT) & KVM_REG_ARM64_SYSREG_OP0_MASK as u64)
    | (((5_u64) << KVM_REG_ARM64_SYSREG_OP2_SHIFT) & KVM_REG_ARM64_SYSREG_OP2_MASK as u64);
--- a/hypervisor/src/kvm/aarch64/mod.rs
+++ b/hypervisor/src/kvm/aarch64/mod.rs
@@ -24,8 +24,8 @@ pub use {kvm_ioctls::Cap, kvm_ioctls::Kvm};
 // This macro gets the offset of a structure (i.e `str`) member (i.e `field`) without having
 // an instance of that structure.
 #[macro_export]
-macro_rules! offset_of {
-    ($str:ty, $field:ident) => {{
+macro_rules! offset__of {
+    ($str:ty, $($field:ident)+) => ({
        let tmp: std::mem::MaybeUninit<$str> = std::mem::MaybeUninit::uninit();
        let base = tmp.as_ptr();

@@ -34,16 +34,14 @@ macro_rules! offset_of {
        // SAFETY: The pointer is valid and aligned, just not initialised. Using `addr_of` ensures
        // that we don't actually read from `base` (which would be UB) nor create an intermediate
        // reference.
-        let member = unsafe { core::ptr::addr_of!((*base).$field) } as *const u8;
+        let member = unsafe { core::ptr::addr_of!((*base).$($field)*) } as *const u8;

        // Avoid warnings when nesting `unsafe` blocks.
        #[allow(unused_unsafe)]
        // SAFETY: The two pointers are within the same allocated object `tmp`. All requirements
        // from offset_from are upheld.
-        unsafe {
-            member.offset_from(base as *const u8) as usize
-        }
-    }};
+        unsafe { member.offset_from(base as *const u8) as usize }
+    });
 }

 // Following are macros that help with getting the ID of a aarch64 core register.
--- a/hypervisor/src/kvm/mod.rs
+++ b/hypervisor/src/kvm/mod.rs
@@ -23,7 +23,7 @@ use crate::vec_with_array_field;
 use crate::vm::{self, InterruptSourceConfig, VmOps};
 use crate::HypervisorType;
 #[cfg(target_arch = "aarch64")]
-use crate::{arm64_core_reg_id, offset_of};
+use crate::{arm64_core_reg_id, offset__of};
 use kvm_ioctls::{NoDatamatch, VcpuFd, VmFd};
 use std::any::Any;
 use std::collections::HashMap;
@@ -330,7 +330,7 @@ impl KvmVm {
        Ok(VfioDeviceFd::new_from_kvm(device_fd))
    }
    /// Checks if a particular `Cap` is available.
-    fn check_extension(&self, c: Cap) -> bool {
+    pub fn check_extension(&self, c: Cap) -> bool {
        self.fd.check_extension(c)
    }
 }
@@ -1114,7 +1114,7 @@ impl cpu::Vcpu for KvmVcpu {
    #[cfg(target_arch = "aarch64")]
    fn get_regs(&self) -> cpu::Result<StandardRegisters> {
        let mut state: StandardRegisters = kvm_regs::default();
-        let mut off = offset_of!(user_pt_regs, regs);
+        let mut off = offset__of!(user_pt_regs, regs);
        // There are 31 user_pt_regs:
        // https://elixir.free-electrons.com/linux/v4.14.174/source/arch/arm64/include/uapi/asm/ptrace.h#L72
        // These actually are the general-purpose registers of the Armv8-a
@@ -1131,7 +1131,7 @@ impl cpu::Vcpu for KvmVcpu {

        // We are now entering the "Other register" section of the ARMv8-a architecture.
        // First one, stack pointer.
-        let off = offset_of!(user_pt_regs, sp);
+        let off = offset__of!(user_pt_regs, sp);
        state.regs.sp = self
            .fd
            .get_one_reg(arm64_core_reg_id!(KVM_REG_SIZE_U64, off))
@@ -1140,7 +1140,7 @@ impl cpu::Vcpu for KvmVcpu {
            .unwrap();

        // Second one, the program counter.
-        let off = offset_of!(user_pt_regs, pc);
+        let off = offset__of!(user_pt_regs, pc);
        state.regs.pc = self
            .fd
            .get_one_reg(arm64_core_reg_id!(KVM_REG_SIZE_U64, off))
@@ -1149,7 +1149,7 @@ impl cpu::Vcpu for KvmVcpu {
            .unwrap();

        // Next is the processor state.
-        let off = offset_of!(user_pt_regs, pstate);
+        let off = offset__of!(user_pt_regs, pstate);
        state.regs.pstate = self
            .fd
            .get_one_reg(arm64_core_reg_id!(KVM_REG_SIZE_U64, off))
@@ -1158,7 +1158,7 @@ impl cpu::Vcpu for KvmVcpu {
            .unwrap();

        // The stack pointer associated with EL1
-        let off = offset_of!(kvm_regs, sp_el1);
+        let off = offset__of!(kvm_regs, sp_el1);
        state.sp_el1 = self
            .fd
            .get_one_reg(arm64_core_reg_id!(KVM_REG_SIZE_U64, off))
@@ -1168,7 +1168,7 @@ impl cpu::Vcpu for KvmVcpu {

        // Exception Link Register for EL1, when taking an exception to EL1, this register
        // holds the address to which to return afterwards.
-        let off = offset_of!(kvm_regs, elr_el1);
+        let off = offset__of!(kvm_regs, elr_el1);
        state.elr_el1 = self
            .fd
            .get_one_reg(arm64_core_reg_id!(KVM_REG_SIZE_U64, off))
@@ -1177,7 +1177,7 @@ impl cpu::Vcpu for KvmVcpu {
            .unwrap();

        // Saved Program Status Registers, there are 5 of them used in the kernel.
-        let mut off = offset_of!(kvm_regs, spsr);
+        let mut off = offset__of!(kvm_regs, spsr);
        for i in 0..KVM_NR_SPSR as usize {
            state.spsr[i] = self
                .fd
@@ -1190,7 +1190,7 @@ impl cpu::Vcpu for KvmVcpu {

        // Now moving on to floting point registers which are stored in the user_fpsimd_state in the kernel:
        // https://elixir.free-electrons.com/linux/v4.9.62/source/arch/arm64/include/uapi/asm/kvm.h#L53
-        let mut off = offset_of!(kvm_regs, fp_regs) + offset_of!(user_fpsimd_state, vregs);
+        let mut off = offset__of!(kvm_regs, fp_regs) + offset__of!(user_fpsimd_state, vregs);
        for i in 0..32 {
            state.fp_regs.vregs[i] = self
                .fd
@@ -1200,7 +1200,7 @@ impl cpu::Vcpu for KvmVcpu {
        }

        // Floating-point Status Register
-        let off = offset_of!(kvm_regs, fp_regs) + offset_of!(user_fpsimd_state, fpsr);
+        let off = offset__of!(kvm_regs, fp_regs) + offset__of!(user_fpsimd_state, fpsr);
        state.fp_regs.fpsr = self
            .fd
            .get_one_reg(arm64_core_reg_id!(KVM_REG_SIZE_U32, off))
@@ -1209,7 +1209,7 @@ impl cpu::Vcpu for KvmVcpu {
            .unwrap();

        // Floating-point Control Register
-        let off = offset_of!(kvm_regs, fp_regs) + offset_of!(user_fpsimd_state, fpcr);
+        let off = offset__of!(kvm_regs, fp_regs) + offset__of!(user_fpsimd_state, fpcr);
        state.fp_regs.fpcr = self
            .fd
            .get_one_reg(arm64_core_reg_id!(KVM_REG_SIZE_U32, off))
@@ -1238,7 +1238,7 @@ impl cpu::Vcpu for KvmVcpu {
    fn set_regs(&self, state: &StandardRegisters) -> cpu::Result<()> {
        // The function follows the exact identical order from `state`. Look there
        // for some additional info on registers.
-        let mut off = offset_of!(user_pt_regs, regs);
+        let mut off = offset__of!(user_pt_regs, regs);
        for i in 0..31 {
            self.fd
                .set_one_reg(
@@ -1249,7 +1249,7 @@ impl cpu::Vcpu for KvmVcpu {
            off += std::mem::size_of::<u64>();
        }

-        let off = offset_of!(user_pt_regs, sp);
+        let off = offset__of!(user_pt_regs, sp);
        self.fd
            .set_one_reg(
                arm64_core_reg_id!(KVM_REG_SIZE_U64, off),
@@ -1257,7 +1257,7 @@ impl cpu::Vcpu for KvmVcpu {
            )
            .map_err(|e| cpu::HypervisorCpuError::SetCoreRegister(e.into()))?;

-        let off = offset_of!(user_pt_regs, pc);
+        let off = offset__of!(user_pt_regs, pc);
        self.fd
            .set_one_reg(
                arm64_core_reg_id!(KVM_REG_SIZE_U64, off),
@@ -1265,7 +1265,7 @@ impl cpu::Vcpu for KvmVcpu {
            )
            .map_err(|e| cpu::HypervisorCpuError::SetCoreRegister(e.into()))?;

-        let off = offset_of!(user_pt_regs, pstate);
+        let off = offset__of!(user_pt_regs, pstate);
        self.fd
            .set_one_reg(
                arm64_core_reg_id!(KVM_REG_SIZE_U64, off),
@@ -1273,7 +1273,7 @@ impl cpu::Vcpu for KvmVcpu {
            )
            .map_err(|e| cpu::HypervisorCpuError::SetCoreRegister(e.into()))?;

-        let off = offset_of!(kvm_regs, sp_el1);
+        let off = offset__of!(kvm_regs, sp_el1);
        self.fd
            .set_one_reg(
                arm64_core_reg_id!(KVM_REG_SIZE_U64, off),
@@ -1281,7 +1281,7 @@ impl cpu::Vcpu for KvmVcpu {
            )
            .map_err(|e| cpu::HypervisorCpuError::SetCoreRegister(e.into()))?;

-        let off = offset_of!(kvm_regs, elr_el1);
+        let off = offset__of!(kvm_regs, elr_el1);
        self.fd
            .set_one_reg(
                arm64_core_reg_id!(KVM_REG_SIZE_U64, off),
@@ -1289,7 +1289,7 @@ impl cpu::Vcpu for KvmVcpu {
            )
            .map_err(|e| cpu::HypervisorCpuError::SetCoreRegister(e.into()))?;

-        let mut off = offset_of!(kvm_regs, spsr);
+        let mut off = offset__of!(kvm_regs, spsr);
        for i in 0..KVM_NR_SPSR as usize {
            self.fd
                .set_one_reg(
@@ -1300,7 +1300,7 @@ impl cpu::Vcpu for KvmVcpu {
            off += std::mem::size_of::<u64>();
        }

-        let mut off = offset_of!(kvm_regs, fp_regs) + offset_of!(user_fpsimd_state, vregs);
+        let mut off = offset__of!(kvm_regs, fp_regs) + offset__of!(user_fpsimd_state, vregs);
        for i in 0..32 {
            self.fd
                .set_one_reg(
@@ -1311,7 +1311,7 @@ impl cpu::Vcpu for KvmVcpu {
            off += mem::size_of::<u128>();
        }

-        let off = offset_of!(kvm_regs, fp_regs) + offset_of!(user_fpsimd_state, fpsr);
+        let off = offset__of!(kvm_regs, fp_regs) + offset__of!(user_fpsimd_state, fpsr);
        self.fd
            .set_one_reg(
                arm64_core_reg_id!(KVM_REG_SIZE_U32, off),
@@ -1319,7 +1319,7 @@ impl cpu::Vcpu for KvmVcpu {
            )
            .map_err(|e| cpu::HypervisorCpuError::SetCoreRegister(e.into()))?;

-        let off = offset_of!(kvm_regs, fp_regs) + offset_of!(user_fpsimd_state, fpcr);
+        let off = offset__of!(kvm_regs, fp_regs) + offset__of!(user_fpsimd_state, fpcr);
        self.fd
            .set_one_reg(
                arm64_core_reg_id!(KVM_REG_SIZE_U32, off),
@@ -1700,8 +1700,8 @@ impl cpu::Vcpu for KvmVcpu {
        // it to the corresponding KVM ID, and call `KVM_GET_ONE_REG` API to
        // get the value of the system parameter.
        //
-        let id: u64 = KVM_REG_ARM64
-            | KVM_REG_SIZE_U64
+        let id: u64 = KVM_REG_ARM64 as u64
+            | KVM_REG_SIZE_U64 as u64
            | KVM_REG_ARM64_SYSREG as u64
            | ((((sys_reg) >> 5)
                & (KVM_REG_ARM64_SYSREG_OP0_MASK
@@ -1733,10 +1733,10 @@ impl cpu::Vcpu for KvmVcpu {
        const PSTATE_FAULT_BITS_64: u64 =
            PSR_MODE_EL1h | PSR_A_BIT | PSR_F_BIT | PSR_I_BIT | PSR_D_BIT;

-        let kreg_off = offset_of!(kvm_regs, regs);
+        let kreg_off = offset__of!(kvm_regs, regs);

        // Get the register index of the PSTATE (Processor State) register.
-        let pstate = offset_of!(user_pt_regs, pstate) + kreg_off;
+        let pstate = offset__of!(user_pt_regs, pstate) + kreg_off;
        self.fd
            .set_one_reg(
                arm64_core_reg_id!(KVM_REG_SIZE_U64, pstate),
@@ -1747,7 +1747,7 @@ impl cpu::Vcpu for KvmVcpu {
        // Other vCPUs are powered off initially awaiting PSCI wakeup.
        if cpu_id == 0 {
            // Setting the PC (Processor Counter) to the current program address (kernel address).
-            let pc = offset_of!(user_pt_regs, pc) + kreg_off;
+            let pc = offset__of!(user_pt_regs, pc) + kreg_off;
            self.fd
                .set_one_reg(arm64_core_reg_id!(KVM_REG_SIZE_U64, pc), boot_ip.into())
                .map_err(|e| cpu::HypervisorCpuError::SetCoreRegister(e.into()))?;
@@ -1756,7 +1756,7 @@ impl cpu::Vcpu for KvmVcpu {
            // "The device tree blob (dtb) must be placed on an 8-byte boundary and must
            // not exceed 2 megabytes in size." -> https://www.kernel.org/doc/Documentation/arm64/booting.txt.
            // We are choosing to place it the end of DRAM. See `get_fdt_addr`.
-            let regs0 = offset_of!(user_pt_regs, regs) + kreg_off;
+            let regs0 = offset__of!(user_pt_regs, regs) + kreg_off;
            self.fd
                .set_one_reg(
                    arm64_core_reg_id!(KVM_REG_SIZE_U64, regs0),
@@ -2066,7 +2066,6 @@ impl cpu::Vcpu for KvmVcpu {
    #[cfg(feature = "tdx")]
    fn get_tdx_exit_details(&mut self) -> cpu::Result<TdxExitDetails> {
        let kvm_run = self.fd.get_kvm_run();
-        // SAFETY: accessing a union field in a valid structure
        let tdx_vmcall = unsafe { &mut kvm_run.__bindgen_anon_1.tdx.u.vmcall };

        tdx_vmcall.status_code = TDG_VP_VMCALL_INVALID_OPERAND;
@@ -2090,7 +2089,6 @@ impl cpu::Vcpu for KvmVcpu {
    #[cfg(feature = "tdx")]
    fn set_tdx_status(&mut self, status: TdxExitStatus) {
        let kvm_run = self.fd.get_kvm_run();
-        // SAFETY: accessing a union field in a valid structure
        let tdx_vmcall = unsafe { &mut kvm_run.__bindgen_anon_1.tdx.u.vmcall };

        tdx_vmcall.status_code = match status {
@@ -2154,6 +2152,23 @@ impl cpu::Vcpu for KvmVcpu {
            .set_device_attr(&cpu_attr)
            .map_err(|_| cpu::HypervisorCpuError::InitializePmu)
    }
+
+    #[cfg(target_arch = "x86_64")]
+    ///
+    /// Get the frequency of the TSC if available
+    ///
+    fn tsc_khz(&self) -> cpu::Result<Option<u32>> {
+        match self.fd.get_tsc_khz() {
+            Err(e) => {
+                if e.errno() == libc::EIO {
+                    Ok(None)
+                } else {
+                    Err(cpu::HypervisorCpuError::GetTscKhz(e.into()))
+                }
+            }
+            Ok(v) => Ok(Some(v)),
+        }
+    }
 }

 impl KvmVcpu {
--- a/hypervisor/src/kvm/x86_64/mod.rs
+++ b/hypervisor/src/kvm/x86_64/mod.rs
@@ -50,6 +50,9 @@ pub fn check_required_kvm_extensions(kvm: &Kvm) -> KvmResult<()> {
    if !kvm.check_extension(Cap::ImmediateExit) {
        return Err(KvmError::CapabilityMissing(Cap::ImmediateExit));
    }
+    if !kvm.check_extension(Cap::GetTscKhz) {
+        return Err(KvmError::CapabilityMissing(Cap::GetTscKhz));
+    }
    Ok(())
 }
 #[derive(Clone, Serialize, Deserialize)]
--- a/hypervisor/src/lib.rs
+++ b/hypervisor/src/lib.rs
@@ -18,6 +18,8 @@
 //! - arm64
 //!

+#![allow(clippy::significant_drop_in_scrutinee)]
+
 #[macro_use]
 extern crate anyhow;
 #[cfg(target_arch = "x86_64")]
@@ -59,11 +61,9 @@ pub use vm::{
    Vm, VmOps,
 };

-#[derive(Debug, Copy, Clone)]
+#[derive(Debug, Copy, Clone, Eq, PartialEq)]
 pub enum HypervisorType {
-    #[cfg(feature = "kvm")]
    Kvm,
-    #[cfg(feature = "mshv")]
    Mshv,
 }

--- a/hypervisor/src/mshv/mod.rs
+++ b/hypervisor/src/mshv/mod.rs
@@ -38,9 +38,7 @@ use std::fs::File;
 use std::os::unix::io::AsRawFd;

 #[cfg(target_arch = "x86_64")]
-use crate::arch::x86::{
-    CpuIdEntry, FpuState, LapicState, MsrEntry, SpecialRegisters, StandardRegisters,
-};
+use crate::arch::x86::{CpuIdEntry, FpuState, MsrEntry};

 const DIRTY_BITMAP_CLEAR_DIRTY: u64 = 0x4;
 const DIRTY_BITMAP_SET_DIRTY: u64 = 0x8;
@@ -299,7 +297,7 @@ impl cpu::Vcpu for MshvVcpu {
    ///
    /// Returns the vCPU general purpose registers.
    ///
-    fn get_regs(&self) -> cpu::Result<StandardRegisters> {
+    fn get_regs(&self) -> cpu::Result<crate::arch::x86::StandardRegisters> {
        Ok(self
            .fd
            .get_regs()
@@ -310,7 +308,7 @@ impl cpu::Vcpu for MshvVcpu {
    ///
    /// Sets the vCPU general purpose registers.
    ///
-    fn set_regs(&self, regs: &StandardRegisters) -> cpu::Result<()> {
+    fn set_regs(&self, regs: &crate::arch::x86::StandardRegisters) -> cpu::Result<()> {
        let regs = (*regs).into();
        self.fd
            .set_regs(&regs)
@@ -320,7 +318,7 @@ impl cpu::Vcpu for MshvVcpu {
    ///
    /// Returns the vCPU special registers.
    ///
-    fn get_sregs(&self) -> cpu::Result<SpecialRegisters> {
+    fn get_sregs(&self) -> cpu::Result<crate::arch::x86::SpecialRegisters> {
        Ok(self
            .fd
            .get_sregs()
@@ -331,7 +329,7 @@ impl cpu::Vcpu for MshvVcpu {
    ///
    /// Sets the vCPU special registers.
    ///
-    fn set_sregs(&self, sregs: &SpecialRegisters) -> cpu::Result<()> {
+    fn set_sregs(&self, sregs: &crate::arch::x86::SpecialRegisters) -> cpu::Result<()> {
        let sregs = (*sregs).into();
        self.fd
            .set_sregs(&sregs)
@@ -446,10 +444,10 @@ impl cpu::Vcpu for MshvVcpu {
                            /* Advance RIP and update RAX */
                            let arr_reg_name_value = [
                                (
-                                    hv_x64_register_name_HV_X64_REGISTER_RIP,
+                                    hv_register_name::HV_X64_REGISTER_RIP,
                                    info.header.rip + insn_len,
                                ),
-                                (hv_x64_register_name_HV_X64_REGISTER_RAX, ret_rax),
+                                (hv_register_name::HV_X64_REGISTER_RAX, ret_rax),
                            ];
                            set_registers_64!(self.fd, arr_reg_name_value)
                                .map_err(|e| cpu::HypervisorCpuError::SetRegister(e.into()))?;
@@ -458,13 +456,12 @@ impl cpu::Vcpu for MshvVcpu {
                        _ => {}
                    }

+                    // SAFETY: access_info is valid, otherwise we won't be here
                    assert!(
-                        // SAFETY: access_info is valid, otherwise we won't be here
                        (unsafe { access_info.__bindgen_anon_1.string_op() } != 1),
                        "String IN/OUT not supported"
                    );
                    assert!(
-                        // SAFETY: access_info is valid, otherwise we won't be here
                        (unsafe { access_info.__bindgen_anon_1.rep_prefix() } != 1),
                        "Rep IN/OUT not supported"
                    );
@@ -495,10 +492,10 @@ impl cpu::Vcpu for MshvVcpu {
                    /* Advance RIP and update RAX */
                    let arr_reg_name_value = [
                        (
-                            hv_x64_register_name_HV_X64_REGISTER_RIP,
+                            hv_register_name::HV_X64_REGISTER_RIP,
                            info.header.rip + insn_len,
                        ),
-                        (hv_x64_register_name_HV_X64_REGISTER_RAX, ret_rax),
+                        (hv_register_name::HV_X64_REGISTER_RAX, ret_rax),
                    ];
                    set_registers_64!(self.fd, arr_reg_name_value)
                        .map_err(|e| cpu::HypervisorCpuError::SetRegister(e.into()))?;
@@ -582,7 +579,7 @@ impl cpu::Vcpu for MshvVcpu {
    ///
    /// Returns the state of the LAPIC (Local Advanced Programmable Interrupt Controller).
    ///
-    fn get_lapic(&self) -> cpu::Result<LapicState> {
+    fn get_lapic(&self) -> cpu::Result<crate::arch::x86::LapicState> {
        Ok(self
            .fd
            .get_lapic()
@@ -593,7 +590,7 @@ impl cpu::Vcpu for MshvVcpu {
    ///
    /// Sets the state of the LAPIC (Local Advanced Programmable Interrupt Controller).
    ///
-    fn set_lapic(&self, lapic: &LapicState) -> cpu::Result<()> {
+    fn set_lapic(&self, lapic: &crate::arch::x86::LapicState) -> cpu::Result<()> {
        let lapic: mshv_bindings::LapicState = (*lapic).clone().into();
        self.fd
            .set_lapic(&lapic)
--- a/net_gen/src/iff.rs
+++ b/net_gen/src/iff.rs
@@ -1187,7 +1187,6 @@ pub struct ifreq {

 impl Default for ifreq {
    fn default() -> Self {
-        // SAFETY: all zeros is a valid pattern for this data type
        unsafe { std::mem::zeroed() }
    }
 }
--- a/net_gen/src/lib.rs
+++ b/net_gen/src/lib.rs
@@ -28,10 +28,12 @@ pub mod inn;
 // generated with bindgen /usr/include/linux/sockios.h --no-unstable-rust
 // --constified-enum '*' --with-derive-default
 pub mod sockios;
-pub use if_tun::*;
-pub use iff::*;
-pub use inn::*;
-pub use sockios::*;
+pub use if_tun::{
+    sock_fprog, IFF_MULTI_QUEUE, IFF_NO_PI, IFF_TAP, IFF_VNET_HDR, TUN_F_CSUM, TUN_F_TSO4,
+    TUN_F_TSO6, TUN_F_TSO_ECN, TUN_F_UFO,
+};
+pub use iff::{ifreq, net_device_flags_IFF_UP, setsockopt, sockaddr, AF_INET};
+pub use inn::sockaddr_in;

 pub const TUNTAP: ::std::os::raw::c_uint = 84;

--- a/net_util/Cargo.toml
+++ b/net_util/Cargo.toml
@@ -7,12 +7,12 @@ edition = "2021"
 [dependencies]
 epoll = "4.3.1"
 getrandom = "0.2.8"
-libc = "0.2.139"
+libc = "0.2.138"
 log = "0.4.17"
 net_gen = { path = "../net_gen" }
 rate_limiter = { path = "../rate_limiter" }
-serde = "1.0.151"
-thiserror = "1.0.38"
+serde = "1.0.150"
+thiserror = "1.0.37"
 versionize = "0.1.9"
 versionize_derive = "0.1.4"
 virtio-bindings = "0.1.0"
@@ -23,6 +23,6 @@ vmm-sys-util = "0.11.0"

 [dev-dependencies]
 once_cell = "1.16.0"
-pnet = "0.31.0"
-pnet_datalink = "0.31.0"
+pnet = "0.34.0"
+pnet_datalink = "0.34.0"
 serde_json = "1.0.89"
--- a/net_util/src/lib.rs
+++ b/net_util/src/lib.rs
@@ -66,34 +66,32 @@ fn create_sockaddr(ip_addr: net::Ipv4Addr) -> net_gen::sockaddr {
    let addr_in = net_gen::sockaddr_in {
        sin_family: net_gen::AF_INET as u16,
        sin_port: 0,
-        // SAFETY: ip_addr can be safely transmute to in_addr
        sin_addr: unsafe { mem::transmute(ip_addr.octets()) },
        __pad: [0; 8usize],
    };

-    // SAFETY: addr_in can be safely transmute to sockaddr
    unsafe { mem::transmute(addr_in) }
 }

 fn create_inet_socket() -> Result<net::UdpSocket> {
-    // SAFETY: we check the return value.
+    // This is safe since we check the return value.
    let sock = unsafe { libc::socket(libc::AF_INET, libc::SOCK_DGRAM, 0) };
    if sock < 0 {
        return Err(Error::CreateSocket(IoError::last_os_error()));
    }

-    // SAFETY: nothing else will use or hold onto the raw sock fd.
+    // This is safe; nothing else will use or hold onto the raw sock fd.
    Ok(unsafe { net::UdpSocket::from_raw_fd(sock) })
 }

 fn create_unix_socket() -> Result<net::UdpSocket> {
-    // SAFETY: we check the return value.
+    // This is safe since we check the return value.
    let sock = unsafe { libc::socket(libc::AF_UNIX, libc::SOCK_DGRAM, 0) };
    if sock < 0 {
        return Err(Error::CreateSocket(IoError::last_os_error()));
    }

-    // SAFETY: nothing else will use or hold onto the raw sock fd.
+    // This is safe; nothing else will use or hold onto the raw sock fd.
    Ok(unsafe { net::UdpSocket::from_raw_fd(sock) })
 }

--- a/net_util/src/queue_pair.rs
+++ b/net_util/src/queue_pair.rs
@@ -83,11 +83,10 @@ impl TxVirtio {
            }

            let len = if !iovecs.is_empty() {
-                // SAFETY: FFI call with correct arguments
                let result = unsafe {
                    libc::writev(
                        tap.as_raw_fd() as libc::c_int,
-                        iovecs.as_ptr() as *const libc::iovec,
+                        iovecs.as_ptr(),
                        iovecs.len() as libc::c_int,
                    )
                };
@@ -222,11 +221,10 @@ impl RxVirtio {
            }

            let len = if !iovecs.is_empty() {
-                // SAFETY: FFI call with correct arguments
                let result = unsafe {
                    libc::readv(
                        tap.as_raw_fd() as libc::c_int,
-                        iovecs.as_ptr() as *const libc::iovec,
+                        iovecs.as_ptr(),
                        iovecs.len() as libc::c_int,
                    )
                };
--- a/net_util/src/tap.rs
+++ b/net_util/src/tap.rs
@@ -28,8 +28,8 @@ pub enum Error {
    GetFeatures(IoError),
    #[error("Missing multiqueue support in the kernel")]
    MultiQueueKernelSupport,
-    #[error("ioctl ({0}) failed: {1}")]
-    IoctlError(c_ulong, IoError),
+    #[error("ioctl failed: {0}")]
+    IoctlError(IoError),
    #[error("Failed to create a socket: {0}")]
    NetUtil(NetUtilError),
    #[error("Invalid interface name")]
@@ -87,37 +87,9 @@ fn build_terminated_if_name(if_name: &str) -> Result<Vec<u8>> {
 }

 impl Tap {
-    unsafe fn ioctl_with_mut_ref<F: AsRawFd, T>(fd: &F, req: c_ulong, arg: &mut T) -> Result<()> {
-        let ret = ioctl_with_mut_ref(fd, req, arg);
-        if ret < 0 {
-            return Err(Error::IoctlError(req, IoError::last_os_error()));
-        }
-
-        Ok(())
-    }
-
-    unsafe fn ioctl_with_ref<F: AsRawFd, T>(fd: &F, req: c_ulong, arg: &T) -> Result<()> {
-        let ret = ioctl_with_ref(fd, req, arg);
-        if ret < 0 {
-            return Err(Error::IoctlError(req, IoError::last_os_error()));
-        }
-
-        Ok(())
-    }
-
-    unsafe fn ioctl_with_val<F: AsRawFd>(fd: &F, req: c_ulong, arg: c_ulong) -> Result<()> {
-        let ret = ioctl_with_val(fd, req, arg);
-        if ret < 0 {
-            return Err(Error::IoctlError(req, IoError::last_os_error()));
-        }
-
-        Ok(())
-    }
-
    pub fn open_named(if_name: &str, num_queue_pairs: usize, flags: Option<i32>) -> Result<Tap> {
        let terminated_if_name = build_terminated_if_name(if_name)?;

-        // SAFETY: FFI call
        let fd = unsafe {
            // Open calls are safe because we give a constant null-terminated
            // string and verify the result.
@@ -130,14 +102,13 @@ impl Tap {
            return Err(Error::OpenTun(IoError::last_os_error()));
        }

-        // SAFETY: We just checked that the fd is valid.
+        // We just checked that the fd is valid.
        let tuntap = unsafe { File::from_raw_fd(fd) };

        // Let's validate some features before going any further.
        // ioctl is safe since we call it with a valid tap fd and check the return
        // value.
        let mut features = 0;
-        // SAFETY: IOCTL with correct arguments
        let ret = unsafe { ioctl_with_mut_ref(&tuntap, net_gen::TUNGETFEATURES(), &mut features) };
        if ret < 0 {
            return Err(Error::GetFeatures(IoError::last_os_error()));
@@ -152,7 +123,6 @@ impl Tap {
        // don't call as_mut on the same union field more than once, this block
        // is safe.
        let mut ifreq: net_gen::ifreq = Default::default();
-        // SAFETY: see the comment above.
        unsafe {
            let ifrn_name = ifreq.ifr_ifrn.ifrn_name.as_mut();
            let name_slice = &mut ifrn_name[..terminated_if_name.len()];
@@ -164,16 +134,16 @@ impl Tap {
            }
        }

-        // SAFETY: ioctl is safe since we call it with a valid tap fd and check the return
+        // ioctl is safe since we call it with a valid tap fd and check the return
        // value.
        let ret = unsafe { ioctl_with_mut_ref(&tuntap, net_gen::TUNSETIFF(), &mut ifreq) };
        if ret < 0 {
            return Err(Error::ConfigureTap(IoError::last_os_error()));
        }

-        // SAFETY: only the name is accessed, and it's cloned out.
        let mut if_name = unsafe { ifreq.ifr_ifrn.ifrn_name }.to_vec();
        if_name.truncate(terminated_if_name.len() - 1);
+        // Safe since only the name is accessed, and it's cloned out.
        Ok(Tap {
            tap_file: tuntap,
            if_name,
@@ -188,7 +158,6 @@ impl Tap {
    pub fn from_tap_fd(fd: RawFd, num_queue_pairs: usize) -> Result<Tap> {
        // Ensure that the file is opened non-blocking, this is particularly
        // needed when opened via the shell for macvtap.
-        // SAFETY: FFI call
        let ret = unsafe {
            let mut flags = libc::fcntl(fd, libc::F_GETFL);
            flags |= libc::O_NONBLOCK;
@@ -198,20 +167,19 @@ impl Tap {
            return Err(Error::ConfigureTap(IoError::last_os_error()));
        }

-        // SAFETY: fd is a tap fd
        let tap_file = unsafe { File::from_raw_fd(fd) };
        let mut ifreq: net_gen::ifreq = Default::default();

        // Get current config including name
-        // SAFETY: IOCTL with correct arugments
-        unsafe { Self::ioctl_with_mut_ref(&tap_file, net_gen::TUNGETIFF(), &mut ifreq)? };
-
-        // SAFETY: We only access one field of the ifru union
+        let ret = unsafe { ioctl_with_mut_ref(&tap_file, net_gen::TUNGETIFF(), &mut ifreq) };
+        if ret < 0 {
+            return Err(Error::IoctlError(IoError::last_os_error()));
+        }
+        // We only access one field of the ifru union, hence this is safe.
        let if_name = unsafe { ifreq.ifr_ifrn.ifrn_name }.to_vec();

        // Try and update flags. Depending on how the tap was created (macvtap
        // or via open_named()) this might return -EEXIST so we just ignore that.
-        // SAFETY: access union fields
        unsafe {
            ifreq.ifr_ifru.ifru_flags =
                (net_gen::IFF_TAP | net_gen::IFF_NO_PI | net_gen::IFF_VNET_HDR) as c_short;
@@ -219,7 +187,6 @@ impl Tap {
                ifreq.ifr_ifru.ifru_flags |= net_gen::IFF_MULTI_QUEUE as c_short;
            }
        }
-        // SAFETY: IOCTL with correct arguments
        let ret = unsafe { ioctl_with_mut_ref(&tap_file, net_gen::TUNSETIFF(), &mut ifreq) };
        if ret < 0 && IoError::last_os_error().raw_os_error().unwrap() != libc::EEXIST {
            return Err(Error::ConfigureTap(IoError::last_os_error()));
@@ -241,8 +208,14 @@ impl Tap {

        ifreq.ifr_ifru.ifru_addr = addr;

-        // SAFETY: ioctl is safe. Called with a valid sock fd, and we check the return.
-        unsafe { Self::ioctl_with_ref(&sock, net_gen::sockios::SIOCSIFADDR as c_ulong, &ifreq) }
+        // ioctl is safe. Called with a valid sock fd, and we check the return.
+        let ret =
+            unsafe { ioctl_with_ref(&sock, net_gen::sockios::SIOCSIFADDR as c_ulong, &ifreq) };
+        if ret < 0 {
+            return Err(Error::IoctlError(IoError::last_os_error()));
+        }
+
+        Ok(())
    }

    /// Set mac addr for tap interface.
@@ -260,10 +233,13 @@ impl Tap {

        let mut ifreq = self.get_ifreq();

-        // SAFETY: ioctl is safe. Called with a valid sock fd, and we check the return.
-        unsafe { Self::ioctl_with_ref(&sock, net_gen::sockios::SIOCGIFHWADDR as c_ulong, &ifreq)? };
-
-        // SAFETY: We only access one field of the ifru union
+        // ioctl is safe. Called with a valid sock fd, and we check the return.
+        let ret =
+            unsafe { ioctl_with_ref(&sock, net_gen::sockios::SIOCGIFHWADDR as c_ulong, &ifreq) };
+        if ret < 0 {
+            return Err(Error::IoctlError(IoError::last_os_error()));
+        }
+        // We only access one field of the ifru union, hence this is safe.
        unsafe {
            let ifru_hwaddr = &mut ifreq.ifr_ifru.ifru_hwaddr;
            for (i, v) in addr.get_bytes().iter().enumerate() {
@@ -271,8 +247,14 @@ impl Tap {
            }
        }

-        // SAFETY: ioctl is safe. Called with a valid sock fd, and we check the return.
-        unsafe { Self::ioctl_with_ref(&sock, net_gen::sockios::SIOCSIFHWADDR as c_ulong, &ifreq) }
+        // ioctl is safe. Called with a valid sock fd, and we check the return.
+        let ret =
+            unsafe { ioctl_with_ref(&sock, net_gen::sockios::SIOCSIFHWADDR as c_ulong, &ifreq) };
+        if ret < 0 {
+            return Err(Error::IoctlError(IoError::last_os_error()));
+        }
+
+        Ok(())
    }

    /// Get mac addr for tap interface.
@@ -281,10 +263,14 @@ impl Tap {

        let ifreq = self.get_ifreq();

-        // SAFETY: ioctl is safe. Called with a valid sock fd, and we check the return.
-        unsafe { Self::ioctl_with_ref(&sock, net_gen::sockios::SIOCGIFHWADDR as c_ulong, &ifreq)? };
+        // ioctl is safe. Called with a valid sock fd, and we check the return.
+        let ret =
+            unsafe { ioctl_with_ref(&sock, net_gen::sockios::SIOCGIFHWADDR as c_ulong, &ifreq) };
+        if ret < 0 {
+            return Err(Error::IoctlError(IoError::last_os_error()));
+        }

-        // SAFETY: We only access one field of the ifru union
+        // We only access one field of the ifru union, hence this is safe.
        let addr = unsafe {
            MacAddr::from_bytes(&ifreq.ifr_ifru.ifru_hwaddr.sa_data[0..MAC_ADDR_LEN])
                .map_err(Error::MacParsing)?
@@ -301,45 +287,57 @@ impl Tap {

        ifreq.ifr_ifru.ifru_addr = addr;

-        // SAFETY: ioctl is safe. Called with a valid sock fd, and we check the return.
-        unsafe { Self::ioctl_with_ref(&sock, net_gen::sockios::SIOCSIFNETMASK as c_ulong, &ifreq) }
+        // ioctl is safe. Called with a valid sock fd, and we check the return.
+        let ret =
+            unsafe { ioctl_with_ref(&sock, net_gen::sockios::SIOCSIFNETMASK as c_ulong, &ifreq) };
+        if ret < 0 {
+            return Err(Error::IoctlError(IoError::last_os_error()));
+        }
+
+        Ok(())
    }

-    #[cfg(not(fuzzing))]
    pub fn mtu(&self) -> Result<i32> {
        let sock = create_unix_socket().map_err(Error::NetUtil)?;

        let ifreq = self.get_ifreq();

-        // SAFETY: ioctl is safe. Called with a valid sock fd, and we check the return.
-        unsafe { Self::ioctl_with_ref(&sock, net_gen::sockios::SIOCGIFMTU as c_ulong, &ifreq)? };
+        // ioctl is safe. Called with a valid sock fd, and we check the return.
+        let ret = unsafe { ioctl_with_ref(&sock, net_gen::sockios::SIOCGIFMTU as c_ulong, &ifreq) };
+        if ret < 0 {
+            return Err(Error::IoctlError(IoError::last_os_error()));
+        }

-        // SAFETY: access a union field
        let mtu = unsafe { ifreq.ifr_ifru.ifru_mtu };

        Ok(mtu)
    }

-    #[cfg(fuzzing)]
-    pub fn mtu(&self) -> Result<i32> {
-        // Consistent with the `virtio_devices::net::MIN_MTU`
-        Ok(1280)
-    }
-
    pub fn set_mtu(&self, mtu: i32) -> Result<()> {
        let sock = create_unix_socket().map_err(Error::NetUtil)?;

        let mut ifreq = self.get_ifreq();
        ifreq.ifr_ifru.ifru_mtu = mtu;

-        // SAFETY: ioctl is safe. Called with a valid sock fd, and we check the return.
-        unsafe { Self::ioctl_with_ref(&sock, net_gen::sockios::SIOCSIFMTU as c_ulong, &ifreq) }
+        // ioctl is safe. Called with a valid sock fd, and we check the return.
+        let ret = unsafe { ioctl_with_ref(&sock, net_gen::sockios::SIOCSIFMTU as c_ulong, &ifreq) };
+        if ret < 0 {
+            return Err(Error::IoctlError(IoError::last_os_error()));
+        }
+
+        Ok(())
    }

    /// Set the offload flags for the tap interface.
    pub fn set_offload(&self, flags: c_uint) -> Result<()> {
-        // SAFETY: ioctl is safe. Called with a valid tap fd, and we check the return.
-        unsafe { Self::ioctl_with_val(&self.tap_file, net_gen::TUNSETOFFLOAD(), flags as c_ulong) }
+        // ioctl is safe. Called with a valid tap fd, and we check the return.
+        let ret =
+            unsafe { ioctl_with_val(&self.tap_file, net_gen::TUNSETOFFLOAD(), flags as c_ulong) };
+        if ret < 0 {
+            return Err(Error::IoctlError(IoError::last_os_error()));
+        }
+
+        Ok(())
    }

    /// Enable the tap interface.
@@ -348,11 +346,13 @@ impl Tap {

        let mut ifreq = self.get_ifreq();

-        // SAFETY: IOCTL with correct arguments
-        unsafe { Self::ioctl_with_ref(&sock, net_gen::sockios::SIOCGIFFLAGS as c_ulong, &ifreq)? };
+        let ret =
+            unsafe { ioctl_with_ref(&sock, net_gen::sockios::SIOCGIFFLAGS as c_ulong, &ifreq) };
+        if ret < 0 {
+            return Err(Error::IoctlError(IoError::last_os_error()));
+        }

        // If TAP device is already up don't try and enable it
-        // SAFETY: access a union field
        let ifru_flags = unsafe { ifreq.ifr_ifru.ifru_flags };
        if ifru_flags & net_gen::net_device_flags_IFF_UP as i16
            == net_gen::net_device_flags_IFF_UP as i16
@@ -362,14 +362,25 @@ impl Tap {

        ifreq.ifr_ifru.ifru_flags = net_gen::net_device_flags_IFF_UP as i16;

-        // SAFETY: ioctl is safe. Called with a valid sock fd, and we check the return.
-        unsafe { Self::ioctl_with_ref(&sock, net_gen::sockios::SIOCSIFFLAGS as c_ulong, &ifreq) }
+        // ioctl is safe. Called with a valid sock fd, and we check the return.
+        let ret =
+            unsafe { ioctl_with_ref(&sock, net_gen::sockios::SIOCSIFFLAGS as c_ulong, &ifreq) };
+        if ret < 0 {
+            return Err(Error::IoctlError(IoError::last_os_error()));
+        }
+
+        Ok(())
    }

    /// Set the size of the vnet hdr.
    pub fn set_vnet_hdr_size(&self, size: c_int) -> Result<()> {
-        // SAFETY: ioctl is safe. Called with a valid tap fd, and we check the return.
-        unsafe { Self::ioctl_with_ref(&self.tap_file, net_gen::TUNSETVNETHDRSZ(), &size) }
+        // ioctl is safe. Called with a valid tap fd, and we check the return.
+        let ret = unsafe { ioctl_with_ref(&self.tap_file, net_gen::TUNSETVNETHDRSZ(), &size) };
+        if ret < 0 {
+            return Err(Error::IoctlError(IoError::last_os_error()));
+        }
+
+        Ok(())
    }

    fn get_ifreq(&self) -> net_gen::ifreq {
@@ -377,7 +388,6 @@ impl Tap {

        // This sets the name of the interface, which is the only entry
        // in a single-field union.
-        // SAFETY: access union fields and we're sure the copy is okay.
        unsafe {
            let ifrn_name = ifreq.ifr_ifrn.ifrn_name.as_mut();
            let name_slice = &mut ifrn_name[..self.if_name.len()];
@@ -390,11 +400,6 @@ impl Tap {
    pub fn get_if_name(&self) -> Vec<u8> {
        self.if_name.clone()
    }
-
-    #[cfg(fuzzing)]
-    pub fn new_for_fuzzing(tap_file: File, if_name: Vec<u8>) -> Self {
-        Tap { tap_file, if_name }
-    }
 }

 impl Read for Tap {
--- a/pci/Cargo.toml
+++ b/pci/Cargo.toml
@@ -10,20 +10,20 @@ kvm = ["vfio-ioctls/kvm"]
 mshv = ["vfio-ioctls/mshv"]

 [dependencies]
-anyhow = "1.0.68"
+anyhow = "1.0.66"
 byteorder = "1.4.3"
 hypervisor = { path = "../hypervisor" }
 vfio-bindings = { git = "https://github.com/rust-vmm/vfio", branch = "main", features = ["fam-wrappers"] }
 vfio-ioctls = { git = "https://github.com/rust-vmm/vfio", branch = "main", default-features = false }
 vfio_user = { path = "../vfio_user" }
 vmm-sys-util = "0.11.0"
-libc = "0.2.139"
+libc = "0.2.138"
 log = "0.4.17"
-serde = { version = "1.0.151", features = ["derive"] }
-thiserror = "1.0.38"
+serde = { version = "1.0.150", features = ["derive"] }
+thiserror = "1.0.37"
 versionize = "0.1.9"
 versionize_derive = "0.1.4"
 vm-allocator = { path = "../vm-allocator" }
 vm-device = { path = "../vm-device" }
 vm-memory = "0.10.0"
-vm-migration = { path = "../vm-migration" }
+vm-migration = { path = "../vm-migration" }
--- a/pci/src/bus.rs
+++ b/pci/src/bus.rs
@@ -62,7 +62,6 @@ impl PciRoot {
                    0,
                    0,
                    None,
-                    None,
                ),
            }
        }
--- a/pci/src/configuration.rs
+++ b/pci/src/configuration.rs
@@ -32,8 +32,6 @@ const CAPABILITY_MAX_OFFSET: usize = 192;

 const INTERRUPT_LINE_PIN_REG: usize = 15;

-pub const PCI_CONFIGURATION_ID: &str = "pci_configuration";
-
 /// Represents the types of PCI headers allowed in the configuration registers.
 #[derive(Copy, Clone)]
 pub enum PciHeaderType {
@@ -396,7 +394,7 @@ fn decode_64_bits_bar_size(bar_size_hi: u32, bar_size_lo: u32) -> Option<u64> {
    None
 }

-#[derive(Debug, Default, Clone, Copy, Versionize)]
+#[derive(Default, Clone, Copy, Versionize)]
 struct PciBar {
    addr: u32,
    size: u32,
@@ -405,7 +403,7 @@ struct PciBar {
 }

 #[derive(Versionize)]
-pub struct PciConfigurationState {
+struct PciConfigurationState {
    registers: Vec<u32>,
    writable_bits: Vec<u32>,
    bars: Vec<PciBar>,
@@ -555,78 +553,46 @@ impl PciConfiguration {
        subsystem_vendor_id: u16,
        subsystem_id: u16,
        msix_config: Option<Arc<Mutex<MsixConfig>>>,
-        state: Option<PciConfigurationState>,
    ) -> Self {
-        let (
-            registers,
-            writable_bits,
-            bars,
-            rom_bar_addr,
-            rom_bar_size,
-            rom_bar_used,
-            last_capability,
-            msix_cap_reg_idx,
-        ) = if let Some(state) = state {
-            (
-                state.registers.try_into().unwrap(),
-                state.writable_bits.try_into().unwrap(),
-                state.bars.try_into().unwrap(),
-                state.rom_bar_addr,
-                state.rom_bar_size,
-                state.rom_bar_used,
-                state.last_capability,
-                state.msix_cap_reg_idx,
-            )
+        let mut registers = [0u32; NUM_CONFIGURATION_REGISTERS];
+        let mut writable_bits = [0u32; NUM_CONFIGURATION_REGISTERS];
+        registers[0] = u32::from(device_id) << 16 | u32::from(vendor_id);
+        // TODO(dverkamp): Status should be write-1-to-clear
+        writable_bits[1] = 0x0000_ffff; // Status (r/o), command (r/w)
+        let pi = if let Some(pi) = programming_interface {
+            pi.get_register_value()
        } else {
-            let mut registers = [0u32; NUM_CONFIGURATION_REGISTERS];
-            let mut writable_bits = [0u32; NUM_CONFIGURATION_REGISTERS];
-            registers[0] = u32::from(device_id) << 16 | u32::from(vendor_id);
-            // TODO(dverkamp): Status should be write-1-to-clear
-            writable_bits[1] = 0x0000_ffff; // Status (r/o), command (r/w)
-            let pi = if let Some(pi) = programming_interface {
-                pi.get_register_value()
-            } else {
-                0
-            };
-            registers[2] = u32::from(class_code.get_register_value()) << 24
-                | u32::from(subclass.get_register_value()) << 16
-                | u32::from(pi) << 8
-                | u32::from(revision_id);
-            writable_bits[3] = 0x0000_00ff; // Cacheline size (r/w)
-            match header_type {
-                PciHeaderType::Device => {
-                    registers[3] = 0x0000_0000; // Header type 0 (device)
-                    writable_bits[15] = 0x0000_00ff; // Interrupt line (r/w)
-                }
-                PciHeaderType::Bridge => {
-                    registers[3] = 0x0001_0000; // Header type 1 (bridge)
-                    writable_bits[9] = 0xfff0_fff0; // Memory base and limit
-                    writable_bits[15] = 0xffff_00ff; // Bridge control (r/w), interrupt line (r/w)
-                }
-            };
-            registers[11] = u32::from(subsystem_id) << 16 | u32::from(subsystem_vendor_id);
-
-            (
-                registers,
-                writable_bits,
-                [PciBar::default(); NUM_BAR_REGS],
-                0,
-                0,
-                false,
-                None,
-                None,
-            )
+            0
        };
+        registers[2] = u32::from(class_code.get_register_value()) << 24
+            | u32::from(subclass.get_register_value()) << 16
+            | u32::from(pi) << 8
+            | u32::from(revision_id);
+        writable_bits[3] = 0x0000_00ff; // Cacheline size (r/w)
+        match header_type {
+            PciHeaderType::Device => {
+                registers[3] = 0x0000_0000; // Header type 0 (device)
+                writable_bits[15] = 0x0000_00ff; // Interrupt line (r/w)
+            }
+            PciHeaderType::Bridge => {
+                registers[3] = 0x0001_0000; // Header type 1 (bridge)
+                writable_bits[9] = 0xfff0_fff0; // Memory base and limit
+                writable_bits[15] = 0xffff_00ff; // Bridge control (r/w), interrupt line (r/w)
+            }
+        };
+        registers[11] = u32::from(subsystem_id) << 16 | u32::from(subsystem_vendor_id);
+
+        let bars = [PciBar::default(); NUM_BAR_REGS];

        PciConfiguration {
            registers,
            writable_bits,
            bars,
-            rom_bar_addr,
-            rom_bar_size,
-            rom_bar_used,
-            last_capability,
-            msix_cap_reg_idx,
+            rom_bar_addr: 0,
+            rom_bar_size: 0,
+            rom_bar_used: false,
+            last_capability: None,
+            msix_cap_reg_idx: None,
            msix_config,
        }
    }
@@ -644,6 +610,18 @@ impl PciConfiguration {
        }
    }

+    fn set_state(&mut self, state: &PciConfigurationState) {
+        self.registers.clone_from_slice(state.registers.as_slice());
+        self.writable_bits
+            .clone_from_slice(state.writable_bits.as_slice());
+        self.bars.clone_from_slice(state.bars.as_slice());
+        self.rom_bar_addr = state.rom_bar_addr;
+        self.rom_bar_size = state.rom_bar_size;
+        self.rom_bar_used = state.rom_bar_used;
+        self.last_capability = state.last_capability;
+        self.msix_cap_reg_idx = state.msix_cap_reg_idx;
+    }
+
    /// Reads a 32bit register from `reg_idx` in the register map.
    pub fn read_reg(&self, reg_idx: usize) -> u32 {
        *(self.registers.get(reg_idx).unwrap_or(&0xffff_ffff))
@@ -1068,11 +1046,16 @@ impl Pausable for PciConfiguration {}

 impl Snapshottable for PciConfiguration {
    fn id(&self) -> String {
-        String::from(PCI_CONFIGURATION_ID)
+        String::from("pci_configuration")
    }

    fn snapshot(&mut self) -> std::result::Result<Snapshot, MigratableError> {
-        Snapshot::new_from_versioned_state(&self.state())
+        Snapshot::new_from_versioned_state(&self.id(), &self.state())
+    }
+
+    fn restore(&mut self, snapshot: Snapshot) -> std::result::Result<(), MigratableError> {
+        self.set_state(&snapshot.to_versioned_state(&self.id())?);
+        Ok(())
    }
 }

@@ -1195,7 +1178,6 @@ mod tests {
            0xABCD,
            0x2468,
            None,
-            None,
        );

        // Add two capabilities with different contents.
@@ -1252,7 +1234,6 @@ mod tests {
            0xABCD,
            0x2468,
            None,
-            None,
        );

        let class_reg = cfg.read_reg(2);
--- a/pci/src/lib.rs
+++ b/pci/src/lib.rs
@@ -19,13 +19,12 @@ pub use self::configuration::{
    PciBarConfiguration, PciBarPrefetchable, PciBarRegionType, PciCapability, PciCapabilityId,
    PciClassCode, PciConfiguration, PciExpressCapabilityId, PciHeaderType, PciMassStorageSubclass,
    PciNetworkControllerSubclass, PciProgrammingInterface, PciSerialBusSubClass, PciSubclass,
-    PCI_CONFIGURATION_ID,
 };
 pub use self::device::{
    BarReprogrammingParams, DeviceRelocation, Error as PciDeviceError, PciDevice,
 };
 pub use self::msi::{msi_num_enabled_vectors, MsiCap, MsiConfig};
-pub use self::msix::{MsixCap, MsixConfig, MsixTableEntry, MSIX_CONFIG_ID, MSIX_TABLE_ENTRY_SIZE};
+pub use self::msix::{MsixCap, MsixConfig, MsixTableEntry, MSIX_TABLE_ENTRY_SIZE};
 pub use self::vfio::{VfioPciDevice, VfioPciError};
 pub use self::vfio_user::{VfioUserDmaMapping, VfioUserPciDevice, VfioUserPciDeviceError};
 use serde::de::Visitor;
--- a/pci/src/msi.rs
+++ b/pci/src/msi.rs
@@ -3,6 +3,7 @@
 // SPDX-License-Identifier: Apache-2.0 OR BSD-3-Clause
 //

+use anyhow::anyhow;
 use byteorder::{ByteOrder, LittleEndian};
 use std::io;
 use std::sync::Arc;
@@ -38,15 +39,13 @@ pub fn msi_num_enabled_vectors(msg_ctl: u16) -> usize {
 }

 #[derive(Error, Debug)]
-pub enum Error {
+enum Error {
    #[error("Failed enabling the interrupt route: {0}")]
    EnableInterruptRoute(io::Error),
    #[error("Failed updating the interrupt route: {0}")]
    UpdateInterruptRoute(io::Error),
 }

-pub const MSI_CONFIG_ID: &str = "msi_config";
-
 #[derive(Clone, Copy, Default, Versionize)]
 pub struct MsiCap {
    // Message Control Register
@@ -173,7 +172,7 @@ impl MsiCap {
 }

 #[derive(Versionize)]
-pub struct MsiConfigState {
+struct MsiConfigState {
    cap: MsiCap,
 }

@@ -185,53 +184,51 @@ pub struct MsiConfig {
 }

 impl MsiConfig {
-    pub fn new(
-        msg_ctl: u16,
-        interrupt_source_group: Arc<dyn InterruptSourceGroup>,
-        state: Option<MsiConfigState>,
-    ) -> Result<Self, Error> {
-        let cap = if let Some(state) = state {
-            if state.cap.enabled() {
-                for idx in 0..state.cap.num_enabled_vectors() {
-                    let config = MsiIrqSourceConfig {
-                        high_addr: state.cap.msg_addr_hi,
-                        low_addr: state.cap.msg_addr_lo,
-                        data: state.cap.msg_data as u32,
-                        devid: 0,
-                    };
-
-                    interrupt_source_group
-                        .update(
-                            idx as InterruptIndex,
-                            InterruptSourceConfig::MsiIrq(config),
-                            state.cap.vector_masked(idx),
-                        )
-                        .map_err(Error::UpdateInterruptRoute)?;
-                }
-
-                interrupt_source_group
-                    .enable()
-                    .map_err(Error::EnableInterruptRoute)?;
-            }
-
-            state.cap
-        } else {
-            MsiCap {
-                msg_ctl,
-                ..Default::default()
-            }
+    pub fn new(msg_ctl: u16, interrupt_source_group: Arc<dyn InterruptSourceGroup>) -> Self {
+        let cap = MsiCap {
+            msg_ctl,
+            ..Default::default()
        };

-        Ok(MsiConfig {
+        MsiConfig {
            cap,
            interrupt_source_group,
-        })
+        }
    }

    fn state(&self) -> MsiConfigState {
        MsiConfigState { cap: self.cap }
    }

+    fn set_state(&mut self, state: &MsiConfigState) -> Result<(), Error> {
+        self.cap = state.cap;
+
+        if self.enabled() {
+            for idx in 0..self.num_enabled_vectors() {
+                let config = MsiIrqSourceConfig {
+                    high_addr: self.cap.msg_addr_hi,
+                    low_addr: self.cap.msg_addr_lo,
+                    data: self.cap.msg_data as u32,
+                    devid: 0,
+                };
+
+                self.interrupt_source_group
+                    .update(
+                        idx as InterruptIndex,
+                        InterruptSourceConfig::MsiIrq(config),
+                        self.cap.vector_masked(idx),
+                    )
+                    .map_err(Error::UpdateInterruptRoute)?;
+            }
+
+            self.interrupt_source_group
+                .enable()
+                .map_err(Error::EnableInterruptRoute)?;
+        }
+
+        Ok(())
+    }
+
    pub fn enabled(&self) -> bool {
        self.cap.enabled()
    }
@@ -284,10 +281,21 @@ impl Pausable for MsiConfig {}

 impl Snapshottable for MsiConfig {
    fn id(&self) -> String {
-        String::from(MSI_CONFIG_ID)
+        String::from("msi_config")
    }

    fn snapshot(&mut self) -> std::result::Result<Snapshot, MigratableError> {
-        Snapshot::new_from_versioned_state(&self.state())
+        Snapshot::new_from_versioned_state(&self.id(), &self.state())
+    }
+
+    fn restore(&mut self, snapshot: Snapshot) -> std::result::Result<(), MigratableError> {
+        self.set_state(&snapshot.to_versioned_state(&self.id())?)
+            .map_err(|e| {
+                MigratableError::Restore(anyhow!(
+                    "Could not restore state for {}: {:?}",
+                    self.id(),
+                    e
+                ))
+            })
    }
 }
--- a/pci/src/msix.rs
+++ b/pci/src/msix.rs
@@ -4,6 +4,7 @@
 //

 use crate::{PciCapability, PciCapabilityId};
+use anyhow::anyhow;
 use byteorder::{ByteOrder, LittleEndian};
 use std::io;
 use std::result;
@@ -25,10 +26,9 @@ const MSIX_ENABLE_BIT: u8 = 15;
 const FUNCTION_MASK_MASK: u16 = (1 << FUNCTION_MASK_BIT) as u16;
 const MSIX_ENABLE_MASK: u16 = (1 << MSIX_ENABLE_BIT) as u16;
 pub const MSIX_TABLE_ENTRY_SIZE: usize = 16;
-pub const MSIX_CONFIG_ID: &str = "msix_config";

 #[derive(Debug)]
-pub enum Error {
+enum Error {
    /// Failed enabling the interrupt route.
    EnableInterruptRoute(io::Error),
    /// Failed updating the interrupt route.
@@ -61,7 +61,7 @@ impl Default for MsixTableEntry {
 }

 #[derive(Versionize)]
-pub struct MsixConfigState {
+struct MsixConfigState {
    table_entries: Vec<MsixTableEntry>,
    pba_entries: Vec<u64>,
    masked: bool,
@@ -84,62 +84,23 @@ impl MsixConfig {
        msix_vectors: u16,
        interrupt_source_group: Arc<dyn InterruptSourceGroup>,
        devid: u32,
-        state: Option<MsixConfigState>,
-    ) -> result::Result<Self, Error> {
+    ) -> Self {
        assert!(msix_vectors <= MAX_MSIX_VECTORS_PER_DEVICE);

-        let (table_entries, pba_entries, masked, enabled) = if let Some(state) = state {
-            if state.enabled && !state.masked {
-                for (idx, table_entry) in state.table_entries.iter().enumerate() {
-                    if table_entry.masked() {
-                        continue;
-                    }
+        let mut table_entries: Vec<MsixTableEntry> = Vec::new();
+        table_entries.resize_with(msix_vectors as usize, Default::default);
+        let mut pba_entries: Vec<u64> = Vec::new();
+        let num_pba_entries: usize = ((msix_vectors as usize) / BITS_PER_PBA_ENTRY) + 1;
+        pba_entries.resize_with(num_pba_entries, Default::default);

-                    let config = MsiIrqSourceConfig {
-                        high_addr: table_entry.msg_addr_hi,
-                        low_addr: table_entry.msg_addr_lo,
-                        data: table_entry.msg_data,
-                        devid,
-                    };
-
-                    interrupt_source_group
-                        .update(
-                            idx as InterruptIndex,
-                            InterruptSourceConfig::MsiIrq(config),
-                            state.masked,
-                        )
-                        .map_err(Error::UpdateInterruptRoute)?;
-
-                    interrupt_source_group
-                        .enable()
-                        .map_err(Error::EnableInterruptRoute)?;
-                }
-            }
-
-            (
-                state.table_entries,
-                state.pba_entries,
-                state.masked,
-                state.enabled,
-            )
-        } else {
-            let mut table_entries: Vec<MsixTableEntry> = Vec::new();
-            table_entries.resize_with(msix_vectors as usize, Default::default);
-            let mut pba_entries: Vec<u64> = Vec::new();
-            let num_pba_entries: usize = ((msix_vectors as usize) / BITS_PER_PBA_ENTRY) + 1;
-            pba_entries.resize_with(num_pba_entries, Default::default);
-
-            (table_entries, pba_entries, true, false)
-        };
-
-        Ok(MsixConfig {
+        MsixConfig {
            table_entries,
            pba_entries,
            devid,
            interrupt_source_group,
-            masked,
-            enabled,
-        })
+            masked: true,
+            enabled: false,
+        }
    }

    fn state(&self) -> MsixConfigState {
@@ -151,6 +112,42 @@ impl MsixConfig {
        }
    }

+    fn set_state(&mut self, state: &MsixConfigState) -> result::Result<(), Error> {
+        self.table_entries = state.table_entries.clone();
+        self.pba_entries = state.pba_entries.clone();
+        self.masked = state.masked;
+        self.enabled = state.enabled;
+
+        if self.enabled && !self.masked {
+            for (idx, table_entry) in self.table_entries.iter().enumerate() {
+                if table_entry.masked() {
+                    continue;
+                }
+
+                let config = MsiIrqSourceConfig {
+                    high_addr: table_entry.msg_addr_hi,
+                    low_addr: table_entry.msg_addr_lo,
+                    data: table_entry.msg_data,
+                    devid: self.devid,
+                };
+
+                self.interrupt_source_group
+                    .update(
+                        idx as InterruptIndex,
+                        InterruptSourceConfig::MsiIrq(config),
+                        self.masked,
+                    )
+                    .map_err(Error::UpdateInterruptRoute)?;
+
+                self.interrupt_source_group
+                    .enable()
+                    .map_err(Error::EnableInterruptRoute)?;
+            }
+        }
+
+        Ok(())
+    }
+
    pub fn masked(&self) -> bool {
        self.masked
    }
@@ -429,11 +426,22 @@ impl Pausable for MsixConfig {}

 impl Snapshottable for MsixConfig {
    fn id(&self) -> String {
-        String::from(MSIX_CONFIG_ID)
+        String::from("msix_config")
    }

    fn snapshot(&mut self) -> std::result::Result<Snapshot, MigratableError> {
-        Snapshot::new_from_versioned_state(&self.state())
+        Snapshot::new_from_versioned_state(&self.id(), &self.state())
+    }
+
+    fn restore(&mut self, snapshot: Snapshot) -> std::result::Result<(), MigratableError> {
+        self.set_state(&snapshot.to_versioned_state(&self.id())?)
+            .map_err(|e| {
+                MigratableError::Restore(anyhow!(
+                    "Could not restore state for {}: {:?}",
+                    self.id(),
+                    e
+                ))
+            })
    }
 }

--- a/pci/src/vfio.rs
+++ b/pci/src/vfio.rs
@@ -3,17 +3,16 @@
 // SPDX-License-Identifier: Apache-2.0 OR BSD-3-Clause
 //

-use crate::msi::{MsiConfigState, MSI_CONFIG_ID};
-use crate::msix::MsixConfigState;
 use crate::{
    msi_num_enabled_vectors, BarReprogrammingParams, MsiCap, MsiConfig, MsixCap, MsixConfig,
    PciBarConfiguration, PciBarPrefetchable, PciBarRegionType, PciBdf, PciCapabilityId,
    PciClassCode, PciConfiguration, PciDevice, PciDeviceError, PciExpressCapabilityId,
-    PciHeaderType, PciSubclass, MSIX_CONFIG_ID, MSIX_TABLE_ENTRY_SIZE, PCI_CONFIGURATION_ID,
+    PciHeaderType, PciSubclass, MSIX_TABLE_ENTRY_SIZE,
 };
 use anyhow::anyhow;
 use byteorder::{ByteOrder, LittleEndian};
 use hypervisor::HypervisorVmError;
+use libc::{sysconf, _SC_PAGESIZE};
 use std::any::Any;
 use std::collections::{BTreeMap, HashMap};
 use std::io;
@@ -38,8 +37,6 @@ use vm_migration::{
 };
 use vmm_sys_util::eventfd::EventFd;

-pub(crate) const VFIO_COMMON_ID: &str = "vfio_common";
-
 #[derive(Debug, Error)]
 pub enum VfioPciError {
    #[error("Failed to create user memory region: {0}")]
@@ -62,14 +59,6 @@ pub enum VfioPciError {
    RegionAlignment,
    #[error("Invalid region size")]
    RegionSize,
-    #[error("Failed to retrieve MsiConfigState: {0}")]
-    RetrieveMsiConfigState(#[source] anyhow::Error),
-    #[error("Failed to retrieve MsixConfigState: {0}")]
-    RetrieveMsixConfigState(#[source] anyhow::Error),
-    #[error("Failed to retrieve PciConfigurationState: {0}")]
-    RetrievePciConfigurationState(#[source] anyhow::Error),
-    #[error("Failed to retrieve VfioCommonState: {0}")]
-    RetrieveVfioCommonState(#[source] anyhow::Error),
 }

 #[derive(Copy, Clone)]
@@ -418,86 +407,6 @@ pub(crate) struct VfioCommon {
 }

 impl VfioCommon {
-    pub(crate) fn new(
-        msi_interrupt_manager: Arc<dyn InterruptManager<GroupConfig = MsiIrqGroupConfig>>,
-        legacy_interrupt_group: Option<Arc<dyn InterruptSourceGroup>>,
-        vfio_wrapper: Arc<dyn Vfio>,
-        subclass: &dyn PciSubclass,
-        bdf: PciBdf,
-        snapshot: Option<Snapshot>,
-    ) -> Result<Self, VfioPciError> {
-        let pci_configuration_state =
-            vm_migration::versioned_state_from_id(snapshot.as_ref(), PCI_CONFIGURATION_ID)
-                .map_err(|e| {
-                    VfioPciError::RetrievePciConfigurationState(anyhow!(
-                        "Failed to get PciConfigurationState from Snapshot: {}",
-                        e
-                    ))
-                })?;
-
-        let configuration = PciConfiguration::new(
-            0,
-            0,
-            0,
-            PciClassCode::Other,
-            subclass,
-            None,
-            PciHeaderType::Device,
-            0,
-            0,
-            None,
-            pci_configuration_state,
-        );
-
-        let mut vfio_common = VfioCommon {
-            mmio_regions: Vec::new(),
-            configuration,
-            interrupt: Interrupt {
-                intx: None,
-                msi: None,
-                msix: None,
-            },
-            msi_interrupt_manager,
-            legacy_interrupt_group,
-            vfio_wrapper,
-            patches: HashMap::new(),
-        };
-
-        let state: Option<VfioCommonState> = snapshot
-            .as_ref()
-            .map(|s| s.to_versioned_state())
-            .transpose()
-            .map_err(|e| {
-                VfioPciError::RetrieveVfioCommonState(anyhow!(
-                    "Failed to get VfioCommonState from Snapshot: {}",
-                    e
-                ))
-            })?;
-        let msi_state = vm_migration::versioned_state_from_id(snapshot.as_ref(), MSI_CONFIG_ID)
-            .map_err(|e| {
-                VfioPciError::RetrieveMsiConfigState(anyhow!(
-                    "Failed to get MsiConfigState from Snapshot: {}",
-                    e
-                ))
-            })?;
-        let msix_state = vm_migration::versioned_state_from_id(snapshot.as_ref(), MSIX_CONFIG_ID)
-            .map_err(|e| {
-            VfioPciError::RetrieveMsixConfigState(anyhow!(
-                "Failed to get MsixConfigState from Snapshot: {}",
-                e
-            ))
-        })?;
-
-        if let Some(state) = state.as_ref() {
-            vfio_common.set_state(state, msi_state, msix_state)?;
-        } else {
-            vfio_common.parse_capabilities(bdf);
-            vfio_common.initialize_legacy_interrupt()?;
-        }
-
-        Ok(vfio_common)
-    }
-
    pub(crate) fn allocate_bars(
        &mut self,
        allocator: &Arc<Mutex<SystemAllocator>>,
@@ -663,7 +572,12 @@ impl VfioCommon {
                PciBarRegionType::Memory64BitRegion => {
                    // BAR allocation must be naturally aligned
                    mmio_allocator
-                        .allocate(restored_bar_addr, region_size, Some(region_size))
+                        .allocate(
+                            restored_bar_addr,
+                            region_size,
+                            // SAFETY: FFI call. Trivially safe.
+                            Some(unsafe { sysconf(_SC_PAGESIZE) as GuestUsize }),
+                        )
                        .ok_or(PciDeviceError::IoAllocationFailed(region_size))?
                }
            };
@@ -742,13 +656,7 @@ impl VfioCommon {
        }
    }

-    pub(crate) fn initialize_msix(
-        &mut self,
-        msix_cap: MsixCap,
-        cap_offset: u32,
-        bdf: PciBdf,
-        state: Option<MsixConfigState>,
-    ) {
+    pub(crate) fn initialize_msix(&mut self, msix_cap: MsixCap, cap_offset: u32, bdf: PciBdf) {
        let interrupt_source_group = self
            .msi_interrupt_manager
            .create_group(MsiIrqGroupConfig {
@@ -761,9 +669,7 @@ impl VfioCommon {
            msix_cap.table_size(),
            interrupt_source_group.clone(),
            bdf.into(),
-            state,
-        )
-        .unwrap();
+        );

        self.interrupt.msix = Some(VfioMsix {
            bar: msix_config,
@@ -777,12 +683,7 @@ impl VfioCommon {
        self.vfio_wrapper.read_config_word((cap + 2).into())
    }

-    pub(crate) fn initialize_msi(
-        &mut self,
-        msg_ctl: u16,
-        cap_offset: u32,
-        state: Option<MsiConfigState>,
-    ) {
+    pub(crate) fn initialize_msi(&mut self, msg_ctl: u16, cap_offset: u32) {
        let interrupt_source_group = self
            .msi_interrupt_manager
            .create_group(MsiIrqGroupConfig {
@@ -791,7 +692,7 @@ impl VfioCommon {
            })
            .unwrap();

-        let msi_config = MsiConfig::new(msg_ctl, interrupt_source_group.clone(), state).unwrap();
+        let msi_config = MsiConfig::new(msg_ctl, interrupt_source_group.clone());

        self.interrupt.msi = Some(VfioMsi {
            cfg: msi_config,
@@ -818,7 +719,7 @@ impl VfioCommon {
                            // Parse capability only if the VFIO device
                            // supports MSI.
                            let msg_ctl = self.parse_msi_capabilities(cap_next);
-                            self.initialize_msi(msg_ctl, cap_next as u32, None);
+                            self.initialize_msi(msg_ctl, cap_next as u32);
                        }
                    }
                }
@@ -829,7 +730,7 @@ impl VfioCommon {
                            // Parse capability only if the VFIO device
                            // supports MSI-X.
                            let msix_cap = self.parse_msix_capabilities(cap_next);
-                            self.initialize_msix(msix_cap, cap_next as u32, bdf, None);
+                            self.initialize_msix(msix_cap, cap_next as u32, bdf);
                        }
                    }
                }
@@ -1196,12 +1097,7 @@ impl VfioCommon {
        }
    }

-    fn set_state(
-        &mut self,
-        state: &VfioCommonState,
-        msi_state: Option<MsiConfigState>,
-        msix_state: Option<MsixConfigState>,
-    ) -> Result<(), VfioPciError> {
+    fn set_state(&mut self, state: &VfioCommonState) -> Result<(), VfioPciError> {
        if let (Some(intx), Some(interrupt_source_group)) =
            (&state.intx_state, self.legacy_interrupt_group.clone())
        {
@@ -1216,11 +1112,11 @@ impl VfioCommon {
        }

        if let Some(msi) = &state.msi_state {
-            self.initialize_msi(msi.cap.msg_ctl, msi.cap_offset, msi_state);
+            self.initialize_msi(msi.cap.msg_ctl, msi.cap_offset);
        }

        if let Some(msix) = &state.msix_state {
-            self.initialize_msix(msix.cap, msix.cap_offset, msix.bdf.into(), msix_state);
+            self.initialize_msix(msix.cap, msix.cap_offset, msix.bdf.into());
        }

        Ok(())
@@ -1231,27 +1127,73 @@ impl Pausable for VfioCommon {}

 impl Snapshottable for VfioCommon {
    fn id(&self) -> String {
-        String::from(VFIO_COMMON_ID)
+        String::from("vfio_common")
    }

    fn snapshot(&mut self) -> std::result::Result<Snapshot, MigratableError> {
-        let mut vfio_common_snapshot = Snapshot::new_from_versioned_state(&self.state())?;
+        let mut vfio_common_snapshot =
+            Snapshot::new_from_versioned_state(&self.id(), &self.state())?;

        // Snapshot PciConfiguration
-        vfio_common_snapshot.add_snapshot(self.configuration.id(), self.configuration.snapshot()?);
+        vfio_common_snapshot.add_snapshot(self.configuration.snapshot()?);

        // Snapshot MSI
        if let Some(msi) = &mut self.interrupt.msi {
-            vfio_common_snapshot.add_snapshot(msi.cfg.id(), msi.cfg.snapshot()?);
+            vfio_common_snapshot.add_snapshot(msi.cfg.snapshot()?);
        }

        // Snapshot MSI-X
        if let Some(msix) = &mut self.interrupt.msix {
-            vfio_common_snapshot.add_snapshot(msix.bar.id(), msix.bar.snapshot()?);
+            vfio_common_snapshot.add_snapshot(msix.bar.snapshot()?);
        }

        Ok(vfio_common_snapshot)
    }
+
+    fn restore(&mut self, snapshot: Snapshot) -> std::result::Result<(), MigratableError> {
+        if let Some(vfio_common_section) = snapshot
+            .snapshot_data
+            .get(&format!("{}-section", self.id()))
+        {
+            // It has to be invoked first as we want Interrupt to be initialized
+            // correctly before we try to restore MSI and MSI-X configurations.
+            self.set_state(&vfio_common_section.to_versioned_state()?)
+                .map_err(|e| {
+                    MigratableError::Restore(anyhow!("Could not restore VFIO_COMMON state {:?}", e))
+                })?;
+
+            // Restore PciConfiguration
+            if let Some(pci_config_snapshot) = snapshot.snapshots.get(&self.configuration.id()) {
+                self.configuration.restore(*pci_config_snapshot.clone())?;
+            }
+
+            // Restore MSI
+            if let Some(msi) = &mut self.interrupt.msi {
+                if let Some(msi_snapshot) = snapshot.snapshots.get(&msi.cfg.id()) {
+                    msi.cfg.restore(*msi_snapshot.clone())?;
+                }
+                if msi.cfg.enabled() {
+                    self.enable_msi().unwrap();
+                }
+            }
+
+            // Restore MSI-X
+            if let Some(msix) = &mut self.interrupt.msix {
+                if let Some(msix_snapshot) = snapshot.snapshots.get(&msix.bar.id()) {
+                    msix.bar.restore(*msix_snapshot.clone())?;
+                }
+                if msix.bar.enabled() {
+                    self.enable_msix().unwrap();
+                }
+            }
+
+            return Ok(());
+        }
+
+        Err(MigratableError::Restore(anyhow!(
+            "Could not find VFIO_COMMON snapshot section"
+        )))
+    }
 }

 /// VfioPciDevice represents a VFIO PCI device.
@@ -1282,22 +1224,48 @@ impl VfioPciDevice {
        legacy_interrupt_group: Option<Arc<dyn InterruptSourceGroup>>,
        iommu_attached: bool,
        bdf: PciBdf,
+        restoring: bool,
        memory_slot: Arc<dyn Fn() -> u32 + Send + Sync>,
-        snapshot: Option<Snapshot>,
    ) -> Result<Self, VfioPciError> {
        let device = Arc::new(device);
        device.reset();

+        let configuration = PciConfiguration::new(
+            0,
+            0,
+            0,
+            PciClassCode::Other,
+            &PciVfioSubclass::VfioSubclass,
+            None,
+            PciHeaderType::Device,
+            0,
+            0,
+            None,
+        );
+
        let vfio_wrapper = VfioDeviceWrapper::new(Arc::clone(&device));

-        let common = VfioCommon::new(
+        let mut common = VfioCommon {
+            mmio_regions: Vec::new(),
+            configuration,
+            interrupt: Interrupt {
+                intx: None,
+                msi: None,
+                msix: None,
+            },
            msi_interrupt_manager,
            legacy_interrupt_group,
-            Arc::new(vfio_wrapper) as Arc<dyn Vfio>,
-            &PciVfioSubclass::VfioSubclass,
-            bdf,
-            vm_migration::snapshot_from_id(snapshot.as_ref(), VFIO_COMMON_ID),
-        )?;
+            vfio_wrapper: Arc::new(vfio_wrapper) as Arc<dyn Vfio>,
+            patches: HashMap::new(),
+        };
+
+        // No need to parse capabilities from the device if on the restore path.
+        // The initialization will be performed later when restore() will be
+        // called.
+        if !restoring {
+            common.parse_capabilities(bdf);
+            common.initialize_legacy_interrupt()?;
+        }

        let vfio_pci_device = VfioPciDevice {
            id,
@@ -1316,8 +1284,10 @@ impl VfioPciDevice {
        self.iommu_attached
    }

-    fn align_4k(address: u64) -> u64 {
-        (address + 0xfff) & 0xffff_ffff_ffff_f000
+    fn align_page_size(address: u64) -> u64 {
+        // SAFETY: FFI call. Trivially safe.
+        let page_size = unsafe { sysconf(_SC_PAGESIZE) as u64 };
+        (address + page_size - 1) & !(page_size - 1)
    }

    fn is_4k_aligned(address: u64) -> bool {
@@ -1377,6 +1347,7 @@ impl VfioPciDevice {
                    let mut sparse_areas = Vec::new();
                    let mut current_offset = 0;
                    for (range_offset, range_size) in inter_ranges {
+                        let range_offset = Self::align_page_size(range_offset);
                        if range_offset > current_offset {
                            sparse_areas.push(VfioRegionSparseMmapArea {
                                offset: current_offset,
@@ -1384,13 +1355,13 @@ impl VfioPciDevice {
                            });
                        }

-                        current_offset = Self::align_4k(range_offset + range_size);
+                        current_offset = Self::align_page_size(range_offset + range_size);
                    }

                    if region_size > current_offset {
                        sparse_areas.push(VfioRegionSparseMmapArea {
-                            offset: current_offset,
-                            size: region_size - current_offset,
+                            offset: Self::align_page_size(current_offset),
+                            size: Self::align_page_size(region_size - current_offset),
                        });
                    }

@@ -1460,7 +1431,6 @@ impl VfioPciDevice {
                )?;

                for area in sparse_areas.iter() {
-                    // SAFETY: FFI call with correct arguments
                    let host_addr = unsafe {
                        libc::mmap(
                            null_mut(),
@@ -1527,7 +1497,6 @@ impl VfioPciDevice {
                    error!("Could not remove the userspace memory region: {}", e);
                }

-                // SAFETY: FFI call with correct arguments
                let ret = unsafe {
                    libc::munmap(
                        user_memory_region.host_addr as *mut libc::c_void,
@@ -1739,13 +1708,28 @@ impl Snapshottable for VfioPciDevice {
    }

    fn snapshot(&mut self) -> std::result::Result<Snapshot, MigratableError> {
-        let mut vfio_pci_dev_snapshot = Snapshot::default();
+        let mut vfio_pci_dev_snapshot = Snapshot::new(&self.id);

        // Snapshot VfioCommon
-        vfio_pci_dev_snapshot.add_snapshot(self.common.id(), self.common.snapshot()?);
+        vfio_pci_dev_snapshot.add_snapshot(self.common.snapshot()?);

        Ok(vfio_pci_dev_snapshot)
    }
+
+    fn restore(&mut self, snapshot: Snapshot) -> std::result::Result<(), MigratableError> {
+        // Restore VfioCommon
+        if let Some(vfio_common_snapshot) = snapshot.snapshots.get(&self.common.id()) {
+            self.common.restore(*vfio_common_snapshot.clone())?;
+            self.map_mmio_regions().map_err(|e| {
+                MigratableError::Restore(anyhow!(
+                    "Could not map MMIO regions for VfioPciDevice on restore {:?}",
+                    e
+                ))
+            })?;
+        }
+
+        Ok(())
+    }
 }
 impl Transportable for VfioPciDevice {}
 impl Migratable for VfioPciDevice {}
--- a/pci/src/vfio_user.rs
+++ b/pci/src/vfio_user.rs
@@ -3,11 +3,15 @@
 // SPDX-License-Identifier: Apache-2.0
 //

-use crate::vfio::{UserMemoryRegion, Vfio, VfioCommon, VfioError, VFIO_COMMON_ID};
+use crate::vfio::{Interrupt, UserMemoryRegion, Vfio, VfioCommon, VfioError};
 use crate::{BarReprogrammingParams, PciBarConfiguration, VfioPciError};
-use crate::{PciBdf, PciDevice, PciDeviceError, PciSubclass};
+use crate::{
+    PciBdf, PciClassCode, PciConfiguration, PciDevice, PciDeviceError, PciHeaderType, PciSubclass,
+};
+use anyhow::anyhow;
 use hypervisor::HypervisorVmError;
 use std::any::Any;
+use std::collections::HashMap;
 use std::os::unix::prelude::AsRawFd;
 use std::ptr::null_mut;
 use std::sync::{Arc, Barrier, Mutex};
@@ -47,8 +51,6 @@ pub enum VfioUserPciDeviceError {
    DmaUnmap(#[source] VfioUserError),
    #[error("Failed to initialize legacy interrupts: {0}")]
    InitializeLegacyInterrupts(#[source] VfioPciError),
-    #[error("Failed to create VfioCommon: {0}")]
-    CreateVfioCommon(#[source] VfioPciError),
 }

 #[derive(Copy, Clone)]
@@ -71,9 +73,22 @@ impl VfioUserPciDevice {
        msi_interrupt_manager: Arc<dyn InterruptManager<GroupConfig = MsiIrqGroupConfig>>,
        legacy_interrupt_group: Option<Arc<dyn InterruptSourceGroup>>,
        bdf: PciBdf,
+        restoring: bool,
        memory_slot: Arc<dyn Fn() -> u32 + Send + Sync>,
-        snapshot: Option<Snapshot>,
    ) -> Result<Self, VfioUserPciDeviceError> {
+        // This is used for the BAR and capabilities only
+        let configuration = PciConfiguration::new(
+            0,
+            0,
+            0,
+            PciClassCode::Other,
+            &PciVfioUserSubclass::VfioUserSubclass,
+            None,
+            PciHeaderType::Device,
+            0,
+            0,
+            None,
+        );
        let resettable = client.lock().unwrap().resettable();
        if resettable {
            client
@@ -87,15 +102,29 @@ impl VfioUserPciDevice {
            client: client.clone(),
        };

-        let common = VfioCommon::new(
+        let mut common = VfioCommon {
+            mmio_regions: Vec::new(),
+            configuration,
+            interrupt: Interrupt {
+                intx: None,
+                msi: None,
+                msix: None,
+            },
            msi_interrupt_manager,
            legacy_interrupt_group,
-            Arc::new(vfio_wrapper) as Arc<dyn Vfio>,
-            &PciVfioUserSubclass::VfioUserSubclass,
-            bdf,
-            vm_migration::snapshot_from_id(snapshot.as_ref(), VFIO_COMMON_ID),
-        )
-        .map_err(VfioUserPciDeviceError::CreateVfioCommon)?;
+            vfio_wrapper: Arc::new(vfio_wrapper) as Arc<dyn Vfio>,
+            patches: HashMap::new(),
+        };
+
+        // No need to parse capabilities from the device if on the restore path.
+        // The initialization will be performed later when restore() will be
+        // called.
+        if !restoring {
+            common.parse_capabilities(bdf);
+            common
+                .initialize_legacy_interrupt()
+                .map_err(VfioUserPciDeviceError::InitializeLegacyInterrupts)?;
+        }

        Ok(Self {
            id,
@@ -152,7 +181,6 @@ impl VfioUserPciDevice {
                };

                for s in mmaps.iter() {
-                    // SAFETY: FFI call with correct arguments
                    let host_addr = unsafe {
                        libc::mmap(
                            null_mut(),
@@ -219,7 +247,6 @@ impl VfioUserPciDevice {
                }

                // Remove mmaps
-                // SAFETY: FFI call with correct arguments
                let ret = unsafe {
                    libc::munmap(
                        user_memory_region.host_addr as *mut libc::c_void,
@@ -535,13 +562,28 @@ impl Snapshottable for VfioUserPciDevice {
    }

    fn snapshot(&mut self) -> std::result::Result<Snapshot, MigratableError> {
-        let mut vfio_pci_dev_snapshot = Snapshot::default();
+        let mut vfio_pci_dev_snapshot = Snapshot::new(&self.id);

        // Snapshot VfioCommon
-        vfio_pci_dev_snapshot.add_snapshot(self.common.id(), self.common.snapshot()?);
+        vfio_pci_dev_snapshot.add_snapshot(self.common.snapshot()?);

        Ok(vfio_pci_dev_snapshot)
    }
+
+    fn restore(&mut self, snapshot: Snapshot) -> std::result::Result<(), MigratableError> {
+        // Restore VfioCommon
+        if let Some(vfio_common_snapshot) = snapshot.snapshots.get(&self.common.id()) {
+            self.common.restore(*vfio_common_snapshot.clone())?;
+            self.map_mmio_regions().map_err(|e| {
+                MigratableError::Restore(anyhow!(
+                    "Could not map MMIO regions for VfioUserPciDevice on restore {:?}",
+                    e
+                ))
+            })?;
+        }
+
+        Ok(())
+    }
 }
 impl Transportable for VfioUserPciDevice {}
 impl Migratable for VfioUserPciDevice {}
--- a/performance-metrics/Cargo.toml
+++ b/performance-metrics/Cargo.toml
@@ -6,10 +6,13 @@ edition = "2021"
 build = "build.rs"

 [dependencies]
-clap = { version = "4.0.32", features = ["wrap_help"] }
+clap = { version = "4.0.29", features = ["wrap_help","cargo"] }
 dirs = "4.0.0"
-serde = { version = "1.0.151", features = ["rc", "derive"] }
+serde = { version = "1.0.150", features = ["rc", "derive"] }
 serde_json = "1.0.89"
 test_infra = { path = "../test_infra" }
-thiserror = "1.0.38"
+thiserror = "1.0.37"
 wait-timeout = "0.2.0"
+
+[build-dependencies]
+clap = { version = "4.0.29", features = ["cargo"] }
--- a/performance-metrics/build.rs
+++ b/performance-metrics/build.rs
@@ -3,11 +3,13 @@
 // SPDX-License-Identifier: Apache-2.0
 //

+#[macro_use(crate_version)]
+extern crate clap;
+
 use std::process::Command;

 fn main() {
-    let mut git_human_readable = "v".to_owned() + env!("CARGO_PKG_VERSION");
-
+    let mut git_human_readable = "v".to_owned() + crate_version!();
    if let Ok(git_out) = Command::new("git").args(["describe", "--dirty"]).output() {
        if git_out.status.success() {
            if let Ok(git_out_str) = String::from_utf8(git_out.stdout) {
--- a/performance-metrics/src/main.rs
+++ b/performance-metrics/src/main.rs
@@ -5,6 +5,8 @@

 // Custom harness to run performance tests
 extern crate test_infra;
+#[macro_use(crate_authors)]
+extern crate clap;

 mod performance_tests;

@@ -127,8 +129,8 @@ pub struct PerformanceTestControl {
    test_iterations: u32,
    num_queues: Option<u32>,
    queue_size: Option<u32>,
-    net_control: Option<(bool, bool)>, // First bool is for RX(true)/TX(false), second bool is for bandwidth or PPS
-    fio_control: Option<(FioOps, bool)>, // Second parameter controls whether we want bandwidth or IOPS
+    net_rx: Option<bool>,
+    fio_ops: Option<FioOps>,
    num_boot_vcpus: Option<u8>,
 }

@@ -144,13 +146,11 @@ impl fmt::Display for PerformanceTestControl {
        if let Some(o) = self.queue_size {
            output = format!("{output}, queue_size = {o}");
        }
-        if let Some(o) = self.net_control {
-            let (rx, bw) = o;
-            output = format!("{output}, rx = {rx}, bandwidth = {bw}");
+        if let Some(o) = self.net_rx {
+            output = format!("{output}, net_rx = {o}");
        }
-        if let Some(o) = &self.fio_control {
-            let (ops, bw) = o;
-            output = format!("{output}, fio_ops = {ops}, bandwidth = {bw}");
+        if let Some(o) = &self.fio_ops {
+            output = format!("{output}, fio_ops = {o}");
        }

        write!(f, "{output}")
@@ -164,8 +164,8 @@ impl PerformanceTestControl {
            test_iterations: 5,
            num_queues: None,
            queue_size: None,
-            net_control: None,
-            fio_control: None,
+            net_rx: None,
+            fio_ops: None,
            num_boot_vcpus: Some(1),
        }
    }
@@ -262,7 +262,7 @@ mod adjuster {
    }
 }

-const TEST_LIST: [PerformanceTest; 29] = [
+const TEST_LIST: [PerformanceTest; 17] = [
    PerformanceTest {
        name: "boot_time_ms",
        func_ptr: performance_boot_time,
@@ -321,7 +321,7 @@ const TEST_LIST: [PerformanceTest; 29] = [
        control: PerformanceTestControl {
            num_queues: Some(2),
            queue_size: Some(256),
-            net_control: Some((true, true)),
+            net_rx: Some(true),
            ..PerformanceTestControl::default()
        },
        unit_adjuster: adjuster::bps_to_gbps,
@@ -332,7 +332,7 @@ const TEST_LIST: [PerformanceTest; 29] = [
        control: PerformanceTestControl {
            num_queues: Some(2),
            queue_size: Some(256),
-            net_control: Some((false, true)),
+            net_rx: Some(false),
            ..PerformanceTestControl::default()
        },
        unit_adjuster: adjuster::bps_to_gbps,
@@ -343,7 +343,7 @@ const TEST_LIST: [PerformanceTest; 29] = [
        control: PerformanceTestControl {
            num_queues: Some(4),
            queue_size: Some(256),
-            net_control: Some((true, true)),
+            net_rx: Some(true),
            ..PerformanceTestControl::default()
        },
        unit_adjuster: adjuster::bps_to_gbps,
@@ -354,62 +354,18 @@ const TEST_LIST: [PerformanceTest; 29] = [
        control: PerformanceTestControl {
            num_queues: Some(4),
            queue_size: Some(256),
-            net_control: Some((false, true)),
+            net_rx: Some(false),
            ..PerformanceTestControl::default()
        },
        unit_adjuster: adjuster::bps_to_gbps,
    },
-    PerformanceTest {
-        name: "virtio_net_throughput_single_queue_rx_pps",
-        func_ptr: performance_net_throughput,
-        control: PerformanceTestControl {
-            num_queues: Some(2),
-            queue_size: Some(256),
-            net_control: Some((true, false)),
-            ..PerformanceTestControl::default()
-        },
-        unit_adjuster: adjuster::identity,
-    },
-    PerformanceTest {
-        name: "virtio_net_throughput_single_queue_tx_pps",
-        func_ptr: performance_net_throughput,
-        control: PerformanceTestControl {
-            num_queues: Some(2),
-            queue_size: Some(256),
-            net_control: Some((false, false)),
-            ..PerformanceTestControl::default()
-        },
-        unit_adjuster: adjuster::identity,
-    },
-    PerformanceTest {
-        name: "virtio_net_throughput_multi_queue_rx_pps",
-        func_ptr: performance_net_throughput,
-        control: PerformanceTestControl {
-            num_queues: Some(4),
-            queue_size: Some(256),
-            net_control: Some((true, false)),
-            ..PerformanceTestControl::default()
-        },
-        unit_adjuster: adjuster::identity,
-    },
-    PerformanceTest {
-        name: "virtio_net_throughput_multi_queue_tx_pps",
-        func_ptr: performance_net_throughput,
-        control: PerformanceTestControl {
-            num_queues: Some(4),
-            queue_size: Some(256),
-            net_control: Some((false, false)),
-            ..PerformanceTestControl::default()
-        },
-        unit_adjuster: adjuster::identity,
-    },
    PerformanceTest {
        name: "block_read_MiBps",
        func_ptr: performance_block_io,
        control: PerformanceTestControl {
            num_queues: Some(1),
            queue_size: Some(128),
-            fio_control: Some((FioOps::Read, true)),
+            fio_ops: Some(FioOps::Read),
            ..PerformanceTestControl::default()
        },
        unit_adjuster: adjuster::Bps_to_MiBps,
@@ -420,7 +376,7 @@ const TEST_LIST: [PerformanceTest; 29] = [
        control: PerformanceTestControl {
            num_queues: Some(1),
            queue_size: Some(128),
-            fio_control: Some((FioOps::Write, true)),
+            fio_ops: Some(FioOps::Write),
            ..PerformanceTestControl::default()
        },
        unit_adjuster: adjuster::Bps_to_MiBps,
@@ -431,7 +387,7 @@ const TEST_LIST: [PerformanceTest; 29] = [
        control: PerformanceTestControl {
            num_queues: Some(1),
            queue_size: Some(128),
-            fio_control: Some((FioOps::RandomRead, true)),
+            fio_ops: Some(FioOps::RandomRead),
            ..PerformanceTestControl::default()
        },
        unit_adjuster: adjuster::Bps_to_MiBps,
@@ -442,7 +398,7 @@ const TEST_LIST: [PerformanceTest; 29] = [
        control: PerformanceTestControl {
            num_queues: Some(1),
            queue_size: Some(128),
-            fio_control: Some((FioOps::RandomWrite, true)),
+            fio_ops: Some(FioOps::RandomWrite),
            ..PerformanceTestControl::default()
        },
        unit_adjuster: adjuster::Bps_to_MiBps,
@@ -453,7 +409,7 @@ const TEST_LIST: [PerformanceTest; 29] = [
        control: PerformanceTestControl {
            num_queues: Some(2),
            queue_size: Some(128),
-            fio_control: Some((FioOps::Read, true)),
+            fio_ops: Some(FioOps::Read),
            ..PerformanceTestControl::default()
        },
        unit_adjuster: adjuster::Bps_to_MiBps,
@@ -464,7 +420,7 @@ const TEST_LIST: [PerformanceTest; 29] = [
        control: PerformanceTestControl {
            num_queues: Some(2),
            queue_size: Some(128),
-            fio_control: Some((FioOps::Write, true)),
+            fio_ops: Some(FioOps::Write),
            ..PerformanceTestControl::default()
        },
        unit_adjuster: adjuster::Bps_to_MiBps,
@@ -475,7 +431,7 @@ const TEST_LIST: [PerformanceTest; 29] = [
        control: PerformanceTestControl {
            num_queues: Some(2),
            queue_size: Some(128),
-            fio_control: Some((FioOps::RandomRead, true)),
+            fio_ops: Some(FioOps::RandomRead),
            ..PerformanceTestControl::default()
        },
        unit_adjuster: adjuster::Bps_to_MiBps,
@@ -486,99 +442,11 @@ const TEST_LIST: [PerformanceTest; 29] = [
        control: PerformanceTestControl {
            num_queues: Some(2),
            queue_size: Some(128),
-            fio_control: Some((FioOps::RandomWrite, true)),
+            fio_ops: Some(FioOps::RandomWrite),
            ..PerformanceTestControl::default()
        },
        unit_adjuster: adjuster::Bps_to_MiBps,
    },
-    PerformanceTest {
-        name: "block_read_IOPS",
-        func_ptr: performance_block_io,
-        control: PerformanceTestControl {
-            num_queues: Some(1),
-            queue_size: Some(128),
-            fio_control: Some((FioOps::Read, false)),
-            ..PerformanceTestControl::default()
-        },
-        unit_adjuster: adjuster::identity,
-    },
-    PerformanceTest {
-        name: "block_write_IOPS",
-        func_ptr: performance_block_io,
-        control: PerformanceTestControl {
-            num_queues: Some(1),
-            queue_size: Some(128),
-            fio_control: Some((FioOps::Write, false)),
-            ..PerformanceTestControl::default()
-        },
-        unit_adjuster: adjuster::identity,
-    },
-    PerformanceTest {
-        name: "block_random_read_IOPS",
-        func_ptr: performance_block_io,
-        control: PerformanceTestControl {
-            num_queues: Some(1),
-            queue_size: Some(128),
-            fio_control: Some((FioOps::RandomRead, false)),
-            ..PerformanceTestControl::default()
-        },
-        unit_adjuster: adjuster::identity,
-    },
-    PerformanceTest {
-        name: "block_random_write_IOPS",
-        func_ptr: performance_block_io,
-        control: PerformanceTestControl {
-            num_queues: Some(1),
-            queue_size: Some(128),
-            fio_control: Some((FioOps::RandomWrite, false)),
-            ..PerformanceTestControl::default()
-        },
-        unit_adjuster: adjuster::identity,
-    },
-    PerformanceTest {
-        name: "block_multi_queue_read_IOPS",
-        func_ptr: performance_block_io,
-        control: PerformanceTestControl {
-            num_queues: Some(2),
-            queue_size: Some(128),
-            fio_control: Some((FioOps::Read, false)),
-            ..PerformanceTestControl::default()
-        },
-        unit_adjuster: adjuster::identity,
-    },
-    PerformanceTest {
-        name: "block_multi_queue_write_IOPS",
-        func_ptr: performance_block_io,
-        control: PerformanceTestControl {
-            num_queues: Some(2),
-            queue_size: Some(128),
-            fio_control: Some((FioOps::Write, false)),
-            ..PerformanceTestControl::default()
-        },
-        unit_adjuster: adjuster::identity,
-    },
-    PerformanceTest {
-        name: "block_multi_queue_random_read_IOPS",
-        func_ptr: performance_block_io,
-        control: PerformanceTestControl {
-            num_queues: Some(2),
-            queue_size: Some(128),
-            fio_control: Some((FioOps::RandomRead, false)),
-            ..PerformanceTestControl::default()
-        },
-        unit_adjuster: adjuster::identity,
-    },
-    PerformanceTest {
-        name: "block_multi_queue_random_write_IOPS",
-        func_ptr: performance_block_io,
-        control: PerformanceTestControl {
-            num_queues: Some(2),
-            queue_size: Some(128),
-            fio_control: Some((FioOps::RandomWrite, false)),
-            ..PerformanceTestControl::default()
-        },
-        unit_adjuster: adjuster::identity,
-    },
 ];

 fn run_test_with_timeout(
@@ -629,7 +497,7 @@ fn date() -> String {
 fn main() {
    let cmd_arguments = ClapCommand::new("performance-metrics")
        .version(env!("GIT_HUMAN_READABLE"))
-        .author(env!("CARGO_PKG_AUTHORS"))
+        .author(crate_authors!())
        .about("Generate the performance metrics data for Cloud Hypervisor")
        .arg(
            Arg::new("test-filter")
--- a/performance-metrics/src/performance_tests.rs
+++ b/performance-metrics/src/performance_tests.rs
@@ -78,7 +78,7 @@ fn direct_kernel_boot_path() -> PathBuf {

 pub fn performance_net_throughput(control: &PerformanceTestControl) -> f64 {
    let test_timeout = control.test_timeout;
-    let (rx, bandwidth) = control.net_control.unwrap();
+    let rx = control.net_rx.unwrap();

    let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string());
    let guest = performance_test_new_guest(Box::new(focal));
@@ -105,7 +105,7 @@ pub fn performance_net_throughput(control: &PerformanceTestControl) -> f64 {

    let r = std::panic::catch_unwind(|| {
        guest.wait_vm_boot(None).unwrap();
-        measure_virtio_net_throughput(test_timeout, num_queues / 2, &guest, rx, bandwidth).unwrap()
+        measure_virtio_net_throughput(test_timeout, num_queues / 2, &guest, rx).unwrap()
    });

    let _ = child.kill();
@@ -168,7 +168,6 @@ fn parse_boot_time_output(output: &[u8]) -> Result<f64, Error> {
    std::panic::catch_unwind(|| {
        let l: Vec<String> = String::from_utf8_lossy(output)
            .lines()
-            .into_iter()
            .filter(|l| l.contains("Debug I/O port: Kernel code"))
            .map(|l| l.to_string())
            .collect();
@@ -335,7 +334,7 @@ pub fn performance_boot_time_pmem(control: &PerformanceTestControl) -> f64 {
 pub fn performance_block_io(control: &PerformanceTestControl) -> f64 {
    let test_timeout = control.test_timeout;
    let num_queues = control.num_queues.unwrap();
-    let (fio_ops, bandwidth) = control.fio_control.as_ref().unwrap();
+    let fio_ops = control.fio_ops.as_ref().unwrap();

    let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string());
    let guest = performance_test_new_guest(Box::new(focal));
@@ -388,11 +387,7 @@ pub fn performance_block_io(control: &PerformanceTestControl) -> f64 {
            .unwrap();

        // Parse fio output
-        if *bandwidth {
-            parse_fio_output(&output, fio_ops, num_queues).unwrap()
-        } else {
-            parse_fio_output_iops(&output, fio_ops, num_queues).unwrap()
-        }
+        parse_fio_output(&output, fio_ops, num_queues).unwrap()
    });

    let _ = child.kill();
@@ -429,7 +424,7 @@ mod tests {
 }
       "#;
        assert_eq!(
-            parse_iperf3_output(output.as_bytes(), true, true).unwrap(),
+            parse_iperf3_output(output.as_bytes(), true).unwrap(),
            23957198874.604115
        );

@@ -448,31 +443,9 @@ mod tests {
 }
              "#;
        assert_eq!(
-            parse_iperf3_output(output.as_bytes(), false, true).unwrap(),
+            parse_iperf3_output(output.as_bytes(), false).unwrap(),
            39520744482.79
        );
-        let output = r#"
-{
-    "end":	{
-        "sum":  {
-            "start":        0,
-            "end":  5.000036,
-            "seconds":      5.000036,
-            "bytes":        29944971264,
-            "bits_per_second":      47911877363.396217,
-            "jitter_ms":    0.0038609822983198556,
-            "lost_packets": 16,
-            "packets":      913848,
-            "lost_percent": 0.0017508382137948542,
-            "sender":       true
-        }
-    }
-}
-              "#;
-        assert_eq!(
-            parse_iperf3_output(output.as_bytes(), true, false).unwrap(),
-            182765.08409139456
-        );
    }

    #[test]
--- a/qcow/Cargo.toml
+++ b/qcow/Cargo.toml
@@ -10,7 +10,7 @@ path = "src/qcow.rs"

 [dependencies]
 byteorder = "1.4.3"
-libc = "0.2.139"
+libc = "0.2.138"
 log = "0.4.17"
-remain = "0.2.6"
+remain = "0.2.5"
 vmm-sys-util = "0.11.0"
--- a/qcow/src/qcow_raw_file.rs
+++ b/qcow/src/qcow_raw_file.rs
@@ -66,7 +66,7 @@ impl QcowRawFile {
        non_zero_flags: u64,
    ) -> io::Result<()> {
        self.file.seek(SeekFrom::Start(offset))?;
-        let mut buffer = BufWriter::with_capacity(table.len() * size_of::<u64>(), &mut self.file);
+        let mut buffer = BufWriter::with_capacity(std::mem::size_of_val(table), &mut self.file);
        for addr in table {
            let val = if *addr == 0 {
                0
@@ -91,7 +91,7 @@ impl QcowRawFile {
    /// Writes a refcount block to the file.
    pub fn write_refcount_block(&mut self, offset: u64, table: &[u16]) -> io::Result<()> {
        self.file.seek(SeekFrom::Start(offset))?;
-        let mut buffer = BufWriter::with_capacity(table.len() * size_of::<u16>(), &mut self.file);
+        let mut buffer = BufWriter::with_capacity(std::mem::size_of_val(table), &mut self.file);
        for count in table {
            buffer.write_u16::<BigEndian>(*count)?;
        }
--- a/qcow/src/raw_file.rs
+++ b/qcow/src/raw_file.rs
@@ -231,7 +231,6 @@ impl Write for RawFile {
                return Err(io::Error::last_os_error());
            }

-            // SAFETY: tmp_ptr is at least rounded_len long
            let tmp_buf = unsafe { slice::from_raw_parts_mut(tmp_ptr, rounded_len) };

            // This can eventually replaced with read_at once its interface
--- a/rate_limiter/Cargo.toml
+++ b/rate_limiter/Cargo.toml
@@ -4,6 +4,6 @@ version = "0.1.0"
 edition = "2021"

 [dependencies]
-libc = "0.2.139"
+libc = "0.2.138"
 log = "0.4.17"
 vmm-sys-util = "0.11.0"
--- a/rate_limiter/src/lib.rs
+++ b/rate_limiter/src/lib.rs
@@ -343,7 +343,6 @@ impl RateLimiter {
        let timer_fd = TimerFd::new()?;
        // Note: vmm_sys_util::TimerFd::new() open the fd w/o O_NONBLOCK. We manually add this flag
        // so that `Self::event_handler` won't be blocked with `vmm_sys_util::TimerFd::wait()`.
-        // SAFETY: FFI calls.
        let ret = unsafe {
            let fd = timer_fd.as_raw_fd();
            let mut flags = libc::fcntl(fd, libc::F_GETFL);
--- a/release-notes.md
+++ b/release-notes.md
@@ -1,15 +1,6 @@
- [v29.0](#v290)
-    - [Release Binary Supports Both MSHV and KVM](#release-binary-supports-both-mshv-and-kvm)
-    - [Snapshot/Restore and Live Migration Improvements](#snapshotrestore-and-live-migration-improvements)
-    - [Heap Allocation Improvements](#heap-allocation-improvements)
-    - [`ch-remote` Improvements](#ch-remote-improvements)
-    - [`AArch64` Documentation Integration](#aarch64-documentation-integration)
-    - [`virtio-block` Counters Enhancement](#virtio-block-counters-enhancement)
-    - [TCP Offload Control](#tcp-offload-control)
-    - [Notable Bug Fixes](#notable-bug-fixes)
-    - [Removals](#removals)
-    - [Deprecations](#deprecations)
-    - [Contributors](#contributors)
+- [v28.4](#v284)
+- [v28.3](#v283)
+- [v28.2](#v282)
 - [v28.1](#v281)
 - [v28.0](#v280)
    - [Community Engagement (Reminder)](#community-engagement-reminder)
@@ -17,9 +8,9 @@
    - [Virtualised TPM Support](#virtualised-tpm-support)
    - [Transparent Huge Page Support](#transparent-huge-page-support)
    - [README Quick Start Improved](#readme-quick-start-improved)
-    - [Notable Bug Fixes](#notable-bug-fixes-1)
-    - [Removals](#removals-1)
-    - [Contributors](#contributors-1)
+    - [Notable Bug Fixes](#notable-bug-fixes)
+    - [Removals](#removals)
+    - [Contributors](#contributors)
 - [v27.0](#v270)
    - [Community Engagement](#community-engagement)
    - [Prebuilt Packages](#prebuilt-packages)
@@ -28,41 +19,41 @@
    - [Simplified Build Feature Flags](#simplified-build-feature-flags)
    - [Asynchronous Kernel Loading](#asynchronous-kernel-loading)
    - [GDB Support for AArch64](#gdb-support-for-aarch64)
-    - [Notable Bug Fixes](#notable-bug-fixes-2)
-    - [Deprecations](#deprecations-1)
-    - [Contributors](#contributors-2)
+    - [Notable Bug Fixes](#notable-bug-fixes-1)
+    - [Deprecations](#deprecations)
+    - [Contributors](#contributors-1)
 - [v26.0](#v260)
    - [SMBIOS Improvements via `--platform`](#smbios-improvements-via---platform)
    - [Unified Binary MSHV and KVM Support](#unified-binary-mshv-and-kvm-support)
+    - [Notable Bug Fixes](#notable-bug-fixes-2)
+    - [Deprecations](#deprecations-1)
+    - [Removals](#removals-1)
+    - [Contributors](#contributors-2)
+- [v25.0](#v250)
+    - [`ch-remote` Improvements](#ch-remote-improvements)
+    - [VM "Coredump" Support](#vm-coredump-support)
    - [Notable Bug Fixes](#notable-bug-fixes-3)
-    - [Deprecations](#deprecations-2)
    - [Removals](#removals-2)
    - [Contributors](#contributors-3)
- [v25.0](#v250)
-    - [`ch-remote` Improvements](#ch-remote-improvements-1)
-    - [VM "Coredump" Support](#vm-coredump-support)
-    - [Notable Bug Fixes](#notable-bug-fixes-4)
-    - [Removals](#removals-3)
-    - [Contributors](#contributors-4)
 - [v24.0](#v240)
    - [Bypass Mode for `virtio-iommu`](#bypass-mode-for-virtio-iommu)
    - [Ensure Identifiers Uniqueness](#ensure-identifiers-uniqueness)
    - [Sparse Mmap support](#sparse-mmap-support)
    - [Expose Platform Serial Number](#expose-platform-serial-number)
-    - [Notable Bug Fixes](#notable-bug-fixes-5)
+    - [Notable Bug Fixes](#notable-bug-fixes-4)
    - [Notable Improvements](#notable-improvements)
-    - [Deprecations](#deprecations-3)
+    - [Deprecations](#deprecations-2)
    - [New on the Website](#new-on-the-website)
-    - [Contributors](#contributors-5)
+    - [Contributors](#contributors-4)
 - [v23.1](#v231)
 - [v23.0](#v230)
    - [vDPA Support](#vdpa-support)
    - [Updated OS Support list](#updated-os-support-list)
    - [`AArch64` Memory Map Improvements](#aarch64-memory-map-improvements)
    - [`AMX` Support](#amx-support)
-    - [Notable Bug Fixes](#notable-bug-fixes-6)
-    - [Deprecations](#deprecations-4)
-    - [Contributors](#contributors-6)
+    - [Notable Bug Fixes](#notable-bug-fixes-5)
+    - [Deprecations](#deprecations-3)
+    - [Contributors](#contributors-5)
 - [v22.1](#v221)
 - [v22.0](#v220)
    - [GDB Debug Stub Support](#gdb-debug-stub-support)
@@ -73,13 +64,13 @@
    - [PMU Support for AArch64](#pmu-support-for-aarch64)
    - [Documentation Under CC-BY-4.0 License](#documentation-under-cc-by-40-license)
    - [Deprecation of "Classic" `virtiofsd`](#deprecation-of-classic-virtiofsd)
-    - [Notable Bug Fixes](#notable-bug-fixes-7)
-    - [Contributors](#contributors-7)
+    - [Notable Bug Fixes](#notable-bug-fixes-6)
+    - [Contributors](#contributors-6)
 - [v21.0](#v210)
    - [Efficient Local Live Migration (for Live Upgrade)](#efficient-local-live-migration-for-live-upgrade)
    - [Recommended Kernel is Now 5.15](#recommended-kernel-is-now-515)
-    - [Notable Bug fixes](#notable-bug-fixes-8)
-    - [Contributors](#contributors-8)
+    - [Notable Bug fixes](#notable-bug-fixes-7)
+    - [Contributors](#contributors-7)
 - [v20.2](#v202)
 - [v20.1](#v201)
 - [v20.0](#v200)
@@ -88,8 +79,8 @@
    - [Improved VFIO support](#improved-vfio-support)
    - [Safer code](#safer-code)
    - [Extended documentation](#extended-documentation)
-    - [Notable bug fixes](#notable-bug-fixes-9)
-    - [Contributors](#contributors-9)
+    - [Notable bug fixes](#notable-bug-fixes-8)
+    - [Contributors](#contributors-8)
 - [v19.0](#v190)
    - [Improved PTY handling for serial and `virtio-console`](#improved-pty-handling-for-serial-and-virtio-console)
    - [PCI boot time optimisations](#pci-boot-time-optimisations)
@@ -97,8 +88,8 @@
    - [Live migration enhancements](#live-migration-enhancements)
    - [`virtio-mem` support with `vfio-user`](#virtio-mem-support-with-vfio-user)
    - [AArch64 for `virtio-iommu`](#aarch64-for-virtio-iommu)
-    - [Notable bug fixes](#notable-bug-fixes-10)
-    - [Contributors](#contributors-10)
+    - [Notable bug fixes](#notable-bug-fixes-9)
+    - [Contributors](#contributors-9)
 - [v18.0](#v180)
    - [Experimental User Device (`vfio-user`) support](#experimental-user-device-vfio-user-support)
    - [Migration support for `vhost-user` devices](#migration-support-for-vhost-user-devices)
@@ -108,31 +99,31 @@
    - [Live migration on MSHV hypervisor](#live-migration-on-mshv-hypervisor)
    - [AArch64 CPU topology support](#aarch64-cpu-topology-support)
    - [Power button support on AArch64](#power-button-support-on-aarch64)
-    - [Notable bug fixes](#notable-bug-fixes-11)
-    - [Contributors](#contributors-11)
+    - [Notable bug fixes](#notable-bug-fixes-10)
+    - [Contributors](#contributors-10)
 - [v17.0](#v170)
    - [ARM64 NUMA support using ACPI](#arm64-numa-support-using-acpi)
    - [`Seccomp` support for MSHV backend](#seccomp-support-for-mshv-backend)
    - [Hotplug of `macvtap` devices](#hotplug-of-macvtap-devices)
    - [Improved SGX support](#improved-sgx-support)
    - [Inflight tracking for `vhost-user` devices](#inflight-tracking-for-vhost-user-devices)
-    - [Notable bug fixes](#notable-bug-fixes-12)
-    - [Contributors](#contributors-12)
+    - [Notable bug fixes](#notable-bug-fixes-11)
+    - [Contributors](#contributors-11)
 - [v16.0](#v160)
    - [Improved live migration support](#improved-live-migration-support)
    - [Improved `vhost-user` support](#improved-vhost-user-support)
    - [ARM64 ACPI and UEFI support](#arm64-acpi-and-uefi-support)
-    - [Notable bug fixes](#notable-bug-fixes-13)
+    - [Notable bug fixes](#notable-bug-fixes-12)
    - [Removed functionality](#removed-functionality)
-    - [Contributors](#contributors-13)
+    - [Contributors](#contributors-12)
 - [v15.0](#v150)
    - [Version numbering and stability guarantees](#version-numbering-and-stability-guarantees)
    - [Network device rate limiting](#network-device-rate-limiting)
    - [Support for runtime control of `virtio-net` guest offload](#support-for-runtime-control-of-virtio-net-guest-offload)
    - [`--api-socket` supports file descriptor parameter](#--api-socket-supports-file-descriptor-parameter)
    - [Bug fixes](#bug-fixes)
-    - [Deprecations](#deprecations-5)
-    - [Contributors](#contributors-14)
+    - [Deprecations](#deprecations-4)
+    - [Contributors](#contributors-13)
 - [v0.14.1](#v0141)
 - [v0.14.0](#v0140)
    - [Structured event monitoring](#structured-event-monitoring)
@@ -141,8 +132,8 @@
    - [Updated hotplug documentation](#updated-hotplug-documentation)
    - [PTY control for serial and `virtio-console`](#pty-control-for-serial-and-virtio-console)
    - [Block device rate limiting](#block-device-rate-limiting)
-    - [Deprecations](#deprecations-6)
-    - [Contributors](#contributors-15)
+    - [Deprecations](#deprecations-5)
+    - [Contributors](#contributors-14)
 - [v0.13.0](#v0130)
    - [Wider VFIO device support](#wider-vfio-device-support)
    - [Improved huge page support](#improved-huge-page-support)
@@ -150,13 +141,13 @@
    - [VHD disk image support](#vhd-disk-image-support)
    - [Improved Virtio device threading](#improved-virtio-device-threading)
    - [Clean shutdown support via synthetic power button](#clean-shutdown-support-via-synthetic-power-button)
-    - [Contributors](#contributors-16)
+    - [Contributors](#contributors-15)
 - [v0.12.0](#v0120)
    - [ARM64 enhancements](#arm64-enhancements)
    - [Removal of `vhost-user-net` and `vhost-user-block` self spawning](#removal-of-vhost-user-net-and-vhost-user-block-self-spawning)
    - [Migration of `vhost-user-fs` backend](#migration-of-vhost-user-fs-backend)
    - [Enhanced "info" API](#enhanced-info-api)
-    - [Contributors](#contributors-17)
+    - [Contributors](#contributors-16)
 - [v0.11.0](#v0110)
    - [`io_uring` support by default for `virtio-block`](#io_uring-support-by-default-for-virtio-block)
    - [Windows Guest Support](#windows-guest-support)
@@ -168,15 +159,15 @@
    - [Default Log Level Changed](#default-log-level-changed)
    - [New `--balloon` Parameter Added](#new---balloon-parameter-added)
    - [Experimental `virtio-watchdog` Support](#experimental-virtio-watchdog-support)
-    - [Notable Bug Fixes](#notable-bug-fixes-14)
-    - [Contributors](#contributors-18)
+    - [Notable Bug Fixes](#notable-bug-fixes-13)
+    - [Contributors](#contributors-17)
 - [v0.10.0](#v0100)
    - [`virtio-block` Support for Multiple Descriptors](#virtio-block-support-for-multiple-descriptors)
    - [Memory Zones](#memory-zones)
    - [`Seccomp` Sandbox Improvements](#seccomp-sandbox-improvements)
    - [Preliminary KVM HyperV Emulation Control](#preliminary-kvm-hyperv-emulation-control)
-    - [Notable Bug Fixes](#notable-bug-fixes-15)
-    - [Contributors](#contributors-19)
+    - [Notable Bug Fixes](#notable-bug-fixes-14)
+    - [Contributors](#contributors-18)
 - [v0.9.0](#v090)
    - [`io_uring` Based Block Device Support](#io_uring-based-block-device-support)
    - [Block and Network Device Statistics](#block-and-network-device-statistics)
@@ -189,17 +180,17 @@
    - [Enhancements to ARM64 Support](#enhancements-to-arm64-support)
    - [Intel SGX Support](#intel-sgx-support)
    - [`Seccomp` Sandbox Improvements](#seccomp-sandbox-improvements-1)
-    - [Notable Bug Fixes](#notable-bug-fixes-16)
-    - [Contributors](#contributors-20)
+    - [Notable Bug Fixes](#notable-bug-fixes-15)
+    - [Contributors](#contributors-19)
 - [v0.8.0](#v080)
    - [Experimental Snapshot and Restore Support](#experimental-snapshot-and-restore-support)
    - [Experimental ARM64 Support](#experimental-arm64-support)
    - [Support for Using 5-level Paging in Guests](#support-for-using-5-level-paging-in-guests)
    - [Virtio Device Interrupt Suppression for Network Devices](#virtio-device-interrupt-suppression-for-network-devices)
    - [`vhost_user_fs` Improvements](#vhost_user_fs-improvements)
-    - [Notable Bug Fixes](#notable-bug-fixes-17)
+    - [Notable Bug Fixes](#notable-bug-fixes-16)
    - [Command Line and API Changes](#command-line-and-api-changes)
-    - [Contributors](#contributors-21)
+    - [Contributors](#contributors-20)
 - [v0.7.0](#v070)
    - [Block, Network, Persistent Memory (PMEM), VirtioFS and Vsock hotplug](#block-network-persistent-memory-pmem-virtiofs-and-vsock-hotplug)
    - [Alternative `libc` Support](#alternative-libc-support)
@@ -209,14 +200,14 @@
    - [`Seccomp` Sandboxing](#seccomp-sandboxing)
    - [Updated Distribution Support](#updated-distribution-support)
    - [Command Line and API Changes](#command-line-and-api-changes-1)
-    - [Contributors](#contributors-22)
+    - [Contributors](#contributors-21)
 - [v0.6.0](#v060)
    - [Directly Assigned Devices Hotplug](#directly-assigned-devices-hotplug)
    - [Shared Filesystem Improvements](#shared-filesystem-improvements)
    - [Block and Networking IO Self Offloading](#block-and-networking-io-self-offloading)
    - [Command Line Interface](#command-line-interface)
    - [PVH Boot](#pvh-boot)
-    - [Contributors](#contributors-23)
+    - [Contributors](#contributors-22)
 - [v0.5.1](#v051)
 - [v0.5.0](#v050)
    - [Virtual Machine Dynamic Resizing](#virtual-machine-dynamic-resizing)
@@ -224,7 +215,7 @@
    - [New Interrupt Management Framework](#new-interrupt-management-framework)
    - [Development Tools](#development-tools)
    - [Kata Containers Integration](#kata-containers-integration)
-    - [Contributors](#contributors-24)
+    - [Contributors](#contributors-23)
 - [v0.4.0](#v040)
    - [Dynamic virtual CPUs addition](#dynamic-virtual-cpus-addition)
    - [Programmatic firmware tables generation](#programmatic-firmware-tables-generation)
@@ -233,7 +224,7 @@
    - [Userspace IOAPIC by default](#userspace-ioapic-by-default)
    - [PCI BAR reprogramming](#pci-bar-reprogramming)
    - [New `cloud-hypervisor` organization](#new-cloud-hypervisor-organization)
-    - [Contributors](#contributors-25)
+    - [Contributors](#contributors-24)
 - [v0.3.0](#v030)
    - [Block device offloading](#block-device-offloading)
    - [Network device backend](#network-device-backend)
@@ -260,93 +251,44 @@
    - [Unit testing](#unit-testing)
    - [Integration tests parallelization](#integration-tests-parallelization)

-# v29.0

-This release has been tracked in our [roadmap
-project](https://github.com/orgs/cloud-hypervisor/projects/6) as iteration
-v29.0. The following user visible changes have been made:
+# v28.4

-### Release Binary Supports Both MSHV and KVM
+This is a bug fix release. The following issues have been addressed:

-On `x86-64` the binary included in releases supports both the KVM and MSHV
-hypervisor with runtime detection to identify the correct hypervisor to use.
+* Fix VFIO on platforms with non-4k page size (#5450)
+* Fix seccomp filter lists related to virtio-console, serial and pty
+(#5506, #5524)
+* Populate APIC ID properly (#5512)
+* Ignore and warn TAP FDs in more situations (#5522)
+* Disallow concurrent CPU resizing (#5668)
+* Use wrapping add for memory offset from instruction emulator (#5719)
+* Replace unsound `static mut` with `once_cell` (#5772)
+* Fix a deadlock when TDX is enabled (#5845)
+* Bug fix to OpenAPI specification file (#5967)
+* Error out early for live migration when TDX is enabled (#6025)

-### Snapshot/Restore and Live Migration Improvements
+# v28.3

-Improvements have been made to the data structures used for both live migration
-and snapshot/restore. Unfortunately this has broken compatibility with older
-versions (support for migrating between major versions is not yet officially
-supported but has worked for some versions.)
+This is a bug fix release. The following issues have been addressed:

-### Heap Allocation Improvements
+* Ignore and warn TAP FDs sent via the HTTP request body (#5350)
+* Properly preserve and close valid FDs for TAP devices (#5373)
+* Only use `KVM_ARM_VCPU_PMU_V3` if available (#5360)
+* Fix seccomp filter lists for vhost-user devices (#5361)
+* Fix the offset setting while removing the entire mapping of
+  `vhost-user` FS client (#5235)
+* Set vcpu thread status properly and signal exit_evt upon thread exit (#5211)
+* Populate CPUID leaf 0x4000_0010 (TSC frequency) (#5179)
+* Bug fix to OpenAPI specification file (#5186)

-Improvements have been made to the volume of heap allocations when running with
-`virtio-block` devices along with a reduction in the peak heap size.
+# v28.2
+This is a bug fix release. The following issues have been addressed:

-### `ch-remote` Improvements
-
-Support for "pinging" the VMM and shutting the VMM down have been added to
-`ch-remote`.
-
-### `AArch64` Documentation Integration
-
-The documentation for `AArch64` support has been integrated into the main
-README.
-
-### `virtio-block` Counters Enhancement
-
-The counters for the `virtio-block` device has extended to include min/mean/max
-latency counters.
-
-### TCP Offload Control
-
-The `virtio-net` device has gained support for controlling the enabling of
-checksum and offloading. This allows the device to be used in environments
-where the hardware lacks support for the offloading.
-
-### Notable Bug Fixes
-
-* Update dependencies including a version of `linux-loader` that addresses an
-  infinite loop issue ([details](https://github.com/rust-vmm/linux-loader/security/advisories/GHSA-52h2-m2cf-9jh6))
-* Fix bugs related to `virtio-net` including an integer overflow issue
-  (#4924, #4949)
-* Use host `cpuid` information for L2 cache for older KVM on x86 (#4920)
-* Memory mapped into the guest is now marked as non-dumpable which prevents large core files (#5016)
 * Fix QCOW2 refcount table size (#5034)
 * Fix unpause support on MSHV in dual binary (#5037)
 * Threads inside `virtio` devices are now shutdown on reboot (#5095)

-### Removals
-
-No functionality has been removed in this release.
-
-### Deprecations
-
-* Support for specifying a directory with `MemoryZoneConfig::file` or
-  `MemoryConfig::file` has been deprecated. This was originally used for
-  supporting huge pages or shared memory backing which is now natively supported
-  (#5085)
-
-### Contributors
-
-Many thanks to everyone who has contributed to our release:
-
-* Bo Chen <chen.bo@intel.com>
-* Claudio Fontana <claudio.fontana@gmail.com>
-* Hao Xu <howeyxu@tencent.com>
-* Henry Wang <Henry.Wang@arm.com>
-* Jinank Jain <jinankjain@microsoft.com>
-* Michael Zhao <michael.zhao@arm.com>
-* Muminul Islam <muislam@microsoft.com>
-* Philipp Schuster <philipp.schuster@cyberus-technology.de>
-* Rob Bradford <robert.bradford@intel.com>
-* Sebastien Boeuf <sebastien.boeuf@intel.com>
-* Shuaiyi Zhang <zhangsy28@lenovo.com>
-* Wei Liu <liuwe@microsoft.com>
-* Yong He <alexyonghe@tencent.com>
-* Yuji Hagiwara <yuuzi41@gmail.com>
-
-
 # v28.1
 This is a bug fix release. The following issues have been addressed:

--- a/resources/Dockerfile
+++ b/resources/Dockerfile
@@ -1,10 +1,10 @@
 # When changing this file don't forget to update the tag name in the
 # .github/workflows/docker-image.yaml file if doing multiple per day

-FROM ubuntu:20.04 as dev
+FROM ubuntu:22.04 as dev

 ARG TARGETARCH
-ARG RUST_TOOLCHAIN="1.62.0"
+ARG RUST_TOOLCHAIN="1.65.0"
 ARG CLH_SRC_DIR="/cloud-hypervisor"
 ARG CLH_BUILD_DIR="$CLH_SRC_DIR/build"
 ARG CARGO_REGISTRY_DIR="$CLH_BUILD_DIR/cargo_registry"
@@ -16,40 +16,43 @@ ENV PATH="$PATH:$CARGO_HOME/bin"

 # Install all CI dependencies
 RUN apt-get update \
-	&& apt-get -yq upgrade \
-	&& DEBIAN_FRONTEND=noninteractive apt-get install -yq \
-	build-essential \
-	bc \
-	curl \
-	wget \
-	sudo \
-	mtools \
-	musl-tools \
-	libssl-dev \
-	pkg-config \
-	flex \
-	bison \
-	libelf-dev \
-	qemu-utils \
-	libglib2.0-dev \
-	libpixman-1-dev \
-	libseccomp-dev \
-	libcap-ng-dev \
-	socat \
-	dosfstools \
-	cpio \
-	python \
-	python3 \
-	python3-setuptools \
-	ntfs-3g \
-	openvswitch-switch-dpdk \
-	python3-distutils \
-	uuid-dev \
-	iperf3 \
-	zip \
-	git-core \
-	dnsmasq \
-	dmsetup \
+        && apt-get -yq upgrade \
+        && apt-get install --no-install-recommends -yq \
+        build-essential \
+        bc \
+        curl \
+        wget \
+        sudo \
+        mtools \
+        musl-tools \
+        libssl-dev \
+        pkg-config \
+        flex \
+        bison \
+        libelf-dev \
+        qemu-utils \
+        libglib2.0-dev \
+        libpixman-1-dev \
+        libseccomp-dev \
+        libcap-ng-dev \
+        socat \
+        dosfstools \
+        cpio \
+        python3 \
+        python3-setuptools \
+        ntfs-3g \
+        python3-distutils \
+        uuid-dev \
+        iperf3 \
+        zip \
+        git-core \
+        dnsmasq \
+        dmsetup \
+        ca-certificates \
+        unzip \
+        iproute2 \
+        dbus \
+    && apt-get install openvswitch-switch-dpdk -yq \
    && apt-get clean \
    && rm -rf /var/lib/apt/lists/*

@@ -112,17 +115,18 @@ RUN echo 'source $CARGO_HOME/env' >> $HOME/.bashrc \
 RUN if [ "$TARGETARCH" = "amd64" ]; then \
       git clone https://github.com/spdk/spdk \
       && cd spdk \
-       && git checkout 6301f8915de32baed10dba1eebed556a6749211a \
+       && git checkout ef8bcce58f3f02b79c0619a297e4f17e81e62b24 \
       && git submodule update --init \
       && apt-get update \
       && ./scripts/pkgdep.sh \
       && apt-get clean \
       && ./configure --with-vfio-user \
-       && make -j `nproc` \
+       && make -j $(nproc) TARGET_ARCHITECTURE=skylake \
+       && rm -rf /usr/local/bin/spdk-nvme \
       && mkdir /usr/local/bin/spdk-nvme \
-       && cp ./build/bin/nvmf_tgt /usr/local/bin/spdk-nvme \
-       && cp ./scripts/rpc.py /usr/local/bin/spdk-nvme \
-       && cp -r ./scripts/rpc /usr/local/bin/spdk-nvme \
+       && cp -f ./build/bin/nvmf_tgt /usr/local/bin/spdk-nvme \
+       && cp -f ./scripts/rpc.py /usr/local/bin/spdk-nvme \
+       && cp -rf ./python /usr/local/bin \
       && cd .. && rm -rf spdk; fi

 # install ethr tool for performance tests
--- a/scripts/common-aarch64.sh
+++ b/scripts/common-aarch64.sh
@@ -1,7 +1,5 @@
 #!/bin/bash

-export BUILD_TARGET=${BUILD_TARGET-aarch64-unknown-linux-gnu}
-
 WORKLOADS_DIR="$HOME/workloads"

 mkdir -p "$WORKLOADS_DIR"
--- a/scripts/dev_cli.sh
+++ b/scripts/dev_cli.sh
@@ -6,9 +6,9 @@

 CLI_NAME="Cloud Hypervisor"

-CTR_IMAGE_TAG="cloudhypervisor/dev"
-CTR_IMAGE_VERSION="20220705-0"
-CTR_IMAGE="${CTR_IMAGE_TAG}:${CTR_IMAGE_VERSION}"
+CTR_IMAGE_TAG="ghcr.io/cloud-hypervisor/cloud-hypervisor"
+CTR_IMAGE_VERSION="20231220-0"
+: "${CTR_IMAGE:=${CTR_IMAGE_TAG}:${CTR_IMAGE_VERSION}}"

 DOCKER_RUNTIME="docker"

@@ -277,11 +277,10 @@ cmd_build() {
    [ $build = "release" ] && cargo_args+=("--release")
    cargo_args+=(--target "$target")

-    rustflags=""
+    rustflags="$RUSTFLAGS"
    target_cc=""
    if [ "$(uname -m)" = "aarch64" ] && [ "$libc" = "musl" ]; then
-        rustflags="-C link-arg=-lgcc -C link_arg=-specs -C link_arg=/usr/lib/aarch64-linux-musl/musl-gcc.specs"
-        target_cc="musl-gcc"
+        rustflags="$rustflags -C link-args=-Wl,-Bstatic -C link-args=-lc"
    fi

    $DOCKER_RUNTIME run \
@@ -387,6 +386,12 @@ cmd_tests() {
    process_volumes_args
    target="$(uname -m)-unknown-linux-${libc}"

+    rustflags="$RUSTFLAGS"
+    target_cc=""
+    if [ "$(uname -m)" = "aarch64" ] && [ "$libc" = "musl" ]; then
+        rustflags="$rustflags -C link-args=-Wl,-Bstatic -C link-args=-lc"
+    fi
+
    if [[ "$unit" = true ]]; then
        say "Running unit tests for $target..."
        $DOCKER_RUNTIME run \
@@ -397,6 +402,8 @@ cmd_tests() {
            --cap-add net_admin \
            --volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR" $exported_volumes \
            --env BUILD_TARGET="$target" \
+            --env RUSTFLAGS="$rustflags" \
+            --env TARGET_CC="$target_cc" \
            "$CTR_IMAGE" \
            ./scripts/run_unit_tests.sh "$@" || fix_dir_perms $? || exit $?
    fi
@@ -415,7 +422,9 @@ cmd_tests() {
            --volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR" $exported_volumes \
            --volume "$CLH_INTEGRATION_WORKLOADS:$CTR_CLH_INTEGRATION_WORKLOADS" \
            --env USER="root" \
-            --env CH_LIBC="${libc}" \
+            --env BUILD_TARGET="$target" \
+            --env RUSTFLAGS="$rustflags" \
+            --env TARGET_CC="$target_cc" \
            "$CTR_IMAGE" \
            ./scripts/run_integration_tests_"$(uname -m)".sh "$@" || fix_dir_perms $? || exit $?
    fi
@@ -434,7 +443,9 @@ cmd_tests() {
            --volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR" $exported_volumes \
            --volume "$CLH_INTEGRATION_WORKLOADS:$CTR_CLH_INTEGRATION_WORKLOADS" \
            --env USER="root" \
-            --env CH_LIBC="${libc}" \
+            --env BUILD_TARGET="$target" \
+            --env RUSTFLAGS="$rustflags" \
+            --env TARGET_CC="$target_cc" \
            "$CTR_IMAGE" \
            ./scripts/run_integration_tests_sgx.sh "$@" || fix_dir_perms $? || exit $?
    fi
@@ -453,7 +464,9 @@ cmd_tests() {
            --volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR" $exported_volumes \
            --volume "$CLH_INTEGRATION_WORKLOADS:$CTR_CLH_INTEGRATION_WORKLOADS" \
            --env USER="root" \
-            --env CH_LIBC="${libc}" \
+            --env BUILD_TARGET="$target" \
+            --env RUSTFLAGS="$rustflags" \
+            --env TARGET_CC="$target_cc" \
            "$CTR_IMAGE" \
            ./scripts/run_integration_tests_vfio.sh "$@" || fix_dir_perms $? || exit $?
    fi
@@ -472,7 +485,9 @@ cmd_tests() {
            --volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR" $exported_volumes \
            --volume "$CLH_INTEGRATION_WORKLOADS:$CTR_CLH_INTEGRATION_WORKLOADS" \
            --env USER="root" \
-            --env CH_LIBC="${libc}" \
+            --env BUILD_TARGET="$target" \
+            --env RUSTFLAGS="$rustflags" \
+            --env TARGET_CC="$target_cc" \
            "$CTR_IMAGE" \
            ./scripts/run_integration_tests_windows_"$(uname -m)".sh "$@" || fix_dir_perms $? || exit $?
    fi
@@ -491,7 +506,9 @@ cmd_tests() {
            --volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR" $exported_volumes \
            --volume "$CLH_INTEGRATION_WORKLOADS:$CTR_CLH_INTEGRATION_WORKLOADS" \
            --env USER="root" \
-            --env CH_LIBC="${libc}" \
+            --env BUILD_TARGET="$target" \
+            --env RUSTFLAGS="$rustflags" \
+            --env TARGET_CC="$target_cc" \
            "$CTR_IMAGE" \
            ./scripts/run_integration_tests_live_migration.sh "$@" || fix_dir_perms $? || exit $?
    fi
@@ -510,7 +527,9 @@ cmd_tests() {
            --volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR" $exported_volumes \
            --volume "$CLH_INTEGRATION_WORKLOADS:$CTR_CLH_INTEGRATION_WORKLOADS" \
            --env USER="root" \
-            --env CH_LIBC="${libc}" \
+            --env BUILD_TARGET="$target" \
+            --env RUSTFLAGS="$rustflags" \
+            --env TARGET_CC="$target_cc" \
            "$CTR_IMAGE" \
            ./scripts/run_integration_tests_rate_limiter.sh "$@" || fix_dir_perms $? || exit $?
    fi
@@ -529,7 +548,10 @@ cmd_tests() {
            --volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR" $exported_volumes \
            --volume "$CLH_INTEGRATION_WORKLOADS:$CTR_CLH_INTEGRATION_WORKLOADS" \
            --env USER="root" \
-            --env CH_LIBC="${libc}" \
+            --env BUILD_TARGET="$target" \
+            --env RUSTFLAGS="$rustflags" \
+            --env TARGET_CC="$target_cc" \
+            --env RUST_BACKTRACE="${RUST_BACKTRACE}" \
            "$CTR_IMAGE" \
            ./scripts/run_metrics.sh "$@" || fix_dir_perms $? || exit $?
    fi
--- a/scripts/run_integration_tests_aarch64.sh
+++ b/scripts/run_integration_tests_aarch64.sh
@@ -11,15 +11,16 @@ build_spdk_nvme() {
    SPDK_DIR="$WORKLOADS_DIR/spdk"
    SPDK_REPO="https://github.com/spdk/spdk.git"
    SPDK_DEPLOY_DIR="/usr/local/bin/spdk-nvme"
-    checkout_repo "$SPDK_DIR" "$SPDK_REPO" master "6301f8915de32baed10dba1eebed556a6749211a"
+    checkout_repo "$SPDK_DIR" "$SPDK_REPO" master "ef8bcce58f3f02b79c0619a297e4f17e81e62b24"

    if [ ! -f "$SPDK_DIR/.built" ]; then
        pushd $SPDK_DIR
        git submodule update --init
        apt-get update
+        sed -i "/grpcio/d" scripts/pkgdep/debian.sh
        ./scripts/pkgdep.sh
        ./configure --with-vfio-user
-        chmod +x /usr/local/lib/python3.8/dist-packages/ninja/data/bin/ninja
+        chmod +x /usr/local/lib/python3.10/dist-packages/ninja/data/bin/ninja
        make -j `nproc` || exit 1
        touch .built
        popd
@@ -29,14 +30,15 @@ build_spdk_nvme() {
    fi
    cp "$WORKLOADS_DIR/spdk/build/bin/nvmf_tgt" $SPDK_DEPLOY_DIR/nvmf_tgt
    cp "$WORKLOADS_DIR/spdk/scripts/rpc.py" $SPDK_DEPLOY_DIR/rpc.py
-    cp -r "$WORKLOADS_DIR/spdk/scripts/rpc" $SPDK_DEPLOY_DIR/rpc
+    cp -r "$WORKLOADS_DIR/spdk/python/spdk/" $SPDK_DEPLOY_DIR/
+    cp -r "$WORKLOADS_DIR/spdk/python" $SPDK_DEPLOY_DIR/../
 }

 build_virtiofsd() {
    VIRTIOFSD_DIR="$WORKLOADS_DIR/virtiofsd_build"
    VIRTIOFSD_REPO="https://gitlab.com/virtio-fs/virtiofsd.git"

-    checkout_repo "$VIRTIOFSD_DIR" "$VIRTIOFSD_REPO" v1.1.0 "220405d7a2606c92636d31992b5cb3036a41047b"
+    checkout_repo "$VIRTIOFSD_DIR" "$VIRTIOFSD_REPO" v1.8.0 "97ea7908fe7f9bc59916671a771bdcfaf4044b45"

    if [ ! -f "$VIRTIOFSD_DIR/.built" ]; then
        pushd $VIRTIOFSD_DIR
@@ -152,8 +154,21 @@ update_workloads() {
    CH_RELEASE_URL="https://github.com/cloud-hypervisor/cloud-hypervisor/releases/download/$LAST_RELEASE_VERSION/cloud-hypervisor-static-aarch64"
    CH_RELEASE_NAME="cloud-hypervisor-static-aarch64"
    pushd $WORKLOADS_DIR
-    time wget --quiet $CH_RELEASE_URL -O "$CH_RELEASE_NAME" || exit 1
-    chmod +x $CH_RELEASE_NAME
+    # Repeat a few times to workaround a random wget failure
+    WGET_RETRY_MAX=10
+    wget_retry=0
+
+    until [ "$wget_retry" -ge "$WGET_RETRY_MAX" ]
+    do
+        time wget $CH_RELEASE_URL -O "$CH_RELEASE_NAME" && break
+        wget_retry=$((wget_retry+1))
+    done
+
+    if [ $wget_retry -ge "$WGET_RETRY_MAX" ]; then
+        exit 1
+    else
+        chmod +x $CH_RELEASE_NAME
+    fi
    popd

    # Build custom kernel for guest VMs
@@ -208,6 +223,8 @@ if [[ "$hypervisor" = "mshv" ]]; then
    exit 1
 fi

+# For now these values are deafult for kvm
+features=""

 # lock the workloads folder to avoid parallel updating by different containers
 (
@@ -223,15 +240,13 @@ if [ $RES -ne 0 ]; then
    exit 1
 fi

-BUILD_TARGET="aarch64-unknown-linux-${CH_LIBC}"
-if [[ "${BUILD_TARGET}" == "aarch64-unknown-linux-musl" ]]; then
-export TARGET_CC="musl-gcc"
-export RUSTFLAGS="-C link-arg=-lgcc -C link_arg=-specs -C link_arg=/usr/lib/aarch64-linux-musl/musl-gcc.specs"
-fi
-
 export RUST_BACKTRACE=1

-cargo build --all --release --target $BUILD_TARGET
+# Test without ACPI
+cargo build --all --release $features --target $BUILD_TARGET
+strip target/$BUILD_TARGET/release/cloud-hypervisor
+strip target/$BUILD_TARGET/release/vhost_user_net
+strip target/$BUILD_TARGET/release/ch-remote

 # Enable KSM with some reasonable parameters so that it won't take too long
 # for the memory to be merged between two processes.
@@ -244,13 +259,13 @@ echo 6144 | sudo tee /proc/sys/vm/nr_hugepages
 sudo chmod a+rwX /dev/hugepages

 # Run all direct kernel boot (Device Tree) test cases in mod `parallel`
-time cargo test "common_parallel::$test_filter" --target $BUILD_TARGET -- ${test_binary_args[*]}
+time cargo test $features "common_parallel::$test_filter" --target $BUILD_TARGET -- ${test_binary_args[*]}
 RES=$?

 # Run some tests in sequence since the result could be affected by other tests
 # running in parallel.
 if [ $RES -eq 0 ]; then
-    time cargo test "common_sequential::$test_filter" --target $BUILD_TARGET -- --test-threads=1 ${test_binary_args[*]}
+    time cargo test $features "common_sequential::$test_filter" --target $BUILD_TARGET -- --test-threads=1 ${test_binary_args[*]}
    RES=$?
 else
    exit $RES
@@ -258,7 +273,7 @@ fi

 # Run all ACPI test cases
 if [ $RES -eq 0 ]; then
-    time cargo test "aarch64_acpi::$test_filter" --target $BUILD_TARGET -- ${test_binary_args[*]}
+    time cargo test $features "aarch64_acpi::$test_filter" --target $BUILD_TARGET -- ${test_binary_args[*]}
    RES=$?
 else
    exit $RES
@@ -266,14 +281,14 @@ fi

 # Run all test cases related to live migration
 if [ $RES -eq 0 ]; then
-    time cargo test "live_migration_parallel::$test_filter" --target $BUILD_TARGET -- ${test_binary_args[*]}
+    time cargo test $features "live_migration_parallel::$test_filter" --target $BUILD_TARGET -- ${test_binary_args[*]}
    RES=$?
 else
    exit $RES
 fi

 if [ $RES -eq 0 ]; then
-    time cargo test "live_migration_sequential::$test_filter" --target $BUILD_TARGET -- --test-threads=1 ${test_binary_args[*]}
+    time cargo test $features "live_migration_sequential::$test_filter" --target $BUILD_TARGET -- --test-threads=1 ${test_binary_args[*]}
    RES=$?
 else
    exit $RES
--- a/scripts/run_integration_tests_live_migration.sh
+++ b/scripts/run_integration_tests_live_migration.sh
@@ -4,18 +4,16 @@ set -x
 source $HOME/.cargo/env
 source $(dirname "$0")/test-util.sh

-export BUILD_TARGET=${BUILD_TARGET-x86_64-unknown-linux-gnu}
-
 WORKLOADS_DIR="$HOME/workloads"
 mkdir -p "$WORKLOADS_DIR"

 process_common_args "$@"

 # For now these values are default for kvm
-test_features=""
+features=""

 if [ "$hypervisor" = "mshv" ] ;  then
-    test_features="--no-default-features --features mshv"
+    features="--no-default-features --features mshv"
 fi

 cp scripts/sha1sums-x86_64 $WORKLOADS_DIR
@@ -78,29 +76,29 @@ if [ -d "$LINUX_CUSTOM_DIR" ]; then
    rm -rf $LINUX_CUSTOM_DIR
 fi

-BUILD_TARGET="$(uname -m)-unknown-linux-${CH_LIBC}"
 CFLAGS=""
-TARGET_CC=""
 if [[ "${BUILD_TARGET}" == "x86_64-unknown-linux-musl" ]]; then
-    TARGET_CC="musl-gcc"
    CFLAGS="-I /usr/include/x86_64-linux-musl/ -idirafter /usr/include/"
 fi

-cargo build --no-default-features --features "kvm,mshv" --all --release --target $BUILD_TARGET
+cargo build --all --release $features --target $BUILD_TARGET
+strip target/$BUILD_TARGET/release/cloud-hypervisor
+strip target/$BUILD_TARGET/release/vhost_user_net
+strip target/$BUILD_TARGET/release/ch-remote

 # Test ovs-dpdk relies on hugepages
 echo 6144 | sudo tee /proc/sys/vm/nr_hugepages
 sudo chmod a+rwX /dev/hugepages

 export RUST_BACKTRACE=1
-time cargo test $test_features "live_migration_parallel::$test_filter" -- ${test_binary_args[*]}
+time cargo test $features "live_migration_parallel::$test_filter" -- ${test_binary_args[*]}
 RES=$?

 # Run some tests in sequence since the result could be affected by other tests
 # running in parallel.
 if [ $RES -eq 0 ]; then
    export RUST_BACKTRACE=1
-    time cargo test $test_features "live_migration_sequential::$test_filter" -- --test-threads=1 ${test_binary_args[*]}
+    time cargo test $features "live_migration_sequential::$test_filter" -- --test-threads=1 ${test_binary_args[*]}
    RES=$?
 fi

--- a/scripts/run_integration_tests_rate_limiter.sh
+++ b/scripts/run_integration_tests_rate_limiter.sh
@@ -4,18 +4,16 @@ set -x
 source $HOME/.cargo/env
 source $(dirname "$0")/test-util.sh

-export BUILD_TARGET=${BUILD_TARGET-x86_64-unknown-linux-gnu}
-
 WORKLOADS_DIR="$HOME/workloads"
 mkdir -p "$WORKLOADS_DIR"

 process_common_args "$@"

 # For now these values are default for kvm
-test_features=""
+features=""

 if [ "$hypervisor" = "mshv" ] ;  then
-    test_features="--no-default-features --features mshv"
+    features="--no-default-features --features mshv"
 fi

 cp scripts/sha1sums-x86_64 $WORKLOADS_DIR
@@ -69,18 +67,18 @@ if [ -d "$LINUX_CUSTOM_DIR" ]; then
    rm -rf $LINUX_CUSTOM_DIR
 fi

-BUILD_TARGET="$(uname -m)-unknown-linux-${CH_LIBC}"
 CFLAGS=""
-TARGET_CC=""
 if [[ "${BUILD_TARGET}" == "x86_64-unknown-linux-musl" ]]; then
-    TARGET_CC="musl-gcc"
    CFLAGS="-I /usr/include/x86_64-linux-musl/ -idirafter /usr/include/"
 fi

-cargo build --no-default-features --features "kvm,mshv" --all --release --target $BUILD_TARGET
+cargo build --all --release $features --target $BUILD_TARGET
+strip target/$BUILD_TARGET/release/cloud-hypervisor
+strip target/$BUILD_TARGET/release/vhost_user_net
+strip target/$BUILD_TARGET/release/ch-remote

 export RUST_BACKTRACE=1
-time cargo test $test_features "rate_limiter::$test_filter" -- --test-threads=1 ${test_binary_args[*]}
+time cargo test $features "rate_limiter::$test_filter" -- --test-threads=1 ${test_binary_args[*]}
 RES=$?

 exit $RES
--- a/scripts/run_integration_tests_sgx.sh
+++ b/scripts/run_integration_tests_sgx.sh
@@ -11,46 +11,20 @@ if [[ "$hypervisor" = "mshv" ]]; then
    exit 1
 fi

-WORKLOADS_DIR="$HOME/workloads"
-mkdir -p "$WORKLOADS_DIR"
+# For now these values are default for kvm
+features=""

-FW_URL=$(curl --silent https://api.github.com/repos/cloud-hypervisor/rust-hypervisor-firmware/releases/latest | grep "browser_download_url" | grep -o 'https://.*[^ "]')
-FW="$WORKLOADS_DIR/hypervisor-fw"
-pushd $WORKLOADS_DIR
-rm -f $FW
-time wget --quiet $FW_URL || exit 1
-popd
-
-JAMMY_OS_IMAGE_NAME="jammy-server-cloudimg-amd64-custom-20221118-1.qcow2"
-JAMMY_OS_IMAGE_URL="https://cloud-hypervisor.azureedge.net/$JAMMY_OS_IMAGE_NAME"
-JAMMY_OS_IMAGE="$WORKLOADS_DIR/$JAMMY_OS_IMAGE_NAME"
-if [ ! -f "$JAMMY_OS_IMAGE" ]; then
-    pushd $WORKLOADS_DIR
-    time wget --quiet $JAMMY_OS_IMAGE_URL || exit 1
-    popd
-fi
-
-JAMMY_OS_RAW_IMAGE_NAME="jammy-server-cloudimg-amd64-custom-20221118-1.raw"
-JAMMY_OS_RAW_IMAGE="$WORKLOADS_DIR/$JAMMY_OS_RAW_IMAGE_NAME"
-if [ ! -f "$JAMMY_OS_RAW_IMAGE" ]; then
-    pushd $WORKLOADS_DIR
-    time qemu-img convert -p -f qcow2 -O raw $JAMMY_OS_IMAGE_NAME $JAMMY_OS_RAW_IMAGE_NAME || exit 1
-    popd
-fi
-
-BUILD_TARGET="$(uname -m)-unknown-linux-${CH_LIBC}"
 CFLAGS=""
-TARGET_CC=""
 if [[ "${BUILD_TARGET}" == "x86_64-unknown-linux-musl" ]]; then
-TARGET_CC="musl-gcc"
-CFLAGS="-I /usr/include/x86_64-linux-musl/ -idirafter /usr/include/"
+    CFLAGS="-I /usr/include/x86_64-linux-musl/ -idirafter /usr/include/"
 fi

-cargo build --no-default-features --features "kvm,mshv" --all --release --target $BUILD_TARGET
+cargo build --all --release $features --target $BUILD_TARGET
+strip target/$BUILD_TARGET/release/cloud-hypervisor

 export RUST_BACKTRACE=1

-time cargo test "sgx::$test_filter" -- ${test_binary_args[*]}
+time cargo test $features "sgx::$test_filter" -- ${test_binary_args[*]}
 RES=$?

 exit $RES
--- a/scripts/run_integration_tests_vfio.sh
+++ b/scripts/run_integration_tests_vfio.sh
@@ -5,16 +5,14 @@ source $HOME/.cargo/env
 source $(dirname "$0")/test-util.sh

 process_common_args "$@"
+# For now these values are default for kvm
+features=""

 WORKLOADS_DIR="$HOME/workloads"
 FOCAL_OS_IMAGE_NAME="focal-server-cloudimg-amd64-custom-20210609-0.raw"
 FOCAL_OS_IMAGE="$WORKLOADS_DIR/$FOCAL_OS_IMAGE_NAME"
 FW="$WORKLOADS_DIR/hypervisor-fw"
-
 VMLINUX_IMAGE="$WORKLOADS_DIR/vmlinux"
-if [ ! -f "$VMLINUX_IMAGE" ]; then
-    build_custom_linux
-fi

 BLK_IMAGE="$WORKLOADS_DIR/blk.img"
 MNT_DIR="mount_image"
@@ -37,15 +35,14 @@ cp $FOCAL_OS_IMAGE $VFIO_DIR
 cp $FW $VFIO_DIR
 cp $VMLINUX_IMAGE $VFIO_DIR || exit 1

-BUILD_TARGET="$(uname -m)-unknown-linux-${CH_LIBC}"
 CFLAGS=""
-TARGET_CC=""
 if [[ "${BUILD_TARGET}" == "x86_64-unknown-linux-musl" ]]; then
-TARGET_CC="musl-gcc"
-CFLAGS="-I /usr/include/x86_64-linux-musl/ -idirafter /usr/include/"
+    CFLAGS="-I /usr/include/x86_64-linux-musl/ -idirafter /usr/include/"
 fi

-cargo build --no-default-features --features "kvm,mshv" --all --release --target $BUILD_TARGET
+cargo build --all --release $features --target $BUILD_TARGET
+strip target/$BUILD_TARGET/release/cloud-hypervisor
+strip target/$BUILD_TARGET/release/ch-remote

 # We always copy a fresh version of our binary for our L2 guest.
 cp target/$BUILD_TARGET/release/cloud-hypervisor $VFIO_DIR
@@ -56,11 +53,11 @@ echo 6144 | sudo tee /proc/sys/vm/nr_hugepages
 sudo chmod a+rwX /dev/hugepages

 export RUST_BACKTRACE=1
-time cargo test "vfio::test_vfio" -- ${test_binary_args[*]}
+time cargo test $features "vfio::test_vfio" -- ${test_binary_args[*]}
 RES=$?

 if [ $RES -eq 0 ]; then
-	time cargo test "vfio::test_nvidia" -- --test-threads=1 ${test_binary_args[*]}
+	time cargo test $features "vfio::test_nvidia" -- --test-threads=1 ${test_binary_args[*]}
 	RES=$?
 fi

--- a/scripts/run_integration_tests_windows_aarch64.sh
+++ b/scripts/run_integration_tests_windows_aarch64.sh
@@ -6,6 +6,8 @@ source $(dirname "$0")/test-util.sh
 source $(dirname "$0")/common-aarch64.sh

 process_common_args "$@"
+# For now these values are default for kvm
+features=""

 # aarch64 not supported for MSHV
 if [[ "$hypervisor" = "mshv" ]]; then
@@ -20,14 +22,6 @@ WIN_IMAGE_FILE="$WORKLOADS_DIR/$WIN_IMAGE_BASENAME"
 OVMF_FW="$WORKLOADS_DIR/CLOUDHV_EFI.fd"
 build_edk2

-BUILD_TARGET="$(uname -m)-unknown-linux-${CH_LIBC}"
-CFLAGS=""
-TARGET_CC=""
-if [[ "${BUILD_TARGET}" == "aarch64-unknown-linux-musl" ]]; then
-export TARGET_CC="musl-gcc"
-export RUSTFLAGS="-C link-arg=-lgcc -C link_arg=-specs -C link_arg=/usr/lib/aarch64-linux-musl/musl-gcc.specs"
-fi
-
 # Check if the images are present
 if [[ ! -f ${WIN_IMAGE_FILE} || ! -f ${OVMF_FW} ]]; then
    echo "Windows image/firmware not present in the host"
@@ -44,11 +38,12 @@ dmsetup mknodes

 export RUST_BACKTRACE=1

-cargo build --all --release --target $BUILD_TARGET
+cargo build --all --release $features --target $BUILD_TARGET
+strip target/$BUILD_TARGET/release/cloud-hypervisor

 # Only run with 1 thread to avoid tests interfering with one another because
 # Windows has a static IP configured
-time cargo test "windows::$test_filter" --target $BUILD_TARGET -- ${test_binary_args[*]}
+time cargo test $features "windows::$test_filter" --target $BUILD_TARGET -- ${test_binary_args[*]}
 RES=$?

 dmsetup remove_all -f
--- a/scripts/run_integration_tests_windows_x86_64.sh
+++ b/scripts/run_integration_tests_windows_x86_64.sh
@@ -6,10 +6,10 @@ source $(dirname "$0")/test-util.sh

 process_common_args "$@"
 # For now these values are default for kvm
-test_features=""
+features=""

 if [ "$hypervisor" = "mshv" ] ;  then
-    test_features="--no-default-features --features mshv"
+    features="--no-default-features --features mshv"
 fi
 WIN_IMAGE_FILE="/root/workloads/windows-server-2019.raw"

@@ -22,12 +22,9 @@ if [ ! -f "$OVMF_FW" ]; then
    popd
 fi

-BUILD_TARGET="$(uname -m)-unknown-linux-${CH_LIBC}"
 CFLAGS=""
-TARGET_CC=""
 if [[ "${BUILD_TARGET}" == "x86_64-unknown-linux-musl" ]]; then
-TARGET_CC="musl-gcc"
-CFLAGS="-I /usr/include/x86_64-linux-musl/ -idirafter /usr/include/"
+    CFLAGS="-I /usr/include/x86_64-linux-musl/ -idirafter /usr/include/"
 fi

 # Check if the images are present
@@ -44,13 +41,14 @@ dmsetup mknodes
 dmsetup create windows-snapshot-base --table "0 $img_blk_size snapshot-origin /dev/mapper/windows-base"
 dmsetup mknodes

-cargo build --no-default-features --features "kvm,mshv" --all --release --target $BUILD_TARGET
+cargo build --all --release $features --target $BUILD_TARGET
+strip target/$BUILD_TARGET/release/cloud-hypervisor

 export RUST_BACKTRACE=1

 # Only run with 1 thread to avoid tests interfering with one another because
 # Windows has a static IP configured
-time cargo test $test_features "windows::$test_filter" -- ${test_binary_args[*]}
+time cargo test $features "windows::$test_filter" -- ${test_binary_args[*]}
 RES=$?

 dmsetup remove_all -f
--- a/scripts/run_integration_tests_x86_64.sh
+++ b/scripts/run_integration_tests_x86_64.sh
@@ -4,18 +4,16 @@ set -x
 source $HOME/.cargo/env
 source $(dirname "$0")/test-util.sh

-export BUILD_TARGET=${BUILD_TARGET-x86_64-unknown-linux-gnu}
-
 WORKLOADS_DIR="$HOME/workloads"
 mkdir -p "$WORKLOADS_DIR"

 process_common_args "$@"

 # For now these values are default for kvm
-test_features=""
+features=""

 if [ "$hypervisor" = "mshv" ] ;  then
-    test_features="--no-default-features --features mshv"
+    features="--no-default-features --features mshv"
 fi

 cp scripts/sha1sums-x86_64 $WORKLOADS_DIR
@@ -36,6 +34,24 @@ if [ ! -f "$OVMF_FW" ]; then
    popd
 fi

+BIONIC_OS_IMAGE_NAME="bionic-server-cloudimg-amd64.qcow2"
+BIONIC_OS_IMAGE_URL="https://cloud-hypervisor.azureedge.net/$BIONIC_OS_IMAGE_NAME"
+BIONIC_OS_IMAGE="$WORKLOADS_DIR/$BIONIC_OS_IMAGE_NAME"
+if [ ! -f "$BIONIC_OS_IMAGE" ]; then
+    pushd $WORKLOADS_DIR
+    time wget --quiet $BIONIC_OS_IMAGE_URL || exit 1
+    popd
+fi
+
+BIONIC_OS_RAW_IMAGE_NAME="bionic-server-cloudimg-amd64.raw"
+BIONIC_OS_RAW_IMAGE="$WORKLOADS_DIR/$BIONIC_OS_RAW_IMAGE_NAME"
+if [ ! -f "$BIONIC_OS_RAW_IMAGE" ]; then
+    pushd $WORKLOADS_DIR
+    time qemu-img convert -p -f qcow2 -O raw $BIONIC_OS_IMAGE_NAME $BIONIC_OS_RAW_IMAGE_NAME || exit 1
+    popd
+fi
+
+
 FOCAL_OS_IMAGE_NAME="focal-server-cloudimg-amd64-custom-20210609-0.qcow2"
 FOCAL_OS_IMAGE_URL="https://cloud-hypervisor.azureedge.net/$FOCAL_OS_IMAGE_NAME"
 FOCAL_OS_IMAGE="$WORKLOADS_DIR/$FOCAL_OS_IMAGE_NAME"
@@ -53,7 +69,7 @@ if [ ! -f "$FOCAL_OS_RAW_IMAGE" ]; then
    popd
 fi

-JAMMY_OS_IMAGE_NAME="jammy-server-cloudimg-amd64-custom-20221118-1.qcow2"
+JAMMY_OS_IMAGE_NAME="jammy-server-cloudimg-amd64-custom-20220329-0.qcow2"
 JAMMY_OS_IMAGE_URL="https://cloud-hypervisor.azureedge.net/$JAMMY_OS_IMAGE_NAME"
 JAMMY_OS_IMAGE="$WORKLOADS_DIR/$JAMMY_OS_IMAGE_NAME"
 if [ ! -f "$JAMMY_OS_IMAGE" ]; then
@@ -62,7 +78,7 @@ if [ ! -f "$JAMMY_OS_IMAGE" ]; then
    popd
 fi

-JAMMY_OS_RAW_IMAGE_NAME="jammy-server-cloudimg-amd64-custom-20221118-1.raw"
+JAMMY_OS_RAW_IMAGE_NAME="jammy-server-cloudimg-amd64-custom-20220329-0.raw"
 JAMMY_OS_RAW_IMAGE="$WORKLOADS_DIR/$JAMMY_OS_RAW_IMAGE_NAME"
 if [ ! -f "$JAMMY_OS_RAW_IMAGE" ]; then
    pushd $WORKLOADS_DIR
@@ -116,7 +132,7 @@ if [ ! -f "$VIRTIOFSD" ]; then
    pushd $WORKLOADS_DIR
    git clone "https://gitlab.com/virtio-fs/virtiofsd.git" $VIRTIOFSD_DIR
    pushd $VIRTIOFSD_DIR
-    git checkout v1.1.0
+    git checkout v1.8.0
    time cargo build --release
    cp target/release/virtiofsd $VIRTIOFSD || exit 1
    popd
@@ -154,9 +170,10 @@ cp $FOCAL_OS_RAW_IMAGE $VFIO_DIR
 cp $FW $VFIO_DIR
 cp $VMLINUX_IMAGE $VFIO_DIR || exit 1

-BUILD_TARGET="$(uname -m)-unknown-linux-${CH_LIBC}"
-
-cargo build --no-default-features --features "kvm,mshv" --all  --release --target $BUILD_TARGET
+cargo build --all  --release $features --target $BUILD_TARGET
+strip target/$BUILD_TARGET/release/cloud-hypervisor
+strip target/$BUILD_TARGET/release/vhost_user_net
+strip target/$BUILD_TARGET/release/ch-remote

 # We always copy a fresh version of our binary for our L2 guest.
 cp target/$BUILD_TARGET/release/cloud-hypervisor $VFIO_DIR
@@ -176,14 +193,14 @@ sudo chmod a+rwX /dev/hugepages
 ulimit -l unlimited

 export RUST_BACKTRACE=1
-time cargo test $test_features "common_parallel::$test_filter" -- ${test_binary_args[*]}
+time cargo test $features "common_parallel::$test_filter" -- ${test_binary_args[*]}
 RES=$?

 # Run some tests in sequence since the result could be affected by other tests
 # running in parallel.
 if [ $RES -eq 0 ]; then
    export RUST_BACKTRACE=1
-    time cargo test $test_features "common_sequential::$test_filter" -- --test-threads=1 ${test_binary_args[*]}
+    time cargo test $features "common_sequential::$test_filter" -- --test-threads=1 ${test_binary_args[*]}
    RES=$?
 fi

--- a/scripts/run_metrics.sh
+++ b/scripts/run_metrics.sh
@@ -5,8 +5,6 @@ source $HOME/.cargo/env
 source $(dirname "$0")/test-util.sh

 export TEST_ARCH=$(uname -m)
-export BUILD_TARGET=${BUILD_TARGET-${TEST_ARCH}-unknown-linux-gnu}
-

 WORKLOADS_DIR="$HOME/workloads"
 mkdir -p "$WORKLOADS_DIR"
@@ -28,6 +26,13 @@ build_fio() {

 process_common_args "$@"

+# For now these values are default for kvm
+features=""
+
+if [ "$hypervisor" = "mshv" ]; then
+    features="--no-default-features --features mshv"
+fi
+
 cp scripts/sha1sums-${TEST_ARCH} $WORKLOADS_DIR

 if [ ${TEST_ARCH} == "aarch64" ]; then
@@ -84,15 +89,16 @@ fi
 # Build custom kernel based on virtio-pmem and virtio-fs upstream patches
 build_custom_linux

-BUILD_TARGET="${TEST_ARCH}-unknown-linux-${CH_LIBC}"
 CFLAGS=""
-TARGET_CC=""
 if [[ "${BUILD_TARGET}" == "${TEST_ARCH}-unknown-linux-musl" ]]; then
-    TARGET_CC="musl-gcc"
    CFLAGS="-I /usr/include/${TEST_ARCH}-linux-musl/ -idirafter /usr/include/"
 fi

-cargo build --no-default-features --features "kvm,mshv" --all --release --target $BUILD_TARGET
+cargo build --all --release $features --target $BUILD_TARGET
+strip target/$BUILD_TARGET/release/cloud-hypervisor
+strip target/$BUILD_TARGET/release/vhost_user_net
+strip target/$BUILD_TARGET/release/ch-remote
+strip target/$BUILD_TARGET/release/performance-metrics

 # setup hugepages
 echo 6144 | sudo tee /proc/sys/vm/nr_hugepages
--- a/scripts/run_unit_tests.sh
+++ b/scripts/run_unit_tests.sh
@@ -5,7 +5,6 @@ source $(dirname "$0")/test-util.sh

 process_common_args "$@"

-BUILD_TARGET=${BUILD_TARGET-x86_64-unknown-linux-gnu}
 cargo_args=("")

 if [[ $hypervisor = "mshv" ]]; then
@@ -15,10 +14,5 @@ elif [[ $(uname -m) = "x86_64" ]]; then
    cargo_args+=("--features tdx")
 fi

-if [[ "${BUILD_TARGET}" == "aarch64-unknown-linux-musl" ]]; then
-    export TARGET_CC="musl-gcc"
-    export RUSTFLAGS="-C link-arg=-lgcc -C link_arg=-specs -C link_arg=/usr/lib/aarch64-linux-musl/musl-gcc.specs"
-fi
-
 export RUST_BACKTRACE=1
 cargo test --lib --bins --target $BUILD_TARGET --workspace ${cargo_args[@]} || exit 1
--- a/scripts/sha1sums-x86_64
+++ b/scripts/sha1sums-x86_64
@@ -1,5 +1,7 @@
+27f3b17962ace69b51f0ddc2012095e3109e6ed8 bionic-server-cloudimg-amd64.qcow2
+8db9cc58b01452ce2d06c313177e6e74d8582d93 bionic-server-cloudimg-amd64.raw
 d4a44acc6014d5f83dea1c625c43d677a95fa75f alpine-minirootfs-x86_64.tar.gz
 f1eccdc5e1b515dbad294426ab081b47ebfb97c0 focal-server-cloudimg-amd64-custom-20210609-0.qcow2
 7f5a8358243a96adf61f5c20139b29f308f2c0e3 focal-server-cloudimg-amd64-custom-20210609-0.raw
-b12e52203bc78400b2f099420b411b49ef4be7b8 jammy-server-cloudimg-amd64-custom-20221118-1.qcow2
-d843d5b249e8db3c0158eaf56ade895ed10342b5 jammy-server-cloudimg-amd64-custom-20221118-1.raw
+272102ac243e544bc5c2303f383eeb217bd2e446 jammy-server-cloudimg-amd64-custom-20220329-0.qcow2
+0b5b4816a0976c458f8e8bab061e9d63ebc1b806 jammy-server-cloudimg-amd64-custom-20220329-0.raw
--- a/src/bin/ch-remote.rs
+++ b/src/bin/ch-remote.rs
@@ -3,9 +3,11 @@
 // SPDX-License-Identifier: Apache-2.0
 //

+#[macro_use(crate_authors)]
+extern crate clap;
+
 use api_client::simple_api_command;
 use api_client::simple_api_command_with_fds;
-use api_client::simple_api_full_command;
 use api_client::Error as ApiClientError;
 use clap::{Arg, ArgAction, ArgMatches, Command};
 use option_parser::{ByteSized, ByteSizedParseError};
@@ -336,11 +338,6 @@ fn do_command(matches: &ArgMatches) -> Result<(), Error> {
        Some("counters") => {
            simple_api_command(&mut socket, "GET", "counters", None).map_err(Error::ApiClient)
        }
-        Some("ping") => {
-            simple_api_full_command(&mut socket, "GET", "vmm.ping", None).map_err(Error::ApiClient)
-        }
-        Some("shutdown-vmm") => simple_api_full_command(&mut socket, "PUT", "vmm.shutdown", None)
-            .map_err(Error::ApiClient),
        Some("resize") => resize_api_command(
            &mut socket,
            matches
@@ -503,7 +500,7 @@ fn do_command(matches: &ArgMatches) -> Result<(), Error> {

 fn main() {
    let app = Command::new("ch-remote")
-        .author(env!("CARGO_PKG_AUTHORS"))
+        .author(crate_authors!())
        .subcommand_required(true)
        .about("Remotely control a cloud-hypervisor VMM.")
        .arg(
@@ -678,9 +675,7 @@ fn main() {
            Command::new("create")
                .about("Create VM from a JSON configuration")
                .arg(Arg::new("path").index(1).default_value("-")),
-        )
-        .subcommand(Command::new("ping").about("Ping the VMM to check for API server availability"))
-        .subcommand(Command::new("shutdown-vmm").about("Shutdown the VMM"));
+        );

    let matches = app.get_matches();

--- a/src/main.rs
+++ b/src/main.rs
@@ -3,6 +3,8 @@
 // SPDX-License-Identifier: Apache-2.0
 //

+#[macro_use(crate_authors)]
+extern crate clap;
 #[macro_use]
 extern crate event_monitor;

@@ -89,7 +91,7 @@ impl log::Log for Logger {
        if record.file().is_some() && record.line().is_some() {
            writeln!(
                *(*(self.output.lock().unwrap())),
-                "cloud-hypervisor: {:.6?}: <{}> {}:{}:{} -- {}",
+                "cloud-hypervisor: {:?}: <{}> {}:{}:{} -- {}",
                duration,
                std::thread::current().name().unwrap_or("anonymous"),
                record.level(),
@@ -100,7 +102,7 @@ impl log::Log for Logger {
        } else {
            writeln!(
                *(*(self.output.lock().unwrap())),
-                "cloud-hypervisor: {:.6?}: <{}> {}:{} -- {}",
+                "cloud-hypervisor: {:?}: <{}> {}:{} -- {}",
                duration,
                std::thread::current().name().unwrap_or("anonymous"),
                record.level(),
@@ -123,12 +125,11 @@ fn prepare_default_values() -> (String, String, String) {
 }

 fn create_app(default_vcpus: String, default_memory: String, default_rng: String) -> Command {
-    #[allow(clippy::let_and_return)]
    let app = Command::new("cloud-hypervisor")
        // 'BUILT_VERSION' is set by the build script 'build.rs' at
        // compile time
        .version(env!("BUILT_VERSION"))
-        .author(env!("CARGO_PKG_AUTHORS"))
+        .author(crate_authors!())
        .about("Launch a cloud-hypervisor VMM.")
        .group(ArgGroup::new("vm-config").multiple(true))
        .group(ArgGroup::new("vmm-config").multiple(true))
@@ -452,7 +453,6 @@ fn start_vmm(cmd_arguments: ArgMatches) -> Result<Option<String>, Error> {
                .convert("fd")
                .map_err(Error::ParsingEventMonitor)?
                .unwrap();
-            // SAFETY: fd is valid
            unsafe { File::from_raw_fd(fd) }
        } else if parser.is_set("path") {
            std::fs::OpenOptions::new()
@@ -592,7 +592,6 @@ fn start_vmm(cmd_arguments: ArgMatches) -> Result<Option<String>, Error> {

 fn main() {
    // Ensure all created files (.e.g sockets) are only accessible by this user
-    // SAFETY: trivially safe
    let _ = unsafe { libc::umask(0o077) };

    let (default_vcpus, default_memory, default_rng) = prepare_default_values();
@@ -608,7 +607,6 @@ fn main() {
        }
    };

-    // SAFETY: trivially safe
    let on_tty = unsafe { libc::isatty(libc::STDIN_FILENO) } != 0;
    if on_tty {
        // Don't forget to set the terminal in canonical mode
@@ -724,6 +722,7 @@ mod unit_tests {
            gdb: false,
            platform: None,
            tpm: None,
+            preserved_fds: None,
        };

        assert_eq!(expected_vm_config, result_vm_config);
@@ -731,7 +730,7 @@ mod unit_tests {

    #[test]
    fn test_valid_vm_config_cpus() {
-        vec![
+        [
            (
                vec![
                    "cloud-hypervisor",
@@ -849,7 +848,7 @@ mod unit_tests {

    #[test]
    fn test_valid_vm_config_kernel() {
-        vec![(
+        [(
            vec!["cloud-hypervisor", "--kernel", "/path/to/kernel"],
            r#"{
                "payload": {"kernel": "/path/to/kernel"}
@@ -864,7 +863,7 @@ mod unit_tests {

    #[test]
    fn test_valid_vm_config_cmdline() {
-        vec![(
+        [(
            vec![
                "cloud-hypervisor",
                "--kernel",
@@ -885,7 +884,7 @@ mod unit_tests {

    #[test]
    fn test_valid_vm_config_disks() {
-        vec![
+        [
            (
                vec![
                    "cloud-hypervisor",
@@ -1157,7 +1156,7 @@ mod unit_tests {

    #[test]
    fn test_valid_vm_config_rng() {
-        vec![(
+        [(
            vec![
                "cloud-hypervisor",
                "--kernel",
@@ -1179,8 +1178,7 @@ mod unit_tests {

    #[test]
    fn test_valid_vm_config_fs() {
-        vec![
-            (
+        [(
                vec![
                    "cloud-hypervisor", "--kernel", "/path/to/kernel",
                    "--memory", "shared=true",
@@ -1248,8 +1246,7 @@ mod unit_tests {
                    ]
                }"#,
                true,
-            ),
-        ]
+            )]
        .iter()
        .for_each(|(cli, openapi, equal)| {
            compare_vm_config_cli_vs_json(cli, openapi, *equal);
@@ -1258,7 +1255,7 @@ mod unit_tests {

    #[test]
    fn test_valid_vm_config_pmem() {
-        vec![
+        [
            (
                vec![
                    "cloud-hypervisor",
@@ -1321,7 +1318,7 @@ mod unit_tests {

    #[test]
    fn test_valid_vm_config_serial_console() {
-        vec![
+        [
            (
                vec!["cloud-hypervisor", "--kernel", "/path/to/kernel"],
                r#"{
@@ -1372,7 +1369,7 @@ mod unit_tests {

    #[test]
    fn test_valid_vm_config_serial_pty_console_pty() {
-        vec![
+        [
            (
                vec!["cloud-hypervisor", "--kernel", "/path/to/kernel"],
                r#"{
@@ -1518,7 +1515,7 @@ mod unit_tests {

    #[test]
    fn test_valid_vm_config_vdpa() {
-        vec![
+        [
            (
                vec![
                    "cloud-hypervisor",
@@ -1563,7 +1560,7 @@ mod unit_tests {

    #[test]
    fn test_valid_vm_config_vsock() {
-        vec![
+        [
            (
                vec![
                    "cloud-hypervisor",
@@ -1646,7 +1643,7 @@ mod unit_tests {

    #[test]
    fn test_valid_vm_config_tpm_socket() {
-        vec![(
+        [(
            vec![
                "cloud-hypervisor",
                "--kernel",
--- a/Show More
+++ b/Show More