build: Release v28.4 (bug fix release)

Signed-off-by: Bo Chen <chen.bo@intel.com>
build: Bump MSRV to 1.65
2026-08-05 02:19:16 +00:00 · 2024-01-19 16:04:30 -08:00 · 2024-01-19 09:09:22 -08:00 · 2024-01-19 09:09:22 -08:00 · 2024-01-19 09:09:22 -08:00 · 2024-01-19 09:09:22 -08:00
68 changed files with 736 additions and 413 deletions
--- a/.github/workflows/build.yaml
+++ b/.github/workflows/build.yaml
@@ -13,7 +13,7 @@ jobs:
          - stable
          - beta
          - nightly
-          - "1.60"
+          - "1.65"
        target:
          - x86_64-unknown-linux-gnu
          - x86_64-unknown-linux-musl
--- a/.github/workflows/release.yaml
+++ b/.github/workflows/release.yaml
@@ -16,29 +16,29 @@ jobs:
      - name: Install Rust toolchain (x86_64-unknown-linux-gnu)
        uses: actions-rs/toolchain@v1
        with:
-          toolchain: "1.62"
+          toolchain: "1.65"
          target: x86_64-unknown-linux-gnu
      - name: Install Rust toolchain (x86_64-unknown-linux-musl)
        uses: actions-rs/toolchain@v1
        with:
-          toolchain: "1.62"
+          toolchain: "1.65"
          target: x86_64-unknown-linux-musl
      - name: Build
        uses: actions-rs/cargo@v1
        with:
-            toolchain: "1.62"
+            toolchain: "1.65"
            command: build
            args: --all --release --target=x86_64-unknown-linux-gnu
      - name: Static Build
        uses: actions-rs/cargo@v1
        with:
-            toolchain: "1.62"
+            toolchain: "1.65"
            command: build
            args: --all --release --target=x86_64-unknown-linux-musl
      - name: Install Rust toolchain (aarch64-unknown-linux-musl)
        uses: actions-rs/toolchain@v1
        with:
-          toolchain: "1.62"
+          toolchain: "1.65"
          target: aarch64-unknown-linux-musl
          override: true
      - name: Create Release
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -11,9 +11,9 @@ dependencies = [

 [[package]]
 name = "aho-corasick"
-version = "0.7.20"
+version = "1.1.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cc936419f96fa211c1b9166887b38e5e40b19958e5b895be7c1f93adec7071ac"
+checksum = "b2969dcb958b36655471fc61f7e416fa76033bdd4bfed0678d8fee1e2d07a1f0"
 dependencies = [
 "memchr",
 ]
@@ -80,6 +80,12 @@ version = "1.3.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"

+[[package]]
+name = "bitflags"
+version = "2.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "327762f6e5a765692301e5bb513e0d9fef63be86bbc14528052b1cd3e6f03e07"
+
 [[package]]
 name = "block_util"
 version = "0.1.0"
@@ -123,7 +129,7 @@ version = "4.0.29"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "4d63b9e9c07271b9957ad22c173bae2a4d9a81127680962039296abcd2f8251d"
 dependencies = [
- "bitflags",
+ "bitflags 1.3.2",
 "clap_lex",
 "is-terminal",
 "once_cell",
@@ -143,7 +149,7 @@ dependencies = [

 [[package]]
 name = "cloud-hypervisor"
-version = "28.2.0"
+version = "28.4.0"
 dependencies = [
 "anyhow",
 "api_client",
@@ -206,7 +212,7 @@ dependencies = [
 "proc-macro2",
 "quote",
 "strsim",
- "syn",
+ "syn 1.0.105",
 ]

 [[package]]
@@ -217,7 +223,7 @@ checksum = "7618812407e9402654622dd402b0a89dff9ba93badd6540781526117b92aab7e"
 dependencies = [
 "darling_core",
 "quote",
- "syn",
+ "syn 1.0.105",
 ]

 [[package]]
@@ -227,7 +233,7 @@ dependencies = [
 "acpi_tables",
 "anyhow",
 "arch",
- "bitflags",
+ "bitflags 2.4.1",
 "byteorder",
 "hypervisor",
 "libc",
@@ -282,7 +288,7 @@ version = "4.3.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "20df693c700404f7e19d4d6fae6b15215d2913c27955d2b9d6f2c0f537511cd0"
 dependencies = [
- "bitflags",
+ "bitflags 1.3.2",
 "libc",
 ]

@@ -312,6 +318,7 @@ name = "event_monitor"
 version = "0.1.0"
 dependencies = [
 "libc",
+ "once_cell",
 "serde",
 "serde_json",
 ]
@@ -334,7 +341,7 @@ version = "0.6.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "32c95766e0414f8bfc1d07055574c621b67739466d6ba516c4fef8e99d30d2e6"
 dependencies = [
- "bitflags",
+ "bitflags 1.3.2",
 "cfg-if",
 "log",
 "managed",
@@ -365,9 +372,9 @@ dependencies = [

 [[package]]
 name = "glob"
-version = "0.3.0"
+version = "0.3.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574"
+checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b"

 [[package]]
 name = "hermit-abi"
@@ -447,15 +454,15 @@ version = "0.5.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7ba34abb5175052fc1a2227a10d2275b7386c9990167de9786c0b88d8b062330"
 dependencies = [
- "bitflags",
+ "bitflags 1.3.2",
 "libc",
 ]

 [[package]]
 name = "ipnetwork"
-version = "0.19.0"
+version = "0.20.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1f84f1612606f3753f205a4e9a2efd6fe5b4c573a6269b2cc6c3003d44a0d127"
+checksum = "bf466541e9d546596ee94f9f69590f89473455f88372423e0008fc1a7daf100e"
 dependencies = [
 "serde",
 ]
@@ -490,9 +497,8 @@ dependencies = [

 [[package]]
 name = "kvm-ioctls"
-version = "0.12.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c3a321cabd827642499c77e27314f388dd83a717a5ca716b86476fb947f73ae4"
+version = "0.13.0"
+source = "git+https://github.com/rust-vmm/kvm-ioctls?branch=main#23a3bb045a467e60bb00328a0b13cea13b5815d0"
 dependencies = [
 "kvm-bindings",
 "libc",
@@ -507,9 +513,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"

 [[package]]
 name = "libc"
-version = "0.2.138"
+version = "0.2.152"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "db6d7e329c562c5dfab7a46a2afabc8b987ab9a4834c9d1ca04dc54c1546cef8"
+checksum = "13e3bf6590cbc649f4d1a3eefc9d5d6eb746f5200ffb04e5e142700b8faa56e7"

 [[package]]
 name = "libssh2-sys"
@@ -579,9 +585,9 @@ checksum = "0ca88d725a0a943b096803bd34e73a4437208b6077654cc4ecb2947a5f91618d"

 [[package]]
 name = "memchr"
-version = "2.5.0"
+version = "2.7.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d"
+checksum = "523dc4f511e55ab87b694dc30d0f820d60906ef06413f93d4d7a1385599cc149"

 [[package]]
 name = "micro_http"
@@ -797,7 +803,7 @@ dependencies = [
 "phf_shared",
 "proc-macro2",
 "quote",
- "syn",
+ "syn 1.0.105",
 ]

 [[package]]
@@ -817,30 +823,32 @@ checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160"

 [[package]]
 name = "pnet"
-version = "0.31.0"
+version = "0.34.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0caaf5b11fd907ff15cf14a4477bfabca4b37ab9e447a4f8dead969a59cdafad"
+checksum = "130c5b738eeda2dc5796fe2671e49027e6935e817ab51b930a36ec9e6a206a64"
 dependencies = [
+ "ipnetwork",
 "pnet_base",
 "pnet_datalink",
 "pnet_packet",
+ "pnet_sys",
 "pnet_transport",
 ]

 [[package]]
 name = "pnet_base"
-version = "0.31.0"
+version = "0.34.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f9d3a993d49e5fd5d4d854d6999d4addca1f72d86c65adf224a36757161c02b6"
+checksum = "fe4cf6fb3ab38b68d01ab2aea03ed3d1132b4868fa4e06285f29f16da01c5f4c"
 dependencies = [
 "no-std-net",
 ]

 [[package]]
 name = "pnet_datalink"
-version = "0.31.0"
+version = "0.34.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e466faf03a98ad27f6e15cd27a2b7cc89e73e640a43527742977bc503c37f8aa"
+checksum = "ad5854abf0067ebbd3967f7d45ebc8976ff577ff0c7bd101c4973ae3c70f98fe"
 dependencies = [
 "ipnetwork",
 "libc",
@@ -851,30 +859,30 @@ dependencies = [

 [[package]]
 name = "pnet_macros"
-version = "0.31.0"
+version = "0.34.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "48dd52a5211fac27e7acb14cfc9f30ae16ae0e956b7b779c8214c74559cef4c3"
+checksum = "688b17499eee04a0408aca0aa5cba5fc86401d7216de8a63fdf7a4c227871804"
 dependencies = [
 "proc-macro2",
 "quote",
 "regex",
- "syn",
+ "syn 2.0.48",
 ]

 [[package]]
 name = "pnet_macros_support"
-version = "0.31.0"
+version = "0.34.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "89de095dc7739349559913aed1ef6a11e73ceade4897dadc77c5e09de6740750"
+checksum = "eea925b72f4bd37f8eab0f221bbe4c78b63498350c983ffa9dd4bcde7e030f56"
 dependencies = [
 "pnet_base",
 ]

 [[package]]
 name = "pnet_packet"
-version = "0.31.0"
+version = "0.34.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bc3b5111e697c39c8b9795b9fdccbc301ab696699e88b9ea5a4e4628978f495f"
+checksum = "a9a005825396b7fe7a38a8e288dbc342d5034dac80c15212436424fef8ea90ba"
 dependencies = [
 "glob",
 "pnet_base",
@@ -884,9 +892,9 @@ dependencies = [

 [[package]]
 name = "pnet_sys"
-version = "0.31.0"
+version = "0.34.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "328e231f0add6d247d82421bf3790b4b33b39c8930637f428eef24c4c6a90805"
+checksum = "417c0becd1b573f6d544f73671070b039051e5ad819cc64aa96377b536128d00"
 dependencies = [
 "libc",
 "winapi",
@@ -894,9 +902,9 @@ dependencies = [

 [[package]]
 name = "pnet_transport"
-version = "0.31.0"
+version = "0.34.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ff597185e6f1f5671b3122e4dba892a1c73e17c17e723d7669bd9299cbe7f124"
+checksum = "2637e14d7de974ee2f74393afccbc8704f3e54e6eb31488715e72481d1662cc3"
 dependencies = [
 "libc",
 "pnet_base",
@@ -906,9 +914,9 @@ dependencies = [

 [[package]]
 name = "proc-macro2"
-version = "1.0.47"
+version = "1.0.76"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5ea3d908b0e36316caf9e9e2c4625cdde190a7e6f440d794667ed17a1855e725"
+checksum = "95fc56cda0b5c3325f5fbbd7ff9fda9e02bb00bb3dac51252d2f1bfa1cb8cc8c"
 dependencies = [
 "unicode-ident",
 ]
@@ -926,9 +934,9 @@ dependencies = [

 [[package]]
 name = "quote"
-version = "1.0.21"
+version = "1.0.35"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bbe448f377a7d6961e30f5955f9b8d106c3f5e449d493ee1b125c1d43c2b5179"
+checksum = "291ec9ab5efd934aaf503a6466c5d5251535d108ee747472c3977cc5acc868ef"
 dependencies = [
 "proc-macro2",
 ]
@@ -963,7 +971,7 @@ version = "0.2.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a"
 dependencies = [
- "bitflags",
+ "bitflags 1.3.2",
 ]

 [[package]]
@@ -979,9 +987,21 @@ dependencies = [

 [[package]]
 name = "regex"
-version = "1.7.0"
+version = "1.10.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e076559ef8e241f2ae3479e36f97bd5741c0330689e217ad51ce2c76808b868a"
+checksum = "380b951a9c5e80ddfd6136919eef32310721aa4aacd4889a8d39124b026ab343"
+dependencies = [
+ "aho-corasick",
+ "memchr",
+ "regex-automata",
+ "regex-syntax",
+]
+
+[[package]]
+name = "regex-automata"
+version = "0.4.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5f804c7828047e88b2d32e2d7fe5a105da8ee3264f01902f796c8e067dc2483f"
 dependencies = [
 "aho-corasick",
 "memchr",
@@ -990,9 +1010,9 @@ dependencies = [

 [[package]]
 name = "regex-syntax"
-version = "0.6.28"
+version = "0.8.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "456c603be3e8d448b072f410900c09faf164fbce2d480456f50eea6e25f9c848"
+checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f"

 [[package]]
 name = "remain"
@@ -1002,7 +1022,7 @@ checksum = "1a81bc6a3aaeb180f767bd2bda8c03bac5b93b3b69a783c66f35d16a4df276cf"
 dependencies = [
 "proc-macro2",
 "quote",
- "syn",
+ "syn 1.0.105",
 ]

 [[package]]
@@ -1020,7 +1040,7 @@ version = "0.36.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a3807b5d10909833d3e9acd1eb5fb988f79376ff10fce42937de71a449c4c588"
 dependencies = [
- "bitflags",
+ "bitflags 1.3.2",
 "errno",
 "io-lifetimes",
 "libc",
@@ -1072,7 +1092,7 @@ checksum = "42a3df25b0713732468deadad63ab9da1f1fd75a48a15024b50363f128db627e"
 dependencies = [
 "proc-macro2",
 "quote",
- "syn",
+ "syn 1.0.105",
 ]

 [[package]]
@@ -1105,7 +1125,7 @@ dependencies = [
 "darling",
 "proc-macro2",
 "quote",
- "syn",
+ "syn 1.0.105",
 ]

 [[package]]
@@ -1149,7 +1169,7 @@ version = "0.9.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "269343e64430067a14937ae0e3c4ec604c178fb896dde0964b1acd22b3e2eeb1"
 dependencies = [
- "bitflags",
+ "bitflags 1.3.2",
 "libc",
 "libssh2-sys",
 "parking_lot",
@@ -1178,6 +1198,17 @@ dependencies = [
 "unicode-ident",
 ]

+[[package]]
+name = "syn"
+version = "2.0.48"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0f3531638e407dfc0814761abb7c00a5b54992b849452a0646b7f65c9f770f3f"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "unicode-ident",
+]
+
 [[package]]
 name = "termcolor"
 version = "1.1.3"
@@ -1229,7 +1260,7 @@ checksum = "982d17546b47146b28f7c22e3d08465f6b8903d0ea13c1660d9d84a6e7adcdbb"
 dependencies = [
 "proc-macro2",
 "quote",
- "syn",
+ "syn 1.0.105",
 ]

 [[package]]
@@ -1288,7 +1319,7 @@ dependencies = [
 "quote",
 "serde",
 "serde_derive",
- "syn",
+ "syn 1.0.105",
 "versionize_derive",
 "vmm-sys-util",
 ]
@@ -1300,13 +1331,13 @@ source = "git+https://github.com/cloud-hypervisor/versionize_derive?branch=ch#ae
 dependencies = [
 "proc-macro2",
 "quote",
- "syn",
+ "syn 1.0.105",
 ]

 [[package]]
 name = "vfio-bindings"
 version = "0.4.0"
-source = "git+https://github.com/rust-vmm/vfio?branch=main#7ef33f6759ae6d37a54cd66f749aa47c1c81509c"
+source = "git+https://github.com/rust-vmm/vfio?branch=main#43439e056ddfa84a4f7906ee7f2f58be70505c08"
 dependencies = [
 "vmm-sys-util",
 ]
@@ -1314,7 +1345,7 @@ dependencies = [
 [[package]]
 name = "vfio-ioctls"
 version = "0.2.0"
-source = "git+https://github.com/rust-vmm/vfio?branch=main#7ef33f6759ae6d37a54cd66f749aa47c1c81509c"
+source = "git+https://github.com/rust-vmm/vfio?branch=main#43439e056ddfa84a4f7906ee7f2f58be70505c08"
 dependencies = [
 "byteorder",
 "kvm-bindings",
@@ -1363,7 +1394,7 @@ version = "0.6.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c9b791c5b0717a0558888a4cf7240cea836f39a99cb342e12ce633dcaa078072"
 dependencies = [
- "bitflags",
+ "bitflags 1.3.2",
 "libc",
 "vm-memory",
 "vmm-sys-util",
@@ -1543,7 +1574,7 @@ dependencies = [
 "anyhow",
 "arc-swap",
 "arch",
- "bitflags",
+ "bitflags 2.4.1",
 "block_util",
 "clap",
 "devices",
@@ -1590,7 +1621,7 @@ version = "0.11.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "cc06a16ee8ebf0d9269aed304030b0d20a866b8b3dd3d4ce532596ac567a0d24"
 dependencies = [
- "bitflags",
+ "bitflags 1.3.2",
 "libc",
 "serde",
 "serde_derive",
@@ -1717,5 +1748,5 @@ checksum = "6505e6815af7de1746a08f69c69606bb45695a17149517680f3b2149713b19a3"
 dependencies = [
 "proc-macro2",
 "quote",
- "syn",
+ "syn 1.0.105",
 ]
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "cloud-hypervisor"
-version = "28.2.0"
+version = "28.4.0"
 authors = ["The Cloud Hypervisor Authors"]
 edition = "2021"
 default-run = "cloud-hypervisor"
@@ -10,7 +10,12 @@ description = "Open source Virtual Machine Monitor (VMM) that runs on top of KVM
 homepage = "https://github.com/cloud-hypervisor/cloud-hypervisor"
 # Minimum buildable version:
 # Keep in sync with version in .github/workflows/build.yaml
-rust-version = "1.60"
+# Policy on MSRV (see #4318):
+# Can only be bumped by:
+# a.) A dependency requires it, 
+# b.) If we want to use a new feature and that MSRV is at least 6 months old,
+# c.) There is a security issue that is addressed by the toolchain update.
+rust-version = "1.65"

 [profile.release]
 lto = true
@@ -44,6 +49,7 @@ clap = { version = "4.0.29", features = ["cargo"] }
 # List of patched crates
 [patch.crates-io]
 kvm-bindings = { git = "https://github.com/cloud-hypervisor/kvm-bindings", branch = "ch-v0.6.0-tdx" }
+kvm-ioctls = { git = "https://github.com/rust-vmm/kvm-ioctls", branch = "main" }
 versionize_derive = { git = "https://github.com/cloud-hypervisor/versionize_derive", branch = "ch" }

 [dev-dependencies]
--- a/acpi_tables/src/aml.rs
+++ b/acpi_tables/src/aml.rs
@@ -1083,7 +1083,7 @@ impl<'a, T> CreateField<'a, T> {
            buffer,
            offset,
            field,
-            phantom: PhantomData::default(),
+            phantom: PhantomData,
        }
    }
 }
--- a/arch/src/x86_64/mod.rs
+++ b/arch/src/x86_64/mod.rs
@@ -16,7 +16,7 @@ use crate::GuestMemoryMmap;
 use crate::InitramfsConfig;
 use crate::RegionType;
 use hypervisor::arch::x86::{CpuIdEntry, CPUID_FLAG_VALID_INDEX};
-use hypervisor::HypervisorError;
+use hypervisor::{HypervisorCpuError, HypervisorError};
 use linux_loader::loader::bootparam::boot_params;
 use linux_loader::loader::elf::start_info::{
    hvm_memmap_table_entry, hvm_modlist_entry, hvm_start_info,
@@ -36,6 +36,7 @@ pub mod tdx;
 const TSC_DEADLINE_TIMER_ECX_BIT: u8 = 24; // tsc deadline timer ecx bit.
 const HYPERVISOR_ECX_BIT: u8 = 31; // Hypervisor ecx bit.
 const MTRR_EDX_BIT: u8 = 12; // Hypervisor ecx bit.
+const INVARIANT_TSC_EDX_BIT: u8 = 8; // Invariant TSC bit on 0x8000_0007 EDX

 // KVM feature bits
 const KVM_FEATURE_ASYNC_PF_INT_BIT: u8 = 14;
@@ -191,6 +192,9 @@ pub enum Error {
    // Error writing EBDA address
    EbdaSetup(vm_memory::GuestMemoryError),

+    // Error getting CPU TSC frequency
+    GetTscFrequency(HypervisorCpuError),
+
    /// Error retrieving TDX capabilities through the hypervisor (kvm/mshv) API
    #[cfg(feature = "tdx")]
    TdxCapabilities(HypervisorError),
@@ -749,6 +753,41 @@ pub fn configure_vcpu(
    CpuidPatch::set_cpuid_reg(&mut cpuid, 0xb, None, CpuidReg::EDX, u32::from(id));
    CpuidPatch::set_cpuid_reg(&mut cpuid, 0x1f, None, CpuidReg::EDX, u32::from(id));

+    // Set ApicId in cpuid for each vcpu
+    // SAFETY: get host cpuid when eax=1
+    let mut cpu_ebx = unsafe { core::arch::x86_64::__cpuid(1) }.ebx;
+    cpu_ebx &= 0xffffff;
+    cpu_ebx |= (id as u32) << 24;
+    CpuidPatch::set_cpuid_reg(&mut cpuid, 0x1, None, CpuidReg::EBX, cpu_ebx);
+
+    // The TSC frequency CPUID leaf should not be included when running with HyperV emulation
+    if !kvm_hyperv {
+        if let Some(tsc_khz) = vcpu.tsc_khz().map_err(Error::GetTscFrequency)? {
+            // Need to check that the TSC doesn't vary with dynamic frequency
+            // SAFETY: cpuid called with valid leaves
+            if unsafe { std::arch::x86_64::__cpuid(0x8000_0007) }.edx
+                & (1u32 << INVARIANT_TSC_EDX_BIT)
+                > 0
+            {
+                CpuidPatch::set_cpuid_reg(
+                    &mut cpuid,
+                    0x4000_0000,
+                    None,
+                    CpuidReg::EAX,
+                    0x4000_0010,
+                );
+                cpuid.retain(|c| c.function != 0x4000_0010);
+                cpuid.push(CpuIdEntry {
+                    function: 0x4000_0010,
+                    eax: tsc_khz,
+                    ebx: 1000000, /* LAPIC resolution of 1ns (freq: 1GHz) is hardcoded in KVM's
+                                   * APIC_BUS_CYCLE_NS */
+                    ..Default::default()
+                });
+            };
+        }
+    }
+
    vcpu.set_cpuid2(&cpuid)
        .map_err(|e| Error::SetSupportedCpusFailed(e.into()))?;

--- a/arch/src/x86_64/tdx/mod.rs
+++ b/arch/src/x86_64/tdx/mod.rs
@@ -46,7 +46,7 @@ pub struct TdvfSection {
 }

 #[repr(u32)]
-#[derive(Clone, Copy, Debug)]
+#[derive(Clone, Copy, Debug, Default)]
 pub enum TdvfSectionType {
    Bfv,
    Cfv,
@@ -55,15 +55,10 @@ pub enum TdvfSectionType {
    PermMem,
    Payload,
    PayloadParam,
+    #[default]
    Reserved = 0xffffffff,
 }

-impl Default for TdvfSectionType {
-    fn default() -> Self {
-        TdvfSectionType::Reserved
-    }
-}
-
 pub fn parse_tdvf_sections(file: &mut File) -> Result<Vec<TdvfSection>, TdvfError> {
    // The 32-bit offset to the TDVF metadata is located 32 bytes from
    // the end of the file.
@@ -120,21 +115,16 @@ pub fn parse_tdvf_sections(file: &mut File) -> Result<Vec<TdvfSection>, TdvfErro
 }

 #[repr(u16)]
-#[derive(Copy, Clone, Debug)]
+#[derive(Copy, Clone, Debug, Default)]
 enum HobType {
    Handoff = 0x1,
    ResourceDescriptor = 0x3,
    GuidExtension = 0x4,
+    #[default]
    Unused = 0xfffe,
    EndOfHobList = 0xffff,
 }

-impl Default for HobType {
-    fn default() -> Self {
-        HobType::Unused
-    }
-}
-
 #[repr(C)]
 #[derive(Copy, Clone, Default, Debug)]
 struct HobHeader {
@@ -184,19 +174,14 @@ struct HobGuidType {
 }

 #[repr(u32)]
-#[derive(Clone, Copy, Debug)]
+#[derive(Clone, Copy, Debug, Default)]
 pub enum PayloadImageType {
+    #[default]
    ExecutablePayload,
    BzImage,
    RawVmLinux,
 }

-impl Default for PayloadImageType {
-    fn default() -> Self {
-        PayloadImageType::ExecutablePayload
-    }
-}
-
 #[repr(C)]
 #[derive(Copy, Clone, Default, Debug)]
 pub struct PayloadInfo {
--- a/block_util/src/raw_sync.rs
+++ b/block_util/src/raw_sync.rs
@@ -71,7 +71,7 @@ impl AsyncIo for RawFileSync {
        let result = unsafe {
            libc::preadv(
                self.fd as libc::c_int,
-                iovecs.as_ptr() as *const libc::iovec,
+                iovecs.as_ptr(),
                iovecs.len() as libc::c_int,
                offset,
            )
@@ -95,7 +95,7 @@ impl AsyncIo for RawFileSync {
        let result = unsafe {
            libc::pwritev(
                self.fd as libc::c_int,
-                iovecs.as_ptr() as *const libc::iovec,
+                iovecs.as_ptr(),
                iovecs.len() as libc::c_int,
                offset,
            )
--- a/devices/Cargo.toml
+++ b/devices/Cargo.toml
@@ -8,7 +8,7 @@ edition = "2021"
 acpi_tables = { path = "../acpi_tables" }
 anyhow = "1.0.66"
 arch = { path = "../arch" }
-bitflags = "1.3.2"
+bitflags = "2.4.1"
 byteorder = "1.4.3"
 hypervisor = { path = "../hypervisor" }
 libc = "0.2.138"
--- a/event_monitor/Cargo.toml
+++ b/event_monitor/Cargo.toml
@@ -6,5 +6,6 @@ edition = "2021"

 [dependencies]
 libc = "0.2.138"
+once_cell = "1.16.0"
 serde = { version = "1.0.150", features = ["rc", "derive"] }
 serde_json = "1.0.89"
--- a/event_monitor/src/lib.rs
+++ b/event_monitor/src/lib.rs
@@ -3,6 +3,7 @@
 // SPDX-License-Identifier: Apache-2.0
 //

+use once_cell::sync::OnceCell;
 use serde::Serialize;
 use std::borrow::Cow;
 use std::collections::HashMap;
@@ -11,12 +12,13 @@ use std::io::Write;
 use std::os::unix::io::AsRawFd;
 use std::time::{Duration, Instant};

-static mut MONITOR: Option<(File, Instant)> = None;
+static MONITOR: OnceCell<(File, Instant)> = OnceCell::new();

 /// This function must only be called once from the main process before any threads
 /// are created to avoid race conditions
 pub fn set_monitor(file: File) -> Result<(), std::io::Error> {
-    assert!(unsafe { MONITOR.is_none() });
+    // There is only one caller of this function, so MONITOR is written to only once
+    assert!(MONITOR.get().is_none());
    let fd = file.as_raw_fd();
    let ret = unsafe {
        let mut flags = libc::fcntl(fd, libc::F_GETFL);
@@ -26,9 +28,9 @@ pub fn set_monitor(file: File) -> Result<(), std::io::Error> {
    if ret < 0 {
        return Err(std::io::Error::last_os_error());
    }
-    unsafe {
-        MONITOR = Some((file, Instant::now()));
-    };
+
+    MONITOR.get_or_init(|| (file, Instant::now()));
+
    Ok(())
 }

@@ -41,7 +43,7 @@ struct Event<'a> {
 }

 pub fn event_log(source: &str, event: &str, properties: Option<&HashMap<Cow<str>, Cow<str>>>) {
-    if let Some((file, start)) = unsafe { MONITOR.as_ref() } {
+    if let Some((file, start)) = MONITOR.get().as_ref() {
        let e = Event {
            timestamp: start.elapsed(),
            source,
--- a/fuzz/Cargo.lock
+++ b/fuzz/Cargo.lock
@@ -71,6 +71,12 @@ version = "1.3.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"

+[[package]]
+name = "bitflags"
+version = "2.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "327762f6e5a765692301e5bb513e0d9fef63be86bbc14528052b1cd3e6f03e07"
+
 [[package]]
 name = "block_util"
 version = "0.1.0"
@@ -117,7 +123,7 @@ version = "4.0.29"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "4d63b9e9c07271b9957ad22c173bae2a4d9a81127680962039296abcd2f8251d"
 dependencies = [
- "bitflags",
+ "bitflags 1.3.2",
 "clap_lex",
 "is-terminal",
 "once_cell",
@@ -137,7 +143,7 @@ dependencies = [

 [[package]]
 name = "cloud-hypervisor"
-version = "28.0.0"
+version = "28.3.0"
 dependencies = [
 "anyhow",
 "api_client",
@@ -240,7 +246,7 @@ dependencies = [
 "acpi_tables",
 "anyhow",
 "arch",
- "bitflags",
+ "bitflags 2.4.1",
 "byteorder",
 "hypervisor",
 "libc",
@@ -262,7 +268,7 @@ version = "4.3.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "20df693c700404f7e19d4d6fae6b15215d2913c27955d2b9d6f2c0f537511cd0"
 dependencies = [
- "bitflags",
+ "bitflags 1.3.2",
 "libc",
 ]

@@ -292,6 +298,7 @@ name = "event_monitor"
 version = "0.1.0"
 dependencies = [
 "libc",
+ "once_cell",
 "serde",
 "serde_json",
 ]
@@ -379,7 +386,7 @@ version = "0.5.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7ba34abb5175052fc1a2227a10d2275b7386c9990167de9786c0b88d8b062330"
 dependencies = [
- "bitflags",
+ "bitflags 1.3.2",
 "libc",
 ]

@@ -422,9 +429,9 @@ dependencies = [

 [[package]]
 name = "kvm-ioctls"
-version = "0.12.0"
+version = "0.13.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c3a321cabd827642499c77e27314f388dd83a717a5ca716b86476fb947f73ae4"
+checksum = "b8f8dc9c1896e5f144ec5d07169bc29f39a047686d29585a91f30489abfaeb6b"
 dependencies = [
 "kvm-bindings",
 "libc",
@@ -598,9 +605,9 @@ dependencies = [

 [[package]]
 name = "proc-macro2"
-version = "1.0.47"
+version = "1.0.76"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5ea3d908b0e36316caf9e9e2c4625cdde190a7e6f440d794667ed17a1855e725"
+checksum = "95fc56cda0b5c3325f5fbbd7ff9fda9e02bb00bb3dac51252d2f1bfa1cb8cc8c"
 dependencies = [
 "unicode-ident",
 ]
@@ -675,7 +682,7 @@ version = "0.36.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a3807b5d10909833d3e9acd1eb5fb988f79376ff10fce42937de71a449c4c588"
 dependencies = [
- "bitflags",
+ "bitflags 1.3.2",
 "errno",
 "io-lifetimes",
 "libc",
@@ -916,7 +923,7 @@ dependencies = [
 [[package]]
 name = "vfio-bindings"
 version = "0.4.0"
-source = "git+https://github.com/rust-vmm/vfio?branch=main#7ef33f6759ae6d37a54cd66f749aa47c1c81509c"
+source = "git+https://github.com/rust-vmm/vfio?branch=main#43439e056ddfa84a4f7906ee7f2f58be70505c08"
 dependencies = [
 "vmm-sys-util",
 ]
@@ -924,7 +931,7 @@ dependencies = [
 [[package]]
 name = "vfio-ioctls"
 version = "0.2.0"
-source = "git+https://github.com/rust-vmm/vfio?branch=main#7ef33f6759ae6d37a54cd66f749aa47c1c81509c"
+source = "git+https://github.com/rust-vmm/vfio?branch=main#43439e056ddfa84a4f7906ee7f2f58be70505c08"
 dependencies = [
 "byteorder",
 "kvm-bindings",
@@ -971,7 +978,7 @@ version = "0.6.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c9b791c5b0717a0558888a4cf7240cea836f39a99cb342e12ce633dcaa078072"
 dependencies = [
- "bitflags",
+ "bitflags 1.3.2",
 "libc",
 "vm-memory",
 "vmm-sys-util",
@@ -1098,7 +1105,7 @@ dependencies = [
 "anyhow",
 "arc-swap",
 "arch",
- "bitflags",
+ "bitflags 2.4.1",
 "block_util",
 "clap",
 "devices",
@@ -1143,7 +1150,7 @@ version = "0.11.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "cc06a16ee8ebf0d9269aed304030b0d20a866b8b3dd3d4ce532596ac567a0d24"
 dependencies = [
- "bitflags",
+ "bitflags 1.3.2",
 "libc",
 "serde",
 "serde_derive",
--- a/hypervisor/Cargo.toml
+++ b/hypervisor/Cargo.toml
@@ -16,7 +16,7 @@ byteorder = "1.4.3"
 thiserror = "1.0.37"
 libc = "0.2.138"
 log = "0.4.17"
-kvm-ioctls = { version = "0.12.0", optional = true }
+kvm-ioctls = { version = "0.13.0", optional = true }
 kvm-bindings = { git = "https://github.com/cloud-hypervisor/kvm-bindings", branch = "ch-v0.6.0-tdx", features = ["with-serde", "fam-wrappers"], optional  = true }
 mshv-bindings = { git = "https://github.com/rust-vmm/mshv", branch = "main", features = ["with-serde", "fam-wrappers"], optional  = true }
 mshv-ioctls = { git = "https://github.com/rust-vmm/mshv", branch = "main", optional  = true}
--- a/hypervisor/src/arch/x86/emulator/instructions/mod.rs
+++ b/hypervisor/src/arch/x86/emulator/instructions/mod.rs
@@ -18,8 +18,8 @@ fn get_op<T: CpuStateManager>(
    insn: &Instruction,
    op_index: u32,
    op_size: usize,
-    state: &mut T,
-    platform: &mut dyn PlatformEmulator<CpuState = T>,
+    state: &T,
+    platform: &dyn PlatformEmulator<CpuState = T>,
 ) -> Result<u64, PlatformError> {
    if insn.op_count() < op_index + 1 {
        return Err(PlatformError::InvalidOperand(anyhow!(
--- a/hypervisor/src/cpu.rs
+++ b/hypervisor/src/cpu.rs
@@ -237,6 +237,12 @@ pub enum HypervisorCpuError {
    ///
    #[error("Failed to initialize PMU")]
    InitializePmu,
+    #[cfg(target_arch = "x86_64")]
+    ///
+    /// Error getting TSC frequency
+    ///
+    #[error("Failed to get TSC frequency: {0}")]
+    GetTscKhz(#[source] anyhow::Error),
 }

 #[derive(Debug)]
@@ -432,4 +438,11 @@ pub trait Vcpu: Send + Sync {
    /// Return the list of initial MSR entries for a VCPU
    ///
    fn boot_msr_entries(&self) -> Vec<MsrEntry>;
+    #[cfg(target_arch = "x86_64")]
+    ///
+    /// Get the frequency of the TSC if available
+    ///
+    fn tsc_khz(&self) -> Result<Option<u32>> {
+        Ok(None)
+    }
 }
--- a/hypervisor/src/kvm/mod.rs
+++ b/hypervisor/src/kvm/mod.rs
@@ -330,7 +330,7 @@ impl KvmVm {
        Ok(VfioDeviceFd::new_from_kvm(device_fd))
    }
    /// Checks if a particular `Cap` is available.
-    fn check_extension(&self, c: Cap) -> bool {
+    pub fn check_extension(&self, c: Cap) -> bool {
        self.fd.check_extension(c)
    }
 }
@@ -2152,6 +2152,23 @@ impl cpu::Vcpu for KvmVcpu {
            .set_device_attr(&cpu_attr)
            .map_err(|_| cpu::HypervisorCpuError::InitializePmu)
    }
+
+    #[cfg(target_arch = "x86_64")]
+    ///
+    /// Get the frequency of the TSC if available
+    ///
+    fn tsc_khz(&self) -> cpu::Result<Option<u32>> {
+        match self.fd.get_tsc_khz() {
+            Err(e) => {
+                if e.errno() == libc::EIO {
+                    Ok(None)
+                } else {
+                    Err(cpu::HypervisorCpuError::GetTscKhz(e.into()))
+                }
+            }
+            Ok(v) => Ok(Some(v)),
+        }
+    }
 }

 impl KvmVcpu {
--- a/hypervisor/src/kvm/x86_64/mod.rs
+++ b/hypervisor/src/kvm/x86_64/mod.rs
@@ -50,6 +50,9 @@ pub fn check_required_kvm_extensions(kvm: &Kvm) -> KvmResult<()> {
    if !kvm.check_extension(Cap::ImmediateExit) {
        return Err(KvmError::CapabilityMissing(Cap::ImmediateExit));
    }
+    if !kvm.check_extension(Cap::GetTscKhz) {
+        return Err(KvmError::CapabilityMissing(Cap::GetTscKhz));
+    }
    Ok(())
 }
 #[derive(Clone, Serialize, Deserialize)]
--- a/hypervisor/src/mshv/mod.rs
+++ b/hypervisor/src/mshv/mod.rs
@@ -38,9 +38,7 @@ use std::fs::File;
 use std::os::unix::io::AsRawFd;

 #[cfg(target_arch = "x86_64")]
-use crate::arch::x86::{
-    CpuIdEntry, FpuState, LapicState, MsrEntry, SpecialRegisters, StandardRegisters,
-};
+use crate::arch::x86::{CpuIdEntry, FpuState, MsrEntry};

 const DIRTY_BITMAP_CLEAR_DIRTY: u64 = 0x4;
 const DIRTY_BITMAP_SET_DIRTY: u64 = 0x8;
@@ -299,7 +297,7 @@ impl cpu::Vcpu for MshvVcpu {
    ///
    /// Returns the vCPU general purpose registers.
    ///
-    fn get_regs(&self) -> cpu::Result<StandardRegisters> {
+    fn get_regs(&self) -> cpu::Result<crate::arch::x86::StandardRegisters> {
        Ok(self
            .fd
            .get_regs()
@@ -310,7 +308,7 @@ impl cpu::Vcpu for MshvVcpu {
    ///
    /// Sets the vCPU general purpose registers.
    ///
-    fn set_regs(&self, regs: &StandardRegisters) -> cpu::Result<()> {
+    fn set_regs(&self, regs: &crate::arch::x86::StandardRegisters) -> cpu::Result<()> {
        let regs = (*regs).into();
        self.fd
            .set_regs(&regs)
@@ -320,7 +318,7 @@ impl cpu::Vcpu for MshvVcpu {
    ///
    /// Returns the vCPU special registers.
    ///
-    fn get_sregs(&self) -> cpu::Result<SpecialRegisters> {
+    fn get_sregs(&self) -> cpu::Result<crate::arch::x86::SpecialRegisters> {
        Ok(self
            .fd
            .get_sregs()
@@ -331,7 +329,7 @@ impl cpu::Vcpu for MshvVcpu {
    ///
    /// Sets the vCPU special registers.
    ///
-    fn set_sregs(&self, sregs: &SpecialRegisters) -> cpu::Result<()> {
+    fn set_sregs(&self, sregs: &crate::arch::x86::SpecialRegisters) -> cpu::Result<()> {
        let sregs = (*sregs).into();
        self.fd
            .set_sregs(&sregs)
@@ -581,7 +579,7 @@ impl cpu::Vcpu for MshvVcpu {
    ///
    /// Returns the state of the LAPIC (Local Advanced Programmable Interrupt Controller).
    ///
-    fn get_lapic(&self) -> cpu::Result<LapicState> {
+    fn get_lapic(&self) -> cpu::Result<crate::arch::x86::LapicState> {
        Ok(self
            .fd
            .get_lapic()
@@ -592,7 +590,7 @@ impl cpu::Vcpu for MshvVcpu {
    ///
    /// Sets the state of the LAPIC (Local Advanced Programmable Interrupt Controller).
    ///
-    fn set_lapic(&self, lapic: &LapicState) -> cpu::Result<()> {
+    fn set_lapic(&self, lapic: &crate::arch::x86::LapicState) -> cpu::Result<()> {
        let lapic: mshv_bindings::LapicState = (*lapic).clone().into();
        self.fd
            .set_lapic(&lapic)
--- a/net_gen/src/lib.rs
+++ b/net_gen/src/lib.rs
@@ -28,10 +28,12 @@ pub mod inn;
 // generated with bindgen /usr/include/linux/sockios.h --no-unstable-rust
 // --constified-enum '*' --with-derive-default
 pub mod sockios;
-pub use if_tun::*;
-pub use iff::*;
-pub use inn::*;
-pub use sockios::*;
+pub use if_tun::{
+    sock_fprog, IFF_MULTI_QUEUE, IFF_NO_PI, IFF_TAP, IFF_VNET_HDR, TUN_F_CSUM, TUN_F_TSO4,
+    TUN_F_TSO6, TUN_F_TSO_ECN, TUN_F_UFO,
+};
+pub use iff::{ifreq, net_device_flags_IFF_UP, setsockopt, sockaddr, AF_INET};
+pub use inn::sockaddr_in;

 pub const TUNTAP: ::std::os::raw::c_uint = 84;

--- a/net_util/Cargo.toml
+++ b/net_util/Cargo.toml
@@ -23,6 +23,6 @@ vmm-sys-util = "0.11.0"

 [dev-dependencies]
 once_cell = "1.16.0"
-pnet = "0.31.0"
-pnet_datalink = "0.31.0"
+pnet = "0.34.0"
+pnet_datalink = "0.34.0"
 serde_json = "1.0.89"
--- a/net_util/src/queue_pair.rs
+++ b/net_util/src/queue_pair.rs
@@ -86,7 +86,7 @@ impl TxVirtio {
                let result = unsafe {
                    libc::writev(
                        tap.as_raw_fd() as libc::c_int,
-                        iovecs.as_ptr() as *const libc::iovec,
+                        iovecs.as_ptr(),
                        iovecs.len() as libc::c_int,
                    )
                };
@@ -224,7 +224,7 @@ impl RxVirtio {
                let result = unsafe {
                    libc::readv(
                        tap.as_raw_fd() as libc::c_int,
-                        iovecs.as_ptr() as *const libc::iovec,
+                        iovecs.as_ptr(),
                        iovecs.len() as libc::c_int,
                    )
                };
--- a/pci/src/vfio.rs
+++ b/pci/src/vfio.rs
@@ -12,6 +12,7 @@ use crate::{
 use anyhow::anyhow;
 use byteorder::{ByteOrder, LittleEndian};
 use hypervisor::HypervisorVmError;
+use libc::{sysconf, _SC_PAGESIZE};
 use std::any::Any;
 use std::collections::{BTreeMap, HashMap};
 use std::io;
@@ -571,7 +572,12 @@ impl VfioCommon {
                PciBarRegionType::Memory64BitRegion => {
                    // BAR allocation must be naturally aligned
                    mmio_allocator
-                        .allocate(restored_bar_addr, region_size, Some(region_size))
+                        .allocate(
+                            restored_bar_addr,
+                            region_size,
+                            // SAFETY: FFI call. Trivially safe.
+                            Some(unsafe { sysconf(_SC_PAGESIZE) as GuestUsize }),
+                        )
                        .ok_or(PciDeviceError::IoAllocationFailed(region_size))?
                }
            };
@@ -1278,8 +1284,10 @@ impl VfioPciDevice {
        self.iommu_attached
    }

-    fn align_4k(address: u64) -> u64 {
-        (address + 0xfff) & 0xffff_ffff_ffff_f000
+    fn align_page_size(address: u64) -> u64 {
+        // SAFETY: FFI call. Trivially safe.
+        let page_size = unsafe { sysconf(_SC_PAGESIZE) as u64 };
+        (address + page_size - 1) & !(page_size - 1)
    }

    fn is_4k_aligned(address: u64) -> bool {
@@ -1339,6 +1347,7 @@ impl VfioPciDevice {
                    let mut sparse_areas = Vec::new();
                    let mut current_offset = 0;
                    for (range_offset, range_size) in inter_ranges {
+                        let range_offset = Self::align_page_size(range_offset);
                        if range_offset > current_offset {
                            sparse_areas.push(VfioRegionSparseMmapArea {
                                offset: current_offset,
@@ -1346,13 +1355,13 @@ impl VfioPciDevice {
                            });
                        }

-                        current_offset = Self::align_4k(range_offset + range_size);
+                        current_offset = Self::align_page_size(range_offset + range_size);
                    }

                    if region_size > current_offset {
                        sparse_areas.push(VfioRegionSparseMmapArea {
-                            offset: current_offset,
-                            size: region_size - current_offset,
+                            offset: Self::align_page_size(current_offset),
+                            size: Self::align_page_size(region_size - current_offset),
                        });
                    }

--- a/performance-metrics/src/performance_tests.rs
+++ b/performance-metrics/src/performance_tests.rs
@@ -168,7 +168,6 @@ fn parse_boot_time_output(output: &[u8]) -> Result<f64, Error> {
    std::panic::catch_unwind(|| {
        let l: Vec<String> = String::from_utf8_lossy(output)
            .lines()
-            .into_iter()
            .filter(|l| l.contains("Debug I/O port: Kernel code"))
            .map(|l| l.to_string())
            .collect();
--- a/qcow/src/qcow_raw_file.rs
+++ b/qcow/src/qcow_raw_file.rs
@@ -66,7 +66,7 @@ impl QcowRawFile {
        non_zero_flags: u64,
    ) -> io::Result<()> {
        self.file.seek(SeekFrom::Start(offset))?;
-        let mut buffer = BufWriter::with_capacity(table.len() * size_of::<u64>(), &mut self.file);
+        let mut buffer = BufWriter::with_capacity(std::mem::size_of_val(table), &mut self.file);
        for addr in table {
            let val = if *addr == 0 {
                0
@@ -91,7 +91,7 @@ impl QcowRawFile {
    /// Writes a refcount block to the file.
    pub fn write_refcount_block(&mut self, offset: u64, table: &[u16]) -> io::Result<()> {
        self.file.seek(SeekFrom::Start(offset))?;
-        let mut buffer = BufWriter::with_capacity(table.len() * size_of::<u16>(), &mut self.file);
+        let mut buffer = BufWriter::with_capacity(std::mem::size_of_val(table), &mut self.file);
        for count in table {
            buffer.write_u16::<BigEndian>(*count)?;
        }
--- a/release-notes.md
+++ b/release-notes.md
@@ -1,3 +1,5 @@
+- [v28.4](#v284)
+- [v28.3](#v283)
 - [v28.2](#v282)
 - [v28.1](#v281)
 - [v28.0](#v280)
@@ -249,6 +251,37 @@
    - [Unit testing](#unit-testing)
    - [Integration tests parallelization](#integration-tests-parallelization)

+
+# v28.4
+
+This is a bug fix release. The following issues have been addressed:
+
+* Fix VFIO on platforms with non-4k page size (#5450)
+* Fix seccomp filter lists related to virtio-console, serial and pty
+(#5506, #5524)
+* Populate APIC ID properly (#5512)
+* Ignore and warn TAP FDs in more situations (#5522)
+* Disallow concurrent CPU resizing (#5668)
+* Use wrapping add for memory offset from instruction emulator (#5719)
+* Replace unsound `static mut` with `once_cell` (#5772)
+* Fix a deadlock when TDX is enabled (#5845)
+* Bug fix to OpenAPI specification file (#5967)
+* Error out early for live migration when TDX is enabled (#6025)
+
+# v28.3
+
+This is a bug fix release. The following issues have been addressed:
+
+* Ignore and warn TAP FDs sent via the HTTP request body (#5350)
+* Properly preserve and close valid FDs for TAP devices (#5373)
+* Only use `KVM_ARM_VCPU_PMU_V3` if available (#5360)
+* Fix seccomp filter lists for vhost-user devices (#5361)
+* Fix the offset setting while removing the entire mapping of
+  `vhost-user` FS client (#5235)
+* Set vcpu thread status properly and signal exit_evt upon thread exit (#5211)
+* Populate CPUID leaf 0x4000_0010 (TSC frequency) (#5179)
+* Bug fix to OpenAPI specification file (#5186)
+
 # v28.2
 This is a bug fix release. The following issues have been addressed:

--- a/resources/Dockerfile
+++ b/resources/Dockerfile
@@ -1,10 +1,10 @@
 # When changing this file don't forget to update the tag name in the
 # .github/workflows/docker-image.yaml file if doing multiple per day

-FROM ubuntu:20.04 as dev
+FROM ubuntu:22.04 as dev

 ARG TARGETARCH
-ARG RUST_TOOLCHAIN="1.62.0"
+ARG RUST_TOOLCHAIN="1.65.0"
 ARG CLH_SRC_DIR="/cloud-hypervisor"
 ARG CLH_BUILD_DIR="$CLH_SRC_DIR/build"
 ARG CARGO_REGISTRY_DIR="$CLH_BUILD_DIR/cargo_registry"
@@ -16,40 +16,43 @@ ENV PATH="$PATH:$CARGO_HOME/bin"

 # Install all CI dependencies
 RUN apt-get update \
-	&& apt-get -yq upgrade \
-	&& DEBIAN_FRONTEND=noninteractive apt-get install -yq \
-	build-essential \
-	bc \
-	curl \
-	wget \
-	sudo \
-	mtools \
-	musl-tools \
-	libssl-dev \
-	pkg-config \
-	flex \
-	bison \
-	libelf-dev \
-	qemu-utils \
-	libglib2.0-dev \
-	libpixman-1-dev \
-	libseccomp-dev \
-	libcap-ng-dev \
-	socat \
-	dosfstools \
-	cpio \
-	python \
-	python3 \
-	python3-setuptools \
-	ntfs-3g \
-	openvswitch-switch-dpdk \
-	python3-distutils \
-	uuid-dev \
-	iperf3 \
-	zip \
-	git-core \
-	dnsmasq \
-	dmsetup \
+        && apt-get -yq upgrade \
+        && apt-get install --no-install-recommends -yq \
+        build-essential \
+        bc \
+        curl \
+        wget \
+        sudo \
+        mtools \
+        musl-tools \
+        libssl-dev \
+        pkg-config \
+        flex \
+        bison \
+        libelf-dev \
+        qemu-utils \
+        libglib2.0-dev \
+        libpixman-1-dev \
+        libseccomp-dev \
+        libcap-ng-dev \
+        socat \
+        dosfstools \
+        cpio \
+        python3 \
+        python3-setuptools \
+        ntfs-3g \
+        python3-distutils \
+        uuid-dev \
+        iperf3 \
+        zip \
+        git-core \
+        dnsmasq \
+        dmsetup \
+        ca-certificates \
+        unzip \
+        iproute2 \
+        dbus \
+    && apt-get install openvswitch-switch-dpdk -yq \
    && apt-get clean \
    && rm -rf /var/lib/apt/lists/*

@@ -112,17 +115,18 @@ RUN echo 'source $CARGO_HOME/env' >> $HOME/.bashrc \
 RUN if [ "$TARGETARCH" = "amd64" ]; then \
       git clone https://github.com/spdk/spdk \
       && cd spdk \
-       && git checkout 6301f8915de32baed10dba1eebed556a6749211a \
+       && git checkout ef8bcce58f3f02b79c0619a297e4f17e81e62b24 \
       && git submodule update --init \
       && apt-get update \
       && ./scripts/pkgdep.sh \
       && apt-get clean \
       && ./configure --with-vfio-user \
-       && make -j `nproc` \
+       && make -j $(nproc) TARGET_ARCHITECTURE=skylake \
+       && rm -rf /usr/local/bin/spdk-nvme \
       && mkdir /usr/local/bin/spdk-nvme \
-       && cp ./build/bin/nvmf_tgt /usr/local/bin/spdk-nvme \
-       && cp ./scripts/rpc.py /usr/local/bin/spdk-nvme \
-       && cp -r ./scripts/rpc /usr/local/bin/spdk-nvme \
+       && cp -f ./build/bin/nvmf_tgt /usr/local/bin/spdk-nvme \
+       && cp -f ./scripts/rpc.py /usr/local/bin/spdk-nvme \
+       && cp -rf ./python /usr/local/bin \
       && cd .. && rm -rf spdk; fi

 # install ethr tool for performance tests
--- a/scripts/common-aarch64.sh
+++ b/scripts/common-aarch64.sh
@@ -1,7 +1,5 @@
 #!/bin/bash

-export BUILD_TARGET=${BUILD_TARGET-aarch64-unknown-linux-gnu}
-
 WORKLOADS_DIR="$HOME/workloads"

 mkdir -p "$WORKLOADS_DIR"
--- a/scripts/dev_cli.sh
+++ b/scripts/dev_cli.sh
@@ -6,9 +6,9 @@

 CLI_NAME="Cloud Hypervisor"

-CTR_IMAGE_TAG="cloudhypervisor/dev"
-CTR_IMAGE_VERSION="20220705-0"
-CTR_IMAGE="${CTR_IMAGE_TAG}:${CTR_IMAGE_VERSION}"
+CTR_IMAGE_TAG="ghcr.io/cloud-hypervisor/cloud-hypervisor"
+CTR_IMAGE_VERSION="20231220-0"
+: "${CTR_IMAGE:=${CTR_IMAGE_TAG}:${CTR_IMAGE_VERSION}}"

 DOCKER_RUNTIME="docker"

@@ -277,11 +277,10 @@ cmd_build() {
    [ $build = "release" ] && cargo_args+=("--release")
    cargo_args+=(--target "$target")

-    rustflags=""
+    rustflags="$RUSTFLAGS"
    target_cc=""
    if [ "$(uname -m)" = "aarch64" ] && [ "$libc" = "musl" ]; then
-        rustflags="-C link-arg=-lgcc -C link_arg=-specs -C link_arg=/usr/lib/aarch64-linux-musl/musl-gcc.specs"
-        target_cc="musl-gcc"
+        rustflags="$rustflags -C link-args=-Wl,-Bstatic -C link-args=-lc"
    fi

    $DOCKER_RUNTIME run \
@@ -387,6 +386,12 @@ cmd_tests() {
    process_volumes_args
    target="$(uname -m)-unknown-linux-${libc}"

+    rustflags="$RUSTFLAGS"
+    target_cc=""
+    if [ "$(uname -m)" = "aarch64" ] && [ "$libc" = "musl" ]; then
+        rustflags="$rustflags -C link-args=-Wl,-Bstatic -C link-args=-lc"
+    fi
+
    if [[ "$unit" = true ]]; then
        say "Running unit tests for $target..."
        $DOCKER_RUNTIME run \
@@ -397,6 +402,8 @@ cmd_tests() {
            --cap-add net_admin \
            --volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR" $exported_volumes \
            --env BUILD_TARGET="$target" \
+            --env RUSTFLAGS="$rustflags" \
+            --env TARGET_CC="$target_cc" \
            "$CTR_IMAGE" \
            ./scripts/run_unit_tests.sh "$@" || fix_dir_perms $? || exit $?
    fi
@@ -415,7 +422,9 @@ cmd_tests() {
            --volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR" $exported_volumes \
            --volume "$CLH_INTEGRATION_WORKLOADS:$CTR_CLH_INTEGRATION_WORKLOADS" \
            --env USER="root" \
-            --env CH_LIBC="${libc}" \
+            --env BUILD_TARGET="$target" \
+            --env RUSTFLAGS="$rustflags" \
+            --env TARGET_CC="$target_cc" \
            "$CTR_IMAGE" \
            ./scripts/run_integration_tests_"$(uname -m)".sh "$@" || fix_dir_perms $? || exit $?
    fi
@@ -434,7 +443,9 @@ cmd_tests() {
            --volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR" $exported_volumes \
            --volume "$CLH_INTEGRATION_WORKLOADS:$CTR_CLH_INTEGRATION_WORKLOADS" \
            --env USER="root" \
-            --env CH_LIBC="${libc}" \
+            --env BUILD_TARGET="$target" \
+            --env RUSTFLAGS="$rustflags" \
+            --env TARGET_CC="$target_cc" \
            "$CTR_IMAGE" \
            ./scripts/run_integration_tests_sgx.sh "$@" || fix_dir_perms $? || exit $?
    fi
@@ -453,7 +464,9 @@ cmd_tests() {
            --volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR" $exported_volumes \
            --volume "$CLH_INTEGRATION_WORKLOADS:$CTR_CLH_INTEGRATION_WORKLOADS" \
            --env USER="root" \
-            --env CH_LIBC="${libc}" \
+            --env BUILD_TARGET="$target" \
+            --env RUSTFLAGS="$rustflags" \
+            --env TARGET_CC="$target_cc" \
            "$CTR_IMAGE" \
            ./scripts/run_integration_tests_vfio.sh "$@" || fix_dir_perms $? || exit $?
    fi
@@ -472,7 +485,9 @@ cmd_tests() {
            --volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR" $exported_volumes \
            --volume "$CLH_INTEGRATION_WORKLOADS:$CTR_CLH_INTEGRATION_WORKLOADS" \
            --env USER="root" \
-            --env CH_LIBC="${libc}" \
+            --env BUILD_TARGET="$target" \
+            --env RUSTFLAGS="$rustflags" \
+            --env TARGET_CC="$target_cc" \
            "$CTR_IMAGE" \
            ./scripts/run_integration_tests_windows_"$(uname -m)".sh "$@" || fix_dir_perms $? || exit $?
    fi
@@ -491,7 +506,9 @@ cmd_tests() {
            --volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR" $exported_volumes \
            --volume "$CLH_INTEGRATION_WORKLOADS:$CTR_CLH_INTEGRATION_WORKLOADS" \
            --env USER="root" \
-            --env CH_LIBC="${libc}" \
+            --env BUILD_TARGET="$target" \
+            --env RUSTFLAGS="$rustflags" \
+            --env TARGET_CC="$target_cc" \
            "$CTR_IMAGE" \
            ./scripts/run_integration_tests_live_migration.sh "$@" || fix_dir_perms $? || exit $?
    fi
@@ -510,7 +527,9 @@ cmd_tests() {
            --volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR" $exported_volumes \
            --volume "$CLH_INTEGRATION_WORKLOADS:$CTR_CLH_INTEGRATION_WORKLOADS" \
            --env USER="root" \
-            --env CH_LIBC="${libc}" \
+            --env BUILD_TARGET="$target" \
+            --env RUSTFLAGS="$rustflags" \
+            --env TARGET_CC="$target_cc" \
            "$CTR_IMAGE" \
            ./scripts/run_integration_tests_rate_limiter.sh "$@" || fix_dir_perms $? || exit $?
    fi
@@ -529,7 +548,10 @@ cmd_tests() {
            --volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR" $exported_volumes \
            --volume "$CLH_INTEGRATION_WORKLOADS:$CTR_CLH_INTEGRATION_WORKLOADS" \
            --env USER="root" \
-            --env CH_LIBC="${libc}" \
+            --env BUILD_TARGET="$target" \
+            --env RUSTFLAGS="$rustflags" \
+            --env TARGET_CC="$target_cc" \
+            --env RUST_BACKTRACE="${RUST_BACKTRACE}" \
            "$CTR_IMAGE" \
            ./scripts/run_metrics.sh "$@" || fix_dir_perms $? || exit $?
    fi
--- a/scripts/run_integration_tests_aarch64.sh
+++ b/scripts/run_integration_tests_aarch64.sh
@@ -11,15 +11,16 @@ build_spdk_nvme() {
    SPDK_DIR="$WORKLOADS_DIR/spdk"
    SPDK_REPO="https://github.com/spdk/spdk.git"
    SPDK_DEPLOY_DIR="/usr/local/bin/spdk-nvme"
-    checkout_repo "$SPDK_DIR" "$SPDK_REPO" master "6301f8915de32baed10dba1eebed556a6749211a"
+    checkout_repo "$SPDK_DIR" "$SPDK_REPO" master "ef8bcce58f3f02b79c0619a297e4f17e81e62b24"

    if [ ! -f "$SPDK_DIR/.built" ]; then
        pushd $SPDK_DIR
        git submodule update --init
        apt-get update
+        sed -i "/grpcio/d" scripts/pkgdep/debian.sh
        ./scripts/pkgdep.sh
        ./configure --with-vfio-user
-        chmod +x /usr/local/lib/python3.8/dist-packages/ninja/data/bin/ninja
+        chmod +x /usr/local/lib/python3.10/dist-packages/ninja/data/bin/ninja
        make -j `nproc` || exit 1
        touch .built
        popd
@@ -29,14 +30,15 @@ build_spdk_nvme() {
    fi
    cp "$WORKLOADS_DIR/spdk/build/bin/nvmf_tgt" $SPDK_DEPLOY_DIR/nvmf_tgt
    cp "$WORKLOADS_DIR/spdk/scripts/rpc.py" $SPDK_DEPLOY_DIR/rpc.py
-    cp -r "$WORKLOADS_DIR/spdk/scripts/rpc" $SPDK_DEPLOY_DIR/rpc
+    cp -r "$WORKLOADS_DIR/spdk/python/spdk/" $SPDK_DEPLOY_DIR/
+    cp -r "$WORKLOADS_DIR/spdk/python" $SPDK_DEPLOY_DIR/../
 }

 build_virtiofsd() {
    VIRTIOFSD_DIR="$WORKLOADS_DIR/virtiofsd_build"
    VIRTIOFSD_REPO="https://gitlab.com/virtio-fs/virtiofsd.git"

-    checkout_repo "$VIRTIOFSD_DIR" "$VIRTIOFSD_REPO" v1.1.0 "220405d7a2606c92636d31992b5cb3036a41047b"
+    checkout_repo "$VIRTIOFSD_DIR" "$VIRTIOFSD_REPO" v1.8.0 "97ea7908fe7f9bc59916671a771bdcfaf4044b45"

    if [ ! -f "$VIRTIOFSD_DIR/.built" ]; then
        pushd $VIRTIOFSD_DIR
@@ -152,8 +154,21 @@ update_workloads() {
    CH_RELEASE_URL="https://github.com/cloud-hypervisor/cloud-hypervisor/releases/download/$LAST_RELEASE_VERSION/cloud-hypervisor-static-aarch64"
    CH_RELEASE_NAME="cloud-hypervisor-static-aarch64"
    pushd $WORKLOADS_DIR
-    time wget --quiet $CH_RELEASE_URL -O "$CH_RELEASE_NAME" || exit 1
-    chmod +x $CH_RELEASE_NAME
+    # Repeat a few times to workaround a random wget failure
+    WGET_RETRY_MAX=10
+    wget_retry=0
+
+    until [ "$wget_retry" -ge "$WGET_RETRY_MAX" ]
+    do
+        time wget $CH_RELEASE_URL -O "$CH_RELEASE_NAME" && break
+        wget_retry=$((wget_retry+1))
+    done
+
+    if [ $wget_retry -ge "$WGET_RETRY_MAX" ]; then
+        exit 1
+    else
+        chmod +x $CH_RELEASE_NAME
+    fi
    popd

    # Build custom kernel for guest VMs
@@ -225,12 +240,6 @@ if [ $RES -ne 0 ]; then
    exit 1
 fi

-BUILD_TARGET="aarch64-unknown-linux-${CH_LIBC}"
-if [[ "${BUILD_TARGET}" == "aarch64-unknown-linux-musl" ]]; then
-export TARGET_CC="musl-gcc"
-export RUSTFLAGS="-C link-arg=-lgcc -C link_arg=-specs -C link_arg=/usr/lib/aarch64-linux-musl/musl-gcc.specs"
-fi
-
 export RUST_BACKTRACE=1

 # Test without ACPI
--- a/scripts/run_integration_tests_live_migration.sh
+++ b/scripts/run_integration_tests_live_migration.sh
@@ -4,8 +4,6 @@ set -x
 source $HOME/.cargo/env
 source $(dirname "$0")/test-util.sh

-export BUILD_TARGET=${BUILD_TARGET-x86_64-unknown-linux-gnu}
-
 WORKLOADS_DIR="$HOME/workloads"
 mkdir -p "$WORKLOADS_DIR"

@@ -78,11 +76,8 @@ if [ -d "$LINUX_CUSTOM_DIR" ]; then
    rm -rf $LINUX_CUSTOM_DIR
 fi

-BUILD_TARGET="$(uname -m)-unknown-linux-${CH_LIBC}"
 CFLAGS=""
-TARGET_CC=""
 if [[ "${BUILD_TARGET}" == "x86_64-unknown-linux-musl" ]]; then
-    TARGET_CC="musl-gcc"
    CFLAGS="-I /usr/include/x86_64-linux-musl/ -idirafter /usr/include/"
 fi

--- a/scripts/run_integration_tests_rate_limiter.sh
+++ b/scripts/run_integration_tests_rate_limiter.sh
@@ -4,8 +4,6 @@ set -x
 source $HOME/.cargo/env
 source $(dirname "$0")/test-util.sh

-export BUILD_TARGET=${BUILD_TARGET-x86_64-unknown-linux-gnu}
-
 WORKLOADS_DIR="$HOME/workloads"
 mkdir -p "$WORKLOADS_DIR"

@@ -69,11 +67,8 @@ if [ -d "$LINUX_CUSTOM_DIR" ]; then
    rm -rf $LINUX_CUSTOM_DIR
 fi

-BUILD_TARGET="$(uname -m)-unknown-linux-${CH_LIBC}"
 CFLAGS=""
-TARGET_CC=""
 if [[ "${BUILD_TARGET}" == "x86_64-unknown-linux-musl" ]]; then
-    TARGET_CC="musl-gcc"
    CFLAGS="-I /usr/include/x86_64-linux-musl/ -idirafter /usr/include/"
 fi

--- a/scripts/run_integration_tests_sgx.sh
+++ b/scripts/run_integration_tests_sgx.sh
@@ -14,12 +14,9 @@ fi
 # For now these values are default for kvm
 features=""

-BUILD_TARGET="$(uname -m)-unknown-linux-${CH_LIBC}"
 CFLAGS=""
-TARGET_CC=""
 if [[ "${BUILD_TARGET}" == "x86_64-unknown-linux-musl" ]]; then
-TARGET_CC="musl-gcc"
-CFLAGS="-I /usr/include/x86_64-linux-musl/ -idirafter /usr/include/"
+    CFLAGS="-I /usr/include/x86_64-linux-musl/ -idirafter /usr/include/"
 fi

 cargo build --all --release $features --target $BUILD_TARGET
--- a/scripts/run_integration_tests_vfio.sh
+++ b/scripts/run_integration_tests_vfio.sh
@@ -35,12 +35,9 @@ cp $FOCAL_OS_IMAGE $VFIO_DIR
 cp $FW $VFIO_DIR
 cp $VMLINUX_IMAGE $VFIO_DIR || exit 1

-BUILD_TARGET="$(uname -m)-unknown-linux-${CH_LIBC}"
 CFLAGS=""
-TARGET_CC=""
 if [[ "${BUILD_TARGET}" == "x86_64-unknown-linux-musl" ]]; then
-TARGET_CC="musl-gcc"
-CFLAGS="-I /usr/include/x86_64-linux-musl/ -idirafter /usr/include/"
+    CFLAGS="-I /usr/include/x86_64-linux-musl/ -idirafter /usr/include/"
 fi

 cargo build --all --release $features --target $BUILD_TARGET
--- a/scripts/run_integration_tests_windows_aarch64.sh
+++ b/scripts/run_integration_tests_windows_aarch64.sh
@@ -22,14 +22,6 @@ WIN_IMAGE_FILE="$WORKLOADS_DIR/$WIN_IMAGE_BASENAME"
 OVMF_FW="$WORKLOADS_DIR/CLOUDHV_EFI.fd"
 build_edk2

-BUILD_TARGET="$(uname -m)-unknown-linux-${CH_LIBC}"
-CFLAGS=""
-TARGET_CC=""
-if [[ "${BUILD_TARGET}" == "aarch64-unknown-linux-musl" ]]; then
-export TARGET_CC="musl-gcc"
-export RUSTFLAGS="-C link-arg=-lgcc -C link_arg=-specs -C link_arg=/usr/lib/aarch64-linux-musl/musl-gcc.specs"
-fi
-
 # Check if the images are present
 if [[ ! -f ${WIN_IMAGE_FILE} || ! -f ${OVMF_FW} ]]; then
    echo "Windows image/firmware not present in the host"
--- a/scripts/run_integration_tests_windows_x86_64.sh
+++ b/scripts/run_integration_tests_windows_x86_64.sh
@@ -22,12 +22,9 @@ if [ ! -f "$OVMF_FW" ]; then
    popd
 fi

-BUILD_TARGET="$(uname -m)-unknown-linux-${CH_LIBC}"
 CFLAGS=""
-TARGET_CC=""
 if [[ "${BUILD_TARGET}" == "x86_64-unknown-linux-musl" ]]; then
-TARGET_CC="musl-gcc"
-CFLAGS="-I /usr/include/x86_64-linux-musl/ -idirafter /usr/include/"
+    CFLAGS="-I /usr/include/x86_64-linux-musl/ -idirafter /usr/include/"
 fi

 # Check if the images are present
--- a/scripts/run_integration_tests_x86_64.sh
+++ b/scripts/run_integration_tests_x86_64.sh
@@ -4,8 +4,6 @@ set -x
 source $HOME/.cargo/env
 source $(dirname "$0")/test-util.sh

-export BUILD_TARGET=${BUILD_TARGET-x86_64-unknown-linux-gnu}
-
 WORKLOADS_DIR="$HOME/workloads"
 mkdir -p "$WORKLOADS_DIR"

@@ -134,7 +132,7 @@ if [ ! -f "$VIRTIOFSD" ]; then
    pushd $WORKLOADS_DIR
    git clone "https://gitlab.com/virtio-fs/virtiofsd.git" $VIRTIOFSD_DIR
    pushd $VIRTIOFSD_DIR
-    git checkout v1.1.0
+    git checkout v1.8.0
    time cargo build --release
    cp target/release/virtiofsd $VIRTIOFSD || exit 1
    popd
@@ -172,8 +170,6 @@ cp $FOCAL_OS_RAW_IMAGE $VFIO_DIR
 cp $FW $VFIO_DIR
 cp $VMLINUX_IMAGE $VFIO_DIR || exit 1

-BUILD_TARGET="$(uname -m)-unknown-linux-${CH_LIBC}"
-
 cargo build --all  --release $features --target $BUILD_TARGET
 strip target/$BUILD_TARGET/release/cloud-hypervisor
 strip target/$BUILD_TARGET/release/vhost_user_net
--- a/scripts/run_metrics.sh
+++ b/scripts/run_metrics.sh
@@ -5,8 +5,6 @@ source $HOME/.cargo/env
 source $(dirname "$0")/test-util.sh

 export TEST_ARCH=$(uname -m)
-export BUILD_TARGET=${BUILD_TARGET-${TEST_ARCH}-unknown-linux-gnu}
-

 WORKLOADS_DIR="$HOME/workloads"
 mkdir -p "$WORKLOADS_DIR"
@@ -91,11 +89,8 @@ fi
 # Build custom kernel based on virtio-pmem and virtio-fs upstream patches
 build_custom_linux

-BUILD_TARGET="${TEST_ARCH}-unknown-linux-${CH_LIBC}"
 CFLAGS=""
-TARGET_CC=""
 if [[ "${BUILD_TARGET}" == "${TEST_ARCH}-unknown-linux-musl" ]]; then
-    TARGET_CC="musl-gcc"
    CFLAGS="-I /usr/include/${TEST_ARCH}-linux-musl/ -idirafter /usr/include/"
 fi

--- a/scripts/run_unit_tests.sh
+++ b/scripts/run_unit_tests.sh
@@ -5,7 +5,6 @@ source $(dirname "$0")/test-util.sh

 process_common_args "$@"

-BUILD_TARGET=${BUILD_TARGET-x86_64-unknown-linux-gnu}
 cargo_args=("")

 if [[ $hypervisor = "mshv" ]]; then
@@ -15,10 +14,5 @@ elif [[ $(uname -m) = "x86_64" ]]; then
    cargo_args+=("--features tdx")
 fi

-if [[ "${BUILD_TARGET}" == "aarch64-unknown-linux-musl" ]]; then
-    export TARGET_CC="musl-gcc"
-    export RUSTFLAGS="-C link-arg=-lgcc -C link_arg=-specs -C link_arg=/usr/lib/aarch64-linux-musl/musl-gcc.specs"
-fi
-
 export RUST_BACKTRACE=1
 cargo test --lib --bins --target $BUILD_TARGET --workspace ${cargo_args[@]} || exit 1
--- a/src/main.rs
+++ b/src/main.rs
@@ -722,6 +722,7 @@ mod unit_tests {
            gdb: false,
            platform: None,
            tpm: None,
+            preserved_fds: None,
        };

        assert_eq!(expected_vm_config, result_vm_config);
@@ -729,7 +730,7 @@ mod unit_tests {

    #[test]
    fn test_valid_vm_config_cpus() {
-        vec![
+        [
            (
                vec![
                    "cloud-hypervisor",
@@ -847,7 +848,7 @@ mod unit_tests {

    #[test]
    fn test_valid_vm_config_kernel() {
-        vec![(
+        [(
            vec!["cloud-hypervisor", "--kernel", "/path/to/kernel"],
            r#"{
                "payload": {"kernel": "/path/to/kernel"}
@@ -862,7 +863,7 @@ mod unit_tests {

    #[test]
    fn test_valid_vm_config_cmdline() {
-        vec![(
+        [(
            vec![
                "cloud-hypervisor",
                "--kernel",
@@ -883,7 +884,7 @@ mod unit_tests {

    #[test]
    fn test_valid_vm_config_disks() {
-        vec![
+        [
            (
                vec![
                    "cloud-hypervisor",
@@ -1155,7 +1156,7 @@ mod unit_tests {

    #[test]
    fn test_valid_vm_config_rng() {
-        vec![(
+        [(
            vec![
                "cloud-hypervisor",
                "--kernel",
@@ -1177,8 +1178,7 @@ mod unit_tests {

    #[test]
    fn test_valid_vm_config_fs() {
-        vec![
-            (
+        [(
                vec![
                    "cloud-hypervisor", "--kernel", "/path/to/kernel",
                    "--memory", "shared=true",
@@ -1246,8 +1246,7 @@ mod unit_tests {
                    ]
                }"#,
                true,
-            ),
-        ]
+            )]
        .iter()
        .for_each(|(cli, openapi, equal)| {
            compare_vm_config_cli_vs_json(cli, openapi, *equal);
@@ -1256,7 +1255,7 @@ mod unit_tests {

    #[test]
    fn test_valid_vm_config_pmem() {
-        vec![
+        [
            (
                vec![
                    "cloud-hypervisor",
@@ -1319,7 +1318,7 @@ mod unit_tests {

    #[test]
    fn test_valid_vm_config_serial_console() {
-        vec![
+        [
            (
                vec!["cloud-hypervisor", "--kernel", "/path/to/kernel"],
                r#"{
@@ -1370,7 +1369,7 @@ mod unit_tests {

    #[test]
    fn test_valid_vm_config_serial_pty_console_pty() {
-        vec![
+        [
            (
                vec!["cloud-hypervisor", "--kernel", "/path/to/kernel"],
                r#"{
@@ -1516,7 +1515,7 @@ mod unit_tests {

    #[test]
    fn test_valid_vm_config_vdpa() {
-        vec![
+        [
            (
                vec![
                    "cloud-hypervisor",
@@ -1561,7 +1560,7 @@ mod unit_tests {

    #[test]
    fn test_valid_vm_config_vsock() {
-        vec![
+        [
            (
                vec![
                    "cloud-hypervisor",
@@ -1644,7 +1643,7 @@ mod unit_tests {

    #[test]
    fn test_valid_vm_config_tpm_socket() {
-        vec![(
+        [(
            vec![
                "cloud-hypervisor",
                "--kernel",
--- a/test_infra/src/lib.rs
+++ b/test_infra/src/lib.rs
@@ -88,7 +88,7 @@ impl GuestNetworkConfig {
            None => DEFAULT_TCP_LISTENER_TIMEOUT,
        };

-        match (|| -> Result<(), WaitForBootError> {
+        let mut closure = || -> Result<(), WaitForBootError> {
            let listener =
                TcpListener::bind(listen_addr.as_str()).map_err(WaitForBootError::Listen)?;
            listener
@@ -106,7 +106,7 @@ impl GuestNetworkConfig {
                epoll::Event::new(epoll::Events::EPOLLIN, 0),
            )
            .expect("Cannot add 'tcp_listener' event to epoll");
-            let mut events = vec![epoll::Event::new(epoll::Events::empty(), 0); 1];
+            let mut events = [epoll::Event::new(epoll::Events::empty(), 0); 1];
            loop {
                let num_events = match epoll::wait(epoll_fd, timeout * 1000_i32, &mut events[..]) {
                    Ok(num_events) => Ok(num_events),
@@ -141,7 +141,9 @@ impl GuestNetworkConfig {
                    Err(WaitForBootError::Accept(e))
                }
            }
-        })() {
+        };
+
+        match closure() {
            Err(e) => {
                let duration = start.elapsed();
                eprintln!(
@@ -249,7 +251,7 @@ impl DiskConfig for UbuntuDiskConfig {
            .join("cloud-init")
            .join("ubuntu");

-        vec!["meta-data"].iter().for_each(|x| {
+        ["meta-data"].iter().for_each(|x| {
            rate_limited_copy(source_file_dir.join(x), cloud_init_directory.join(x))
                .expect("Expect copying cloud-init meta-data to succeed");
        });
@@ -305,7 +307,7 @@ impl DiskConfig for UbuntuDiskConfig {
            .output()
            .expect("Expect creating disk image to succeed");

-        vec!["user-data", "meta-data", "network-config"]
+        ["user-data", "meta-data", "network-config"]
            .iter()
            .for_each(|x| {
                std::process::Command::new("mcopy")
@@ -556,7 +558,7 @@ fn scp_to_guest_with_auth(
 ) -> Result<(), SshCommandError> {
    let mut counter = 0;
    loop {
-        match (|| -> Result<(), SshCommandError> {
+        let closure = || -> Result<(), SshCommandError> {
            let tcp =
                TcpStream::connect(format!("{ip}:22")).map_err(SshCommandError::Connection)?;
            let mut sess = Session::new().unwrap();
@@ -589,7 +591,9 @@ fn scp_to_guest_with_auth(
            let _ = channel.wait_close();

            Ok(())
-        })() {
+        };
+
+        match closure() {
            Ok(_) => break,
            Err(e) => {
                counter += 1;
@@ -644,7 +648,7 @@ pub fn ssh_command_ip_with_auth(

    let mut counter = 0;
    loop {
-        match (|| -> Result<(), SshCommandError> {
+        let mut closure = || -> Result<(), SshCommandError> {
            let tcp =
                TcpStream::connect(format!("{ip}:22")).map_err(SshCommandError::Connection)?;
            let mut sess = Session::new().unwrap();
@@ -673,7 +677,9 @@ pub fn ssh_command_ip_with_auth(
            } else {
                Ok(())
            }
-        })() {
+        };
+
+        match closure() {
            Ok(_) => break,
            Err(e) => {
                counter += 1;
--- a/tests/integration.rs
+++ b/tests/integration.rs
@@ -2933,16 +2933,19 @@ mod common_parallel {
    }

    #[test]
+    #[cfg(not(target_arch = "aarch64"))]
    fn test_vhost_user_blk_default() {
        test_vhost_user_blk(2, false, false, Some(&prepare_vubd))
    }

    #[test]
+    #[cfg(not(target_arch = "aarch64"))]
    fn test_vhost_user_blk_readonly() {
        test_vhost_user_blk(1, true, false, Some(&prepare_vubd))
    }

    #[test]
+    #[cfg(not(target_arch = "aarch64"))]
    fn test_vhost_user_blk_direct() {
        test_vhost_user_blk(1, false, true, Some(&prepare_vubd))
    }
@@ -3457,7 +3460,7 @@ mod common_parallel {
        let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string());
        let guest = Guest::new(Box::new(focal));

-        let serial_path = guest.tmp_dir.as_path().join("/tmp/serial-output");
+        let serial_path = guest.tmp_dir.as_path().join("serial-output");
        #[cfg(target_arch = "x86_64")]
        let console_str: &str = "console=ttyS0";
        #[cfg(target_arch = "aarch64")]
@@ -3656,7 +3659,7 @@ mod common_parallel {
        let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string());
        let guest = Guest::new(Box::new(focal));

-        let console_path = guest.tmp_dir.as_path().join("/tmp/console-output");
+        let console_path = guest.tmp_dir.as_path().join("console-output");
        let mut child = GuestCommand::new(&guest)
            .args(["--cpus", "boot=1"])
            .args(["--memory", "size=512M"])
@@ -3718,8 +3721,8 @@ mod common_parallel {
    fn test_vfio() {
        setup_vfio_network_interfaces();

-        let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string());
-        let guest = Guest::new_from_ip_range(Box::new(focal), "172.18", 0);
+        let jammy = UbuntuDiskConfig::new(JAMMY_IMAGE_NAME.to_string());
+        let guest = Guest::new_from_ip_range(Box::new(jammy), "172.18", 0);

        let mut workload_path = dirs::home_dir().unwrap();
        workload_path.push("workloads");
@@ -6226,6 +6229,18 @@ mod common_parallel {
                    .unwrap_or_default(),
                2
            );
+
+            guest.reboot_linux(0, None);
+
+            assert_eq!(
+                guest
+                    .ssh_command("ip -o link | wc -l")
+                    .unwrap()
+                    .trim()
+                    .parse::<u32>()
+                    .unwrap_or_default(),
+                2
+            );
        });

        let _ = child.kill();
@@ -6631,6 +6646,7 @@ mod common_parallel {

    #[test]
    #[cfg(target_arch = "x86_64")]
+    #[ignore = "See #5756"]
    fn test_vdpa_net() {
        // Before trying to run the test, verify the vdpa_sim_net module is correctly loaded.
        if !exec_host_command_status("lsmod | grep vdpa_sim_net").success() {
@@ -7431,6 +7447,7 @@ mod windows {
    }

    #[test]
+    #[ignore = "See #6037"]
    #[cfg(not(feature = "mshv"))]
    #[cfg(not(target_arch = "aarch64"))]
    fn test_windows_guest_disk_hotplug() {
@@ -7526,6 +7543,7 @@ mod windows {
    }

    #[test]
+    #[ignore = "See #6037"]
    #[cfg(not(feature = "mshv"))]
    #[cfg(not(target_arch = "aarch64"))]
    fn test_windows_guest_disk_hotplug_multi() {
@@ -9288,6 +9306,7 @@ mod live_migration {

        // Require to run ovs-dpdk tests sequentially because they rely on the same ovs-dpdk setup
        #[test]
+        #[ignore = "See #5532"]
        #[cfg(target_arch = "x86_64")]
        #[cfg(not(feature = "mshv"))]
        fn test_live_migration_ovs_dpdk() {
@@ -9302,6 +9321,7 @@ mod live_migration {
        }

        #[test]
+        #[ignore = "See #5532"]
        #[cfg(target_arch = "x86_64")]
        #[cfg(not(feature = "mshv"))]
        fn test_live_upgrade_ovs_dpdk() {
--- a/tpm/src/emulator.rs
+++ b/tpm/src/emulator.rs
@@ -277,11 +277,7 @@ impl Emulator {
        }

        self.established_flag_cached = true;
-        if est.resp.bit != 0 {
-            self.established_flag = false;
-        } else {
-            self.established_flag = true;
-        }
+        self.established_flag = est.resp.bit == 0;

        self.established_flag
    }
--- a/vfio_user/src/lib.rs
+++ b/vfio_user/src/lib.rs
@@ -24,8 +24,9 @@ extern crate log;

 #[allow(dead_code)]
 #[repr(u16)]
-#[derive(Clone, Copy, Debug)]
+#[derive(Clone, Copy, Debug, Default)]
 enum Command {
+    #[default]
    Unknown = 0,
    Version = 1,
    DmaMap = 2,
@@ -43,28 +44,17 @@ enum Command {
    UserDirtyPages = 14,
 }

-impl Default for Command {
-    fn default() -> Self {
-        Command::Unknown
-    }
-}
-
 #[allow(dead_code)]
 #[repr(u32)]
-#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Default)]
 enum HeaderFlags {
+    #[default]
    Command = 0,
    Reply = 1,
    NoReply = 1 << 4,
    Error = 1 << 5,
 }

-impl Default for HeaderFlags {
-    fn default() -> Self {
-        HeaderFlags::Command
-    }
-}
-
 #[repr(C)]
 #[derive(Default, Clone, Copy, Debug)]
 struct Header {
@@ -331,11 +321,8 @@ impl Client {

        debug!("Reply: {:?}", server_version);

-        let mut server_version_data = Vec::new();
-        server_version_data.resize(
-            server_version.header.message_size as usize - size_of::<Version>(),
-            0,
-        );
+        let mut server_version_data =
+            vec![0; server_version.header.message_size as usize - size_of::<Version>()];
        self.stream
            .read_exact(server_version_data.as_mut_slice())
            .map_err(Error::StreamRead)?;
@@ -576,7 +563,7 @@ impl Client {
        let mmap_cap_size = size_of::<vfio_region_info_cap_sparse_mmap>() as u32;
        let mmap_area_size = size_of::<vfio_region_sparse_mmap_area>() as u32;

-        let cap_data_ptr = cap_data.as_ptr() as *const u8;
+        let cap_data_ptr = cap_data.as_ptr();
        let mut region_info_offset = region_info.cap_offset;
        while region_info_offset != 0 {
            // calculate the offset from the begining of the cap_data based on the offset
--- a/vhdx/src/vhdx_metadata.rs
+++ b/vhdx/src/vhdx_metadata.rs
@@ -133,7 +133,7 @@ impl DiskSpec {
                    .map_err(VhdxMetadataError::ReadMetadata)?;

                // MUST be at least 1 MiB and not greater than 256 MiB
-                if disk_spec.block_size < BLOCK_SIZE_MIN && disk_spec.block_size > BLOCK_SIZE_MAX {
+                if disk_spec.block_size < BLOCK_SIZE_MIN || disk_spec.block_size > BLOCK_SIZE_MAX {
                    return Err(VhdxMetadataError::InvalidBlockSize);
                }

@@ -145,11 +145,7 @@ impl DiskSpec {
                let bits = f
                    .read_u32::<LittleEndian>()
                    .map_err(VhdxMetadataError::ReadMetadata)?;
-                if bits & BLOCK_HAS_PARENT != 0 {
-                    disk_spec.has_parent = true;
-                } else {
-                    disk_spec.has_parent = false;
-                }
+                disk_spec.has_parent = bits & BLOCK_HAS_PARENT != 0;

                metadata_presence |= METADATA_FILE_PARAMETER_PRESENT;
            } else if metadata_entry.item_id
--- a/virtio-devices/src/lib.rs
+++ b/virtio-devices/src/lib.rs
@@ -41,19 +41,24 @@ pub mod vhost_user;
 pub mod vsock;
 pub mod watchdog;

-pub use self::balloon::*;
-pub use self::block::*;
-pub use self::console::*;
-pub use self::device::*;
-pub use self::epoll_helper::*;
-pub use self::iommu::*;
-pub use self::mem::*;
-pub use self::net::*;
-pub use self::pmem::*;
-pub use self::rng::*;
-pub use self::vdpa::*;
-pub use self::vsock::*;
-pub use self::watchdog::*;
+pub use self::balloon::Balloon;
+pub use self::block::{Block, BlockState};
+pub use self::console::{Console, ConsoleResizer, Endpoint};
+pub use self::device::{
+    DmaRemapping, UserspaceMapping, VirtioCommon, VirtioDevice, VirtioInterrupt,
+    VirtioInterruptType, VirtioSharedMemoryList,
+};
+pub use self::epoll_helper::{
+    EpollHelper, EpollHelperError, EpollHelperHandler, EPOLL_HELPER_EVENT_LAST,
+};
+pub use self::iommu::{AccessPlatformMapping, Iommu, IommuMapping};
+pub use self::mem::{BlocksState, Mem, VirtioMemMappingSource, VIRTIO_MEM_ALIGN_SIZE};
+pub use self::net::{Net, NetCtrlEpollHandler};
+pub use self::pmem::Pmem;
+pub use self::rng::Rng;
+pub use self::vdpa::{Vdpa, VdpaDmaMapping};
+pub use self::vsock::Vsock;
+pub use self::watchdog::Watchdog;
 use vm_memory::{bitmap::AtomicBitmap, GuestAddress, GuestMemory};
 use vm_virtio::VirtioDeviceType;

--- a/virtio-devices/src/seccomp_filters.rs
+++ b/virtio-devices/src/seccomp_filters.rs
@@ -151,6 +151,7 @@ fn virtio_rng_thread_rules() -> Vec<(i64, Vec<SeccompRule>)> {

 fn virtio_vhost_fs_thread_rules() -> Vec<(i64, Vec<SeccompRule>)> {
    vec![
+        (libc::SYS_clock_nanosleep, vec![]),
        (libc::SYS_connect, vec![]),
        (libc::SYS_nanosleep, vec![]),
        (libc::SYS_pread64, vec![]),
@@ -170,8 +171,11 @@ fn virtio_vhost_net_thread_rules() -> Vec<(i64, Vec<SeccompRule>)> {
    vec![
        (libc::SYS_accept4, vec![]),
        (libc::SYS_bind, vec![]),
+        (libc::SYS_clock_nanosleep, vec![]),
+        (libc::SYS_connect, vec![]),
        (libc::SYS_getcwd, vec![]),
        (libc::SYS_listen, vec![]),
+        (libc::SYS_nanosleep, vec![]),
        (libc::SYS_recvmsg, vec![]),
        (libc::SYS_sendmsg, vec![]),
        (libc::SYS_sendto, vec![]),
@@ -184,7 +188,14 @@ fn virtio_vhost_net_thread_rules() -> Vec<(i64, Vec<SeccompRule>)> {
 }

 fn virtio_vhost_block_thread_rules() -> Vec<(i64, Vec<SeccompRule>)> {
-    vec![]
+    vec![
+        (libc::SYS_clock_nanosleep, vec![]),
+        (libc::SYS_connect, vec![]),
+        (libc::SYS_nanosleep, vec![]),
+        (libc::SYS_recvmsg, vec![]),
+        (libc::SYS_sendmsg, vec![]),
+        (libc::SYS_socket, vec![]),
+    ]
 }

 fn create_vsock_ioctl_seccomp_rule() -> Vec<SeccompRule> {
@@ -248,6 +259,7 @@ fn virtio_thread_common() -> Vec<(i64, Vec<SeccompRule>)> {
        (libc::SYS_madvise, vec![]),
        (libc::SYS_mmap, vec![]),
        (libc::SYS_mprotect, vec![]),
+        (libc::SYS_mremap, vec![]),
        (libc::SYS_munmap, vec![]),
        (libc::SYS_openat, vec![]),
        (libc::SYS_read, vec![]),
--- a/virtio-devices/src/vhost_user/fs.rs
+++ b/virtio-devices/src/vhost_user/fs.rs
@@ -117,7 +117,6 @@ impl VhostUserMasterReqHandler for SlaveReqHandler {
        debug!("fs_slave_unmap");

        for i in 0..VHOST_USER_FS_SLAVE_ENTRIES {
-            let offset = fs.cache_offset[i];
            let mut len = fs.len[i];

            // Ignore if the length is 0.
@@ -127,9 +126,12 @@ impl VhostUserMasterReqHandler for SlaveReqHandler {

            // Need to handle a special case where the slave ask for the unmapping
            // of the entire mapping.
-            if len == 0xffff_ffff_ffff_ffff {
+            let offset = if len == 0xffff_ffff_ffff_ffff {
                len = self.cache_size;
-            }
+                0
+            } else {
+                fs.cache_offset[i]
+            };

            if !self.is_req_valid(offset, len) {
                return Err(io::Error::from_raw_os_error(libc::EINVAL));
@@ -592,7 +594,7 @@ impl VirtioDevice for Fs {
        &mut self,
        shm_regions: VirtioSharedMemoryList,
    ) -> std::result::Result<(), crate::Error> {
-        if let Some(mut cache) = self.cache.as_mut() {
+        if let Some(cache) = self.cache.as_mut() {
            cache.0 = shm_regions;
            Ok(())
        } else {
--- a/virtio-devices/src/vhost_user/vu_common_ctrl.rs
+++ b/virtio-devices/src/vhost_user/vu_common_ctrl.rs
@@ -7,7 +7,7 @@ use crate::{
    get_host_address_range, GuestMemoryMmap, GuestRegionMmap, MmapRegion, VirtioInterrupt,
    VirtioInterruptType,
 };
-use std::convert::TryInto;
+
 use std::ffi;
 use std::fs::File;
 use std::os::unix::io::{AsRawFd, FromRawFd, RawFd};
@@ -201,7 +201,7 @@ impl VhostUserHandle {

        let mut vrings_info = Vec::new();
        for (queue_index, queue, queue_evt) in queues.iter() {
-            let actual_size: usize = queue.size().try_into().unwrap();
+            let actual_size: usize = queue.size().into();

            let config_data = VringConfigData {
                queue_max_size: queue.max_size(),
--- a/virtio-devices/src/vsock/csm/txbuf.rs
+++ b/virtio-devices/src/vsock/csm/txbuf.rs
@@ -213,9 +213,7 @@ mod tests {
    fn test_push_wrap() {
        let mut txbuf = TxBuf::new();
        let mut sink = TestSink::new();
-        let mut tmp: Vec<u8> = Vec::new();
-
-        tmp.resize(TxBuf::SIZE - 2, 0);
+        let tmp: Vec<u8> = vec![0; TxBuf::SIZE - 2];
        txbuf.push(tmp.as_slice()).unwrap();
        txbuf.flush_to(&mut sink).unwrap();
        sink.clear();
--- a/virtio-devices/src/vsock/device.rs
+++ b/virtio-devices/src/vsock/device.rs
@@ -333,7 +333,7 @@ impl VersionMapped for VsockState {}

 impl<B> Vsock<B>
 where
-    B: VsockBackend,
+    B: VsockBackend + Sync,
 {
    /// Create a new virtio-vsock device with the given VM CID and vsock
    /// backend.
--- a/virtio-devices/src/vsock/packet.rs
+++ b/virtio-devices/src/vsock/packet.rs
@@ -136,7 +136,7 @@ impl VsockPacket {
                    .translate_gva(access_platform, head.len() as usize),
                VSOCK_PKT_HDR_SIZE,
            )
-            .ok_or(VsockError::GuestMemory)? as *mut u8,
+            .ok_or(VsockError::GuestMemory)?,
            buf: None,
            buf_size: 0,
        };
@@ -175,7 +175,7 @@ impl VsockPacket {
                    .translate_gva(access_platform, buf_desc.len() as usize),
                pkt.buf_size,
            )
-            .ok_or(VsockError::GuestMemory)? as *mut u8,
+            .ok_or(VsockError::GuestMemory)?,
        );

        Ok(pkt)
@@ -221,7 +221,7 @@ impl VsockPacket {
                    .translate_gva(access_platform, head.len() as usize),
                VSOCK_PKT_HDR_SIZE,
            )
-            .ok_or(VsockError::GuestMemory)? as *mut u8,
+            .ok_or(VsockError::GuestMemory)?,
            buf: Some(
                get_host_address_range(
                    desc_chain.memory(),
@@ -230,7 +230,7 @@ impl VsockPacket {
                        .translate_gva(access_platform, buf_desc.len() as usize),
                    buf_size,
                )
-                .ok_or(VsockError::GuestMemory)? as *mut u8,
+                .ok_or(VsockError::GuestMemory)?,
            ),
            buf_size,
        })
@@ -427,8 +427,8 @@ mod tests {

    fn set_pkt_len(len: u32, guest_desc: &GuestQDesc, mem: &GuestMemoryMmap) {
        let hdr_gpa = guest_desc.addr.get();
-        let hdr_ptr = get_host_address_range(mem, GuestAddress(hdr_gpa), VSOCK_PKT_HDR_SIZE)
-            .unwrap() as *mut u8;
+        let hdr_ptr =
+            get_host_address_range(mem, GuestAddress(hdr_gpa), VSOCK_PKT_HDR_SIZE).unwrap();
        let len_ptr = unsafe { hdr_ptr.add(HDROFF_LEN) };

        LittleEndian::write_u32(unsafe { std::slice::from_raw_parts_mut(len_ptr, 4) }, len);
--- a/virtio-devices/src/vsock/unix/muxer.rs
+++ b/virtio-devices/src/vsock/unix/muxer.rs
@@ -963,7 +963,7 @@ mod tests {
            self.init_pkt(local_port, peer_port, uapi::VSOCK_OP_RESPONSE);
            self.send();

-            let mut buf = vec![0u8; 32];
+            let mut buf = [0u8; 32];
            let len = stream.read(&mut buf[..]).unwrap();
            assert_eq!(&buf[..len], format!("OK {local_port}\n").as_bytes());

--- a/vm-device/src/bus.rs
+++ b/vm-device/src/bus.rs
@@ -85,7 +85,7 @@ impl Ord for BusRange {

 impl PartialOrd for BusRange {
    fn partial_cmp(&self, other: &BusRange) -> Option<Ordering> {
-        self.base.partial_cmp(&other.base)
+        Some(self.cmp(other))
    }
 }

@@ -110,8 +110,7 @@ impl Bus {
        let devices = self.devices.read().unwrap();
        let (range, dev) = devices
            .range(..=BusRange { base: addr, len: 1 })
-            .rev()
-            .next()?;
+            .next_back()?;
        dev.upgrade().map(|d| (*range, d.clone()))
    }

--- a/vm-migration/src/lib.rs
+++ b/vm-migration/src/lib.rs
@@ -13,7 +13,7 @@ pub mod protocol;

 /// Global VMM version for versioning
 const MAJOR_VERSION: u16 = 28;
-const MINOR_VERSION: u16 = 2;
+const MINOR_VERSION: u16 = 4;
 const VMM_VERSION: u16 = MAJOR_VERSION << 12 | MINOR_VERSION & 0b1111;

 pub trait VersionMapped {
--- a/vm-migration/src/protocol.rs
+++ b/vm-migration/src/protocol.rs
@@ -274,10 +274,7 @@ impl MemoryRangeTable {

    pub fn write_to(&self, fd: &mut dyn Write) -> Result<(), MigratableError> {
        fd.write_all(unsafe {
-            std::slice::from_raw_parts(
-                self.data.as_ptr() as *const MemoryRange as *const u8,
-                self.length() as usize,
-            )
+            std::slice::from_raw_parts(self.data.as_ptr() as *const u8, self.length() as usize)
        })
        .map_err(MigratableError::MigrateSocket)
    }
--- a/vmm/Cargo.toml
+++ b/vmm/Cargo.toml
@@ -17,7 +17,7 @@ acpi_tables = { path = "../acpi_tables" }
 anyhow = "1.0.66"
 arc-swap = "1.5.1"
 arch = { path = "../arch" }
-bitflags = "1.3.2"
+bitflags = "2.4.1"
 block_util = { path = "../block_util" }
 clap = "4.0.29"
 devices = { path = "../devices" }
--- a/vmm/src/api/http_endpoint.rs
+++ b/vmm/src/api/http_endpoint.rs
@@ -36,13 +36,22 @@ impl EndpointHandler for VmCreate {
                match &req.body {
                    Some(body) => {
                        // Deserialize into a VmConfig
-                        let vm_config: VmConfig = match serde_json::from_slice(body.raw())
+                        let mut vm_config: VmConfig = match serde_json::from_slice(body.raw())
                            .map_err(HttpError::SerdeJsonDeserialize)
                        {
                            Ok(config) => config,
                            Err(e) => return error_response(e, StatusCode::BadRequest),
                        };

+                        if let Some(ref mut nets) = vm_config.net {
+                            if nets.iter().any(|net| net.fds.is_some()) {
+                                warn!("Ignoring FDs sent via the HTTP request body");
+                            }
+                            for net in nets {
+                                net.fds = None;
+                            }
+                        }
+
                        // Call vm_create()
                        match vm_create(api_notifier, api_sender, Arc::new(Mutex::new(vm_config)))
                            .map_err(HttpError::ApiError)
@@ -105,6 +114,10 @@ impl EndpointHandler for VmActionHandler {
                ),
                AddNet(_) => {
                    let mut net_cfg: NetConfig = serde_json::from_slice(body.raw())?;
+                    if net_cfg.fds.is_some() {
+                        warn!("Ignoring FDs sent via the HTTP request body");
+                        net_cfg.fds = None;
+                    }
                    // Update network config with optional files that might have
                    // been sent through control message.
                    if !files.is_empty() {
--- a/vmm/src/api/openapi/cloud-hypervisor.yaml
+++ b/vmm/src/api/openapi/cloud-hypervisor.yaml
@@ -347,6 +347,28 @@ paths:
        500:
          description: The new vDPA device could not be added to the VM instance.

+  /vm.add-user-device:
+    put:
+      requestBody:
+        content:
+          application/json:
+            schema:
+              $ref: '#/components/schemas/VmAddUserDevice'
+        description: The path of the new device
+        required: true
+      responses:
+        "200":
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/PciDeviceInfo'
+          description: The new device was successfully added to the VM instance.
+        "204":
+          description: The new device was successfully (cold) added to the VM instance.
+        "404":
+          description: The new device could not be added to the VM instance.
+      summary: Add a new userspace device to the VM
+
  /vm.snapshot:
    put:
      summary: Returns a VM snapshot.
@@ -578,6 +600,9 @@ components:
      description: Virtual machine configuration

    CpuAffinity:
+      required:
+        - vcpu
+        - host_cpus
      type: object
      properties:
        vcpu:
@@ -1135,3 +1160,11 @@ components:
          type: string
        local:
          type: boolean
+
+    VmAddUserDevice:
+      required:
+        - socket
+      type: object
+      properties:
+        socket:
+          type: string
--- a/vmm/src/config.rs
+++ b/vmm/src/config.rs
@@ -2191,21 +2191,83 @@ impl VmConfig {
            gdb,
            platform,
            tpm,
+            preserved_fds: None,
        };
        config.validate().map_err(Error::Validation)?;
        Ok(config)
    }

+    /// # Safety
+    /// To use this safely, the caller must guarantee that the input
+    /// fds are all valid.
+    pub unsafe fn add_preserved_fds(&mut self, mut fds: Vec<i32>) {
+        if fds.is_empty() {
+            return;
+        }
+
+        if let Some(preserved_fds) = &self.preserved_fds {
+            fds.append(&mut preserved_fds.clone());
+        }
+
+        self.preserved_fds = Some(fds);
+    }
+
    #[cfg(feature = "tdx")]
    pub fn is_tdx_enabled(&self) -> bool {
        self.platform.as_ref().map(|p| p.tdx).unwrap_or(false)
    }
 }

+impl Clone for VmConfig {
+    fn clone(&self) -> Self {
+        VmConfig {
+            cpus: self.cpus.clone(),
+            memory: self.memory.clone(),
+            payload: self.payload.clone(),
+            disks: self.disks.clone(),
+            net: self.net.clone(),
+            rng: self.rng.clone(),
+            balloon: self.balloon.clone(),
+            fs: self.fs.clone(),
+            pmem: self.pmem.clone(),
+            serial: self.serial.clone(),
+            console: self.console.clone(),
+            devices: self.devices.clone(),
+            user_devices: self.user_devices.clone(),
+            vdpa: self.vdpa.clone(),
+            vsock: self.vsock.clone(),
+            #[cfg(target_arch = "x86_64")]
+            sgx_epc: self.sgx_epc.clone(),
+            numa: self.numa.clone(),
+            platform: self.platform.clone(),
+            tpm: self.tpm.clone(),
+            preserved_fds: self
+                .preserved_fds
+                .as_ref()
+                // SAFETY: FFI call with valid FDs
+                .map(|fds| fds.iter().map(|fd| unsafe { libc::dup(*fd) }).collect()),
+            ..*self
+        }
+    }
+}
+
+impl Drop for VmConfig {
+    fn drop(&mut self) {
+        if let Some(mut fds) = self.preserved_fds.take() {
+            for fd in fds.drain(..) {
+                // SAFETY: FFI call with valid FDs
+                unsafe { libc::close(fd) };
+            }
+        }
+    }
+}
+
 #[cfg(test)]
 mod tests {
    use super::*;
    use net_util::MacAddr;
+    use std::fs::File;
+    use std::os::unix::io::AsRawFd;

    #[test]
    fn test_cpu_parsing() -> Result<()> {
@@ -2812,6 +2874,7 @@ mod tests {
            gdb: false,
            platform: None,
            tpm: None,
+            preserved_fds: None,
        };

        assert!(valid_config.validate().is_ok());
@@ -3211,7 +3274,7 @@ mod tests {
        ]);
        assert!(still_valid_config.validate().is_ok());

-        let mut invalid_config = valid_config;
+        let mut invalid_config = valid_config.clone();
        invalid_config.devices = Some(vec![
            DeviceConfig {
                path: "/device1".into(),
@@ -3223,5 +3286,16 @@ mod tests {
            },
        ]);
        assert!(invalid_config.validate().is_err());
+
+        let mut still_valid_config = valid_config;
+        // SAFETY: Safe as the file was just opened
+        let fd1 = unsafe { libc::dup(File::open("/dev/null").unwrap().as_raw_fd()) };
+        // SAFETY: Safe as the file was just opened
+        let fd2 = unsafe { libc::dup(File::open("/dev/null").unwrap().as_raw_fd()) };
+        // SAFETY: safe as both FDs are valid
+        unsafe {
+            still_valid_config.add_preserved_fds(vec![fd1, fd2]);
+        }
+        let _still_valid_config = still_valid_config.clone();
    }
 }
--- a/vmm/src/cpu.rs
+++ b/vmm/src/cpu.rs
@@ -52,6 +52,8 @@ use hypervisor::arch::x86::MsrEntry;
 use hypervisor::arch::x86::{SpecialRegisters, StandardRegisters};
 #[cfg(target_arch = "aarch64")]
 use hypervisor::kvm::kvm_bindings;
+#[cfg(all(target_arch = "aarch64", feature = "kvm"))]
+use hypervisor::kvm::kvm_ioctls::Cap;
 #[cfg(feature = "tdx")]
 use hypervisor::kvm::{TdxExitDetails, TdxExitStatus};
 use hypervisor::{CpuState, HypervisorCpuError, HypervisorType, VmExit, VmOps};
@@ -114,6 +116,9 @@ pub enum Error {
    #[error("Error configuring vCPU: {0}")]
    VcpuConfiguration(#[source] arch::Error),

+    #[error("Still pending removed vcpu")]
+    VcpuPendingRemovedVcpu,
+
    #[cfg(target_arch = "aarch64")]
    #[error("Error fetching preferred target: {0}")]
    VcpuArmPreferredTarget(#[source] hypervisor::HypervisorVmError),
@@ -378,7 +383,14 @@ impl Vcpu {
            .map_err(Error::VcpuArmPreferredTarget)?;
        // We already checked that the capability is supported.
        kvi.features[0] |= 1 << kvm_bindings::KVM_ARM_VCPU_PSCI_0_2;
-        kvi.features[0] |= 1 << kvm_bindings::KVM_ARM_VCPU_PMU_V3;
+        if vm
+            .as_any()
+            .downcast_ref::<hypervisor::kvm::KvmVm>()
+            .unwrap()
+            .check_extension(Cap::ArmPmuV3)
+        {
+            kvi.features[0] |= 1 << kvm_bindings::KVM_ARM_VCPU_PMU_V3;
+        }
        // Non-boot cpus are powered off initially.
        if self.id > 0 {
            kvi.features[0] |= 1 << kvm_bindings::KVM_ARM_VCPU_POWER_OFF;
@@ -549,6 +561,7 @@ impl BusDevice for CpuManager {
 struct VcpuState {
    inserting: bool,
    removing: bool,
+    pending_removal: Arc<AtomicBool>,
    handle: Option<thread::JoinHandle<()>>,
    kill: Arc<AtomicBool>,
    vcpu_run_interrupted: Arc<AtomicBool>,
@@ -1035,12 +1048,16 @@ impl CpuManager {
                                            "VCPU generated error: {:?}",
                                            Error::UnexpectedVmExit
                                        );
+                                        vcpu_run_interrupted.store(true, Ordering::SeqCst);
+                                        exit_evt.write(1).unwrap();
                                        break;
                                    }
                                },

                                Err(e) => {
                                    error!("VCPU generated error: {:?}", Error::VcpuRun(e.into()));
+                                    vcpu_run_interrupted.store(true, Ordering::SeqCst);
+                                    exit_evt.write(1).unwrap();
                                    break;
                                }
                            }
@@ -1114,12 +1131,24 @@ impl CpuManager {
        // Mark vCPUs for removal, actual removal happens on ejection
        for cpu_id in desired_vcpus..self.present_vcpus() {
            self.vcpu_states[usize::from(cpu_id)].removing = true;
+            self.vcpu_states[usize::from(cpu_id)]
+                .pending_removal
+                .store(true, Ordering::SeqCst);
        }
    }

+    pub fn check_pending_removed_vcpu(&mut self) -> bool {
+        for state in self.vcpu_states.iter() {
+            if state.active() && state.pending_removal.load(Ordering::SeqCst) {
+                return true;
+            }
+        }
+        false
+    }
+
    fn remove_vcpu(&mut self, cpu_id: u8) -> Result<()> {
        info!("Removing vCPU: cpu_id = {}", cpu_id);
-        let mut state = &mut self.vcpu_states[usize::from(cpu_id)];
+        let state = &mut self.vcpu_states[usize::from(cpu_id)];
        state.kill.store(true, Ordering::SeqCst);
        state.signal_thread();
        state.join_thread()?;
@@ -1127,6 +1156,7 @@ impl CpuManager {

        // Once the thread has exited, clear the "kill" so that it can reused
        state.kill.store(false, Ordering::SeqCst);
+        state.pending_removal.store(false, Ordering::SeqCst);

        Ok(())
    }
@@ -1160,6 +1190,10 @@ impl CpuManager {
            return Ok(false);
        }

+        if self.check_pending_removed_vcpu() {
+            return Err(Error::VcpuPendingRemovedVcpu);
+        }
+
        match desired_vcpus.cmp(&self.present_vcpus()) {
            cmp::Ordering::Greater => {
                self.create_vcpus(desired_vcpus, None)?;
@@ -1699,8 +1733,7 @@ impl Cpu {
            flags: 1 << MADT_CPU_ENABLE_FLAG,
        };

-        let mut mat_data: Vec<u8> = Vec::new();
-        mat_data.resize(std::mem::size_of_val(&lapic), 0);
+        let mut mat_data: Vec<u8> = vec![0; std::mem::size_of_val(&lapic)];
        unsafe { *(mat_data.as_mut_ptr() as *mut LocalApic) = lapic };

        mat_data
--- a/vmm/src/device_manager.rs
+++ b/vmm/src/device_manager.rs
@@ -2317,23 +2317,28 @@ impl DeviceManager {
                    .map_err(DeviceManagerError::CreateVirtioNet)?,
                ))
            } else if let Some(fds) = &net_cfg.fds {
-                Arc::new(Mutex::new(
-                    virtio_devices::Net::from_tap_fds(
-                        id.clone(),
-                        fds,
-                        Some(net_cfg.mac),
-                        net_cfg.mtu,
-                        self.force_iommu | net_cfg.iommu,
-                        net_cfg.queue_size,
-                        self.seccomp_action.clone(),
-                        net_cfg.rate_limiter_config,
-                        self.exit_evt
-                            .try_clone()
-                            .map_err(DeviceManagerError::EventFd)?,
-                        state,
-                    )
-                    .map_err(DeviceManagerError::CreateVirtioNet)?,
-                ))
+                let net = virtio_devices::Net::from_tap_fds(
+                    id.clone(),
+                    fds,
+                    Some(net_cfg.mac),
+                    net_cfg.mtu,
+                    self.force_iommu | net_cfg.iommu,
+                    net_cfg.queue_size,
+                    self.seccomp_action.clone(),
+                    net_cfg.rate_limiter_config,
+                    self.exit_evt
+                        .try_clone()
+                        .map_err(DeviceManagerError::EventFd)?,
+                    state,
+                )
+                .map_err(DeviceManagerError::CreateVirtioNet)?;
+
+                // SAFETY: 'fds' are valid because TAP devices are created successfully
+                unsafe {
+                    self.config.lock().unwrap().add_preserved_fds(fds.clone());
+                }
+
+                Arc::new(Mutex::new(net))
            } else {
                Arc::new(Mutex::new(
                    virtio_devices::Net::new(
--- a/vmm/src/lib.rs
+++ b/vmm/src/lib.rs
@@ -43,6 +43,7 @@ use std::os::unix::net::UnixListener;
 use std::os::unix::net::UnixStream;
 use std::panic::AssertUnwindSafe;
 use std::path::PathBuf;
+use std::rc::Rc;
 use std::sync::mpsc::{Receiver, RecvError, SendError, Sender};
 use std::sync::{Arc, Mutex};
 use std::time::Instant;
@@ -318,9 +319,9 @@ pub fn start_vmm_thread(
                vmm.setup_signal_handler()?;

                vmm.control_loop(
-                    Arc::new(api_receiver),
+                    Rc::new(api_receiver),
                    #[cfg(feature = "guest_debug")]
-                    Arc::new(gdb_receiver),
+                    Rc::new(gdb_receiver),
                )
            })
            .map_err(Error::VmmThreadSpawn)?
@@ -1471,6 +1472,12 @@ impl Vmm {
        let vm_config = vm.get_config();
        #[cfg(all(feature = "kvm", target_arch = "x86_64"))]
        let common_cpuid = {
+            #[cfg(feature = "tdx")]
+            if vm_config.lock().unwrap().is_tdx_enabled() {
+                return Err(MigratableError::MigrateSend(anyhow!(
+                    "Live Migration is not supported when TDX is enabled"
+                )));
+            };
            let phys_bits = vm::physical_bits(vm_config.lock().unwrap().cpus.max_phys_bits);
            arch::generate_common_cpuid(
                hypervisor,
@@ -1479,7 +1486,7 @@ impl Vmm {
                phys_bits,
                vm_config.lock().unwrap().cpus.kvm_hyperv,
                #[cfg(feature = "tdx")]
-                vm_config.lock().unwrap().is_tdx_enabled(),
+                false,
            )
            .map_err(|e| {
                MigratableError::MigrateReceive(anyhow!("Error generating common cpuid': {:?}", e))
@@ -1656,6 +1663,13 @@ impl Vmm {
        src_vm_config: &Arc<Mutex<VmConfig>>,
        src_vm_cpuid: &[hypervisor::arch::x86::CpuIdEntry],
    ) -> result::Result<(), MigratableError> {
+        #[cfg(feature = "tdx")]
+        if src_vm_config.lock().unwrap().is_tdx_enabled() {
+            return Err(MigratableError::MigrateReceive(anyhow!(
+                "Live Migration is not supported when TDX is enabled"
+            )));
+        };
+
        // We check the `CPUID` compatibility of between the source vm and destination, which is
        // mostly about feature compatibility and "topology/sgx" leaves are not relevant.
        let dest_cpuid = &{
@@ -1669,7 +1683,7 @@ impl Vmm {
                phys_bits,
                vm_config.cpus.kvm_hyperv,
                #[cfg(feature = "tdx")]
-                vm_config.is_tdx_enabled(),
+                false,
            )
            .map_err(|e| {
                MigratableError::MigrateReceive(anyhow!("Error generating common cpuid: {:?}", e))
@@ -1685,8 +1699,8 @@ impl Vmm {

    fn control_loop(
        &mut self,
-        api_receiver: Arc<Receiver<ApiRequest>>,
-        #[cfg(feature = "guest_debug")] gdb_receiver: Arc<Receiver<gdb::GdbRequest>>,
+        api_receiver: Rc<Receiver<ApiRequest>>,
+        #[cfg(feature = "guest_debug")] gdb_receiver: Rc<Receiver<gdb::GdbRequest>>,
    ) -> Result<()> {
        const EPOLL_EVENTS_LEN: usize = 100;

@@ -2100,6 +2114,7 @@ mod unit_tests {
            gdb: false,
            platform: None,
            tpm: None,
+            preserved_fds: None,
        }))
    }

--- a/vmm/src/memory_manager.rs
+++ b/vmm/src/memory_manager.rs
@@ -1506,7 +1506,7 @@ impl MemoryManager {
            .ok_or(Error::MemoryRangeAllocation)?;

        // Update the slot so that it can be queried via the I/O port
-        let mut slot = &mut self.hotplug_slots[self.next_hotplug_slot];
+        let slot = &mut self.hotplug_slots[self.next_hotplug_slot];
        slot.active = true;
        slot.inserting = true;
        slot.base = region.start_addr().0;
--- a/vmm/src/pci_segment.rs
+++ b/vmm/src/pci_segment.rs
@@ -159,7 +159,7 @@ impl PciSegment {

        // There are 32 devices on the PCI bus, let's assign them an IRQ.
        for i in 0..32 {
-            pci_irq_slots[i] = irqs[(i % num_irqs)];
+            pci_irq_slots[i] = irqs[i % num_irqs];
        }

        Ok(())
--- a/vmm/src/seccomp_filters.rs
+++ b/vmm/src/seccomp_filters.rs
@@ -341,6 +341,7 @@ fn create_vmm_ioctl_seccomp_rule_kvm() -> Result<Vec<SeccompRule>, BackendError>
    const KVM_GET_MSR_INDEX_LIST: u64 = 0xc004_ae02;
    const KVM_GET_MSRS: u64 = 0xc008_ae88;
    const KVM_GET_SREGS: u64 = 0x8138_ae83;
+    const KVM_GET_TSC_KHZ: u64 = 0xaea3;
    const KVM_GET_XCRS: u64 = 0x8188_aea6;
    const KVM_GET_XSAVE: u64 = 0x9000_aea4;
    const KVM_KVMCLOCK_CTRL: u64 = 0xaead;
@@ -367,6 +368,7 @@ fn create_vmm_ioctl_seccomp_rule_kvm() -> Result<Vec<SeccompRule>, BackendError>
        and![Cond::new(1, ArgLen::Dword, Eq, KVM_GET_MSR_INDEX_LIST)?],
        and![Cond::new(1, ArgLen::Dword, Eq, KVM_GET_MSRS)?],
        and![Cond::new(1, ArgLen::Dword, Eq, KVM_GET_SREGS)?],
+        and![Cond::new(1, ArgLen::Dword, Eq, KVM_GET_TSC_KHZ)?],
        and![Cond::new(1, ArgLen::Dword, Eq, KVM_GET_XCRS,)?],
        and![Cond::new(1, ArgLen::Dword, Eq, KVM_GET_XSAVE,)?],
        and![Cond::new(1, ArgLen::Dword, Eq, KVM_KVMCLOCK_CTRL)?],
@@ -476,6 +478,7 @@ fn pty_foreground_thread_rules() -> Result<Vec<(i64, Vec<SeccompRule>)>, Backend
        #[cfg(target_arch = "aarch64")]
        (libc::SYS_ppoll, vec![]),
        (libc::SYS_read, vec![]),
+        (libc::SYS_restart_syscall, vec![]),
        (libc::SYS_rt_sigaction, vec![]),
        (libc::SYS_rt_sigreturn, vec![]),
        (libc::SYS_setsid, vec![]),
@@ -706,6 +709,7 @@ fn vcpu_thread_rules(
        (libc::SYS_madvise, vec![]),
        (libc::SYS_mmap, vec![]),
        (libc::SYS_mprotect, vec![]),
+        (libc::SYS_mremap, vec![]),
        (libc::SYS_munmap, vec![]),
        (libc::SYS_nanosleep, vec![]),
        (libc::SYS_newfstatat, vec![]),
--- a/vmm/src/serial_manager.rs
+++ b/vmm/src/serial_manager.rs
@@ -225,7 +225,7 @@ impl SerialManager {
                    const EPOLL_EVENTS_LEN: usize = 3;

                    let mut events =
-                        vec![epoll::Event::new(epoll::Events::empty(), 0); EPOLL_EVENTS_LEN];
+                        [epoll::Event::new(epoll::Events::empty(), 0); EPOLL_EVENTS_LEN];

                    loop {
                        let num_events = match epoll::wait(epoll_fd, timeout, &mut events[..]) {
--- a/vmm/src/vm.rs
+++ b/vmm/src/vm.rs
@@ -1319,7 +1319,7 @@ impl Vm {
                .resize(desired_memory)
                .map_err(Error::MemoryManager)?;

-            let mut memory_config = &mut self.config.lock().unwrap().memory;
+            let memory_config = &mut self.config.lock().unwrap().memory;

            if let Some(new_region) = &new_region {
                self.device_manager
@@ -2605,12 +2605,9 @@ impl Snapshottable for Vm {
    fn snapshot(&mut self) -> std::result::Result<Snapshot, MigratableError> {
        event!("vm", "snapshotting");

-        #[cfg(feature = "tdx")]
-        let tdx_enabled = self.config.lock().unwrap().is_tdx_enabled();
-
        #[cfg(feature = "tdx")]
        {
-            if tdx_enabled {
+            if self.config.lock().unwrap().is_tdx_enabled() {
                return Err(MigratableError::Snapshot(anyhow!(
                    "Snapshot not possible with TDX VM"
                )));
@@ -2634,7 +2631,7 @@ impl Snapshottable for Vm {
                phys_bits,
                self.config.lock().unwrap().cpus.kvm_hyperv,
                #[cfg(feature = "tdx")]
-                tdx_enabled,
+                false,
            )
            .map_err(|e| {
                MigratableError::MigrateReceive(anyhow!("Error generating common cpuid: {:?}", e))
--- a/vmm/src/vm_config.rs
+++ b/vmm/src/vm_config.rs
@@ -127,18 +127,13 @@ pub struct MemoryZoneConfig {
    pub prefault: bool,
 }

-#[derive(Clone, Copy, Debug, PartialEq, Eq, Deserialize, Serialize)]
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Deserialize, Serialize, Default)]
 pub enum HotplugMethod {
+    #[default]
    Acpi,
    VirtioMem,
 }

-impl Default for HotplugMethod {
-    fn default() -> Self {
-        HotplugMethod::Acpi
-    }
-}
-
 fn default_memoryconfig_thp() -> bool {
    true
 }
@@ -188,18 +183,13 @@ impl Default for MemoryConfig {
    }
 }

-#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize)]
+#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize, Default)]
 pub enum VhostMode {
+    #[default]
    Client,
    Server,
 }

-impl Default for VhostMode {
-    fn default() -> Self {
-        VhostMode::Client
-    }
-}
-
 #[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize)]
 pub struct DiskConfig {
    pub path: Option<PathBuf>,
@@ -560,7 +550,7 @@ pub struct TpmConfig {
    pub socket: PathBuf,
 }

-#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize)]
+#[derive(Debug, PartialEq, Eq, Deserialize, Serialize)]
 pub struct VmConfig {
    #[serde(default)]
    pub cpus: CpusConfig,
@@ -593,4 +583,10 @@ pub struct VmConfig {
    pub gdb: bool,
    pub platform: Option<PlatformConfig>,
    pub tpm: Option<TpmConfig>,
+    // Preseved FDs are the ones that share the same life-time as its holding
+    // VmConfig instance, such as FDs for creating TAP devices.
+    // Perserved FDs will stay open as long as the holding VmConfig instance is
+    // valid, and will be closed when the holding VmConfig instance is destroyed.
+    #[serde(skip)]
+    pub preserved_fds: Option<Vec<i32>>,
 }