build: Release v36.1 (bug fix release)

Signed-off-by: Bo Chen <chen.bo@intel.com>
tests: Temporarily disable vhost_user_blk tests on aarch64
2026-08-05 02:19:16 +00:00 · 2024-01-10 11:36:39 -08:00 · 2024-01-10 09:09:15 -08:00 · 2024-01-10 09:09:15 -08:00 · 2024-01-10 09:09:15 -08:00 · 2024-01-10 09:09:15 -08:00
80 changed files with 3768 additions and 1946 deletions
--- a/.github/workflows/build.yaml
+++ b/.github/workflows/build.yaml
@@ -13,7 +13,7 @@ jobs:
          - stable
          - beta
          - nightly
-          - "1.64"
+          - "1.66"
        target:
          - x86_64-unknown-linux-gnu
          - x86_64-unknown-linux-musl
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -5,9 +5,9 @@ version = 3
 [[package]]
 name = "acpi_tables"
 version = "0.1.0"
-source = "git+https://github.com/rust-vmm/acpi_tables?branch=main#1029d22777f07b04849234bbe756da34a6df2913"
+source = "git+https://github.com/rust-vmm/acpi_tables?branch=main#1a733bf690ccc10bdfeacad33e3c9f6cce0008fd"
 dependencies = [
- "zerocopy 0.6.1",
+ "zerocopy",
 ]

 [[package]]
@@ -34,6 +34,55 @@ dependencies = [
 "memchr",
 ]

+[[package]]
+name = "anstream"
+version = "0.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0ca84f3628370c59db74ee214b3263d58f9aadd9b4fe7e711fd87dc452b7f163"
+dependencies = [
+ "anstyle",
+ "anstyle-parse",
+ "anstyle-query",
+ "anstyle-wincon",
+ "colorchoice",
+ "is-terminal",
+ "utf8parse",
+]
+
+[[package]]
+name = "anstyle"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3a30da5c5f2d5e72842e00bcb57657162cdabef0931f40e2deb9b4140440cecd"
+
+[[package]]
+name = "anstyle-parse"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "938874ff5980b03a87c5524b3ae5b59cf99b1d6bc836848df7bc5ada9643c333"
+dependencies = [
+ "utf8parse",
+]
+
+[[package]]
+name = "anstyle-query"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5ca11d4be1bab0c8bc8734a9aa7bf4ee8316d462a08c6ac5052f888fef5b494b"
+dependencies = [
+ "windows-sys 0.48.0",
+]
+
+[[package]]
+name = "anstyle-wincon"
+version = "1.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c677ab05e09154296dd37acecd46420c17b9713e8366facafa8fc0885167cf4c"
+dependencies = [
+ "anstyle",
+ "windows-sys 0.48.0",
+]
+
 [[package]]
 name = "anyhow"
 version = "1.0.75"
@@ -75,41 +124,13 @@ dependencies = [
 "vmm-sys-util",
 ]

-[[package]]
-name = "argh"
-version = "0.1.10"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ab257697eb9496bf75526f0217b5ed64636a9cfafa78b8365c71bd283fcef93e"
-dependencies = [
- "argh_derive",
- "argh_shared",
-]
-
-[[package]]
-name = "argh_derive"
-version = "0.1.10"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b382dbd3288e053331f03399e1db106c9fb0d8562ad62cb04859ae926f324fa6"
-dependencies = [
- "argh_shared",
- "proc-macro2",
- "quote",
- "syn 1.0.109",
-]
-
-[[package]]
-name = "argh_shared"
-version = "0.1.10"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "64cb94155d965e3d37ffbbe7cc5b82c3dd79dd33bd48e536f73d2cfb8d85506f"
-
 [[package]]
 name = "async-broadcast"
 version = "0.5.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7c48ccdbf6ca6b121e0f586cbc0e73ae440e56c67c30fa0873b4e110d9c26d2b"
 dependencies = [
- "event-listener",
+ "event-listener 2.5.3",
 "futures-core",
 ]

@@ -120,7 +141,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "81953c529336010edd6d8e358f886d9581267795c61b19475b71314bffa46d35"
 dependencies = [
 "concurrent-queue",
- "event-listener",
+ "event-listener 2.5.3",
 "futures-core",
 ]

@@ -164,7 +185,7 @@ dependencies = [
 "log",
 "parking",
 "polling",
- "rustix 0.37.21",
+ "rustix 0.37.25",
 "slab",
 "socket2",
 "waker-fn",
@@ -176,24 +197,23 @@ version = "2.7.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "fa24f727524730b077666307f2734b4a1a1c57acb79193127dcc8914d5242dd7"
 dependencies = [
- "event-listener",
+ "event-listener 2.5.3",
 ]

 [[package]]
 name = "async-process"
-version = "1.7.0"
+version = "1.8.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7a9d28b1d97e08915212e2e45310d47854eafa69600756fc735fb788f75199c9"
+checksum = "bf012553ce51eb7aa6dc2143804cc8252bd1cb681a1c5cb7fa94ca88682dee1d"
 dependencies = [
 "async-io",
 "async-lock",
- "autocfg",
+ "async-signal",
 "blocking",
 "cfg-if",
- "event-listener",
+ "event-listener 3.0.0",
 "futures-lite",
- "rustix 0.37.21",
- "signal-hook",
+ "rustix 0.38.8",
 "windows-sys 0.48.0",
 ]

@@ -208,6 +228,25 @@ dependencies = [
 "syn 2.0.31",
 ]

+[[package]]
+name = "async-signal"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c99f3cb3f9ff89f7d718fbb942c9eb91bedff12e396adf09a622dfe7ffec2bc2"
+dependencies = [
+ "async-io",
+ "async-lock",
+ "atomic-waker",
+ "cfg-if",
+ "concurrent-queue",
+ "futures-core",
+ "futures-io",
+ "libc",
+ "signal-hook-registry",
+ "slab",
+ "windows-sys 0.48.0",
+]
+
 [[package]]
 name = "async-task"
 version = "4.4.0"
@@ -216,9 +255,9 @@ checksum = "ecc7ab41815b3c653ccd2978ec3255c81349336702dfdf62ee6f7069b12a3aae"

 [[package]]
 name = "async-trait"
-version = "0.1.73"
+version = "0.1.74"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bc00ceb34980c03614e35a3a4e218276a0a824e911d07651cd0d858a51e8c0f0"
+checksum = "a66537f1bb974b254c98ed142ff995236e81b9d0fe4db0575f46612cb15eb0f9"
 dependencies = [
 "proc-macro2",
 "quote",
@@ -227,9 +266,9 @@ dependencies = [

 [[package]]
 name = "atomic-waker"
-version = "1.1.1"
+version = "1.1.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1181e1e0d1fce796a03db1ae795d67167da795f9cf4a39c37589e85ef57f26d3"
+checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0"

 [[package]]
 name = "autocfg"
@@ -261,6 +300,17 @@ dependencies = [
 "serde",
 ]

+[[package]]
+name = "bitfield-struct"
+version = "0.5.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "eac32db62a43cf33353ce30b4a208b08193ea2086a1c6c004acb0073c706a29d"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.31",
+]
+
 [[package]]
 name = "bitflags"
 version = "1.3.2"
@@ -269,9 +319,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"

 [[package]]
 name = "bitflags"
-version = "2.3.3"
+version = "2.4.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "630be753d4e58660abd17930c71b647fe46c27ea6b63cc59e1e3851406972e42"
+checksum = "327762f6e5a765692301e5bb513e0d9fef63be86bbc14528052b1cd3e6f03e07"

 [[package]]
 name = "block"
@@ -346,13 +396,42 @@ version = "1.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"

+[[package]]
+name = "clap"
+version = "4.3.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1640e5cc7fb47dbb8338fd471b105e7ed6c3cb2aeb00c2e067127ffd3764a05d"
+dependencies = [
+ "clap_builder",
+]
+
+[[package]]
+name = "clap_builder"
+version = "4.3.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "98c59138d527eeaf9b53f35a77fcc1fad9d883116070c63d5de1c7dc7b00c72b"
+dependencies = [
+ "anstream",
+ "anstyle",
+ "clap_lex",
+ "once_cell",
+ "strsim",
+ "terminal_size",
+]
+
+[[package]]
+name = "clap_lex"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2da6da31387c7e4ef160ffab6d5e7f00c42626fe39aea70a7b0f1773f7dd6c1b"
+
 [[package]]
 name = "cloud-hypervisor"
-version = "35.0.0"
+version = "36.1.0"
 dependencies = [
 "anyhow",
 "api_client",
- "argh",
+ "clap",
 "dhat",
 "dirs",
 "epoll",
@@ -377,6 +456,12 @@ dependencies = [
 "zbus",
 ]

+[[package]]
+name = "colorchoice"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7"
+
 [[package]]
 name = "concurrent-queue"
 version = "2.2.0"
@@ -404,6 +489,15 @@ dependencies = [
 "rustc_version",
 ]

+[[package]]
+name = "crc32fast"
+version = "1.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d"
+dependencies = [
+ "cfg-if",
+]
+
 [[package]]
 name = "crc64"
 version = "1.0.0"
@@ -482,7 +576,7 @@ dependencies = [
 "acpi_tables",
 "anyhow",
 "arch",
- "bitflags 2.3.3",
+ "bitflags 2.4.1",
 "byteorder",
 "event_monitor",
 "hypervisor",
@@ -549,9 +643,9 @@ dependencies = [

 [[package]]
 name = "enumflags2"
-version = "0.7.7"
+version = "0.7.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c041f5090df68b32bcd905365fd51769c8b9d553fe87fde0b683534f10c01bd2"
+checksum = "5998b4f30320c9d93aed72f63af821bfdac50465b75428fce77b48ec482c3939"
 dependencies = [
 "enumflags2_derive",
 "serde",
@@ -559,9 +653,9 @@ dependencies = [

 [[package]]
 name = "enumflags2_derive"
-version = "0.7.7"
+version = "0.7.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5e9a1f9f7d83e59740248a6e14ecf93929ade55027844dfcea78beafccc15745"
+checksum = "f95e2801cd355d4a1a3e3953ce6ee5ae9603a5c833455343a8bfe3f44d418246"
 dependencies = [
 "proc-macro2",
 "quote",
@@ -587,37 +681,43 @@ version = "4.3.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "74351c3392ea1ff6cd2628e0042d268ac2371cb613252ff383b6dfa50d22fa79"
 dependencies = [
- "bitflags 2.3.3",
+ "bitflags 2.4.1",
 "libc",
 ]

+[[package]]
+name = "equivalent"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5"
+
 [[package]]
 name = "errno"
-version = "0.3.2"
+version = "0.3.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6b30f669a7961ef1631673d2766cc92f52d64f7ef354d4fe0ddfd30ed52f0f4f"
+checksum = "ac3e13f66a2f95e32a39eaa81f6b95d42878ca0e1db0c7543723dfe12557e860"
 dependencies = [
- "errno-dragonfly",
 "libc",
 "windows-sys 0.48.0",
 ]

-[[package]]
-name = "errno-dragonfly"
-version = "0.1.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf"
-dependencies = [
- "cc",
- "libc",
-]
-
 [[package]]
 name = "event-listener"
 version = "2.5.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0"

+[[package]]
+name = "event-listener"
+version = "3.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "29e56284f00d94c1bc7fd3c77027b4623c88c1f53d8d2394c6199f2921dea325"
+dependencies = [
+ "concurrent-queue",
+ "parking",
+ "pin-project-lite",
+]
+
 [[package]]
 name = "event_monitor"
 version = "0.1.0"
@@ -834,15 +934,15 @@ checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b"

 [[package]]
 name = "hashbrown"
-version = "0.12.3"
+version = "0.14.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888"
+checksum = "f93e7192158dbcda357bdec5fb5788eebf8bbac027f3f33e719d29135ae84156"

 [[package]]
 name = "hermit-abi"
-version = "0.3.2"
+version = "0.3.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "443144c8cdadd93ebf52ddb4056d257f5b52c04d3c804e657d19eb73fc33668b"
+checksum = "d77f7ec81a6d05a3abb01ab6eb7590f6083d08449fe5a1c8b1e620283546ccb7"

 [[package]]
 name = "hex"
@@ -864,6 +964,8 @@ dependencies = [
 "byteorder",
 "env_logger",
 "iced-x86",
+ "igvm",
+ "igvm_defs",
 "kvm-bindings",
 "kvm-ioctls",
 "libc",
@@ -880,9 +982,9 @@ dependencies = [

 [[package]]
 name = "iced-x86"
-version = "1.19.0"
+version = "1.20.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b7cc8d38244d84278262c8ebe6930cc44283d194cbabae2651f6112103802fb5"
+checksum = "cdd366a53278429c028367e0ba22a46cab6d565a57afb959f06e92c7a69e7828"
 dependencies = [
 "lazy_static",
 ]
@@ -894,12 +996,39 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39"

 [[package]]
-name = "indexmap"
-version = "1.9.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99"
+name = "igvm"
+version = "0.1.0"
+source = "git+https://github.com/microsoft/igvm?branch=main#c1b0201d8286cb23b9f30cb16ba435484666cfa3"
 dependencies = [
- "autocfg",
+ "bitfield-struct",
+ "crc32fast",
+ "hex",
+ "igvm_defs",
+ "open-enum",
+ "range_map_vec",
+ "thiserror",
+ "tracing",
+ "zerocopy",
+]
+
+[[package]]
+name = "igvm_defs"
+version = "0.1.0"
+source = "git+https://github.com/microsoft/igvm?branch=main#c1b0201d8286cb23b9f30cb16ba435484666cfa3"
+dependencies = [
+ "bitfield-struct",
+ "open-enum",
+ "static_assertions",
+ "zerocopy",
+]
+
+[[package]]
+name = "indexmap"
+version = "2.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8adf3ddd720272c6ea8bf59463c04e0f93d0bbf7c5439b691bca2987e0270897"
+dependencies = [
+ "equivalent",
 "hashbrown",
 ]

@@ -1118,19 +1247,19 @@ dependencies = [
 [[package]]
 name = "mshv-bindings"
 version = "0.1.1"
-source = "git+https://github.com/rust-vmm/mshv?branch=main#c5a60508595dc504da469b89102b8b49e91714a9"
+source = "git+https://github.com/rust-vmm/mshv?branch=main#af397ea8514303d3a19d21d33730e867f7415ba9"
 dependencies = [
 "libc",
 "serde",
 "serde_derive",
 "vmm-sys-util",
- "zerocopy 0.7.1",
+ "zerocopy",
 ]

 [[package]]
 name = "mshv-ioctls"
 version = "0.1.1"
-source = "git+https://github.com/rust-vmm/mshv?branch=main#c5a60508595dc504da469b89102b8b49e91714a9"
+source = "git+https://github.com/rust-vmm/mshv?branch=main#af397ea8514303d3a19d21d33730e867f7415ba9"
 dependencies = [
 "libc",
 "mshv-bindings",
@@ -1198,9 +1327,9 @@ checksum = "43794a0ace135be66a25d3ae77d41b91615fb68ae937f904090203e81f755b65"

 [[package]]
 name = "num-traits"
-version = "0.2.15"
+version = "0.2.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd"
+checksum = "f30b0abd723be7e2ffca1272140fac1a2f084c77ec3e123c192b66af1ee9e6c2"
 dependencies = [
 "autocfg",
 ]
@@ -1221,10 +1350,30 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d"

 [[package]]
-name = "openssl-src"
-version = "300.1.3+3.1.2"
+name = "open-enum"
+version = "0.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cd2c101a165fff9935e34def4669595ab1c7847943c42be86e21503e482be107"
+checksum = "9807f1199cf84ec7cc801a79e5ee9aa5178e4762c6b9c7066c30b3cabdcd911e"
+dependencies = [
+ "open-enum-derive",
+]
+
+[[package]]
+name = "open-enum-derive"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "894ae443e59fecf7173ab3b963473f44193fa71b3c8953c61a5bd5f30880bb88"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 1.0.109",
+]
+
+[[package]]
+name = "openssl-src"
+version = "300.1.5+3.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "559068e4c12950d7dcaa1857a61725c0d38d4fc03ff8e070ab31a75d6e316491"
 dependencies = [
 "cc",
 ]
@@ -1264,9 +1413,9 @@ dependencies = [

 [[package]]
 name = "parking"
-version = "2.1.0"
+version = "2.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "14f2252c834a40ed9bb5422029649578e63aa341ac401f74e719dd1afda8394e"
+checksum = "e52c774a4c39359c1d1c52e43f73dd91a75a614652c825408eec30c95a9b2067"

 [[package]]
 name = "parking_lot"
@@ -1318,9 +1467,9 @@ dependencies = [

 [[package]]
 name = "paste"
-version = "1.0.12"
+version = "1.0.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9f746c4065a8fa3fe23974dd82f15431cc8d40779821001404d10d2e79ca7d79"
+checksum = "de3145af08024dea9fa9914f381a17b8fc6034dfb00f3a84013f7ff43f29ed4c"

 [[package]]
 name = "pci"
@@ -1349,7 +1498,7 @@ dependencies = [
 name = "performance-metrics"
 version = "0.1.0"
 dependencies = [
- "argh",
+ "clap",
 "dirs",
 "serde",
 "serde_json",
@@ -1567,6 +1716,12 @@ dependencies = [
 "getrandom",
 ]

+[[package]]
+name = "range_map_vec"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8edc89eaa583cf6bc4c6ef16a219f0a60d342ca3bf0eae793560038ac8af1795"
+
 [[package]]
 name = "rate_limiter"
 version = "0.1.0"
@@ -1607,9 +1762,9 @@ dependencies = [

 [[package]]
 name = "regex"
-version = "1.9.1"
+version = "1.9.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b2eae68fc220f7cf2532e4494aded17545fce192d59cd996e0fe7887f4ceb575"
+checksum = "12de2eff854e5fa4b1295edd650e227e9d8fb0c9e90b12e7f36d6a6811791a29"
 dependencies = [
 "aho-corasick",
 "memchr",
@@ -1668,9 +1823,9 @@ dependencies = [

 [[package]]
 name = "rustix"
-version = "0.37.21"
+version = "0.37.25"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "62f25693a73057a1b4cb56179dd3c7ea21a7c6c5ee7d85781f5749b46f34b79c"
+checksum = "d4eb579851244c2c03e7c24f501c3432bed80b8f720af1d6e5b0e0f01555a035"
 dependencies = [
 "bitflags 1.3.2",
 "errno",
@@ -1686,7 +1841,7 @@ version = "0.38.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "19ed4fa021d81c8392ce04db050a3da9a60299050b7ae1cf482d862b54a7218f"
 dependencies = [
- "bitflags 2.3.3",
+ "bitflags 2.4.1",
 "errno",
 "libc",
 "linux-raw-sys 0.4.5",
@@ -1707,18 +1862,18 @@ checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"

 [[package]]
 name = "seccompiler"
-version = "0.3.0"
+version = "0.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6f6575e3c2b3a0fe2ef3e53855b6a8dead7c29f783da5e123d378c8c6a89017e"
+checksum = "345a3e4dddf721a478089d4697b83c6c0a8f5bf16086f6c13397e4534eb6e2e5"
 dependencies = [
 "libc",
 ]

 [[package]]
 name = "semver"
-version = "1.0.18"
+version = "1.0.20"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b0293b4b29daaf487284529cc2f5675b8e57c61f70167ba415a463651fd6a918"
+checksum = "836fa6a3e1e547f9a2c4040802ec865b5d85f4014efe00555d7090a3dcaa1090"

 [[package]]
 name = "serde"
@@ -1764,9 +1919,9 @@ dependencies = [

 [[package]]
 name = "serde_with"
-version = "3.0.0"
+version = "3.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9f02d8aa6e3c385bf084924f660ce2a3a6bd333ba55b35e8590b321f35d88513"
+checksum = "64cd236ccc1b7a29e7e2739f27c0b2dd199804abc4290e32f59f3b68d6405c23"
 dependencies = [
 "serde",
 "serde_with_macros",
@@ -1774,9 +1929,9 @@ dependencies = [

 [[package]]
 name = "serde_with_macros"
-version = "3.0.0"
+version = "3.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "edc7d5d3932fb12ce722ee5e64dd38c504efba37567f0c402f6ca728c3b8b070"
+checksum = "93634eb5f75a2323b16de4748022ac4297f9e76b6dced2be287a099f41b5e788"
 dependencies = [
 "darling",
 "proc-macro2",
@@ -1790,9 +1945,9 @@ version = "0.1.0"

 [[package]]
 name = "sha1"
-version = "0.10.5"
+version = "0.10.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f04293dc80c3993519f2d7f6f511707ee7094fe0c6d3406feb330cdb3540eba3"
+checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba"
 dependencies = [
 "cfg-if",
 "cpufeatures",
@@ -1910,9 +2065,9 @@ dependencies = [

 [[package]]
 name = "tempfile"
-version = "3.7.1"
+version = "3.8.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dc02fddf48964c42031a0b3fe0428320ecf3a73c401040fc0096f97794310651"
+checksum = "cb94d2f3cc536af71caac6b6fcebf65860b347e7ce0cc9ebe8f70d3e521054ef"
 dependencies = [
 "cfg-if",
 "fastrand 2.0.0",
@@ -1930,6 +2085,16 @@ dependencies = [
 "winapi-util",
 ]

+[[package]]
+name = "terminal_size"
+version = "0.2.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8e6bf6f19e9f8ed8d4048dc22981458ebcf406d67e94cd422e5ecd73d63b3237"
+dependencies = [
+ "rustix 0.37.25",
+ "windows-sys 0.48.0",
+]
+
 [[package]]
 name = "test_infra"
 version = "0.1.0"
@@ -1979,9 +2144,9 @@ checksum = "7cda73e2f1397b1262d6dfdcef8aafae14d1de7748d66822d3bfeeb6d03e5e4b"

 [[package]]
 name = "toml_edit"
-version = "0.19.8"
+version = "0.19.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "239410c8609e8125456927e6707163a3b1fdb40561e4b803bc041f466ccfdc13"
+checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421"
 dependencies = [
 "indexmap",
 "toml_datetime",
@@ -2046,9 +2211,9 @@ dependencies = [

 [[package]]
 name = "typenum"
-version = "1.16.0"
+version = "1.17.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba"
+checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825"

 [[package]]
 name = "uds_windows"
@@ -2066,6 +2231,12 @@ version = "1.0.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "301abaae475aa91687eb82514b328ab47a211a533026cb25fc3e519b86adfc3c"

+[[package]]
+name = "utf8parse"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a"
+
 [[package]]
 name = "uuid"
 version = "1.3.4"
@@ -2188,8 +2359,8 @@ dependencies = [
 name = "vhost_user_block"
 version = "0.1.0"
 dependencies = [
- "argh",
 "block",
+ "clap",
 "env_logger",
 "epoll",
 "libc",
@@ -2207,7 +2378,7 @@ dependencies = [
 name = "vhost_user_net"
 version = "0.1.0"
 dependencies = [
- "argh",
+ "clap",
 "env_logger",
 "epoll",
 "libc",
@@ -2342,10 +2513,11 @@ dependencies = [
 "anyhow",
 "arc-swap",
 "arch",
- "bitflags 2.3.3",
+ "bitflags 2.4.1",
 "block",
 "blocking",
 "cfg-if",
+ "clap",
 "devices",
 "epoll",
 "event_monitor",
@@ -2383,7 +2555,7 @@ dependencies = [
 "vm-virtio",
 "vmm-sys-util",
 "zbus",
- "zerocopy 0.6.1",
+ "zerocopy",
 ]

 [[package]]
@@ -2409,9 +2581,9 @@ dependencies = [

 [[package]]
 name = "waker-fn"
-version = "1.1.0"
+version = "1.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9d5b2c62b4012a3e1eca5a7e077d13b3bf498c4073e33ccd58626607748ceeca"
+checksum = "f3c4517f54858c779bbcbf228f4fca63d121bf85fbecb2dc578cdf4a39395690"

 [[package]]
 name = "wasi"
@@ -2491,9 +2663,9 @@ checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"

 [[package]]
 name = "winapi-util"
-version = "0.1.5"
+version = "0.1.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178"
+checksum = "f29e6f9198ba0d26b4c9f07dbe6f9ed633e1f3d5b8b414090084349e46a52596"
 dependencies = [
 "winapi",
 ]
@@ -2638,9 +2810,9 @@ checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a"

 [[package]]
 name = "winnow"
-version = "0.4.9"
+version = "0.5.18"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "81a2094c43cc94775293eaa0e499fbc30048a6d824ac82c0351a8c0bf9112529"
+checksum = "176b6138793677221d420fd2f0aeeced263f197688b36484660da767bca2fa32"
 dependencies = [
 "memchr",
 ]
@@ -2674,7 +2846,7 @@ dependencies = [
 "byteorder",
 "derivative",
 "enumflags2",
- "event-listener",
+ "event-listener 2.5.3",
 "futures-core",
 "futures-sink",
 "futures-util",
@@ -2723,40 +2895,19 @@ dependencies = [

 [[package]]
 name = "zerocopy"
-version = "0.6.1"
+version = "0.7.21"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "332f188cc1bcf1fe1064b8c58d150f497e697f49774aa846f2dc949d9a25f236"
+checksum = "686b7e407015242119c33dab17b8f61ba6843534de936d94368856528eae4dcc"
 dependencies = [
 "byteorder",
- "zerocopy-derive 0.3.2",
-]
-
-[[package]]
-name = "zerocopy"
-version = "0.7.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2f00a66029e63d181fa590cc5694cf2afbc0974a4604824e80017b1789f99c07"
-dependencies = [
- "byteorder",
- "zerocopy-derive 0.7.1",
+ "zerocopy-derive",
 ]

 [[package]]
 name = "zerocopy-derive"
-version = "0.3.2"
+version = "0.7.21"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6505e6815af7de1746a08f69c69606bb45695a17149517680f3b2149713b19a3"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn 1.0.109",
-]
-
-[[package]]
-name = "zerocopy-derive"
-version = "0.7.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e9c682f46403e5d567cb27b79f6279c145759528ba9450fe371f43b921b452bd"
+checksum = "020f3dfe25dfc38dfea49ce62d5d45ecdd7f0d8a724fa63eb36b6eba4ec76806"
 dependencies = [
 "proc-macro2",
 "quote",
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "cloud-hypervisor"
-version = "35.0.0"
+version = "36.1.0"
 authors = ["The Cloud Hypervisor Authors"]
 edition = "2021"
 default-run = "cloud-hypervisor"
@@ -15,7 +15,7 @@ homepage = "https://github.com/cloud-hypervisor/cloud-hypervisor"
 # a.) A dependency requires it,
 # b.) If we want to use a new feature and that MSRV is at least 6 months old,
 # c.) There is a security issue that is addressed by the toolchain update.
-rust-version = "1.64"
+rust-version = "1.66"

 [profile.release]
 lto = true
@@ -31,7 +31,7 @@ debug = true
 [dependencies]
 anyhow = "1.0.75"
 api_client = { path = "api_client" }
-argh = "0.1.9"
+clap = { version = "4.3.11", features = ["string"] }
 dhat = { version = "0.3.2", optional = true }
 epoll = "4.3.3"
 event_monitor = { path = "event_monitor" }
@@ -39,7 +39,7 @@ hypervisor = { path = "hypervisor" }
 libc = "0.2.147"
 log = { version = "0.4.17", features = ["std"] }
 option_parser = { path = "option_parser" }
-seccompiler = "0.3.0"
+seccompiler = "0.4.0"
 serde_json = "1.0.107"
 signal-hook = "0.3.17"
 thiserror = "1.0.40"
@@ -64,6 +64,8 @@ serde_json = "1.0.107"
 test_infra = { path = "test_infra" }
 wait-timeout = "0.2.0"

+# Please adjust `vmm::feature_list()` accordingly when changing the
+# feature list below
 [features]
 default = ["kvm", "io_uring"]
 dbus_api = ["zbus", "vmm/dbus_api"]
--- a/README.md
+++ b/README.md
@@ -150,7 +150,7 @@ $ sudo setcap cap_net_admin+ep ./cloud-hypervisor
 $ ./create-cloud-init.sh
 $ ./cloud-hypervisor \
 	--kernel ./hypervisor-fw \
-	--disk path=focal-server-cloudimg-amd64.raw --disk path=/tmp/ubuntu-cloudinit.img \
+	--disk path=focal-server-cloudimg-amd64.raw path=/tmp/ubuntu-cloudinit.img \
 	--cpus boot=4 \
 	--memory size=1024M \
 	--net "tap=,mac=,ip=,mask="
@@ -163,7 +163,7 @@ GRUB) is required then it necessary to switch to the serial console instead of
 ```shell
 $ ./cloud-hypervisor \
 	--kernel ./hypervisor-fw \
-	--disk path=focal-server-cloudimg-amd64.raw --disk path=/tmp/ubuntu-cloudinit.img \
+	--disk path=focal-server-cloudimg-amd64.raw path=/tmp/ubuntu-cloudinit.img \
 	--cpus boot=4 \
 	--memory size=1024M \
 	--net "tap=,mac=,ip=,mask=" \
@@ -225,7 +225,7 @@ $ sudo setcap cap_net_admin+ep ./cloud-hypervisor
 $ ./create-cloud-init.sh
 $ ./cloud-hypervisor \
 	--kernel ./linux-cloud-hypervisor/arch/x86/boot/compressed/vmlinux.bin \
-	--disk path=focal-server-cloudimg-amd64.raw --disk path=/tmp/ubuntu-cloudinit.img \
+	--disk path=focal-server-cloudimg-amd64.raw path=/tmp/ubuntu-cloudinit.img \
 	--cmdline "console=hvc0 root=/dev/vda1 rw" \
 	--cpus boot=4 \
 	--memory size=1024M \
@@ -239,7 +239,7 @@ $ sudo setcap cap_net_admin+ep ./cloud-hypervisor
 $ ./create-cloud-init.sh
 $ ./cloud-hypervisor \
 	--kernel ./linux-cloud-hypervisor/arch/arm64/boot/Image \
-	--disk path=focal-server-cloudimg-arm64.raw --disk path=/tmp/ubuntu-cloudinit.img \
+	--disk path=focal-server-cloudimg-arm64.raw path=/tmp/ubuntu-cloudinit.img \
 	--cmdline "console=hvc0 root=/dev/vda1 rw" \
 	--cpus boot=4 \
 	--memory size=1024M \
--- a/arch/src/aarch64/fdt.rs
+++ b/arch/src/aarch64/fdt.rs
@@ -290,10 +290,7 @@ fn create_cpu_nodes(
    fdt.property_u32("#size-cells", 0x0)?;

    let num_cpus = vcpu_mpidr.len();
-    let threads_per_core = vcpu_topology.unwrap_or_default().0;
-    let cores_per_package = vcpu_topology.unwrap_or_default().1;
-    let packages = vcpu_topology.unwrap_or_default().2;
-
+    let (threads_per_core, cores_per_package, packages) = vcpu_topology.unwrap_or((1, 1, 1));
    let max_cpus: u32 = (threads_per_core * cores_per_package * packages).into();

    // Add cache info.
--- a/arch/src/lib.rs
+++ b/arch/src/lib.rs
@@ -92,7 +92,7 @@ pub mod x86_64;
 pub use x86_64::{
    arch_memory_regions, configure_system, configure_vcpu, generate_common_cpuid,
    get_host_cpu_phys_bits, initramfs_load_addr, layout, layout::CMDLINE_MAX_SIZE,
-    layout::CMDLINE_START, regs, CpuidFeatureEntry, EntryPoint, _NSIG,
+    layout::CMDLINE_START, regs, CpuidConfig, CpuidFeatureEntry, EntryPoint, _NSIG,
 };

 /// Safe wrapper for `sysconf(_SC_PAGESIZE)`.
@@ -108,6 +108,7 @@ pub struct NumaNode {
    pub memory_regions: Vec<Arc<GuestRegionMmap>>,
    pub hotplug_regions: Vec<Arc<GuestRegionMmap>>,
    pub cpus: Vec<u8>,
+    pub pci_segments: Vec<u16>,
    pub distances: BTreeMap<u32, u8>,
    pub memory_zones: Vec<String>,
    #[cfg(target_arch = "x86_64")]
--- a/arch/src/x86_64/mod.rs
+++ b/arch/src/x86_64/mod.rs
@@ -37,6 +37,9 @@ const TSC_DEADLINE_TIMER_ECX_BIT: u8 = 24; // tsc deadline timer ecx bit.
 const HYPERVISOR_ECX_BIT: u8 = 31; // Hypervisor ecx bit.
 const MTRR_EDX_BIT: u8 = 12; // Hypervisor ecx bit.
 const INVARIANT_TSC_EDX_BIT: u8 = 8; // Invariant TSC bit on 0x8000_0007 EDX
+const AMX_BF16: u8 = 22; // AMX tile computation on bfloat16 numbers
+const AMX_TILE: u8 = 24; // AMX tile load/store instructions
+const AMX_INT8: u8 = 25; // AMX tile computation on 8-bit integers

 // KVM feature bits
 #[cfg(feature = "tdx")]
@@ -145,6 +148,15 @@ struct BootParamsWrapper(boot_params);
 // SAFETY: BootParamsWrap is a wrapper over `boot_params` (a series of ints).
 unsafe impl ByteValued for BootParamsWrapper {}

+pub struct CpuidConfig {
+    pub sgx_epc_sections: Option<Vec<SgxEpcSection>>,
+    pub phys_bits: u8,
+    pub kvm_hyperv: bool,
+    #[cfg(feature = "tdx")]
+    pub tdx: bool,
+    pub amx: bool,
+}
+
 #[derive(Debug)]
 pub enum Error {
    /// Error writing MP table to memory.
@@ -552,11 +564,7 @@ impl CpuidFeatureEntry {

 pub fn generate_common_cpuid(
    hypervisor: &Arc<dyn hypervisor::Hypervisor>,
-    topology: Option<(u8, u8, u8)>,
-    sgx_epc_sections: Option<Vec<SgxEpcSection>>,
-    phys_bits: u8,
-    kvm_hyperv: bool,
-    #[cfg(feature = "tdx")] tdx_enabled: bool,
+    config: &CpuidConfig,
 ) -> super::Result<Vec<CpuIdEntry>> {
    // SAFETY: cpuid called with valid leaves
    if unsafe { x86_64::__cpuid(1) }.ecx & 1 << HYPERVISOR_ECX_BIT == 1 << HYPERVISOR_ECX_BIT {
@@ -574,7 +582,10 @@ pub fn generate_common_cpuid(
        );
    }

-    info!("Generating guest CPUID for with physical address size: {phys_bits}");
+    info!(
+        "Generating guest CPUID for with physical address size: {}",
+        config.phys_bits
+    );
    let cpuid_patches = vec![
        // Patch tsc deadline timer bit
        CpuidPatch {
@@ -615,16 +626,12 @@ pub fn generate_common_cpuid(

    CpuidPatch::patch_cpuid(&mut cpuid, cpuid_patches);

-    if let Some(t) = topology {
-        update_cpuid_topology(&mut cpuid, t.0, t.1, t.2);
-    }
-
-    if let Some(sgx_epc_sections) = sgx_epc_sections {
+    if let Some(sgx_epc_sections) = &config.sgx_epc_sections {
        update_cpuid_sgx(&mut cpuid, sgx_epc_sections)?;
    }

    #[cfg(feature = "tdx")]
-    let tdx_capabilities = if tdx_enabled {
+    let tdx_capabilities = if config.tdx {
        let caps = hypervisor
            .tdx_capabilities()
            .map_err(Error::TdxCapabilities)?;
@@ -637,6 +644,12 @@ pub fn generate_common_cpuid(
    // Update some existing CPUID
    for entry in cpuid.as_mut_slice().iter_mut() {
        match entry.function {
+            // Clear AMX related bits if the AMX feature is not enabled
+            0x7 => {
+                if !config.amx && entry.index == 0 {
+                    entry.edx &= !(1 << AMX_BF16 | 1 << AMX_TILE | 1 << AMX_INT8)
+                }
+            }
            0xd =>
            {
                #[cfg(feature = "tdx")]
@@ -672,12 +685,12 @@ pub fn generate_common_cpuid(
            }
            // Set CPU physical bits
            0x8000_0008 => {
-                entry.eax = (entry.eax & 0xffff_ff00) | (phys_bits as u32 & 0xff);
+                entry.eax = (entry.eax & 0xffff_ff00) | (config.phys_bits as u32 & 0xff);
            }
            0x4000_0001 => {
                // These features are not supported by TDX
                #[cfg(feature = "tdx")]
-                if tdx_enabled {
+                if config.tdx {
                    entry.eax &= !(1 << KVM_FEATURE_CLOCKSOURCE_BIT
                        | 1 << KVM_FEATURE_CLOCKSOURCE2_BIT
                        | 1 << KVM_FEATURE_CLOCKSOURCE_STABLE_BIT
@@ -705,7 +718,7 @@ pub fn generate_common_cpuid(
        });
    }

-    if kvm_hyperv {
+    if config.kvm_hyperv {
        // Remove conflicting entries
        cpuid.retain(|c| c.function != 0x4000_0000);
        cpuid.retain(|c| c.function != 0x4000_0001);
@@ -761,11 +774,26 @@ pub fn configure_vcpu(
    boot_setup: Option<(EntryPoint, &GuestMemoryAtomic<GuestMemoryMmap>)>,
    cpuid: Vec<CpuIdEntry>,
    kvm_hyperv: bool,
+    cpu_vendor: CpuVendor,
+    topology: Option<(u8, u8, u8)>,
 ) -> super::Result<()> {
    // Per vCPU CPUID changes; common are handled via generate_common_cpuid()
    let mut cpuid = cpuid;
    CpuidPatch::set_cpuid_reg(&mut cpuid, 0xb, None, CpuidReg::EDX, u32::from(id));
    CpuidPatch::set_cpuid_reg(&mut cpuid, 0x1f, None, CpuidReg::EDX, u32::from(id));
+    if matches!(cpu_vendor, CpuVendor::AMD) {
+        CpuidPatch::set_cpuid_reg(
+            &mut cpuid,
+            0x8000_001e,
+            Some(0),
+            CpuidReg::EAX,
+            u32::from(id),
+        );
+    }
+
+    if let Some(t) = topology {
+        update_cpuid_topology(&mut cpuid, t.0, t.1, t.2, cpu_vendor, id);
+    }

    // Set ApicId in cpuid for each vcpu
    // SAFETY: get host cpuid when eax=1
@@ -1160,6 +1188,8 @@ fn update_cpuid_topology(
    threads_per_core: u8,
    cores_per_die: u8,
    dies_per_package: u8,
+    cpu_vendor: CpuVendor,
+    id: u8,
 ) {
    let thread_width = 8 - (threads_per_core - 1).leading_zeros();
    let core_width = (8 - (cores_per_die - 1).leading_zeros()) + thread_width;
@@ -1216,13 +1246,72 @@ fn update_cpuid_topology(
        u32::from(dies_per_package * cores_per_die * threads_per_core),
    );
    CpuidPatch::set_cpuid_reg(cpuid, 0x1f, Some(2), CpuidReg::ECX, 5 << 8);
+
+    if matches!(cpu_vendor, CpuVendor::AMD) {
+        CpuidPatch::set_cpuid_reg(
+            cpuid,
+            0x8000_001e,
+            Some(0),
+            CpuidReg::EBX,
+            ((threads_per_core as u32 - 1) << 8) | (id as u32 & 0xff),
+        );
+        CpuidPatch::set_cpuid_reg(
+            cpuid,
+            0x8000_001e,
+            Some(0),
+            CpuidReg::ECX,
+            ((dies_per_package as u32 - 1) << 8) | (thread_width + die_width) & 0xff,
+        );
+        CpuidPatch::set_cpuid_reg(cpuid, 0x8000_001e, Some(0), CpuidReg::EDX, 0);
+        if cores_per_die * threads_per_core > 1 {
+            CpuidPatch::set_cpuid_reg(
+                cpuid,
+                0x8000_0001,
+                Some(0),
+                CpuidReg::ECX,
+                (1u32 << 1) | (1u32 << 22),
+            );
+            CpuidPatch::set_cpuid_reg(
+                cpuid,
+                0x0000_0001,
+                Some(0),
+                CpuidReg::EBX,
+                ((id as u32) << 24)
+                    | (8 << 8)
+                    | (((cores_per_die * threads_per_core) as u32) << 16),
+            );
+            let cpuid_patches = vec![
+                // Patch tsc deadline timer bit
+                CpuidPatch {
+                    function: 1,
+                    index: 0,
+                    flags_bit: None,
+                    eax_bit: None,
+                    ebx_bit: None,
+                    ecx_bit: None,
+                    edx_bit: Some(28),
+                },
+            ];
+            CpuidPatch::patch_cpuid(cpuid, cpuid_patches);
+            CpuidPatch::set_cpuid_reg(
+                cpuid,
+                0x8000_0008,
+                Some(0),
+                CpuidReg::ECX,
+                ((thread_width + core_width + die_width) << 12)
+                    | ((cores_per_die * threads_per_core) - 1) as u32,
+            );
+        } else {
+            CpuidPatch::set_cpuid_reg(cpuid, 0x8000_0008, Some(0), CpuidReg::ECX, 0u32);
+        }
+    }
 }

 // The goal is to update the CPUID sub-leaves to reflect the number of EPC
 // sections exposed to the guest.
 fn update_cpuid_sgx(
    cpuid: &mut Vec<CpuIdEntry>,
-    epc_sections: Vec<SgxEpcSection>,
+    epc_sections: &[SgxEpcSection],
 ) -> Result<(), Error> {
    // Something's wrong if there's no EPC section.
    if epc_sections.is_empty() {
--- a/block/src/lib.rs
+++ b/block/src/lib.rs
@@ -24,6 +24,7 @@ pub mod qcow_sync;
 ///
 /// Enabled with the `"io_uring"` feature
 pub mod raw_async;
+pub mod raw_async_aio;
 pub mod raw_sync;
 pub mod vhd;
 pub mod vhdx;
@@ -61,6 +62,7 @@ use vm_memory::{
    GuestMemoryError, GuestMemoryLoadGuard,
 };
 use vm_virtio::{AccessPlatform, Translatable};
+use vmm_sys_util::aio;
 use vmm_sys_util::eventfd::EventFd;
 use vmm_sys_util::{ioctl_io_nr, ioctl_ioc_nr};

@@ -565,6 +567,11 @@ unsafe impl ByteValued for VirtioBlockConfig {}
 // SAFETY: data structure only contain a series of integers
 unsafe impl ByteValued for VirtioBlockGeometry {}

+/// Check if aio can be used on the current system.
+pub fn block_aio_is_supported() -> bool {
+    aio::IoContext::new(1).is_ok()
+}
+
 /// Check if io_uring for block device can be used on the current system, as
 /// it correctly supports the expected io_uring features.
 pub fn block_io_uring_is_supported() -> bool {
--- a/block/src/raw_async_aio.rs
+++ b/block/src/raw_async_aio.rs
@@ -0,0 +1,150 @@
+// Copyright © 2023 Intel Corporation
+//
+// SPDX-License-Identifier: Apache-2.0 AND BSD-3-Clause
+//
+// Copyright © 2023 Crusoe Energy Systems LLC
+//
+
+use crate::async_io::{
+    AsyncIo, AsyncIoError, AsyncIoResult, DiskFile, DiskFileError, DiskFileResult,
+};
+use crate::DiskTopology;
+use std::fs::File;
+use std::io::{Seek, SeekFrom};
+use std::os::unix::io::{AsRawFd, RawFd};
+use vmm_sys_util::aio;
+use vmm_sys_util::eventfd::EventFd;
+
+pub struct RawFileDiskAio {
+    file: File,
+}
+
+impl RawFileDiskAio {
+    pub fn new(file: File) -> Self {
+        RawFileDiskAio { file }
+    }
+}
+
+impl DiskFile for RawFileDiskAio {
+    fn size(&mut self) -> DiskFileResult<u64> {
+        self.file
+            .seek(SeekFrom::End(0))
+            .map_err(DiskFileError::Size)
+    }
+
+    fn new_async_io(&self, ring_depth: u32) -> DiskFileResult<Box<dyn AsyncIo>> {
+        Ok(Box::new(
+            RawFileAsyncAio::new(self.file.as_raw_fd(), ring_depth)
+                .map_err(DiskFileError::NewAsyncIo)?,
+        ) as Box<dyn AsyncIo>)
+    }
+
+    fn topology(&mut self) -> DiskTopology {
+        if let Ok(topology) = DiskTopology::probe(&self.file) {
+            topology
+        } else {
+            warn!("Unable to get device topology. Using default topology");
+            DiskTopology::default()
+        }
+    }
+}
+
+pub struct RawFileAsyncAio {
+    fd: RawFd,
+    ctx: aio::IoContext,
+    eventfd: EventFd,
+}
+
+impl RawFileAsyncAio {
+    pub fn new(fd: RawFd, queue_depth: u32) -> std::io::Result<Self> {
+        let eventfd = EventFd::new(libc::EFD_NONBLOCK)?;
+        let ctx = aio::IoContext::new(queue_depth)?;
+
+        Ok(RawFileAsyncAio { fd, ctx, eventfd })
+    }
+}
+
+impl AsyncIo for RawFileAsyncAio {
+    fn notifier(&self) -> &EventFd {
+        &self.eventfd
+    }
+
+    fn read_vectored(
+        &mut self,
+        offset: libc::off_t,
+        iovecs: &[libc::iovec],
+        user_data: u64,
+    ) -> AsyncIoResult<()> {
+        let iocbs = [&mut aio::IoControlBlock {
+            aio_fildes: self.fd.as_raw_fd() as u32,
+            aio_lio_opcode: aio::IOCB_CMD_PREADV as u16,
+            aio_buf: iovecs.as_ptr() as u64,
+            aio_nbytes: iovecs.len() as u64,
+            aio_offset: offset,
+            aio_data: user_data,
+            aio_flags: aio::IOCB_FLAG_RESFD,
+            aio_resfd: self.eventfd.as_raw_fd() as u32,
+            ..Default::default()
+        }];
+        let _ = self
+            .ctx
+            .submit(&iocbs[..])
+            .map_err(AsyncIoError::ReadVectored)?;
+
+        Ok(())
+    }
+
+    fn write_vectored(
+        &mut self,
+        offset: libc::off_t,
+        iovecs: &[libc::iovec],
+        user_data: u64,
+    ) -> AsyncIoResult<()> {
+        let iocbs = [&mut aio::IoControlBlock {
+            aio_fildes: self.fd.as_raw_fd() as u32,
+            aio_lio_opcode: aio::IOCB_CMD_PWRITEV as u16,
+            aio_buf: iovecs.as_ptr() as u64,
+            aio_nbytes: iovecs.len() as u64,
+            aio_offset: offset,
+            aio_data: user_data,
+            aio_flags: aio::IOCB_FLAG_RESFD,
+            aio_resfd: self.eventfd.as_raw_fd() as u32,
+            ..Default::default()
+        }];
+        let _ = self
+            .ctx
+            .submit(&iocbs[..])
+            .map_err(AsyncIoError::WriteVectored)?;
+
+        Ok(())
+    }
+
+    fn fsync(&mut self, user_data: Option<u64>) -> AsyncIoResult<()> {
+        if let Some(user_data) = user_data {
+            let iocbs = [&mut aio::IoControlBlock {
+                aio_fildes: self.fd.as_raw_fd() as u32,
+                aio_lio_opcode: aio::IOCB_CMD_FSYNC as u16,
+                aio_data: user_data,
+                aio_flags: aio::IOCB_FLAG_RESFD,
+                aio_resfd: self.eventfd.as_raw_fd() as u32,
+                ..Default::default()
+            }];
+            let _ = self.ctx.submit(&iocbs[..]).map_err(AsyncIoError::Fsync)?;
+        } else {
+            // SAFETY: FFI call with a valid fd
+            unsafe { libc::fsync(self.fd) };
+        }
+
+        Ok(())
+    }
+
+    fn next_completed_request(&mut self) -> Option<(u64, i32)> {
+        let mut events: [aio::IoEvent; 1] = [aio::IoEvent::default()];
+        let rc = self.ctx.get_events(0, &mut events, None).unwrap();
+        if rc == 0 {
+            None
+        } else {
+            Some((events[0].data, events[0].res as i32))
+        }
+    }
+}
--- a/devices/Cargo.toml
+++ b/devices/Cargo.toml
@@ -8,7 +8,7 @@ edition = "2021"
 acpi_tables = { git = "https://github.com/rust-vmm/acpi_tables", branch = "main"  }
 anyhow = "1.0.75"
 arch = { path = "../arch" }
-bitflags = "2.3.3"
+bitflags = "2.4.1"
 byteorder = "1.4.3"
 event_monitor = { path = "../event_monitor" }
 hypervisor = { path = "../hypervisor" }
--- a/devices/src/legacy/serial.rs
+++ b/devices/src/legacy/serial.rs
@@ -166,8 +166,8 @@ impl Serial {
        Self::new(id, interrupt, None, state)
    }

-    pub fn set_out(&mut self, out: Box<dyn io::Write + Send>) {
-        self.out = Some(out);
+    pub fn set_out(&mut self, out: Option<Box<dyn io::Write + Send>>) {
+        self.out = out;
    }

    /// Queues raw bytes for the guest to read and signals the interrupt if the line status would
--- a/devices/src/legacy/uart_pl011.rs
+++ b/devices/src/legacy/uart_pl011.rs
@@ -201,8 +201,8 @@ impl Pl011 {
        }
    }

-    pub fn set_out(&mut self, out: Box<dyn io::Write + Send>) {
-        self.out = Some(out);
+    pub fn set_out(&mut self, out: Option<Box<dyn io::Write + Send>>) {
+        self.out = out;
    }

    fn state(&self) -> Pl011State {
--- a/devices/src/pvpanic.rs
+++ b/devices/src/pvpanic.rs
@@ -26,6 +26,7 @@ const PVPANIC_VENDOR_ID: u16 = 0x1b36;
 const PVPANIC_DEVICE_ID: u16 = 0x0011;

 pub const PVPANIC_DEVICE_MMIO_SIZE: u64 = 0x2;
+pub const PVPANIC_DEVICE_MMIO_ALIGNMENT: u64 = 0x10;

 const PVPANIC_PANICKED: u8 = 1 << 0;
 const PVPANIC_CRASH_LOADED: u8 = 1 << 1;
@@ -192,7 +193,7 @@ impl PciDevice for PvPanicDevice {
        let bar_addr = allocator
            .lock()
            .unwrap()
-            .allocate_mmio_hole_addresses(None, region_size, None)
+            .allocate_mmio_hole_addresses(None, region_size, Some(PVPANIC_DEVICE_MMIO_ALIGNMENT))
            .ok_or(PciDeviceError::IoAllocationFailed(region_size))?;

        let bar = PciBarConfiguration::default()
--- a/docs/api.md
+++ b/docs/api.md
@@ -295,7 +295,7 @@ From the CLI, one can:
 The REST API, D-Bus API and the CLI all rely on a common, [internal API](#internal-api).

 The CLI options are parsed by the
-[argh crate](https://docs.rs/argh/latest/argh/) and then translated into
+[clap crate](https://docs.rs/clap/4.3.11/clap/) and then translated into
 [internal API](#internal-api) commands.

 The REST API is processed by an HTTP thread using the
@@ -327,7 +327,7 @@ As a summary, the REST API, the D-Bus API and the CLI are essentially frontends
                       |                                      |      +------------------------+
                       |            +----------+              |                 VMM
                       |     CLI    |          |              |
-                       +----------->+   argh   +--------------+
+                       +----------->+   clap   +--------------+
                                    |          |
                                    +----------+

--- a/docs/balloon.md
+++ b/docs/balloon.md
@@ -54,7 +54,7 @@ Value is a boolean set to `off` by default.
 _Example_

 ```
--ballloon size=2G,deflate_on_oom=on
+--balloon size=2G,deflate_on_oom=on
 ```

 ### `free_page_reporting`
@@ -72,5 +72,5 @@ Value is a boolean set to `off` by default.
 _Example_

 ```
--ballloon size=0,free_page_reporting=on
+--balloon size=0,free_page_reporting=on
 ```
--- a/docs/coverage.md
+++ b/docs/coverage.md
@@ -0,0 +1,81 @@
+# Code coverage
+
+LLVM provides a set of tools to collect code coverage data and present the data
+in human-consumable forms.
+
+## Building a suitable binary
+
+The compiler flag to generate code coverage data has been stabilized since Rust
+1.60.
+
+An instrumented binary can be built with the following command:
+
+```shell
+cargo clean && RUSTFLAGS='-C instrument-coverage' cargo build
+```
+
+Using either `debug` or `release` profile is fine. You will need to adjust
+the path for some commands.
+
+## Running the binary
+
+Run the binary as you normally would. When the process exits, you will see
+files with the prefix `profraw`.
+
+Multiple runs of the same binary will produce multiple `profraw` files.
+
+The more diverse the runs are, the better. Try to exercise different features
+as much as possible.
+
+## Combining raw data
+
+Raw data files can be combined with `llvm-profdata`.
+
+```shell
+rustup component add llvm-tools-preview
+# Assuming profraw files reside in the current directory and its children directories
+find . -name '*.profraw' -exec llvm-profdata merge -sparse {} -o coverage.profdata \;
+```
+
+A file named `coverage.profdata` will be generated.
+
+## Generating HTML files for human consumption
+
+This can be done either with LLVM or `grcov`.
+
+Here is an example using grcov.
+
+```shell
+cargo install grcov
+# Assuming the profdata file is in the top level directory of the Cloud Hypervisor repository
+grcov . --binary-path ./target/x86_64-unknown-linux-gnu/release -s . -t html --branch --ignore-not-existing -o coverage-html-output/
+```
+
+You can then open the `index.html` file under coverage-html-output to see the
+results.
+
+## Notes on running the in-tree integration tests and unit tests
+
+Please set RUSTFLAGS the same way while invoking `dev_cli.sh`. The script will
+pass RUSTFLAGS to the container.
+
+Since the `profraw` files are generated from within the container, the file
+paths embedded in the data files are going to be different. It is easier to do
+the data processing from within the container if you don't want to fight the
+tool chain.
+
+```shell
+# Get a shell
+./scripts/dev_cli.sh shell
+
+# Install llvm-tools-preview for llvm-profdata
+rustup component add llvm-tools-preview
+# Merge data files by using the following command
+find . -name '*.profraw' -exec `rustc --print sysroot`/lib/rustlib/x86_64-unknown-linux-gnu/bin/llvm-profdata merge -sparse {} -o coverage.profdata \;
+
+# As of writing, the container has Rust 1.67.1. It is too old for grcov.
+rustup install stable
+cargo +stable install grcov
+# Run grcov as usual
+grcov . --binary-path ./target/x86_64-unknown-linux-gnu/release -s . -t html --branch --ignore-not-existing -o coverage-html-output/
+```
--- a/docs/debug-port.md
+++ b/docs/debug-port.md
@@ -36,7 +36,7 @@ Assuming parts of the guest software stack have been instrumented to use the
 `cloud-hypervisor` debug I/O port, we may want to gather the related logs.

 To do so we need to start `cloud-hypervisor` with the right debug level
-(`-v -v -v`). It is also recommended to have it log into a dedicated file in order
+(`-vvv`). It is also recommended to have it log into a dedicated file in order
 to easily grep for the tracing logs (e.g.
 `--log-file /tmp/cloud-hypervisor.log`):

@@ -48,7 +48,7 @@ to easily grep for the tracing logs (e.g.
    --memory size=1024M \
    --rng \
    --log-file /tmp/ch-fw.log \
-    -v -v -v
+    -vvv
 ```

 After booting the guest, we then have to grep for the debug I/O port traces in
--- a/docs/gdb.md
+++ b/docs/gdb.md
@@ -8,7 +8,7 @@ To enable debugging with GDB, build with the `guest_debug` feature enabled:
 cargo build --features guest_debug
 ```

-To use the `--gdb` option, specify the Unix Domain Socket with `path` that Cloud Hypervisor will use to communicate with the host's GDB:
+To use the `--gdb` option, specify the Unix Domain Socket with `--path` that Cloud Hypervisor will use to communicate with the host's GDB:

 ```bash
 ./cloud-hypervisor \
--- a/docs/hotplug.md
+++ b/docs/hotplug.md
@@ -27,16 +27,16 @@ $ ./cloud-hypervisor/target/release/cloud-hypervisor \
 	--memory size=1024M \
 	--net "tap=,mac=,ip=,mask=" \
 	--rng \
-	--api-socket /tmp/ch-socket
+	--api-socket=/tmp/ch-socket
 $ popd
 ```

-Notice the addition of `--api-socket /tmp/ch-socket` and a `max` parameter on `--cpus boot=4,max=8`.
+Notice the addition of `--api-socket=/tmp/ch-socket` and a `max` parameter on `--cpus boot=4,max=8`.

 To ask the VMM to add additional vCPUs then use the resize API:

 ```shell
-./ch-remote --api-socket /tmp/ch-socket resize --cpus 8
+./ch-remote --api-socket=/tmp/ch-socket resize --cpus 8
 ```

 The extra vCPU threads will be created and advertised to the running kernel. The kernel does not bring up the CPUs immediately and instead the user must "online" them from inside the VM:
@@ -56,7 +56,7 @@ After a reboot the added CPUs will remain.
 Removing CPUs works similarly by reducing the number in the "desired_vcpus" field of the reisze API. The CPUs will be automatically offlined inside the guest so there is no need to run any commands inside the guest:

 ```shell
-./ch-remote --api-socket /tmp/ch-socket resize --cpus 2
+./ch-remote --api-socket=/tmp/ch-socket resize --cpus 2
 ```

 As per adding CPUs to the guest, after a reboot the VM will be running with the reduced number of vCPUs.
@@ -85,7 +85,7 @@ $ ./cloud-hypervisor/target/release/cloud-hypervisor \
 	--memory size=1024M,hotplug_size=8192M \
 	--net "tap=,mac=,ip=,mask=" \
 	--rng \
-	--api-socket /tmp/ch-socket
+	--api-socket=/tmp/ch-socket
 $ popd
 ```

@@ -98,7 +98,7 @@ root@ch-guest ~ # echo online | sudo tee /sys/devices/system/memory/auto_online_
 To ask the VMM to expand the RAM for the VM:

 ```shell
-./ch-remote --api-socket /tmp/ch-socket resize --memory 3G
+./ch-remote --api-socket=/tmp/ch-socket resize --memory 3G
 ```

 The new memory is now available to use inside the VM:
@@ -134,14 +134,14 @@ $ ./cloud-hypervisor/target/release/cloud-hypervisor \
 	--disk path=focal-server-cloudimg-amd64.raw \
 	--memory size=1024M,hotplug_size=8192M,hotplug_method=virtio-mem \
 	--net "tap=,mac=,ip=,mask=" \
-	--api-socket /tmp/ch-socket
+	--api-socket=/tmp/ch-socket
 $ popd
 ```

 To ask the VMM to expand the RAM for the VM (request is in bytes):

 ```shell
-./ch-remote --api-socket /tmp/ch-socket resize --memory 3G
+./ch-remote --api-socket=/tmp/ch-socket resize --memory 3G
 ```

 The new memory is now available to use inside the VM:
@@ -172,17 +172,17 @@ $ ./cloud-hypervisor/target/release/cloud-hypervisor \
 	--cpus boot=4 \
 	--memory size=1024M \
 	--net "tap=,mac=,ip=,mask=" \
-	--api-socket /tmp/ch-socket
+	--api-socket=/tmp/ch-socket
 ```

-Notice the addition of `--api-socket /tmp/ch-socket`.
+Notice the addition of `--api-socket=/tmp/ch-socket`.

 ### Add VFIO Device

 To ask the VMM to add additional VFIO device then use the `add-device` API.

 ```shell
-./ch-remote --api-socket /tmp/ch-socket add-device path=/sys/bus/pci/devices/0000:01:00.0/
+./ch-remote --api-socket=/tmp/ch-socket add-device path=/sys/bus/pci/devices/0000:01:00.0/
 ```

 ### Add Disk Device
@@ -190,7 +190,7 @@ To ask the VMM to add additional VFIO device then use the `add-device` API.
 To ask the VMM to add additional disk device then use the `add-disk` API.

 ```shell
-./ch-remote --api-socket /tmp/ch-socket add-disk path=/foo/bar/cloud.img
+./ch-remote --api-socket=/tmp/ch-socket add-disk path=/foo/bar/cloud.img
 ```

 ### Add Fs Device
@@ -198,7 +198,7 @@ To ask the VMM to add additional disk device then use the `add-disk` API.
 To ask the VMM to add additional fs device then use the `add-fs` API.

 ```shell
-./ch-remote --api-socket /tmp/ch-socket add-fs tag=myfs,socket=/foo/bar/virtiofs.sock
+./ch-remote --api-socket=/tmp/ch-socket add-fs tag=myfs,socket=/foo/bar/virtiofs.sock
 ```

 ### Add Net Device
@@ -206,7 +206,7 @@ To ask the VMM to add additional fs device then use the `add-fs` API.
 To ask the VMM to add additional network device then use the `add-net` API.

 ```shell
-./ch-remote --api-socket /tmp/ch-socket add-net tap=chtap0
+./ch-remote --api-socket=/tmp/ch-socket add-net tap=chtap0
 ```

 ### Add Pmem Device
@@ -214,7 +214,7 @@ To ask the VMM to add additional network device then use the `add-net` API.
 To ask the VMM to add additional PMEM device then use the `add-pmem` API.

 ```shell
-./ch-remote --api-socket /tmp/ch-socket add-pmem file=/foo/bar.cloud.img
+./ch-remote --api-socket=/tmp/ch-socket add-pmem file=/foo/bar.cloud.img
 ```

 ### Add Vsock Device
@@ -222,7 +222,7 @@ To ask the VMM to add additional PMEM device then use the `add-pmem` API.
 To ask the VMM to add additional vsock device then use the `add-vsock` API.

 ```shell
-./ch-remote --api-socket /tmp/ch-socket add-vsock cid=3,socket=/foo/bar/vsock.sock
+./ch-remote --api-socket=/tmp/ch-socket add-vsock cid=3,socket=/foo/bar/vsock.sock
 ```

 ### Common Across All PCI Devices
@@ -244,7 +244,7 @@ After a reboot the added PCI device will remain.
 Removing a PCI device works the same way for all kind of PCI devices. The unique identifier related to the device must be provided. This identifier can be provided by the user when adding the new device, or by default Cloud Hypervisor will assign one.

 ```shell
-./ch-remote --api-socket /tmp/ch-socket remove-device _disk0
+./ch-remote --api-socket=/tmp/ch-socket remove-device _disk0
 ```

 As per adding a PCI device to the guest, after a reboot the VM will be running without the removed PCI device.
--- a/docs/iommu.md
+++ b/docs/iommu.md
@@ -245,7 +245,7 @@ e.g.

 ```bash
 ./cloud-hypervisor \
-    --api-socket /tmp/api \
+    --api-socket=/tmp/api \
    --cpus boot=1 \
    --memory size=4G,hugepages=on \
    --disk path=focal-server-cloudimg-amd64.raw \
@@ -260,7 +260,7 @@ requiring the IOMMU then may be hotplugged:
 e.g.

 ```bash
-./ch-remote --api-socket /tmp/api add-device path=/sys/bus/pci/devices/0000:00:04.0,iommu=on,pci_segment=1
+./ch-remote --api-socket=/tmp/api add-device path=/sys/bus/pci/devices/0000:00:04.0,iommu=on,pci_segment=1
 ```

 Devices that cannot be placed behind an IOMMU (e.g. lacking an `iommu=` option)
--- a/docs/live_migration.md
+++ b/docs/live_migration.md
@@ -16,22 +16,22 @@ $ target/release/cloud-hypervisor
    --disk path=~/workloads/focal.raw \
    --cpus boot=1 --memory size=1G,shared=on \
    --cmdline "root=/dev/vda1 console=ttyS0"  \
-    --serial tty --console off --api-socket /tmp/api1
+    --serial tty --console off --api-socket=/tmp/api1
 ```

 Launch the destination VM from the same directory (on the host machine):
 ```bash
-$ target/release/cloud-hypervisor --api-socket /tmp/api2
+$ target/release/cloud-hypervisor --api-socket=/tmp/api2
 ```

 Get ready for receiving migration for the destination VM (on the host machine):
 ```bash
-$ target/release/ch-remote --api-socket /tmp/api2 receive-migration unix:/tmp/sock
+$ target/release/ch-remote --api-socket=/tmp/api2 receive-migration unix:/tmp/sock
 ```

 Start to send migration for the source VM (on the host machine):
 ```bash
-$ target/release/ch-remote --api-socket /tmp/api1 send-migration --local unix:/tmp/sock
+$ target/release/ch-remote --api-socket=/tmp/api1 send-migration --local unix:/tmp/sock
 ```

 When the above commands completed, the source VM should be successfully
@@ -51,7 +51,7 @@ $ sudo /target/release/cloud-hypervisor \
        --cpus boot=1 --memory size=512M \
        --kernel vmlinux \
        --cmdline "root=/dev/vda1 console=ttyS0"  \
-        --disk path=focal-1.raw path=focal-nested.raw --disk path=tmp.img\
+        --disk path=focal-1.raw path=focal-nested.raw path=tmp.img\
        --net ip=192.168.101.1
 ```

@@ -63,7 +63,7 @@ $ sudo /target/release/cloud-hypervisor \
        --cpus boot=1 --memory size=512M \
        --kernel vmlinux \
        --cmdline "root=/dev/vda1 console=ttyS0"  \
-        --disk path=focal-2.raw path=focal-nested.raw --disk path=tmp.img\
+        --disk path=focal-2.raw path=focal-nested.raw path=tmp.img\
        --net ip=192.168.102.1
 ```

@@ -74,8 +74,8 @@ vm-1:~$ sudo ./cloud-hypervisor \
        --memory size=128M \
        --kernel vmlinux \
        --cmdline "console=ttyS0 root=/dev/vda1" \
-        --disk path=/dev/vdb --disk path=/dev/vdc \
-        --api-socket /tmp/api1 \
+        --disk path=/dev/vdb path=/dev/vdc \
+        --api-socket=/tmp/api1 \
        --net ip=192.168.100.1
 vm-1:~$ # setup the guest network if needed
 vm-1:~$ sudo ip addr add 192.168.101.2/24 dev ens4
@@ -108,7 +108,7 @@ echo "tmp = $tmp"

 Launch the nested destination VM (inside the guest OS of the VM 2):
 ```bash
-vm-2:~$ sudo ./cloud-hypervisor --api-socket /tmp/api2
+vm-2:~$ sudo ./cloud-hypervisor --api-socket=/tmp/api2
 vm-2:~$ # setup the guest network with the following commands if needed
 vm-2:~$ sudo ip addr add 192.168.102.2/24 dev ens4
 vm-2:~$ sudo ip link set up dev ens4
@@ -122,7 +122,7 @@ vm-2:~$ ping 192.168.101.2 # This should succeed
 Get ready for receiving migration for the nested destination VM (inside
 the guest OS of the VM 2):
 ```bash
-vm-2:~$ sudo ./ch-remote --api-socket /tmp/api2 receive-migration unix:/tmp/sock2
+vm-2:~$ sudo ./ch-remote --api-socket=/tmp/api2 receive-migration unix:/tmp/sock2
 vm-2:~$ sudo socat TCP-LISTEN:6000,reuseaddr UNIX-CLIENT:/tmp/sock2
 ```

@@ -130,7 +130,7 @@ Start to send migration for the nested source VM (inside the guest OS of
 the VM 1):
 ```bash
 vm-1:~$ sudo socat UNIX-LISTEN:/tmp/sock1,reuseaddr TCP:192.168.102.2:6000
-vm-1:~$ sudo ./ch-remote --api-socket /tmp/api1 send-migration unix:/tmp/sock1
+vm-1:~$ sudo ./ch-remote --api-socket=/tmp/api1 send-migration unix:/tmp/sock1
 ```

 When the above commands completed, the source VM should be successfully
--- a/docs/logging.md
+++ b/docs/logging.md
@@ -38,6 +38,6 @@ This level is for the benefit of developers. It should be used for sporadic and

 ### `debug!()`

-Use `-v -v` to enable.
+Use `-vv` to enable.

-For the most verbose of logging messages. It is acceptable to "spam" the log with repeated invocations of the same message. This level of logging would be combined with `--log-file`.
+For the most verbose of logging messages. It is acceptable to "spam" the log with repeated invocations of the same message. This level of logging would be combined with `--log-file`.
--- a/docs/memory.md
+++ b/docs/memory.md
@@ -516,7 +516,7 @@ different distances, it can be described with the following example.
 _Example_

 ```
--numa guest_numa_id=0,distances=[1@15,2@25] --numa guest_numa_id=1,distances=[0@15,2@20] guest_numa_id=2,distances=[0@25,1@20]
+--numa guest_numa_id=0,distances=[1@15,2@25] guest_numa_id=1,distances=[0@15,2@20] guest_numa_id=2,distances=[0@25,1@20]
 ```

 ### `memory_zones`
@@ -540,14 +540,14 @@ demarcate the list.

 Note that a memory zone must belong to a single NUMA node. The following
 configuration is incorrect, therefore not allowed:
-`--numa guest_numa_id=0,memory_zones=mem0 --numa guest_numa_id=1,memory_zones=mem0`
+`--numa guest_numa_id=0,memory_zones=mem0 guest_numa_id=1,memory_zones=mem0`

 _Example_

 ```
 --memory size=0
--memory-zone id=mem0,size=1G id=mem1,size=1G --memory-zone id=mem2,size=1G
--numa guest_numa_id=0,memory_zones=[mem0,mem2] --numa guest_numa_id=1,memory_zones=mem1
+--memory-zone id=mem0,size=1G id=mem1,size=1G id=mem2,size=1G
+--numa guest_numa_id=0,memory_zones=[mem0,mem2] guest_numa_id=1,memory_zones=mem1
 ```

 ### `sgx_epc_sections`
@@ -567,12 +567,21 @@ _Example_

 ```
 --sgx-epc id=epc0,size=32M id=epc1,size=64M id=epc2,size=32M
--numa guest_numa_id=0,sgx_epc_sections=epc1 --numa guest_numa_id=1,sgx_epc_sections=[epc0,epc2]
+--numa guest_numa_id=0,sgx_epc_sections=epc1 guest_numa_id=1,sgx_epc_sections=[epc0,epc2]
 ```

 ### PCI bus

-Cloud Hypervisor supports only one PCI bus, which is why it has been tied to
-the NUMA node 0 by default. It is the user responsibility to organize the NUMA
-nodes correctly so that vCPUs and guest RAM which should be located on the same
-NUMA node as the PCI bus end up on the NUMA node 0.
+Cloud Hypervisor supports guests with one or more PCI segments. The default PCI segment always
+has affinity to NUMA node 0. Be default, all other PCI segments have afffinity to NUMA node 0.
+The user may configure the NUMA affinity for any additional PCI segments.
+
+_Example_
+
+```
+--platform num_pci_segments=2
+--memory-zone size=16G,host_numa_node=0,id=mem0
+--memory-zone size=16G,host_numa_node=1,id=mem1
+--numa guest_numa_id=0,memory_zones=mem0,pci_segments=[0]
+--numa guest_numa_id=1,memory_zones=mem1,pci_segments=[1]
+```
--- a/docs/profiling.md
+++ b/docs/profiling.md
@@ -25,7 +25,7 @@ $ perf record -g target/profiling/cloud-hypervisor \
        --cpus boot=1 --memory size=1G \
        --cmdline "root=/dev/pmem0p1 console=ttyS0" \
        --serial tty --console off \
-        --api-socket /tmp/api1
+        --api-socket=/tmp/api1
 ```

 For analysing the samples:
@@ -52,5 +52,5 @@ $ perf record --call-graph lbr --all-user --user-callchains -g target/release/cl
        --cpus boot=1 --memory size=1G \
        --cmdline "root=/dev/pmem0p1 console=ttyS0" \
        --serial tty --console off \
-        --api-socket /tmp/api1
+        --api-socket=/tmp/api1
 ```
--- a/docs/snapshot_restore.md
+++ b/docs/snapshot_restore.md
@@ -25,14 +25,14 @@ First thing, we must run a Cloud Hypervisor VM:
 At any point in time when the VM is running, one might choose to pause it:

 ```bash
-./ch-remote --api-socket /tmp/cloud-hypervisor.sock pause
+./ch-remote --api-socket=/tmp/cloud-hypervisor.sock pause
 ```

 Once paused, the VM can be safely snapshot into the specified directory and
 using the following command:

 ```bash
-./ch-remote --api-socket /tmp/cloud-hypervisor.sock snapshot file:///home/foo/snapshot
+./ch-remote --api-socket=/tmp/cloud-hypervisor.sock snapshot file:///home/foo/snapshot
 ```

 Given the directory was present on the system, the snapshot will succeed and
@@ -79,7 +79,7 @@ Or using two different commands from two terminals:
 ./cloud-hypervisor --api-socket /tmp/cloud-hypervisor.sock

 # Second terminal
-./ch-remote --api-socket /tmp/cloud-hypervisor.sock restore source_url=file:///home/foo/snapshot
+./ch-remote --api-socket=/tmp/cloud-hypervisor.sock restore source_url=file:///home/foo/snapshot
 ```

 Remember the VM is restored in a `paused` state, which was the VM's state when
@@ -87,7 +87,7 @@ it was snapshot. For this reason, one must explicitly `resume` the VM before to
 start using it.

 ```bash
-./ch-remote --api-socket /tmp/cloud-hypervisor.sock resume
+./ch-remote --api-socket=/tmp/cloud-hypervisor.sock resume
 ```

 At this point, the VM is fully restored and is identical to the VM which was
--- a/docs/vhost-user-blk-testing.md
+++ b/docs/vhost-user-blk-testing.md
@@ -94,7 +94,7 @@ VMs run in client mode. They connect to the socket created by the `dpdkvhostuser
        --memory size=1024M,hugepages=on,shared=true \
        --kernel linux/arch/x86/boot/compressed/vmlinux.bin \
        --cmdline "console=ttyS0 root=/dev/vda1 rw iommu=off" \
-        --disk path=images/focal-server-cloudimg-amd64.raw --disk vhost_user=true,socket=/var/tmp/vhost.1,num_queues=4,queue_size=128 \
+        --disk path=images/focal-server-cloudimg-amd64.raw vhost_user=true,socket=/var/tmp/vhost.1,num_queues=4,queue_size=128 \
        --console off \
        --serial tty \
        --rng
--- a/docs/vsock.md
+++ b/docs/vsock.md
@@ -50,14 +50,14 @@ The examples use __socat__ `>=1.7.4` to illustrate the VSOCK functionality. Howe

 ### Connecting from Host to Guest

-The host starts to listen on the defined port:
+The guest starts to listen on the defined port:

 `$ socat - VSOCK-LISTEN:1234`

-Once the host is listening, the guest can send data:
+Once the guest is listening, the host can send data:

 `echo -e "CONNECT 1234\\nHello from host!" | socat - UNIX-CONNECT:/tmp/ch.vsock
- 
+
 Note the string `CONNECT <port>` prepended to the actual data. It is possible for the guest to start listening on different ports, thus the specific command is needed to instruct VSOCK to which listener the host wants to connect. It needs to be sent once per connection. Once the connection established, data transfers can take place directly.

 ### Connecting from Guest to Host
--- a/fuzz/Cargo.lock
+++ b/fuzz/Cargo.lock
@@ -5,11 +5,59 @@ version = 3
 [[package]]
 name = "acpi_tables"
 version = "0.1.0"
-source = "git+https://github.com/rust-vmm/acpi_tables?branch=main#1029d22777f07b04849234bbe756da34a6df2913"
+source = "git+https://github.com/rust-vmm/acpi_tables?branch=main#1a733bf690ccc10bdfeacad33e3c9f6cce0008fd"
 dependencies = [
 "zerocopy",
 ]

+[[package]]
+name = "anstream"
+version = "0.6.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2ab91ebe16eb252986481c5b62f6098f3b698a45e34b5b98200cf20dd2484a44"
+dependencies = [
+ "anstyle",
+ "anstyle-parse",
+ "anstyle-query",
+ "anstyle-wincon",
+ "colorchoice",
+ "utf8parse",
+]
+
+[[package]]
+name = "anstyle"
+version = "1.0.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7079075b41f533b8c61d2a4d073c4676e1f8b249ff94a393b0595db304e0dd87"
+
+[[package]]
+name = "anstyle-parse"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "317b9a89c1868f5ea6ff1d9539a69f45dffc21ce321ac1fd1160dfa48c8e2140"
+dependencies = [
+ "utf8parse",
+]
+
+[[package]]
+name = "anstyle-query"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5ca11d4be1bab0c8bc8734a9aa7bf4ee8316d462a08c6ac5052f888fef5b494b"
+dependencies = [
+ "windows-sys",
+]
+
+[[package]]
+name = "anstyle-wincon"
+version = "3.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f0699d10d2f4d628a98ee7b57b289abbc98ff3bad977cb3152709d4bf2330628"
+dependencies = [
+ "anstyle",
+ "windows-sys",
+]
+
 [[package]]
 name = "anyhow"
 version = "1.0.75"
@@ -25,9 +73,9 @@ dependencies = [

 [[package]]
 name = "arbitrary"
-version = "1.3.0"
+version = "1.3.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e2d098ff73c1ca148721f37baad5ea6a465a13f9573aba8641fbbbae8164a54e"
+checksum = "a2e1373abdaa212b704512ec2bd8b26bd0b7d5c3f70117411a5d9a451383c859"

 [[package]]
 name = "arc-swap"
@@ -57,37 +105,6 @@ dependencies = [
 "vmm-sys-util",
 ]

-[[package]]
-name = "argh"
-version = "0.1.12"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7af5ba06967ff7214ce4c7419c7d185be7ecd6cc4965a8f6e1d8ce0398aad219"
-dependencies = [
- "argh_derive",
- "argh_shared",
-]
-
-[[package]]
-name = "argh_derive"
-version = "0.1.12"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "56df0aeedf6b7a2fc67d06db35b09684c3e8da0c95f8f27685cb17e08413d87a"
-dependencies = [
- "argh_shared",
- "proc-macro2",
- "quote",
- "syn 2.0.23",
-]
-
-[[package]]
-name = "argh_shared"
-version = "0.1.12"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5693f39141bda5760ecc4111ab08da40565d1771038c4a0250f03457ec707531"
-dependencies = [
- "serde",
-]
-
 [[package]]
 name = "autocfg"
 version = "1.1.0"
@@ -111,9 +128,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"

 [[package]]
 name = "bitflags"
-version = "2.3.3"
+version = "2.4.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "630be753d4e58660abd17930c71b647fe46c27ea6b63cc59e1e3851406972e42"
+checksum = "327762f6e5a765692301e5bb513e0d9fef63be86bbc14528052b1cd3e6f03e07"

 [[package]]
 name = "block"
@@ -139,15 +156,15 @@ dependencies = [

 [[package]]
 name = "bumpalo"
-version = "3.13.0"
+version = "3.14.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a3e2c3daef883ecc1b5d58c15adae93470a91d425f3532ba1695849656af3fc1"
+checksum = "7f30e7476521f6f8af1a1c4c0b8cc94f0bee37d91763d0ca2665f299b6cd8aec"

 [[package]]
 name = "byteorder"
-version = "1.4.3"
+version = "1.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610"
+checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"

 [[package]]
 name = "cc"
@@ -165,13 +182,40 @@ version = "1.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"

+[[package]]
+name = "clap"
+version = "4.4.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ac495e00dcec98c83465d5ad66c5c4fabd652fd6686e7c6269b117e729a6f17b"
+dependencies = [
+ "clap_builder",
+]
+
+[[package]]
+name = "clap_builder"
+version = "4.4.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c77ed9a32a62e6ca27175d00d29d05ca32e396ea1eb5fb01d8256b669cec7663"
+dependencies = [
+ "anstream",
+ "anstyle",
+ "clap_lex",
+ "strsim",
+]
+
+[[package]]
+name = "clap_lex"
+version = "0.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "702fc72eb24e5a1e48ce58027a675bc24edd52096d5397d4aea7c6dd9eca0bd1"
+
 [[package]]
 name = "cloud-hypervisor"
-version = "34.0.0"
+version = "35.0.0"
 dependencies = [
 "anyhow",
 "api_client",
- "argh",
+ "clap",
 "epoll",
 "event_monitor",
 "hypervisor",
@@ -213,6 +257,12 @@ dependencies = [
 "vmm-sys-util",
 ]

+[[package]]
+name = "colorchoice"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7"
+
 [[package]]
 name = "crc32c"
 version = "0.6.4"
@@ -249,7 +299,7 @@ dependencies = [
 "proc-macro2",
 "quote",
 "strsim",
- "syn 2.0.23",
+ "syn 2.0.32",
 ]

 [[package]]
@@ -260,7 +310,7 @@ checksum = "836a9bbc7ad63342d6d6e7b815ccab164bc77a2d95d84bc3117a8c0d5c98e2d5"
 dependencies = [
 "darling_core",
 "quote",
- "syn 2.0.23",
+ "syn 2.0.32",
 ]

 [[package]]
@@ -270,7 +320,7 @@ dependencies = [
 "acpi_tables",
 "anyhow",
 "arch",
- "bitflags 2.3.3",
+ "bitflags 2.4.1",
 "byteorder",
 "event_monitor",
 "hypervisor",
@@ -294,7 +344,7 @@ version = "4.3.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "74351c3392ea1ff6cd2628e0042d268ac2371cb613252ff383b6dfa50d22fa79"
 dependencies = [
- "bitflags 2.3.3",
+ "bitflags 2.4.1",
 "libc",
 ]

@@ -336,9 +386,9 @@ checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"

 [[package]]
 name = "futures-core"
-version = "0.3.28"
+version = "0.3.29"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4bca583b7e26f571124fe5b7561d49cb2868d79116cfa0eefce955557c6fee8c"
+checksum = "eb1d22c66e66d9d72e1758f0bd7d4fd0bee04cad842ee34587d68c07e45d088c"

 [[package]]
 name = "futures-sink"
@@ -401,9 +451,9 @@ checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38"

 [[package]]
 name = "jobserver"
-version = "0.1.26"
+version = "0.1.27"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "936cfd212a0155903bcbc060e316fb6cc7cbf2e1907329391ebadc1fe0ce77c2"
+checksum = "8c37f63953c4c63420ed5fd3d6d398c719489b9f872b9fa683262f8edd363c7d"
 dependencies = [
 "libc",
 ]
@@ -440,9 +490,9 @@ dependencies = [

 [[package]]
 name = "libc"
-version = "0.2.148"
+version = "0.2.149"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9cdc71e17332e86d2e1d38c1f99edcb6288ee11b815fb1a4b049eaa2114d369b"
+checksum = "a08173bc88b7955d1b3145aa561539096c421ac8debde8cbc3612ec635fee29b"

 [[package]]
 name = "libfuzzer-sys"
@@ -466,9 +516,9 @@ dependencies = [

 [[package]]
 name = "lock_api"
-version = "0.4.10"
+version = "0.4.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c1cc9717a20b1bb222f333e6a92fd32f7d8a18ddc5a3191a11af45dcbf4dcd16"
+checksum = "3c168f8615b12bc01f9c17e2eb0cc07dcae1940121185446edc3744920e8ef45"
 dependencies = [
 "autocfg",
 "scopeguard",
@@ -483,7 +533,7 @@ checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f"
 [[package]]
 name = "micro_http"
 version = "0.1.0"
-source = "git+https://github.com/firecracker-microvm/micro-http?branch=main#0d0fdcd50ea10c1b4777f9a958873fc848a5b7bb"
+source = "git+https://github.com/firecracker-microvm/micro-http?branch=main#a4d632f2c5ea45712c0d2002dc909a63879e85c3"
 dependencies = [
 "libc",
 "vmm-sys-util",
@@ -576,14 +626,14 @@ checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405"
 dependencies = [
 "proc-macro2",
 "quote",
- "syn 2.0.23",
+ "syn 2.0.32",
 ]

 [[package]]
 name = "proc-macro2"
-version = "1.0.66"
+version = "1.0.69"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "18fb31db3f9bddb2ea821cde30a9f70117e3f119938b5ee630b7403aa6e2ead9"
+checksum = "134c189feb4956b20f6f547d2cf727d4c0fe06722b20a0eec87ed445a97f92da"
 dependencies = [
 "unicode-ident",
 ]
@@ -614,7 +664,7 @@ checksum = "bce3a7139d2ee67d07538ee5dba997364fbc243e7e7143e96eb830c74bfaa082"
 dependencies = [
 "proc-macro2",
 "quote",
- "syn 2.0.23",
+ "syn 2.0.32",
 ]

 [[package]]
@@ -640,44 +690,44 @@ checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"

 [[package]]
 name = "seccompiler"
-version = "0.3.0"
+version = "0.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6f6575e3c2b3a0fe2ef3e53855b6a8dead7c29f783da5e123d378c8c6a89017e"
+checksum = "345a3e4dddf721a478089d4697b83c6c0a8f5bf16086f6c13397e4534eb6e2e5"
 dependencies = [
 "libc",
 ]

 [[package]]
 name = "semver"
-version = "1.0.18"
+version = "1.0.20"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b0293b4b29daaf487284529cc2f5675b8e57c61f70167ba415a463651fd6a918"
+checksum = "836fa6a3e1e547f9a2c4040802ec865b5d85f4014efe00555d7090a3dcaa1090"

 [[package]]
 name = "serde"
-version = "1.0.168"
+version = "1.0.189"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d614f89548720367ded108b3c843be93f3a341e22d5674ca0dd5cd57f34926af"
+checksum = "8e422a44e74ad4001bdc8eede9a4570ab52f71190e9c076d14369f38b9200537"
 dependencies = [
 "serde_derive",
 ]

 [[package]]
 name = "serde_derive"
-version = "1.0.168"
+version = "1.0.189"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d4fe589678c688e44177da4f27152ee2d190757271dc7f1d5b6b9f68d869d641"
+checksum = "1e48d1f918009ce3145511378cf68d613e3b3d9137d67272562080d68a2b32d5"
 dependencies = [
 "proc-macro2",
 "quote",
- "syn 2.0.23",
+ "syn 2.0.32",
 ]

 [[package]]
 name = "serde_json"
-version = "1.0.107"
+version = "1.0.108"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6b420ce6e3d8bd882e9b243c6eed35dbc9a6110c9769e74b584e0d68d1f20c65"
+checksum = "3d1c7e3eac408d115102c4c24ad393e0821bb3a5df4d506a80f85f7a742a526b"
 dependencies = [
 "itoa",
 "ryu",
@@ -686,9 +736,9 @@ dependencies = [

 [[package]]
 name = "serde_with"
-version = "3.3.0"
+version = "3.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1ca3b16a3d82c4088f343b7480a93550b3eabe1a358569c2dfe38bbcead07237"
+checksum = "64cd236ccc1b7a29e7e2739f27c0b2dd199804abc4290e32f59f3b68d6405c23"
 dependencies = [
 "serde",
 "serde_with_macros",
@@ -696,14 +746,14 @@ dependencies = [

 [[package]]
 name = "serde_with_macros"
-version = "3.3.0"
+version = "3.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2e6be15c453eb305019bfa438b1593c731f36a289a7853f7707ee29e870b3b3c"
+checksum = "93634eb5f75a2323b16de4748022ac4297f9e76b6dced2be287a099f41b5e788"
 dependencies = [
 "darling",
 "proc-macro2",
 "quote",
- "syn 2.0.23",
+ "syn 2.0.32",
 ]

 [[package]]
@@ -763,9 +813,9 @@ dependencies = [

 [[package]]
 name = "syn"
-version = "2.0.23"
+version = "2.0.32"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "59fb7d6d8281a51045d62b8eb3a7d1ce347b76f312af50cd3dc0af39c87c1737"
+checksum = "239814284fd6f1a4ffe4ca893952cdd93c224b6a1571c9a9eadd670295c0c9e2"
 dependencies = [
 "proc-macro2",
 "quote",
@@ -774,22 +824,22 @@ dependencies = [

 [[package]]
 name = "thiserror"
-version = "1.0.48"
+version = "1.0.50"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9d6d7a740b8a666a7e828dd00da9c0dc290dff53154ea77ac109281de90589b7"
+checksum = "f9a7210f5c9a7156bb50aa36aed4c95afb51df0df00713949448cf9e97d382d2"
 dependencies = [
 "thiserror-impl",
 ]

 [[package]]
 name = "thiserror-impl"
-version = "1.0.48"
+version = "1.0.50"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "49922ecae66cc8a249b77e68d1d0623c1b2c514f0060c27cdc68bd62a1219d35"
+checksum = "266b2e40bc00e5a6c09c3584011e08b06f123c00362c92b975ba9843aaaa14b8"
 dependencies = [
 "proc-macro2",
 "quote",
- "syn 2.0.23",
+ "syn 2.0.32",
 ]

 [[package]]
@@ -823,10 +873,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b"

 [[package]]
-name = "uuid"
-version = "1.4.1"
+name = "utf8parse"
+version = "0.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "79daa5ed5740825c40b389c5e50312b9c86df53fccd33f281df655642b43869d"
+checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a"
+
+[[package]]
+name = "uuid"
+version = "1.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "88ad59a7560b41a70d191093a945f0b87bc1deeda46fb237479708a1d6b6cdfc"
 dependencies = [
 "getrandom",
 ]
@@ -1032,9 +1088,10 @@ dependencies = [
 "anyhow",
 "arc-swap",
 "arch",
- "bitflags 2.3.3",
+ "bitflags 2.4.1",
 "block",
 "cfg-if",
+ "clap",
 "devices",
 "epoll",
 "event_monitor",
@@ -1110,7 +1167,7 @@ dependencies = [
 "once_cell",
 "proc-macro2",
 "quote",
- "syn 2.0.23",
+ "syn 2.0.32",
 "wasm-bindgen-shared",
 ]

@@ -1132,7 +1189,7 @@ checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b"
 dependencies = [
 "proc-macro2",
 "quote",
- "syn 2.0.23",
+ "syn 2.0.32",
 "wasm-bindgen-backend",
 "wasm-bindgen-shared",
 ]
@@ -1166,10 +1223,76 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"

 [[package]]
-name = "zerocopy"
-version = "0.6.4"
+name = "windows-sys"
+version = "0.48.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "20707b61725734c595e840fb3704378a0cd2b9c74cc9e6e20724838fc6a1e2f9"
+checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9"
+dependencies = [
+ "windows-targets",
+]
+
+[[package]]
+name = "windows-targets"
+version = "0.48.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c"
+dependencies = [
+ "windows_aarch64_gnullvm",
+ "windows_aarch64_msvc",
+ "windows_i686_gnu",
+ "windows_i686_msvc",
+ "windows_x86_64_gnu",
+ "windows_x86_64_gnullvm",
+ "windows_x86_64_msvc",
+]
+
+[[package]]
+name = "windows_aarch64_gnullvm"
+version = "0.48.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8"
+
+[[package]]
+name = "windows_aarch64_msvc"
+version = "0.48.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc"
+
+[[package]]
+name = "windows_i686_gnu"
+version = "0.48.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e"
+
+[[package]]
+name = "windows_i686_msvc"
+version = "0.48.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406"
+
+[[package]]
+name = "windows_x86_64_gnu"
+version = "0.48.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e"
+
+[[package]]
+name = "windows_x86_64_gnullvm"
+version = "0.48.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc"
+
+[[package]]
+name = "windows_x86_64_msvc"
+version = "0.48.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538"
+
+[[package]]
+name = "zerocopy"
+version = "0.7.20"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dd66a62464e3ffd4e37bd09950c2b9dd6c4f8767380fabba0d523f9a775bc85a"
 dependencies = [
 "byteorder",
 "zerocopy-derive",
@@ -1177,11 +1300,11 @@ dependencies = [

 [[package]]
 name = "zerocopy-derive"
-version = "0.6.4"
+version = "0.7.20"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "56097d5b91d711293a42be9289403896b68654625021732067eac7a4ca388a1f"
+checksum = "255c4596d41e6916ced49cfafea18727b24d67878fa180ddfd69b9df34fd1726"
 dependencies = [
 "proc-macro2",
 "quote",
- "syn 2.0.23",
+ "syn 2.0.32",
 ]
--- a/fuzz/Cargo.toml
+++ b/fuzz/Cargo.toml
@@ -12,13 +12,13 @@ cargo-fuzz = true
 block = { path = "../block" }
 devices = { path = "../devices" }
 epoll = "4.3.1"
-libc = "0.2.148"
+libc = "0.2.149"
 libfuzzer-sys = "0.4.7"
 linux-loader = { version = "0.9.1", features = ["elf", "bzimage", "pe"] }
 micro_http = { git = "https://github.com/firecracker-microvm/micro-http", branch = "main" }
 net_util = { path = "../net_util" }
 once_cell = "1.18.0"
-seccompiler = "0.3.0"
+seccompiler = "0.4.0"
 virtio-devices = { path = "../virtio-devices" }
 virtio-queue = "0.9.0"
 vmm = { path = "../vmm" }
--- a/hypervisor/Cargo.toml
+++ b/hypervisor/Cargo.toml
@@ -8,13 +8,14 @@ license = "Apache-2.0 OR BSD-3-Clause"
 [features]
 kvm = ["kvm-ioctls", "kvm-bindings", "vfio-ioctls/kvm"]
 mshv = ["mshv-ioctls", "mshv-bindings", "vfio-ioctls/mshv", "iced-x86"]
-sev_snp = []
+sev_snp = ["igvm_parser", "igvm_defs"]
 tdx = []

 [dependencies]
 anyhow = "1.0.75"
 byteorder = "1.4.3"
-thiserror = "1.0.40"
+igvm_defs = { git = "https://github.com/microsoft/igvm", branch = "main" , package = "igvm_defs", optional  = true }
+igvm_parser = { git = "https://github.com/microsoft/igvm", branch = "main" , package = "igvm", optional  = true }
 libc = "0.2.147"
 log = "0.4.17"
 kvm-ioctls = { version = "0.13.0", optional = true }
@@ -22,14 +23,15 @@ kvm-bindings = { git = "https://github.com/cloud-hypervisor/kvm-bindings", branc
 mshv-bindings = { git = "https://github.com/rust-vmm/mshv", branch = "main", features = ["with-serde", "fam-wrappers"], optional  = true }
 mshv-ioctls = { git = "https://github.com/rust-vmm/mshv", branch = "main", optional  = true}
 serde = { version = "1.0.168", features = ["rc", "derive"] }
-serde_with = { version = "3.0.0", default-features = false, features = ["macros"] }
+serde_with = { version = "3.4.0", default-features = false, features = ["macros"] }
 vfio-ioctls = { git = "https://github.com/rust-vmm/vfio", branch = "main", default-features = false }
 vm-memory = { version = "0.12.2", features = ["backend-mmap", "backend-atomic"] }
 vmm-sys-util = { version = "0.11.0", features = ["with-serde"] }
+thiserror = "1.0.40"

 [target.'cfg(target_arch = "x86_64")'.dependencies.iced-x86]
 optional = true
-version = "1.19.0"
+version = "1.20.0"
 default-features = false
 features = ["std", "decoder", "op_code_info", "instr_info", "fast_fmt"]

--- a/hypervisor/src/cpu.rs
+++ b/hypervisor/src/cpu.rs
@@ -257,6 +257,16 @@ pub enum HypervisorCpuError {
    ///
    #[error("Failed to set TSC frequency: {0}")]
    SetTscKhz(#[source] anyhow::Error),
+    ///
+    /// Error reading value at given GPA
+    ///
+    #[error("Failed to read from GPA: {0}")]
+    GpaRead(#[source] anyhow::Error),
+    ///
+    /// Error writing value at given GPA
+    ///
+    #[error("Failed to write to GPA: {0}")]
+    GpaWrite(#[source] anyhow::Error),
 }

 #[derive(Debug)]
--- a/hypervisor/src/mshv/mod.rs
+++ b/hypervisor/src/mshv/mod.rs
@@ -20,13 +20,21 @@ use std::collections::HashMap;
 use std::sync::{Arc, RwLock};
 use vfio_ioctls::VfioDeviceFd;
 use vm::DataMatch;
+
+#[cfg(feature = "sev_snp")]
+mod snp_constants;
 // x86_64 dependencies
 #[cfg(target_arch = "x86_64")]
 pub mod x86_64;
+#[cfg(feature = "sev_snp")]
+use snp_constants::*;
+
 use crate::{
    ClockData, CpuState, IoEventAddress, IrqRoutingEntry, MpState, UserMemoryRegion,
    USER_MEMORY_REGION_EXECUTE, USER_MEMORY_REGION_READ, USER_MEMORY_REGION_WRITE,
 };
+#[cfg(feature = "sev_snp")]
+use igvm_defs::IGVM_VHS_SNP_ID_BLOCK;
 use vmm_sys_util::eventfd::EventFd;
 #[cfg(target_arch = "x86_64")]
 pub use x86_64::VcpuMshvState;
@@ -234,6 +242,33 @@ impl hypervisor::Hypervisor for MshvHypervisor {
            break;
        }

+        // Set additional partition property for SEV-SNP partition.
+        if mshv_vm_type == VmType::Snp {
+            let snp_policy = snp::get_default_snp_guest_policy();
+            let vmgexit_offloads = snp::get_default_vmgexit_offload_features();
+            // SAFETY: access union fields
+            unsafe {
+                debug!(
+                    "Setting the partition isolation policy as: 0x{:x}",
+                    snp_policy.as_uint64
+                );
+                fd.set_partition_property(
+                    hv_partition_property_code_HV_PARTITION_PROPERTY_ISOLATION_POLICY,
+                    snp_policy.as_uint64,
+                )
+                .map_err(|e| hypervisor::HypervisorError::SetPartitionProperty(e.into()))?;
+                debug!(
+                    "Setting the partition property to enable VMGEXIT offloads as : 0x{:x}",
+                    vmgexit_offloads.as_uint64
+                );
+                fd.set_partition_property(
+                    hv_partition_property_code_HV_PARTITION_PROPERTY_SEV_VMGEXIT_OFFLOADS,
+                    vmgexit_offloads.as_uint64,
+                )
+                .map_err(|e| hypervisor::HypervisorError::SetPartitionProperty(e.into()))?;
+            }
+        }
+
        // Default Microsoft Hypervisor behavior for unimplemented MSR is to
        // send a fault to the guest if it tries to access it. It is possible
        // to override this behavior with a more suitable option i.e., ignore
@@ -584,6 +619,374 @@ impl cpu::Vcpu for MshvVcpu {
                        info.interrupt_vector.try_into().unwrap(),
                    ))
                }
+                #[cfg(feature = "sev_snp")]
+                hv_message_type_HVMSG_X64_SEV_VMGEXIT_INTERCEPT => {
+                    let info = x.to_vmg_intercept_info().unwrap();
+                    let ghcb_data = info.ghcb_msr >> GHCB_INFO_BIT_WIDTH;
+                    let ghcb_msr = svm_ghcb_msr {
+                        as_uint64: info.ghcb_msr,
+                    };
+                    // SAFETY: Accessing a union element from bindgen generated bindings.
+                    let ghcb_op = unsafe { ghcb_msr.__bindgen_anon_2.ghcb_info() as u32 };
+                    // Sanity check on the header fields before handling other operations.
+                    assert!(info.header.intercept_access_type == HV_INTERCEPT_ACCESS_EXECUTE as u8);
+
+                    match ghcb_op {
+                        GHCB_INFO_HYP_FEATURE_REQUEST => {
+                            // Pre-condition: GHCB data must be zero
+                            assert!(ghcb_data == 0);
+                            let mut ghcb_response = GHCB_INFO_HYP_FEATURE_RESPONSE as u64;
+                            // Indicate support for basic SEV-SNP features
+                            ghcb_response |=
+                                (GHCB_HYP_FEATURE_SEV_SNP << GHCB_INFO_BIT_WIDTH) as u64;
+                            // Indicate support for SEV-SNP AP creation
+                            ghcb_response |= (GHCB_HYP_FEATURE_SEV_SNP_AP_CREATION
+                                << GHCB_INFO_BIT_WIDTH)
+                                as u64;
+                            debug!(
+                                "GHCB_INFO_HYP_FEATURE_REQUEST: Supported features: {:0x}",
+                                ghcb_response
+                            );
+                            let arr_reg_name_value =
+                                [(hv_register_name_HV_X64_REGISTER_GHCB, ghcb_response)];
+                            set_registers_64!(self.fd, arr_reg_name_value)
+                                .map_err(|e| cpu::HypervisorCpuError::SetRegister(e.into()))?;
+                        }
+                        GHCB_INFO_REGISTER_REQUEST => {
+                            let mut ghcb_gpa = hv_x64_register_sev_ghcb::default();
+                            // SAFETY: Accessing a union element from bindgen generated bindings.
+                            unsafe {
+                                ghcb_gpa.__bindgen_anon_1.set_enabled(1);
+                                ghcb_gpa
+                                    .__bindgen_anon_1
+                                    .set_page_number(ghcb_msr.__bindgen_anon_2.gpa_page_number());
+                            }
+                            // SAFETY: Accessing a union element from bindgen generated bindings.
+                            let reg_name_value = unsafe {
+                                [(
+                                    hv_register_name_HV_X64_REGISTER_SEV_GHCB_GPA,
+                                    ghcb_gpa.as_uint64,
+                                )]
+                            };
+
+                            set_registers_64!(self.fd, reg_name_value)
+                                .map_err(|e| cpu::HypervisorCpuError::SetRegister(e.into()))?;
+
+                            let mut resp_ghcb_msr = svm_ghcb_msr::default();
+                            // SAFETY: Accessing a union element from bindgen generated bindings.
+                            unsafe {
+                                resp_ghcb_msr
+                                    .__bindgen_anon_2
+                                    .set_ghcb_info(GHCB_INFO_REGISTER_RESPONSE as u64);
+                                resp_ghcb_msr.__bindgen_anon_2.set_gpa_page_number(
+                                    ghcb_msr.__bindgen_anon_2.gpa_page_number(),
+                                );
+                            }
+                            // SAFETY: Accessing a union element from bindgen generated bindings.
+                            let reg_name_value = unsafe {
+                                [(
+                                    hv_register_name_HV_X64_REGISTER_GHCB,
+                                    resp_ghcb_msr.as_uint64,
+                                )]
+                            };
+
+                            set_registers_64!(self.fd, reg_name_value)
+                                .map_err(|e| cpu::HypervisorCpuError::SetRegister(e.into()))?;
+                        }
+                        GHCB_INFO_SEV_INFO_REQUEST => {
+                            let sev_cpuid_function = 0x8000_001F;
+                            let cpu_leaf = self
+                                .fd
+                                .get_cpuid_values(sev_cpuid_function, 0, 0, 0)
+                                .unwrap();
+                            let ebx = cpu_leaf[1];
+                            // First 6-byte of EBX represents page table encryption bit number
+                            let pbit_encryption = (ebx & 0x3f) as u8;
+                            let mut ghcb_response = GHCB_INFO_SEV_INFO_RESPONSE as u64;
+
+                            // GHCBData[63:48] specifies the maximum GHCB protocol version supported
+                            ghcb_response |= (GHCB_PROTOCOL_VERSION_MAX as u64) << 48;
+                            // GHCBData[47:32] specifies the minimum GHCB protocol version supported
+                            ghcb_response |= (GHCB_PROTOCOL_VERSION_MIN as u64) << 32;
+                            // GHCBData[31:24] specifies the SEV page table encryption bit number.
+                            ghcb_response |= (pbit_encryption as u64) << 24;
+
+                            let arr_reg_name_value =
+                                [(hv_register_name_HV_X64_REGISTER_GHCB, ghcb_response)];
+                            set_registers_64!(self.fd, arr_reg_name_value)
+                                .map_err(|e| cpu::HypervisorCpuError::SetRegister(e.into()))?;
+                        }
+                        GHCB_INFO_NORMAL => {
+                            let exit_code =
+                                info.__bindgen_anon_2.__bindgen_anon_1.sw_exit_code as u32;
+                            // SAFETY: Accessing a union element from bindgen generated bindings.
+                            let pfn = unsafe { ghcb_msr.__bindgen_anon_2.gpa_page_number() };
+                            let ghcb_gpa = pfn << GHCB_INFO_BIT_WIDTH;
+                            match exit_code {
+                                SVM_EXITCODE_HV_DOORBELL_PAGE => {
+                                    let exit_info1 =
+                                        info.__bindgen_anon_2.__bindgen_anon_1.sw_exit_info1 as u32;
+                                    match exit_info1 {
+                                        SVM_NAE_HV_DOORBELL_PAGE_GET_PREFERRED => {
+                                            // Hypervisor does not have any preference for doorbell GPA.
+                                            let preferred_doorbell_gpa: u64 = 0xFFFFFFFFFFFFFFFF;
+                                            let mut swei2_rw_gpa_arg =
+                                                mshv_bindings::mshv_read_write_gpa {
+                                                    base_gpa: ghcb_gpa + GHCB_SW_EXITINFO2_OFFSET,
+                                                    byte_count: std::mem::size_of::<u64>() as u32,
+                                                    ..Default::default()
+                                                };
+                                            swei2_rw_gpa_arg.data.copy_from_slice(
+                                                &preferred_doorbell_gpa.to_le_bytes(),
+                                            );
+                                            self.fd.gpa_write(&mut swei2_rw_gpa_arg).map_err(
+                                                |e| cpu::HypervisorCpuError::GpaWrite(e.into()),
+                                            )?;
+                                        }
+                                        SVM_NAE_HV_DOORBELL_PAGE_SET => {
+                                            let exit_info2 = info
+                                                .__bindgen_anon_2
+                                                .__bindgen_anon_1
+                                                .sw_exit_info2;
+                                            let mut ghcb_doorbell_gpa =
+                                                hv_x64_register_sev_hv_doorbell::default();
+                                            // SAFETY: Accessing a union element from bindgen generated bindings.
+                                            unsafe {
+                                                ghcb_doorbell_gpa.__bindgen_anon_1.set_enabled(1);
+                                                ghcb_doorbell_gpa
+                                                    .__bindgen_anon_1
+                                                    .set_page_number(exit_info2 >> PAGE_SHIFT);
+                                            }
+                                            // SAFETY: Accessing a union element from bindgen generated bindings.
+                                            let reg_names = unsafe {
+                                                [(
+                                                    hv_register_name_HV_X64_REGISTER_SEV_DOORBELL_GPA,
+                                                    ghcb_doorbell_gpa.as_uint64,
+                                                )]
+                                            };
+                                            set_registers_64!(self.fd, reg_names).map_err(|e| {
+                                                cpu::HypervisorCpuError::SetRegister(e.into())
+                                            })?;
+
+                                            let mut swei2_rw_gpa_arg =
+                                                mshv_bindings::mshv_read_write_gpa {
+                                                    base_gpa: ghcb_gpa + GHCB_SW_EXITINFO2_OFFSET,
+                                                    byte_count: std::mem::size_of::<u64>() as u32,
+                                                    ..Default::default()
+                                                };
+                                            swei2_rw_gpa_arg.data[0..8]
+                                                .copy_from_slice(&exit_info2.to_le_bytes());
+                                            self.fd.gpa_write(&mut swei2_rw_gpa_arg).map_err(
+                                                |e| cpu::HypervisorCpuError::GpaWrite(e.into()),
+                                            )?;
+
+                                            // Clear the SW_EXIT_INFO1 register to indicate no error
+                                            let mut swei1_rw_gpa_arg =
+                                                mshv_bindings::mshv_read_write_gpa {
+                                                    base_gpa: ghcb_gpa + GHCB_SW_EXITINFO1_OFFSET,
+                                                    byte_count: std::mem::size_of::<u64>() as u32,
+                                                    ..Default::default()
+                                                };
+                                            self.fd.gpa_write(&mut swei1_rw_gpa_arg).map_err(
+                                                |e| cpu::HypervisorCpuError::GpaWrite(e.into()),
+                                            )?;
+                                        }
+                                        SVM_NAE_HV_DOORBELL_PAGE_QUERY => {
+                                            let mut reg_assocs = [ hv_register_assoc {
+                                                name: hv_register_name_HV_X64_REGISTER_SEV_DOORBELL_GPA,
+                                                ..Default::default()
+                                            } ];
+                                            self.fd.get_reg(&mut reg_assocs).unwrap();
+                                            // SAFETY: Accessing a union element from bindgen generated bindings.
+                                            let doorbell_gpa = unsafe { reg_assocs[0].value.reg64 };
+                                            let mut swei2_rw_gpa_arg =
+                                                mshv_bindings::mshv_read_write_gpa {
+                                                    base_gpa: ghcb_gpa + GHCB_SW_EXITINFO2_OFFSET,
+                                                    byte_count: std::mem::size_of::<u64>() as u32,
+                                                    ..Default::default()
+                                                };
+                                            swei2_rw_gpa_arg
+                                                .data
+                                                .copy_from_slice(&doorbell_gpa.to_le_bytes());
+                                            self.fd.gpa_write(&mut swei2_rw_gpa_arg).map_err(
+                                                |e| cpu::HypervisorCpuError::GpaWrite(e.into()),
+                                            )?;
+                                        }
+                                        SVM_NAE_HV_DOORBELL_PAGE_CLEAR => {
+                                            let mut swei2_rw_gpa_arg =
+                                                mshv_bindings::mshv_read_write_gpa {
+                                                    base_gpa: ghcb_gpa + GHCB_SW_EXITINFO2_OFFSET,
+                                                    byte_count: std::mem::size_of::<u64>() as u32,
+                                                    ..Default::default()
+                                                };
+                                            self.fd.gpa_write(&mut swei2_rw_gpa_arg).map_err(
+                                                |e| cpu::HypervisorCpuError::GpaWrite(e.into()),
+                                            )?;
+                                        }
+                                        _ => {
+                                            panic!(
+                                                "SVM_EXITCODE_HV_DOORBELL_PAGE: Unhandled exit code: {:0x}",
+                                                exit_info1
+                                            );
+                                        }
+                                    }
+                                }
+                                SVM_EXITCODE_SNP_EXTENDED_GUEST_REQUEST => {
+                                    warn!("Fetching extended guest request is not supported");
+                                    // Extended guest request is not supported by the Hypervisor
+                                    // Returning the error to the guest
+                                    // 0x6 means `The NAE event was not valid`
+                                    // Reference: GHCB Spec, page 42
+                                    let value: u64 = 0x6;
+                                    let mut swei2_rw_gpa_arg = mshv_bindings::mshv_read_write_gpa {
+                                        base_gpa: ghcb_gpa + GHCB_SW_EXITINFO2_OFFSET,
+                                        byte_count: std::mem::size_of::<u64>() as u32,
+                                        ..Default::default()
+                                    };
+                                    swei2_rw_gpa_arg.data.copy_from_slice(&value.to_le_bytes());
+                                    self.fd
+                                        .gpa_write(&mut swei2_rw_gpa_arg)
+                                        .map_err(|e| cpu::HypervisorCpuError::GpaWrite(e.into()))?;
+                                }
+                                SVM_EXITCODE_IOIO_PROT => {
+                                    let exit_info1 =
+                                        info.__bindgen_anon_2.__bindgen_anon_1.sw_exit_info1 as u32;
+                                    let port_info = hv_sev_vmgexit_port_info {
+                                        as_uint32: exit_info1,
+                                    };
+
+                                    let port =
+                                        // SAFETY: Accessing a union element from bindgen generated bindings.
+                                        unsafe { port_info.__bindgen_anon_1.intercepted_port() };
+                                    let mut len = 4;
+                                    // SAFETY: Accessing a union element from bindgen generated bindings.
+                                    unsafe {
+                                        if port_info.__bindgen_anon_1.operand_size_16bit() == 1 {
+                                            len = 2;
+                                        } else if port_info.__bindgen_anon_1.operand_size_8bit()
+                                            == 1
+                                        {
+                                            len = 1;
+                                        }
+                                    }
+                                    let is_write =
+                                        // SAFETY: Accessing a union element from bindgen generated bindings.
+                                        unsafe { port_info.__bindgen_anon_1.access_type() == 0 };
+                                    let mut rax_rw_gpa_arg: mshv_read_write_gpa =
+                                        mshv_bindings::mshv_read_write_gpa {
+                                            base_gpa: ghcb_gpa + GHCB_RAX_OFFSET,
+                                            byte_count: std::mem::size_of::<u64>() as u32,
+                                            ..Default::default()
+                                        };
+                                    self.fd
+                                        .gpa_read(&mut rax_rw_gpa_arg)
+                                        .map_err(|e| cpu::HypervisorCpuError::GpaRead(e.into()))?;
+
+                                    if is_write {
+                                        if let Some(vm_ops) = &self.vm_ops {
+                                            vm_ops
+                                                .pio_write(
+                                                    port.into(),
+                                                    &rax_rw_gpa_arg.data[0..len],
+                                                )
+                                                .map_err(|e| {
+                                                    cpu::HypervisorCpuError::RunVcpu(e.into())
+                                                })?;
+                                        }
+                                    } else {
+                                        if let Some(vm_ops) = &self.vm_ops {
+                                            vm_ops
+                                                .pio_read(
+                                                    port.into(),
+                                                    &mut rax_rw_gpa_arg.data[0..len],
+                                                )
+                                                .map_err(|e| {
+                                                    cpu::HypervisorCpuError::RunVcpu(e.into())
+                                                })?;
+                                        }
+
+                                        self.fd.gpa_write(&mut rax_rw_gpa_arg).map_err(|e| {
+                                            cpu::HypervisorCpuError::GpaWrite(e.into())
+                                        })?;
+                                    }
+
+                                    // Clear the SW_EXIT_INFO1 register to indicate no error
+                                    let mut swei1_rw_gpa_arg = mshv_bindings::mshv_read_write_gpa {
+                                        base_gpa: ghcb_gpa + GHCB_SW_EXITINFO1_OFFSET,
+                                        byte_count: std::mem::size_of::<u64>() as u32,
+                                        ..Default::default()
+                                    };
+                                    self.fd
+                                        .gpa_write(&mut swei1_rw_gpa_arg)
+                                        .map_err(|e| cpu::HypervisorCpuError::GpaWrite(e.into()))?;
+                                }
+                                SVM_EXITCODE_MMIO_READ => {
+                                    let src_gpa =
+                                        info.__bindgen_anon_2.__bindgen_anon_1.sw_exit_info1;
+                                    let dst_gpa = info.__bindgen_anon_2.__bindgen_anon_1.sw_scratch;
+                                    let data_len =
+                                        info.__bindgen_anon_2.__bindgen_anon_1.sw_exit_info2
+                                            as usize;
+                                    // Sanity check to make sure data len is within supported range.
+                                    assert!(data_len <= 0x8);
+
+                                    let mut data: Vec<u8> = vec![0; data_len];
+                                    if let Some(vm_ops) = &self.vm_ops {
+                                        vm_ops.mmio_read(src_gpa, &mut data[0..data_len]).map_err(
+                                            |e| cpu::HypervisorCpuError::RunVcpu(e.into()),
+                                        )?;
+                                    }
+                                    let mut arg: mshv_read_write_gpa =
+                                        mshv_bindings::mshv_read_write_gpa {
+                                            base_gpa: dst_gpa,
+                                            byte_count: data_len as u32,
+                                            ..Default::default()
+                                        };
+                                    arg.data[0..data_len].copy_from_slice(&data);
+
+                                    self.fd
+                                        .gpa_write(&mut arg)
+                                        .map_err(|e| cpu::HypervisorCpuError::GpaWrite(e.into()))?;
+                                }
+                                SVM_EXITCODE_MMIO_WRITE => {
+                                    let dst_gpa =
+                                        info.__bindgen_anon_2.__bindgen_anon_1.sw_exit_info1;
+                                    let src_gpa = info.__bindgen_anon_2.__bindgen_anon_1.sw_scratch;
+                                    let data_len =
+                                        info.__bindgen_anon_2.__bindgen_anon_1.sw_exit_info2
+                                            as usize;
+                                    // Sanity check to make sure data len is within supported range.
+                                    assert!(data_len <= 0x8);
+                                    let mut arg: mshv_read_write_gpa =
+                                        mshv_bindings::mshv_read_write_gpa {
+                                            base_gpa: src_gpa,
+                                            byte_count: data_len as u32,
+                                            ..Default::default()
+                                        };
+
+                                    self.fd
+                                        .gpa_read(&mut arg)
+                                        .map_err(|e| cpu::HypervisorCpuError::GpaRead(e.into()))?;
+
+                                    if let Some(vm_ops) = &self.vm_ops {
+                                        vm_ops
+                                            .mmio_write(dst_gpa, &arg.data[0..data_len])
+                                            .map_err(|e| {
+                                                cpu::HypervisorCpuError::RunVcpu(e.into())
+                                            })?;
+                                    }
+                                }
+                                _ => panic!(
+                                    "GHCB_INFO_NORMAL: Unhandled exit code: {:0x}",
+                                    exit_code
+                                ),
+                            }
+                        }
+                        _ => panic!("Unsupported VMGEXIT operation: {:0x}", ghcb_op),
+                    }
+
+                    Ok(cpu::VmExit::Ignore)
+                }
                exit => Err(cpu::HypervisorCpuError::RunVcpu(anyhow!(
                    "Unhandled VCPU exit {:?}",
                    exit
@@ -1249,4 +1652,89 @@ impl vm::Vm for MshvVm {
    fn as_any(&self) -> &dyn Any {
        self
    }
+    /// Initialize the SEV-SNP VM
+    #[cfg(feature = "sev_snp")]
+    fn sev_snp_init(&self) -> vm::Result<()> {
+        self.fd
+            .set_partition_property(
+                hv_partition_property_code_HV_PARTITION_PROPERTY_ISOLATION_STATE,
+                hv_partition_isolation_state_HV_PARTITION_ISOLATION_SECURE as u64,
+            )
+            .map_err(|e| vm::HypervisorVmError::InitializeSevSnp(e.into()))
+    }
+
+    #[cfg(feature = "sev_snp")]
+    fn import_isolated_pages(
+        &self,
+        page_type: u32,
+        page_size: u32,
+        pages: &[u64],
+    ) -> vm::Result<()> {
+        if pages.is_empty() {
+            return Ok(());
+        }
+
+        let mut isolated_pages =
+            vec_with_array_field::<mshv_import_isolated_pages, u64>(pages.len());
+        isolated_pages[0].num_pages = pages.len() as u64;
+        isolated_pages[0].page_type = page_type;
+        isolated_pages[0].page_size = page_size;
+        // SAFETY: isolated_pages initialized with pages.len() and now it is being turned into
+        // pages_slice with pages.len() again. It is guaranteed to be large enough to hold
+        // everything from pages.
+        unsafe {
+            let pages_slice: &mut [u64] = isolated_pages[0].page_number.as_mut_slice(pages.len());
+            pages_slice.copy_from_slice(pages);
+        }
+        self.fd
+            .import_isolated_pages(&isolated_pages[0])
+            .map_err(|e| vm::HypervisorVmError::ImportIsolatedPages(e.into()))
+    }
+    #[cfg(feature = "sev_snp")]
+    fn complete_isolated_import(
+        &self,
+        snp_id_block: IGVM_VHS_SNP_ID_BLOCK,
+        host_data: &[u8],
+        id_block_enabled: u8,
+    ) -> vm::Result<()> {
+        let mut auth_info = hv_snp_id_auth_info {
+            id_key_algorithm: snp_id_block.id_key_algorithm,
+            auth_key_algorithm: snp_id_block.author_key_algorithm,
+            ..Default::default()
+        };
+        // Each of r/s component is 576 bits long
+        auth_info.id_block_signature[..SIG_R_COMPONENT_SIZE_IN_BYTES]
+            .copy_from_slice(snp_id_block.id_key_signature.r_comp.as_ref());
+        auth_info.id_block_signature
+            [SIG_R_COMPONENT_SIZE_IN_BYTES..SIG_R_AND_S_COMPONENT_SIZE_IN_BYTES]
+            .copy_from_slice(snp_id_block.id_key_signature.s_comp.as_ref());
+        auth_info.id_key[..ECDSA_CURVE_ID_SIZE_IN_BYTES]
+            .copy_from_slice(snp_id_block.id_public_key.curve.to_le_bytes().as_ref());
+        auth_info.id_key[ECDSA_SIG_X_COMPONENT_START..ECDSA_SIG_X_COMPONENT_END]
+            .copy_from_slice(snp_id_block.id_public_key.qx.as_ref());
+        auth_info.id_key[ECDSA_SIG_Y_COMPONENT_START..ECDSA_SIG_Y_COMPONENT_END]
+            .copy_from_slice(snp_id_block.id_public_key.qy.as_ref());
+
+        let data = mshv_complete_isolated_import {
+            import_data: hv_partition_complete_isolated_import_data {
+                psp_parameters: hv_psp_launch_finish_data {
+                    id_block: hv_snp_id_block {
+                        launch_digest: snp_id_block.ld,
+                        family_id: snp_id_block.family_id,
+                        image_id: snp_id_block.image_id,
+                        version: snp_id_block.version,
+                        guest_svn: snp_id_block.guest_svn,
+                        policy: get_default_snp_guest_policy(),
+                    },
+                    id_auth_info: auth_info,
+                    host_data: host_data[0..32].try_into().unwrap(),
+                    id_block_enabled,
+                    author_key_enabled: 0,
+                },
+            },
+        };
+        self.fd
+            .complete_isolated_import(&data)
+            .map_err(|e| vm::HypervisorVmError::CompleteIsolatedImport(e.into()))
+    }
 }
--- a/hypervisor/src/mshv/snp_constants.rs
+++ b/hypervisor/src/mshv/snp_constants.rs
@@ -0,0 +1,24 @@
+// SPDX-License-Identifier: Apache-2.0 OR BSD-3-Clause
+//
+// Copyright © 2020, Microsoft Corporation
+//
+
+// Reference: https://www.amd.com/content/dam/amd/en/documents/epyc-technical-docs/specifications/56860.pdf
+// Chapter 10
+pub const SIG_R_COMPONENT_SIZE_IN_BYTES: usize = 72;
+pub const SIG_R_AND_S_COMPONENT_SIZE_IN_BYTES: usize = 144;
+pub const ECDSA_CURVE_ID_SIZE_IN_BYTES: usize = 4;
+pub const ECDSA_SIG_X_COMPONENT_SIZE_IN_BYTES: usize = 72;
+pub const ECDSA_SIG_Y_COMPONENT_SIZE_IN_BYTES: usize = 72;
+pub const ECDSA_SIG_X_COMPONENT_START: usize = ECDSA_CURVE_ID_SIZE_IN_BYTES;
+pub const ECDSA_SIG_X_COMPONENT_END: usize =
+    ECDSA_SIG_X_COMPONENT_START + ECDSA_SIG_X_COMPONENT_SIZE_IN_BYTES;
+pub const ECDSA_SIG_Y_COMPONENT_START: usize = ECDSA_SIG_X_COMPONENT_END;
+pub const ECDSA_SIG_Y_COMPONENT_END: usize =
+    ECDSA_SIG_X_COMPONENT_END + ECDSA_SIG_Y_COMPONENT_SIZE_IN_BYTES;
+
+// These constants are derived from GHCB spec Sect. 2.6 Table 3 GHCB Layout
+// Link: https://www.amd.com/content/dam/amd/en/documents/epyc-technical-docs/specifications/56421.pdf
+pub const GHCB_RAX_OFFSET: u64 = 0x01F8;
+pub const GHCB_SW_EXITINFO1_OFFSET: u64 = 0x398;
+pub const GHCB_SW_EXITINFO2_OFFSET: u64 = 0x3A0;
--- a/hypervisor/src/vm.rs
+++ b/hypervisor/src/vm.rs
@@ -19,6 +19,8 @@ use crate::cpu::Vcpu;
 use crate::ClockData;
 use crate::UserMemoryRegion;
 use crate::{IoEventAddress, IrqRoutingEntry};
+#[cfg(feature = "sev_snp")]
+use igvm_defs::IGVM_VHS_SNP_ID_BLOCK;
 use std::any::Any;
 #[cfg(target_arch = "x86_64")]
 use std::fs::File;
@@ -190,6 +192,13 @@ pub enum HypervisorVmError {
    #[error("Failed to assert virtual Interrupt: {0}")]
    AsserttVirtualInterrupt(#[source] anyhow::Error),

+    #[cfg(feature = "sev_snp")]
+    ///
+    /// Error initializing SEV-SNP on the VM
+    ///
+    #[error("Failed to initialize SEV-SNP: {0}")]
+    InitializeSevSnp(#[source] std::io::Error),
+
    #[cfg(feature = "tdx")]
    ///
    /// Error initializing TDX on the VM
@@ -213,6 +222,15 @@ pub enum HypervisorVmError {
    ///
    #[error("Failed to create Vgic: {0}")]
    CreateVgic(#[source] anyhow::Error),
+    ///
+    /// Import isolated pages error
+    ///
+    #[error("Failed to import isolated pages: {0}")]
+    ImportIsolatedPages(#[source] anyhow::Error),
+    /// Failed to complete isolated import
+    ///
+    #[error("Failed to complete isolated import: {0}")]
+    CompleteIsolatedImport(#[source] anyhow::Error),
 }
 ///
 /// Result type for returning from a function
@@ -324,6 +342,11 @@ pub trait Vm: Send + Sync + Any {
    fn stop_dirty_log(&self) -> Result<()>;
    /// Get dirty pages bitmap
    fn get_dirty_log(&self, slot: u32, base_gpa: u64, memory_size: u64) -> Result<Vec<u64>>;
+    #[cfg(feature = "sev_snp")]
+    /// Initialize SEV-SNP on this VM
+    fn sev_snp_init(&self) -> Result<()> {
+        unimplemented!()
+    }
    #[cfg(feature = "tdx")]
    /// Initialize TDX on this VM
    fn tdx_init(&self, _cpuid: &[CpuIdEntry], _max_vcpus: u32) -> Result<()> {
@@ -347,6 +370,26 @@ pub trait Vm: Send + Sync + Any {
    }
    /// Downcast to the underlying hypervisor VM type
    fn as_any(&self) -> &dyn Any;
+    /// Import the isolated pages
+    #[cfg(feature = "sev_snp")]
+    fn import_isolated_pages(
+        &self,
+        _page_type: u32,
+        _page_size: u32,
+        _pages: &[u64],
+    ) -> Result<()> {
+        unimplemented!()
+    }
+    /// Complete the isolated import
+    #[cfg(feature = "sev_snp")]
+    fn complete_isolated_import(
+        &self,
+        _snp_id_block: IGVM_VHS_SNP_ID_BLOCK,
+        _host_data: &[u8],
+        _id_block_enabled: u8,
+    ) -> Result<()> {
+        unimplemented!()
+    }
 }

 pub trait VmOps: Send + Sync {
--- a/performance-metrics/Cargo.toml
+++ b/performance-metrics/Cargo.toml
@@ -6,7 +6,7 @@ edition = "2021"
 build = "../build.rs"

 [dependencies]
-argh = "0.1.9"
+clap = { version = "4.0.32", features = ["wrap_help"] }
 dirs = "5.0.0"
 serde = { version = "1.0.168", features = ["rc", "derive"] }
 serde_json = "1.0.107"
--- a/performance-metrics/src/main.rs
+++ b/performance-metrics/src/main.rs
@@ -8,11 +8,11 @@ extern crate test_infra;

 mod performance_tests;

-use argh::FromArgs;
+use clap::{Arg, ArgAction, Command as ClapCommand};
 use performance_tests::*;
 use serde::{Deserialize, Serialize};
 use std::{
-    fmt,
+    env, fmt,
    process::Command,
    sync::{mpsc::channel, Arc},
    thread,
@@ -639,41 +639,45 @@ fn date() -> String {
    String::from_utf8_lossy(&output.stdout).trim().to_string()
 }

-#[derive(FromArgs)]
-/// Generate the performance metrics data for Cloud Hypervisor
-struct Options {
-    #[argh(switch, long = "list-tests")]
-    /// print the list of available metrics tests
-    list_tests: bool,
-
-    #[argh(option, long = "test-filter")]
-    /// filter metrics tests to run based on provided keywords
-    keywords: Vec<String>,
-
-    #[argh(option, long = "report-file")]
-    /// report file. Stderr is used if not specified
-    report_file: Option<String>,
-
-    #[argh(option, long = "iterations")]
-    /// override number of test iterations
-    iterations: Option<u32>,
-
-    #[argh(option, long = "timeout")]
-    /// override test timeout, Ex. --timeout 5
-    timeout: Option<u32>,
-
-    #[argh(switch, short = 'V', long = "version")]
-    /// print version information
-    version: bool,
-}
-
 fn main() {
-    let opts: Options = argh::from_env();
-
-    if opts.version {
-        println!("{} {}", env!("CARGO_BIN_NAME"), env!("BUILD_VERSION"));
-        return;
-    }
+    let cmd_arguments = ClapCommand::new("performance-metrics")
+        .version(env!("CARGO_PKG_VERSION"))
+        .author(env!("CARGO_PKG_AUTHORS"))
+        .about("Generate the performance metrics data for Cloud Hypervisor")
+        .arg(
+            Arg::new("test-filter")
+                .long("test-filter")
+                .help("Filter metrics tests to run based on provided keywords")
+                .num_args(1)
+                .required(false),
+        )
+        .arg(
+            Arg::new("list-tests")
+                .long("list-tests")
+                .help("Print the list of available metrics tests")
+                .num_args(0)
+                .action(ArgAction::SetTrue)
+                .required(false),
+        )
+        .arg(
+            Arg::new("report-file")
+                .long("report-file")
+                .help("Report file. Standard error is used if not specified")
+                .num_args(1),
+        )
+        .arg(
+            Arg::new("iterations")
+                .long("iterations")
+                .help("Override number of test iterations")
+                .num_args(1),
+        )
+        .arg(
+            Arg::new("timeout")
+                .long("timeout")
+                .help("Override test timeout, Ex. --timeout 5")
+                .num_args(1),
+        )
+        .get_matches();

    // It seems that the tool (ethr) used for testing the virtio-net latency
    // is not stable on AArch64, and therefore the latency test is currently
@@ -683,7 +687,7 @@ fn main() {
        .filter(|t| !(cfg!(target_arch = "aarch64") && t.name == "virtio_net_latency_us"))
        .collect();

-    if opts.list_tests {
+    if cmd_arguments.get_flag("list-tests") {
        for test in test_list.iter() {
            println!("\"{}\" ({})", test.name, test.control);
        }
@@ -691,7 +695,10 @@ fn main() {
        return;
    }

-    let test_filter = opts.keywords.iter().collect::<Vec<&String>>();
+    let test_filter = match cmd_arguments.get_many::<String>("test-filter") {
+        Some(s) => s.collect(),
+        None => Vec::new(),
+    };

    // Run performance tests sequentially and report results (in both readable/json format)
    let mut metrics_report: MetricsReport = Default::default();
@@ -699,8 +706,16 @@ fn main() {
    init_tests();

    let overrides = Arc::new(PerformanceTestOverrides {
-        test_iterations: opts.iterations,
-        test_timeout: opts.timeout,
+        test_iterations: cmd_arguments
+            .get_one::<String>("iterations")
+            .map(|s| s.parse())
+            .transpose()
+            .unwrap_or_default(),
+        test_timeout: cmd_arguments
+            .get_one::<String>("timeout")
+            .map(|s| s.parse())
+            .transpose()
+            .unwrap_or_default(),
    });

    for test in test_list.iter() {
@@ -719,18 +734,19 @@ fn main() {

    cleanup_tests();

-    let mut report_file: Box<dyn std::io::Write + Send> = if let Some(ref file) = opts.report_file {
-        Box::new(
-            std::fs::File::create(std::path::Path::new(file))
-                .map_err(|e| {
-                    eprintln!("Error opening report file: {file}: {e}");
-                    std::process::exit(1);
-                })
-                .unwrap(),
-        )
-    } else {
-        Box::new(std::io::stdout())
-    };
+    let mut report_file: Box<dyn std::io::Write + Send> =
+        if let Some(file) = cmd_arguments.get_one::<String>("report-file") {
+            Box::new(
+                std::fs::File::create(std::path::Path::new(file))
+                    .map_err(|e| {
+                        eprintln!("Error opening report file: {file}: {e}");
+                        std::process::exit(1);
+                    })
+                    .unwrap(),
+            )
+        } else {
+            Box::new(std::io::stdout())
+        };

    report_file
        .write_all(
--- a/performance-metrics/src/performance_tests.rs
+++ b/performance-metrics/src/performance_tests.rs
@@ -358,13 +358,11 @@ pub fn performance_block_io(control: &PerformanceTestControl) -> f64 {
                guest.disk_config.disk(DiskType::OperatingSystem).unwrap()
            )
            .as_str(),
-            "--disk",
            format!(
                "path={}",
                guest.disk_config.disk(DiskType::CloudInit).unwrap()
            )
            .as_str(),
-            "--disk",
            format!("path={BLK_IO_TEST_IMG}").as_str(),
        ])
        .default_net()
--- a/release-notes.md
+++ b/release-notes.md
@@ -1,42 +1,53 @@
+- [v36.1](#v361)
+- [v36.0](#v360)
+    - [Command Line Changes](#command-line-changes)
+    - [Enabled Features Reported via API Endpoint and CLI](#enabled-features-reported-via-api-endpoint-and-cli)
+    - [NUMA Support for PCI segments](#numa-support-for-pci-segments)
+    - [CPU Topology Support on AMD Platforms](#cpu-topology-support-on-amd-platforms)
+    - [Unix Socket Backend for Serial Port](#unix-socket-backend-for-serial-port)
+    - [AIO Backend for Block Devices](#aio-backend-for-block-devices)
+    - [Documentation Improvements](#documentation-improvements)
+    - [Notable Bug Fixes](#notable-bug-fixes)
+    - [Contributors](#contributors)
 - [v35.0](#v350)
    - [`virtio-vsock` Support for Linux Guest Kernel v6.3+](#virtio-vsock-support-for-linux-guest-kernel-v63)
    - [User Specified Serial Number for `virtio-block`](#user-specified-serial-number-for-virtio-block)
    - [vCPU TSC Frequency Included in Migration State](#vcpu-tsc-frequency-included-in-migration-state)
-    - [Notable Bug Fixes](#notable-bug-fixes)
-    - [Contributors](#contributors)
+    - [Notable Bug Fixes](#notable-bug-fixes-1)
+    - [Contributors](#contributors-1)
 - [v34.0](#v340)
    - [Paravirtualised Panic Device Support](#paravirtualised-panic-device-support)
    - [Improvements to VM Core Dump](#improvements-to-vm-core-dump)
    - [QCOW2 Support for Backing Files](#qcow2-support-for-backing-files)
    - [Minimum Host Kernel Bump](#minimum-host-kernel-bump)
-    - [Notable Bug Fixes](#notable-bug-fixes-1)
-    - [Contributors](#contributors-1)
+    - [Notable Bug Fixes](#notable-bug-fixes-2)
+    - [Contributors](#contributors-2)
 - [v33.0](#v330)
    - [D-Bus based API](#d-bus-based-api)
    - [Expose Host CPU Cache Details for AArch64](#expose-host-cpu-cache-details-for-aarch64)
-    - [Notable Bug Fixes](#notable-bug-fixes-2)
-    - [Contributors](#contributors-2)
+    - [Notable Bug Fixes](#notable-bug-fixes-3)
+    - [Contributors](#contributors-3)
 - [v32.0](#v320)
    - [Increased PCI Segment Limit](#increased-pci-segment-limit)
    - [API Changes](#api-changes)
-    - [Notable Bug Fixes](#notable-bug-fixes-3)
-    - [Contributors](#contributors-3)
+    - [Notable Bug Fixes](#notable-bug-fixes-4)
+    - [Contributors](#contributors-4)
 - [v31.1](#v311)
 - [v31.0](#v310)
    - [Update to Latest `acpi_tables`](#update-to-latest-acpi_tables)
    - [Update Reference Kernel to 6.2](#update-reference-kernel-to-62)
    - [Improvements on Console `SIGWINCH` Handler](#improvements-on-console-sigwinch-handler)
    - [Remove Directory Support from `MemoryZoneConfig::file`](#remove-directory-support-from-memoryzoneconfigfile)
-    - [Documentation Improvements](#documentation-improvements)
-    - [Notable Bug Fixes](#notable-bug-fixes-4)
-    - [Contributors](#contributors-4)
+    - [Documentation Improvements](#documentation-improvements-1)
+    - [Notable Bug Fixes](#notable-bug-fixes-5)
+    - [Contributors](#contributors-5)
 - [v30.0](#v300)
    - [Command Line Changes for Reduced Binary Size](#command-line-changes-for-reduced-binary-size)
    - [Basic vfio-user Server Support](#basic-vfio-user-server-support)
    - [Heap Profiling Support](#heap-profiling-support)
-    - [Documentation Improvements](#documentation-improvements-1)
-    - [Notable Bug Fixes](#notable-bug-fixes-5)
-    - [Contributors](#contributors-5)
+    - [Documentation Improvements](#documentation-improvements-2)
+    - [Notable Bug Fixes](#notable-bug-fixes-6)
+    - [Contributors](#contributors-6)
 - [v28.2](#v282)
 - [v29.0](#v290)
    - [Release Binary Supports Both MSHV and KVM](#release-binary-supports-both-mshv-and-kvm)
@@ -46,10 +57,10 @@
    - [`AArch64` Documentation Integration](#aarch64-documentation-integration)
    - [`virtio-block` Counters Enhancement](#virtio-block-counters-enhancement)
    - [TCP Offload Control](#tcp-offload-control)
-    - [Notable Bug Fixes](#notable-bug-fixes-6)
+    - [Notable Bug Fixes](#notable-bug-fixes-7)
    - [Removals](#removals)
    - [Deprecations](#deprecations)
-    - [Contributors](#contributors-6)
+    - [Contributors](#contributors-7)
 - [v28.1](#v281)
 - [v28.0](#v280)
    - [Community Engagement (Reminder)](#community-engagement-reminder)
@@ -57,9 +68,9 @@
    - [Virtualised TPM Support](#virtualised-tpm-support)
    - [Transparent Huge Page Support](#transparent-huge-page-support)
    - [README Quick Start Improved](#readme-quick-start-improved)
-    - [Notable Bug Fixes](#notable-bug-fixes-7)
+    - [Notable Bug Fixes](#notable-bug-fixes-8)
    - [Removals](#removals-1)
-    - [Contributors](#contributors-7)
+    - [Contributors](#contributors-8)
 - [v27.0](#v270)
    - [Community Engagement](#community-engagement)
    - [Prebuilt Packages](#prebuilt-packages)
@@ -68,41 +79,41 @@
    - [Simplified Build Feature Flags](#simplified-build-feature-flags)
    - [Asynchronous Kernel Loading](#asynchronous-kernel-loading)
    - [GDB Support for AArch64](#gdb-support-for-aarch64)
-    - [Notable Bug Fixes](#notable-bug-fixes-8)
+    - [Notable Bug Fixes](#notable-bug-fixes-9)
    - [Deprecations](#deprecations-1)
-    - [Contributors](#contributors-8)
+    - [Contributors](#contributors-9)
 - [v26.0](#v260)
    - [SMBIOS Improvements via `--platform`](#smbios-improvements-via---platform)
    - [Unified Binary MSHV and KVM Support](#unified-binary-mshv-and-kvm-support)
-    - [Notable Bug Fixes](#notable-bug-fixes-9)
+    - [Notable Bug Fixes](#notable-bug-fixes-10)
    - [Deprecations](#deprecations-2)
    - [Removals](#removals-2)
-    - [Contributors](#contributors-9)
+    - [Contributors](#contributors-10)
 - [v25.0](#v250)
    - [`ch-remote` Improvements](#ch-remote-improvements-1)
    - [VM "Coredump" Support](#vm-coredump-support)
-    - [Notable Bug Fixes](#notable-bug-fixes-10)
+    - [Notable Bug Fixes](#notable-bug-fixes-11)
    - [Removals](#removals-3)
-    - [Contributors](#contributors-10)
+    - [Contributors](#contributors-11)
 - [v24.0](#v240)
    - [Bypass Mode for `virtio-iommu`](#bypass-mode-for-virtio-iommu)
    - [Ensure Identifiers Uniqueness](#ensure-identifiers-uniqueness)
    - [Sparse Mmap support](#sparse-mmap-support)
    - [Expose Platform Serial Number](#expose-platform-serial-number)
-    - [Notable Bug Fixes](#notable-bug-fixes-11)
+    - [Notable Bug Fixes](#notable-bug-fixes-12)
    - [Notable Improvements](#notable-improvements)
    - [Deprecations](#deprecations-3)
    - [New on the Website](#new-on-the-website)
-    - [Contributors](#contributors-11)
+    - [Contributors](#contributors-12)
 - [v23.1](#v231)
 - [v23.0](#v230)
    - [vDPA Support](#vdpa-support)
    - [Updated OS Support list](#updated-os-support-list)
    - [`AArch64` Memory Map Improvements](#aarch64-memory-map-improvements)
    - [`AMX` Support](#amx-support)
-    - [Notable Bug Fixes](#notable-bug-fixes-12)
+    - [Notable Bug Fixes](#notable-bug-fixes-13)
    - [Deprecations](#deprecations-4)
-    - [Contributors](#contributors-12)
+    - [Contributors](#contributors-13)
 - [v22.1](#v221)
 - [v22.0](#v220)
    - [GDB Debug Stub Support](#gdb-debug-stub-support)
@@ -113,13 +124,13 @@
    - [PMU Support for AArch64](#pmu-support-for-aarch64)
    - [Documentation Under CC-BY-4.0 License](#documentation-under-cc-by-40-license)
    - [Deprecation of "Classic" `virtiofsd`](#deprecation-of-classic-virtiofsd)
-    - [Notable Bug Fixes](#notable-bug-fixes-13)
-    - [Contributors](#contributors-13)
+    - [Notable Bug Fixes](#notable-bug-fixes-14)
+    - [Contributors](#contributors-14)
 - [v21.0](#v210)
    - [Efficient Local Live Migration (for Live Upgrade)](#efficient-local-live-migration-for-live-upgrade)
    - [Recommended Kernel is Now 5.15](#recommended-kernel-is-now-515)
-    - [Notable Bug fixes](#notable-bug-fixes-14)
-    - [Contributors](#contributors-14)
+    - [Notable Bug fixes](#notable-bug-fixes-15)
+    - [Contributors](#contributors-15)
 - [v20.2](#v202)
 - [v20.1](#v201)
 - [v20.0](#v200)
@@ -128,8 +139,8 @@
    - [Improved VFIO support](#improved-vfio-support)
    - [Safer code](#safer-code)
    - [Extended documentation](#extended-documentation)
-    - [Notable bug fixes](#notable-bug-fixes-15)
-    - [Contributors](#contributors-15)
+    - [Notable bug fixes](#notable-bug-fixes-16)
+    - [Contributors](#contributors-16)
 - [v19.0](#v190)
    - [Improved PTY handling for serial and `virtio-console`](#improved-pty-handling-for-serial-and-virtio-console)
    - [PCI boot time optimisations](#pci-boot-time-optimisations)
@@ -137,8 +148,8 @@
    - [Live migration enhancements](#live-migration-enhancements)
    - [`virtio-mem` support with `vfio-user`](#virtio-mem-support-with-vfio-user)
    - [AArch64 for `virtio-iommu`](#aarch64-for-virtio-iommu)
-    - [Notable bug fixes](#notable-bug-fixes-16)
-    - [Contributors](#contributors-16)
+    - [Notable bug fixes](#notable-bug-fixes-17)
+    - [Contributors](#contributors-17)
 - [v18.0](#v180)
    - [Experimental User Device (`vfio-user`) support](#experimental-user-device-vfio-user-support)
    - [Migration support for `vhost-user` devices](#migration-support-for-vhost-user-devices)
@@ -148,23 +159,23 @@
    - [Live migration on MSHV hypervisor](#live-migration-on-mshv-hypervisor)
    - [AArch64 CPU topology support](#aarch64-cpu-topology-support)
    - [Power button support on AArch64](#power-button-support-on-aarch64)
-    - [Notable bug fixes](#notable-bug-fixes-17)
-    - [Contributors](#contributors-17)
+    - [Notable bug fixes](#notable-bug-fixes-18)
+    - [Contributors](#contributors-18)
 - [v17.0](#v170)
    - [ARM64 NUMA support using ACPI](#arm64-numa-support-using-acpi)
    - [`Seccomp` support for MSHV backend](#seccomp-support-for-mshv-backend)
    - [Hotplug of `macvtap` devices](#hotplug-of-macvtap-devices)
    - [Improved SGX support](#improved-sgx-support)
    - [Inflight tracking for `vhost-user` devices](#inflight-tracking-for-vhost-user-devices)
-    - [Notable bug fixes](#notable-bug-fixes-18)
-    - [Contributors](#contributors-18)
+    - [Notable bug fixes](#notable-bug-fixes-19)
+    - [Contributors](#contributors-19)
 - [v16.0](#v160)
    - [Improved live migration support](#improved-live-migration-support)
    - [Improved `vhost-user` support](#improved-vhost-user-support)
    - [ARM64 ACPI and UEFI support](#arm64-acpi-and-uefi-support)
-    - [Notable bug fixes](#notable-bug-fixes-19)
+    - [Notable bug fixes](#notable-bug-fixes-20)
    - [Removed functionality](#removed-functionality)
-    - [Contributors](#contributors-19)
+    - [Contributors](#contributors-20)
 - [v15.0](#v150)
    - [Version numbering and stability guarantees](#version-numbering-and-stability-guarantees)
    - [Network device rate limiting](#network-device-rate-limiting)
@@ -172,7 +183,7 @@
    - [`--api-socket` supports file descriptor parameter](#--api-socket-supports-file-descriptor-parameter)
    - [Bug fixes](#bug-fixes)
    - [Deprecations](#deprecations-5)
-    - [Contributors](#contributors-20)
+    - [Contributors](#contributors-21)
 - [v0.14.1](#v0141)
 - [v0.14.0](#v0140)
    - [Structured event monitoring](#structured-event-monitoring)
@@ -182,7 +193,7 @@
    - [PTY control for serial and `virtio-console`](#pty-control-for-serial-and-virtio-console)
    - [Block device rate limiting](#block-device-rate-limiting)
    - [Deprecations](#deprecations-6)
-    - [Contributors](#contributors-21)
+    - [Contributors](#contributors-22)
 - [v0.13.0](#v0130)
    - [Wider VFIO device support](#wider-vfio-device-support)
    - [Improved huge page support](#improved-huge-page-support)
@@ -190,13 +201,13 @@
    - [VHD disk image support](#vhd-disk-image-support)
    - [Improved Virtio device threading](#improved-virtio-device-threading)
    - [Clean shutdown support via synthetic power button](#clean-shutdown-support-via-synthetic-power-button)
-    - [Contributors](#contributors-22)
+    - [Contributors](#contributors-23)
 - [v0.12.0](#v0120)
    - [ARM64 enhancements](#arm64-enhancements)
    - [Removal of `vhost-user-net` and `vhost-user-block` self spawning](#removal-of-vhost-user-net-and-vhost-user-block-self-spawning)
    - [Migration of `vhost-user-fs` backend](#migration-of-vhost-user-fs-backend)
    - [Enhanced "info" API](#enhanced-info-api)
-    - [Contributors](#contributors-23)
+    - [Contributors](#contributors-24)
 - [v0.11.0](#v0110)
    - [`io_uring` support by default for `virtio-block`](#io_uring-support-by-default-for-virtio-block)
    - [Windows Guest Support](#windows-guest-support)
@@ -208,15 +219,15 @@
    - [Default Log Level Changed](#default-log-level-changed)
    - [New `--balloon` Parameter Added](#new---balloon-parameter-added)
    - [Experimental `virtio-watchdog` Support](#experimental-virtio-watchdog-support)
-    - [Notable Bug Fixes](#notable-bug-fixes-20)
-    - [Contributors](#contributors-24)
+    - [Notable Bug Fixes](#notable-bug-fixes-21)
+    - [Contributors](#contributors-25)
 - [v0.10.0](#v0100)
    - [`virtio-block` Support for Multiple Descriptors](#virtio-block-support-for-multiple-descriptors)
    - [Memory Zones](#memory-zones)
    - [`Seccomp` Sandbox Improvements](#seccomp-sandbox-improvements)
    - [Preliminary KVM HyperV Emulation Control](#preliminary-kvm-hyperv-emulation-control)
-    - [Notable Bug Fixes](#notable-bug-fixes-21)
-    - [Contributors](#contributors-25)
+    - [Notable Bug Fixes](#notable-bug-fixes-22)
+    - [Contributors](#contributors-26)
 - [v0.9.0](#v090)
    - [`io_uring` Based Block Device Support](#io_uring-based-block-device-support)
    - [Block and Network Device Statistics](#block-and-network-device-statistics)
@@ -229,17 +240,17 @@
    - [Enhancements to ARM64 Support](#enhancements-to-arm64-support)
    - [Intel SGX Support](#intel-sgx-support)
    - [`Seccomp` Sandbox Improvements](#seccomp-sandbox-improvements-1)
-    - [Notable Bug Fixes](#notable-bug-fixes-22)
-    - [Contributors](#contributors-26)
+    - [Notable Bug Fixes](#notable-bug-fixes-23)
+    - [Contributors](#contributors-27)
 - [v0.8.0](#v080)
    - [Experimental Snapshot and Restore Support](#experimental-snapshot-and-restore-support)
    - [Experimental ARM64 Support](#experimental-arm64-support)
    - [Support for Using 5-level Paging in Guests](#support-for-using-5-level-paging-in-guests)
    - [Virtio Device Interrupt Suppression for Network Devices](#virtio-device-interrupt-suppression-for-network-devices)
    - [`vhost_user_fs` Improvements](#vhost_user_fs-improvements)
-    - [Notable Bug Fixes](#notable-bug-fixes-23)
+    - [Notable Bug Fixes](#notable-bug-fixes-24)
    - [Command Line and API Changes](#command-line-and-api-changes)
-    - [Contributors](#contributors-27)
+    - [Contributors](#contributors-28)
 - [v0.7.0](#v070)
    - [Block, Network, Persistent Memory (PMEM), VirtioFS and Vsock hotplug](#block-network-persistent-memory-pmem-virtiofs-and-vsock-hotplug)
    - [Alternative `libc` Support](#alternative-libc-support)
@@ -249,14 +260,14 @@
    - [`Seccomp` Sandboxing](#seccomp-sandboxing)
    - [Updated Distribution Support](#updated-distribution-support)
    - [Command Line and API Changes](#command-line-and-api-changes-1)
-    - [Contributors](#contributors-28)
+    - [Contributors](#contributors-29)
 - [v0.6.0](#v060)
    - [Directly Assigned Devices Hotplug](#directly-assigned-devices-hotplug)
    - [Shared Filesystem Improvements](#shared-filesystem-improvements)
    - [Block and Networking IO Self Offloading](#block-and-networking-io-self-offloading)
    - [Command Line Interface](#command-line-interface)
    - [PVH Boot](#pvh-boot)
-    - [Contributors](#contributors-29)
+    - [Contributors](#contributors-30)
 - [v0.5.1](#v051)
 - [v0.5.0](#v050)
    - [Virtual Machine Dynamic Resizing](#virtual-machine-dynamic-resizing)
@@ -264,7 +275,7 @@
    - [New Interrupt Management Framework](#new-interrupt-management-framework)
    - [Development Tools](#development-tools)
    - [Kata Containers Integration](#kata-containers-integration)
-    - [Contributors](#contributors-30)
+    - [Contributors](#contributors-31)
 - [v0.4.0](#v040)
    - [Dynamic virtual CPUs addition](#dynamic-virtual-cpus-addition)
    - [Programmatic firmware tables generation](#programmatic-firmware-tables-generation)
@@ -273,7 +284,7 @@
    - [Userspace IOAPIC by default](#userspace-ioapic-by-default)
    - [PCI BAR reprogramming](#pci-bar-reprogramming)
    - [New `cloud-hypervisor` organization](#new-cloud-hypervisor-organization)
-    - [Contributors](#contributors-31)
+    - [Contributors](#contributors-32)
 - [v0.3.0](#v030)
    - [Block device offloading](#block-device-offloading)
    - [Network device backend](#network-device-backend)
@@ -300,6 +311,91 @@
    - [Unit testing](#unit-testing)
    - [Integration tests parallelization](#integration-tests-parallelization)

+# v36.1
+
+This is a bug fix release. The following issues have been addressed:
+
+* Fix aio backend behavior for block devices when writeback cache
+disabled (#5930)
+* Fix PvPanic device PCI BAR alignment (#5956)
+* Bug fix to OpenAPI specification file (#5967)
+* Error out early for live migration when TDX is enabled (#6025)
+
+# v36.0
+
+This release has been tracked in our [roadmap
+project](https://github.com/orgs/cloud-hypervisor/projects/6) as iteration
+v36.0. The following user visible changes have been made:
+
+### Command Line Changes
+
+We switched back to use the `clap` crate to create our command line,
+since the `argh` crate is barely maintained. There were several syntax
+changes:
+
+* All `--option value` commands now are `--option=value`.
+* The `--disk DISK1 --disk DISK2` command now is `--disk DISK1 DISK2`.
+* The `-v -v -v`command now is `-vvv`.
+
+Note: the released binary size increased around 0.3M due to this change.
+
+### Enabled Features Reported via API Endpoint and CLI
+
+Now the enabled (Cargo) features of the running Cloud Hypervisor
+instance can be queried via API endpoint (`/vmm.ping`) and CLI
+(`--version -v`).
+
+### NUMA Support for PCI segments
+
+The `--numa` command is augmented with a new option `pci_segment=`, so
+that users can define the relationship between PCI segments and NUMA
+nodes. Examples can be found from the [memory documentation](docs/memory.md)
+
+### CPU Topology Support on AMD Platforms
+
+Now the CPU topology on x86_64 platforms supports multiple vendors.
+
+### Unix Socket Backend for Serial Port
+
+The `--serial` command is augmented with a new option `socket=`, allowing
+users to access the serial port using a Unix socket.
+
+### AIO Backend for Block Devices
+
+An AIO backend is added for `virtio-block` devices to improve block
+device performance when the `io_uring` feature is not supported by the
+host Operating System.
+
+### Documentation Improvements
+
+* New [documentation](docs/coverage.md) for collecting coverage data
+* Various typo fixes
+
+### Notable Bug Fixes
+
+* Fix a deadlock when TDX is enabled (#5845)
+* Only advertise AMX feature bits to guest when the AMX cpu feature is
+  enabled (#5834)
+* Correct default value for vCPU topology on AArch64 (#5893)
+
+### Contributors
+
+Many thanks to everyone who has contributed to our release:
+
+*  Anatol Belski <anbelski@linux.microsoft.com>
+*  Bo Chen <chen.bo@intel.com>
+*  Dario Nieuwenhuis <dirbaio@dirbaio.net>
+*  Jianyong Wu <jianyong.wu@arm.com>
+*  Jinank Jain <jinankjain@microsoft.com>
+*  Muminul Islam <muislam@microsoft.com>
+*  Praveen K Paladugu <prapal@linux.microsoft.com>
+*  Ravi kumar Veeramally <ravikumar.veeramally@intel.com>
+*  Rob Bradford <rbradford@rivosinc.com>
+*  Thomas Barrett <tbarrett@crusoeenergy.com>
+*  Wei Liu <liuwe@microsoft.com>
+*  Yi Wang <foxywang@tencent.com>
+*  dom.song <dom.song@amperecomputing.com>
+
 # v35.0

 This release has been tracked in our [roadmap
--- a/resources/Dockerfile
+++ b/resources/Dockerfile
@@ -3,7 +3,7 @@
 # When changing this file don't forget to update the tag name in the
 # .github/workflows/docker-image.yaml file if doing multiple per day

-FROM ubuntu:20.04 as dev
+FROM ubuntu:22.04 as dev

 ARG TARGETARCH
 ARG RUST_TOOLCHAIN="1.67.1"
@@ -43,7 +43,6 @@ RUN apt-get update \
        socat \
        dosfstools \
        cpio \
-        python \
        python3 \
        python3-setuptools \
        ntfs-3g \
@@ -130,17 +129,18 @@ RUN echo 'source $CARGO_HOME/env' >> "$HOME"/.bashrc \
 RUN if [ "$TARGETARCH" = "amd64" ]; then \
       git clone https://github.com/spdk/spdk \
       && cd spdk \
-       && git checkout 6301f8915de32baed10dba1eebed556a6749211a \
+       && git checkout ef8bcce58f3f02b79c0619a297e4f17e81e62b24 \
       && git submodule update --init \
       && apt-get update \
       && ./scripts/pkgdep.sh \
       && apt-get clean \
       && ./configure --with-vfio-user \
       && make -j $(nproc) TARGET_ARCHITECTURE=skylake \
+       && rm -rf /usr/local/bin/spdk-nvme \
       && mkdir /usr/local/bin/spdk-nvme \
-       && cp ./build/bin/nvmf_tgt /usr/local/bin/spdk-nvme \
-       && cp ./scripts/rpc.py /usr/local/bin/spdk-nvme \
-       && cp -r ./scripts/rpc /usr/local/bin/spdk-nvme \
+       && cp -f ./build/bin/nvmf_tgt /usr/local/bin/spdk-nvme \
+       && cp -f ./scripts/rpc.py /usr/local/bin/spdk-nvme \
+       && cp -rf ./python /usr/local/bin \
       && cd .. && rm -rf spdk; fi

 # install swtpm only for x86_64 arch
--- a/resources/linux-config-x86_64
+++ b/resources/linux-config-x86_64
@@ -639,7 +639,7 @@ CONFIG_VIRTUALIZATION=y
 CONFIG_KVM=y
 # CONFIG_KVM_WERROR is not set
 CONFIG_KVM_INTEL=y
-# CONFIG_KVM_AMD is not set
+CONFIG_KVM_AMD=y
 CONFIG_KVM_SMM=y
 # CONFIG_KVM_XEN is not set
 CONFIG_AS_AVX512=y
--- a/scripts/common-aarch64.sh
+++ b/scripts/common-aarch64.sh
@@ -1,7 +1,5 @@
 #!/bin/bash

-export BUILD_TARGET=${BUILD_TARGET-aarch64-unknown-linux-gnu}
-
 WORKLOADS_DIR="$HOME/workloads"

 mkdir -p "$WORKLOADS_DIR"
--- a/scripts/dev_cli.sh
+++ b/scripts/dev_cli.sh
@@ -7,7 +7,7 @@
 CLI_NAME="Cloud Hypervisor"

 CTR_IMAGE_TAG="ghcr.io/cloud-hypervisor/cloud-hypervisor"
-CTR_IMAGE_VERSION="20230804-0"
+CTR_IMAGE_VERSION="20231220-0"
 : "${CTR_IMAGE:=${CTR_IMAGE_TAG}:${CTR_IMAGE_VERSION}}"

 DOCKER_RUNTIME="docker"
@@ -282,11 +282,10 @@ cmd_build() {
    [ $build = "release" ] && cargo_args+=("--release")
    cargo_args+=(--target "$target")

-    rustflags=""
+    rustflags="$RUSTFLAGS"
    target_cc=""
    if [ "$(uname -m)" = "aarch64" ] && [ "$libc" = "musl" ]; then
-        rustflags="-C link-arg=-lgcc -C link_arg=-specs -C link_arg=/usr/lib/aarch64-linux-musl/musl-gcc.specs"
-        target_cc="musl-gcc"
+        rustflags="$rustflags -C link-args=-Wl,-Bstatic -C link-args=-lc"
    fi

    $DOCKER_RUNTIME run \
@@ -396,6 +395,12 @@ cmd_tests() {
    process_volumes_args
    target="$(uname -m)-unknown-linux-${libc}"

+    rustflags="$RUSTFLAGS"
+    target_cc=""
+    if [ "$(uname -m)" = "aarch64" ] && [ "$libc" = "musl" ]; then
+        rustflags="$rustflags -C link-args=-Wl,-Bstatic -C link-args=-lc"
+    fi
+
    if [[ "$unit" = true ]]; then
        say "Running unit tests for $target..."
        $DOCKER_RUNTIME run \
@@ -406,6 +411,8 @@ cmd_tests() {
            --cap-add net_admin \
            --volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR" $exported_volumes \
            --env BUILD_TARGET="$target" \
+            --env RUSTFLAGS="$rustflags" \
+            --env TARGET_CC="$target_cc" \
            "$CTR_IMAGE" \
            ./scripts/run_unit_tests.sh "$@" || fix_dir_perms $? || exit $?
    fi
@@ -424,7 +431,9 @@ cmd_tests() {
            --volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR" $exported_volumes \
            --volume "$CLH_INTEGRATION_WORKLOADS:$CTR_CLH_INTEGRATION_WORKLOADS" \
            --env USER="root" \
-            --env CH_LIBC="${libc}" \
+            --env BUILD_TARGET="$target" \
+            --env RUSTFLAGS="$rustflags" \
+            --env TARGET_CC="$target_cc" \
            "$CTR_IMAGE" \
            dbus-run-session ./scripts/run_integration_tests_"$(uname -m)".sh "$@" || fix_dir_perms $? || exit $?
    fi
@@ -443,7 +452,9 @@ cmd_tests() {
            --volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR" $exported_volumes \
            --volume "$CLH_INTEGRATION_WORKLOADS:$CTR_CLH_INTEGRATION_WORKLOADS" \
            --env USER="root" \
-            --env CH_LIBC="${libc}" \
+            --env BUILD_TARGET="$target" \
+            --env RUSTFLAGS="$rustflags" \
+            --env TARGET_CC="$target_cc" \
            "$CTR_IMAGE" \
            ./scripts/run_integration_tests_sgx.sh "$@" || fix_dir_perms $? || exit $?
    fi
@@ -462,7 +473,9 @@ cmd_tests() {
            --volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR" $exported_volumes \
            --volume "$CLH_INTEGRATION_WORKLOADS:$CTR_CLH_INTEGRATION_WORKLOADS" \
            --env USER="root" \
-            --env CH_LIBC="${libc}" \
+            --env BUILD_TARGET="$target" \
+            --env RUSTFLAGS="$rustflags" \
+            --env TARGET_CC="$target_cc" \
            "$CTR_IMAGE" \
            ./scripts/run_integration_tests_vfio.sh "$@" || fix_dir_perms $? || exit $?
    fi
@@ -481,7 +494,9 @@ cmd_tests() {
            --volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR" $exported_volumes \
            --volume "$CLH_INTEGRATION_WORKLOADS:$CTR_CLH_INTEGRATION_WORKLOADS" \
            --env USER="root" \
-            --env CH_LIBC="${libc}" \
+            --env BUILD_TARGET="$target" \
+            --env RUSTFLAGS="$rustflags" \
+            --env TARGET_CC="$target_cc" \
            "$CTR_IMAGE" \
            ./scripts/run_integration_tests_windows_"$(uname -m)".sh "$@" || fix_dir_perms $? || exit $?
    fi
@@ -500,7 +515,9 @@ cmd_tests() {
            --volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR" $exported_volumes \
            --volume "$CLH_INTEGRATION_WORKLOADS:$CTR_CLH_INTEGRATION_WORKLOADS" \
            --env USER="root" \
-            --env CH_LIBC="${libc}" \
+            --env BUILD_TARGET="$target" \
+            --env RUSTFLAGS="$rustflags" \
+            --env TARGET_CC="$target_cc" \
            "$CTR_IMAGE" \
            ./scripts/run_integration_tests_live_migration.sh "$@" || fix_dir_perms $? || exit $?
    fi
@@ -519,7 +536,9 @@ cmd_tests() {
            --volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR" $exported_volumes \
            --volume "$CLH_INTEGRATION_WORKLOADS:$CTR_CLH_INTEGRATION_WORKLOADS" \
            --env USER="root" \
-            --env CH_LIBC="${libc}" \
+            --env BUILD_TARGET="$target" \
+            --env RUSTFLAGS="$rustflags" \
+            --env TARGET_CC="$target_cc" \
            "$CTR_IMAGE" \
            ./scripts/run_integration_tests_rate_limiter.sh "$@" || fix_dir_perms $? || exit $?
    fi
@@ -538,7 +557,9 @@ cmd_tests() {
            --volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR" $exported_volumes \
            --volume "$CLH_INTEGRATION_WORKLOADS:$CTR_CLH_INTEGRATION_WORKLOADS" \
            --env USER="root" \
-            --env CH_LIBC="${libc}" \
+            --env BUILD_TARGET="$target" \
+            --env RUSTFLAGS="$rustflags" \
+            --env TARGET_CC="$target_cc" \
            --env RUST_BACKTRACE="${RUST_BACKTRACE}" \
            "$CTR_IMAGE" \
            ./scripts/run_metrics.sh "$@" || fix_dir_perms $? || exit $?
--- a/scripts/run_integration_tests_aarch64.sh
+++ b/scripts/run_integration_tests_aarch64.sh
@@ -11,15 +11,16 @@ build_spdk_nvme() {
    SPDK_DIR="$WORKLOADS_DIR/spdk"
    SPDK_REPO="https://github.com/spdk/spdk.git"
    SPDK_DEPLOY_DIR="/usr/local/bin/spdk-nvme"
-    checkout_repo "$SPDK_DIR" "$SPDK_REPO" master "6301f8915de32baed10dba1eebed556a6749211a"
+    checkout_repo "$SPDK_DIR" "$SPDK_REPO" master "ef8bcce58f3f02b79c0619a297e4f17e81e62b24"

    if [ ! -f "$SPDK_DIR/.built" ]; then
        pushd $SPDK_DIR
        git submodule update --init
        apt-get update
+        sed -i "/grpcio/d" scripts/pkgdep/debian.sh
        ./scripts/pkgdep.sh
        ./configure --with-vfio-user
-        chmod +x /usr/local/lib/python3.8/dist-packages/ninja/data/bin/ninja
+        chmod +x /usr/local/lib/python3.10/dist-packages/ninja/data/bin/ninja
        make -j `nproc` || exit 1
        touch .built
        popd
@@ -29,18 +30,20 @@ build_spdk_nvme() {
    fi
    cp "$WORKLOADS_DIR/spdk/build/bin/nvmf_tgt" $SPDK_DEPLOY_DIR/nvmf_tgt
    cp "$WORKLOADS_DIR/spdk/scripts/rpc.py" $SPDK_DEPLOY_DIR/rpc.py
-    cp -r "$WORKLOADS_DIR/spdk/scripts/rpc" $SPDK_DEPLOY_DIR/rpc
+    cp -r "$WORKLOADS_DIR/spdk/python/spdk/" $SPDK_DEPLOY_DIR/
+    cp -r "$WORKLOADS_DIR/spdk/python" $SPDK_DEPLOY_DIR/../
 }

 build_virtiofsd() {
    VIRTIOFSD_DIR="$WORKLOADS_DIR/virtiofsd_build"
    VIRTIOFSD_REPO="https://gitlab.com/virtio-fs/virtiofsd.git"

-    checkout_repo "$VIRTIOFSD_DIR" "$VIRTIOFSD_REPO" v1.1.0 "220405d7a2606c92636d31992b5cb3036a41047b"
+    checkout_repo "$VIRTIOFSD_DIR" "$VIRTIOFSD_REPO" v1.8.0 "97ea7908fe7f9bc59916671a771bdcfaf4044b45"

    if [ ! -f "$VIRTIOFSD_DIR/.built" ]; then
        pushd $VIRTIOFSD_DIR
-        time cargo build --release
+        rm -rf target/
+        time RUSTFLAGS="" TARGET_CC="" cargo build --release
        cp target/release/virtiofsd "$WORKLOADS_DIR/" || exit 1
        touch .built
        popd
@@ -156,12 +159,25 @@ update_workloads() {
    popd

    # Download Cloud Hypervisor binary from its last stable release
-    LAST_RELEASE_VERSION="v34.0"
+    LAST_RELEASE_VERSION="v36.0"
    CH_RELEASE_URL="https://github.com/cloud-hypervisor/cloud-hypervisor/releases/download/$LAST_RELEASE_VERSION/cloud-hypervisor-static-aarch64"
    CH_RELEASE_NAME="cloud-hypervisor-static-aarch64"
    pushd $WORKLOADS_DIR
-    time wget --quiet $CH_RELEASE_URL -O "$CH_RELEASE_NAME" || exit 1
-    chmod +x $CH_RELEASE_NAME
+    # Repeat a few times to workaround a random wget failure
+    WGET_RETRY_MAX=10
+    wget_retry=0
+
+    until [ "$wget_retry" -ge "$WGET_RETRY_MAX" ]
+    do
+        time wget $CH_RELEASE_URL -O "$CH_RELEASE_NAME" && break
+        wget_retry=$((wget_retry+1))
+    done
+
+    if [ $wget_retry -ge "$WGET_RETRY_MAX" ]; then
+        exit 1
+    else
+        chmod +x $CH_RELEASE_NAME
+    fi
    popd

    # Build custom kernel for guest VMs
@@ -231,12 +247,6 @@ if [ $RES -ne 0 ]; then
    exit 1
 fi

-BUILD_TARGET="aarch64-unknown-linux-${CH_LIBC}"
-if [[ "${BUILD_TARGET}" == "aarch64-unknown-linux-musl" ]]; then
-    export TARGET_CC="musl-gcc"
-    export RUSTFLAGS="-C link-arg=-lgcc -C link_arg=-specs -C link_arg=/usr/lib/aarch64-linux-musl/musl-gcc.specs"
-fi
-
 export RUST_BACKTRACE=1

 cargo build --all --release --target $BUILD_TARGET
--- a/scripts/run_integration_tests_live_migration.sh
+++ b/scripts/run_integration_tests_live_migration.sh
@@ -4,8 +4,6 @@ set -x
 source $HOME/.cargo/env
 source $(dirname "$0")/test-util.sh

-export BUILD_TARGET=${BUILD_TARGET-x86_64-unknown-linux-gnu}
-
 WORKLOADS_DIR="$HOME/workloads"
 mkdir -p "$WORKLOADS_DIR"

@@ -46,7 +44,7 @@ fi
 popd

 # Download Cloud Hypervisor binary from its last stable release
-LAST_RELEASE_VERSION="v34.0"
+LAST_RELEASE_VERSION="v36.0"
 CH_RELEASE_URL="https://github.com/cloud-hypervisor/cloud-hypervisor/releases/download/$LAST_RELEASE_VERSION/cloud-hypervisor-static"
 CH_RELEASE_NAME="cloud-hypervisor-static"
 pushd $WORKLOADS_DIR
@@ -60,11 +58,8 @@ if [ ! -f "$VMLINUX_IMAGE" ]; then
    build_custom_linux
 fi

-BUILD_TARGET="$(uname -m)-unknown-linux-${CH_LIBC}"
 CFLAGS=""
-TARGET_CC=""
 if [[ "${BUILD_TARGET}" == "x86_64-unknown-linux-musl" ]]; then
-    TARGET_CC="musl-gcc"
    CFLAGS="-I /usr/include/x86_64-linux-musl/ -idirafter /usr/include/"
 fi

--- a/scripts/run_integration_tests_rate_limiter.sh
+++ b/scripts/run_integration_tests_rate_limiter.sh
@@ -4,8 +4,6 @@ set -x
 source $HOME/.cargo/env
 source $(dirname "$0")/test-util.sh

-export BUILD_TARGET=${BUILD_TARGET-x86_64-unknown-linux-gnu}
-
 WORKLOADS_DIR="$HOME/workloads"
 mkdir -p "$WORKLOADS_DIR"

@@ -47,11 +45,8 @@ popd

 build_custom_linux

-BUILD_TARGET="$(uname -m)-unknown-linux-${CH_LIBC}"
 CFLAGS=""
-TARGET_CC=""
 if [[ "${BUILD_TARGET}" == "x86_64-unknown-linux-musl" ]]; then
-    TARGET_CC="musl-gcc"
    CFLAGS="-I /usr/include/x86_64-linux-musl/ -idirafter /usr/include/"
 fi

--- a/scripts/run_integration_tests_sgx.sh
+++ b/scripts/run_integration_tests_sgx.sh
@@ -38,11 +38,8 @@ if [ ! -f "$JAMMY_OS_RAW_IMAGE" ]; then
    popd
 fi

-BUILD_TARGET="$(uname -m)-unknown-linux-${CH_LIBC}"
 CFLAGS=""
-TARGET_CC=""
 if [[ "${BUILD_TARGET}" == "x86_64-unknown-linux-musl" ]]; then
-    TARGET_CC="musl-gcc"
    CFLAGS="-I /usr/include/x86_64-linux-musl/ -idirafter /usr/include/"
 fi

--- a/scripts/run_integration_tests_vfio.sh
+++ b/scripts/run_integration_tests_vfio.sh
@@ -1,100 +1,34 @@
 #!/bin/bash
 set -x

+# This set of vfio tests require to be ran on a specific machine with
+# specific hardware (e.g. the "vfio" bera-metal worker equipped with a
+# Nvidia Tesla T4 card). So the provisioning of the running machine is
+# out of the scope of this script, including the custom guest image with
+# Nvidia drivers installed, and properly configured Nvidia Tesla T4 card.
+
 source $HOME/.cargo/env
 source $(dirname "$0")/test-util.sh

 process_common_args "$@"

 WORKLOADS_DIR="$HOME/workloads"
-mkdir -p "$WORKLOADS_DIR"
-
-cp scripts/sha1sums-x86_64 $WORKLOADS_DIR

+# Always download the latest "hypervisor-fw"
 FW_URL=$(curl --silent https://api.github.com/repos/cloud-hypervisor/rust-hypervisor-firmware/releases/latest | grep "browser_download_url" | grep -o 'https://.*[^ "]')
-FW="$WORKLOADS_DIR/hypervisor-fw"
-if [ ! -f "$FW" ]; then
-    pushd $WORKLOADS_DIR
-    time wget --quiet $FW_URL || exit 1
-    popd
-fi
-
-FOCAL_OS_IMAGE_NAME="focal-server-cloudimg-amd64-custom-20210609-0.qcow2"
-FOCAL_OS_IMAGE_URL="https://cloud-hypervisor.azureedge.net/$FOCAL_OS_IMAGE_NAME"
-FOCAL_OS_IMAGE="$WORKLOADS_DIR/$FOCAL_OS_IMAGE_NAME"
-if [ ! -f "$FOCAL_OS_IMAGE" ]; then
-    pushd $WORKLOADS_DIR
-    time wget --quiet $FOCAL_OS_IMAGE_URL || exit 1
-    popd
-fi
-
-FOCAL_OS_RAW_IMAGE_NAME="focal-server-cloudimg-amd64-custom-20210609-0.raw"
-FOCAL_OS_RAW_IMAGE="$WORKLOADS_DIR/$FOCAL_OS_RAW_IMAGE_NAME"
-if [ ! -f "$FOCAL_OS_RAW_IMAGE" ]; then
-    pushd $WORKLOADS_DIR
-    time qemu-img convert -p -f qcow2 -O raw $FOCAL_OS_IMAGE_NAME $FOCAL_OS_RAW_IMAGE_NAME || exit 1
-    popd
-fi
-
 pushd $WORKLOADS_DIR
-sha1sum sha1sums-x86_64 --check --ignore-missing
-if [ $? -ne 0 ]; then
-    echo "sha1sum validation of images failed, remove invalid images to fix the issue."
-    exit 1
-fi
+time wget --quiet $FW_URL -O hypervisor-fw || exit 1
 popd

-VMLINUX_IMAGE="$WORKLOADS_DIR/vmlinux"
-build_custom_linux
-
-BLK_IMAGE="$WORKLOADS_DIR/blk.img"
-MNT_DIR="mount_image"
-rm -rf $BLK_IMAGE
-pushd $WORKLOADS_DIR
-fallocate -l 16M $BLK_IMAGE
-mkfs.ext4 -j $BLK_IMAGE
-mkdir $MNT_DIR
-sudo mount -t ext4 $BLK_IMAGE $MNT_DIR
-sudo bash -c "echo bar > $MNT_DIR/foo" || exit 1
-sudo umount $BLK_IMAGE
-rm -r $MNT_DIR
-popd
-
-VFIO_DIR="$WORKLOADS_DIR/vfio"
-VFIO_DISK_IMAGE="$WORKLOADS_DIR/vfio.img"
-rm -rf $VFIO_DIR $VFIO_DISK_IMAGE
-mkdir -p $VFIO_DIR
-cp $FOCAL_OS_RAW_IMAGE $VFIO_DIR
-cp $FW $VFIO_DIR
-cp $VMLINUX_IMAGE $VFIO_DIR || exit 1
-
-BUILD_TARGET="$(uname -m)-unknown-linux-${CH_LIBC}"
 CFLAGS=""
-TARGET_CC=""
 if [[ "${BUILD_TARGET}" == "x86_64-unknown-linux-musl" ]]; then
-TARGET_CC="musl-gcc"
-CFLAGS="-I /usr/include/x86_64-linux-musl/ -idirafter /usr/include/"
+    CFLAGS="-I /usr/include/x86_64-linux-musl/ -idirafter /usr/include/"
 fi

 cargo build --features mshv --all --release --target $BUILD_TARGET

-# We always copy a fresh version of our binary for our L2 guest.
-cp target/$BUILD_TARGET/release/cloud-hypervisor $VFIO_DIR
-cp target/$BUILD_TARGET/release/ch-remote $VFIO_DIR
-
-# test_vfio rely on hugepages
-HUGEPAGESIZE=`grep Hugepagesize /proc/meminfo | awk '{print $2}'`
-PAGE_NUM=`echo $((12288 * 1024 / $HUGEPAGESIZE))`
-echo $PAGE_NUM | sudo tee /proc/sys/vm/nr_hugepages
-sudo chmod a+rwX /dev/hugepages
-
 export RUST_BACKTRACE=1
-time cargo test "vfio::test_vfio" -- ${test_binary_args[*]}
+time cargo test "vfio::test_nvidia" -- --test-threads=1 ${test_binary_args[*]}
 RES=$?

-if [ $RES -eq 0 ]; then
-	time cargo test "vfio::test_nvidia" -- --test-threads=1 ${test_binary_args[*]}
-	RES=$?
-fi
-
 exit $RES
--- a/scripts/run_integration_tests_windows_aarch64.sh
+++ b/scripts/run_integration_tests_windows_aarch64.sh
@@ -20,14 +20,6 @@ WIN_IMAGE_FILE="$WORKLOADS_DIR/$WIN_IMAGE_BASENAME"
 OVMF_FW="$WORKLOADS_DIR/CLOUDHV_EFI.fd"
 build_edk2

-BUILD_TARGET="$(uname -m)-unknown-linux-${CH_LIBC}"
-CFLAGS=""
-TARGET_CC=""
-if [[ "${BUILD_TARGET}" == "aarch64-unknown-linux-musl" ]]; then
-    export TARGET_CC="musl-gcc"
-    export RUSTFLAGS="-C link-arg=-lgcc -C link_arg=-specs -C link_arg=/usr/lib/aarch64-linux-musl/musl-gcc.specs"
-fi
-
 # Check if the images are present
 if [[ ! -f ${WIN_IMAGE_FILE} || ! -f ${OVMF_FW} ]]; then
    echo "Windows image/firmware not present in the host"
--- a/scripts/run_integration_tests_windows_x86_64.sh
+++ b/scripts/run_integration_tests_windows_x86_64.sh
@@ -22,11 +22,8 @@ if [ ! -f "$OVMF_FW" ]; then
    popd
 fi

-BUILD_TARGET="$(uname -m)-unknown-linux-${CH_LIBC}"
 CFLAGS=""
-TARGET_CC=""
 if [[ "${BUILD_TARGET}" == "x86_64-unknown-linux-musl" ]]; then
-    TARGET_CC="musl-gcc"
    CFLAGS="-I /usr/include/x86_64-linux-musl/ -idirafter /usr/include/"
 fi

--- a/scripts/run_integration_tests_x86_64.sh
+++ b/scripts/run_integration_tests_x86_64.sh
@@ -4,8 +4,6 @@ set -x
 source $HOME/.cargo/env
 source $(dirname "$0")/test-util.sh

-export BUILD_TARGET=${BUILD_TARGET-x86_64-unknown-linux-gnu}
-
 WORKLOADS_DIR="$HOME/workloads"
 mkdir -p "$WORKLOADS_DIR"

@@ -124,7 +122,7 @@ if [ ! -f "$VIRTIOFSD" ]; then
    pushd $WORKLOADS_DIR
    git clone "https://gitlab.com/virtio-fs/virtiofsd.git" $VIRTIOFSD_DIR
    pushd $VIRTIOFSD_DIR
-    git checkout v1.1.0
+    git checkout v1.8.0
    time cargo build --release
    cp target/release/virtiofsd $VIRTIOFSD || exit 1
    popd
@@ -162,8 +160,6 @@ cp $FOCAL_OS_RAW_IMAGE $VFIO_DIR
 cp $FW $VFIO_DIR
 cp $VMLINUX_IMAGE $VFIO_DIR || exit 1

-BUILD_TARGET="$(uname -m)-unknown-linux-${CH_LIBC}"
-
 cargo build --features mshv --all  --release --target $BUILD_TARGET

 # We always copy a fresh version of our binary for our L2 guest.
--- a/scripts/run_metrics.sh
+++ b/scripts/run_metrics.sh
@@ -5,8 +5,6 @@ source $HOME/.cargo/env
 source $(dirname "$0")/test-util.sh

 export TEST_ARCH=$(uname -m)
-export BUILD_TARGET=${BUILD_TARGET-${TEST_ARCH}-unknown-linux-gnu}
-

 WORKLOADS_DIR="$HOME/workloads"
 mkdir -p "$WORKLOADS_DIR"
@@ -84,11 +82,8 @@ fi
 # Build custom kernel based on virtio-pmem and virtio-fs upstream patches
 build_custom_linux

-BUILD_TARGET="${TEST_ARCH}-unknown-linux-${CH_LIBC}"
 CFLAGS=""
-TARGET_CC=""
 if [[ "${BUILD_TARGET}" == "${TEST_ARCH}-unknown-linux-musl" ]]; then
-    TARGET_CC="musl-gcc"
    CFLAGS="-I /usr/include/${TEST_ARCH}-linux-musl/ -idirafter /usr/include/"
 fi

--- a/scripts/run_unit_tests.sh
+++ b/scripts/run_unit_tests.sh
@@ -5,7 +5,6 @@ source $(dirname "$0")/test-util.sh

 process_common_args "$@"

-BUILD_TARGET=${BUILD_TARGET-x86_64-unknown-linux-gnu}
 cargo_args=("")

 if [[ $hypervisor = "mshv" ]]; then
@@ -14,11 +13,6 @@ elif [[ $(uname -m) = "x86_64" ]]; then
    cargo_args+=("--features tdx")
 fi

-if [[ "${BUILD_TARGET}" == "aarch64-unknown-linux-musl" ]]; then
-    export TARGET_CC="musl-gcc"
-    export RUSTFLAGS="-C link-arg=-lgcc -C link_arg=-specs -C link_arg=/usr/lib/aarch64-linux-musl/musl-gcc.specs"
-fi
-
 export RUST_BACKTRACE=1
 cargo test --lib --bins --target $BUILD_TARGET --workspace ${cargo_args[@]} || exit 1
 cargo test --doc --target $BUILD_TARGET --workspace ${cargo_args[@]} || exit 1
--- a/src/bin/ch-remote.rs
+++ b/src/bin/ch-remote.rs
--- a/src/main.rs
+++ b/src/main.rs
@@ -6,7 +6,7 @@
 #[macro_use]
 extern crate event_monitor;

-use argh::FromArgs;
+use clap::{Arg, ArgAction, ArgGroup, ArgMatches, Command};
 use libc::EFD_NONBLOCK;
 use log::{warn, LevelFilter};
 use option_parser::OptionParser;
@@ -128,6 +128,10 @@ impl log::Log for Logger {
    fn flush(&self) {}
 }

+fn prepare_default_values() -> (String, String, String) {
+    (default_vcpus(), default_memory(), default_rng())
+}
+
 fn default_vcpus() -> String {
    format!(
        "boot={},max_phys_bits={}",
@@ -144,280 +148,323 @@ fn default_rng() -> String {
    format!("src={}", config::DEFAULT_RNG_SOURCE)
 }

-#[derive(FromArgs)]
-/// Launch a cloud-hypervisor VMM.
-pub struct TopLevel {
-    #[argh(option, long = "cpus", default = "default_vcpus()")]
-    /// boot=<boot_vcpus>, max=<max_vcpus>, topology=<threads_per_core>:<cores_per_die>:<dies_per_package>:<packages>, kvm_hyperv=on|off, max_phys_bits=<maximum_number_of_physical_bits>, affinity=<list_of_vcpus_with_their_associated_cpuset>, features=<list_of_features_to_enable>
-    cpus: String,
-
-    #[argh(option, long = "platform")]
-    /// num_pci_segments=<num_pci_segments>, iommu_segments=<list_of_segments>, serial_number=<dmi_device_serial_number>, uuid=<dmi_device_uuid>, oem_strings=<list_of_strings>
-    platform: Option<String>,
-
-    #[argh(option, long = "memory", default = "default_memory()")]
-    /// size=<guest_memory_size>, mergeable=on|off, shared=on|off, hugepages=on|off, hugepage_size=<hugepage_size>, hotplug_method=acpi|virtio-mem, hotplug_size=<hotpluggable_memory_size>, hotplugged_size=<hotplugged_memory_size>, prefault=on|off, thp=on|off
-    memory: String,
-
-    #[argh(option, long = "memory-zone")]
-    /// size=<guest_memory_region_size>, file=<backing_file>, shared=on|off, hugepages=on|off, hugepage_size=<hugepage_size>, host_numa_node=<node_id>, id=<zone_identifier>, hotplug_size=<hotpluggable_memory_size>, hotplugged_size=<hotplugged_memory_size>, prefault=on|off
-    memory_zone: Vec<String>,
-
-    #[argh(option, long = "firmware")]
-    /// path to firmware that is loaded in an architectural specific way
-    firmware: Option<String>,
-
-    #[argh(option, long = "kernel")]
-    /// path to kernel or firmware that supports a PVH entry point or architecture equivalent
-    kernel: Option<String>,
-
-    #[argh(option, long = "initramfs")]
-    /// path to initramfs image
-    initramfs: Option<String>,
-
-    #[argh(option, long = "cmdline")]
-    /// kernel command line
-    cmdline: Option<String>,
-
-    #[argh(option, long = "disk")]
-    /// path=<disk_image_path>, readonly=on|off, direct=on|off, iommu=on|off, num_queues=<number_of_queues>, queue_size=<size_of_each_queue>, vhost_user=on|off, socket=<vhost_user_socket_path>, bw_size=<bytes>, bw_one_time_burst=<bytes>, bw_refill_time=<ms>, ops_size=<io_ops>, ops_one_time_burst=<io_ops>, ops_refill_time=<ms>, id=<device_id>, pci_segment=<segment_id>
-    disk: Vec<String>,
-
-    #[argh(option, long = "net")]
-    /// tap=<if_name>, ip=<ip_addr>, mask=<net_mask>, mac=<mac_addr>, fd=<fd1,fd2...>, iommu=on|off, num_queues=<number_of_queues>, queue_size=<size_of_each_queue>, id=<device_id>, vhost_user=<vhost_user_enable>, socket=<vhost_user_socket_path>, vhost_mode=client|server, bw_size=<bytes>, bw_one_time_burst=<bytes>, bw_refill_time=<ms>, ops_size=<io_ops>, ops_one_time_burst=<io_ops>, ops_refill_time=<ms>, pci_segment=<segment_id>, offload_tso=on|off, offload_ufo=on|off, offload_csum=on|off
-    net: Vec<String>,
-
-    #[argh(option, long = "rng", default = "default_rng()")]
-    /// src=<entropy_source_path>, iommu=on|off
-    rng: String,
-
-    #[argh(option, long = "balloon")]
-    /// size=<balloon_size>, deflate_on_oom=on|off, free_page_reporting=on|off
-    balloon: Option<String>,
-
-    #[argh(option, long = "fs")]
-    /// tag=<tag_name>, socket=<socket_path>, num_queues=<number_of_queues>, queue_size=<size_of_each_queue>, id=<device_id>, pci_segment=<segment_id>
-    fs: Vec<String>,
-
-    #[argh(option, long = "pmem")]
-    /// file=<backing_file_path>, size=<persistent_memory_size>, iommu=on|off, discard_writes=on|off, id=<device_id>, pci_segment=<segment_id>
-    pmem: Vec<String>,
-
-    #[argh(option, long = "serial", default = "String::from(\"null\")")]
-    /// off|null|pty|tty|file=/path/to/a/file
-    serial: String,
-
-    #[argh(option, long = "console", default = "String::from(\"tty\")")]
-    /// off|null|pty|tty|file=/path/to/a/file, iommu=on|off
-    console: String,
-
-    #[argh(option, long = "device")]
-    /// path=<device_path>, iommu=on|off, id=<device_id>, pci_segment=<segment_id>
-    device: Vec<String>,
-
-    #[argh(option, long = "user-device")]
-    /// socket=<socket_path>, id=<device_id>, pci_segment=<segment_id>
-    user_device: Vec<String>,
-
-    #[argh(option, long = "vdpa")]
-    /// path=<device_path>, num_queues=<number_of_queues>, iommu=on|off, id=<device_id>, pci_segment=<segment_id>
-    vdpa: Vec<String>,
-
-    #[argh(option, long = "vsock")]
-    /// cid=<context_id>, socket=<socket_path>, iommu=on|off, id=<device_id>, pci_segment=<segment_id>
-    vsock: Option<String>,
-
-    #[argh(switch, long = "pvpanic")]
-    /// enable pvpanic device
-    pvpanic: bool,
-
-    #[argh(option, long = "numa")]
-    /// guest_numa_id=<node_id>, cpus=<cpus_id>, distances=<list_of_distances_to_destination_nodes>, memory_zones=<list_of_memory_zones>, sgx_epc_sections=<list_of_sgx_epc_sections>
-    numa: Vec<String>,
-
-    #[argh(switch, long = "watchdog")]
-    /// enable virtio-watchdog
-    watchdog: bool,
-
-    #[argh(switch, short = 'v')]
-    /// set the level of debugging output
-    verbosity: u8,
-
-    #[argh(option, long = "log-file")]
-    /// path to log file
-    log_file: Option<String>,
-
-    #[argh(option, long = "api-socket")]
-    /// path=<path/to/a/file>|fd=<fd>
-    api_socket: Option<String>,
-
-    #[cfg(feature = "dbus_api")]
-    #[argh(option, long = "dbus-service-name")]
-    /// well known name of the service
-    dbus_name: Option<String>,
-
-    #[cfg(feature = "dbus_api")]
-    #[argh(option, long = "dbus-object-path")]
-    /// object path to serve the dbus interface
-    dbus_path: Option<String>,
-
-    #[cfg(feature = "dbus_api")]
-    #[argh(switch, long = "dbus-system-bus")]
-    /// use the system bus instead of a session bus
-    dbus_system_bus: bool,
-
-    #[argh(option, long = "event-monitor")]
-    /// path=<path/to/a/file>|fd=<fd>
-    event_monitor: Option<String>,
-
-    #[argh(option, long = "restore")]
-    /// source_url=<source_url>, prefault=on|off
-    restore: Option<String>,
-
-    #[argh(option, long = "seccomp", default = "String::from(\"true\")")]
-    /// seccomp configuration (true, false or log)
-    seccomp: String,
-
-    #[argh(option, long = "tpm")]
-    /// socket=<path/to/a/socket>
-    tpm: Option<String>,
+fn create_app(default_vcpus: String, default_memory: String, default_rng: String) -> Command {
+    #[allow(clippy::let_and_return)]
+    let app = Command::new("cloud-hypervisor")
+        // 'BUILD_VERSION' is set by the build script 'build.rs' at
+        // compile time
+        .author(env!("CARGO_PKG_AUTHORS"))
+        .about("Launch a cloud-hypervisor VMM.")
+        .group(ArgGroup::new("vm-config").multiple(true))
+        .group(ArgGroup::new("vmm-config").multiple(true))
+        .group(ArgGroup::new("logging").multiple(true))
+        .arg(
+            Arg::new("cpus")
+                .long("cpus")
+                .help(
+                    "boot=<boot_vcpus>,max=<max_vcpus>,\
+                    topology=<threads_per_core>:<cores_per_die>:<dies_per_package>:<packages>,\
+                    kvm_hyperv=on|off,max_phys_bits=<maximum_number_of_physical_bits>,\
+                    affinity=<list_of_vcpus_with_their_associated_cpuset>,\
+                    features=<list_of_features_to_enable>",
+                )
+                .default_value(default_vcpus)
+                .group("vm-config"),
+        )
+        .arg(
+            Arg::new("platform")
+                .long("platform")
+                .help("num_pci_segments=<num_pci_segments>,iommu_segments=<list_of_segments>,serial_number=<dmi_device_serial_number>,uuid=<dmi_device_uuid>,oem_strings=<list_of_strings>")
+                .num_args(1)
+                .group("vm-config"),
+        )
+        .arg(
+            Arg::new("memory")
+                .long("memory")
+                .help(
+                    "Memory parameters \
+                     \"size=<guest_memory_size>,mergeable=on|off,shared=on|off,\
+                     hugepages=on|off,hugepage_size=<hugepage_size>,\
+                     hotplug_method=acpi|virtio-mem,\
+                     hotplug_size=<hotpluggable_memory_size>,\
+                     hotplugged_size=<hotplugged_memory_size>,\
+                     prefault=on|off,thp=on|off\"",
+                )
+                .default_value(default_memory)
+                .group("vm-config"),
+        )
+        .arg(
+            Arg::new("memory-zone")
+                .long("memory-zone")
+                .help(
+                    "User defined memory zone parameters \
+                     \"size=<guest_memory_region_size>,file=<backing_file>,\
+                     shared=on|off,\
+                     hugepages=on|off,hugepage_size=<hugepage_size>,\
+                     host_numa_node=<node_id>,\
+                     id=<zone_identifier>,hotplug_size=<hotpluggable_memory_size>,\
+                     hotplugged_size=<hotplugged_memory_size>,\
+                     prefault=on|off\"",
+                )
+                .num_args(1..)
+                .group("vm-config"),
+        )
+        .arg(
+            Arg::new("firmware")
+                .long("firmware")
+                .help("Path to firmware that is loaded in an architectural specific way")
+                .num_args(1)
+                .group("vm-config"),
+        )
+        .arg(
+            Arg::new("kernel")
+                .long("kernel")
+                .help(
+                    "Path to kernel to load. This may be a kernel or firmware that supports a PVH \
+                entry point (e.g. vmlinux) or architecture equivalent",
+                )
+                .num_args(1)
+                .group("vm-config"),
+        )
+        .arg(
+            Arg::new("initramfs")
+                .long("initramfs")
+                .help("Path to initramfs image")
+                .num_args(1)
+                .group("vm-config"),
+        )
+        .arg(
+            Arg::new("cmdline")
+                .long("cmdline")
+                .help("Kernel command line")
+                .num_args(1)
+                .group("vm-config"),
+        )
+        .arg(
+            Arg::new("disk")
+                .long("disk")
+                .help(config::DiskConfig::SYNTAX)
+                .num_args(1..)
+                .group("vm-config"),
+        )
+        .arg(
+            Arg::new("net")
+                .long("net")
+                .help(config::NetConfig::SYNTAX)
+                .num_args(1..)
+                .group("vm-config"),
+        )
+        .arg(
+            Arg::new("rng")
+                .long("rng")
+                .help(
+                    "Random number generator parameters \"src=<entropy_source_path>,iommu=on|off\"",
+                )
+                .default_value(default_rng)
+                .group("vm-config"),
+        )
+        .arg(
+            Arg::new("balloon")
+                .long("balloon")
+                .help(config::BalloonConfig::SYNTAX)
+                .num_args(1)
+                .group("vm-config"),
+        )
+        .arg(
+            Arg::new("fs")
+                .long("fs")
+                .help(config::FsConfig::SYNTAX)
+                .num_args(1..)
+                .group("vm-config"),
+        )
+        .arg(
+            Arg::new("pmem")
+                .long("pmem")
+                .help(config::PmemConfig::SYNTAX)
+                .num_args(1..)
+                .group("vm-config"),
+        )
+        .arg(
+            Arg::new("serial")
+                .long("serial")
+                .help("Control serial port: off|null|pty|tty|file=</path/to/a/file>|socket=</path/to/a/file>")
+                .default_value("null")
+                .group("vm-config"),
+        )
+        .arg(
+            Arg::new("console")
+                .long("console")
+                .help(
+                    "Control (virtio) console: \"off|null|pty|tty|file=</path/to/a/file>,iommu=on|off\"",
+                )
+                .default_value("tty")
+                .group("vm-config"),
+        )
+        .arg(
+            Arg::new("device")
+                .long("device")
+                .help(config::DeviceConfig::SYNTAX)
+                .num_args(1..)
+                .group("vm-config"),
+        )
+        .arg(
+            Arg::new("user-device")
+                .long("user-device")
+                .help(config::UserDeviceConfig::SYNTAX)
+                .num_args(1..)
+                .group("vm-config"),
+        )
+        .arg(
+            Arg::new("vdpa")
+                .long("vdpa")
+                .help(config::VdpaConfig::SYNTAX)
+                .num_args(1..)
+                .group("vm-config"),
+        )
+        .arg(
+            Arg::new("vsock")
+                .long("vsock")
+                .help(config::VsockConfig::SYNTAX)
+                .num_args(1)
+                .group("vm-config"),
+        )
+        .arg(
+            Arg::new("pvpanic")
+                .long("pvpanic")
+                .help("Enable pvpanic device")
+                .num_args(0)
+                .action(ArgAction::SetTrue)
+                .group("vm-config"),
+        )
+        .arg(
+            Arg::new("numa")
+                .long("numa")
+                .help(config::NumaConfig::SYNTAX)
+                .num_args(1..)
+                .group("vm-config"),
+        )
+        .arg(
+            Arg::new("watchdog")
+                .long("watchdog")
+                .help("Enable virtio-watchdog")
+                .num_args(0)
+                .action(ArgAction::SetTrue)
+                .group("vm-config"),
+        )
+        .arg(
+            Arg::new("v")
+                .short('v')
+                .action(ArgAction::Count)
+                .help("Sets the level of debugging output")
+                .group("logging"),
+        )
+        .arg(
+            Arg::new("log-file")
+                .long("log-file")
+                .help("Log file. Standard error is used if not specified")
+                .num_args(1)
+                .group("logging"),
+        )
+        .arg(
+            Arg::new("api-socket")
+                .long("api-socket")
+                .help("HTTP API socket (UNIX domain socket): path=</path/to/a/file> or fd=<fd>.")
+                .num_args(1)
+                .group("vmm-config"),
+        )
+        .arg(
+            Arg::new("event-monitor")
+                .long("event-monitor")
+                .help("File to report events on: path=</path/to/a/file> or fd=<fd>")
+                .num_args(1)
+                .group("vmm-config"),
+        )
+        .arg(
+            Arg::new("restore")
+                .long("restore")
+                .help(config::RestoreConfig::SYNTAX)
+                .num_args(1)
+                .group("vmm-config"),
+        )
+        .arg(
+            Arg::new("seccomp")
+                .long("seccomp")
+                .num_args(1)
+                .value_parser(["true", "false", "log"])
+                .default_value("true"),
+        )
+        .arg(
+            Arg::new("tpm")
+                .long("tpm")
+                .num_args(1)
+                .help(config::TpmConfig::SYNTAX)
+                .group("vmm-config"),
+        );

    #[cfg(target_arch = "x86_64")]
-    #[argh(option, long = "sgx-epc")]
-    /// id=<epc_section_identifier>, size=<epc_section_size>, prefault=on|off
-    sgx_epc: Vec<String>,
+    let app = app.arg(
+        Arg::new("sgx-epc")
+            .long("sgx-epc")
+            .help(config::SgxEpcConfig::SYNTAX)
+            .num_args(1..)
+            .group("vm-config"),
+    );

    #[cfg(feature = "guest_debug")]
-    #[argh(option, long = "gdb")]
-    /// path=<path/to/a/file>
-    gdb: Option<String>,
+    let app = app.arg(
+        Arg::new("gdb")
+            .long("gdb")
+            .help("GDB socket (UNIX domain socket): path=</path/to/a/file>")
+            .num_args(1)
+            .group("vmm-config"),
+    );

-    #[argh(switch, short = 'V', long = "version")]
-    /// print version information
-    version: bool,
+    #[cfg(feature = "dbus_api")]
+    let app = app
+        .arg(
+            Arg::new("dbus-service-name")
+                .long("dbus-service-name")
+                .help("Well known name of the device")
+                .num_args(1)
+                .group("vmm-config"),
+        )
+        .arg(
+            Arg::new("dbus-object-path")
+                .long("dbus-object-path")
+                .help("Object path to serve the dbus interface")
+                .num_args(1)
+                .group("vmm-config"),
+        )
+        .arg(
+            Arg::new("dbus-system-bus")
+                .long("dbus-system-bus")
+                .action(ArgAction::SetTrue)
+                .help("Use the system bus instead of a session bus")
+                .num_args(0)
+                .group("vmm-config"),
+        );
+
+    app.arg(
+        Arg::new("version")
+            .short('V')
+            .long("version")
+            .action(ArgAction::SetTrue)
+            .help("Print version")
+            .num_args(0),
+    )
 }

-impl TopLevel {
-    fn to_vm_params(&self) -> config::VmParams<'_> {
-        let cpus = &self.cpus;
-        let memory = &self.memory;
-        let memory_zones = if !self.memory_zone.is_empty() {
-            Some(self.memory_zone.iter().map(|x| x.as_str()).collect())
-        } else {
-            None
-        };
-        let rng = &self.rng;
-        let serial = &self.serial;
-        let firmware = self.firmware.as_deref();
-        let kernel = self.kernel.as_deref();
-        let initramfs = self.initramfs.as_deref();
-        let cmdline = self.cmdline.as_deref();
-
-        let disks = if !self.disk.is_empty() {
-            Some(self.disk.iter().map(|x| x.as_str()).collect())
-        } else {
-            None
-        };
-
-        let net = if !self.net.is_empty() {
-            Some(self.net.iter().map(|x| x.as_str()).collect())
-        } else {
-            None
-        };
-
-        let console = &self.console;
-        let balloon = self.balloon.as_deref();
-        let fs = if !self.fs.is_empty() {
-            Some(self.fs.iter().map(|x| x.as_str()).collect())
-        } else {
-            None
-        };
-
-        let pmem = if !self.pmem.is_empty() {
-            Some(self.pmem.iter().map(|x| x.as_str()).collect())
-        } else {
-            None
-        };
-        let devices = if !self.device.is_empty() {
-            Some(self.device.iter().map(|x| x.as_str()).collect())
-        } else {
-            None
-        };
-        let user_devices = if !self.user_device.is_empty() {
-            Some(self.user_device.iter().map(|x| x.as_str()).collect())
-        } else {
-            None
-        };
-        let vdpa = if !self.vdpa.is_empty() {
-            Some(self.vdpa.iter().map(|x| x.as_str()).collect())
-        } else {
-            None
-        };
-
-        let vsock = self.vsock.as_deref();
-
-        let pvpanic = self.pvpanic;
-
-        #[cfg(target_arch = "x86_64")]
-        let sgx_epc = if !self.sgx_epc.is_empty() {
-            Some(self.sgx_epc.iter().map(|x| x.as_str()).collect())
-        } else {
-            None
-        };
-
-        let numa = if !self.numa.is_empty() {
-            Some(self.numa.iter().map(|x| x.as_str()).collect())
-        } else {
-            None
-        };
-        let watchdog = self.watchdog;
-        let platform = self.platform.as_deref();
-        #[cfg(feature = "guest_debug")]
-        let gdb = self.gdb.is_some();
-        let tpm = self.tpm.as_deref();
-
-        config::VmParams {
-            cpus,
-            memory,
-            memory_zones,
-            firmware,
-            kernel,
-            initramfs,
-            cmdline,
-            disks,
-            net,
-            rng,
-            balloon,
-            fs,
-            pmem,
-            serial,
-            console,
-            devices,
-            user_devices,
-            vdpa,
-            vsock,
-            pvpanic,
-            #[cfg(target_arch = "x86_64")]
-            sgx_epc,
-            numa,
-            watchdog,
-            #[cfg(feature = "guest_debug")]
-            gdb,
-            platform,
-            tpm,
-        }
-    }
-}
-
-fn start_vmm(toplevel: TopLevel) -> Result<Option<String>, Error> {
-    let log_level = match toplevel.verbosity {
+fn start_vmm(cmd_arguments: ArgMatches) -> Result<Option<String>, Error> {
+    let log_level = match cmd_arguments.get_count("v") {
        0 => LevelFilter::Warn,
        1 => LevelFilter::Info,
        2 => LevelFilter::Debug,
        _ => LevelFilter::Trace,
    };

-    let log_file: Box<dyn std::io::Write + Send> = if let Some(ref file) = toplevel.log_file {
+    let log_file: Box<dyn std::io::Write + Send> = if let Some(ref file) =
+        cmd_arguments.get_one::<String>("log-file")
+    {
        Box::new(std::fs::File::create(std::path::Path::new(file)).map_err(Error::LogFileCreation)?)
    } else {
        Box::new(std::io::stderr())
@@ -430,37 +477,47 @@ fn start_vmm(toplevel: TopLevel) -> Result<Option<String>, Error> {
    .map(|()| log::set_max_level(log_level))
    .map_err(Error::LoggerSetup)?;

-    let (api_socket_path, api_socket_fd) = if let Some(ref socket_config) = toplevel.api_socket {
-        let mut parser = OptionParser::new();
-        parser.add("path").add("fd");
-        parser.parse(socket_config).unwrap_or_default();
+    let (api_socket_path, api_socket_fd) =
+        if let Some(socket_config) = cmd_arguments.get_one::<String>("api-socket") {
+            let mut parser = OptionParser::new();
+            parser.add("path").add("fd");
+            parser.parse(socket_config).unwrap_or_default();

-        if let Some(fd) = parser.get("fd") {
-            (
-                None,
-                Some(fd.parse::<RawFd>().map_err(Error::ParsingApiSocket)?),
-            )
-        } else if let Some(path) = parser.get("path") {
-            (Some(path), None)
+            if let Some(fd) = parser.get("fd") {
+                (
+                    None,
+                    Some(fd.parse::<RawFd>().map_err(Error::ParsingApiSocket)?),
+                )
+            } else if let Some(path) = parser.get("path") {
+                (Some(path), None)
+            } else {
+                (
+                    cmd_arguments
+                        .get_one::<String>("api-socket")
+                        .map(|s| s.to_string()),
+                    None,
+                )
+            }
        } else {
-            (toplevel.api_socket.as_ref().map(|s| s.to_string()), None)
-        }
-    } else {
-        (None, None)
-    };
+            (None, None)
+        };

    let (api_request_sender, api_request_receiver) = channel();
    let api_evt = EventFd::new(EFD_NONBLOCK).map_err(Error::CreateApiEventFd)?;

    let api_request_sender_clone = api_request_sender.clone();
-    let seccomp_action = match &toplevel.seccomp as &str {
-        "true" => SeccompAction::Trap,
-        "false" => SeccompAction::Allow,
-        "log" => SeccompAction::Log,
-        val => {
-            // The user providing an invalid value will be rejected
-            panic!("Invalid parameter {val} for \"--seccomp\" flag");
+    let seccomp_action = if let Some(seccomp_value) = cmd_arguments.get_one::<String>("seccomp") {
+        match seccomp_value as &str {
+            "true" => SeccompAction::Trap,
+            "false" => SeccompAction::Allow,
+            "log" => SeccompAction::Log,
+            val => {
+                // The user providing an invalid value will be rejected
+                panic!("Invalid parameter {val} for \"--seccomp\" flag");
+            }
        }
+    } else {
+        SeccompAction::Trap
    };

    if seccomp_action == SeccompAction::Trap {
@@ -498,7 +555,7 @@ fn start_vmm(toplevel: TopLevel) -> Result<Option<String>, Error> {
    let hypervisor = hypervisor::new().map_err(Error::CreateHypervisor)?;

    #[cfg(feature = "guest_debug")]
-    let gdb_socket_path = if let Some(ref gdb_config) = toplevel.gdb {
+    let gdb_socket_path = if let Some(gdb_config) = cmd_arguments.get_one::<String>("gdb") {
        let mut parser = OptionParser::new();
        parser.add("path");
        parser.parse(gdb_config).map_err(Error::ParsingGdb)?;
@@ -519,8 +576,8 @@ fn start_vmm(toplevel: TopLevel) -> Result<Option<String>, Error> {
    let exit_evt = EventFd::new(EFD_NONBLOCK).map_err(Error::CreateExitEventFd)?;

    #[allow(unused_mut)]
-    let mut event_monitor = toplevel
-        .event_monitor
+    let mut event_monitor = cmd_arguments
+        .get_one::<String>("event-monitor")
        .as_ref()
        .map(|monitor_config| {
            let mut parser = OptionParser::new();
@@ -555,8 +612,11 @@ fn start_vmm(toplevel: TopLevel) -> Result<Option<String>, Error> {
        .transpose()?;

    #[cfg(feature = "dbus_api")]
-    let dbus_options = match (&toplevel.dbus_name, &toplevel.dbus_path) {
-        (Some(ref name), Some(ref path)) => {
+    let dbus_options = match (
+        cmd_arguments.get_one::<String>("dbus-service-name"),
+        cmd_arguments.get_one::<String>("dbus-object-path"),
+    ) {
+        (Some(name), Some(path)) => {
            // monitor is either set (file based) or not.
            // if it's not set, create one without file support.
            let mut monitor = match event_monitor.take() {
@@ -564,9 +624,9 @@ fn start_vmm(toplevel: TopLevel) -> Result<Option<String>, Error> {
                None => event_monitor::set_monitor(None).map_err(Error::EventMonitorIo)?,
            };
            let options = DBusApiOptions {
-                service_name: name.to_owned(),
-                object_path: path.to_owned(),
-                system_bus: toplevel.dbus_system_bus,
+                service_name: name.to_string(),
+                object_path: path.to_string(),
+                system_bus: cmd_arguments.get_flag("dbus-system-bus"),
                event_monitor_rx: monitor.subscribe(),
            };

@@ -612,10 +672,11 @@ fn start_vmm(toplevel: TopLevel) -> Result<Option<String>, Error> {
    .map_err(Error::StartVmmThread)?;

    let r: Result<(), Error> = (|| {
-        let payload_present = toplevel.kernel.is_some() || toplevel.firmware.is_some();
+        let payload_present =
+            cmd_arguments.contains_id("kernel") || cmd_arguments.contains_id("firmware");

        if payload_present {
-            let vm_params = toplevel.to_vm_params();
+            let vm_params = config::VmParams::from_arg_matches(&cmd_arguments);
            let vm_config = config::VmConfig::parse(vm_params).map_err(Error::ParsingConfig)?;

            // Create and boot the VM based off the VM config we just built.
@@ -627,12 +688,12 @@ fn start_vmm(toplevel: TopLevel) -> Result<Option<String>, Error> {
            )
            .map_err(Error::VmCreate)?;
            vmm::api::vm_boot(api_evt.try_clone().unwrap(), sender).map_err(Error::VmBoot)?;
-        } else if let Some(restore_params) = toplevel.restore {
+        } else if let Some(restore_params) = cmd_arguments.get_one::<String>("restore") {
            vmm::api::vm_restore(
                api_evt.try_clone().unwrap(),
                api_request_sender,
                Arc::new(
-                    config::RestoreConfig::parse(&restore_params).map_err(Error::ParsingRestore)?,
+                    config::RestoreConfig::parse(restore_params).map_err(Error::ParsingRestore)?,
                ),
            )
            .map_err(Error::VmRestore)?;
@@ -672,14 +733,20 @@ fn main() {
    // SAFETY: trivially safe
    let _ = unsafe { libc::umask(0o077) };

-    let toplevel: TopLevel = argh::from_env();
+    let (default_vcpus, default_memory, default_rng) = prepare_default_values();
+    let cmd_arguments = create_app(default_vcpus, default_memory, default_rng).get_matches();

-    if toplevel.version {
+    if cmd_arguments.get_flag("version") {
        println!("{} {}", env!("CARGO_BIN_NAME"), env!("BUILD_VERSION"));
+
+        if cmd_arguments.get_count("v") != 0 {
+            println!("Enabled features: {:?}", vmm::feature_list());
+        }
+
        return;
    }

-    let exit_code = match start_vmm(toplevel) {
+    let exit_code = match start_vmm(cmd_arguments) {
        Ok(path) => {
            path.map(|s| std::fs::remove_file(s).ok());
            0
@@ -699,45 +766,18 @@ fn main() {
 #[cfg(test)]
 mod unit_tests {
    use crate::config::HotplugMethod;
-    use crate::TopLevel;
+    use crate::{create_app, prepare_default_values};
    use std::path::PathBuf;
    use vmm::config::{
        ConsoleConfig, ConsoleOutputMode, CpuFeatures, CpusConfig, MemoryConfig, PayloadConfig,
-        RngConfig, VmConfig,
+        RngConfig, VmConfig, VmParams,
    };

-    // Taken from argh
-    fn cmd<'a>(default: &'a str, path: &'a str) -> &'a str {
-        std::path::Path::new(path)
-            .file_name()
-            .and_then(|s| s.to_str())
-            .unwrap_or(default)
-    }
-
-    // Some code taken from argh since it does not provide a helper to parse arbitrary strings
    fn get_vm_config_from_vec(args: &[&str]) -> VmConfig {
-        let strings: Vec<String> = args.iter().map(|x| x.to_string()).collect();
-        let cmd = cmd(&strings[0], &strings[0]);
-        let strs: Vec<&str> = strings.iter().map(|s| s.as_str()).collect();
-        let toplevel = <TopLevel as argh::FromArgs>::from_args(&[cmd], &strs[1..]).unwrap_or_else(
-            |early_exit| {
-                std::process::exit(match early_exit.status {
-                    Ok(()) => {
-                        println!("{}", early_exit.output);
-                        0
-                    }
-                    Err(()) => {
-                        eprintln!(
-                            "{}\nRun {} --help for more information.",
-                            early_exit.output, cmd
-                        );
-                        1
-                    }
-                })
-            },
-        );
-
-        let vm_params = toplevel.to_vm_params();
+        let (default_vcpus, default_memory, default_rng) = prepare_default_values();
+        let cmd_arguments =
+            create_app(default_vcpus, default_memory, default_rng).get_matches_from(args);
+        let vm_params = VmParams::from_arg_matches(&cmd_arguments);

        VmConfig::parse(vm_params).unwrap()
    }
@@ -808,11 +848,13 @@ mod unit_tests {
                file: None,
                mode: ConsoleOutputMode::Null,
                iommu: false,
+                socket: None,
            },
            console: ConsoleConfig {
                file: None,
                mode: ConsoleOutputMode::Tty,
                iommu: false,
+                socket: None,
            },
            devices: None,
            user_devices: None,
@@ -998,7 +1040,6 @@ mod unit_tests {
                    "/path/to/kernel",
                    "--disk",
                    "path=/path/to/disk/1",
-                    "--disk",
                    "path=/path/to/disk/2",
                ],
                r#"{
@@ -1017,7 +1058,6 @@ mod unit_tests {
                    "/path/to/kernel",
                    "--disk",
                    "path=/path/to/disk/1",
-                    "--disk",
                    "path=/path/to/disk/2",
                ],
                r#"{
@@ -1292,7 +1332,6 @@ mod unit_tests {
                    "--memory", "shared=true",
                    "--fs",
                    "tag=virtiofs1,socket=/path/to/sock1",
-                    "--fs",
                    "tag=virtiofs2,socket=/path/to/sock2",
                ],
                r#"{
@@ -1311,7 +1350,6 @@ mod unit_tests {
                    "--memory", "shared=true",
                    "--fs",
                    "tag=virtiofs1,socket=/path/to/sock1",
-                    "--fs",
                    "tag=virtiofs2,socket=/path/to/sock2",
                ],
                r#"{
@@ -1373,7 +1411,6 @@ mod unit_tests {
                    "/path/to/kernel",
                    "--pmem",
                    "file=/path/to/img/1,size=1G",
-                    "--pmem",
                    "file=/path/to/img/2,size=2G",
                ],
                r#"{
@@ -1540,7 +1577,6 @@ mod unit_tests {
                    "/path/to/kernel",
                    "--device",
                    "path=/path/to/device/1",
-                    "--device",
                    "path=/path/to/device/2",
                ],
                r#"{
@@ -1559,7 +1595,6 @@ mod unit_tests {
                    "/path/to/kernel",
                    "--device",
                    "path=/path/to/device/1",
-                    "--device",
                    "path=/path/to/device/2",
                ],
                r#"{
@@ -1636,7 +1671,6 @@ mod unit_tests {
                    "/path/to/kernel",
                    "--vdpa",
                    "path=/path/to/device/1",
-                    "--vdpa",
                    "path=/path/to/device/2,num_queues=2",
                ],
                r#"{
@@ -1655,7 +1689,6 @@ mod unit_tests {
                    "/path/to/kernel",
                    "--vdpa",
                    "path=/path/to/device/1",
-                    "--vdpa",
                    "path=/path/to/device/2",
                ],
                r#"{
--- a/test_data/cloud-init/ubuntu/ci/user-data
+++ b/test_data/cloud-init/ubuntu/ci/user-data
@@ -47,7 +47,7 @@ write_files:
        # 1G ram requires 512 pages
        echo 512 | sudo tee /proc/sys/vm/nr_hugepages
        sudo chmod a+rwX /dev/hugepages
-        /mnt/cloud-hypervisor --kernel /mnt/vmlinux --cmdline "console=hvc0 reboot=k panic=1 nomodules root=/dev/vda1 VFIOTAG" --disk path=/mnt/focal-server-cloudimg-amd64-custom-20210609-0.raw --disk path=/mnt/cloudinit.img --cpus boot=1 --memory size=512M,hotplug_size=1G,hugepages=on --device path=/sys/bus/pci/devices/0000:00:05.0/ --device path=/sys/bus/pci/devices/0000:00:07.0/ --device path=/sys/bus/pci/devices/0000:00:08.0/ --api-socket /tmp/ch_api.sock
+        /mnt/cloud-hypervisor --kernel /mnt/vmlinux --cmdline "console=hvc0 reboot=k panic=1 nomodules root=/dev/vda1 VFIOTAG" --disk path=/mnt/focal-server-cloudimg-amd64-custom-20210609-0.raw path=/mnt/cloudinit.img --cpus boot=1 --memory size=512M,hotplug_size=1G,hugepages=on --device path=/sys/bus/pci/devices/0000:00:05.0/ path=/sys/bus/pci/devices/0000:00:07.0/ path=/sys/bus/pci/devices/0000:00:08.0/ --api-socket=/tmp/ch_api.sock

  -
    path: /etc/systemd/system/notify-booted.service
--- a/test_infra/src/lib.rs
+++ b/test_infra/src/lib.rs
@@ -90,7 +90,7 @@ impl GuestNetworkConfig {
            None => DEFAULT_TCP_LISTENER_TIMEOUT,
        };

-        match (|| -> Result<(), WaitForBootError> {
+        let mut closure = || -> Result<(), WaitForBootError> {
            let listener =
                TcpListener::bind(listen_addr.as_str()).map_err(WaitForBootError::Listen)?;
            listener
@@ -143,7 +143,9 @@ impl GuestNetworkConfig {
                    Err(WaitForBootError::Accept(e))
                }
            }
-        })() {
+        };
+
+        match closure() {
            Err(e) => {
                let duration = start.elapsed();
                eprintln!(
@@ -559,7 +561,7 @@ fn scp_to_guest_with_auth(
 ) -> Result<(), SshCommandError> {
    let mut counter = 0;
    loop {
-        match (|| -> Result<(), SshCommandError> {
+        let closure = || -> Result<(), SshCommandError> {
            let tcp =
                TcpStream::connect(format!("{ip}:22")).map_err(SshCommandError::Connection)?;
            let mut sess = Session::new().unwrap();
@@ -592,7 +594,9 @@ fn scp_to_guest_with_auth(
            let _ = channel.wait_close();

            Ok(())
-        })() {
+        };
+
+        match closure() {
            Ok(_) => break,
            Err(e) => {
                counter += 1;
@@ -647,7 +651,7 @@ pub fn ssh_command_ip_with_auth(

    let mut counter = 0;
    loop {
-        match (|| -> Result<(), SshCommandError> {
+        let mut closure = || -> Result<(), SshCommandError> {
            let tcp =
                TcpStream::connect(format!("{ip}:22")).map_err(SshCommandError::Connection)?;
            let mut sess = Session::new().unwrap();
@@ -676,7 +680,9 @@ pub fn ssh_command_ip_with_auth(
            } else {
                Ok(())
            }
-        })() {
+        };
+
+        match closure() {
            Ok(_) => break,
            Err(e) => {
                counter += 1;
@@ -1152,7 +1158,7 @@ impl ToString for VerbosityLevel {
        match self {
            Warn => "".to_string(),
            Info => "-v".to_string(),
-            Debug => "-v -v".to_string(),
+            Debug => "-vv".to_string(),
        }
    }
 }
@@ -1203,7 +1209,7 @@ impl<'a> GuestCommand<'a> {
                self.command.arg("-v");
            }
            Debug => {
-                self.command.args(["-v", "-v"]);
+                self.command.args(["-vv"]);
            }
        };

@@ -1271,7 +1277,6 @@ impl<'a> GuestCommand<'a> {
                        .unwrap()
                )
                .as_str(),
-                "--disk",
                format!(
                    "path={}",
                    self.guest.disk_config.disk(DiskType::CloudInit).unwrap()
--- a/tests/integration.rs
+++ b/tests/integration.rs
--- a/vhost_user_block/Cargo.toml
+++ b/vhost_user_block/Cargo.toml
@@ -6,7 +6,7 @@ edition = "2021"
 build = "../build.rs"

 [dependencies]
-argh = "0.1.9"
+clap = { version = "4.3.11", features = ["wrap_help","cargo"] }
 block = { path = "../block" }
 env_logger = "0.10.0"
 epoll = "4.3.3"
--- a/vhost_user_block/src/lib.rs
+++ b/vhost_user_block/src/lib.rs
@@ -74,6 +74,11 @@ enum Error {
    SocketParameterMissing,
 }

+pub const SYNTAX: &str = "vhost-user-block backend parameters \
+ \"path=<image_path>,socket=<socket_path>,num_queues=<number_of_queues>,\
+ queue_size=<size_of_each_queue>,readonly=true|false,direct=true|false,\
+ poll_queue=true|false\"";
+
 impl fmt::Display for Error {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        write!(f, "vhost_user_block_error: {self:?}")
--- a/vhost_user_block/src/main.rs
+++ b/vhost_user_block/src/main.rs
@@ -10,36 +10,25 @@

 extern crate vhost_user_block;

-use argh::FromArgs;
+use clap::{Arg, Command};
 use vhost_user_block::start_block_backend;

-#[derive(FromArgs)]
-/// Launch a vhost-user-blk backend.
-struct TopLevel {
-    #[argh(option, long = "block-backend")]
-    /// vhost-user-block backend parameters
-    /// path=<image_path>,socket=<socket_path>,num_queues=<number_of_queues>,queue_size=<size_of_each_queue>,readonly=true|false,direct=true|false,poll_queue=true|false
-    backend_command: Option<String>,
-
-    #[argh(switch, short = 'V', long = "version")]
-    /// print version information
-    version: bool,
-}
-
 fn main() {
    env_logger::init();

-    let toplevel: TopLevel = argh::from_env();
+    let cmd_arguments = Command::new("vhost-user-blk backend")
+        .version(env!("CARGO_PKG_VERSION"))
+        .author(env!("CARGO_PKG_AUTHORS"))
+        .about("Launch a vhost-user-blk backend.")
+        .arg(
+            Arg::new("block-backend")
+                .long("block-backend")
+                .help(vhost_user_block::SYNTAX)
+                .num_args(1)
+                .required(true),
+        )
+        .get_matches();

-    if toplevel.version {
-        println!("{} {}", env!("CARGO_BIN_NAME"), env!("BUILD_VERSION"));
-        return;
-    }
-
-    if toplevel.backend_command.is_none() {
-        println!("Please specify --block-backend");
-        std::process::exit(1)
-    }
-
-    start_block_backend(&toplevel.backend_command.unwrap());
+    let backend_command = cmd_arguments.get_one::<String>("block-backend").unwrap();
+    start_block_backend(backend_command);
 }
--- a/vhost_user_net/Cargo.toml
+++ b/vhost_user_net/Cargo.toml
@@ -6,7 +6,7 @@ edition = "2021"
 build = "../build.rs"

 [dependencies]
-argh = "0.1.9"
+clap = { version = "4.3.11", features = ["wrap_help","cargo"] }
 env_logger = "0.10.0"
 epoll = "4.3.3"
 libc = "0.2.147"
--- a/vhost_user_net/src/main.rs
+++ b/vhost_user_net/src/main.rs
@@ -6,36 +6,25 @@
 //
 // SPDX-License-Identifier: (Apache-2.0 AND BSD-3-Clause)

-use argh::FromArgs;
+use clap::{Arg, Command};
 use vhost_user_net::start_net_backend;

-#[derive(FromArgs)]
-/// Launch a vhost-user-net backend.
-struct TopLevel {
-    #[argh(option, long = "net-backend")]
-    /// vhost-user-net backend parameters
-    /// ip=<ip_addr>,mask=<net_mask>,socket=<socket_path>,client=on|off,num_queues=<number_of_queues>,queue_size=<size_of_each_queue>,tap=<if_name>
-    backend_command: Option<String>,
-
-    #[argh(switch, short = 'V', long = "version")]
-    /// print version information
-    version: bool,
-}
-
 fn main() {
    env_logger::init();

-    let toplevel: TopLevel = argh::from_env();
+    let cmd_arguments = Command::new("vhost-user-net backend")
+        .version(env!("CARGO_PKG_VERSION"))
+        .author(env!("CARGO_PKG_AUTHORS"))
+        .about("Launch a vhost-user-net backend.")
+        .arg(
+            Arg::new("net-backend")
+                .long("net-backend")
+                .help(vhost_user_net::SYNTAX)
+                .num_args(1)
+                .required(true),
+        )
+        .get_matches();

-    if toplevel.version {
-        println!("{} {}", env!("CARGO_BIN_NAME"), env!("BUILD_VERSION"));
-        return;
-    }
-
-    if toplevel.backend_command.is_none() {
-        println!("Please specify --net-backend");
-        std::process::exit(1)
-    }
-
-    start_net_backend(&toplevel.backend_command.unwrap());
+    let backend_command = cmd_arguments.get_one::<String>("net-backend").unwrap();
+    start_net_backend(backend_command);
 }
--- a/virtio-devices/Cargo.toml
+++ b/virtio-devices/Cargo.toml
@@ -20,7 +20,7 @@ net_gen = { path = "../net_gen" }
 net_util = { path = "../net_util" }
 pci = { path = "../pci" }
 rate_limiter = { path = "../rate_limiter" }
-seccompiler = "0.3.0"
+seccompiler = "0.4.0"
 serde = { version = "1.0.168", features = ["derive"] }
 serde_json = "1.0.107"
 serial_buffer = { path = "../serial_buffer" }
--- a/virtio-devices/src/block.rs
+++ b/virtio-devices/src/block.rs
@@ -616,7 +616,9 @@ impl Block {
                (disk_nsectors, avail_features, 0, config, false)
            };

-        let serial = serial.map(Vec::from).unwrap_or(build_serial(&disk_path));
+        let serial = serial
+            .map(Vec::from)
+            .unwrap_or_else(|| build_serial(&disk_path));

        Ok(Block {
            common: VirtioCommon {
--- a/virtio-devices/src/seccomp_filters.rs
+++ b/virtio-devices/src/seccomp_filters.rs
@@ -86,9 +86,11 @@ fn virtio_block_thread_rules() -> Vec<(i64, Vec<SeccompRule>)> {
        (libc::SYS_fsync, vec![]),
        (libc::SYS_ftruncate, vec![]),
        (libc::SYS_getrandom, vec![]),
+        (libc::SYS_io_destroy, vec![]),
+        (libc::SYS_io_getevents, vec![]),
+        (libc::SYS_io_submit, vec![]),
        (libc::SYS_io_uring_enter, vec![]),
        (libc::SYS_lseek, vec![]),
-        (libc::SYS_prctl, vec![]),
        (libc::SYS_pread64, vec![]),
        (libc::SYS_preadv, vec![]),
        (libc::SYS_pwritev, vec![]),
@@ -102,7 +104,6 @@ fn virtio_block_thread_rules() -> Vec<(i64, Vec<SeccompRule>)> {
 fn virtio_console_thread_rules() -> Vec<(i64, Vec<SeccompRule>)> {
    vec![
        (libc::SYS_ioctl, create_virtio_console_ioctl_seccomp_rule()),
-        (libc::SYS_prctl, vec![]),
        (libc::SYS_sched_getaffinity, vec![]),
        (libc::SYS_set_robust_list, vec![]),
    ]
@@ -143,7 +144,6 @@ fn virtio_pmem_thread_rules() -> Vec<(i64, Vec<SeccompRule>)> {

 fn virtio_rng_thread_rules() -> Vec<(i64, Vec<SeccompRule>)> {
    vec![
-        (libc::SYS_prctl, vec![]),
        (libc::SYS_sched_getaffinity, vec![]),
        (libc::SYS_set_robust_list, vec![]),
    ]
@@ -214,7 +214,6 @@ fn virtio_vsock_thread_rules() -> Vec<(i64, Vec<SeccompRule>)> {

 fn virtio_watchdog_thread_rules() -> Vec<(i64, Vec<SeccompRule>)> {
    vec![
-        (libc::SYS_prctl, vec![]),
        (libc::SYS_sched_getaffinity, vec![]),
        (libc::SYS_set_robust_list, vec![]),
        (libc::SYS_timerfd_settime, vec![]),
--- a/virtio-devices/src/vhost_user/vu_common_ctrl.rs
+++ b/virtio-devices/src/vhost_user/vu_common_ctrl.rs
@@ -7,7 +7,6 @@ use crate::{
    get_host_address_range, GuestMemoryMmap, GuestRegionMmap, MmapRegion, VirtioInterrupt,
    VirtioInterruptType,
 };
-use std::convert::TryInto;
 use std::ffi;
 use std::fs::File;
 use std::os::unix::io::{AsRawFd, FromRawFd, RawFd};
@@ -201,7 +200,7 @@ impl VhostUserHandle {

        let mut vrings_info = Vec::new();
        for (queue_index, queue, queue_evt) in queues.iter() {
-            let actual_size: usize = queue.size().try_into().unwrap();
+            let actual_size: usize = queue.size().into();

            let config_data = VringConfigData {
                queue_max_size: queue.max_size(),
--- a/vm-allocator/src/system.rs
+++ b/vm-allocator/src/system.rs
@@ -119,7 +119,7 @@ impl SystemAllocator {
        self.platform_mmio_address_space.allocate(
            address,
            size,
-            Some(align_size.unwrap_or(get_page_size())),
+            Some(align_size.unwrap_or_else(get_page_size)),
        )
    }

@@ -133,7 +133,7 @@ impl SystemAllocator {
        self.mmio_hole_address_space.allocate(
            address,
            size,
-            Some(align_size.unwrap_or(get_page_size())),
+            Some(align_size.unwrap_or_else(get_page_size)),
        )
    }

--- a/vm-migration/src/lib.rs
+++ b/vm-migration/src/lib.rs
@@ -12,8 +12,8 @@ use versionize::{VersionMap, Versionize};
 pub mod protocol;

 /// Global VMM version for versioning
-const MAJOR_VERSION: u16 = 35;
-const MINOR_VERSION: u16 = 0;
+const MAJOR_VERSION: u16 = 36;
+const MINOR_VERSION: u16 = 1;
 const VMM_VERSION: u16 = MAJOR_VERSION << 12 | MINOR_VERSION & 0b1111;

 pub trait VersionMapped {
--- a/vmm/Cargo.toml
+++ b/vmm/Cargo.toml
@@ -20,10 +20,11 @@ acpi_tables = { git = "https://github.com/rust-vmm/acpi_tables", branch = "main"
 anyhow = "1.0.75"
 arc-swap = "1.5.1"
 arch = { path = "../arch" }
-bitflags = "2.3.3"
+bitflags = "2.4.1"
 block = { path = "../block" }
 blocking = { version = "1.3.0", optional = true }
 cfg-if = "1.0.0"
+clap = "4.3.11"
 devices = { path = "../devices" }
 epoll = "4.3.3"
 event_monitor = { path = "../event_monitor" }
@@ -40,7 +41,7 @@ net_util = { path = "../net_util" }
 once_cell = "1.18.0"
 option_parser = { path = "../option_parser" }
 pci = { path = "../pci" }
-seccompiler = "0.3.0"
+seccompiler = "0.4.0"
 serde = { version = "1.0.168", features = ["rc", "derive"] }
 serde_json = "1.0.107"
 serial_buffer = { path = "../serial_buffer" }
@@ -61,4 +62,4 @@ vm-migration = { path = "../vm-migration" }
 vm-virtio = { path = "../vm-virtio" }
 vmm-sys-util = { version = "0.11.0", features = ["with-serde"] }
 zbus = { version = "3.11.1", optional = true }
-zerocopy = "0.6.1"
+zerocopy = { version = "0.7.21", features = ["derive"] }
--- a/vmm/src/api/mod.rs
+++ b/vmm/src/api/mod.rs
@@ -171,6 +171,7 @@ pub struct VmmPingResponse {
    pub build_version: String,
    pub version: String,
    pub pid: i64,
+    pub features: Vec<String>,
 }

 #[derive(Clone, Deserialize, Serialize, Default, Debug)]
--- a/vmm/src/api/openapi/cloud-hypervisor.yaml
+++ b/vmm/src/api/openapi/cloud-hypervisor.yaml
@@ -347,6 +347,28 @@ paths:
        500:
          description: The new vDPA device could not be added to the VM instance.

+  /vm.add-user-device:
+    put:
+      requestBody:
+        content:
+          application/json:
+            schema:
+              $ref: '#/components/schemas/VmAddUserDevice'
+        description: The path of the new device
+        required: true
+      responses:
+        "200":
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/PciDeviceInfo'
+          description: The new device was successfully added to the VM instance.
+        "204":
+          description: The new device was successfully (cold) added to the VM instance.
+        "404":
+          description: The new device could not be added to the VM instance.
+      summary: Add a new userspace device to the VM
+
  /vm.snapshot:
    put:
      summary: Returns a VM snapshot.
@@ -445,6 +467,10 @@ components:
        pid:
          type: integer
          format: int64
+        features:
+          type: array
+          items:
+            type: string
      description: Virtual Machine Monitor information

    VmInfo:
@@ -947,9 +973,11 @@ components:
      properties:
        file:
          type: string
+        socket:
+          type: string
        mode:
          type: string
-          enum: [Off, Pty, Tty, File, Null]
+          enum: [Off, Pty, Tty, File, Socket, Null]
        iommu:
          type: boolean
          default: false
@@ -1074,6 +1102,11 @@ components:
          type: array
          items:
            type: string
+        pci_segments:
+          type: array
+          items:
+            type: integer
+            format: int32

    VmResize:
      type: object
@@ -1145,3 +1178,11 @@ components:
          type: string
        local:
          type: boolean
+
+    VmAddUserDevice:
+      required:
+        - socket
+      type: object
+      properties:
+        socket:
+          type: string
--- a/vmm/src/config.rs
+++ b/vmm/src/config.rs
@@ -4,6 +4,7 @@
 //

 pub use crate::vm_config::*;
+use clap::ArgMatches;
 use option_parser::{
    ByteSized, IntegerList, OptionParser, OptionParserError, StringList, Toggle, Tuple,
 };
@@ -111,6 +112,8 @@ pub enum ValidationError {
    KernelMissing,
    /// Missing file value for console
    ConsoleFileMissing,
+    /// Missing socket path for console
+    ConsoleSocketPathMissing,
    /// Max is less than boot
    CpusMaxLowerThanBoot,
    /// Both socket and path specified
@@ -174,6 +177,10 @@ pub enum ValidationError {
    DuplicateDevicePath(String),
    /// Provided MTU is lower than what the VIRTIO specification expects
    InvalidMtu(u16),
+    /// PCI segment is reused across NUMA nodes
+    PciSegmentReused(u16, u32, u32),
+    /// Default PCI segment is assigned to NUMA node other than 0.
+    DefaultPciSegmentInvalidNode(u32),
 }

 type ValidationResult<T> = std::result::Result<T, ValidationError>;
@@ -185,6 +192,7 @@ impl fmt::Display for ValidationError {
            DoubleTtyMode => write!(f, "Console mode tty specified for both serial and console"),
            KernelMissing => write!(f, "No kernel specified"),
            ConsoleFileMissing => write!(f, "Path missing when using file console mode"),
+            ConsoleSocketPathMissing => write!(f, "Path missing when using socket console mode"),
            CpusMaxLowerThanBoot => write!(f, "Max CPUs lower than boot CPUs"),
            DiskSocketAndPath => write!(f, "Disk path and vhost socket both provided"),
            VhostUserRequiresSharedMemory => {
@@ -285,6 +293,15 @@ impl fmt::Display for ValidationError {
                    "Provided MTU {mtu} is lower than 1280 (expected by VIRTIO specification)"
                )
            }
+            PciSegmentReused(pci_segment, u1, u2) => {
+                write!(
+                    f,
+                    "PCI segment: {pci_segment} belongs to multiple NUMA nodes {u1} and {u2}"
+                )
+            }
+            DefaultPciSegmentInvalidNode(u1) => {
+                write!(f, "Default PCI segment assigned to non-zero NUMA node {u1}")
+            }
        }
    }
 }
@@ -386,6 +403,90 @@ pub struct VmParams<'a> {
    pub tpm: Option<&'a str>,
 }

+impl<'a> VmParams<'a> {
+    pub fn from_arg_matches(args: &'a ArgMatches) -> Self {
+        // These .unwrap()s cannot fail as there is a default value defined
+        let cpus = args.get_one::<String>("cpus").unwrap();
+        let memory = args.get_one::<String>("memory").unwrap();
+        let memory_zones: Option<Vec<&str>> = args
+            .get_many::<String>("memory-zone")
+            .map(|x| x.map(|y| y as &str).collect());
+        let rng = args.get_one::<String>("rng").unwrap();
+        let serial = args.get_one::<String>("serial").unwrap();
+        let firmware = args.get_one::<String>("firmware").map(|x| x as &str);
+        let kernel = args.get_one::<String>("kernel").map(|x| x as &str);
+        let initramfs = args.get_one::<String>("initramfs").map(|x| x as &str);
+        let cmdline = args.get_one::<String>("cmdline").map(|x| x as &str);
+        let disks: Option<Vec<&str>> = args
+            .get_many::<String>("disk")
+            .map(|x| x.map(|y| y as &str).collect());
+        let net: Option<Vec<&str>> = args
+            .get_many::<String>("net")
+            .map(|x| x.map(|y| y as &str).collect());
+        let console = args.get_one::<String>("console").unwrap();
+        let balloon = args.get_one::<String>("balloon").map(|x| x as &str);
+        let fs: Option<Vec<&str>> = args
+            .get_many::<String>("fs")
+            .map(|x| x.map(|y| y as &str).collect());
+        let pmem: Option<Vec<&str>> = args
+            .get_many::<String>("pmem")
+            .map(|x| x.map(|y| y as &str).collect());
+        let devices: Option<Vec<&str>> = args
+            .get_many::<String>("device")
+            .map(|x| x.map(|y| y as &str).collect());
+        let user_devices: Option<Vec<&str>> = args
+            .get_many::<String>("user-device")
+            .map(|x| x.map(|y| y as &str).collect());
+        let vdpa: Option<Vec<&str>> = args
+            .get_many::<String>("vdpa")
+            .map(|x| x.map(|y| y as &str).collect());
+        let vsock: Option<&str> = args.get_one::<String>("vsock").map(|x| x as &str);
+        let pvpanic = args.get_flag("pvpanic");
+        #[cfg(target_arch = "x86_64")]
+        let sgx_epc: Option<Vec<&str>> = args
+            .get_many::<String>("sgx-epc")
+            .map(|x| x.map(|y| y as &str).collect());
+        let numa: Option<Vec<&str>> = args
+            .get_many::<String>("numa")
+            .map(|x| x.map(|y| y as &str).collect());
+        let watchdog = args.get_flag("watchdog");
+        let platform = args.get_one::<String>("platform").map(|x| x as &str);
+        #[cfg(feature = "guest_debug")]
+        let gdb = args.contains_id("gdb");
+        let tpm: Option<&str> = args.get_one::<String>("tpm").map(|x| x as &str);
+        VmParams {
+            cpus,
+            memory,
+            memory_zones,
+            firmware,
+            kernel,
+            initramfs,
+            cmdline,
+            disks,
+            net,
+            rng,
+            balloon,
+            fs,
+            pmem,
+            serial,
+            console,
+            devices,
+            user_devices,
+            vdpa,
+            vsock,
+            pvpanic,
+            #[cfg(target_arch = "x86_64")]
+            sgx_epc,
+            numa,
+            watchdog,
+            #[cfg(feature = "guest_debug")]
+            gdb,
+            platform,
+            tpm,
+        }
+    }
+}
+
 #[derive(Debug)]
 pub enum ParseHotplugMethodError {
    InvalidValue(String),
@@ -760,6 +861,14 @@ impl MemoryConfig {
 }

 impl DiskConfig {
+    pub const SYNTAX: &'static str = "Disk parameters \
+         \"path=<disk_image_path>,readonly=on|off,direct=on|off,iommu=on|off,\
+         num_queues=<number_of_queues>,queue_size=<size_of_each_queue>,\
+         vhost_user=on|off,socket=<vhost_user_socket_path>,\
+         bw_size=<bytes>,bw_one_time_burst=<bytes>,bw_refill_time=<ms>,\
+         ops_size=<io_ops>,ops_one_time_burst=<io_ops>,ops_refill_time=<ms>,\
+         id=<device_id>,pci_segment=<segment_id>\"";
+
    pub fn parse(disk: &str) -> Result<Self> {
        let mut parser = OptionParser::new();
        parser
@@ -779,6 +888,7 @@ impl DiskConfig {
            .add("ops_refill_time")
            .add("id")
            .add("_disable_io_uring")
+            .add("_disable_aio")
            .add("pci_segment")
            .add("serial");
        parser.parse(disk).map_err(Error::ParseDisk)?;
@@ -819,6 +929,11 @@ impl DiskConfig {
            .map_err(Error::ParseDisk)?
            .unwrap_or(Toggle(false))
            .0;
+        let disable_aio = parser
+            .convert::<Toggle>("_disable_aio")
+            .map_err(Error::ParseDisk)?
+            .unwrap_or(Toggle(false))
+            .0;
        let pci_segment = parser
            .convert("pci_segment")
            .map_err(Error::ParseDisk)?
@@ -887,6 +1002,7 @@ impl DiskConfig {
            rate_limiter_config,
            id,
            disable_io_uring,
+            disable_aio,
            pci_segment,
            serial,
        })
@@ -935,6 +1051,14 @@ impl FromStr for VhostMode {
 }

 impl NetConfig {
+    pub const SYNTAX: &'static str = "Network parameters \
+    \"tap=<if_name>,ip=<ip_addr>,mask=<net_mask>,mac=<mac_addr>,fd=<fd1,fd2...>,iommu=on|off,\
+    num_queues=<number_of_queues>,queue_size=<size_of_each_queue>,id=<device_id>,\
+    vhost_user=<vhost_user_enable>,socket=<vhost_user_socket_path>,vhost_mode=client|server,\
+    bw_size=<bytes>,bw_one_time_burst=<bytes>,bw_refill_time=<ms>,\
+    ops_size=<io_ops>,ops_one_time_burst=<io_ops>,ops_refill_time=<ms>,pci_segment=<segment_id>\
+    offload_tso=on|off,offload_ufo=on|off,offload_csum=on|off\"";
+
    pub fn parse(net: &str) -> Result<Self> {
        let mut parser = OptionParser::new();

@@ -1175,6 +1299,10 @@ impl RngConfig {
 }

 impl BalloonConfig {
+    pub const SYNTAX: &'static str =
+        "Balloon parameters \"size=<balloon_size>,deflate_on_oom=on|off,\
+        free_page_reporting=on|off\"";
+
    pub fn parse(balloon: &str) -> Result<Self> {
        let mut parser = OptionParser::new();
        parser.add("size");
@@ -1209,6 +1337,10 @@ impl BalloonConfig {
 }

 impl FsConfig {
+    pub const SYNTAX: &'static str = "virtio-fs parameters \
+    \"tag=<tag_name>,socket=<socket_path>,num_queues=<number_of_queues>,\
+    queue_size=<size_of_each_queue>,id=<device_id>,pci_segment=<segment_id>\"";
+
    pub fn parse(fs: &str) -> Result<Self> {
        let mut parser = OptionParser::new();
        parser
@@ -1273,6 +1405,10 @@ impl FsConfig {
 }

 impl PmemConfig {
+    pub const SYNTAX: &'static str = "Persistent memory parameters \
+    \"file=<backing_file_path>,size=<persistent_memory_size>,iommu=on|off,\
+    discard_writes=on|off,id=<device_id>,pci_segment=<segment_id>\"";
+
    pub fn parse(pmem: &str) -> Result<Self> {
        let mut parser = OptionParser::new();
        parser
@@ -1341,10 +1477,12 @@ impl ConsoleConfig {
            .add_valueless("tty")
            .add_valueless("null")
            .add("file")
-            .add("iommu");
+            .add("iommu")
+            .add("socket");
        parser.parse(console).map_err(Error::ParseConsole)?;

        let mut file: Option<PathBuf> = default_consoleconfig_file();
+        let mut socket: Option<PathBuf> = None;
        let mut mode: ConsoleOutputMode = ConsoleOutputMode::Off;

        if parser.is_set("off") {
@@ -1360,6 +1498,11 @@ impl ConsoleConfig {
                Some(PathBuf::from(parser.get("file").ok_or(
                    Error::Validation(ValidationError::ConsoleFileMissing),
                )?));
+        } else if parser.is_set("socket") {
+            mode = ConsoleOutputMode::Socket;
+            socket = Some(PathBuf::from(parser.get("socket").ok_or(
+                Error::Validation(ValidationError::ConsoleSocketPathMissing),
+            )?));
        } else {
            return Err(Error::ParseConsoleInvalidModeGiven);
        }
@@ -1369,11 +1512,19 @@ impl ConsoleConfig {
            .unwrap_or(Toggle(false))
            .0;

-        Ok(Self { file, mode, iommu })
+        Ok(Self {
+            file,
+            mode,
+            iommu,
+            socket,
+        })
    }
 }

 impl DeviceConfig {
+    pub const SYNTAX: &'static str =
+        "Direct device assignment parameters \"path=<device_path>,iommu=on|off,id=<device_id>,pci_segment=<segment_id>\"";
+
    pub fn parse(device: &str) -> Result<Self> {
        let mut parser = OptionParser::new();
        parser.add("path").add("id").add("iommu").add("pci_segment");
@@ -1420,6 +1571,9 @@ impl DeviceConfig {
 }

 impl UserDeviceConfig {
+    pub const SYNTAX: &'static str =
+        "Userspace device socket=<socket_path>,id=<device_id>,pci_segment=<segment_id>\"";
+
    pub fn parse(user_device: &str) -> Result<Self> {
        let mut parser = OptionParser::new();
        parser.add("socket").add("id").add("pci_segment");
@@ -1462,6 +1616,10 @@ impl UserDeviceConfig {
 }

 impl VdpaConfig {
+    pub const SYNTAX: &'static str = "vDPA device \
+        \"path=<device_path>,num_queues=<number_of_queues>,iommu=on|off,\
+        id=<device_id>,pci_segment=<segment_id>\"";
+
    pub fn parse(vdpa: &str) -> Result<Self> {
        let mut parser = OptionParser::new();
        parser
@@ -1518,6 +1676,9 @@ impl VdpaConfig {
 }

 impl VsockConfig {
+    pub const SYNTAX: &'static str = "Virtio VSOCK parameters \
+        \"cid=<context_id>,socket=<socket_path>,iommu=on|off,id=<device_id>,pci_segment=<segment_id>\"";
+
    pub fn parse(vsock: &str) -> Result<Self> {
        let mut parser = OptionParser::new();
        parser
@@ -1575,6 +1736,9 @@ impl VsockConfig {

 #[cfg(target_arch = "x86_64")]
 impl SgxEpcConfig {
+    pub const SYNTAX: &'static str = "SGX EPC parameters \
+        \"id=<epc_section_identifier>,size=<epc_section_size>,prefault=on|off\"";
+
    pub fn parse(sgx_epc: &str) -> Result<Self> {
        let mut parser = OptionParser::new();
        parser.add("id").add("size").add("prefault");
@@ -1597,6 +1761,11 @@ impl SgxEpcConfig {
 }

 impl NumaConfig {
+    pub const SYNTAX: &'static str = "Settings related to a given NUMA node \
+        \"guest_numa_id=<node_id>,cpus=<cpus_id>,distances=<list_of_distances_to_destination_nodes>,\
+        memory_zones=<list_of_memory_zones>,sgx_epc_sections=<list_of_sgx_epc_sections>,\
+        pci_segments=<list_of_pci_segments>\"";
+
    pub fn parse(numa: &str) -> Result<Self> {
        let mut parser = OptionParser::new();
        parser
@@ -1604,7 +1773,9 @@ impl NumaConfig {
            .add("cpus")
            .add("distances")
            .add("memory_zones")
-            .add("sgx_epc_sections");
+            .add("sgx_epc_sections")
+            .add("pci_segments");
+
        parser.parse(numa).map_err(Error::ParseNuma)?;

        let guest_numa_id = parser
@@ -1635,7 +1806,10 @@ impl NumaConfig {
            .convert::<StringList>("sgx_epc_sections")
            .map_err(Error::ParseNuma)?
            .map(|v| v.0);
-
+        let pci_segments = parser
+            .convert::<IntegerList>("pci_segments")
+            .map_err(Error::ParseNuma)?
+            .map(|v| v.0.iter().map(|e| *e as u16).collect());
        Ok(NumaConfig {
            guest_numa_id,
            cpus,
@@ -1643,6 +1817,7 @@ impl NumaConfig {
            memory_zones,
            #[cfg(target_arch = "x86_64")]
            sgx_epc_sections,
+            pci_segments,
        })
    }
 }
@@ -1655,6 +1830,11 @@ pub struct RestoreConfig {
 }

 impl RestoreConfig {
+    pub const SYNTAX: &'static str = "Restore from a VM snapshot. \
+        \nRestore parameters \"source_url=<source_url>,prefault=on|off\" \
+        \n`source_url` should be a valid URL (e.g file:///foo/bar or tcp://192.168.1.10/foo) \
+        \n`prefault` brings memory pages in when enabled (disabled by default)";
+
    pub fn parse(restore: &str) -> Result<Self> {
        let mut parser = OptionParser::new();
        parser.add("source_url").add("prefault");
@@ -1678,6 +1858,9 @@ impl RestoreConfig {
 }

 impl TpmConfig {
+    pub const SYNTAX: &'static str = "TPM device \
+        \"(UNIX Domain Socket from swtpm) socket=</path/to/a/socket>\"";
+
    pub fn parse(tpm: &str) -> Result<Self> {
        let mut parser = OptionParser::new();
        parser.add("socket");
@@ -1910,19 +2093,48 @@ impl VmConfig {
            Self::validate_identifier(&mut id_list, &vsock.id)?;
        }

+        let num_pci_segments = match &self.platform {
+            Some(platform_config) => platform_config.num_pci_segments,
+            None => 1,
+        };
        if let Some(numa) = &self.numa {
            let mut used_numa_node_memory_zones = HashMap::new();
+            let mut used_pci_segments = HashMap::new();
            for numa_node in numa.iter() {
-                for memory_zone in numa_node.memory_zones.clone().unwrap().iter() {
-                    if !used_numa_node_memory_zones.contains_key(memory_zone) {
-                        used_numa_node_memory_zones
-                            .insert(memory_zone.to_string(), numa_node.guest_numa_id);
-                    } else {
-                        return Err(ValidationError::MemoryZoneReused(
-                            memory_zone.to_string(),
-                            *used_numa_node_memory_zones.get(memory_zone).unwrap(),
-                            numa_node.guest_numa_id,
-                        ));
+                if let Some(memory_zones) = numa_node.memory_zones.clone() {
+                    for memory_zone in memory_zones.iter() {
+                        if !used_numa_node_memory_zones.contains_key(memory_zone) {
+                            used_numa_node_memory_zones
+                                .insert(memory_zone.to_string(), numa_node.guest_numa_id);
+                        } else {
+                            return Err(ValidationError::MemoryZoneReused(
+                                memory_zone.to_string(),
+                                *used_numa_node_memory_zones.get(memory_zone).unwrap(),
+                                numa_node.guest_numa_id,
+                            ));
+                        }
+                    }
+                }
+
+                if let Some(pci_segments) = numa_node.pci_segments.clone() {
+                    for pci_segment in pci_segments.iter() {
+                        if *pci_segment >= num_pci_segments {
+                            return Err(ValidationError::InvalidPciSegment(*pci_segment));
+                        }
+                        if *pci_segment == 0 && numa_node.guest_numa_id != 0 {
+                            return Err(ValidationError::DefaultPciSegmentInvalidNode(
+                                numa_node.guest_numa_id,
+                            ));
+                        }
+                        if !used_pci_segments.contains_key(pci_segment) {
+                            used_pci_segments.insert(*pci_segment, numa_node.guest_numa_id);
+                        } else {
+                            return Err(ValidationError::PciSegmentReused(
+                                *pci_segment,
+                                *used_pci_segments.get(pci_segment).unwrap(),
+                                numa_node.guest_numa_id,
+                            ));
+                        }
                    }
                }
            }
@@ -2659,6 +2871,7 @@ mod tests {
                mode: ConsoleOutputMode::Off,
                iommu: false,
                file: None,
+                socket: None,
            }
        );
        assert_eq!(
@@ -2667,6 +2880,7 @@ mod tests {
                mode: ConsoleOutputMode::Pty,
                iommu: false,
                file: None,
+                socket: None,
            }
        );
        assert_eq!(
@@ -2675,6 +2889,7 @@ mod tests {
                mode: ConsoleOutputMode::Tty,
                iommu: false,
                file: None,
+                socket: None,
            }
        );
        assert_eq!(
@@ -2683,6 +2898,7 @@ mod tests {
                mode: ConsoleOutputMode::Null,
                iommu: false,
                file: None,
+                socket: None,
            }
        );
        assert_eq!(
@@ -2690,7 +2906,8 @@ mod tests {
            ConsoleConfig {
                mode: ConsoleOutputMode::File,
                iommu: false,
-                file: Some(PathBuf::from("/tmp/console"))
+                file: Some(PathBuf::from("/tmp/console")),
+                socket: None,
            }
        );
        assert_eq!(
@@ -2699,6 +2916,7 @@ mod tests {
                mode: ConsoleOutputMode::Null,
                iommu: true,
                file: None,
+                socket: None,
            }
        );
        assert_eq!(
@@ -2706,7 +2924,17 @@ mod tests {
            ConsoleConfig {
                mode: ConsoleOutputMode::File,
                iommu: true,
-                file: Some(PathBuf::from("/tmp/console"))
+                file: Some(PathBuf::from("/tmp/console")),
+                socket: None,
+            }
+        );
+        assert_eq!(
+            ConsoleConfig::parse("socket=/tmp/serial.sock,iommu=on")?,
+            ConsoleConfig {
+                mode: ConsoleOutputMode::Socket,
+                iommu: true,
+                file: None,
+                socket: Some(PathBuf::from("/tmp/serial.sock")),
            }
        );
        Ok(())
@@ -2852,11 +3080,13 @@ mod tests {
                file: None,
                mode: ConsoleOutputMode::Null,
                iommu: false,
+                socket: None,
            },
            console: ConsoleConfig {
                file: None,
                mode: ConsoleOutputMode::Tty,
                iommu: false,
+                socket: None,
            },
            devices: None,
            user_devices: None,
@@ -3271,6 +3501,63 @@ mod tests {
            Err(ValidationError::IommuNotSupportedOnSegment(1))
        );

+        let mut invalid_config = valid_config.clone();
+        invalid_config.platform = Some(PlatformConfig {
+            num_pci_segments: 2,
+            ..Default::default()
+        });
+        invalid_config.numa = Some(vec![
+            NumaConfig {
+                guest_numa_id: 0,
+                pci_segments: Some(vec![1]),
+                ..Default::default()
+            },
+            NumaConfig {
+                guest_numa_id: 1,
+                pci_segments: Some(vec![1]),
+                ..Default::default()
+            },
+        ]);
+        assert_eq!(
+            invalid_config.validate(),
+            Err(ValidationError::PciSegmentReused(1, 0, 1))
+        );
+
+        let mut invalid_config = valid_config.clone();
+        invalid_config.numa = Some(vec![
+            NumaConfig {
+                guest_numa_id: 0,
+                ..Default::default()
+            },
+            NumaConfig {
+                guest_numa_id: 1,
+                pci_segments: Some(vec![0]),
+                ..Default::default()
+            },
+        ]);
+        assert_eq!(
+            invalid_config.validate(),
+            Err(ValidationError::DefaultPciSegmentInvalidNode(1))
+        );
+
+        let mut invalid_config = valid_config.clone();
+        invalid_config.numa = Some(vec![
+            NumaConfig {
+                guest_numa_id: 0,
+                pci_segments: Some(vec![0]),
+                ..Default::default()
+            },
+            NumaConfig {
+                guest_numa_id: 1,
+                pci_segments: Some(vec![1]),
+                ..Default::default()
+            },
+        ]);
+        assert_eq!(
+            invalid_config.validate(),
+            Err(ValidationError::InvalidPciSegment(1))
+        );
+
        let mut still_valid_config = valid_config.clone();
        still_valid_config.devices = Some(vec![
            DeviceConfig {
--- a/vmm/src/cpu.rs
+++ b/vmm/src/cpu.rs
@@ -56,6 +56,8 @@ use hypervisor::kvm::kvm_bindings;
 use hypervisor::kvm::kvm_ioctls::Cap;
 #[cfg(feature = "tdx")]
 use hypervisor::kvm::{TdxExitDetails, TdxExitStatus};
+#[cfg(target_arch = "x86_64")]
+use hypervisor::CpuVendor;
 use hypervisor::{CpuState, HypervisorCpuError, HypervisorType, VmExit, VmOps};
 use libc::{c_void, siginfo_t};
 #[cfg(all(target_arch = "x86_64", feature = "guest_debug"))]
@@ -85,7 +87,6 @@ use vm_migration::{
 use vmm_sys_util::eventfd::EventFd;
 use vmm_sys_util::signal::{register_signal_handler, SIGRTMIN};
 use zerocopy::AsBytes;
-
 #[cfg(all(target_arch = "aarch64", feature = "guest_debug"))]
 /// Extract the specified bits of a 64-bit integer.
 /// For example, to extrace 2 bits from offset 1 (zero based) of `6u64`,
@@ -315,6 +316,8 @@ pub struct Vcpu {
    #[cfg(target_arch = "aarch64")]
    mpidr: u64,
    saved_state: Option<CpuState>,
+    #[cfg(target_arch = "x86_64")]
+    vendor: CpuVendor,
 }

 impl Vcpu {
@@ -325,10 +328,12 @@ impl Vcpu {
    /// * `id` - Represents the CPU number between [0, max vcpus).
    /// * `vm` - The virtual machine this vcpu will get attached to.
    /// * `vm_ops` - Optional object for exit handling.
+    /// * `cpu_vendor` - CPU vendor as reported by __cpuid(0x0)
    pub fn new(
        id: u8,
        vm: &Arc<dyn hypervisor::Vm>,
        vm_ops: Option<Arc<dyn VmOps>>,
+        #[cfg(target_arch = "x86_64")] cpu_vendor: CpuVendor,
    ) -> Result<Self> {
        let vcpu = vm
            .create_vcpu(id, vm_ops)
@@ -340,6 +345,8 @@ impl Vcpu {
            #[cfg(target_arch = "aarch64")]
            mpidr: 0,
            saved_state: None,
+            #[cfg(target_arch = "x86_64")]
+            vendor: cpu_vendor,
        })
    }

@@ -356,6 +363,7 @@ impl Vcpu {
        boot_setup: Option<(EntryPoint, &GuestMemoryAtomic<GuestMemoryMmap>)>,
        #[cfg(target_arch = "x86_64")] cpuid: Vec<CpuIdEntry>,
        #[cfg(target_arch = "x86_64")] kvm_hyperv: bool,
+        #[cfg(target_arch = "x86_64")] topology: Option<(u8, u8, u8)>,
    ) -> Result<()> {
        #[cfg(target_arch = "aarch64")]
        {
@@ -365,8 +373,16 @@ impl Vcpu {
        }
        info!("Configuring vCPU: cpu_id = {}", self.id);
        #[cfg(target_arch = "x86_64")]
-        arch::configure_vcpu(&self.vcpu, self.id, boot_setup, cpuid, kvm_hyperv)
-            .map_err(Error::VcpuConfiguration)?;
+        arch::configure_vcpu(
+            &self.vcpu,
+            self.id,
+            boot_setup,
+            cpuid,
+            kvm_hyperv,
+            self.vendor,
+            topology,
+        )
+        .map_err(Error::VcpuConfiguration)?;

        Ok(())
    }
@@ -427,7 +443,7 @@ impl Snapshottable for Vcpu {
        let saved_state = self
            .vcpu
            .state()
-            .map_err(|e| MigratableError::Pause(anyhow!("Could not get vCPU state {:?}", e)))?;
+            .map_err(|e| MigratableError::Snapshot(anyhow!("Could not get vCPU state {:?}", e)))?;

        self.saved_state = Some(saved_state.clone());

@@ -438,7 +454,6 @@ impl Snapshottable for Vcpu {
 }

 pub struct CpuManager {
-    hypervisor_type: HypervisorType,
    config: CpusConfig,
    #[cfg_attr(target_arch = "aarch64", allow(dead_code))]
    interrupt_controller: Option<Arc<Mutex<dyn InterruptController>>>,
@@ -463,6 +478,7 @@ pub struct CpuManager {
    proximity_domain_per_cpu: BTreeMap<u8, u32>,
    affinity: BTreeMap<u8, Vec<u8>>,
    dynamic: bool,
+    hypervisor: Arc<dyn hypervisor::Hypervisor>,
 }

 const CPU_ENABLE_FLAG: usize = 0;
@@ -619,6 +635,8 @@ impl CpuManager {
        let mut vcpu_states = Vec::with_capacity(usize::from(config.max_vcpus));
        vcpu_states.resize_with(usize::from(config.max_vcpus), VcpuState::default);
        let hypervisor_type = hypervisor.hypervisor_type();
+        #[cfg(target_arch = "x86_64")]
+        let cpu_vendor = hypervisor.get_cpu_vendor();

        #[cfg(target_arch = "x86_64")]
        if config.features.amx {
@@ -679,7 +697,6 @@ impl CpuManager {
        let dynamic = true;

        Ok(Arc::new(Mutex::new(CpuManager {
-            hypervisor_type,
            config: config.clone(),
            interrupt_controller: None,
            #[cfg(target_arch = "x86_64")]
@@ -700,6 +717,7 @@ impl CpuManager {
            proximity_domain_per_cpu,
            affinity,
            dynamic,
+            hypervisor: hypervisor.clone(),
        })))
    }

@@ -708,7 +726,7 @@ impl CpuManager {
        &mut self,
        memory_manager: &Arc<Mutex<MemoryManager>>,
        hypervisor: &Arc<dyn hypervisor::Hypervisor>,
-        #[cfg(feature = "tdx")] tdx_enabled: bool,
+        #[cfg(feature = "tdx")] tdx: bool,
    ) -> Result<()> {
        let sgx_epc_sections = memory_manager
            .lock()
@@ -717,27 +735,18 @@ impl CpuManager {
            .as_ref()
            .map(|sgx_epc_region| sgx_epc_region.epc_sections().values().cloned().collect());

-        let topology = self.config.topology.clone().map_or_else(
-            || {
-                #[cfg(feature = "mshv")]
-                if matches!(hypervisor.hypervisor_type(), HypervisorType::Mshv) {
-                    return Some((1, self.boot_vcpus(), 1));
-                }
-                None
-            },
-            |t| Some((t.threads_per_core, t.cores_per_die, t.dies_per_package)),
-        );
-
        self.cpuid = {
            let phys_bits = physical_bits(hypervisor, self.config.max_phys_bits);
            arch::generate_common_cpuid(
                hypervisor,
-                topology,
-                sgx_epc_sections,
-                phys_bits,
-                self.config.kvm_hyperv,
-                #[cfg(feature = "tdx")]
-                tdx_enabled,
+                &arch::CpuidConfig {
+                    sgx_epc_sections,
+                    phys_bits,
+                    kvm_hyperv: self.config.kvm_hyperv,
+                    #[cfg(feature = "tdx")]
+                    tdx,
+                    amx: self.config.features.amx,
+                },
            )
            .map_err(Error::CommonCpuId)?
        };
@@ -748,7 +757,13 @@ impl CpuManager {
    fn create_vcpu(&mut self, cpu_id: u8, snapshot: Option<Snapshot>) -> Result<Arc<Mutex<Vcpu>>> {
        info!("Creating vCPU: cpu_id = {}", cpu_id);

-        let mut vcpu = Vcpu::new(cpu_id, &self.vm, Some(self.vm_ops.clone()))?;
+        let mut vcpu = Vcpu::new(
+            cpu_id,
+            &self.vm,
+            Some(self.vm_ops.clone()),
+            #[cfg(target_arch = "x86_64")]
+            self.hypervisor.get_cpu_vendor(),
+        )?;

        if let Some(snapshot) = snapshot {
            // AArch64 vCPUs should be initialized after created.
@@ -784,7 +799,23 @@ impl CpuManager {
        assert!(!self.cpuid.is_empty());

        #[cfg(target_arch = "x86_64")]
-        vcpu.configure(boot_setup, self.cpuid.clone(), self.config.kvm_hyperv)?;
+        let topology = self.config.topology.clone().map_or_else(
+            || {
+                #[cfg(feature = "mshv")]
+                if matches!(self.hypervisor.hypervisor_type(), HypervisorType::Mshv) {
+                    return Some((1, self.boot_vcpus(), 1));
+                }
+                None
+            },
+            |t| Some((t.threads_per_core, t.cores_per_die, t.dies_per_package)),
+        );
+        #[cfg(target_arch = "x86_64")]
+        vcpu.configure(
+            boot_setup,
+            self.cpuid.clone(),
+            self.config.kvm_hyperv,
+            topology,
+        )?;

        #[cfg(target_arch = "aarch64")]
        vcpu.configure(&self.vm, boot_setup)?;
@@ -857,7 +888,7 @@ impl CpuManager {
        let reset_evt = self.reset_evt.try_clone().unwrap();
        let exit_evt = self.exit_evt.try_clone().unwrap();
        #[cfg(feature = "kvm")]
-        let hypervisor_type = self.hypervisor_type;
+        let hypervisor_type = self.hypervisor.hypervisor_type();
        #[cfg(feature = "guest_debug")]
        let vm_debug_evt = self.vm_debug_evt.try_clone().unwrap();
        let panic_exit_evt = self.exit_evt.try_clone().unwrap();
@@ -885,9 +916,12 @@ impl CpuManager {
        });

        // Retrieve seccomp filter for vcpu thread
-        let vcpu_seccomp_filter =
-            get_seccomp_filter(&self.seccomp_action, Thread::Vcpu, self.hypervisor_type)
-                .map_err(Error::CreateSeccompFilter)?;
+        let vcpu_seccomp_filter = get_seccomp_filter(
+            &self.seccomp_action,
+            Thread::Vcpu,
+            self.hypervisor.hypervisor_type(),
+        )
+        .map_err(Error::CreateSeccompFilter)?;

        #[cfg(target_arch = "x86_64")]
        let interrupt_controller_clone = self.interrupt_controller.as_ref().cloned();
--- a/vmm/src/device_manager.rs
+++ b/vmm/src/device_manager.rs
@@ -35,9 +35,9 @@ use arch::NumaNodes;
 #[cfg(target_arch = "aarch64")]
 use arch::{DeviceType, MmioDeviceInfo};
 use block::{
-    async_io::DiskFile, block_io_uring_is_supported, detect_image_type,
-    fixed_vhd_sync::FixedVhdDiskSync, qcow, qcow_sync::QcowDiskSync, raw_sync::RawFileDiskSync,
-    vhdx, vhdx_sync::VhdxDiskSync, ImageType,
+    async_io::DiskFile, block_aio_is_supported, block_io_uring_is_supported, detect_image_type,
+    fixed_vhd_sync::FixedVhdDiskSync, qcow, qcow_sync::QcowDiskSync, raw_async_aio::RawFileDiskAio,
+    raw_sync::RawFileDiskSync, vhdx, vhdx_sync::VhdxDiskSync, ImageType,
 };
 #[cfg(feature = "io_uring")]
 use block::{fixed_vhd_async::FixedVhdDiskAsync, raw_async::RawFileDisk};
@@ -395,6 +395,9 @@ pub enum DeviceManagerError {
    /// No support for device passthrough
    NoDevicePassthroughSupport,

+    /// No socket option support for console device
+    NoSocketOptionSupportForConsoleDevice,
+
    /// Failed to resize virtio-balloon
    VirtioBalloonResize(virtio_devices::balloon::Error),

@@ -946,6 +949,9 @@ pub struct DeviceManager {
    // io_uring availability if detected
    io_uring_supported: Option<bool>,

+    // aio availability if detected
+    aio_supported: Option<bool>,
+
    // List of unique identifiers provided at boot through the configuration.
    boot_id_list: BTreeSet<String>,

@@ -1062,6 +1068,7 @@ impl DeviceManager {
        for i in 1..num_pci_segments as usize {
            pci_segments.push(PciSegment::new(
                i as u16,
+                numa_node_id_from_pci_segment_id(&numa_nodes, i as u16),
                &address_manager,
                Arc::clone(&address_manager.pci_mmio_allocators[i]),
                &pci_irq_slots,
@@ -1134,6 +1141,7 @@ impl DeviceManager {
            pvpanic_device: None,
            force_iommu,
            io_uring_supported: None,
+            aio_supported: None,
            boot_id_list,
            timestamp,
            pending_activations: Arc::new(Mutex::new(Vec::default())),
@@ -1995,6 +2003,9 @@ impl DeviceManager {
                    Endpoint::File(stdout)
                }
            }
+            ConsoleOutputMode::Socket => {
+                return Err(DeviceManagerError::NoSocketOptionSupportForConsoleDevice);
+            }
            ConsoleOutputMode::Null => Endpoint::Null,
            ConsoleOutputMode::Off => return Ok(None),
        };
@@ -2074,15 +2085,19 @@ impl DeviceManager {
                let _ = self.set_raw_mode(&out);
                Some(Box::new(out))
            }
-            ConsoleOutputMode::Off | ConsoleOutputMode::Null => None,
+            ConsoleOutputMode::Off | ConsoleOutputMode::Null | ConsoleOutputMode::Socket => None,
        };
        if serial_config.mode != ConsoleOutputMode::Off {
            let serial = self.add_serial_device(interrupt_manager, serial_writer)?;
            self.serial_manager = match serial_config.mode {
-                ConsoleOutputMode::Pty | ConsoleOutputMode::Tty => {
-                    let serial_manager =
-                        SerialManager::new(serial, self.serial_pty.clone(), serial_config.mode)
-                            .map_err(DeviceManagerError::CreateSerialManager)?;
+                ConsoleOutputMode::Pty | ConsoleOutputMode::Tty | ConsoleOutputMode::Socket => {
+                    let serial_manager = SerialManager::new(
+                        serial,
+                        self.serial_pty.clone(),
+                        serial_config.mode,
+                        serial_config.socket,
+                    )
+                    .map_err(DeviceManagerError::CreateSerialManager)?;
                    if let Some(mut serial_manager) = serial_manager {
                        serial_manager
                            .start_thread(
@@ -2160,6 +2175,17 @@ impl DeviceManager {
        Ok(devices)
    }

+    // Cache whether aio is supported to avoid checking for very block device
+    fn aio_is_supported(&mut self) -> bool {
+        if let Some(supported) = self.aio_supported {
+            return supported;
+        }
+
+        let supported = block_aio_is_supported();
+        self.aio_supported = Some(supported);
+        supported
+    }
+
    // Cache whether io_uring is supported to avoid probing for very block device
    fn io_uring_is_supported(&mut self) -> bool {
        if let Some(supported) = self.io_uring_supported {
@@ -2281,6 +2307,9 @@ impl DeviceManager {
                        {
                            Box::new(RawFileDisk::new(file)) as Box<dyn DiskFile>
                        }
+                    } else if !disk_cfg.disable_aio && self.aio_is_supported() {
+                        info!("Using asynchronous RAW disk file (aio)");
+                        Box::new(RawFileDiskAio::new(file)) as Box<dyn DiskFile>
                    } else {
                        info!("Using synchronous RAW disk file");
                        Box::new(RawFileDiskSync::new(file)) as Box<dyn DiskFile>
@@ -4333,6 +4362,16 @@ fn numa_node_id_from_memory_zone_id(numa_nodes: &NumaNodes, memory_zone_id: &str
    None
 }

+fn numa_node_id_from_pci_segment_id(numa_nodes: &NumaNodes, pci_segment_id: u16) -> u32 {
+    for (numa_node_id, numa_node) in numa_nodes.iter() {
+        if numa_node.pci_segments.contains(&pci_segment_id) {
+            return *numa_node_id;
+        }
+    }
+
+    0
+}
+
 struct TpmDevice {}

 impl Aml for TpmDevice {
--- a/vmm/src/lib.rs
+++ b/vmm/src/lib.rs
@@ -293,6 +293,29 @@ impl Serialize for PciDeviceInfo {
    }
 }

+pub fn feature_list() -> Vec<String> {
+    vec![
+        #[cfg(feature = "dbus_api")]
+        "dbus_api".to_string(),
+        #[cfg(feature = "dhat-heap")]
+        "dhat-heap".to_string(),
+        #[cfg(feature = "guest_debug")]
+        "guest_debug".to_string(),
+        #[cfg(feature = "io_uring")]
+        "io_uring".to_string(),
+        #[cfg(feature = "kvm")]
+        "kvm".to_string(),
+        #[cfg(feature = "mshv")]
+        "mshv".to_string(),
+        #[cfg(feature = "sev_snp")]
+        "sev_snp".to_string(),
+        #[cfg(feature = "tdx")]
+        "tdx".to_string(),
+        #[cfg(feature = "tracing")]
+        "tracing".to_string(),
+    ]
+}
+
 pub fn start_event_monitor_thread(
    mut monitor: event_monitor::Monitor,
    seccomp_action: &SeccompAction,
@@ -931,6 +954,7 @@ impl Vmm {
            build_version,
            version,
            pid: std::process::id() as i64,
+            features: feature_list(),
        }
    }

@@ -1639,16 +1663,26 @@ impl Vmm {
        let vm_config = vm.get_config();
        #[cfg(all(feature = "kvm", target_arch = "x86_64"))]
        let common_cpuid = {
+            #[cfg(feature = "tdx")]
+            if vm_config.lock().unwrap().is_tdx_enabled() {
+                return Err(MigratableError::MigrateSend(anyhow!(
+                    "Live Migration is not supported when TDX is enabled"
+                )));
+            };
+
+            let amx = vm_config.lock().unwrap().cpus.features.amx;
            let phys_bits =
                vm::physical_bits(&hypervisor, vm_config.lock().unwrap().cpus.max_phys_bits);
            arch::generate_common_cpuid(
                &hypervisor,
-                None,
-                None,
-                phys_bits,
-                vm_config.lock().unwrap().cpus.kvm_hyperv,
-                #[cfg(feature = "tdx")]
-                vm_config.lock().unwrap().is_tdx_enabled(),
+                &arch::CpuidConfig {
+                    sgx_epc_sections: None,
+                    phys_bits,
+                    kvm_hyperv: vm_config.lock().unwrap().cpus.kvm_hyperv,
+                    #[cfg(feature = "tdx")]
+                    tdx: false,
+                    amx,
+                },
            )
            .map_err(|e| {
                MigratableError::MigrateReceive(anyhow!("Error generating common cpuid': {:?}", e))
@@ -1825,6 +1859,13 @@ impl Vmm {
        src_vm_config: &Arc<Mutex<VmConfig>>,
        src_vm_cpuid: &[hypervisor::arch::x86::CpuIdEntry],
    ) -> result::Result<(), MigratableError> {
+        #[cfg(feature = "tdx")]
+        if src_vm_config.lock().unwrap().is_tdx_enabled() {
+            return Err(MigratableError::MigrateReceive(anyhow!(
+                "Live Migration is not supported when TDX is enabled"
+            )));
+        };
+
        // We check the `CPUID` compatibility of between the source vm and destination, which is
        // mostly about feature compatibility and "topology/sgx" leaves are not relevant.
        let dest_cpuid = &{
@@ -1833,12 +1874,14 @@ impl Vmm {
            let phys_bits = vm::physical_bits(&self.hypervisor, vm_config.cpus.max_phys_bits);
            arch::generate_common_cpuid(
                &self.hypervisor.clone(),
-                None,
-                None,
-                phys_bits,
-                vm_config.cpus.kvm_hyperv,
-                #[cfg(feature = "tdx")]
-                vm_config.is_tdx_enabled(),
+                &arch::CpuidConfig {
+                    sgx_epc_sections: None,
+                    phys_bits,
+                    kvm_hyperv: vm_config.cpus.kvm_hyperv,
+                    #[cfg(feature = "tdx")]
+                    tdx: false,
+                    amx: vm_config.cpus.features.amx,
+                },
            )
            .map_err(|e| {
                MigratableError::MigrateReceive(anyhow!("Error generating common cpuid: {:?}", e))
@@ -2250,11 +2293,13 @@ mod unit_tests {
                file: None,
                mode: ConsoleOutputMode::Null,
                iommu: false,
+                socket: None,
            },
            console: ConsoleConfig {
                file: None,
                mode: ConsoleOutputMode::Tty,
                iommu: false,
+                socket: None,
            },
            devices: None,
            user_devices: None,
--- a/vmm/src/pci_segment.rs
+++ b/vmm/src/pci_segment.rs
@@ -25,6 +25,7 @@ pub(crate) struct PciSegment {
    pub(crate) pci_bus: Arc<Mutex<PciBus>>,
    pub(crate) pci_config_mmio: Arc<Mutex<PciConfigMmio>>,
    pub(crate) mmio_config_address: u64,
+    pub(crate) proximity_domain: u32,

    #[cfg(target_arch = "x86_64")]
    pub(crate) pci_config_io: Option<Arc<Mutex<PciConfigIo>>>,
@@ -46,6 +47,7 @@ pub(crate) struct PciSegment {
 impl PciSegment {
    pub(crate) fn new(
        id: u16,
+        numa_node: u32,
        address_manager: &Arc<AddressManager>,
        allocator: Arc<Mutex<AddressAllocator>>,
        pci_irq_slots: &[u8; 32],
@@ -77,6 +79,7 @@ impl PciSegment {
            pci_bus,
            pci_config_mmio,
            mmio_config_address,
+            proximity_domain: numa_node,
            pci_devices_up: 0,
            pci_devices_down: 0,
            #[cfg(target_arch = "x86_64")]
@@ -100,7 +103,7 @@ impl PciSegment {
        allocator: Arc<Mutex<AddressAllocator>>,
        pci_irq_slots: &[u8; 32],
    ) -> DeviceManagerResult<PciSegment> {
-        let mut segment = Self::new(0, address_manager, allocator, pci_irq_slots)?;
+        let mut segment = Self::new(0, 0, address_manager, allocator, pci_irq_slots)?;
        let pci_config_io = Arc::new(Mutex::new(PciConfigIo::new(Arc::clone(&segment.pci_bus))));

        address_manager
@@ -123,7 +126,7 @@ impl PciSegment {
        allocator: Arc<Mutex<AddressAllocator>>,
        pci_irq_slots: &[u8; 32],
    ) -> DeviceManagerResult<PciSegment> {
-        Self::new(0, address_manager, allocator, pci_irq_slots)
+        Self::new(0, 0, address_manager, allocator, pci_irq_slots)
    }

    pub(crate) fn next_device_bdf(&self) -> DeviceManagerResult<PciBdf> {
@@ -329,10 +332,7 @@ impl Aml for PciSegment {
        let supp = aml::Name::new("SUPP".into(), &aml::ZERO);
        pci_dsdt_inner_data.push(&supp);

-        // Since Cloud Hypervisor supports only one PCI bus, it can be tied
-        // to the NUMA node 0. It's up to the user to organize the NUMA nodes
-        // so that the PCI bus relates to the expected vCPUs and guest RAM.
-        let proximity_domain = 0u32;
+        let proximity_domain = self.proximity_domain;
        let pxm_return = aml::Return::new(&proximity_domain);
        let pxm = aml::Method::new("_PXM".into(), 0, false, vec![&pxm_return]);
        pci_dsdt_inner_data.push(&pxm);
--- a/vmm/src/seccomp_filters.rs
+++ b/vmm/src/seccomp_filters.rs
@@ -553,6 +553,11 @@ fn vmm_thread_rules(
            libc::SYS_ioctl,
            create_vmm_ioctl_seccomp_rule(hypervisor_type)?,
        ),
+        (libc::SYS_io_cancel, vec![]),
+        (libc::SYS_io_destroy, vec![]),
+        (libc::SYS_io_getevents, vec![]),
+        (libc::SYS_io_setup, vec![]),
+        (libc::SYS_io_submit, vec![]),
        (libc::SYS_io_uring_enter, vec![]),
        (libc::SYS_io_uring_setup, vec![]),
        (libc::SYS_io_uring_register, vec![]),
@@ -602,6 +607,7 @@ fn vmm_thread_rules(
        (libc::SYS_rt_sigreturn, vec![]),
        (libc::SYS_sched_getaffinity, vec![]),
        (libc::SYS_sched_setaffinity, vec![]),
+        (libc::SYS_seccomp, vec![]),
        (libc::SYS_sendmsg, vec![]),
        (libc::SYS_sendto, vec![]),
        (libc::SYS_set_robust_list, vec![]),
--- a/vmm/src/serial_manager.rs
+++ b/vmm/src/serial_manager.rs
@@ -13,8 +13,11 @@ use libc::EFD_NONBLOCK;
 use serial_buffer::SerialBuffer;
 use std::fs::File;
 use std::io::Read;
-use std::os::unix::io::{AsRawFd, FromRawFd};
+use std::net::Shutdown;
+use std::os::unix::io::{AsRawFd, FromRawFd, IntoRawFd};
+use std::os::unix::net::{UnixListener, UnixStream};
 use std::panic::AssertUnwindSafe;
+use std::path::PathBuf;
 use std::sync::atomic::{AtomicBool, Ordering};
 use std::sync::{Arc, Mutex};
 use std::{io, result, thread};
@@ -54,6 +57,26 @@ pub enum Error {
    /// Cannot spawn SerialManager thread.
    #[error("Error spawning SerialManager thread: {0}")]
    SpawnSerialManager(#[source] io::Error),
+
+    /// Cannot bind to Unix socket
+    #[error("Error binding to socket: {0}")]
+    BindUnixSocket(#[source] io::Error),
+
+    /// Cannot accept connection from Unix socket
+    #[error("Error accepting connection: {0}")]
+    AcceptConnection(#[source] io::Error),
+
+    /// Cannot clone the UnixStream
+    #[error("Error cloning UnixStream: {0}")]
+    CloneUnixStream(#[source] io::Error),
+
+    /// Cannot shutdown the connection
+    #[error("Error shutting down a connection: {0}")]
+    ShutdownConnection(#[source] io::Error),
+
+    /// Cannot remove the serial socket
+    #[error("Error removing serial socket: {0}")]
+    RemoveUnixSocket(#[source] io::Error),
 }
 pub type Result<T> = result::Result<T, Error>;

@@ -62,6 +85,7 @@ pub type Result<T> = result::Result<T, Error>;
 pub enum EpollDispatch {
    File = 0,
    Kill = 1,
+    Socket = 2,
    Unknown,
 }

@@ -71,6 +95,7 @@ impl From<u64> for EpollDispatch {
        match v {
            0 => File,
            1 => Kill,
+            2 => Socket,
            _ => Unknown,
        }
    }
@@ -86,6 +111,8 @@ pub struct SerialManager {
    kill_evt: EventFd,
    handle: Option<thread::JoinHandle<()>>,
    pty_write_out: Option<Arc<AtomicBool>>,
+    mode: ConsoleOutputMode,
+    socket_path: Option<PathBuf>,
 }

 impl SerialManager {
@@ -94,7 +121,10 @@ impl SerialManager {
        #[cfg(target_arch = "aarch64")] serial: Arc<Mutex<Pl011>>,
        pty_pair: Option<Arc<Mutex<PtyPair>>>,
        mode: ConsoleOutputMode,
+        socket: Option<PathBuf>,
    ) -> Result<Option<Self>> {
+        let mut socket_path: Option<PathBuf> = None;
+
        let in_file = match mode {
            ConsoleOutputMode::Pty => {
                if let Some(pty_pair) = pty_pair {
@@ -130,6 +160,17 @@ impl SerialManager {
                    return Ok(None);
                }
            }
+            ConsoleOutputMode::Socket => {
+                if let Some(path_in_socket) = socket {
+                    socket_path = Some(path_in_socket.clone());
+                    let listener = UnixListener::bind(path_in_socket.as_path())
+                        .map_err(Error::BindUnixSocket)?;
+                    // SAFETY: listener is valid and will return valid fd
+                    unsafe { File::from_raw_fd(listener.into_raw_fd()) }
+                } else {
+                    return Ok(None);
+                }
+            }
            _ => return Ok(None),
        };

@@ -144,11 +185,17 @@ impl SerialManager {
        )
        .map_err(Error::Epoll)?;

+        let epoll_fd_data = if mode == ConsoleOutputMode::Socket {
+            EpollDispatch::Socket
+        } else {
+            EpollDispatch::File
+        };
+
        epoll::ctl(
            epoll_fd,
            epoll::ControlOptions::EPOLL_CTL_ADD,
            in_file.as_raw_fd(),
-            epoll::Event::new(epoll::Events::EPOLLIN, EpollDispatch::File as u64),
+            epoll::Event::new(epoll::Events::EPOLLIN, epoll_fd_data as u64),
        )
        .map_err(Error::Epoll)?;

@@ -158,7 +205,11 @@ impl SerialManager {
            pty_write_out = Some(write_out.clone());
            let writer = in_file.try_clone().map_err(Error::FileClone)?;
            let buffer = SerialBuffer::new(Box::new(writer), write_out);
-            serial.as_ref().lock().unwrap().set_out(Box::new(buffer));
+            serial
+                .as_ref()
+                .lock()
+                .unwrap()
+                .set_out(Some(Box::new(buffer)));
        }

        // Use 'File' to enforce closing on 'epoll_fd'
@@ -172,6 +223,8 @@ impl SerialManager {
            kill_evt,
            handle: None,
            pty_write_out,
+            mode,
+            socket_path,
        }))
    }

@@ -212,6 +265,10 @@ impl SerialManager {
        let mut in_file = self.in_file.try_clone().map_err(Error::FileClone)?;
        let serial = self.serial.clone();
        let pty_write_out = self.pty_write_out.clone();
+        //SAFETY: in_file is has a valid fd
+        let listener = unsafe { UnixListener::from_raw_fd(self.in_file.as_raw_fd()) };
+        let mut reader: Option<UnixStream> = None;
+        let mode = self.mode.clone();

        // In case of PTY, we want to be able to detect a connection on the
        // other end of the PTY. This is done by detecting there's no event
@@ -250,7 +307,7 @@ impl SerialManager {
                            }
                        };

-                        if num_events == 0 {
+                        if mode != ConsoleOutputMode::Socket && num_events == 0 {
                            // This very specific case happens when the serial is connected
                            // to a PTY. We know EPOLLHUP is always present when there's nothing
                            // connected at the other end of the PTY. That's why getting no event
@@ -266,11 +323,66 @@ impl SerialManager {
                                    let event = event.data;
                                    warn!("Unknown serial manager loop event: {}", event);
                                }
+                                EpollDispatch::Socket => {
+                                    // New connection request arrived.
+                                    // Shutdown the previous connection, if any
+                                    if let Some(previous_reader) = reader {
+                                        previous_reader
+                                            .shutdown(Shutdown::Both)
+                                            .map_err(Error::AcceptConnection)?;
+                                    }
+                                    // Events on the listening socket will be connection requests.
+                                    // Accept them, create a reader and a writer.
+                                    let (unix_stream, _) =
+                                        listener.accept().map_err(Error::AcceptConnection)?;
+                                    let writer =
+                                        unix_stream.try_clone().map_err(Error::CloneUnixStream)?;
+                                    reader = Some(
+                                        unix_stream.try_clone().map_err(Error::CloneUnixStream)?,
+                                    );
+
+                                    epoll::ctl(
+                                        epoll_fd,
+                                        epoll::ControlOptions::EPOLL_CTL_ADD,
+                                        unix_stream.into_raw_fd(),
+                                        epoll::Event::new(
+                                            epoll::Events::EPOLLIN,
+                                            EpollDispatch::File as u64,
+                                        ),
+                                    )
+                                    .map_err(Error::Epoll)?;
+                                    serial.lock().unwrap().set_out(Some(Box::new(writer)));
+                                }
                                EpollDispatch::File => {
                                    if event.events & libc::EPOLLIN as u32 != 0 {
                                        let mut input = [0u8; 64];
-                                        let count =
-                                            in_file.read(&mut input).map_err(Error::ReadInput)?;
+                                        let count = match mode {
+                                            ConsoleOutputMode::Socket => {
+                                                if let Some(mut serial_reader) = reader.as_ref() {
+                                                    let count = serial_reader
+                                                        .read(&mut input)
+                                                        .map_err(Error::ReadInput)?;
+                                                    if count == 0 {
+                                                        info!("Remote end closed serial socket");
+                                                        serial_reader
+                                                            .shutdown(Shutdown::Both)
+                                                            .map_err(Error::ShutdownConnection)?;
+                                                        reader = None;
+                                                        serial
+                                                            .as_ref()
+                                                            .lock()
+                                                            .unwrap()
+                                                            .set_out(None);
+                                                    }
+                                                    count
+                                                } else {
+                                                    0
+                                                }
+                                            }
+                                            _ => in_file
+                                                .read(&mut input)
+                                                .map_err(Error::ReadInput)?,
+                                        };

                                        // Replace "\n" with "\r" to deal with Windows SAC (#1170)
                                        if count == 1 && input[0] == 0x0a {
@@ -325,5 +437,12 @@ impl Drop for SerialManager {
        if let Some(handle) = self.handle.take() {
            handle.join().ok();
        }
+        if self.mode == ConsoleOutputMode::Socket {
+            if let Some(socket_path) = self.socket_path.as_ref() {
+                std::fs::remove_file(socket_path.as_os_str())
+                    .map_err(Error::RemoveUnixSocket)
+                    .ok();
+            }
+        }
    }
 }
--- a/vmm/src/vm.rs
+++ b/vmm/src/vm.rs
@@ -245,6 +245,10 @@ pub enum Error {
    #[error("Failed to copy firmware to memory: {0}")]
    FirmwareLoad(#[source] vm_memory::GuestMemoryError),

+    #[cfg(feature = "sev_snp")]
+    #[error("Error enabling SEV-SNP VM: {0}")]
+    InitializeSevSnpVm(#[source] hypervisor::HypervisorVmError),
+
    #[cfg(feature = "tdx")]
    #[error("Error performing I/O on TDX firmware file: {0}")]
    LoadTdvf(#[source] std::io::Error),
@@ -491,6 +495,8 @@ impl Vm {

        #[cfg(feature = "tdx")]
        let tdx_enabled = config.lock().unwrap().is_tdx_enabled();
+        #[cfg(feature = "sev_snp")]
+        let sev_snp_enabled = config.lock().unwrap().is_sev_snp_enabled();
        #[cfg(feature = "tdx")]
        let force_iommu = tdx_enabled;
        #[cfg(not(feature = "tdx"))]
@@ -558,6 +564,14 @@ impl Vm {
            .create_boot_vcpus(snapshot_from_id(snapshot.as_ref(), CPU_MANAGER_SNAPSHOT_ID))
            .map_err(Error::CpuManager)?;

+        // This initial SEV-SNP configuration must be done immediately after
+        // vCPUs are created. As part of this initialization we are
+        // transitioning the guest into secure state.
+        #[cfg(feature = "sev_snp")]
+        if sev_snp_enabled {
+            vm.sev_snp_init().map_err(Error::InitializeSevSnpVm)?;
+        }
+
        #[cfg(feature = "tdx")]
        let dynamic = !tdx_enabled;
        #[cfg(not(feature = "tdx"))]
@@ -687,6 +701,10 @@ impl Vm {
                    node.cpus.extend(cpus);
                }

+                if let Some(pci_segments) = &config.pci_segments {
+                    node.pci_segments.extend(pci_segments);
+                }
+
                if let Some(distances) = &config.distances {
                    for distance in distances.iter() {
                        let dest = distance.destination;
@@ -836,20 +854,18 @@ impl Vm {

        cfg_if::cfg_if! {
            if #[cfg(feature = "tdx")] {
+                // Passing KVM_X86_TDX_VM: 1 if tdx_enabled is true
+                // Otherwise KVM_X86_LEGACY_VM: 0
+                // value of tdx_enabled is mapped to KVM_X86_TDX_VM or KVM_X86_LEGACY_VM
                let vm = hypervisor
-                    .create_vm_with_type(if tdx_enabled {
-                        1 // KVM_X86_TDX_VM
-                    } else {
-                        0 // KVM_X86_LEGACY_VM
-                    })
+                    .create_vm_with_type(u64::from(tdx_enabled))
                    .unwrap();
            } else if #[cfg(feature = "sev_snp")] {
+                // Passing SEV_SNP_ENABLED: 1 if sev_snp_enabled is true
+                // Otherwise SEV_SNP_DISABLED: 0
+                // value of sev_snp_enabled is mapped to SEV_SNP_ENABLED for true or SEV_SNP_DISABLED for false
                let vm = hypervisor
-                    .create_vm_with_type(if sev_snp_enabled {
-                        1 // SEV_SNP_ENABLED
-                    } else {
-                        0 // SEV_SNP_DISABLED
-                    })
+                    .create_vm_with_type(u64::from(sev_snp_enabled))
                    .unwrap();
            } else {
                let vm = hypervisor.create_vm().unwrap();
@@ -2374,12 +2390,9 @@ impl Snapshottable for Vm {
    fn snapshot(&mut self) -> std::result::Result<Snapshot, MigratableError> {
        event!("vm", "snapshotting");

-        #[cfg(feature = "tdx")]
-        let tdx_enabled = self.config.lock().unwrap().is_tdx_enabled();
-
        #[cfg(feature = "tdx")]
        {
-            if tdx_enabled {
+            if self.config.lock().unwrap().is_tdx_enabled() {
                return Err(MigratableError::Snapshot(anyhow!(
                    "Snapshot not possible with TDX VM"
                )));
@@ -2395,18 +2408,21 @@ impl Snapshottable for Vm {

        #[cfg(all(feature = "kvm", target_arch = "x86_64"))]
        let common_cpuid = {
+            let amx = self.config.lock().unwrap().cpus.features.amx;
            let phys_bits = physical_bits(
                &self.hypervisor,
                self.config.lock().unwrap().cpus.max_phys_bits,
            );
            arch::generate_common_cpuid(
                &self.hypervisor,
-                None,
-                None,
-                phys_bits,
-                self.config.lock().unwrap().cpus.kvm_hyperv,
-                #[cfg(feature = "tdx")]
-                tdx_enabled,
+                &arch::CpuidConfig {
+                    sgx_epc_sections: None,
+                    phys_bits,
+                    kvm_hyperv: self.config.lock().unwrap().cpus.kvm_hyperv,
+                    #[cfg(feature = "tdx")]
+                    tdx: false,
+                    amx,
+                },
            )
            .map_err(|e| {
                MigratableError::MigrateReceive(anyhow!("Error generating common cpuid: {:?}", e))
--- a/vmm/src/vm_config.rs
+++ b/vmm/src/vm_config.rs
@@ -218,6 +218,9 @@ pub struct DiskConfig {
    // For testing use only. Not exposed in API.
    #[serde(default)]
    pub disable_io_uring: bool,
+    // For testing use only. Not exposed in API.
+    #[serde(default)]
+    pub disable_aio: bool,
    #[serde(default)]
    pub pci_segment: u16,
    #[serde(default)]
@@ -249,6 +252,7 @@ impl Default for DiskConfig {
            vhost_socket: None,
            id: None,
            disable_io_uring: false,
+            disable_aio: false,
            rate_limiter_config: None,
            pci_segment: 0,
            serial: None,
@@ -440,6 +444,7 @@ pub enum ConsoleOutputMode {
    Pty,
    Tty,
    File,
+    Socket,
    Null,
 }

@@ -450,6 +455,7 @@ pub struct ConsoleConfig {
    pub mode: ConsoleOutputMode,
    #[serde(default)]
    pub iommu: bool,
+    pub socket: Option<PathBuf>,
 }

 pub fn default_consoleconfig_file() -> Option<PathBuf> {
@@ -536,6 +542,8 @@ pub struct NumaConfig {
    #[cfg(target_arch = "x86_64")]
    #[serde(default)]
    pub sgx_epc_sections: Option<Vec<String>>,
+    #[serde(default)]
+    pub pci_segments: Option<Vec<u16>>,
 }

 #[derive(Clone, Debug, Default, Deserialize, Eq, PartialEq, Serialize)]
@@ -555,6 +563,7 @@ pub fn default_serial() -> ConsoleConfig {
        file: None,
        mode: ConsoleOutputMode::Null,
        iommu: false,
+        socket: None,
    }
 }

@@ -563,6 +572,7 @@ pub fn default_console() -> ConsoleConfig {
        file: None,
        mode: ConsoleOutputMode::Tty,
        iommu: false,
+        socket: None,
    }
 }