.github: Add aarch64 cross build to release

Fixes: #3099

Signed-off-by: Rob Bradford <robert.bradford@intel.com>

.github/workflows/build.yaml

@@ -31,7 +31,7 @@ jobs:
 
       - name: Debug Check (default features)
         run: |
-          git rev-list origin/master..$GITHUB_SHA | xargs -t -I % sh -c 'git checkout %; cargo check --all --target=${{ matrix.target }}'
+          git rev-list origin/main..$GITHUB_SHA | xargs -t -I % sh -c 'git checkout %; cargo check --all --target=${{ matrix.target }}'
           git checkout $GITHUB_SHA
 
       - name: Build (acpi,kvm)
@@ -39,6 +39,12 @@ jobs:
 
       - name: Build (kvm)
         run: cargo rustc --bin cloud-hypervisor --no-default-features --features "kvm"  -- -D warnings
+
+      - name: Build (acpi,mshv)
+        run: cargo rustc --bin cloud-hypervisor --no-default-features --features "acpi,mshv"  -- -D warnings
+
+      - name: Build (mshv)
+        run: cargo rustc --bin cloud-hypervisor --no-default-features --features "mshv"  -- -D warnings
         
       - name: Release Build (default features)
         run: cargo build --all --release --target=${{ matrix.target }}

.github/workflows/docker-image.yaml

@@ -2,7 +2,7 @@ name: Cloud-Hypervisor's Docker image update
 
 on:
   push:
-    branches: master
+    branches: main
     paths: resources/Dockerfile
 
 jobs:

.github/workflows/release.yaml

@@ -25,6 +25,18 @@ jobs:
         run: cargo build --all --release --target=x86_64-unknown-linux-musl
       - name: Strip cloud-hypervisor binaries
         run: strip target/*/release/cloud-hypervisor
+      - name: Install Rust toolchain (aarch64-unknown-linux-musl)
+        uses: actions-rs/toolchain@v1
+        with:
+            toolchain: stable
+            target: aarch64-unknown-linux-musl
+            override: true
+      - name: Static Build (AArch64)
+        uses: actions-rs/cargo@v1
+        with:
+            use-cross: true
+            command: build
+            args: --all --release --target=aarch64-unknown-linux-musl
       - name: Create Release
         id: create_release
         uses: actions/create-release@v1
@@ -65,3 +77,23 @@ jobs:
           asset_path: target/x86_64-unknown-linux-gnu/release/ch-remote
           asset_name: ch-remote
           asset_content_type: application/octet-stream
+      - name: Upload static AArch64 cloud-hypervisor
+        id: upload-release-static-aarch64-cloud-hypervisor
+        uses: actions/upload-release-asset@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          upload_url: ${{ steps.create_release.outputs.upload_url }}
+          asset_path: target/aarch64-unknown-linux-musl/release/cloud-hypervisor
+          asset_name: cloud-hypervisor-static-aarch64
+          asset_content_type: application/octet-stream
+      - name: Upload static AArch64 ch-remote
+        id: upload-release-static-aarch64-ch-remote
+        uses: actions/upload-release-asset@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          upload_url: ${{ steps.create_release.outputs.upload_url }}
+          asset_path: target/aarch64-unknown-linux-musl/release/ch-remote
+          asset_name: ch-remote-static-aarch64
+          asset_content_type: application/octet-stream

CONTRIBUTING.md

@@ -55,7 +55,7 @@ you want to merge your changes to `cloud-hypervisor`:
    into your github organization.
 2. Within your fork, create a branch for your contribution.
 3. [Create a pull request](https://help.github.com/articles/creating-a-pull-request-from-a-fork/)
-   against the master branch of the Cloud Hypervisor repository.
+   against the main branch of the Cloud Hypervisor repository.
 4. Add reviewers to your pull request and then work with your reviewers to address
    any comments and obtain minimum of 2 [maintainers](MAINTAINERS.md) approvals.
    To update your pull request amend existing commits whenever applicable and

Cargo.lock

@@ -11,9 +11,9 @@ dependencies = [
 
 [[package]]
 name = "addr2line"
-version = "0.15.2"
+version = "0.16.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e7a2e47a1fbe209ee101dd6d61285226744c6c8d3c21c8dc878ba6cb9f467f3a"
+checksum = "3e61f2b7f93d2c7d2b08263acaa4a363b3e276806c68af6134c44f523bf1aacd"
 dependencies = [
  "gimli",
 ]
@@ -44,9 +44,9 @@ dependencies = [
 
 [[package]]
 name = "anyhow"
-version = "1.0.42"
+version = "1.0.43"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "595d3cfa7a60d4555cb5067b99f07142a08ea778de5cf993f7b75c7d8fabc486"
+checksum = "28ae2b3dec75a406790005a200b1bd89785afc02517a00ca99ecfe093ee9e6cf"
 
 [[package]]
 name = "api_client"
@@ -57,9 +57,9 @@ dependencies = [
 
 [[package]]
 name = "arc-swap"
-version = "1.3.0"
+version = "1.3.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e906254e445520903e7fc9da4f709886c84ae4bc4ddaf0e093188d66df4dc820"
+checksum = "b5ab7d9e73059c86c36473f459b52adbd99c3554a4fec492caef460806006f00"
 
 [[package]]
 name = "arch"
@@ -102,9 +102,9 @@ checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a"
 
 [[package]]
 name = "backtrace"
-version = "0.3.60"
+version = "0.3.61"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b7815ea54e4d821e791162e078acbebfd6d8c8939cd559c9335dceb1c8ca7282"
+checksum = "e7a905d892734eea339e896738c14b9afce22b5318f64b951e70bf3844419b01"
 dependencies = [
  "addr2line",
  "cc",
@@ -126,9 +126,9 @@ dependencies = [
 
 [[package]]
 name = "bitflags"
-version = "1.2.1"
+version = "1.3.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693"
+checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
 
 [[package]]
 name = "block_util"
@@ -141,6 +141,7 @@ dependencies = [
  "thiserror",
  "versionize",
  "versionize_derive",
+ "vhdx",
  "virtio-bindings",
  "vm-memory",
  "vm-virtio",
@@ -155,9 +156,9 @@ checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610"
 
 [[package]]
 name = "cc"
-version = "1.0.69"
+version = "1.0.70"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e70cc2f62c6ce1868963827bd677764c62d07c3d9a3e1fb1177ee1a9ab199eb2"
+checksum = "d26a6ce4b6a484fa3edb70f7efa6fc430fd2b87285fe8b84304fd0936faa0dc0"
 
 [[package]]
 name = "cfg-if"
@@ -189,7 +190,7 @@ dependencies = [
 
 [[package]]
 name = "cloud-hypervisor"
-version = "17.0.0"
+version = "18.0.0"
 dependencies = [
  "anyhow",
  "api_client",
@@ -204,7 +205,7 @@ dependencies = [
  "log",
  "net_util",
  "option_parser",
- "seccomp",
+ "seccompiler",
  "serde_json",
  "signal-hook",
  "test_infra",
@@ -224,6 +225,15 @@ dependencies = [
  "bitflags",
 ]
 
+[[package]]
+name = "crc32c"
+version = "0.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "210cdf933e6a81212bfabf90cd8762f471b5922e5f6b709547673ad8e04b9448"
+dependencies = [
+ "rustc_version",
+]
+
 [[package]]
 name = "crc64"
 version = "1.0.0"
@@ -354,9 +364,9 @@ dependencies = [
 
 [[package]]
 name = "gimli"
-version = "0.24.0"
+version = "0.25.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0e4075386626662786ddb0ec9081e7c7eeb1ba31951f447ca780ef9f5d568189"
+checksum = "f0a01e0497841a3b2db4f8afa483cce65f7e96a3498bd6c541734792aeac8fe7"
 
 [[package]]
 name = "glob"
@@ -403,9 +413,9 @@ dependencies = [
 
 [[package]]
 name = "iced-x86"
-version = "1.13.0"
+version = "1.14.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "94ef7eabb0e712d4f12aea976b2deb876739dc70fd787df3876c75c9b7bcf0be"
+checksum = "7383772b06135cede839b7270023b46403656a9148024886e721e82639d3f90e"
 dependencies = [
  "lazy_static",
  "static_assertions",
@@ -413,9 +423,9 @@ dependencies = [
 
 [[package]]
 name = "io-uring"
-version = "0.5.1"
+version = "0.5.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fb82832e05cc4ca298f198a8db108837b4f7b7b1248e3cba8e48f151aece80cf"
+checksum = "8d75829ed9377bab6c90039fe47b9d84caceb4b5063266142e21bcce6550cda8"
 dependencies = [
  "bitflags",
  "libc",
@@ -432,9 +442,9 @@ dependencies = [
 
 [[package]]
 name = "itoa"
-version = "0.4.7"
+version = "0.4.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dd25036021b0de88a0aff6b850051563c6516d0bf53f8638938edbb9de732736"
+checksum = "b71991ff56294aa922b450139ee08b3bfc70982c6b2c7562771375cf73542dd4"
 
 [[package]]
 name = "kvm-bindings"
@@ -465,9 +475,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
 
 [[package]]
 name = "libc"
-version = "0.2.98"
+version = "0.2.101"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "320cfe77175da3a483efed4bc0adc1968ca050b098ce4f2f1c13a56626128790"
+checksum = "3cb00336871be5ed2c8ed44b60ae9959dc5b9f08539422ed43f09e34ecaeba21"
 
 [[package]]
 name = "libssh2-sys"
@@ -524,14 +534,14 @@ dependencies = [
 
 [[package]]
 name = "memchr"
-version = "2.4.0"
+version = "2.4.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b16bd47d9e329435e309c58469fe0791c2d0d1ba96ec0954152a5ae2b04387dc"
+checksum = "308cc39be01b73d0d18f82a0e7b2a3df85245f84af96fdddc5d202d27e47b86a"
 
 [[package]]
 name = "micro_http"
 version = "0.1.0"
-source = "git+https://github.com/firecracker-microvm/micro-http?branch=main#9517a300370a158a7af0996b7eebf040d171e1a4"
+source = "git+https://github.com/firecracker-microvm/micro-http?branch=main#36e59a083e76a2449e0f58e4283d201bc72fdf13"
 dependencies = [
  "libc",
  "vmm-sys-util",
@@ -550,7 +560,7 @@ dependencies = [
 [[package]]
 name = "mshv-bindings"
 version = "0.1.0"
-source = "git+https://github.com/cloud-hypervisor/mshv?branch=master#9481c711c329e1e004eb13dc3f1df946316afda0"
+source = "git+https://github.com/rust-vmm/mshv?branch=main#0d6e4e82b9b575798de333e362129515e81049cb"
 dependencies = [
  "libc",
  "serde",
@@ -562,7 +572,7 @@ dependencies = [
 [[package]]
 name = "mshv-ioctls"
 version = "0.1.0"
-source = "git+https://github.com/cloud-hypervisor/mshv?branch=master#9481c711c329e1e004eb13dc3f1df946316afda0"
+source = "git+https://github.com/rust-vmm/mshv?branch=main#0d6e4e82b9b575798de333e362129515e81049cb"
 dependencies = [
  "libc",
  "mshv-bindings",
@@ -599,18 +609,18 @@ dependencies = [
 
 [[package]]
 name = "object"
-version = "0.25.3"
+version = "0.26.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a38f2be3697a57b4060074ff41b44c16870d916ad7877c17696e063257482bc7"
+checksum = "39f37e50073ccad23b6d09bcb5b263f4e76d3bb6038e4a3c08e52162ffa8abc2"
 dependencies = [
  "memchr",
 ]
 
 [[package]]
 name = "openssl-sys"
-version = "0.9.65"
+version = "0.9.66"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7a7907e3bfa08bb85105209cdfcb6c63d109f8f6c1ed6ca318fff5c1853fbc1d"
+checksum = "1996d2d305e561b70d1ee0c53f1542833f4e1ac6ce9a6708b6ff2738ca67dc82"
 dependencies = [
  "autocfg",
  "cc",
@@ -656,10 +666,12 @@ dependencies = [
  "hypervisor",
  "libc",
  "log",
+ "thiserror",
  "versionize",
  "versionize_derive",
  "vfio-bindings",
  "vfio-ioctls",
+ "vfio_user",
  "vm-allocator",
  "vm-device",
  "vm-memory",
@@ -763,9 +775,9 @@ dependencies = [
 
 [[package]]
 name = "proc-macro2"
-version = "1.0.27"
+version = "1.0.29"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f0d8caf72986c1a598726adc988bb5984792ef84f5ee5aa50209145ee8077038"
+checksum = "b9f5105d4fdaab20335ca9565e106a5d9b82b6219b5ba735731124ac6711d23d"
 dependencies = [
  "unicode-xid",
 ]
@@ -807,9 +819,9 @@ checksum = "41cc0f7e4d5d4544e8861606a285bb08d3e70712ccc7d2b84d7c0ccfaf4b05ce"
 
 [[package]]
 name = "redox_syscall"
-version = "0.2.8"
+version = "0.2.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "742739e41cd49414de871ea5e549afb7e2a3ac77b589bcbebe8c82fab37147fc"
+checksum = "8383f39639269cde97d255a32bdb68c047337295414940c68bdd30c2e13203ff"
 dependencies = [
  "bitflags",
 ]
@@ -821,7 +833,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "528532f3d801c87aec9def2add9ca802fe569e44a544afe633765267840abe64"
 dependencies = [
  "getrandom",
- "redox_syscall 0.2.8",
+ "redox_syscall 0.2.10",
 ]
 
 [[package]]
@@ -854,9 +866,18 @@ dependencies = [
 
 [[package]]
 name = "rustc-demangle"
-version = "0.1.20"
+version = "0.1.21"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dead70b0b5e03e9c814bcb6b01e03e68f7c57a80aa48c72ec92152ab3e818d49"
+checksum = "7ef03e0a2b150c7a90d01faf6254c9c48a41e95fb2a8c2ac1c6f0d2b9aefc342"
+
+[[package]]
+name = "rustc_version"
+version = "0.2.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a"
+dependencies = [
+ "semver",
+]
 
 [[package]]
 name = "ryu"
@@ -871,24 +892,39 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
 
 [[package]]
-name = "seccomp"
+name = "seccompiler"
 version = "0.1.0"
-source = "git+https://github.com/firecracker-microvm/firecracker?tag=v0.24.4#8f44986a0e956a77f2b2324c12f73bec16130c82"
+source = "git+https://github.com/rust-vmm/seccompiler#da5788d52f1ae8886d8ed4624199b7e9fa64ac04"
 dependencies = [
  "libc",
 ]
 
 [[package]]
-name = "serde"
-version = "1.0.126"
+name = "semver"
+version = "0.9.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ec7505abeacaec74ae4778d9d9328fe5a5d04253220a85c4ee022239fc996d03"
+checksum = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403"
+dependencies = [
+ "semver-parser",
+]
+
+[[package]]
+name = "semver-parser"
+version = "0.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3"
+
+[[package]]
+name = "serde"
+version = "1.0.130"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f12d06de37cf59146fbdecab66aa99f9fe4f78722e3607577a5375d66bd0c913"
 
 [[package]]
 name = "serde_derive"
-version = "1.0.126"
+version = "1.0.130"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "963a7dbc9895aeac7ac90e74f34a5d5261828f79df35cbed41e10189d3804d43"
+checksum = "d7bc1a1ab1961464eae040d96713baa5a724a8152c1222492465b54322ec508b"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -897,9 +933,9 @@ dependencies = [
 
 [[package]]
 name = "serde_json"
-version = "1.0.64"
+version = "1.0.67"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "799e97dc9fdae36a5c8b8f2cae9ce2ee9fdce2058c57a93e6099d919fd982f79"
+checksum = "a7f9e390c27c3c0ce8bc5d725f6e4d30a29d26659494aa4b17535f7522c5c950"
 dependencies = [
  "itoa",
  "ryu",
@@ -908,9 +944,9 @@ dependencies = [
 
 [[package]]
 name = "signal-hook"
-version = "0.3.9"
+version = "0.3.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "470c5a6397076fae0094aaf06a08e6ba6f37acb77d3b1b91ea92b4d6c8650c39"
+checksum = "9c98891d737e271a2954825ef19e46bd16bdb98e2746f2eec4f7a4ef7946efd1"
 dependencies = [
  "libc",
  "signal-hook-registry",
@@ -957,9 +993,9 @@ checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a"
 
 [[package]]
 name = "syn"
-version = "1.0.74"
+version = "1.0.76"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1873d832550d4588c3dbc20f01361ab00bfe741048f71e3fecf145a7cc18b29c"
+checksum = "c6f107db402c2c2055242dbf4d2af0e69197202e9faacbef9571bbe47f5a1b84"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -1021,18 +1057,18 @@ dependencies = [
 
 [[package]]
 name = "thiserror"
-version = "1.0.26"
+version = "1.0.29"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "93119e4feac1cbe6c798c34d3a53ea0026b0b1de6a120deef895137c0529bfe2"
+checksum = "602eca064b2d83369e2b2f34b09c70b605402801927c65c11071ac911d299b88"
 dependencies = [
  "thiserror-impl",
 ]
 
 [[package]]
 name = "thiserror-impl"
-version = "1.0.26"
+version = "1.0.29"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "060d69a0afe7796bf42e9e2ff91f5ee691fb15c53d38b4b62a9a53eb23164745"
+checksum = "bad553cc2c78e8de258400763a647e80e6d1b31ee237275d756f6836d204494c"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -1056,6 +1092,9 @@ name = "uuid"
 version = "0.8.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "bc5cf98d8186244414c848017f0e2676b3fcb46807f6668a97dfe67359a3c4b7"
+dependencies = [
+ "getrandom",
+]
 
 [[package]]
 name = "vcpkg"
@@ -1108,24 +1147,58 @@ dependencies = [
 [[package]]
 name = "vfio-ioctls"
 version = "0.1.0"
-source = "git+https://github.com/rust-vmm/vfio-ioctls?branch=master#9b84069e9f419c5369b9a313859cac7e9828d331"
+source = "git+https://github.com/rust-vmm/vfio-ioctls?branch=main#a8ee64b978ccd000df88732814d626651a091891"
 dependencies = [
  "byteorder",
  "kvm-bindings",
  "kvm-ioctls",
  "log",
+ "mshv-bindings",
+ "mshv-ioctls",
  "vfio-bindings",
  "vm-memory",
  "vmm-sys-util",
 ]
 
+[[package]]
+name = "vfio_user"
+version = "0.1.0"
+dependencies = [
+ "anyhow",
+ "libc",
+ "log",
+ "serde",
+ "serde_derive",
+ "serde_json",
+ "thiserror",
+ "vfio-bindings",
+ "vm-memory",
+ "vmm-sys-util",
+]
+
+[[package]]
+name = "vhdx"
+version = "0.1.0"
+dependencies = [
+ "byteorder",
+ "crc32c",
+ "libc",
+ "log",
+ "remain",
+ "thiserror",
+ "uuid",
+ "vmm-sys-util",
+]
+
 [[package]]
 name = "vhost"
 version = "0.1.0"
-source = "git+https://github.com/rust-vmm/vhost?branch=master#99fbfc969287ea87398d3755bd2613c14d77579f"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8a6b90237e10f1a61b35fba73885c3567e1a5a8c40d44daae335f7710210a7dc"
 dependencies = [
  "bitflags",
  "libc",
+ "vm-memory",
  "vmm-sys-util",
 ]
 
@@ -1149,6 +1222,7 @@ version = "0.1.0"
 dependencies = [
  "block_util",
  "clap",
+ "env_logger",
  "epoll",
  "libc",
  "log",
@@ -1166,6 +1240,7 @@ name = "vhost_user_net"
 version = "0.1.0"
 dependencies = [
  "clap",
+ "env_logger",
  "epoll",
  "libc",
  "log",
@@ -1201,7 +1276,7 @@ dependencies = [
  "net_util",
  "pci",
  "rate_limiter",
- "seccomp",
+ "seccompiler",
  "serde",
  "serde_derive",
  "serde_json",
@@ -1243,12 +1318,13 @@ dependencies = [
 [[package]]
 name = "vm-fdt"
 version = "0.1.0"
-source = "git+https://github.com/rust-vmm/vm-fdt?branch=master#3c05f2dc84e7c0849c332969f831bdde329d55aa"
+source = "git+https://github.com/rust-vmm/vm-fdt?branch=main#06cbff3a0248ec654467a87b1be7c0bcf78a41a8"
 
 [[package]]
 name = "vm-memory"
-version = "0.5.0"
-source = "git+https://github.com/rust-vmm/vm-memory?rev=5bd7138758183a73ac0da27ce40c004d95f1a7e9#5bd7138758183a73ac0da27ce40c004d95f1a7e9"
+version = "0.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0a8ebcb86ca457f9d6e14cf97009f679952eba42f0113de5db596e514cd0e43b"
 dependencies = [
  "arc-swap",
  "libc",
@@ -1303,7 +1379,7 @@ dependencies = [
  "option_parser",
  "pci",
  "qcow",
- "seccomp",
+ "seccompiler",
  "serde",
  "serde_derive",
  "serde_json",
@@ -1313,6 +1389,7 @@ dependencies = [
  "versionize",
  "versionize_derive",
  "vfio-ioctls",
+ "vhdx",
  "virtio-devices",
  "vm-allocator",
  "vm-device",
@@ -1382,9 +1459,9 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
 
 [[package]]
 name = "zerocopy"
-version = "0.3.0"
+version = "0.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6580539ad917b7c026220c4b3f2c08d52ce54d6ce0dc491e66002e35388fab46"
+checksum = "5e59ec1d2457bd6c0dd89b50e7d9d6b0b647809bf3f0a59ac85557046950b7b2"
 dependencies = [
  "byteorder",
  "zerocopy-derive",
@@ -1392,9 +1469,9 @@ dependencies = [
 
 [[package]]
 name = "zerocopy-derive"
-version = "0.2.0"
+version = "0.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d498dbd1fd7beb83c86709ae1c33ca50942889473473d287d56ce4770a18edfb"
+checksum = "0af017aca1fa6181f5dd7a802456fe6f7666ecdcc18d0910431f0fc89d474e51"
 dependencies = [
  "proc-macro2",
  "syn",

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "cloud-hypervisor"
-version = "17.0.0"
+version = "18.0.0"
 authors = ["The Cloud Hypervisor Authors"]
 edition = "2018"
 default-run = "cloud-hypervisor"
@@ -13,22 +13,22 @@ homepage = "https://github.com/cloud-hypervisor/cloud-hypervisor"
 lto = true
 
 [dependencies]
-anyhow = "1.0.42"
+anyhow = "1.0.43"
 api_client = { path = "api_client" }
 clap = { version = "2.33.3", features = ["wrap_help"] }
 epoll = "4.3.1"
 event_monitor = { path = "event_monitor" }
 hypervisor = { path = "hypervisor" }
-libc = "0.2.98"
+libc = "0.2.101"
 log = { version = "0.4.14", features = ["std"] }
 option_parser = { path = "option_parser" }
-seccomp = { git = "https://github.com/firecracker-microvm/firecracker", tag = "v0.24.4" }
-serde_json = "1.0.64"
-signal-hook = "0.3.9"
-thiserror = "1.0.26"
+seccompiler = { git = "https://github.com/rust-vmm/seccompiler"}
+serde_json = "1.0.67"
+signal-hook = "0.3.10"
+thiserror = "1.0.29"
 vmm = { path = "vmm" }
 vmm-sys-util = "0.8.0"
-vm-memory = "0.5.0"
+vm-memory = "0.6.0"
 
 [build-dependencies]
 clap = { version = "2.33.3", features = ["wrap_help"] }
@@ -37,14 +37,13 @@ clap = { version = "2.33.3", features = ["wrap_help"] }
 [patch.crates-io]
 kvm-bindings = { git = "https://github.com/cloud-hypervisor/kvm-bindings", branch = "ch-v0.4.0", features = ["with-serde", "fam-wrappers"] }
 versionize_derive = { git = "https://github.com/cloud-hypervisor/versionize_derive", branch = "ch" }
-vm-memory = { git = "https://github.com/rust-vmm/vm-memory", rev = "5bd7138758183a73ac0da27ce40c004d95f1a7e9"}
 
 [dev-dependencies]
 credibility = "0.1.3"
 dirs = "3.0.2"
 lazy_static= "1.4.0"
 net_util = { path = "net_util" }
-serde_json = "1.0.64"
+serde_json = "1.0.67"
 test_infra = { path = "test_infra" }
 wait-timeout = "0.2.0"
 
@@ -78,6 +77,8 @@ members = [
     "pci",
     "qcow",
     "rate_limiter",
+    "vfio_user",
+    "vhdx",
     "vhost_user_backend",
     "vhost_user_block",
     "vhost_user_net",

Jenkinsfile

@@ -2,7 +2,7 @@ pipeline{
 	agent none
 	stages {
 		stage ('Early checks') {
-			agent { node { label 'master' } }
+			agent { node { label 'built-in' } }
 			stages {
 				stage ('Check for RFC/WIP builds') {
 					when {
@@ -14,7 +14,7 @@ pipeline{
 					}
 				}
 				stage ('Cancel older builds') {
-					when { not { branch 'master' } }
+					when { not { branch 'main' } }
 					steps {
 						cancelPreviousBuilds()
 					}
@@ -110,7 +110,7 @@ pipeline{
 					agent { node { label 'bionic-sgx' } }
 					when {
 						beforeAgent true
-						branch 'master'
+						branch 'main'
 					}
 					stages {
 						stage ('Checkout') {
@@ -146,7 +146,7 @@ pipeline{
 					agent { node { label 'bionic-vfio' } }
 					when {
 						beforeAgent true
-						branch 'master'
+						branch 'main'
 					}
 					stages {
 						stage ('Checkout') {
@@ -215,21 +215,47 @@ pipeline{
 						}
 					}
 				}
+				stage ('Worker build - Live Migration') {
+					agent { node { label 'hirsute-small' } }
+					stages {
+						stage ('Checkout') {
+							steps {
+								checkout scm
+							}
+						}
+						stage ('Run live-migration integration tests') {
+							options {
+								timeout(time: 1, unit: 'HOURS')
+							}
+							steps {
+								sh "scripts/dev_cli.sh tests --integration-live-migration"
+							}
+						}
+						stage ('Run live-migration integration tests for musl') {
+							options {
+								timeout(time: 1, unit: 'HOURS')
+							}
+							steps {
+								sh "scripts/dev_cli.sh tests --integration-live-migration --libc musl"
+							}
+						}
+					}
+				}
 			}
 		}
 	}
 	post {
 		regression {
 			script {
-				if (env.BRANCH_NAME == 'master') {
-					slackSend (color: '#ff0000', message: '"master" branch build is now failing')
+				if (env.BRANCH_NAME == 'main') {
+					slackSend (color: '#ff0000', message: '"main" branch build is now failing')
 				}
 			}
 		}
 		fixed {
 			script {
-				if (env.BRANCH_NAME == 'master') {
-					slackSend (color: '#00ff00', message: '"master" branch build is now fixed')
+				if (env.BRANCH_NAME == 'main') {
+					slackSend (color: '#00ff00', message: '"main" branch build is now fixed')
 				}
 			}
 		}

MAINTAINERS.md

@@ -1,6 +1,7 @@
 # Maintainers
 
-- Sebastien Boeuf <sebastien.boeuf@intel.com> @sboeuf
-- Robert Bradford <robert.bradford@intel.com> @rbradford
-- Samuel Ortiz <sameo@linux.intel.com> @sameo
-- Chao P Peng <chao.p.peng@linux.intel.com> @chao-p
+- Sebastien Boeuf - @sboeuf
+- Robert Bradford - @rbradford
+- Samuel Ortiz - @sameo
+- Wei Liu - @liuw
+- Michael Zhao - @michael2012z

README.md

@@ -1,11 +1,10 @@
-[![Build Status](https://travis-ci.com/cloud-hypervisor/cloud-hypervisor.svg?branch=master)](https://travis-ci.com/cloud-hypervisor/cloud-hypervisor)
-
 - [1. What is Cloud Hypervisor?](#1-what-is-cloud-hypervisor)
   - [Objectives](#objectives)
     - [High Level](#high-level)
     - [Architectures](#architectures)
     - [Guest OS](#guest-os)
 - [2. Getting Started](#2-getting-started)
+  - [Install prerequisites](#install-prerequisites)
   - [Clone and build](#clone-and-build)
     - [Containerized builds and tests](#containerized-builds-and-tests)
   - [Run](#run)
@@ -23,6 +22,7 @@
 - [5. Community](#5-community)
   - [Contribute](#contribute)
   - [Join us](#join-us)
+  - [Security issues](#security-issues)
 
 # 1. What is Cloud Hypervisor?
 
@@ -149,7 +149,7 @@ We need to get the latest `rust-hypervisor-firmware` release and also a working
 $ pushd $CLOUDH
 $ wget https://cloud-images.ubuntu.com/focal/current/focal-server-cloudimg-amd64.img
 $ qemu-img convert -p -f qcow2 -O raw focal-server-cloudimg-amd64.img focal-server-cloudimg-amd64.raw
-$ wget https://github.com/cloud-hypervisor/rust-hypervisor-firmware/releases/download/0.3.1/hypervisor-fw
+$ wget https://github.com/cloud-hypervisor/rust-hypervisor-firmware/releases/download/0.3.2/hypervisor-fw
 $ popd
 ```
 
@@ -180,7 +180,7 @@ To build the kernel:
 
 # Clone the Cloud Hypervisor Linux branch
 $ pushd $CLOUDH
-$ git clone --depth 1 https://github.com/cloud-hypervisor/linux.git -b ch-5.13-rc5 linux-cloud-hypervisor
+$ git clone --depth 1 https://github.com/cloud-hypervisor/linux.git -b ch-5.14 linux-cloud-hypervisor
 $ pushd linux-cloud-hypervisor
 
 # Use the cloud-hypervisor kernel config to build your kernel

acpi_tables/Cargo.toml

@@ -5,4 +5,4 @@ authors = ["The Cloud Hypervisor Authors"]
 edition = "2018"
 
 [dependencies]
-vm-memory = "0.5.0"
+vm-memory = "0.6.0"

api_client/Cargo.toml

@@ -5,4 +5,4 @@ authors = ["The Cloud Hypervisor Authors"]
 edition = "2018"
 
 [dependencies]
-vmm-sys-util = "0.8.0"
+vmm-sys-util = "0.8.0"

arch/Cargo.toml

@@ -11,20 +11,20 @@ tdx = []
 
 [dependencies]
 acpi_tables = { path = "../acpi_tables", optional = true }
-anyhow = "1.0"
+anyhow = "1.0.43"
 byteorder = "1.4.3"
 hypervisor = { path = "../hypervisor" }
-libc = "0.2.98"
+libc = "0.2.101"
 linux-loader = { version = "0.3.0", features = ["elf", "bzimage", "pe"] }
 log = "0.4.14"
-serde = {version = ">=1.0.27", features = ["rc"] }
-thiserror = "1.0"
+serde = { version = "1.0.130", features = ["rc"] }
+thiserror = "1.0.29"
 versionize = "0.1.6"
 versionize_derive = "0.1.4"
-vm-memory = { version = "0.5.0", features = ["backend-mmap", "backend-bitmap"] }
+vm-memory = { version = "0.6.0", features = ["backend-mmap", "backend-bitmap"] }
 vm-migration = { path = "../vm-migration" }
-vmm-sys-util = { version = ">=0.5.0", features = ["with-serde"] }
+vmm-sys-util = { version = "0.8.0", features = ["with-serde"] }
 
 [target.'cfg(target_arch = "aarch64")'.dependencies]
 fdt_parser = { version = "0.1.3", package = 'fdt'}
-vm-fdt = { git = "https://github.com/rust-vmm/vm-fdt", branch = "master" }
+vm-fdt = { git = "https://github.com/rust-vmm/vm-fdt", branch = "main" }

arch/src/aarch64/fdt.rs

@@ -6,6 +6,7 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the THIRD-PARTY file.
 
+use crate::NumaNodes;
 use byteorder::{BigEndian, ByteOrder};
 use std::cmp;
 use std::collections::HashMap;
@@ -24,7 +25,7 @@ use super::layout::{
     PCI_HIGH_BASE, PCI_MMCONFIG_SIZE, PCI_MMCONFIG_START,
 };
 use vm_fdt::{FdtWriter, FdtWriterResult};
-use vm_memory::{Address, Bytes, GuestAddress, GuestMemory, GuestMemoryError};
+use vm_memory::{Address, Bytes, GuestAddress, GuestMemory, GuestMemoryError, GuestMemoryRegion};
 
 // This is a value for uniquely identifying the FDT node declaring the interrupt controller.
 const GIC_PHANDLE: u32 = 1;
@@ -34,6 +35,9 @@ const MSI_PHANDLE: u32 = 2;
 const CLOCK_PHANDLE: u32 = 3;
 // This is a value for uniquely identifying the FDT node containing the gpio controller.
 const GPIO_PHANDLE: u32 = 4;
+// This is a value for uniquely identifying the FDT node containing the first vCPU.
+// The last number of vCPU phandle depends on the number of vCPUs.
+const FIRST_VCPU_PHANDLE: u32 = 5;
 
 // Read the documentation specified when appending the root node to the FDT.
 const ADDRESS_CELLS: u32 = 0x2;
@@ -72,14 +76,17 @@ pub enum Error {
 type Result<T> = result::Result<T, Error>;
 
 /// Creates the flattened device tree for this aarch64 VM.
+#[allow(clippy::too_many_arguments)]
 pub fn create_fdt<T: DeviceInfoForFdt + Clone + Debug, S: ::std::hash::BuildHasher>(
     guest_mem: &GuestMemoryMmap,
     cmdline: &CStr,
     vcpu_mpidr: Vec<u64>,
+    vcpu_topology: Option<(u8, u8, u8)>,
     device_info: &HashMap<(DeviceType, String), T, S>,
     gic_device: &dyn GicDevice,
     initrd: &Option<InitramfsConfig>,
     pci_space_address: &(u64, u64),
+    numa_nodes: &NumaNodes,
 ) -> FdtWriterResult<Vec<u8>> {
     // Allocate stuff necessary for the holding the blob.
     let mut fdt = FdtWriter::new().unwrap();
@@ -98,8 +105,8 @@ pub fn create_fdt<T: DeviceInfoForFdt + Clone + Debug, S: ::std::hash::BuildHash
     // This is not mandatory but we use it to point the root node to the node
     // containing description of the interrupt controller for this VM.
     fdt.property_u32("interrupt-parent", GIC_PHANDLE)?;
-    create_cpu_nodes(&mut fdt, &vcpu_mpidr)?;
-    create_memory_node(&mut fdt, guest_mem)?;
+    create_cpu_nodes(&mut fdt, &vcpu_mpidr, vcpu_topology, numa_nodes)?;
+    create_memory_node(&mut fdt, guest_mem, numa_nodes)?;
     create_chosen_node(&mut fdt, cmdline.to_str().unwrap(), initrd)?;
     create_gic_node(&mut fdt, gic_device)?;
     create_timer_node(&mut fdt)?;
@@ -107,6 +114,9 @@ pub fn create_fdt<T: DeviceInfoForFdt + Clone + Debug, S: ::std::hash::BuildHash
     create_psci_node(&mut fdt)?;
     create_devices_node(&mut fdt, device_info)?;
     create_pci_nodes(&mut fdt, pci_space_address.0, pci_space_address.1)?;
+    if numa_nodes.len() > 1 {
+        create_distance_map_node(&mut fdt, numa_nodes)?;
+    }
 
     // End Header node.
     fdt.end_node(root_node)?;
@@ -126,12 +136,21 @@ pub fn write_fdt_to_memory(fdt_final: Vec<u8>, guest_mem: &GuestMemoryMmap) -> R
 }
 
 // Following are the auxiliary function for creating the different nodes that we append to our FDT.
-fn create_cpu_nodes(fdt: &mut FdtWriter, vcpu_mpidr: &[u64]) -> FdtWriterResult<()> {
+fn create_cpu_nodes(
+    fdt: &mut FdtWriter,
+    vcpu_mpidr: &[u64],
+    vcpu_topology: Option<(u8, u8, u8)>,
+    numa_nodes: &NumaNodes,
+) -> FdtWriterResult<()> {
     // See https://github.com/torvalds/linux/blob/master/Documentation/devicetree/bindings/arm/cpus.yaml.
     let cpus_node = fdt.begin_node("cpus")?;
     fdt.property_u32("#address-cells", 0x1)?;
     fdt.property_u32("#size-cells", 0x0)?;
+
     let num_cpus = vcpu_mpidr.len();
+    let threads_per_core = vcpu_topology.unwrap_or_default().0 as u8;
+    let cores_per_package = vcpu_topology.unwrap_or_default().1 as u8;
+    let packages = vcpu_topology.unwrap_or_default().2 as u8;
 
     for (cpu_id, mpidr) in vcpu_mpidr.iter().enumerate().take(num_cpus) {
         let cpu_name = format!("cpu@{:x}", cpu_id);
@@ -145,22 +164,129 @@ fn create_cpu_nodes(fdt: &mut FdtWriter, vcpu_mpidr: &[u64]) -> FdtWriterResult<
         // Set the field to first 24 bits of the MPIDR - Multiprocessor Affinity Register.
         // See http://infocenter.arm.com/help/index.jsp?topic=/com.arm.doc.ddi0488c/BABHBJCI.html.
         fdt.property_u32("reg", (mpidr & 0x7FFFFF) as u32)?;
+        fdt.property_u32("phandle", cpu_id as u32 + FIRST_VCPU_PHANDLE)?;
+
+        // Add `numa-node-id` property if there is any numa config.
+        if numa_nodes.len() > 1 {
+            for numa_node_idx in 0..numa_nodes.len() {
+                let numa_node = numa_nodes.get(&(numa_node_idx as u32));
+                if numa_node.unwrap().cpus.contains(&(cpu_id as u8)) {
+                    fdt.property_u32("numa-node-id", numa_node_idx as u32)?;
+                }
+            }
+        }
+
         fdt.end_node(cpu_node)?;
     }
+
+    // If there is a valid cpu topology config, create the cpu-map node.
+    if (threads_per_core > 0)
+        && (cores_per_package > 0)
+        && (packages > 0)
+        && (num_cpus as u8 == threads_per_core * cores_per_package * packages)
+    {
+        let cpu_map_node = fdt.begin_node("cpu-map")?;
+        // Create mappings between CPU index and cluster,core, and thread.
+        let mut cluster_core_thread_to_cpuidx = HashMap::new();
+        for cpu_idx in 0..num_cpus as u8 {
+            if threads_per_core > 1 {
+                cluster_core_thread_to_cpuidx.insert(
+                    (
+                        cpu_idx / (cores_per_package * threads_per_core),
+                        (cpu_idx / threads_per_core) % cores_per_package,
+                        cpu_idx % threads_per_core,
+                    ),
+                    cpu_idx,
+                );
+            } else {
+                cluster_core_thread_to_cpuidx.insert(
+                    (cpu_idx / cores_per_package, cpu_idx % cores_per_package, 0),
+                    cpu_idx,
+                );
+            }
+        }
+
+        // Create device tree nodes with regard of above mapping.
+        for cluster_idx in 0..packages {
+            let cluster_name = format!("cluster{:x}", cluster_idx);
+            let cluster_node = fdt.begin_node(&cluster_name)?;
+
+            for core_idx in 0..cores_per_package {
+                let core_name = format!("core{:x}", core_idx);
+                let core_node = fdt.begin_node(&core_name)?;
+
+                if threads_per_core > 1 {
+                    for thread_idx in 0..threads_per_core {
+                        let thread_name = format!("thread{:x}", thread_idx);
+                        let thread_node = fdt.begin_node(&thread_name)?;
+                        let cpu_idx = cluster_core_thread_to_cpuidx
+                            .get(&(cluster_idx, core_idx, thread_idx))
+                            .unwrap();
+
+                        fdt.property_u32("cpu", *cpu_idx as u32 + FIRST_VCPU_PHANDLE)?;
+                        fdt.end_node(thread_node)?;
+                    }
+                } else {
+                    let cpu_idx = cluster_core_thread_to_cpuidx
+                        .get(&(cluster_idx, core_idx, 0))
+                        .unwrap();
+
+                    fdt.property_u32("cpu", *cpu_idx as u32 + FIRST_VCPU_PHANDLE)?;
+                }
+                fdt.end_node(core_node)?;
+            }
+            fdt.end_node(cluster_node)?;
+        }
+        fdt.end_node(cpu_map_node)?;
+    } else {
+        debug!("Boot using device tree, CPU topology is not (correctly) specified");
+    }
+
     fdt.end_node(cpus_node)?;
+
     Ok(())
 }
 
-fn create_memory_node(fdt: &mut FdtWriter, guest_mem: &GuestMemoryMmap) -> FdtWriterResult<()> {
-    let mem_size = guest_mem.last_addr().raw_value() - super::layout::RAM_64BIT_START + 1;
-    // See https://github.com/torvalds/linux/blob/master/Documentation/devicetree/booting-without-of.txt#L960
-    // for an explanation of this.
-    let mem_reg_prop = [super::layout::RAM_64BIT_START as u64, mem_size as u64];
+fn create_memory_node(
+    fdt: &mut FdtWriter,
+    guest_mem: &GuestMemoryMmap,
+    numa_nodes: &NumaNodes,
+) -> FdtWriterResult<()> {
+    if numa_nodes.len() > 1 {
+        for numa_node_idx in 0..numa_nodes.len() {
+            let numa_node = numa_nodes.get(&(numa_node_idx as u32));
+            // Each memory zone of numa will have its own memory node, but
+            // different numa nodes should not share same memory zones.
+            for memory_region in numa_node.unwrap().memory_regions.iter() {
+                let memory_region_start_addr: u64 = memory_region.start_addr().raw_value();
+                let memory_region_size: u64 = memory_region.size() as u64;
+                let mem_reg_prop = [memory_region_start_addr, memory_region_size];
+                // With feature `acpi` enabled, RAM at 0-4M is for edk2 only
+                // and should be hidden to the guest.
+                #[cfg(feature = "acpi")]
+                if memory_region_start_addr == 0 {
+                    continue;
+                }
 
-    let memory_node = fdt.begin_node("memory")?;
-    fdt.property_string("device_type", "memory")?;
-    fdt.property_array_u64("reg", &mem_reg_prop)?;
-    fdt.end_node(memory_node)?;
+                let memory_node_name = format!("memory@{:x}", memory_region_start_addr);
+                let memory_node = fdt.begin_node(&memory_node_name)?;
+                fdt.property_string("device_type", "memory")?;
+                fdt.property_array_u64("reg", &mem_reg_prop)?;
+                fdt.property_u32("numa-node-id", numa_node_idx as u32)?;
+                fdt.end_node(memory_node)?;
+            }
+        }
+    } else {
+        let mem_size = guest_mem.last_addr().raw_value() - super::layout::RAM_64BIT_START + 1;
+        // See https://github.com/torvalds/linux/blob/master/Documentation/devicetree/booting-without-of.txt#L960
+        // for an explanation of this.
+        let mem_reg_prop = [super::layout::RAM_64BIT_START as u64, mem_size as u64];
+        let memory_node = fdt.begin_node("memory")?;
+
+        fdt.property_string("device_type", "memory")?;
+        fdt.property_array_u64("reg", &mem_reg_prop)?;
+        fdt.end_node(memory_node)?;
+    }
 
     Ok(())
 }
@@ -482,6 +608,47 @@ fn create_pci_nodes(
     Ok(())
 }
 
+fn create_distance_map_node(fdt: &mut FdtWriter, numa_nodes: &NumaNodes) -> FdtWriterResult<()> {
+    let distance_map_node = fdt.begin_node("distance-map")?;
+    fdt.property_string("compatible", "numa-distance-map-v1")?;
+    // Construct the distance matrix.
+    // 1. We use the word entry to describe a distance from a node to
+    // its destination, e.g. 0 -> 1 = 20 is described as <0 1 20>.
+    // 2. Each entry represents distance from first node to second node.
+    // The distances are equal in either direction.
+    // 3. The distance from a node to self (local distance) is represented
+    // with value 10 and all internode distance should be represented with
+    // a value greater than 10.
+    // 4. distance-matrix should have entries in lexicographical ascending
+    // order of nodes.
+    let mut distance_matrix = Vec::new();
+    for numa_node_idx in 0..numa_nodes.len() {
+        let numa_node = numa_nodes.get(&(numa_node_idx as u32));
+        for dest_numa_node in 0..numa_node.unwrap().distances.len() + 1 {
+            if numa_node_idx == dest_numa_node {
+                distance_matrix.push(numa_node_idx as u32);
+                distance_matrix.push(dest_numa_node as u32);
+                distance_matrix.push(10_u32);
+                continue;
+            }
+
+            distance_matrix.push(numa_node_idx as u32);
+            distance_matrix.push(dest_numa_node as u32);
+            distance_matrix.push(
+                *numa_node
+                    .unwrap()
+                    .distances
+                    .get(&(dest_numa_node as u32))
+                    .unwrap() as u32,
+            );
+        }
+    }
+    fdt.property_array_u32("distance-matrix", distance_matrix.as_ref())?;
+    fdt.end_node(distance_map_node)?;
+
+    Ok(())
+}
+
 // Parse the DTB binary and print for debugging
 pub fn print_fdt(dtb: &[u8]) {
     match fdt_parser::Fdt::new(dtb) {

arch/src/aarch64/gic/gicv3.rs

@@ -109,9 +109,6 @@ pub mod kvm {
 
         /// Save the state of GIC.
         fn state(&self, gicr_typers: &[u64]) -> Result<Gicv3State> {
-            // Flush redistributors pending tables to guest RAM.
-            save_pending_tables(self.device()).map_err(Error::SavePendingTables)?;
-
             let gicd_ctlr =
                 read_ctlr(self.device()).map_err(Error::SaveDistributorCtrlRegisters)?;
 
@@ -255,7 +252,16 @@ pub mod kvm {
         }
     }
 
-    impl Pausable for KvmGicV3 {}
+    impl Pausable for KvmGicV3 {
+        fn pause(&mut self) -> std::result::Result<(), MigratableError> {
+            // Flush redistributors pending tables to guest RAM.
+            save_pending_tables(self.device()).map_err(|e| {
+                MigratableError::Pause(anyhow!("Could not save GICv3 GIC pending tables {:?}", e))
+            })?;
+
+            Ok(())
+        }
+    }
     impl Transportable for KvmGicV3 {}
     impl Migratable for KvmGicV3 {}
 }

arch/src/aarch64/gic/gicv3_its.rs

@@ -191,9 +191,6 @@ pub mod kvm {
 
         /// Save the state of GICv3ITS.
         fn state(&self, gicr_typers: &[u64]) -> Result<Gicv3ItsState> {
-            // Flush redistributors pending tables to guest RAM.
-            save_pending_tables(self.device()).map_err(Error::SavePendingTables)?;
-
             let gicd_ctlr =
                 read_ctlr(self.device()).map_err(Error::SaveDistributorCtrlRegisters)?;
 
@@ -206,10 +203,6 @@ pub mod kvm {
             let icc_state =
                 get_icc_regs(self.device(), gicr_typers).map_err(Error::SaveIccRegisters)?;
 
-            // Save GICv3ITS registers
-            gicv3_its_tables_access(self.its_device().unwrap(), true)
-                .map_err(Error::SaveITSTables)?;
-
             let its_baser_state: [u64; 8] = [0; 8];
             for i in 0..8 {
                 gicv3_its_attr_access(
@@ -501,7 +494,23 @@ pub mod kvm {
         }
     }
 
-    impl Pausable for KvmGicV3Its {}
+    impl Pausable for KvmGicV3Its {
+        fn pause(&mut self) -> std::result::Result<(), MigratableError> {
+            // Flush redistributors pending tables to guest RAM.
+            save_pending_tables(self.device()).map_err(|e| {
+                MigratableError::Pause(anyhow!(
+                    "Could not save GICv3ITS GIC pending tables {:?}",
+                    e
+                ))
+            })?;
+            // Flush ITS tables to guest RAM.
+            gicv3_its_tables_access(self.its_device().unwrap(), true).map_err(|e| {
+                MigratableError::Pause(anyhow!("Could not save GICv3ITS ITS tables {:?}", e))
+            })?;
+
+            Ok(())
+        }
+    }
     impl Transportable for KvmGicV3Its {}
     impl Migratable for KvmGicV3Its {}
 }

arch/src/aarch64/mod.rs

@@ -14,9 +14,7 @@ pub mod regs;
 pub mod uefi;
 
 pub use self::fdt::DeviceInfoForFdt;
-use crate::DeviceType;
-use crate::GuestMemoryMmap;
-use crate::RegionType;
+use crate::{DeviceType, GuestMemoryMmap, NumaNodes, RegionType};
 use gic::GicDevice;
 use log::{log_enabled, Level};
 use std::collections::HashMap;
@@ -135,23 +133,28 @@ pub fn arch_memory_regions(size: GuestUsize) -> Vec<(GuestAddress, usize, Region
 }
 
 /// Configures the system and should be called once per vm before starting vcpu threads.
+#[allow(clippy::too_many_arguments)]
 pub fn configure_system<T: DeviceInfoForFdt + Clone + Debug, S: ::std::hash::BuildHasher>(
     guest_mem: &GuestMemoryMmap,
     cmdline_cstring: &CStr,
     vcpu_mpidr: Vec<u64>,
+    vcpu_topology: Option<(u8, u8, u8)>,
     device_info: &HashMap<(DeviceType, String), T, S>,
     initrd: &Option<super::InitramfsConfig>,
     pci_space_address: &(u64, u64),
     gic_device: &dyn GicDevice,
+    numa_nodes: &NumaNodes,
 ) -> super::Result<()> {
     let fdt_final = fdt::create_fdt(
         guest_mem,
         cmdline_cstring,
         vcpu_mpidr,
+        vcpu_topology,
         device_info,
         gic_device,
         initrd,
         pci_space_address,
+        numa_nodes,
     )
     .map_err(|_| Error::SetupFdt)?;
 

arch/src/lib.rs

@@ -11,10 +11,15 @@
 #[macro_use]
 extern crate log;
 
+#[cfg(target_arch = "x86_64")]
+use crate::x86_64::SgxEpcSection;
+use std::collections::BTreeMap;
 use std::fmt;
 use std::result;
+use std::sync::Arc;
 
 type GuestMemoryMmap = vm_memory::GuestMemoryMmap<vm_memory::bitmap::AtomicBitmap>;
+type GuestRegionMmap = vm_memory::GuestRegionMmap<vm_memory::bitmap::AtomicBitmap>;
 
 /// Type for returning error code.
 #[derive(Debug)]
@@ -85,7 +90,7 @@ pub mod x86_64;
 pub use x86_64::{
     arch_memory_regions, configure_system, configure_vcpu, generate_common_cpuid,
     get_host_cpu_phys_bits, initramfs_load_addr, layout, layout::CMDLINE_MAX_SIZE,
-    layout::CMDLINE_START, regs, EntryPoint,
+    layout::CMDLINE_START, regs, CpuidFeatureEntry, EntryPoint,
 };
 
 /// Safe wrapper for `sysconf(_SC_PAGESIZE)`.
@@ -96,6 +101,19 @@ fn pagesize() -> usize {
     unsafe { libc::sysconf(libc::_SC_PAGESIZE) as usize }
 }
 
+#[derive(Clone, Default)]
+pub struct NumaNode {
+    pub memory_regions: Vec<Arc<GuestRegionMmap>>,
+    pub hotplug_regions: Vec<Arc<GuestRegionMmap>>,
+    pub cpus: Vec<u8>,
+    pub distances: BTreeMap<u32, u8>,
+    pub memory_zones: Vec<String>,
+    #[cfg(target_arch = "x86_64")]
+    pub sgx_epc_sections: Vec<SgxEpcSection>,
+}
+
+pub type NumaNodes = BTreeMap<u32, NumaNode>;
+
 /// Type for passing information about the initramfs in the guest memory.
 pub struct InitramfsConfig {
     /// Load address of initramfs in guest memory

arch/src/x86_64/mod.rs

@@ -185,6 +185,9 @@ pub enum Error {
 
     /// Error populating CPUID with CPU identification
     CpuidIdentification(vmm_sys_util::fam::Error),
+
+    /// Error checking CPUID compatibility
+    CpuidCheckCompatibility,
 }
 
 impl From<Error> for super::Error {
@@ -194,7 +197,7 @@ impl From<Error> for super::Error {
 }
 
 #[allow(dead_code, clippy::upper_case_acronyms)]
-#[derive(Copy, Clone)]
+#[derive(Copy, Clone, Debug)]
 pub enum CpuidReg {
     EAX,
     EBX,
@@ -338,6 +341,224 @@ impl CpuidPatch {
     }
 }
 
+#[derive(Debug)]
+enum CpuidCompatibleCheck {
+    BitwiseSubset, // bitwise subset
+    Equal,         // equal in value
+    NumNotGreater, // smaller or equal as a number
+}
+
+pub struct CpuidFeatureEntry {
+    function: u32,
+    index: u32,
+    feature_reg: CpuidReg,
+    compatible_check: CpuidCompatibleCheck,
+}
+
+impl CpuidFeatureEntry {
+    fn checked_feature_entry_list() -> Vec<CpuidFeatureEntry> {
+        vec![
+            // The following list includes all hardware features bits from
+            // the CPUID Wiki Page: https://en.wikipedia.org/wiki/CPUID
+            // Leaf 0x1, ECX/EDX, feature bits
+            CpuidFeatureEntry {
+                function: 1,
+                index: 0,
+                feature_reg: CpuidReg::ECX,
+                compatible_check: CpuidCompatibleCheck::BitwiseSubset,
+            },
+            CpuidFeatureEntry {
+                function: 1,
+                index: 0,
+                feature_reg: CpuidReg::EDX,
+                compatible_check: CpuidCompatibleCheck::BitwiseSubset,
+            },
+            // Leaf 0x7, EAX/EBX/ECX/EDX, extended features
+            CpuidFeatureEntry {
+                function: 7,
+                index: 0,
+                feature_reg: CpuidReg::EAX,
+                compatible_check: CpuidCompatibleCheck::NumNotGreater,
+            },
+            CpuidFeatureEntry {
+                function: 7,
+                index: 0,
+                feature_reg: CpuidReg::EBX,
+                compatible_check: CpuidCompatibleCheck::BitwiseSubset,
+            },
+            CpuidFeatureEntry {
+                function: 7,
+                index: 0,
+                feature_reg: CpuidReg::ECX,
+                compatible_check: CpuidCompatibleCheck::BitwiseSubset,
+            },
+            CpuidFeatureEntry {
+                function: 7,
+                index: 0,
+                feature_reg: CpuidReg::EDX,
+                compatible_check: CpuidCompatibleCheck::BitwiseSubset,
+            },
+            // Leaf 0x7 subleaf 0x1, EAX, extended features
+            CpuidFeatureEntry {
+                function: 7,
+                index: 1,
+                feature_reg: CpuidReg::EAX,
+                compatible_check: CpuidCompatibleCheck::BitwiseSubset,
+            },
+            // Leaf 0x8000_0001, ECX/EDX, CPUID features bits
+            CpuidFeatureEntry {
+                function: 0x8000_0001,
+                index: 0,
+                feature_reg: CpuidReg::ECX,
+                compatible_check: CpuidCompatibleCheck::BitwiseSubset,
+            },
+            CpuidFeatureEntry {
+                function: 0x8000_0001,
+                index: 0,
+                feature_reg: CpuidReg::EDX,
+                compatible_check: CpuidCompatibleCheck::BitwiseSubset,
+            },
+            // KVM CPUID bits: https://www.kernel.org/doc/html/latest/virt/kvm/cpuid.html
+            // Leaf 0x4000_0000, EAX/EBX/ECX/EDX, KVM CPUID SIGNATURE
+            CpuidFeatureEntry {
+                function: 0x4000_0000,
+                index: 0,
+                feature_reg: CpuidReg::EAX,
+                compatible_check: CpuidCompatibleCheck::NumNotGreater,
+            },
+            CpuidFeatureEntry {
+                function: 0x4000_0000,
+                index: 0,
+                feature_reg: CpuidReg::EBX,
+                compatible_check: CpuidCompatibleCheck::Equal,
+            },
+            CpuidFeatureEntry {
+                function: 0x4000_0000,
+                index: 0,
+                feature_reg: CpuidReg::ECX,
+                compatible_check: CpuidCompatibleCheck::Equal,
+            },
+            CpuidFeatureEntry {
+                function: 0x4000_0000,
+                index: 0,
+                feature_reg: CpuidReg::EDX,
+                compatible_check: CpuidCompatibleCheck::Equal,
+            },
+            // Leaf 0x4000_0001, EAX/EBX/ECX/EDX, KVM CPUID features
+            CpuidFeatureEntry {
+                function: 0x4000_0001,
+                index: 0,
+                feature_reg: CpuidReg::EAX,
+                compatible_check: CpuidCompatibleCheck::BitwiseSubset,
+            },
+            CpuidFeatureEntry {
+                function: 0x4000_0001,
+                index: 0,
+                feature_reg: CpuidReg::EBX,
+                compatible_check: CpuidCompatibleCheck::BitwiseSubset,
+            },
+            CpuidFeatureEntry {
+                function: 0x4000_0001,
+                index: 0,
+                feature_reg: CpuidReg::ECX,
+                compatible_check: CpuidCompatibleCheck::BitwiseSubset,
+            },
+            CpuidFeatureEntry {
+                function: 0x4000_0001,
+                index: 0,
+                feature_reg: CpuidReg::EDX,
+                compatible_check: CpuidCompatibleCheck::BitwiseSubset,
+            },
+        ]
+    }
+
+    fn get_features_from_cpuid(
+        cpuid: &CpuId,
+        feature_entry_list: &[CpuidFeatureEntry],
+    ) -> Vec<u32> {
+        let mut features = vec![0; feature_entry_list.len()];
+        for (i, feature_entry) in feature_entry_list.iter().enumerate() {
+            for cpuid_entry in cpuid.as_slice().iter() {
+                if cpuid_entry.function == feature_entry.function
+                    && cpuid_entry.index == feature_entry.index
+                {
+                    match feature_entry.feature_reg {
+                        CpuidReg::EAX => {
+                            features[i] = cpuid_entry.eax;
+                        }
+                        CpuidReg::EBX => {
+                            features[i] = cpuid_entry.ebx;
+                        }
+                        CpuidReg::ECX => {
+                            features[i] = cpuid_entry.ecx;
+                        }
+                        CpuidReg::EDX => {
+                            features[i] = cpuid_entry.edx;
+                        }
+                    }
+
+                    break;
+                }
+            }
+        }
+
+        features
+    }
+
+    // The function returns `Error` (a.k.a. "incompatible"), when the CPUID features from `src_vm_cpuid`
+    // is not a subset of those of the `dest_vm_cpuid`.
+    pub fn check_cpuid_compatibility(
+        src_vm_cpuid: &CpuId,
+        dest_vm_cpuid: &CpuId,
+    ) -> Result<(), Error> {
+        let feature_entry_list = &Self::checked_feature_entry_list();
+        let src_vm_features = Self::get_features_from_cpuid(src_vm_cpuid, feature_entry_list);
+        let dest_vm_features = Self::get_features_from_cpuid(dest_vm_cpuid, feature_entry_list);
+
+        // Loop on feature bit and check if the 'source vm' feature is a subset
+        // of those of the 'destination vm' feature
+        let mut compatible = true;
+        for (i, (src_vm_feature, dest_vm_feature)) in src_vm_features
+            .iter()
+            .zip(dest_vm_features.iter())
+            .enumerate()
+        {
+            let entry = &feature_entry_list[i];
+            let entry_compatible;
+            match entry.compatible_check {
+                CpuidCompatibleCheck::BitwiseSubset => {
+                    let different_feature_bits = src_vm_feature ^ dest_vm_feature;
+                    let src_vm_feature_bits_only = different_feature_bits & src_vm_feature;
+                    entry_compatible = src_vm_feature_bits_only == 0;
+                }
+                CpuidCompatibleCheck::Equal => {
+                    entry_compatible = src_vm_feature == dest_vm_feature;
+                }
+                CpuidCompatibleCheck::NumNotGreater => {
+                    entry_compatible = src_vm_feature <= dest_vm_feature;
+                }
+            };
+            if !entry_compatible {
+                error!(
+                    "Detected incompatible CPUID entry: leaf={:#02x} (subleaf={:#02x}), register='{:?}', \
+                    compatilbe_check='{:?}', source VM feature='{:#04x}', destination VM feature'{:#04x}'.",
+                    entry.function, entry.index, entry.feature_reg,
+                    entry.compatible_check, src_vm_feature, dest_vm_feature
+                    );
+
+                compatible = false;
+            }
+        }
+
+        if compatible {
+            info!("No CPU incompatibility detected.");
+            Ok(())
+        } else {
+            Err(Error::CpuidCheckCompatibility)
+        }
+    }
+}
+
 pub fn generate_common_cpuid(
     hypervisor: Arc<dyn hypervisor::Hypervisor>,
     topology: Option<(u8, u8, u8)>,

block_util/Cargo.toml

@@ -9,15 +9,16 @@ default = []
 io_uring = []
 
 [dependencies]
-io-uring = ">=0.4.0"
-libc = "0.2.98"
+io-uring = "0.5.2"
+libc = "0.2.101"
 log = "0.4.14"
 qcow = { path = "../qcow" }
-thiserror = "1.0"
+thiserror = "1.0.29"
 versionize = "0.1.6"
 versionize_derive = "0.1.4"
-virtio-bindings = { version = "0.1", features = ["virtio-v5_0_0"]}
-vm-memory = { version = "0.5.0", features = ["backend-mmap", "backend-atomic", "backend-bitmap"] }
+vhdx = { path = "../vhdx" }
+virtio-bindings = { version = "0.1.0", features = ["virtio-v5_0_0"] }
+vm-memory = { version = "0.6.0", features = ["backend-mmap", "backend-atomic", "backend-bitmap"] }
 vm-virtio = { path = "../vm-virtio" }
-vmm-sys-util = ">=0.3.1"
+vmm-sys-util = "0.8.0"
 

block_util/src/lib.rs

@@ -18,6 +18,7 @@ pub mod qcow_sync;
 pub mod raw_async;
 pub mod raw_sync;
 pub mod vhd;
+pub mod vhdx_sync;
 
 use crate::async_io::{AsyncIo, AsyncIoError, AsyncIoResult, DiskFileError, DiskFileResult};
 #[cfg(feature = "io_uring")]
@@ -602,9 +603,11 @@ pub enum ImageType {
     FixedVhd,
     Qcow2,
     Raw,
+    Vhdx,
 }
 
 const QCOW_MAGIC: u32 = 0x5146_49fb;
+const VHDX_SIGN: u64 = 0x656C_6966_7864_6876;
 
 /// Determine image type through file parsing.
 pub fn detect_image_type(f: &mut File) -> std::io::Result<ImageType> {
@@ -623,6 +626,8 @@ pub fn detect_image_type(f: &mut File) -> std::io::Result<ImageType> {
         ImageType::Qcow2
     } else if vhd::is_fixed_vhd(f)? {
         ImageType::FixedVhd
+    } else if u64::from_le_bytes(s.data[0..8].try_into().unwrap()) == VHDX_SIGN {
+        ImageType::Vhdx
     } else {
         ImageType::Raw
     };

block_util/src/qcow_sync.rs

@@ -4,7 +4,7 @@
 
 use crate::async_io::{AsyncIo, AsyncIoResult, DiskFile, DiskFileResult};
 use crate::{disk_size, fsync_sync, read_vectored_sync, write_vectored_sync};
-use qcow::{QcowFile, RawFile};
+use qcow::{QcowFile, RawFile, Result as QcowResult};
 use std::fs::File;
 use std::sync::{Arc, Mutex};
 use vmm_sys_util::eventfd::EventFd;
@@ -15,12 +15,11 @@ pub struct QcowDiskSync {
 }
 
 impl QcowDiskSync {
-    pub fn new(file: File, direct_io: bool) -> Self {
-        QcowDiskSync {
-            qcow_file: QcowFile::from(RawFile::new(file, direct_io))
-                .expect("Failed creating QcowFile"),
+    pub fn new(file: File, direct_io: bool) -> QcowResult<Self> {
+        Ok(QcowDiskSync {
+            qcow_file: QcowFile::from(RawFile::new(file, direct_io))?,
             semaphore: Arc::new(Mutex::new(())),
-        }
+        })
     }
 }
 

block_util/src/vhdx_sync.rs

@@ -0,0 +1,113 @@
+// Copyright © 2021 Intel Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+
+use crate::async_io::{AsyncIo, AsyncIoResult, DiskFile, DiskFileError, DiskFileResult};
+use crate::{fsync_sync, read_vectored_sync, write_vectored_sync};
+use std::fs::File;
+use std::ops::DerefMut;
+use std::sync::{Arc, Mutex};
+use vhdx::vhdx::{Result as VhdxResult, Vhdx};
+use vmm_sys_util::eventfd::EventFd;
+
+pub struct VhdxDiskSync {
+    vhdx_file: Arc<Mutex<Vhdx>>,
+    semaphore: Arc<Mutex<()>>,
+}
+
+impl VhdxDiskSync {
+    pub fn new(f: File) -> VhdxResult<Self> {
+        let vhdx = Vhdx::new(f)?;
+        let vhdx_file = Arc::new(Mutex::new(vhdx));
+
+        Ok(VhdxDiskSync {
+            vhdx_file,
+            semaphore: Arc::new(Mutex::new(())),
+        })
+    }
+}
+
+impl DiskFile for VhdxDiskSync {
+    fn size(&mut self) -> DiskFileResult<u64> {
+        Ok(self.vhdx_file.lock().unwrap().virtual_disk_size())
+    }
+
+    fn new_async_io(&self, _ring_depth: u32) -> DiskFileResult<Box<dyn AsyncIo>> {
+        Ok(Box::new(
+            VhdxSync::new(self.vhdx_file.clone(), self.semaphore.clone())
+                .map_err(DiskFileError::NewAsyncIo)?,
+        ) as Box<dyn AsyncIo>)
+    }
+}
+
+pub struct VhdxSync {
+    vhdx_file: Arc<Mutex<Vhdx>>,
+    eventfd: EventFd,
+    completion_list: Vec<(u64, i32)>,
+    semaphore: Arc<Mutex<()>>,
+}
+
+impl VhdxSync {
+    pub fn new(vhdx_file: Arc<Mutex<Vhdx>>, semaphore: Arc<Mutex<()>>) -> std::io::Result<Self> {
+        Ok(VhdxSync {
+            vhdx_file,
+            eventfd: EventFd::new(libc::EFD_NONBLOCK)?,
+            completion_list: Vec::new(),
+            semaphore,
+        })
+    }
+}
+
+impl AsyncIo for VhdxSync {
+    fn notifier(&self) -> &EventFd {
+        &self.eventfd
+    }
+
+    fn read_vectored(
+        &mut self,
+        offset: libc::off_t,
+        iovecs: Vec<libc::iovec>,
+        user_data: u64,
+    ) -> AsyncIoResult<()> {
+        read_vectored_sync(
+            offset,
+            iovecs,
+            user_data,
+            self.vhdx_file.lock().unwrap().deref_mut(),
+            &self.eventfd,
+            &mut self.completion_list,
+            &mut self.semaphore,
+        )
+    }
+
+    fn write_vectored(
+        &mut self,
+        offset: libc::off_t,
+        iovecs: Vec<libc::iovec>,
+        user_data: u64,
+    ) -> AsyncIoResult<()> {
+        write_vectored_sync(
+            offset,
+            iovecs,
+            user_data,
+            self.vhdx_file.lock().unwrap().deref_mut(),
+            &self.eventfd,
+            &mut self.completion_list,
+            &mut self.semaphore,
+        )
+    }
+
+    fn fsync(&mut self, user_data: Option<u64>) -> AsyncIoResult<()> {
+        fsync_sync(
+            user_data,
+            self.vhdx_file.lock().unwrap().deref_mut(),
+            &self.eventfd,
+            &mut self.completion_list,
+            &mut self.semaphore,
+        )
+    }
+
+    fn complete(&mut self) -> Vec<(u64, i32)> {
+        self.completion_list.drain(..).collect()
+    }
+}

devices/Cargo.toml

@@ -6,19 +6,19 @@ edition = "2018"
 
 [dependencies]
 acpi_tables = { path = "../acpi_tables", optional = true }
-anyhow = "1.0"
+anyhow = "1.0.43"
 arch = { path = "../arch" }
-bitflags = ">=1.2.1"
+bitflags = "1.3.2"
 byteorder = "1.4.3"
-epoll = ">=4.0.1"
-libc = "0.2.98"
+epoll = "4.3.1"
+libc = "0.2.101"
 log = "0.4.14"
 versionize = "0.1.6"
 versionize_derive = "0.1.4"
 vm-device = { path = "../vm-device" }
-vm-memory = "0.5.0"
+vm-memory = "0.6.0"
 vm-migration = { path = "../vm-migration" }
-vmm-sys-util = ">=0.3.1"
+vmm-sys-util = "0.8.0"
 
 [features]
 default = []

devices/src/acpi.rs

@@ -41,7 +41,7 @@ impl BusDevice for AcpiShutdownDevice {
 
     fn write(&mut self, _base: u64, _offset: u64, data: &[u8]) -> Option<Arc<Barrier>> {
         if data[0] == 1 {
-            debug!("ACPI Reboot signalled");
+            info!("ACPI Reboot signalled");
             if let Err(e) = self.reset_evt.write(1) {
                 error!("Error triggering ACPI reset event: {}", e);
             }
@@ -51,7 +51,7 @@ impl BusDevice for AcpiShutdownDevice {
         const SLEEP_STATUS_EN_BIT: u8 = 5;
         const SLEEP_VALUE_BIT: u8 = 2;
         if data[0] == (S5_SLEEP_VALUE << SLEEP_VALUE_BIT) | (1 << SLEEP_STATUS_EN_BIT) {
-            debug!("ACPI Shutdown signalled");
+            info!("ACPI Shutdown signalled");
             if let Err(e) = self.exit_evt.write(1) {
                 error!("Error triggering ACPI shutdown event: {}", e);
             }
@@ -62,7 +62,7 @@ impl BusDevice for AcpiShutdownDevice {
 
 /// A device for handling ACPI GED event generation
 pub struct AcpiGedDevice {
-    interrupt: Arc<Box<dyn InterruptSourceGroup>>,
+    interrupt: Arc<dyn InterruptSourceGroup>,
     notification_type: AcpiNotificationFlags,
     ged_irq: u32,
     address: GuestAddress,
@@ -70,7 +70,7 @@ pub struct AcpiGedDevice {
 
 impl AcpiGedDevice {
     pub fn new(
-        interrupt: Arc<Box<dyn InterruptSourceGroup>>,
+        interrupt: Arc<dyn InterruptSourceGroup>,
         ged_irq: u32,
         address: GuestAddress,
     ) -> AcpiGedDevice {

devices/src/gic.rs

@@ -26,7 +26,7 @@ pub const IRQ_LEGACY_COUNT: usize = 32;
 //   1. Move Gic*.rs from arch/ folder here.
 //   2. Move this file and ioapic.rs to arch/, as they are architecture specific.
 pub struct Gic {
-    interrupt_source_group: Arc<Box<dyn InterruptSourceGroup>>,
+    interrupt_source_group: Arc<dyn InterruptSourceGroup>,
     gic_device: Option<Arc<Mutex<Box<dyn GicDevice>>>>,
 }
 

devices/src/ioapic.rs

@@ -134,7 +134,7 @@ pub struct Ioapic {
     reg_entries: [RedirectionTableEntry; NUM_IOAPIC_PINS],
     used_entries: [bool; NUM_IOAPIC_PINS],
     apic_address: GuestAddress,
-    interrupt_source_group: Arc<Box<dyn InterruptSourceGroup>>,
+    interrupt_source_group: Arc<dyn InterruptSourceGroup>,
 }
 
 #[derive(Versionize)]

devices/src/legacy/cmos.rs

@@ -2,12 +2,16 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
-use libc::{clock_gettime, gmtime_r, time_t, timespec, tm, CLOCK_REALTIME};
+use libc::{clock_gettime, gmtime_r, timespec, tm, CLOCK_REALTIME};
 use std::cmp::min;
 use std::mem;
 use std::sync::{Arc, Barrier};
 use vm_device::BusDevice;
 
+// https://github.com/rust-lang/libc/issues/1848
+#[cfg_attr(target_env = "musl", allow(deprecated))]
+use libc::time_t;
+
 const INDEX_MASK: u8 = 0x7f;
 const INDEX_OFFSET: u64 = 0x0;
 const DATA_OFFSET: u64 = 0x1;
@@ -85,6 +89,8 @@ impl BusDevice for Cmos {
                     let mut timespec: timespec = mem::zeroed();
                     clock_gettime(CLOCK_REALTIME, &mut timespec as *mut _);
 
+                    // https://github.com/rust-lang/libc/issues/1848
+                    #[cfg_attr(target_env = "musl", allow(deprecated))]
                     let now: time_t = timespec.tv_sec;
                     let mut tm: tm = mem::zeroed();
                     gmtime_r(&now, &mut tm as *mut _);

devices/src/legacy/gpio_pl061.rs

@@ -86,7 +86,7 @@ pub struct Gpio {
     // Mode Control Select Register
     afsel: u32,
     // GPIO irq_field
-    interrupt: Arc<Box<dyn InterruptSourceGroup>>,
+    interrupt: Arc<dyn InterruptSourceGroup>,
 }
 
 #[derive(Versionize)]
@@ -106,7 +106,7 @@ impl VersionMapped for GpioState {}
 
 impl Gpio {
     /// Constructs an PL061 GPIO device.
-    pub fn new(id: String, interrupt: Arc<Box<dyn InterruptSourceGroup>>) -> Self {
+    pub fn new(id: String, interrupt: Arc<dyn InterruptSourceGroup>) -> Self {
         Self {
             id,
             data: 0,
@@ -376,7 +376,7 @@ mod tests {
         let intr_evt = EventFd::new(libc::EFD_NONBLOCK).unwrap();
         let mut gpio = Gpio::new(
             String::from(GPIO_NAME),
-            Arc::new(Box::new(TestInterrupt::new(intr_evt.try_clone().unwrap()))),
+            Arc::new(TestInterrupt::new(intr_evt.try_clone().unwrap())),
         );
         let mut data = [0; 4];
 

devices/src/legacy/i8042.rs

@@ -34,7 +34,7 @@ impl BusDevice for I8042Device {
 
     fn write(&mut self, _base: u64, offset: u64, data: &[u8]) -> Option<Arc<Barrier>> {
         if data.len() == 1 && data[0] == 0xfe && offset == 3 {
-            debug!("i8042 reset signalled");
+            info!("i8042 reset signalled");
             if let Err(e) = self.reset_evt.write(1) {
                 error!("Error triggering i8042 reset event: {}", e);
             }

devices/src/legacy/rtc_pl031.rs

@@ -224,12 +224,12 @@ pub struct Rtc {
     load: u32,
     imsc: u32,
     ris: u32,
-    interrupt: Arc<Box<dyn InterruptSourceGroup>>,
+    interrupt: Arc<dyn InterruptSourceGroup>,
 }
 
 impl Rtc {
     /// Constructs an AMBA PL031 RTC device.
-    pub fn new(interrupt: Arc<Box<dyn InterruptSourceGroup>>) -> Self {
+    pub fn new(interrupt: Arc<dyn InterruptSourceGroup>) -> Self {
         Self {
             // This is used only for duration measuring purposes.
             previous_now: Instant::now(),
@@ -450,9 +450,7 @@ mod tests {
     fn test_rtc_read_write_and_event() {
         let intr_evt = EventFd::new(libc::EFD_NONBLOCK).unwrap();
 
-        let mut rtc = Rtc::new(Arc::new(Box::new(TestInterrupt::new(
-            intr_evt.try_clone().unwrap(),
-        ))));
+        let mut rtc = Rtc::new(Arc::new(TestInterrupt::new(intr_evt.try_clone().unwrap())));
         let mut data = [0; 4];
 
         // Read and write to the MR register.

devices/src/legacy/serial.rs

@@ -63,7 +63,7 @@ pub struct Serial {
     id: String,
     interrupt_enable: u8,
     interrupt_identification: u8,
-    interrupt: Arc<Box<dyn InterruptSourceGroup>>,
+    interrupt: Arc<dyn InterruptSourceGroup>,
     line_control: u8,
     line_status: u8,
     modem_control: u8,
@@ -91,7 +91,7 @@ impl VersionMapped for SerialState {}
 impl Serial {
     pub fn new(
         id: String,
-        interrupt: Arc<Box<dyn InterruptSourceGroup>>,
+        interrupt: Arc<dyn InterruptSourceGroup>,
         out: Option<Box<dyn io::Write + Send>>,
     ) -> Serial {
         Serial {
@@ -113,14 +113,14 @@ impl Serial {
     /// Constructs a Serial port ready for output.
     pub fn new_out(
         id: String,
-        interrupt: Arc<Box<dyn InterruptSourceGroup>>,
+        interrupt: Arc<dyn InterruptSourceGroup>,
         out: Box<dyn io::Write + Send>,
     ) -> Serial {
         Self::new(id, interrupt, Some(out))
     }
 
     /// Constructs a Serial port with no connected output.
-    pub fn new_sink(id: String, interrupt: Arc<Box<dyn InterruptSourceGroup>>) -> Serial {
+    pub fn new_sink(id: String, interrupt: Arc<dyn InterruptSourceGroup>) -> Serial {
         Self::new(id, interrupt, None)
     }
 
@@ -370,7 +370,7 @@ mod tests {
         let serial_out = SharedBuffer::new();
         let mut serial = Serial::new_out(
             String::from(SERIAL_NAME),
-            Arc::new(Box::new(TestInterrupt::new(intr_evt.try_clone().unwrap()))),
+            Arc::new(TestInterrupt::new(intr_evt.try_clone().unwrap())),
             Box::new(serial_out.clone()),
         );
 
@@ -390,7 +390,7 @@ mod tests {
         let serial_out = SharedBuffer::new();
         let mut serial = Serial::new_out(
             String::from(SERIAL_NAME),
-            Arc::new(Box::new(TestInterrupt::new(intr_evt.try_clone().unwrap()))),
+            Arc::new(TestInterrupt::new(intr_evt.try_clone().unwrap())),
             Box::new(serial_out),
         );
 
@@ -427,7 +427,7 @@ mod tests {
         let intr_evt = EventFd::new(0).unwrap();
         let mut serial = Serial::new_sink(
             String::from(SERIAL_NAME),
-            Arc::new(Box::new(TestInterrupt::new(intr_evt.try_clone().unwrap()))),
+            Arc::new(TestInterrupt::new(intr_evt.try_clone().unwrap())),
         );
 
         // write 1 to the interrupt event fd, so that read doesn't block in case the event fd
@@ -449,7 +449,7 @@ mod tests {
         let intr_evt = EventFd::new(0).unwrap();
         let mut serial = Serial::new_sink(
             String::from(SERIAL_NAME),
-            Arc::new(Box::new(TestInterrupt::new(intr_evt.try_clone().unwrap()))),
+            Arc::new(TestInterrupt::new(intr_evt.try_clone().unwrap())),
         );
 
         serial.write(0, LCR as u64, &[LCR_DLAB_BIT]);
@@ -470,7 +470,7 @@ mod tests {
         let intr_evt = EventFd::new(0).unwrap();
         let mut serial = Serial::new_sink(
             String::from(SERIAL_NAME),
-            Arc::new(Box::new(TestInterrupt::new(intr_evt.try_clone().unwrap()))),
+            Arc::new(TestInterrupt::new(intr_evt.try_clone().unwrap())),
         );
 
         serial.write(0, MCR as u64, &[MCR_LOOP_BIT]);
@@ -496,7 +496,7 @@ mod tests {
         let intr_evt = EventFd::new(0).unwrap();
         let mut serial = Serial::new_sink(
             String::from(SERIAL_NAME),
-            Arc::new(Box::new(TestInterrupt::new(intr_evt.try_clone().unwrap()))),
+            Arc::new(TestInterrupt::new(intr_evt.try_clone().unwrap())),
         );
 
         serial.write(0, SCR as u64, &[0x12]);

devices/src/legacy/uart_pl011.rs

@@ -86,7 +86,7 @@ pub struct Pl011 {
     ifl: u32,
     read_count: u32,
     read_trigger: u32,
-    irq: Arc<Box<dyn InterruptSourceGroup>>,
+    irq: Arc<dyn InterruptSourceGroup>,
     out: Option<Box<dyn io::Write + Send>>,
 }
 
@@ -114,7 +114,7 @@ impl Pl011 {
     /// Constructs an AMBA PL011 UART device.
     pub fn new(
         id: String,
-        irq: Arc<Box<dyn InterruptSourceGroup>>,
+        irq: Arc<dyn InterruptSourceGroup>,
         out: Option<Box<dyn io::Write + Send>>,
     ) -> Self {
         Self {
@@ -440,7 +440,7 @@ mod tests {
         let pl011_out = SharedBuffer::new();
         let mut pl011 = Pl011::new(
             String::from(SERIAL_NAME),
-            Arc::new(Box::new(TestInterrupt::new(intr_evt.try_clone().unwrap()))),
+            Arc::new(TestInterrupt::new(intr_evt.try_clone().unwrap())),
             Some(Box::new(pl011_out.clone())),
         );
 
@@ -460,7 +460,7 @@ mod tests {
         let pl011_out = SharedBuffer::new();
         let mut pl011 = Pl011::new(
             String::from(SERIAL_NAME),
-            Arc::new(Box::new(TestInterrupt::new(intr_evt.try_clone().unwrap()))),
+            Arc::new(TestInterrupt::new(intr_evt.try_clone().unwrap())),
             Some(Box::new(pl011_out)),
         );
 

docs/api.md

@@ -78,27 +78,29 @@ Shut the VMM down                   | `/vmm.shutdown` | N/A          | N/A
 
 #### Virtual Machine (VM) Actions
 
-Action                             | Endpoint            | Request Body              | Response Body            | Prerequisites
------------------------------------|---------------------|---------------------------|--------------------------|---------------------------
-Create the VM                      | `/vm.create`        | `/schemas/VmConfig`       | N/A                      | The VM is not created yet
-Delete the VM                      | `/vm.delete`        | N/A                       | N/A                      | N/A
-Boot the VM                        | `/vm.boot`          | N/A                       | N/A                      | The VM is created but not booted
-Shut the VM down                   | `/vm.shutdown`      | N/A                       | N/A                      | The VM is booted
-Reboot the VM                      | `/vm.reboot`        | N/A                       | N/A                      | The VM is booted
-Pause the VM                       | `/vm.pause`         | N/A                       | N/A                      | The VM is booted
-Resume the VM                      | `/vm.resume`        | N/A                       | N/A                      | The VM is paused
-Add/remove CPUs to/from the VM     | `/vm.resize`        | `/schemas/VmResize`       | N/A                      | The VM is booted
-Add/remove memory from the VM      | `/vm.resize`        | `/schemas/VmResize`       | N/A                      | The VM is booted
-Add/remove memory from a zone      | `/vm.resize-zone`   | `/schemas/VmResizeZone`   | N/A                      | The VM is booted
-Dump the VM information            | `/vm.info`          | N/A                       | `/schemas/VmInfo`        | The VM is created
-Add VFIO PCI device to the VM      | `/vm.add-device`    | `/schemas/VmAddDevice`    | `/schemas/PciDeviceInfo` | The VM is booted
-Add disk device to the VM          | `/vm.add-disk`      | `/schemas/DiskConfig`     | `/schemas/PciDeviceInfo` | The VM is booted
-Add fs device to the VM            | `/vm.add-fs`        | `/schemas/FsConfig`       | `/schemas/PciDeviceInfo` | The VM is booted
-Add pmem device to the VM          | `/vm.add-pmem`      | `/schemas/PmemConfig`     | `/schemas/PciDeviceInfo` | The VM is booted
-Add network device to the VM       | `/vm.add-net`       | `/schemas/NetConfig`      | `/schemas/PciDeviceInfo` | The VM is booted
-Add vsock device to the VM         | `/vm.add-vsock`     | `/schemas/VsockConfig`    | `/schemas/PciDeviceInfo` | The VM is booted
-Remove device from the VM          | `/vm.remove-device` | `/schemas/VmRemoveDevice` | N/A                      | The VM is booted
-Dump the VM counters               | `/vm.counters`      | N/A                       | `/schemas/VmCounters`    | The VM is booted
+Action                             | Endpoint             | Request Body              | Response Body            | Prerequisites
+-----------------------------------|----------------------|---------------------------|--------------------------|---------------------------
+Create the VM                      | `/vm.create`         | `/schemas/VmConfig`       | N/A                      | The VM is not created yet
+Delete the VM                      | `/vm.delete`         | N/A                       | N/A                      | N/A
+Boot the VM                        | `/vm.boot`           | N/A                       | N/A                      | The VM is created but not booted
+Shut the VM down                   | `/vm.shutdown`       | N/A                       | N/A                      | The VM is booted
+Reboot the VM                      | `/vm.reboot`         | N/A                       | N/A                      | The VM is booted
+Trigger power button of the VM     | `/vm.power-button`   | N/A                       | N/A                      | The VM is booted
+Pause the VM                       | `/vm.pause`          | N/A                       | N/A                      | The VM is booted
+Resume the VM                      | `/vm.resume`         | N/A                       | N/A                      | The VM is paused
+Add/remove CPUs to/from the VM     | `/vm.resize`         | `/schemas/VmResize`       | N/A                      | The VM is booted
+Add/remove memory from the VM      | `/vm.resize`         | `/schemas/VmResize`       | N/A                      | The VM is booted
+Add/remove memory from a zone      | `/vm.resize-zone`    | `/schemas/VmResizeZone`   | N/A                      | The VM is booted
+Dump the VM information            | `/vm.info`           | N/A                       | `/schemas/VmInfo`        | The VM is created
+Add VFIO PCI device to the VM      | `/vm.add-device`     | `/schemas/VmAddDevice`    | `/schemas/PciDeviceInfo` | The VM is booted
+Add disk device to the VM          | `/vm.add-disk`       | `/schemas/DiskConfig`     | `/schemas/PciDeviceInfo` | The VM is booted
+Add fs device to the VM            | `/vm.add-fs`         | `/schemas/FsConfig`       | `/schemas/PciDeviceInfo` | The VM is booted
+Add pmem device to the VM          | `/vm.add-pmem`       | `/schemas/PmemConfig`     | `/schemas/PciDeviceInfo` | The VM is booted
+Add network device to the VM       | `/vm.add-net`        | `/schemas/NetConfig`      | `/schemas/PciDeviceInfo` | The VM is booted
+Add userspace PCI device to the VM | `/vm.add-user-device`| `/schemas/VmAddUserDevice`| `/schemas/PciDeviceInfo` | The VM is booted
+Add vsock device to the VM         | `/vm.add-vsock`      | `/schemas/VsockConfig`    | `/schemas/PciDeviceInfo` | The VM is booted
+Remove device from the VM          | `/vm.remove-device`  | `/schemas/VmRemoveDevice` | N/A                      | The VM is booted
+Dump the VM counters               | `/vm.counters`       | N/A                       | `/schemas/VmCounters`    | The VM is booted
 
 ### REST API Examples
 

docs/fuzzing.md

@@ -20,10 +20,10 @@ cargo install cargo-fuzz
 
 ## Running the fuzzers
 
-e.g. To run the `qcow` fuzzer using all available CPUs:
+e.g. To run the `block` fuzzer using all available CPUs:
 
 ```
-cargo fuzz run qcow -j `nproc`
+cargo fuzz run block -j `nproc`
 ```
 
 ## Adding a new fuzzer
@@ -32,4 +32,4 @@ cargo fuzz run qcow -j `nproc`
 cargo fuzz add <new_fuzzer>
 ```
 
-Inspiration for fuzzers can be found in [crosvm](https://chromium.googlesource.com/chromiumos/platform/crosvm/+/refs/heads/master/fuzz/)
+Inspiration for fuzzers can be found in [crosvm](https://chromium.googlesource.com/chromiumos/platform/crosvm/+/refs/heads/master/fuzz/)

docs/live_migration.md

@@ -36,7 +36,7 @@ $ target/release/ch-remote --api-socket=/tmp/api1 send-migration unix:/tmp/sock
 
 When the above commands completed, the source VM should be successfully
 migrated to the destination VM. Now the destination VM is running while
-the source VM is paused.
+the source VM is terminated gracefully.
 
 ## Nested-VM Migration
 
@@ -136,4 +136,4 @@ vm-1:~$ sudo ./ch-remote --api-socket=/tmp/api1 send-migration unix:/tmp/sock1
 When the above commands completed, the source VM should be successfully
 migrated to the destination VM without interrupting our testing guest
 workload. Now the destination VM is running the testing guest workload
-while the source VM is paused.
+while the source VM is terminated gracefully.

docs/memory.md

@@ -24,7 +24,7 @@ struct MemoryConfig {
 ```
 
 ```
---memory <memory>	Memory parameters "size=<guest_memory_size>,mergeable=on|off,shared=on|off,hugepages=on|off,hotplug_method=acpi|virtio-mem,hotplug_size=<hotpluggable_memory_size>,hotplugged_size=<hotplugged_memory_size>"
+--memory <memory>	Memory parameters "size=<guest_memory_size>,mergeable=on|off,shared=on|off,hugepages=on|off,hugepage_size=<hugepage_size>,hotplug_method=acpi|virtio-mem,hotplug_size=<hotpluggable_memory_size>,hotplugged_size=<hotplugged_memory_size>"
 ```
 
 ### `size`
@@ -87,7 +87,9 @@ the guest will allocate hugepages as well. Another interesting use case is VFIO
 as it speeds up the VM's boot time since the amount of IOMMU mappings are
 reduced.
 
-The user is responsible for ensuring there are sufficient huge pages of the specified size for the VMM to use. Failure to do so may result in strange VMM behaviour.
+The user is responsible for ensuring there are sufficient huge pages of the specified size for the VMM to use.
+Failure to do so may result in strange VMM behaviour, e.g. error with `ReadKernelImage` is common.
+If there is a strange error with `hugepages` enabled, just disable it or check whether there are enough huge pages.
 
 By default this option is turned off.
 
@@ -439,6 +441,10 @@ Multiple values can be provided to define the list. Each value is a string
 referring to an existing memory zone identifier. Values are separated from
 each other with the `:` separator.
 
+Note that a memory zone must belong to a single NUMA node. The following
+configuration is incorrect, therefore not allowed:
+`--numa guest_numa_id=0,memory_zones=mem0 guest_numa_id=1,memory_zones=mem0`
+
 _Example_
 
 ```
@@ -469,4 +475,4 @@ _Example_
 Cloud Hypervisor supports only one PCI bus, which is why it has been tied to
 the NUMA node 0 by default. It is the user responsibility to organize the NUMA
 nodes correctly so that vCPUs and guest RAM which should be located on the same
-NUMA node as the PCI bus end up on the NUMA node 0.
+NUMA node as the PCI bus end up on the NUMA node 0.

docs/uefi.md

@@ -41,19 +41,20 @@ CSM is a module that allows to boot legacy operating systems using the OVMF firm
 
 ```shell
 sudo apt-get update
-sudo apt-get install uuid-dev nasm iasl build-essential python3-distutils git
+sudo apt-get install uuid-dev nasm iasl build-essential python3-distutils git libncurses-dev
 
-git checkout https://github.com/cloud-hypervisor/seabios -b ch
+git clone https://github.com/cloud-hypervisor/seabios -b ch
 cd seabios
 make menuconfig
 # Enable `CONFIG_CSM` and `CONFIG_QEMU_HARDWARE`
+# Please make sure `python` is in your path
 make CONFIG_CSM=y CONFIG_QEMU_HARDWARE=y
 cd ..
 
 git clone https://github.com/cloud-hypervisor/edk2 -b ch
 cd edk2
 . edksetup.sh
-git submodule update --init
+git submodule update --init --recursive
 cp ../seabios/out/Csm16.bin OvmfPkg/Csm/Csm16/
 
 echo "ACTIVE_PLATFORM=OvmfPkg/OvmfCh.dsc" >> Conf/target.txt
@@ -61,7 +62,7 @@ echo "TARGET_ARCH=X64" >> Conf/target.txt
 echo "TOOL_CHAIN_TAG=GCC5" >> Conf/target.txt
 
 make -C ./BaseTools
-build
+build -DCSM_ENABLE
 
 ```
 

docs/vfio-user.md

@@ -0,0 +1,77 @@
+# Cloud Hypervisor VFIO-user HOWTO
+
+VFIO-user is an *experimental* protocol for allowing devices to be implemented in another process and communicate over a socket; ie.e VFIO-user is to VFIO as virtio is to vhost-user.
+
+The protocol is documented here: https://github.com/nutanix/libvfio-user/blob/master/docs/vfio-user.rst
+
+The Cloud Hypervisor support for such devices is *experimental*. Not all Cloud Hypervisor functionality is supported in particular: virtio-mem and iommu are not supported.
+
+## Usage
+
+The `--user-device socket=<path>` parameter is used to create a vfio-user device when creating the VM specifying the socket to connect to. The device can also be hotplugged with `ch-remote add-user-device socket=<path>`.
+
+## Example (GPIO device)
+
+There is a simple GPIO device included in the libvfio-user repository: https://github.com/nutanix/libvfio-user#gpio
+
+Run the example from the libvfio-user repository:
+
+```sh
+rm /tmp/vfio-user.sock
+./build/dbg/samples/gpio-pci-idio-16 -v /tmp/vfio-user.sock &
+```
+
+Start Cloud Hypervisor:
+
+```sh
+target/debug/cloud-hypervisor \
+    --memory size=1G,shared=on \
+    --disk path=~/images/focal-server-cloudimg-amd64.raw \
+    --kernel ~/src/linux/vmlinux \
+    --cmdline "root=/dev/vda1 console=hvc0" \
+    --user-device socket=/tmp/vfio-user.sock 
+```
+
+Inside the VM you can test the device with:
+
+```sh
+cat /sys/class/gpio/gpiochip480/base > /sys/class/gpio/export
+for ((i=0;i<12;i++)); do cat /sys/class/gpio/OUT0/value; done
+```
+
+## Example (NVMe device)
+
+Use SPDK: https://github.com/spdk/spdk
+
+Compile with `./configure --with-vfio-user`
+
+Create an NVMe controller listening on a vfio-user socket with a simple block device:
+
+```sh
+sudo scripts/setup.sh
+rm ~/images/test-disk.raw
+truncate ~/images/test-disk.raw -s 128M
+mkfs.ext4  ~/images/test-disk.raw
+sudo killall ./build/bin/nvmf_tgt
+sudo ./build/bin/nvmf_tgt -i 0 -e 0xFFFF -m 0x1 &
+sleep 2
+sudo ./scripts/rpc.py nvmf_create_transport -t VFIOUSER
+sudo rm -rf /tmp/nvme-vfio-user
+sudo mkdir -p /tmp/nvme-vfio-user
+sudo ./scripts/rpc.py bdev_aio_create ~/images/test-disk.raw test 512
+sudo ./scripts/rpc.py nvmf_create_subsystem nqn.2019-07.io.spdk:cnode -a -s test
+sudo ./scripts/rpc.py nvmf_subsystem_add_ns nqn.2019-07.io.spdk:cnode test
+sudo ./scripts/rpc.py nvmf_subsystem_add_listener nqn.2019-07.io.spdk:cnode -t VFIOUSER -a /tmp/nvme-vfio-user -s 0
+sudo chown $USER.$USER -R /tmp/nvme-vfio-user
+```
+
+Start Cloud Hypervisor:
+
+```sh
+target/debug/cloud-hypervisor \
+    --memory size=1G,shared=on \
+    --disk path=~/images/focal-server-cloudimg-amd64.raw \
+    --kernel ~/src/linux/vmlinux \
+    --cmdline "root=/dev/vda1 console=hvc0" \
+    --user-device socket=/tmp/nvme-vfio-user/cntrl 
+```

docs/windows.md

@@ -30,15 +30,15 @@ This step currently requires QEMU to install Windows onto the guest. QEMU is onl
 Preparing several command parts as these will be used in the follow up sections as well.
 
 ```shell
-IMG_FILE=windows-disk.qcow
+IMG_FILE=windows-disk.raw
 WIN_ISO_FILE=en_windows_server_version_2004_updated_may_2020_x64_dvd_1e7f1cfa.iso
 VIRTIO_ISO_FILE=virtio-win-0.1.185.iso
 OVMF_DIR=./FV
 ```
 
-Create an empty image file, `qcow` or `raw` is supported.
+Create an empty image file, `raw` is supported.
 ```shell
-qemu-img create -f qcow2 $IMG_FILE 30G
+qemu-img create -f raw $IMG_FILE 30G
 ```
 
 Begin the Windows installation process under QEMU
@@ -221,7 +221,7 @@ qemu-system-x86_64 \
 	-m 4G \
 	-cdrom ./$WIN_ISO_FILE \
 	-drive file=./$VIRTIO_ISO_FILE,index=0,media=cdrom
-	-drive if=none,id=root,file=./windbg-disk.qcow \
+	-drive if=none,id=root,file=./windbg-disk.raw \
 	-device virtio-blk-pci,drive=root,disable-legacy=on \
 	-device virtio-net-pci,netdev=mynet0,disable-legacy=on \
 	-netdev user,id=mynet0,net=192.168.178.0/24,host=192.168.178.1,dhcpstart=192.168.178.64,hostname=windbg-host \
@@ -280,7 +280,7 @@ qemu-system-x86_64 \
 	-cpu host \
 	-smp 1 \
 	-m 4G \
-	-drive if=none,id=root,file=./windbg-disk.qcow \
+	-drive if=none,id=root,file=./windbg-disk.raw \
 	-device virtio-blk-pci,drive=root,disable-legacy=on \
 	-serial tcp::4445,server,nowait \
 	-device virtio-net-pci,netdev=mynet0,disable-legacy=on \

event_monitor/Cargo.toml

@@ -5,7 +5,7 @@ authors = ["The Cloud Hypervisor Authors"]
 edition = "2018"
 
 [dependencies]
-libc = "0.2.98"
-serde = {version = ">=1.0.27", features = ["rc"] }
-serde_derive = ">=1.0.27"
-serde_json = ">=1.0.9"
+libc = "0.2.101"
+serde = { version = "1.0.130", features = ["rc"] }
+serde_derive = "1.0.130"
+serde_json = "1.0.67"

fuzz/Cargo.lock

@@ -20,9 +20,9 @@ dependencies = [
 
 [[package]]
 name = "anyhow"
-version = "1.0.42"
+version = "1.0.43"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "595d3cfa7a60d4555cb5067b99f07142a08ea778de5cf993f7b75c7d8fabc486"
+checksum = "28ae2b3dec75a406790005a200b1bd89785afc02517a00ca99ecfe093ee9e6cf"
 
 [[package]]
 name = "api_client"
@@ -33,15 +33,15 @@ dependencies = [
 
 [[package]]
 name = "arbitrary"
-version = "1.0.1"
+version = "1.0.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "237430fd6ed3740afe94eefcc278ae21e050285be882804e0d6e8695f0c94691"
+checksum = "577b08a4acd7b99869f863c50011b01eb73424ccc798ecd996f2e24817adfca7"
 
 [[package]]
 name = "arc-swap"
-version = "1.3.0"
+version = "1.3.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e906254e445520903e7fc9da4f709886c84ae4bc4ddaf0e093188d66df4dc820"
+checksum = "b5ab7d9e73059c86c36473f459b52adbd99c3554a4fec492caef460806006f00"
 
 [[package]]
 name = "arch"
@@ -87,9 +87,9 @@ dependencies = [
 
 [[package]]
 name = "bitflags"
-version = "1.2.1"
+version = "1.3.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693"
+checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
 
 [[package]]
 name = "block_util"
@@ -102,6 +102,7 @@ dependencies = [
  "thiserror",
  "versionize",
  "versionize_derive",
+ "vhdx",
  "virtio-bindings",
  "vm-memory",
  "vm-virtio",
@@ -116,9 +117,9 @@ checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610"
 
 [[package]]
 name = "cc"
-version = "1.0.69"
+version = "1.0.70"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e70cc2f62c6ce1868963827bd677764c62d07c3d9a3e1fb1177ee1a9ab199eb2"
+checksum = "d26a6ce4b6a484fa3edb70f7efa6fc430fd2b87285fe8b84304fd0936faa0dc0"
 
 [[package]]
 name = "cfg-if"
@@ -144,7 +145,7 @@ dependencies = [
 
 [[package]]
 name = "cloud-hypervisor"
-version = "16.0.0"
+version = "17.0.0"
 dependencies = [
  "anyhow",
  "api_client",
@@ -155,7 +156,7 @@ dependencies = [
  "libc",
  "log",
  "option_parser",
- "seccomp",
+ "seccompiler",
  "serde_json",
  "signal-hook",
  "thiserror",
@@ -173,13 +174,23 @@ dependencies = [
  "libc",
  "libfuzzer-sys",
  "qcow",
- "seccomp",
+ "seccompiler",
+ "vhdx",
  "virtio-devices",
  "vm-memory",
  "vm-virtio",
  "vmm-sys-util",
 ]
 
+[[package]]
+name = "crc32c"
+version = "0.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "210cdf933e6a81212bfabf90cd8762f471b5922e5f6b709547673ad8e04b9448"
+dependencies = [
+ "rustc_version",
+]
+
 [[package]]
 name = "crc64"
 version = "1.0.0"
@@ -232,6 +243,17 @@ version = "0.1.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b643857cf70949306b81d7e92cb9d47add673868edac9863c4a49c42feaf3f1e"
 
+[[package]]
+name = "getrandom"
+version = "0.2.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7fcd999463524c52659517fe2cea98493cfe485d10565e7b0fb07dbba7ad2753"
+dependencies = [
+ "cfg-if",
+ "libc",
+ "wasi",
+]
+
 [[package]]
 name = "hermit-abi"
 version = "0.1.19"
@@ -262,9 +284,9 @@ dependencies = [
 
 [[package]]
 name = "iced-x86"
-version = "1.13.0"
+version = "1.14.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "94ef7eabb0e712d4f12aea976b2deb876739dc70fd787df3876c75c9b7bcf0be"
+checksum = "7383772b06135cede839b7270023b46403656a9148024886e721e82639d3f90e"
 dependencies = [
  "lazy_static",
  "static_assertions",
@@ -272,9 +294,9 @@ dependencies = [
 
 [[package]]
 name = "io-uring"
-version = "0.5.1"
+version = "0.5.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fb82832e05cc4ca298f198a8db108837b4f7b7b1248e3cba8e48f151aece80cf"
+checksum = "8d75829ed9377bab6c90039fe47b9d84caceb4b5063266142e21bcce6550cda8"
 dependencies = [
  "bitflags",
  "libc",
@@ -282,9 +304,9 @@ dependencies = [
 
 [[package]]
 name = "itoa"
-version = "0.4.7"
+version = "0.4.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dd25036021b0de88a0aff6b850051563c6516d0bf53f8638938edbb9de732736"
+checksum = "b71991ff56294aa922b450139ee08b3bfc70982c6b2c7562771375cf73542dd4"
 
 [[package]]
 name = "kvm-bindings"
@@ -315,9 +337,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
 
 [[package]]
 name = "libc"
-version = "0.2.98"
+version = "0.2.101"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "320cfe77175da3a483efed4bc0adc1968ca050b098ce4f2f1c13a56626128790"
+checksum = "3cb00336871be5ed2c8ed44b60ae9959dc5b9f08539422ed43f09e34ecaeba21"
 
 [[package]]
 name = "libfuzzer-sys"
@@ -351,7 +373,7 @@ dependencies = [
 [[package]]
 name = "micro_http"
 version = "0.1.0"
-source = "git+https://github.com/firecracker-microvm/micro-http?branch=main#9b605a8b61df602cda62076d30d9f70307ea336f"
+source = "git+https://github.com/firecracker-microvm/micro-http?branch=main#9517a300370a158a7af0996b7eebf040d171e1a4"
 dependencies = [
  "libc",
  "vmm-sys-util",
@@ -401,10 +423,12 @@ dependencies = [
  "hypervisor",
  "libc",
  "log",
+ "thiserror",
  "versionize",
  "versionize_derive",
  "vfio-bindings",
  "vfio-ioctls",
+ "vfio_user",
  "vm-allocator",
  "vm-device",
  "vm-memory",
@@ -414,9 +438,9 @@ dependencies = [
 
 [[package]]
 name = "proc-macro2"
-version = "1.0.27"
+version = "1.0.29"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f0d8caf72986c1a598726adc988bb5984792ef84f5ee5aa50209145ee8077038"
+checksum = "b9f5105d4fdaab20335ca9565e106a5d9b82b6219b5ba735731124ac6711d23d"
 dependencies = [
  "unicode-xid",
 ]
@@ -461,6 +485,15 @@ dependencies = [
  "syn",
 ]
 
+[[package]]
+name = "rustc_version"
+version = "0.2.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a"
+dependencies = [
+ "semver",
+]
+
 [[package]]
 name = "ryu"
 version = "1.0.5"
@@ -468,24 +501,39 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e"
 
 [[package]]
-name = "seccomp"
+name = "seccompiler"
 version = "0.1.0"
-source = "git+https://github.com/firecracker-microvm/firecracker?tag=v0.24.4#8f44986a0e956a77f2b2324c12f73bec16130c82"
+source = "git+https://github.com/rust-vmm/seccompiler#da5788d52f1ae8886d8ed4624199b7e9fa64ac04"
 dependencies = [
  "libc",
 ]
 
 [[package]]
-name = "serde"
-version = "1.0.126"
+name = "semver"
+version = "0.9.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ec7505abeacaec74ae4778d9d9328fe5a5d04253220a85c4ee022239fc996d03"
+checksum = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403"
+dependencies = [
+ "semver-parser",
+]
+
+[[package]]
+name = "semver-parser"
+version = "0.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3"
+
+[[package]]
+name = "serde"
+version = "1.0.130"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f12d06de37cf59146fbdecab66aa99f9fe4f78722e3607577a5375d66bd0c913"
 
 [[package]]
 name = "serde_derive"
-version = "1.0.126"
+version = "1.0.130"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "963a7dbc9895aeac7ac90e74f34a5d5261828f79df35cbed41e10189d3804d43"
+checksum = "d7bc1a1ab1961464eae040d96713baa5a724a8152c1222492465b54322ec508b"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -494,9 +542,9 @@ dependencies = [
 
 [[package]]
 name = "serde_json"
-version = "1.0.64"
+version = "1.0.67"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "799e97dc9fdae36a5c8b8f2cae9ce2ee9fdce2058c57a93e6099d919fd982f79"
+checksum = "a7f9e390c27c3c0ce8bc5d725f6e4d30a29d26659494aa4b17535f7522c5c950"
 dependencies = [
  "itoa",
  "ryu",
@@ -505,9 +553,9 @@ dependencies = [
 
 [[package]]
 name = "signal-hook"
-version = "0.3.9"
+version = "0.3.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "470c5a6397076fae0094aaf06a08e6ba6f37acb77d3b1b91ea92b4d6c8650c39"
+checksum = "9c98891d737e271a2954825ef19e46bd16bdb98e2746f2eec4f7a4ef7946efd1"
 dependencies = [
  "libc",
  "signal-hook-registry",
@@ -536,9 +584,9 @@ checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a"
 
 [[package]]
 name = "syn"
-version = "1.0.74"
+version = "1.0.76"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1873d832550d4588c3dbc20f01361ab00bfe741048f71e3fecf145a7cc18b29c"
+checksum = "c6f107db402c2c2055242dbf4d2af0e69197202e9faacbef9571bbe47f5a1b84"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -567,18 +615,18 @@ dependencies = [
 
 [[package]]
 name = "thiserror"
-version = "1.0.26"
+version = "1.0.29"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "93119e4feac1cbe6c798c34d3a53ea0026b0b1de6a120deef895137c0529bfe2"
+checksum = "602eca064b2d83369e2b2f34b09c70b605402801927c65c11071ac911d299b88"
 dependencies = [
  "thiserror-impl",
 ]
 
 [[package]]
 name = "thiserror-impl"
-version = "1.0.26"
+version = "1.0.29"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "060d69a0afe7796bf42e9e2ff91f5ee691fb15c53d38b4b62a9a53eb23164745"
+checksum = "bad553cc2c78e8de258400763a647e80e6d1b31ee237275d756f6836d204494c"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -602,6 +650,9 @@ name = "uuid"
 version = "0.8.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "bc5cf98d8186244414c848017f0e2676b3fcb46807f6668a97dfe67359a3c4b7"
+dependencies = [
+ "getrandom",
+]
 
 [[package]]
 name = "vec_map"
@@ -648,7 +699,7 @@ dependencies = [
 [[package]]
 name = "vfio-ioctls"
 version = "0.1.0"
-source = "git+https://github.com/rust-vmm/vfio-ioctls?branch=master#9b84069e9f419c5369b9a313859cac7e9828d331"
+source = "git+https://github.com/rust-vmm/vfio-ioctls?branch=main#a8ee64b978ccd000df88732814d626651a091891"
 dependencies = [
  "byteorder",
  "kvm-bindings",
@@ -659,13 +710,45 @@ dependencies = [
  "vmm-sys-util",
 ]
 
+[[package]]
+name = "vfio_user"
+version = "0.1.0"
+dependencies = [
+ "anyhow",
+ "libc",
+ "log",
+ "serde",
+ "serde_derive",
+ "serde_json",
+ "thiserror",
+ "vfio-bindings",
+ "vm-memory",
+ "vmm-sys-util",
+]
+
+[[package]]
+name = "vhdx"
+version = "0.1.0"
+dependencies = [
+ "byteorder",
+ "crc32c",
+ "libc",
+ "log",
+ "remain",
+ "thiserror",
+ "uuid",
+ "vmm-sys-util",
+]
+
 [[package]]
 name = "vhost"
 version = "0.1.0"
-source = "git+https://github.com/rust-vmm/vhost?branch=master#9982541776a603d30556a06df55e8c0491072763"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8a6b90237e10f1a61b35fba73885c3567e1a5a8c40d44daae335f7710210a7dc"
 dependencies = [
  "bitflags",
  "libc",
+ "vm-memory",
  "vmm-sys-util",
 ]
 
@@ -692,7 +775,7 @@ dependencies = [
  "net_util",
  "pci",
  "rate_limiter",
- "seccomp",
+ "seccompiler",
  "serde",
  "serde_derive",
  "serde_json",
@@ -734,12 +817,13 @@ dependencies = [
 [[package]]
 name = "vm-fdt"
 version = "0.1.0"
-source = "git+https://github.com/rust-vmm/vm-fdt?branch=master#13ab882e2f8387753064351c69fbe35c422b448b"
+source = "git+https://github.com/rust-vmm/vm-fdt?branch=main#720e48e435b791ec6cbe8d2b229448b71dcb1ab9"
 
 [[package]]
 name = "vm-memory"
-version = "0.5.0"
-source = "git+https://github.com/rust-vmm/vm-memory?rev=5bd7138758183a73ac0da27ce40c004d95f1a7e9#5bd7138758183a73ac0da27ce40c004d95f1a7e9"
+version = "0.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0a8ebcb86ca457f9d6e14cf97009f679952eba42f0113de5db596e514cd0e43b"
 dependencies = [
  "arc-swap",
  "libc",
@@ -793,7 +877,7 @@ dependencies = [
  "option_parser",
  "pci",
  "qcow",
- "seccomp",
+ "seccompiler",
  "serde",
  "serde_derive",
  "serde_json",
@@ -803,6 +887,7 @@ dependencies = [
  "versionize",
  "versionize_derive",
  "vfio-ioctls",
+ "vhdx",
  "virtio-devices",
  "vm-allocator",
  "vm-device",
@@ -824,6 +909,12 @@ dependencies = [
  "serde_derive",
 ]
 
+[[package]]
+name = "wasi"
+version = "0.10.2+wasi-snapshot-preview1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6"
+
 [[package]]
 name = "winapi"
 version = "0.3.9"
@@ -845,3 +936,8 @@ name = "winapi-x86_64-pc-windows-gnu"
 version = "0.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
+
+[[patch.unused]]
+name = "vm-memory"
+version = "0.5.0"
+source = "git+https://github.com/rust-vmm/vm-memory?rev=5bd7138758183a73ac0da27ce40c004d95f1a7e9#5bd7138758183a73ac0da27ce40c004d95f1a7e9"

fuzz/Cargo.toml

@@ -10,14 +10,15 @@ cargo-fuzz = true
 
 [dependencies]
 block_util = { path = "../block_util" }
-libc = "0.2.98"
+libc = "0.2.101"
 libfuzzer-sys = "0.4.2"
 qcow = { path = "../qcow" }
-seccomp = { git = "https://github.com/firecracker-microvm/firecracker", tag = "v0.24.4" }
+seccompiler = { git = "https://github.com/rust-vmm/seccompiler"}
+vhdx = { path = "../vhdx" }
 virtio-devices = { path = "../virtio-devices" }
 vmm-sys-util = "0.8.0"
 vm-virtio = { path = "../vm-virtio" }
-vm-memory = "0.5.0"
+vm-memory = "0.6.0"
 
 [dependencies.cloud-hypervisor]
 path = ".."
@@ -42,3 +43,9 @@ name = "block"
 path = "fuzz_targets/block.rs"
 test = false
 doc = false
+
+[[bin]]
+name = "vhdx"
+path = "fuzz_targets/vhdx.rs"
+test = false
+doc = false

fuzz/fuzz_targets/block.rs

@@ -4,9 +4,9 @@
 
 #![no_main]
 
-use block_util::{async_io::DiskFile, qcow_sync::QcowDiskSync};
+use block_util::{async_io::DiskFile, raw_sync::RawFileDiskSync};
 use libfuzzer_sys::fuzz_target;
-use seccomp::SeccompAction;
+use seccompiler::SeccompAction;
 use std::ffi;
 use std::fs::File;
 use std::io::{self, Cursor, Read, Seek, SeekFrom};
@@ -17,7 +17,7 @@ use std::sync::Arc;
 use virtio_devices::{Block, VirtioDevice, VirtioInterrupt, VirtioInterruptType};
 use vm_memory::{Bytes, GuestAddress, GuestMemoryAtomic, GuestMemoryMmap};
 use vm_virtio::Queue;
-use vmm_sys_util::eventfd::EventFd;
+use vmm_sys_util::eventfd::{EventFd, EFD_NONBLOCK};
 
 const MEM_SIZE: u64 = 256 * 1024 * 1024;
 const DESC_SIZE: u64 = 16; // Bytes in one virtio descriptor.
@@ -84,7 +84,7 @@ fuzz_target!(|bytes| {
 
     let shm = memfd_create(&ffi::CString::new("fuzz").unwrap(), 0).unwrap();
     let disk_file: File = unsafe { File::from_raw_fd(shm) };
-    let qcow_disk = Box::new(QcowDiskSync::new(disk_file, false)) as Box<dyn DiskFile>;
+    let qcow_disk = Box::new(RawFileDiskSync::new(disk_file)) as Box<dyn DiskFile>;
 
     let mut block = Block::new(
         "tmp".to_owned(),
@@ -96,6 +96,7 @@ fuzz_target!(|bytes| {
         256,
         SeccompAction::Allow,
         None,
+        EventFd::new(EFD_NONBLOCK).unwrap(),
     )
     .unwrap();
 

fuzz/fuzz_targets/vhdx.rs

@@ -0,0 +1,53 @@
+// Copyright © 2021 Intel Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+
+#![no_main]
+use libfuzzer_sys::fuzz_target;
+use std::ffi;
+use std::fs::File;
+use std::io::{self, Read, Seek, SeekFrom, Write};
+use std::os::unix::io::{FromRawFd, RawFd};
+use vhdx::vhdx::Vhdx;
+
+// Populate the corpus directory with a test file:
+// truncate -s 16M /tmp/source
+// qemu-img convert -O vhdx /tmp/source fuzz/corpus/vhdx/test.vhdx
+// Run with:
+// cargo fuzz run vhdx -j 32 -- -max_len=16777216
+fuzz_target!(|bytes| {
+    let shm = memfd_create(&ffi::CString::new("fuzz").unwrap(), 0).unwrap();
+    let mut disk_file: File = unsafe { File::from_raw_fd(shm) };
+    disk_file.write_all(&bytes[..]).unwrap();
+    disk_file.seek(SeekFrom::Start(0)).unwrap();
+
+    if let Ok(mut vhdx) = Vhdx::new(disk_file) {
+        if vhdx.seek(SeekFrom::Start(0)).is_ok() {
+            let mut offset = 0;
+            while offset < bytes.len() {
+                let mut data = vec![0; 8192];
+                vhdx.read_exact(&mut data).ok();
+                offset += data.len();
+            }
+        }
+
+        if vhdx.seek(SeekFrom::Start(0)).is_ok() {
+            let mut offset = 0;
+            while offset < bytes.len() {
+                let data = vec![0; 8192];
+                vhdx.write_all(&data).ok();
+                offset += data.len();
+            }
+        }
+    }
+});
+
+fn memfd_create(name: &ffi::CStr, flags: u32) -> Result<RawFd, io::Error> {
+    let res = unsafe { libc::syscall(libc::SYS_memfd_create, name.as_ptr(), flags) };
+
+    if res < 0 {
+        Err(io::Error::last_os_error())
+    } else {
+        Ok(res as RawFd)
+    }
+}

hypervisor/Cargo.toml

@@ -11,23 +11,23 @@ mshv = ["mshv-ioctls", "mshv-bindings"]
 tdx = []
 
 [dependencies]
-anyhow = "1.0"
-epoll = ">=4.0.1"
-thiserror = "1.0"
-libc = "0.2.98"
+anyhow = "1.0.43"
+epoll = "4.3.1"
+thiserror = "1.0.29"
+libc = "0.2.101"
 log = "0.4.14"
-kvm-ioctls = {  version = "0.9.0", optional = true }
+kvm-ioctls = { version = "0.9.0", optional = true }
 kvm-bindings = { git = "https://github.com/cloud-hypervisor/kvm-bindings", branch = "ch-v0.4.0", features = ["with-serde", "fam-wrappers"], optional  = true }
-mshv-bindings = {git = "https://github.com/cloud-hypervisor/mshv", branch = "master", features = ["with-serde", "fam-wrappers"], optional  = true }
-mshv-ioctls = { git = "https://github.com/cloud-hypervisor/mshv", branch = "master", optional  = true}
-serde = {version = ">=1.0.27", features = ["rc"] }
-serde_derive = ">=1.0.27"
-serde_json = ">=1.0.9"
-vm-memory = { version = "0.5.0", features = ["backend-mmap", "backend-atomic"] }
-vmm-sys-util = { version = ">=0.5.0", features = ["with-serde"] }
+mshv-bindings = {git = "https://github.com/rust-vmm/mshv", branch = "main", features = ["with-serde", "fam-wrappers"], optional  = true }
+mshv-ioctls = { git = "https://github.com/rust-vmm/mshv", branch = "main", optional  = true}
+serde = { version = "1.0.130", features = ["rc"] }
+serde_derive = "1.0.130"
+serde_json = "1.0.67"
+vm-memory = { version = "0.6.0", features = ["backend-mmap", "backend-atomic"] }
+vmm-sys-util = { version = "0.8.0", features = ["with-serde"] }
 
 [target.'cfg(target_arch = "x86_64")'.dependencies.iced-x86]
-version = "1.13"
+version = "1.14.0"
 default-features = false
 features = ["std", "decoder", "op_code_info", "instr_info", "fast_fmt"]
 

hypervisor/src/kvm/mod.rs

@@ -22,6 +22,7 @@ use crate::vm::{self, VmmOps};
 use crate::{arm64_core_reg_id, offset__of};
 use kvm_ioctls::{NoDatamatch, VcpuFd, VmFd};
 use serde_derive::{Deserialize, Serialize};
+use std::collections::HashMap;
 #[cfg(target_arch = "aarch64")]
 use std::convert::TryInto;
 #[cfg(target_arch = "x86_64")]
@@ -30,7 +31,7 @@ use std::os::unix::io::{AsRawFd, RawFd};
 use std::result;
 #[cfg(target_arch = "x86_64")]
 use std::sync::atomic::{AtomicBool, Ordering};
-use std::sync::Arc;
+use std::sync::{Arc, RwLock};
 #[cfg(target_arch = "x86_64")]
 use vm_memory::Address;
 use vmm_sys_util::eventfd::EventFd;
@@ -110,12 +111,21 @@ enum TdxCommand {
 pub struct KvmVmState {}
 
 pub use KvmVmState as VmState;
+
+struct KvmDirtyLogSlot {
+    slot: u32,
+    guest_phys_addr: u64,
+    memory_size: u64,
+    userspace_addr: u64,
+}
+
 /// Wrapper over KVM VM ioctls.
 pub struct KvmVm {
     fd: Arc<VmFd>,
     #[cfg(target_arch = "x86_64")]
     msrs: MsrEntries,
     state: KvmVmState,
+    dirty_log_slots: Arc<RwLock<HashMap<u32, KvmDirtyLogSlot>>>,
 }
 
 ///
@@ -168,7 +178,7 @@ impl vm::Vm for KvmVm {
     fn create_vcpu(
         &self,
         id: u8,
-        vmmops: Option<Arc<Box<dyn VmmOps>>>,
+        vmmops: Option<Arc<dyn VmmOps>>,
     ) -> vm::Result<Arc<dyn cpu::Vcpu>> {
         let vc = self
             .fd
@@ -267,10 +277,35 @@ impl vm::Vm for KvmVm {
     /// Creates a guest physical memory region.
     ///
     fn create_user_memory_region(&self, user_memory_region: MemoryRegion) -> vm::Result<()> {
+        let mut region = user_memory_region;
+
+        if (region.flags & KVM_MEM_LOG_DIRTY_PAGES) != 0 {
+            if (region.flags & KVM_MEM_READONLY) != 0 {
+                return Err(vm::HypervisorVmError::CreateUserMemory(anyhow!(
+                    "Error creating regions with both 'dirty-pages-log' and 'read-only'."
+                )));
+            }
+
+            // Keep track of the regions that need dirty pages log
+            self.dirty_log_slots.write().unwrap().insert(
+                region.slot,
+                KvmDirtyLogSlot {
+                    slot: region.slot,
+                    guest_phys_addr: region.guest_phys_addr,
+                    memory_size: region.memory_size,
+                    userspace_addr: region.userspace_addr,
+                },
+            );
+
+            // Always create guest physical memory region without `KVM_MEM_LOG_DIRTY_PAGES`.
+            // For regions that need this flag, dirty pages log will be turned on in `start_dirty_log`.
+            region.flags = 0;
+        }
+
         // Safe because guest regions are guaranteed not to overlap.
         unsafe {
             self.fd
-                .set_user_memory_region(user_memory_region)
+                .set_user_memory_region(region)
                 .map_err(|e| vm::HypervisorVmError::CreateUserMemory(e.into()))
         }
     }
@@ -280,6 +315,9 @@ impl vm::Vm for KvmVm {
     fn remove_user_memory_region(&self, user_memory_region: MemoryRegion) -> vm::Result<()> {
         let mut region = user_memory_region;
 
+        // Remove the corresponding entry from "self.dirty_log_slots" if needed
+        self.dirty_log_slots.write().unwrap().remove(&region.slot);
+
         // Setting the size to 0 means "remove"
         region.memory_size = 0;
         // Safe because guest regions are guaranteed not to overlap.
@@ -383,10 +421,58 @@ impl vm::Vm for KvmVm {
         Ok(())
     }
 
+    ///
+    /// Start logging dirty pages
+    ///
+    fn start_dirty_log(&self) -> vm::Result<()> {
+        let dirty_log_slots = self.dirty_log_slots.read().unwrap();
+        for (_, s) in dirty_log_slots.iter() {
+            let region = MemoryRegion {
+                slot: s.slot,
+                guest_phys_addr: s.guest_phys_addr,
+                memory_size: s.memory_size,
+                userspace_addr: s.userspace_addr,
+                flags: KVM_MEM_LOG_DIRTY_PAGES,
+            };
+            // Safe because guest regions are guaranteed not to overlap.
+            unsafe {
+                self.fd
+                    .set_user_memory_region(region)
+                    .map_err(|e| vm::HypervisorVmError::StartDirtyLog(e.into()))?;
+            }
+        }
+
+        Ok(())
+    }
+
+    ///
+    /// Stop logging dirty pages
+    ///
+    fn stop_dirty_log(&self) -> vm::Result<()> {
+        let dirty_log_slots = self.dirty_log_slots.read().unwrap();
+        for (_, s) in dirty_log_slots.iter() {
+            let region = MemoryRegion {
+                slot: s.slot,
+                guest_phys_addr: s.guest_phys_addr,
+                memory_size: s.memory_size,
+                userspace_addr: s.userspace_addr,
+                flags: 0,
+            };
+            // Safe because guest regions are guaranteed not to overlap.
+            unsafe {
+                self.fd
+                    .set_user_memory_region(region)
+                    .map_err(|e| vm::HypervisorVmError::StartDirtyLog(e.into()))?;
+            }
+        }
+
+        Ok(())
+    }
+
     ///
     /// Get dirty pages bitmap (one bit per page)
     ///
-    fn get_dirty_log(&self, slot: u32, memory_size: u64) -> vm::Result<Vec<u64>> {
+    fn get_dirty_log(&self, slot: u32, _base_gpa: u64, memory_size: u64) -> vm::Result<Vec<u64>> {
         self.fd
             .get_dirty_log(slot, memory_size as usize)
             .map_err(|e| vm::HypervisorVmError::GetDirtyLog(e.into()))
@@ -400,15 +486,23 @@ impl vm::Vm for KvmVm {
         #[repr(C)]
         struct TdxInitVm {
             max_vcpus: u32,
-            reserved: u32,
+            tsc_khz: u32,
             attributes: u64,
             cpuid: u64,
+            mrconfigid: [u64; 6],
+            mrowner: [u64; 6],
+            mrownerconfig: [u64; 6],
+            reserved: [u64; 43],
         }
         let data = TdxInitVm {
             max_vcpus,
-            reserved: 0,
+            tsc_khz: 0,
             attributes: 1, // TDX1_TD_ATTRIBUTE_DEBUG,
             cpuid: cpuid.as_fam_struct_ptr() as u64,
+            mrconfigid: [0; 6],
+            mrowner: [0; 6],
+            mrownerconfig: [0; 6],
+            reserved: [0; 43],
         };
 
         tdx_command(
@@ -571,6 +665,7 @@ impl hypervisor::Hypervisor for KvmHypervisor {
                 fd: vm_fd,
                 msrs,
                 state: VmState {},
+                dirty_log_slots: Arc::new(RwLock::new(HashMap::new())),
             }))
         }
 
@@ -579,6 +674,7 @@ impl hypervisor::Hypervisor for KvmHypervisor {
             Ok(Arc::new(KvmVm {
                 fd: vm_fd,
                 state: VmState {},
+                dirty_log_slots: Arc::new(RwLock::new(HashMap::new())),
             }))
         }
     }
@@ -643,7 +739,7 @@ pub struct KvmVcpu {
     fd: VcpuFd,
     #[cfg(target_arch = "x86_64")]
     msrs: MsrEntries,
-    vmmops: Option<Arc<Box<dyn vm::VmmOps>>>,
+    vmmops: Option<Arc<dyn vm::VmmOps>>,
     #[cfg(target_arch = "x86_64")]
     hyperv_synic: AtomicBool,
 }
@@ -1177,19 +1273,12 @@ impl cpu::Vcpu for KvmVcpu {
         // call KVM_GET_ONE_REG on each id in order to save all of them. We carve out from the list
         // the core registers which are represented in the kernel by kvm_regs structure and for which
         // we can calculate the id based on the offset in the structure.
-
-        reg_list.retain(|regid| *regid != 0);
-        reg_list.as_slice().to_vec().sort_unstable();
-
         reg_list.retain(|regid| is_system_register(*regid));
 
         // Now, for the rest of the registers left in the previously fetched register list, we are
         // simply calling KVM_GET_ONE_REG.
         let indices = reg_list.as_slice();
-        for (_pos, index) in indices.iter().enumerate() {
-            if _pos > 230 {
-                break;
-            }
+        for index in indices.iter() {
             state.push(kvm_bindings::kvm_one_reg {
                 id: *index,
                 addr: self

hypervisor/src/mshv/mod.rs

@@ -16,7 +16,8 @@ pub use mshv_bindings::*;
 pub use mshv_ioctls::IoEventAddress;
 use mshv_ioctls::{set_registers_64, Mshv, NoDatamatch, VcpuFd, VmFd};
 use serde_derive::{Deserialize, Serialize};
-use std::sync::Arc;
+use std::collections::HashMap;
+use std::sync::{Arc, RwLock};
 use vm::DataMatch;
 // x86_64 dependencies
 #[cfg(target_arch = "x86_64")]
@@ -30,8 +31,19 @@ pub use x86_64::*;
 
 #[cfg(target_arch = "x86_64")]
 use std::fs::File;
-use std::os::unix::io::AsRawFd;
-use std::sync::RwLock;
+use std::os::unix::io::{AsRawFd, RawFd};
+
+const DIRTY_BITMAP_CLEAR_DIRTY: u64 = 0x4;
+const DIRTY_BITMAP_SET_DIRTY: u64 = 0x8;
+
+///
+/// Export generically-named wrappers of mshv-bindings for Unix-based platforms
+///
+pub use {
+    mshv_bindings::mshv_create_device as CreateDevice,
+    mshv_bindings::mshv_device_attr as DeviceAttr,
+    mshv_bindings::mshv_msi_routing_entry as IrqRoutingEntry, mshv_ioctls::DeviceFd,
+};
 
 pub const PAGE_SHIFT: usize = 12;
 
@@ -42,6 +54,11 @@ pub struct HvState {
 
 pub use HvState as VmState;
 
+struct MshvDirtyLogSlot {
+    guest_pfn: u64,
+    memory_size: u64,
+}
+
 /// Wrapper over mshv system ioctls.
 pub struct MshvHypervisor {
     mshv: Mshv,
@@ -106,6 +123,7 @@ impl hypervisor::Hypervisor for MshvHypervisor {
             msrs,
             hv_state: hv_state_init(),
             vmmops: None,
+            dirty_log_slots: Arc::new(RwLock::new(HashMap::new())),
         }))
     }
     ///
@@ -133,7 +151,7 @@ pub struct MshvVcpu {
     cpuid: CpuId,
     msrs: MsrEntries,
     hv_state: Arc<RwLock<HvState>>, // Mshv State
-    vmmops: Option<Arc<Box<dyn vm::VmmOps>>>,
+    vmmops: Option<Arc<dyn vm::VmmOps>>,
 }
 
 /// Implementation of Vcpu trait for Microsoft Hypervisor
@@ -547,6 +565,36 @@ impl cpu::Vcpu for MshvVcpu {
     }
 }
 
+/// Device struct for MSHV
+pub struct MshvDevice {
+    fd: DeviceFd,
+}
+
+impl device::Device for MshvDevice {
+    ///
+    /// Set device attribute
+    ///
+    fn set_device_attr(&self, attr: &DeviceAttr) -> device::Result<()> {
+        self.fd
+            .set_device_attr(attr)
+            .map_err(|e| device::HypervisorDeviceError::SetDeviceAttribute(e.into()))
+    }
+    ///
+    /// Get device attribute
+    ///
+    fn get_device_attr(&self, attr: &mut DeviceAttr) -> device::Result<()> {
+        self.fd
+            .get_device_attr(attr)
+            .map_err(|e| device::HypervisorDeviceError::GetDeviceAttribute(e.into()))
+    }
+}
+
+impl AsRawFd for MshvDevice {
+    fn as_raw_fd(&self) -> RawFd {
+        self.fd.as_raw_fd()
+    }
+}
+
 struct MshvEmulatorContext<'a> {
     vcpu: &'a MshvVcpu,
     map: (u64, u64), // Initial GVA to GPA mapping provided by the hypervisor
@@ -680,7 +728,8 @@ pub struct MshvVm {
     msrs: MsrEntries,
     // Hypervisor State
     hv_state: Arc<RwLock<HvState>>,
-    vmmops: Option<Arc<Box<dyn vm::VmmOps>>>,
+    vmmops: Option<Arc<dyn vm::VmmOps>>,
+    dirty_log_slots: Arc<RwLock<HashMap<u64, MshvDirtyLogSlot>>>,
 }
 
 fn hv_state_init() -> Arc<RwLock<HvState>> {
@@ -742,7 +791,7 @@ impl vm::Vm for MshvVm {
     fn create_vcpu(
         &self,
         id: u8,
-        vmmops: Option<Arc<Box<dyn VmmOps>>>,
+        vmmops: Option<Arc<dyn VmmOps>>,
     ) -> vm::Result<Arc<dyn cpu::Vcpu>> {
         let vcpu_fd = self
             .fd
@@ -806,6 +855,17 @@ impl vm::Vm for MshvVm {
 
     /// Creates a guest physical memory region.
     fn create_user_memory_region(&self, user_memory_region: MemoryRegion) -> vm::Result<()> {
+        // No matter read only or not we keep track the slots.
+        // For readonly hypervisor can enable the dirty bits,
+        // but a VM exit happens before setting the dirty bits
+        self.dirty_log_slots.write().unwrap().insert(
+            user_memory_region.guest_pfn,
+            MshvDirtyLogSlot {
+                guest_pfn: user_memory_region.guest_pfn,
+                memory_size: user_memory_region.size,
+            },
+        );
+
         self.fd
             .map_user_memory(user_memory_region)
             .map_err(|e| vm::HypervisorVmError::CreateUserMemory(e.into()))?;
@@ -814,6 +874,12 @@ impl vm::Vm for MshvVm {
 
     /// Removes a guest physical memory region.
     fn remove_user_memory_region(&self, user_memory_region: MemoryRegion) -> vm::Result<()> {
+        // Remove the corresponding entry from "self.dirty_log_slots" if needed
+        self.dirty_log_slots
+            .write()
+            .unwrap()
+            .remove(&user_memory_region.guest_pfn);
+
         self.fd
             .unmap_user_memory(user_memory_region)
             .map_err(|e| vm::HypervisorVmError::RemoveUserMemory(e.into()))?;
@@ -842,10 +908,28 @@ impl vm::Vm for MshvVm {
         }
     }
 
+    ///
+    /// Creates an in-kernel device.
+    ///
+    /// See the documentation for `MSHV_CREATE_DEVICE`.
+    fn create_device(&self, device: &mut CreateDevice) -> vm::Result<Arc<dyn device::Device>> {
+        let fd = self
+            .fd
+            .create_device(device)
+            .map_err(|e| vm::HypervisorVmError::CreateDevice(e.into()))?;
+        let device = MshvDevice { fd };
+        Ok(Arc::new(device))
+    }
+
     fn create_passthrough_device(&self) -> vm::Result<Arc<dyn device::Device>> {
-        Err(vm::HypervisorVmError::CreatePassthroughDevice(anyhow!(
-            "No passthrough support"
-        )))
+        let mut vfio_dev = mshv_create_device {
+            type_: mshv_device_type_MSHV_DEV_TYPE_VFIO,
+            fd: 0,
+            flags: 0,
+        };
+
+        self.create_device(&mut vfio_dev)
+            .map_err(|e| vm::HypervisorVmError::CreatePassthroughDevice(e.into()))
     }
 
     fn set_gsi_routing(&self, entries: &[IrqRoutingEntry]) -> vm::Result<()> {
@@ -877,16 +961,44 @@ impl vm::Vm for MshvVm {
         Ok(())
     }
     ///
+    /// Start logging dirty pages
+    ///
+    fn start_dirty_log(&self) -> vm::Result<()> {
+        self.fd
+            .enable_dirty_page_tracking()
+            .map_err(|e| vm::HypervisorVmError::StartDirtyLog(e.into()))
+    }
+    ///
+    /// Stop logging dirty pages
+    ///
+    fn stop_dirty_log(&self) -> vm::Result<()> {
+        let dirty_log_slots = self.dirty_log_slots.read().unwrap();
+        // Before disabling the dirty page tracking we need
+        // to set the dirty bits in the Hypervisor
+        // This is a requirement from Microsoft Hypervisor
+        for (_, s) in dirty_log_slots.iter() {
+            self.fd
+                .get_dirty_log(s.guest_pfn, s.memory_size as usize, DIRTY_BITMAP_SET_DIRTY)
+                .map_err(|e| vm::HypervisorVmError::StartDirtyLog(e.into()))?;
+        }
+        self.fd
+            .disable_dirty_page_tracking()
+            .map_err(|e| vm::HypervisorVmError::StartDirtyLog(e.into()))?;
+        Ok(())
+    }
+    ///
     /// Get dirty pages bitmap (one bit per page)
     ///
-    fn get_dirty_log(&self, _slot: u32, _memory_size: u64) -> vm::Result<Vec<u64>> {
-        Err(vm::HypervisorVmError::GetDirtyLog(anyhow!(
-            "get_dirty_log not implemented"
-        )))
+    fn get_dirty_log(&self, _slot: u32, base_gpa: u64, memory_size: u64) -> vm::Result<Vec<u64>> {
+        self.fd
+            .get_dirty_log(
+                base_gpa >> PAGE_SHIFT,
+                memory_size as usize,
+                DIRTY_BITMAP_CLEAR_DIRTY,
+            )
+            .map_err(|e| vm::HypervisorVmError::GetDirtyLog(e.into()))
     }
 }
 pub use hv_cpuid_entry as CpuIdEntry;
 
-pub type IrqRoutingEntry = mshv_msi_routing_entry;
-
 pub const CPUID_FLAG_VALID_INDEX: u32 = 0;

hypervisor/src/mshv/x86_64/mod.rs

@@ -61,8 +61,6 @@ impl fmt::Display for VcpuMshvState {
     }
 }
 
-pub struct CreateDevice {}
-pub struct DeviceAttr {}
 pub struct IrqRouting {}
 pub enum VcpuExit {}
 pub struct MpState {}

hypervisor/src/vm.rs

@@ -16,7 +16,6 @@ use crate::device::Device;
 use crate::x86_64::CpuId;
 #[cfg(all(feature = "kvm", target_arch = "x86_64"))]
 use crate::ClockData;
-#[cfg(feature = "kvm")]
 use crate::CreateDevice;
 #[cfg(feature = "mshv")]
 use crate::HvState as VmState;
@@ -168,6 +167,16 @@ pub enum HypervisorVmError {
     #[error("Failed to write to IO Bus: {0}")]
     IoBusWrite(#[source] anyhow::Error),
     ///
+    /// Start dirty log error
+    ///
+    #[error("Failed to get dirty log: {0}")]
+    StartDirtyLog(#[source] anyhow::Error),
+    ///
+    /// Stop dirty log error
+    ///
+    #[error("Failed to get dirty log: {0}")]
+    StopDirtyLog(#[source] anyhow::Error),
+    ///
     /// Get dirty log error
     ///
     #[error("Failed to get dirty log: {0}")]
@@ -218,7 +227,7 @@ pub trait Vm: Send + Sync {
     /// Unregister an event that will, when signaled, trigger the `gsi` IRQ.
     fn unregister_irqfd(&self, fd: &EventFd, gsi: u32) -> Result<()>;
     /// Creates a new KVM vCPU file descriptor and maps the memory corresponding
-    fn create_vcpu(&self, id: u8, vmmops: Option<Arc<Box<dyn VmmOps>>>) -> Result<Arc<dyn Vcpu>>;
+    fn create_vcpu(&self, id: u8, vmmops: Option<Arc<dyn VmmOps>>) -> Result<Arc<dyn Vcpu>>;
     /// Registers an event to be signaled whenever a certain address is written to.
     fn register_ioevent(
         &self,
@@ -244,7 +253,6 @@ pub trait Vm: Send + Sync {
     fn create_user_memory_region(&self, user_memory_region: MemoryRegion) -> Result<()>;
     /// Removes a guest physical memory slot.
     fn remove_user_memory_region(&self, user_memory_region: MemoryRegion) -> Result<()>;
-    #[cfg(feature = "kvm")]
     /// Creates an emulated device in the kernel.
     fn create_device(&self, device: &mut CreateDevice) -> Result<Arc<dyn Device>>;
     /// Returns the preferred CPU target type which can be emulated by KVM on underlying host.
@@ -270,8 +278,12 @@ pub trait Vm: Send + Sync {
     fn state(&self) -> Result<VmState>;
     /// Set the VM state
     fn set_state(&self, state: VmState) -> Result<()>;
+    /// Start logging dirty pages
+    fn start_dirty_log(&self) -> Result<()>;
+    /// Stop logging dirty pages
+    fn stop_dirty_log(&self) -> Result<()>;
     /// Get dirty pages bitmap
-    fn get_dirty_log(&self, slot: u32, memory_size: u64) -> Result<Vec<u64>>;
+    fn get_dirty_log(&self, slot: u32, base_gpa: u64, memory_size: u64) -> Result<Vec<u64>>;
     #[cfg(feature = "tdx")]
     /// Initalize TDX on this VM
     fn tdx_init(&self, cpuid: &CpuId, max_vcpus: u32) -> Result<()>;

net_gen/Cargo.toml

@@ -5,4 +5,4 @@ authors = ["The Chromium OS Authors"]
 edition = "2018"
 
 [dependencies]
-vmm-sys-util = ">=0.3.1"
+vmm-sys-util = "0.8.0"

net_util/Cargo.toml

@@ -5,20 +5,20 @@ authors = ["The Chromium OS Authors"]
 edition = "2018"
 
 [dependencies]
-epoll = ">=4.0.1"
-libc = "0.2.98"
+epoll = "4.3.1"
+libc = "0.2.101"
 log = "0.4.14"
 net_gen = { path = "../net_gen" }
 rate_limiter = { path = "../rate_limiter" }
-serde = "1.0.126"
+serde = "1.0.130"
 versionize = "0.1.6"
 versionize_derive = "0.1.4"
 virtio-bindings = "0.1.0"
-vm-memory = { version = "0.5.0", features = ["backend-mmap", "backend-atomic", "backend-bitmap"] }
+vm-memory = { version = "0.6.0", features = ["backend-mmap", "backend-atomic", "backend-bitmap"] }
 vm-virtio = { path = "../vm-virtio" }
-vmm-sys-util = ">=0.3.1"
+vmm-sys-util = "0.8.0"
 
 [dev-dependencies]
-lazy_static = "1.3.0"
+lazy_static = "1.4.0"
 pnet = "0.28.0"
-serde_json = "1.0.64"
+serde_json = "1.0.67"

pci/Cargo.toml

@@ -4,19 +4,26 @@ version = "0.1.0"
 authors = ["Samuel Ortiz <sameo@linux.intel.com>"]
 edition = "2018"
 
+[features]
+default = []
+kvm = ["vfio-ioctls/kvm"]
+mshv = ["vfio-ioctls/mshv"]
+
 [dependencies]
-anyhow = "1.0"
+anyhow = "1.0.43"
 byteorder = "1.4.3"
 hypervisor = { path = "../hypervisor" }
-vfio-ioctls = { git = "https://github.com/rust-vmm/vfio-ioctls", branch = "master" }
-vmm-sys-util = ">=0.3.1"
-libc = "0.2.98"
+vfio-ioctls = { git = "https://github.com/rust-vmm/vfio-ioctls", branch = "main", default-features = false }
+vfio_user = { path = "../vfio_user" }
+vmm-sys-util = "0.8.0"
+libc = "0.2.101"
 log = "0.4.14"
+thiserror = "1.0.29"
 versionize = "0.1.6"
 versionize_derive = "0.1.4"
 vm-allocator = { path = "../vm-allocator" }
 vm-device = { path = "../vm-device" }
-vm-memory = "0.5.0"
+vm-memory = "0.6.0"
 vm-migration = { path = "../vm-migration" }
 
 [dependencies.vfio-bindings]

pci/src/configuration.rs

@@ -317,7 +317,7 @@ pub struct PciConfiguration {
 }
 
 /// See pci_regs.h in kernel
-#[derive(Copy, Clone, PartialEq, Versionize)]
+#[derive(Copy, Clone, PartialEq, Versionize, Debug)]
 pub enum PciBarRegionType {
     Memory32BitRegion = 0,
     IoRegion = 0x01,
@@ -794,9 +794,9 @@ impl PciConfiguration {
                     return None;
                 }
 
-                debug!(
-                    "DETECT BAR REPROG: current 0x{:x}, new 0x{:x}",
-                    self.registers[reg_idx], value
+                info!(
+                    "Detected BAR reprogramming: (BAR {}) 0x{:x}->0x{:x}",
+                    reg_idx, self.registers[reg_idx], value
                 );
                 let old_base = u64::from(self.bars[bar_idx].addr & mask);
                 let new_base = u64::from(value & mask);
@@ -820,9 +820,9 @@ impl PciConfiguration {
                     != (self.bars[bar_idx - 1].addr & self.writable_bits[reg_idx - 1])
                     || (value & mask) != (self.bars[bar_idx].addr & mask))
             {
-                debug!(
-                    "DETECT BAR REPROG: current 0x{:x}, new 0x{:x}",
-                    self.registers[reg_idx], value
+                info!(
+                    "Detected BAR reprogramming: (BAR {}) 0x{:x}->0x{:x}",
+                    reg_idx, self.registers[reg_idx], value
                 );
                 let old_base = u64::from(self.bars[bar_idx].addr & mask) << 32
                     | u64::from(self.bars[bar_idx - 1].addr & self.writable_bits[reg_idx - 1]);
@@ -850,9 +850,9 @@ impl PciConfiguration {
                 return None;
             }
 
-            debug!(
-                "DETECT ROM BAR REPROG: current 0x{:x}, new 0x{:x}",
-                self.registers[reg_idx], value
+            info!(
+                "Detected ROM BAR reprogramming: (BAR {}) 0x{:x}->0x{:x}",
+                reg_idx, self.registers[reg_idx], value
             );
             let old_base = u64::from(self.rom_bar_addr & mask);
             let new_base = u64::from(value & mask);

pci/src/lib.rs

@@ -12,6 +12,7 @@ mod device;
 mod msi;
 mod msix;
 mod vfio;
+mod vfio_user;
 
 pub use self::bus::{PciBus, PciConfigIo, PciConfigMmio, PciRoot, PciRootError};
 pub use self::configuration::{
@@ -25,6 +26,7 @@ pub use self::device::{
 pub use self::msi::{msi_num_enabled_vectors, MsiCap, MsiConfig};
 pub use self::msix::{MsixCap, MsixConfig, MsixTableEntry, MSIX_TABLE_ENTRY_SIZE};
 pub use self::vfio::{VfioPciDevice, VfioPciError};
+pub use self::vfio_user::{VfioUserPciDevice, VfioUserPciDeviceError};
 
 /// PCI has four interrupt pins A->D.
 #[derive(Copy, Clone)]

pci/src/msi.rs

@@ -159,11 +159,11 @@ impl MsiCap {
 
 pub struct MsiConfig {
     cap: MsiCap,
-    interrupt_source_group: Arc<Box<dyn InterruptSourceGroup>>,
+    interrupt_source_group: Arc<dyn InterruptSourceGroup>,
 }
 
 impl MsiConfig {
-    pub fn new(msg_ctl: u16, interrupt_source_group: Arc<Box<dyn InterruptSourceGroup>>) -> Self {
+    pub fn new(msg_ctl: u16, interrupt_source_group: Arc<dyn InterruptSourceGroup>) -> Self {
         let cap = MsiCap {
             msg_ctl,
             ..Default::default()

pci/src/msix.rs

@@ -74,7 +74,7 @@ pub struct MsixConfig {
     pub table_entries: Vec<MsixTableEntry>,
     pub pba_entries: Vec<u64>,
     pub devid: u32,
-    interrupt_source_group: Arc<Box<dyn InterruptSourceGroup>>,
+    interrupt_source_group: Arc<dyn InterruptSourceGroup>,
     masked: bool,
     enabled: bool,
 }
@@ -82,7 +82,7 @@ pub struct MsixConfig {
 impl MsixConfig {
     pub fn new(
         msix_vectors: u16,
-        interrupt_source_group: Arc<Box<dyn InterruptSourceGroup>>,
+        interrupt_source_group: Arc<dyn InterruptSourceGroup>,
         devid: u32,
     ) -> Self {
         assert!(msix_vectors <= MAX_MSIX_VECTORS_PER_DEVICE);

pci/src/vfio_user.rs

@@ -0,0 +1,485 @@
+// Copyright © 2021 Intel Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+use crate::vfio::{Interrupt, Vfio, VfioCommon, VfioError};
+use crate::{BarReprogrammingParams, PciBarRegionType, VfioPciError};
+use crate::{
+    PciClassCode, PciConfiguration, PciDevice, PciDeviceError, PciHeaderType, PciSubclass,
+};
+use hypervisor::HypervisorVmError;
+use std::any::Any;
+use std::os::unix::prelude::AsRawFd;
+use std::path::Path;
+use std::ptr::null_mut;
+use std::sync::{Arc, Barrier, Mutex};
+use std::u32;
+use thiserror::Error;
+use vfio_bindings::bindings::vfio::*;
+use vfio_ioctls::VfioIrq;
+use vfio_user::{Client, Error as VfioUserError};
+use vm_allocator::SystemAllocator;
+use vm_device::interrupt::{InterruptManager, InterruptSourceGroup, MsiIrqGroupConfig};
+use vm_device::BusDevice;
+use vm_memory::bitmap::AtomicBitmap;
+use vm_memory::{Address, GuestAddress, GuestMemoryRegion, GuestRegionMmap, GuestUsize};
+use vmm_sys_util::eventfd::EventFd;
+
+pub struct VfioUserPciDevice {
+    vm: Arc<dyn hypervisor::Vm>,
+    client: Arc<Mutex<Client>>,
+    vfio_wrapper: VfioUserClientWrapper,
+    common: VfioCommon,
+}
+
+#[derive(Error, Debug)]
+pub enum VfioUserPciDeviceError {
+    #[error("Client error: {0}")]
+    Client(#[source] VfioUserError),
+    #[error("Failed to map VFIO PCI region into guest: {0}")]
+    MapRegionGuest(#[source] HypervisorVmError),
+    #[error("Failed to DMA map: {0}")]
+    DmaMap(#[source] VfioUserError),
+    #[error("Failed to DMA unmap: {0}")]
+    DmaUnmap(#[source] VfioUserError),
+    #[error("Failed to initialize legacy interrupts: {0}")]
+    InitializeLegacyInterrupts(#[source] VfioPciError),
+}
+
+#[derive(Copy, Clone)]
+enum PciVfioUserSubclass {
+    VfioUserSubclass = 0xff,
+}
+
+impl PciSubclass for PciVfioUserSubclass {
+    fn get_register_value(&self) -> u8 {
+        *self as u8
+    }
+}
+
+impl VfioUserPciDevice {
+    pub fn new(
+        vm: &Arc<dyn hypervisor::Vm>,
+        path: &Path,
+        msi_interrupt_manager: &Arc<dyn InterruptManager<GroupConfig = MsiIrqGroupConfig>>,
+        legacy_interrupt_group: Option<Arc<dyn InterruptSourceGroup>>,
+    ) -> Result<Self, VfioUserPciDeviceError> {
+        let mut client = Client::new(path).map_err(VfioUserPciDeviceError::Client)?;
+
+        // This is used for the BAR and capabilities only
+        let configuration = PciConfiguration::new(
+            0,
+            0,
+            0,
+            PciClassCode::Other,
+            &PciVfioUserSubclass::VfioUserSubclass,
+            None,
+            PciHeaderType::Device,
+            0,
+            0,
+            None,
+        );
+        if client.resettable() {
+            client.reset().map_err(VfioUserPciDeviceError::Client)?;
+        }
+
+        let client = Arc::new(Mutex::new(client));
+
+        let vfio_wrapper = VfioUserClientWrapper {
+            client: client.clone(),
+        };
+
+        let mut common = VfioCommon {
+            mmio_regions: Vec::new(),
+            configuration,
+            interrupt: Interrupt {
+                intx: None,
+                msi: None,
+                msix: None,
+            },
+        };
+
+        common.parse_capabilities(msi_interrupt_manager, &vfio_wrapper);
+        common
+            .initialize_legacy_interrupt(legacy_interrupt_group, &vfio_wrapper)
+            .map_err(VfioUserPciDeviceError::InitializeLegacyInterrupts)?;
+
+        Ok(Self {
+            vm: vm.clone(),
+            client,
+            vfio_wrapper,
+            common,
+        })
+    }
+}
+
+impl BusDevice for VfioUserPciDevice {
+    fn read(&mut self, base: u64, offset: u64, data: &mut [u8]) {
+        self.read_bar(base, offset, data)
+    }
+
+    fn write(&mut self, base: u64, offset: u64, data: &[u8]) -> Option<Arc<Barrier>> {
+        self.write_bar(base, offset, data)
+    }
+}
+
+#[repr(u32)]
+#[derive(Debug, PartialEq, Eq, PartialOrd, Ord)]
+#[allow(dead_code)]
+enum Regions {
+    Bar0,
+    Bar1,
+    Bar2,
+    Bar3,
+    Bar4,
+    Bar5,
+    Rom,
+    Config,
+    Vga,
+    Migration,
+}
+
+struct VfioUserClientWrapper {
+    client: Arc<Mutex<Client>>,
+}
+
+impl Vfio for VfioUserClientWrapper {
+    fn region_read(&self, index: u32, offset: u64, data: &mut [u8]) {
+        self.client
+            .lock()
+            .unwrap()
+            .region_read(index, offset, data)
+            .ok();
+    }
+
+    fn region_write(&self, index: u32, offset: u64, data: &[u8]) {
+        self.client
+            .lock()
+            .unwrap()
+            .region_write(index, offset, data)
+            .ok();
+    }
+
+    fn get_irq_info(&self, irq_index: u32) -> Option<VfioIrq> {
+        self.client
+            .lock()
+            .unwrap()
+            .get_irq_info(irq_index)
+            .ok()
+            .map(|i| VfioIrq {
+                index: i.index,
+                flags: i.flags,
+                count: i.count,
+            })
+    }
+
+    fn enable_irq(&self, irq_index: u32, event_fds: Vec<&EventFd>) -> Result<(), VfioError> {
+        info!(
+            "Enabling IRQ {:x} number of fds = {:?}",
+            irq_index,
+            event_fds.len()
+        );
+        let fds: Vec<i32> = event_fds.iter().map(|e| e.as_raw_fd()).collect();
+
+        self.client
+            .lock()
+            .unwrap()
+            .set_irqs(
+                irq_index,
+                VFIO_IRQ_SET_DATA_EVENTFD | VFIO_IRQ_SET_ACTION_TRIGGER,
+                0,
+                event_fds.len() as u32,
+                &fds,
+            )
+            .map_err(VfioError::VfioUser)
+    }
+
+    fn disable_irq(&self, irq_index: u32) -> Result<(), VfioError> {
+        info!("Disabling IRQ {:x}", irq_index);
+        self.client
+            .lock()
+            .unwrap()
+            .set_irqs(
+                irq_index,
+                VFIO_IRQ_SET_DATA_NONE | VFIO_IRQ_SET_ACTION_TRIGGER,
+                0,
+                0,
+                &[],
+            )
+            .map_err(VfioError::VfioUser)
+    }
+
+    fn unmask_irq(&self, irq_index: u32) -> Result<(), VfioError> {
+        info!("Unmasking IRQ {:x}", irq_index);
+        self.client
+            .lock()
+            .unwrap()
+            .set_irqs(
+                irq_index,
+                VFIO_IRQ_SET_DATA_NONE | VFIO_IRQ_SET_ACTION_UNMASK,
+                0,
+                1,
+                &[],
+            )
+            .map_err(VfioError::VfioUser)
+    }
+}
+
+impl PciDevice for VfioUserPciDevice {
+    fn allocate_bars(
+        &mut self,
+        allocator: &mut SystemAllocator,
+    ) -> Result<Vec<(GuestAddress, GuestUsize, PciBarRegionType)>, PciDeviceError> {
+        self.common.allocate_bars(allocator, &self.vfio_wrapper)
+    }
+
+    fn free_bars(&mut self, allocator: &mut SystemAllocator) -> Result<(), PciDeviceError> {
+        self.common.free_bars(allocator)
+    }
+
+    fn as_any(&mut self) -> &mut dyn Any {
+        self
+    }
+
+    fn detect_bar_reprogramming(
+        &mut self,
+        reg_idx: usize,
+        data: &[u8],
+    ) -> Option<BarReprogrammingParams> {
+        self.common
+            .configuration
+            .detect_bar_reprogramming(reg_idx, data)
+    }
+
+    fn write_config_register(
+        &mut self,
+        reg_idx: usize,
+        offset: u64,
+        data: &[u8],
+    ) -> Option<Arc<Barrier>> {
+        self.common
+            .write_config_register(reg_idx, offset, data, &self.vfio_wrapper)
+    }
+
+    fn read_config_register(&mut self, reg_idx: usize) -> u32 {
+        self.common
+            .read_config_register(reg_idx, &self.vfio_wrapper)
+    }
+
+    fn read_bar(&mut self, base: u64, offset: u64, data: &mut [u8]) {
+        self.common.read_bar(base, offset, data, &self.vfio_wrapper)
+    }
+
+    fn write_bar(&mut self, base: u64, offset: u64, data: &[u8]) -> Option<Arc<Barrier>> {
+        self.common
+            .write_bar(base, offset, data, &self.vfio_wrapper)
+    }
+
+    fn move_bar(&mut self, old_base: u64, new_base: u64) -> Result<(), std::io::Error> {
+        info!("Moving BAR 0x{:x} -> 0x{:x}", old_base, new_base);
+        for mmio_region in self.common.mmio_regions.iter_mut() {
+            if mmio_region.start.raw_value() == old_base {
+                mmio_region.start = GuestAddress(new_base);
+
+                if let Some(mem_slot) = mmio_region.mem_slot {
+                    if let Some(host_addr) = mmio_region.host_addr {
+                        // Remove original region
+                        let old_region = self.vm.make_user_memory_region(
+                            mem_slot,
+                            old_base,
+                            mmio_region.length as u64,
+                            host_addr as u64,
+                            false,
+                            false,
+                        );
+
+                        self.vm
+                            .remove_user_memory_region(old_region)
+                            .map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e))?;
+
+                        let new_region = self.vm.make_user_memory_region(
+                            mem_slot,
+                            new_base,
+                            mmio_region.length as u64,
+                            host_addr as u64,
+                            false,
+                            false,
+                        );
+
+                        self.vm
+                            .create_user_memory_region(new_region)
+                            .map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e))?;
+                    }
+                }
+                info!("Moved bar 0x{:x} -> 0x{:x}", old_base, new_base);
+            }
+        }
+
+        Ok(())
+    }
+}
+
+impl VfioUserPciDevice {
+    pub fn map_mmio_regions<F>(
+        &mut self,
+        vm: &Arc<dyn hypervisor::Vm>,
+        mem_slot: F,
+    ) -> Result<(), VfioUserPciDeviceError>
+    where
+        F: Fn() -> u32,
+    {
+        for mmio_region in &mut self.common.mmio_regions {
+            let region_flags = self
+                .client
+                .lock()
+                .unwrap()
+                .region(mmio_region.index)
+                .unwrap()
+                .flags;
+            let file_offset = self
+                .client
+                .lock()
+                .unwrap()
+                .region(mmio_region.index)
+                .unwrap()
+                .file_offset
+                .clone();
+
+            if region_flags & VFIO_REGION_INFO_FLAG_MMAP != 0 {
+                let mut prot = 0;
+                if region_flags & VFIO_REGION_INFO_FLAG_READ != 0 {
+                    prot |= libc::PROT_READ;
+                }
+                if region_flags & VFIO_REGION_INFO_FLAG_WRITE != 0 {
+                    prot |= libc::PROT_WRITE;
+                }
+
+                let host_addr = unsafe {
+                    libc::mmap(
+                        null_mut(),
+                        mmio_region.length as usize,
+                        prot,
+                        libc::MAP_SHARED,
+                        file_offset.as_ref().unwrap().file().as_raw_fd(),
+                        file_offset.as_ref().unwrap().start() as libc::off_t,
+                    )
+                };
+
+                if host_addr == libc::MAP_FAILED {
+                    error!(
+                        "Could not mmap regions, error:{}",
+                        std::io::Error::last_os_error()
+                    );
+                    continue;
+                }
+
+                let slot = mem_slot();
+                let mem_region = vm.make_user_memory_region(
+                    slot,
+                    mmio_region.start.0,
+                    mmio_region.length as u64,
+                    host_addr as u64,
+                    false,
+                    false,
+                );
+
+                vm.create_user_memory_region(mem_region)
+                    .map_err(VfioUserPciDeviceError::MapRegionGuest)?;
+
+                mmio_region.mem_slot = Some(slot);
+                mmio_region.host_addr = Some(host_addr as u64);
+                mmio_region.mmap_size = Some(mmio_region.length as usize);
+            }
+        }
+
+        Ok(())
+    }
+
+    pub fn unmap_mmio_regions(&mut self) {
+        for mmio_region in self.common.mmio_regions.iter() {
+            if let (Some(host_addr), Some(mmap_size), Some(mem_slot)) = (
+                mmio_region.host_addr,
+                mmio_region.mmap_size,
+                mmio_region.mem_slot,
+            ) {
+                // Remove region
+                let r = self.vm.make_user_memory_region(
+                    mem_slot,
+                    mmio_region.start.raw_value(),
+                    mmap_size as u64,
+                    host_addr as u64,
+                    false,
+                    false,
+                );
+
+                if let Err(e) = self.vm.remove_user_memory_region(r) {
+                    error!("Could not remove the userspace memory region: {}", e);
+                }
+
+                let ret = unsafe { libc::munmap(host_addr as *mut libc::c_void, mmap_size) };
+                if ret != 0 {
+                    error!(
+                        "Could not unmap region {}, error:{}",
+                        mmio_region.index,
+                        std::io::Error::last_os_error()
+                    );
+                }
+            }
+        }
+    }
+
+    pub fn dma_map(
+        &mut self,
+        region: &GuestRegionMmap<AtomicBitmap>,
+    ) -> Result<(), VfioUserPciDeviceError> {
+        let (fd, offset) = match region.file_offset() {
+            Some(_file_offset) => (_file_offset.file().as_raw_fd(), _file_offset.start()),
+            None => return Ok(()),
+        };
+
+        self.client
+            .lock()
+            .unwrap()
+            .dma_map(
+                offset,
+                region.start_addr().raw_value(),
+                region.len() as u64,
+                fd,
+            )
+            .map_err(VfioUserPciDeviceError::DmaMap)
+    }
+
+    pub fn dma_unmap(
+        &mut self,
+        region: &GuestRegionMmap<AtomicBitmap>,
+    ) -> Result<(), VfioUserPciDeviceError> {
+        self.client
+            .lock()
+            .unwrap()
+            .dma_unmap(region.start_addr().raw_value(), region.len() as u64)
+            .map_err(VfioUserPciDeviceError::DmaUnmap)
+    }
+}
+
+impl Drop for VfioUserPciDevice {
+    fn drop(&mut self) {
+        self.unmap_mmio_regions();
+
+        if let Some(msix) = &self.common.interrupt.msix {
+            if msix.bar.enabled() {
+                self.common.disable_msix(&self.vfio_wrapper);
+            }
+        }
+
+        if let Some(msi) = &self.common.interrupt.msi {
+            if msi.cfg.enabled() {
+                self.common.disable_msi(&self.vfio_wrapper)
+            }
+        }
+
+        if self.common.interrupt.intx_in_use() {
+            self.common.disable_intx(&self.vfio_wrapper);
+        }
+    }
+}

qcow/Cargo.toml

@@ -10,7 +10,7 @@ path = "src/qcow.rs"
 
 [dependencies]
 byteorder = "1.4.3"
-libc = "0.2.98"
+libc = "0.2.101"
 log = "0.4.14"
 remain = "0.2.2"
-vmm-sys-util = ">=0.3.1"
+vmm-sys-util = "0.8.0"

rate_limiter/Cargo.toml

@@ -4,6 +4,6 @@ version = "0.1.0"
 edition = "2018"
 
 [dependencies]
-libc = "0.2.98"
+libc = "0.2.101"
 log = "0.4.14"
 vmm-sys-util = "0.8.0"

release-notes.md

@@ -1,18 +1,29 @@
+- [v18.0](#v180)
+    - [Experimental User Device (`vfio-user`) support](#experimental-user-device-vfio-user-support)
+    - [Migration support for `vhost-user` devices](#migration-support-for-vhost-user-devices)
+    - [VHDX disk image support](#vhdx-disk-image-support)
+    - [Device pass through on MSHV hypervisor](#device-pass-through-on-mshv-hypervisor)
+    - [AArch64 for support `virtio-mem`](#aarch64-for-support-virtio-mem)
+    - [Live migration on MSHV hypervisor](#live-migration-on-mshv-hypervisor)
+    - [AArch64 CPU topology support](#aarch64-cpu-topology-support)
+    - [Power button support on AArch64](#power-button-support-on-aarch64)
+    - [Notable bug fixes](#notable-bug-fixes)
+    - [Contributors](#contributors)
 - [v17.0](#v170)
     - [ARM64 NUMA support using ACPI](#arm64-numa-support-using-acpi)
     - [`Seccomp` support for MSHV backend](#seccomp-support-for-mshv-backend)
     - [Hotplug of `macvtap` devices](#hotplug-of-macvtap-devices)
     - [Improved SGX support](#improved-sgx-support)
     - [Inflight tracking for `vhost-user` devices](#inflight-tracking-for-vhost-user-devices)
-    - [Notable bug fixes](#notable-bug-fixes)
-    - [Contributors](#contributors)
+    - [Notable bug fixes](#notable-bug-fixes-1)
+    - [Contributors](#contributors-1)
 - [v16.0](#v160)
     - [Improved live migration support](#improved-live-migration-support)
     - [Improved `vhost-user` support](#improved-vhost-user-support)
     - [ARM64 ACPI and UEFI support](#arm64-acpi-and-uefi-support)
-    - [Notable bug fixes](#notable-bug-fixes-1)
+    - [Notable bug fixes](#notable-bug-fixes-2)
     - [Removed functionality](#removed-functionality)
-    - [Contributors](#contributors-1)
+    - [Contributors](#contributors-2)
 - [v15.0](#v150)
     - [Version numbering and stability guarantees](#version-numbering-and-stability-guarantees)
     - [Network device rate limiting](#network-device-rate-limiting)
@@ -20,7 +31,7 @@
     - [`--api-socket` supports file descriptor parameter](#--api-socket-supports-file-descriptor-parameter)
     - [Bug fixes](#bug-fixes)
     - [Deprecations](#deprecations)
-    - [Contributors](#contributors-2)
+    - [Contributors](#contributors-3)
 - [v0.14.1](#v0141)
 - [v0.14.0](#v0140)
     - [Structured event monitoring](#structured-event-monitoring)
@@ -30,7 +41,7 @@
     - [PTY control for serial and `virtio-console`](#pty-control-for-serial-and-virtio-console)
     - [Block device rate limiting](#block-device-rate-limiting)
     - [Deprecations](#deprecations-1)
-    - [Contributors](#contributors-3)
+    - [Contributors](#contributors-4)
 - [v0.13.0](#v0130)
     - [Wider VFIO device support](#wider-vfio-device-support)
     - [Improved huge page support](#improved-huge-page-support)
@@ -38,13 +49,13 @@
     - [VHD disk image support](#vhd-disk-image-support)
     - [Improved Virtio device threading](#improved-virtio-device-threading)
     - [Clean shutdown support via synthetic power button](#clean-shutdown-support-via-synthetic-power-button)
-    - [Contributors](#contributors-4)
+    - [Contributors](#contributors-5)
 - [v0.12.0](#v0120)
     - [ARM64 enhancements](#arm64-enhancements)
     - [Removal of `vhost-user-net` and `vhost-user-block` self spawning](#removal-of-vhost-user-net-and-vhost-user-block-self-spawning)
     - [Migration of `vhost-user-fs` backend](#migration-of-vhost-user-fs-backend)
     - [Enhanced "info" API](#enhanced-info-api)
-    - [Contributors](#contributors-5)
+    - [Contributors](#contributors-6)
 - [v0.11.0](#v0110)
     - [`io_uring` support by default for `virtio-block`](#io_uring-support-by-default-for-virtio-block)
     - [Windows Guest Support](#windows-guest-support)
@@ -56,15 +67,15 @@
     - [Default Log Level Changed](#default-log-level-changed)
     - [New `--balloon` Parameter Added](#new---balloon-parameter-added)
     - [Experimental `virtio-watchdog` Support](#experimental-virtio-watchdog-support)
-    - [Notable Bug Fixes](#notable-bug-fixes-2)
-    - [Contributors](#contributors-6)
+    - [Notable Bug Fixes](#notable-bug-fixes-3)
+    - [Contributors](#contributors-7)
 - [v0.10.0](#v0100)
     - [`virtio-block` Support for Multiple Descriptors](#virtio-block-support-for-multiple-descriptors)
     - [Memory Zones](#memory-zones)
     - [`Seccomp` Sandbox Improvements](#seccomp-sandbox-improvements)
     - [Preliminary KVM HyperV Emulation Control](#preliminary-kvm-hyperv-emulation-control)
-    - [Notable Bug Fixes](#notable-bug-fixes-3)
-    - [Contributors](#contributors-7)
+    - [Notable Bug Fixes](#notable-bug-fixes-4)
+    - [Contributors](#contributors-8)
 - [v0.9.0](#v090)
     - [`io_uring` Based Block Device Support](#io_uring-based-block-device-support)
     - [Block and Network Device Statistics](#block-and-network-device-statistics)
@@ -77,17 +88,17 @@
     - [Enhancements to ARM64 Support](#enhancements-to-arm64-support)
     - [Intel SGX Support](#intel-sgx-support)
     - [`Seccomp` Sandbox Improvements](#seccomp-sandbox-improvements-1)
-    - [Notable Bug Fixes](#notable-bug-fixes-4)
-    - [Contributors](#contributors-8)
+    - [Notable Bug Fixes](#notable-bug-fixes-5)
+    - [Contributors](#contributors-9)
 - [v0.8.0](#v080)
     - [Experimental Snapshot and Restore Support](#experimental-snapshot-and-restore-support)
     - [Experimental ARM64 Support](#experimental-arm64-support)
     - [Support for Using 5-level Paging in Guests](#support-for-using-5-level-paging-in-guests)
     - [Virtio Device Interrupt Suppression for Network Devices](#virtio-device-interrupt-suppression-for-network-devices)
     - [`vhost_user_fs` Improvements](#vhost_user_fs-improvements)
-    - [Notable Bug Fixes](#notable-bug-fixes-5)
+    - [Notable Bug Fixes](#notable-bug-fixes-6)
     - [Command Line and API Changes](#command-line-and-api-changes)
-    - [Contributors](#contributors-9)
+    - [Contributors](#contributors-10)
 - [v0.7.0](#v070)
     - [Block, Network, Persistent Memory (PMEM), VirtioFS and Vsock hotplug](#block-network-persistent-memory-pmem-virtiofs-and-vsock-hotplug)
     - [Alternative `libc` Support](#alternative-libc-support)
@@ -97,14 +108,14 @@
     - [`Seccomp` Sandboxing](#seccomp-sandboxing)
     - [Updated Distribution Support](#updated-distribution-support)
     - [Command Line and API Changes](#command-line-and-api-changes-1)
-    - [Contributors](#contributors-10)
+    - [Contributors](#contributors-11)
 - [v0.6.0](#v060)
     - [Directly Assigned Devices Hotplug](#directly-assigned-devices-hotplug)
     - [Shared Filesystem Improvements](#shared-filesystem-improvements)
     - [Block and Networking IO Self Offloading](#block-and-networking-io-self-offloading)
     - [Command Line Interface](#command-line-interface)
     - [PVH Boot](#pvh-boot)
-    - [Contributors](#contributors-11)
+    - [Contributors](#contributors-12)
 - [v0.5.1](#v051)
 - [v0.5.0](#v050)
     - [Virtual Machine Dynamic Resizing](#virtual-machine-dynamic-resizing)
@@ -112,7 +123,7 @@
     - [New Interrupt Management Framework](#new-interrupt-management-framework)
     - [Development Tools](#development-tools)
     - [Kata Containers Integration](#kata-containers-integration)
-    - [Contributors](#contributors-12)
+    - [Contributors](#contributors-13)
 - [v0.4.0](#v040)
     - [Dynamic virtual CPUs addition](#dynamic-virtual-cpus-addition)
     - [Programmatic firmware tables generation](#programmatic-firmware-tables-generation)
@@ -121,7 +132,7 @@
     - [Userspace IOAPIC by default](#userspace-ioapic-by-default)
     - [PCI BAR reprogramming](#pci-bar-reprogramming)
     - [New `cloud-hypervisor` organization](#new-cloud-hypervisor-organization)
-    - [Contributors](#contributors-13)
+    - [Contributors](#contributors-14)
 - [v0.3.0](#v030)
     - [Block device offloading](#block-device-offloading)
     - [Network device backend](#network-device-backend)
@@ -148,6 +159,78 @@
     - [Unit testing](#unit-testing)
     - [Integration tests parallelization](#integration-tests-parallelization)
 
+# v18.0
+
+This release has been tracked through the [v18.0
+project](https://github.com/cloud-hypervisor/cloud-hypervisor/projects/21).
+
+### Experimental User Device (`vfio-user`) support
+
+Experimental support for running PCI devices in userspace via `vfio-user`
+has been included. This allows the use of the SPDK NVMe `vfio-user` controller
+with Cloud Hypervisor. This is enabled by `--user-device` on the command line.
+
+### Migration support for `vhost-user` devices
+
+Devices exposed into the VM via `vhost-user` can now be migrated using the live
+migration support. This requires support from the backend however the commonly
+used DPDK `vhost-user` backend does support this.
+
+### VHDX disk image support
+
+Images using the VHDX disk image format can now be used with Cloud Hypervisor.
+
+### Device pass through on MSHV hypervisor
+
+When running on the MSHV hypervisor it is possible to pass through devices from
+the host through to the guest (e.g with `--device`)
+
+### AArch64 for support `virtio-mem`
+
+The reference Linux kernel we recommend for using with Cloud Hypervisor now supports `virtio-mem` on AArch64.
+
+### Live migration on MSHV hypervisor
+
+Live migration is now supported when running on the MSHV hypervisor including
+efficient tracking of dirty pages.
+
+### AArch64 CPU topology support
+
+The CPU topology (as configured through `--cpu topology=`) can now be
+configured on AArch64 platforms and is conveyed through either ACPI or device
+tree.
+
+### Power button support on AArch64
+
+Use of the ACPI power button (e.g `ch-remote --api-socket=<API socket> power-button`) 
+is now supported when running on AArch64.
+
+### Notable bug fixes
+
+* Using two PTY outputs e.g. `--serial pty --console pty` now works correctly (#3012)
+* TTY input is now always sent to the correct destination (#3005)
+* The boot is no longer blocked when using a unattached PTY on the serial console (#3004)
+* Live migration is now supported on AArch64 (#3049)
+* Ensure signal handlers are run on the correct thread (#3069)
+
+### Contributors
+
+Many thanks to everyone who has contributed to our release:
+
+* Alyssa Ross <hi@alyssa.is>
+* Anatol Belski <anbelski@linux.microsoft.com>
+* Arafatms <arafatms@outlook.com>
+* Bo Chen <chen.bo@intel.com>
+* Fazla Mehrab <akm.fazla.mehrab@vt.edu>
+* Henry Wang <Henry.Wang@arm.com>
+* Jianyong Wu <jianyong.wu@arm.com>
+* Jiaqi Gao <jiaqi.gao@intel.com>
+* Markus Theil <markus.theil@tu-ilmenau.de>
+* Muminul Islam <muislam@microsoft.com>
+* Rob Bradford <robert.bradford@intel.com>
+* Sebastien Boeuf <sebastien.boeuf@intel.com>
+* Wei Liu <liuwe@microsoft.com>
+* Yu Li <liyu.yukiteru@bytedance.com>
 
 # v17.0
 

resources/Dockerfile

@@ -1,7 +1,7 @@
 FROM ubuntu:20.04 as dev
 
 ARG TARGETARCH
-ARG RUST_TOOLCHAIN="1.52.1"
+ARG RUST_TOOLCHAIN="1.54.0"
 ARG CLH_SRC_DIR="/cloud-hypervisor"
 ARG CLH_BUILD_DIR="$CLH_SRC_DIR/build"
 ARG CARGO_REGISTRY_DIR="$CLH_BUILD_DIR/cargo_registry"

resources/linux-config-aarch64

@@ -1,11 +1,13 @@
 #
 # Automatically generated file; DO NOT EDIT.
-# Linux/arm64 5.12.0 Kernel Configuration
+# Linux/arm64 5.14.0 Kernel Configuration
 #
 CONFIG_CC_VERSION_TEXT="gcc (Ubuntu 9.3.0-17ubuntu1~20.04) 9.3.0"
 CONFIG_CC_IS_GCC=y
 CONFIG_GCC_VERSION=90300
 CONFIG_CLANG_VERSION=0
+CONFIG_AS_IS_GNU=y
+CONFIG_AS_VERSION=23400
 CONFIG_LD_IS_BFD=y
 CONFIG_LD_VERSION=23400
 CONFIG_LLD_VERSION=0
@@ -13,6 +15,7 @@ CONFIG_CC_CAN_LINK=y
 CONFIG_CC_CAN_LINK_STATIC=y
 CONFIG_CC_HAS_ASM_GOTO=y
 CONFIG_CC_HAS_ASM_INLINE=y
+CONFIG_CC_HAS_NO_PROFILE_FN_ATTR=y
 CONFIG_IRQ_WORK=y
 CONFIG_BUILDTIME_TABLE_SORT=y
 CONFIG_THREAD_INFO_IN_TASK=y
@@ -60,7 +63,6 @@ CONFIG_SPARSE_IRQ=y
 # CONFIG_GENERIC_IRQ_DEBUGFS is not set
 # end of IRQ subsystem
 
-CONFIG_GENERIC_IRQ_MULTI_HANDLER=y
 CONFIG_GENERIC_TIME_VSYSCALL=y
 CONFIG_GENERIC_CLOCKEVENTS=y
 CONFIG_ARCH_HAS_TICK_BROADCAST=y
@@ -78,9 +80,22 @@ CONFIG_NO_HZ=y
 CONFIG_HIGH_RES_TIMERS=y
 # end of Timers subsystem
 
+CONFIG_BPF=y
+CONFIG_HAVE_EBPF_JIT=y
+CONFIG_ARCH_WANT_DEFAULT_BPF_JIT=y
+
+#
+# BPF subsystem
+#
+CONFIG_BPF_SYSCALL=y
+# CONFIG_BPF_UNPRIV_DEFAULT_OFF is not set
+# CONFIG_BPF_PRELOAD is not set
+# end of BPF subsystem
+
 CONFIG_PREEMPT_NONE=y
 # CONFIG_PREEMPT_VOLUNTARY is not set
 # CONFIG_PREEMPT is not set
+# CONFIG_SCHED_CORE is not set
 
 #
 # CPU/Task time and stats accounting
@@ -151,6 +166,7 @@ CONFIG_CGROUP_DEVICE=y
 CONFIG_CGROUP_CPUACCT=y
 CONFIG_CGROUP_PERF=y
 CONFIG_CGROUP_BPF=y
+# CONFIG_CGROUP_MISC is not set
 # CONFIG_CGROUP_DEBUG is not set
 CONFIG_SOCK_CGROUP_DATA=y
 CONFIG_NAMESPACES=y
@@ -179,7 +195,6 @@ CONFIG_CC_OPTIMIZE_FOR_PERFORMANCE=y
 CONFIG_LD_ORPHAN_WARN=y
 CONFIG_SYSCTL=y
 CONFIG_SYSCTL_EXCEPTION_TRACE=y
-CONFIG_BPF=y
 CONFIG_EXPERT=y
 CONFIG_MULTIUSER=y
 CONFIG_SGETMASK_SYSCALL=y
@@ -202,13 +217,11 @@ CONFIG_SHMEM=y
 CONFIG_AIO=y
 CONFIG_IO_URING=y
 CONFIG_ADVISE_SYSCALLS=y
+CONFIG_HAVE_ARCH_USERFAULTFD_MINOR=y
 CONFIG_MEMBARRIER=y
 CONFIG_KALLSYMS=y
 # CONFIG_KALLSYMS_ALL is not set
 CONFIG_KALLSYMS_BASE_RELATIVE=y
-CONFIG_BPF_SYSCALL=y
-CONFIG_ARCH_WANT_DEFAULT_BPF_JIT=y
-# CONFIG_BPF_PRELOAD is not set
 CONFIG_USERFAULTFD=y
 CONFIG_ARCH_HAS_MEMBARRIER_SYNC_CORE=y
 # CONFIG_KCMP is not set
@@ -258,10 +271,7 @@ CONFIG_GENERIC_BUG_RELATIVE_POINTERS=y
 CONFIG_GENERIC_HWEIGHT=y
 CONFIG_GENERIC_CSUM=y
 CONFIG_GENERIC_CALIBRATE_DELAY=y
-CONFIG_ZONE_DMA=y
-CONFIG_ZONE_DMA32=y
-CONFIG_ARCH_ENABLE_MEMORY_HOTPLUG=y
-CONFIG_ARCH_ENABLE_MEMORY_HOTREMOVE=y
+CONFIG_ARCH_MHP_MEMMAP_ON_MEMORY_ENABLE=y
 CONFIG_SMP=y
 CONFIG_KERNEL_MODE_NEON=y
 CONFIG_FIX_EARLYCON_MEM=y
@@ -273,10 +283,9 @@ CONFIG_ARCH_PROC_KCORE_TEXT=y
 # Platform selection
 #
 # CONFIG_ARCH_ACTIONS is not set
-# CONFIG_ARCH_AGILEX is not set
-# CONFIG_ARCH_N5X is not set
 # CONFIG_ARCH_SUNXI is not set
 # CONFIG_ARCH_ALPINE is not set
+# CONFIG_ARCH_APPLE is not set
 # CONFIG_ARCH_BCM2835 is not set
 # CONFIG_ARCH_BCM4908 is not set
 # CONFIG_ARCH_BCM_IPROC is not set
@@ -300,7 +309,7 @@ CONFIG_ARCH_PROC_KCORE_TEXT=y
 # CONFIG_ARCH_ROCKCHIP is not set
 # CONFIG_ARCH_S32 is not set
 # CONFIG_ARCH_SEATTLE is not set
-# CONFIG_ARCH_STRATIX10 is not set
+# CONFIG_ARCH_INTEL_SOCFPGA is not set
 # CONFIG_ARCH_SYNQUACER is not set
 # CONFIG_ARCH_TEGRA is not set
 # CONFIG_ARCH_SPRD is not set
@@ -328,6 +337,7 @@ CONFIG_ARM64_ERRATUM_819472=y
 CONFIG_ARM64_ERRATUM_832075=y
 CONFIG_ARM64_ERRATUM_834220=y
 CONFIG_ARM64_ERRATUM_843419=y
+CONFIG_ARM64_LD_HAS_FIX_ERRATUM_843419=y
 CONFIG_ARM64_ERRATUM_1024718=y
 CONFIG_ARM64_WORKAROUND_SPECULATIVE_AT=y
 CONFIG_ARM64_ERRATUM_1165522=y
@@ -373,7 +383,6 @@ CONFIG_NODES_SHIFT=10
 CONFIG_USE_PERCPU_NUMA_NODE_ID=y
 CONFIG_HAVE_SETUP_PER_CPU_AREA=y
 CONFIG_NEED_PER_CPU_EMBED_FIRST_CHUNK=y
-CONFIG_HOLES_IN_ZONE=y
 # CONFIG_HZ_100 is not set
 CONFIG_HZ_250=y
 # CONFIG_HZ_300 is not set
@@ -381,12 +390,8 @@ CONFIG_HZ_250=y
 CONFIG_HZ=250
 CONFIG_SCHED_HRTICK=y
 CONFIG_ARCH_SPARSEMEM_ENABLE=y
-CONFIG_ARCH_SPARSEMEM_DEFAULT=y
-CONFIG_ARCH_SELECT_MEMORY_MODEL=y
 CONFIG_HW_PERF_EVENTS=y
-CONFIG_SYS_SUPPORTS_HUGETLBFS=y
-CONFIG_ARCH_HAS_CACHE_LINE_SIZE=y
-CONFIG_ARCH_ENABLE_SPLIT_PMD_PTLOCK=y
+CONFIG_ARCH_HAS_FILTER_PGPROT=y
 CONFIG_PARAVIRT=y
 CONFIG_PARAVIRT_TIME_ACCOUNTING=y
 # CONFIG_KEXEC is not set
@@ -411,7 +416,6 @@ CONFIG_AS_HAS_LDAPR=y
 CONFIG_AS_HAS_LSE_ATOMICS=y
 CONFIG_ARM64_LSE_ATOMICS=y
 CONFIG_ARM64_USE_LSE_ATOMICS=y
-CONFIG_ARM64_VHE=y
 # end of ARMv8.1 architectural features
 
 #
@@ -426,10 +430,9 @@ CONFIG_ARM64_CNP=y
 # ARMv8.3 architectural features
 #
 CONFIG_ARM64_PTR_AUTH=y
-CONFIG_CC_HAS_BRANCH_PROT_PAC_RET=y
+CONFIG_ARM64_PTR_AUTH_KERNEL=y
 CONFIG_CC_HAS_SIGN_RETURN_ADDRESS=y
 CONFIG_AS_HAS_PAC=y
-CONFIG_AS_HAS_CFI_NEGATE_RA_STATE=y
 # end of ARMv8.3 architectural features
 
 #
@@ -443,22 +446,22 @@ CONFIG_ARM64_TLB_RANGE=y
 #
 # ARMv8.5 architectural features
 #
-CONFIG_AS_HAS_ARMV8_5=y
 CONFIG_ARM64_BTI=y
-CONFIG_CC_HAS_BRANCH_PROT_PAC_RET_BTI=y
 CONFIG_ARM64_E0PD=y
 CONFIG_ARCH_RANDOM=y
-CONFIG_ARM64_AS_HAS_MTE=y
-CONFIG_ARM64_MTE=y
 # end of ARMv8.5 architectural features
 
+#
+# ARMv8.7 architectural features
+#
+CONFIG_ARM64_EPAN=y
+# end of ARMv8.7 architectural features
+
 CONFIG_ARM64_SVE=y
 # CONFIG_ARM64_PSEUDO_NMI is not set
 CONFIG_RELOCATABLE=y
 CONFIG_RANDOMIZE_BASE=y
 CONFIG_RANDOMIZE_MODULE_REGION_FULL=y
-CONFIG_CC_HAVE_STACKPROTECTOR_SYSREG=y
-CONFIG_STACKPROTECTOR_PER_TASK=y
 # end of Kernel Features
 
 #
@@ -471,9 +474,6 @@ CONFIG_EFI=y
 CONFIG_DMI=y
 # end of Boot options
 
-CONFIG_ARCH_ENABLE_HUGEPAGE_MIGRATION=y
-CONFIG_ARCH_ENABLE_THP_MIGRATION=y
-
 #
 # Power management options
 #
@@ -553,6 +553,7 @@ CONFIG_FIRMWARE_MEMMAP=y
 CONFIG_DMIID=y
 CONFIG_DMI_SYSFS=y
 # CONFIG_FW_CFG_SYSFS is not set
+# CONFIG_ARM_FFA_TRANSPORT is not set
 # CONFIG_GOOGLE_FIRMWARE is not set
 
 #
@@ -623,6 +624,7 @@ CONFIG_ACPI_IORT=y
 CONFIG_ACPI_GTDT=y
 CONFIG_ACPI_PPTT=y
 # CONFIG_PMIC_OPREGION is not set
+CONFIG_ACPI_VIOT=y
 CONFIG_IRQ_BYPASS_MANAGER=y
 CONFIG_VIRTUALIZATION=y
 CONFIG_KVM=y
@@ -661,6 +663,7 @@ CONFIG_ARCH_HAS_KEEPINITRD=y
 CONFIG_ARCH_HAS_SET_MEMORY=y
 CONFIG_ARCH_HAS_SET_DIRECT_MAP=y
 CONFIG_HAVE_ARCH_THREAD_STRUCT_WHITELIST=y
+CONFIG_ARCH_WANTS_NO_INSTR=y
 CONFIG_HAVE_ASM_MODVERSIONS=y
 CONFIG_HAVE_REGS_AND_STACK_ACCESS_API=y
 CONFIG_HAVE_RSEQ=y
@@ -688,6 +691,7 @@ CONFIG_STACKPROTECTOR_STRONG=y
 CONFIG_ARCH_SUPPORTS_LTO_CLANG=y
 CONFIG_ARCH_SUPPORTS_LTO_CLANG_THIN=y
 CONFIG_LTO_NONE=y
+CONFIG_ARCH_SUPPORTS_CFI_CLANG=y
 CONFIG_HAVE_CONTEXT_TRACKING=y
 CONFIG_HAVE_VIRT_CPU_ACCOUNTING_GEN=y
 CONFIG_HAVE_IRQ_TIME_ACCOUNTING=y
@@ -705,6 +709,8 @@ CONFIG_CLONE_BACKWARDS=y
 # CONFIG_COMPAT_32BIT_TIME is not set
 CONFIG_HAVE_ARCH_VMAP_STACK=y
 CONFIG_VMAP_STACK=y
+CONFIG_HAVE_ARCH_RANDOMIZE_KSTACK_OFFSET=y
+# CONFIG_RANDOMIZE_KSTACK_OFFSET_DEFAULT is not set
 CONFIG_ARCH_HAS_STRICT_KERNEL_RWX=y
 CONFIG_STRICT_KERNEL_RWX=y
 CONFIG_ARCH_HAS_STRICT_MODULE_RWX=y
@@ -742,9 +748,10 @@ CONFIG_BLK_DEV_THROTTLING=y
 # CONFIG_BLK_DEV_THROTTLING_LOW is not set
 CONFIG_BLK_CMDLINE_PARSER=y
 CONFIG_BLK_WBT=y
+CONFIG_BLK_WBT_MQ=y
 # CONFIG_BLK_CGROUP_IOLATENCY is not set
 # CONFIG_BLK_CGROUP_IOCOST is not set
-CONFIG_BLK_WBT_MQ=y
+# CONFIG_BLK_CGROUP_IOPRIO is not set
 CONFIG_BLK_DEBUG_FS=y
 # CONFIG_BLK_SED_OPAL is not set
 # CONFIG_BLK_INLINE_ENCRYPTION is not set
@@ -863,10 +870,7 @@ CONFIG_COREDUMP=y
 #
 # Memory Management options
 #
-CONFIG_SELECT_MEMORY_MODEL=y
-CONFIG_SPARSEMEM_MANUAL=y
 CONFIG_SPARSEMEM=y
-CONFIG_NEED_MULTIPLE_NODES=y
 CONFIG_SPARSEMEM_EXTREME=y
 CONFIG_SPARSEMEM_VMEMMAP_ENABLE=y
 CONFIG_SPARSEMEM_VMEMMAP=y
@@ -874,19 +878,24 @@ CONFIG_HAVE_FAST_GUP=y
 CONFIG_ARCH_KEEP_MEMBLOCK=y
 CONFIG_NUMA_KEEP_MEMINFO=y
 CONFIG_MEMORY_ISOLATION=y
+CONFIG_ARCH_ENABLE_MEMORY_HOTPLUG=y
 CONFIG_MEMORY_HOTPLUG=y
 CONFIG_MEMORY_HOTPLUG_SPARSE=y
-# CONFIG_MEMORY_HOTPLUG_DEFAULT_ONLINE is not set
+CONFIG_MEMORY_HOTPLUG_DEFAULT_ONLINE=y
+CONFIG_ARCH_ENABLE_MEMORY_HOTREMOVE=y
 CONFIG_MEMORY_HOTREMOVE=y
+CONFIG_MHP_MEMMAP_ON_MEMORY=y
 CONFIG_SPLIT_PTLOCK_CPUS=4
+CONFIG_ARCH_ENABLE_SPLIT_PMD_PTLOCK=y
 CONFIG_MEMORY_BALLOON=y
 CONFIG_BALLOON_COMPACTION=y
 CONFIG_COMPACTION=y
 CONFIG_PAGE_REPORTING=y
 CONFIG_MIGRATION=y
+CONFIG_ARCH_ENABLE_HUGEPAGE_MIGRATION=y
+CONFIG_ARCH_ENABLE_THP_MIGRATION=y
 CONFIG_CONTIG_ALLOC=y
 CONFIG_PHYS_ADDR_T_64BIT=y
-CONFIG_BOUNCE=y
 CONFIG_MMU_NOTIFIER=y
 CONFIG_KSM=y
 CONFIG_DEFAULT_MMAP_MIN_ADDR=4096
@@ -919,15 +928,19 @@ CONFIG_ZSMALLOC_STAT=y
 CONFIG_GENERIC_EARLY_IOREMAP=y
 # CONFIG_DEFERRED_STRUCT_PAGE_INIT is not set
 # CONFIG_IDLE_PAGE_TRACKING is not set
+CONFIG_ARCH_HAS_CACHE_LINE_SIZE=y
 CONFIG_ARCH_HAS_PTE_DEVMAP=y
+CONFIG_ARCH_HAS_ZONE_DMA_SET=y
+CONFIG_ZONE_DMA=y
+CONFIG_ZONE_DMA32=y
 CONFIG_ZONE_DEVICE=y
 CONFIG_DEV_PAGEMAP_OPS=y
 # CONFIG_DEVICE_PRIVATE is not set
-CONFIG_ARCH_USES_HIGH_VMA_FLAGS=y
 CONFIG_PERCPU_STATS=y
 # CONFIG_GUP_TEST is not set
 # CONFIG_READ_ONLY_THP_FOR_FS is not set
 CONFIG_ARCH_HAS_PTE_SPECIAL=y
+CONFIG_SECRETMEM=y
 # end of Memory Management options
 
 CONFIG_NET=y
@@ -991,7 +1004,6 @@ CONFIG_NET_PTP_CLASSIFY=y
 # CONFIG_ATM is not set
 # CONFIG_L2TP is not set
 # CONFIG_BRIDGE is not set
-CONFIG_HAVE_NET_DSA=y
 # CONFIG_NET_DSA is not set
 # CONFIG_VLAN_8021Q is not set
 # CONFIG_DECNET is not set
@@ -1019,6 +1031,7 @@ CONFIG_VIRTIO_VSOCKETS_COMMON=y
 # CONFIG_NET_L3_MASTER_DEV is not set
 # CONFIG_QRTR is not set
 # CONFIG_NET_NCSI is not set
+CONFIG_PCPU_DEV_REFCNT=y
 CONFIG_RPS=y
 CONFIG_RFS_ACCEL=y
 CONFIG_SOCK_RX_QUEUE_MAPPING=y
@@ -1056,7 +1069,6 @@ CONFIG_GRO_CELLS=y
 CONFIG_NET_SOCK_MSG=y
 CONFIG_FAILOVER=y
 # CONFIG_ETHTOOL_NETLINK is not set
-CONFIG_HAVE_EBPF_JIT=y
 
 #
 # Device Drivers
@@ -1128,7 +1140,6 @@ CONFIG_PCI_HOST_GENERIC=y
 #
 # Mobiveil PCIe Core Support
 #
-# CONFIG_PCIE_LAYERSCAPE_GEN4 is not set
 # end of Mobiveil PCIe Core Support
 
 #
@@ -1229,7 +1240,6 @@ CONFIG_ZRAM_DEF_COMP_LZORLE=y
 CONFIG_ZRAM_DEF_COMP="lzo-rle"
 # CONFIG_ZRAM_WRITEBACK is not set
 # CONFIG_ZRAM_MEMORY_TRACKING is not set
-# CONFIG_BLK_DEV_UMEM is not set
 CONFIG_BLK_DEV_LOOP=y
 CONFIG_BLK_DEV_LOOP_MIN_COUNT=8
 CONFIG_BLK_DEV_CRYPTOLOOP=y
@@ -1250,6 +1260,7 @@ CONFIG_VIRTIO_BLK=y
 #
 # CONFIG_BLK_DEV_NVME is not set
 # CONFIG_NVME_FC is not set
+# CONFIG_NVME_TCP is not set
 # CONFIG_NVME_TARGET is not set
 # end of NVME Support
 
@@ -1262,9 +1273,9 @@ CONFIG_VIRTIO_BLK=y
 # CONFIG_ENCLOSURE_SERVICES is not set
 # CONFIG_HP_ILO is not set
 # CONFIG_SRAM is not set
+# CONFIG_DW_XDATA_PCIE is not set
 # CONFIG_PCI_ENDPOINT_TEST is not set
 # CONFIG_XILINX_SDFEC is not set
-# CONFIG_PVPANIC is not set
 # CONFIG_C2PORT is not set
 
 #
@@ -1291,6 +1302,7 @@ CONFIG_VIRTIO_BLK=y
 # CONFIG_MISC_RTSX_PCI is not set
 # CONFIG_HABANA_AI is not set
 # CONFIG_UACCE is not set
+# CONFIG_PVPANIC is not set
 # end of Misc devices
 
 #
@@ -1335,13 +1347,6 @@ CONFIG_VETH=y
 CONFIG_VIRTIO_NET=y
 # CONFIG_NLMON is not set
 # CONFIG_ARCNET is not set
-
-#
-# Distributed Switch Architecture drivers
-#
-# CONFIG_NET_DSA_MV88E6XXX_PTP is not set
-# end of Distributed Switch Architecture drivers
-
 # CONFIG_ETHERNET is not set
 # CONFIG_FDDI is not set
 # CONFIG_HIPPI is not set
@@ -1362,6 +1367,13 @@ CONFIG_VIRTIO_NET=y
 #
 # CONFIG_WLAN is not set
 # CONFIG_WAN is not set
+
+#
+# Wireless WAN
+#
+# CONFIG_WWAN is not set
+# end of Wireless WAN
+
 # CONFIG_VMXNET3 is not set
 # CONFIG_FUJITSU_ES is not set
 # CONFIG_NETDEVSIM is not set
@@ -1497,7 +1509,6 @@ CONFIG_SERIAL_MCTRL_GPIO=y
 # CONFIG_N_GSM is not set
 # CONFIG_NOZOMI is not set
 # CONFIG_NULL_TTY is not set
-# CONFIG_TRACE_SINK is not set
 CONFIG_HVC_DRIVER=y
 # CONFIG_HVC_DCC is not set
 CONFIG_SERIAL_DEV_BUS=y
@@ -1514,8 +1525,6 @@ CONFIG_HW_RANDOM_CAVIUM=y
 # CONFIG_HW_RANDOM_XIPHERA is not set
 # CONFIG_APPLICOM is not set
 CONFIG_DEVMEM=y
-CONFIG_RAW_DRIVER=y
-CONFIG_MAX_RAW_DEVS=256
 CONFIG_DEVPORT=y
 # CONFIG_TCG_TPM is not set
 # CONFIG_XILLYBUS is not set
@@ -1556,6 +1565,7 @@ CONFIG_PTP_1588_CLOCK=y
 #
 # Enable PHYLIB and NETWORK_PHY_TIMESTAMPING to see the additional clocks.
 #
+CONFIG_PTP_1588_CLOCK_KVM=y
 # CONFIG_PTP_1588_CLOCK_OCP is not set
 # end of PTP clock support
 
@@ -1632,6 +1642,7 @@ CONFIG_POWER_SUPPLY=y
 # CONFIG_CHARGER_MAX8903 is not set
 # CONFIG_CHARGER_GPIO is not set
 # CONFIG_CHARGER_LT3651 is not set
+# CONFIG_BATTERY_GOLDFISH is not set
 # CONFIG_HWMON is not set
 CONFIG_THERMAL=y
 # CONFIG_THERMAL_NETLINK is not set
@@ -1655,6 +1666,7 @@ CONFIG_WATCHDOG_CORE=y
 CONFIG_WATCHDOG_HANDLE_BOOT_ENABLED=y
 CONFIG_WATCHDOG_OPEN_TIMEOUT=0
 # CONFIG_WATCHDOG_SYSFS is not set
+# CONFIG_WATCHDOG_HRTIMER_PRETIMEOUT is not set
 
 #
 # Watchdog Pretimeout Governors
@@ -1705,7 +1717,6 @@ CONFIG_BCMA_POSSIBLE=y
 # CONFIG_MFD_MT6397 is not set
 # CONFIG_MFD_RDC321X is not set
 # CONFIG_MFD_SM501 is not set
-# CONFIG_ABX500_CORE is not set
 # CONFIG_MFD_SYSCON is not set
 # CONFIG_MFD_TI_AM335X_TSCADC is not set
 # CONFIG_MFD_TQMX86 is not set
@@ -1869,6 +1880,7 @@ CONFIG_HID_REDRAGON=y
 # CONFIG_HID_PRIMAX is not set
 # CONFIG_HID_SAITEK is not set
 # CONFIG_HID_SAMSUNG is not set
+# CONFIG_HID_SEMITEK is not set
 # CONFIG_HID_SPEEDLINK is not set
 # CONFIG_HID_STEAM is not set
 # CONFIG_HID_STEELSERIES is not set
@@ -1878,7 +1890,6 @@ CONFIG_HID_REDRAGON=y
 # CONFIG_HID_SMARTJOYPLUS is not set
 # CONFIG_HID_TIVO is not set
 # CONFIG_HID_TOPSEED is not set
-# CONFIG_HID_THRUSTMASTER is not set
 # CONFIG_HID_UDRAW_PS3 is not set
 # CONFIG_HID_XINMO is not set
 # CONFIG_HID_ZEROPLUS is not set
@@ -2033,6 +2044,7 @@ CONFIG_VIRTIO_PCI=y
 CONFIG_VIRTIO_PCI_LEGACY=y
 CONFIG_VIRTIO_PMEM=y
 CONFIG_VIRTIO_BALLOON=y
+CONFIG_VIRTIO_MEM=y
 CONFIG_VIRTIO_INPUT=y
 CONFIG_VIRTIO_MMIO=y
 CONFIG_VIRTIO_MMIO_CMDLINE_DEVICES=y
@@ -2048,15 +2060,23 @@ CONFIG_VHOST_MENU=y
 # end of Microsoft Hyper-V guest support
 
 # CONFIG_GREYBUS is not set
+# CONFIG_COMEDI is not set
 # CONFIG_STAGING is not set
 # CONFIG_GOLDFISH is not set
 # CONFIG_CHROME_PLATFORMS is not set
 # CONFIG_MELLANOX_PLATFORM is not set
 # CONFIG_SURFACE_PLATFORMS is not set
 CONFIG_HAVE_CLK=y
-CONFIG_CLKDEV_LOOKUP=y
 CONFIG_HAVE_CLK_PREPARE=y
 CONFIG_COMMON_CLK=y
+
+#
+# Clock driver for ARM Reference designs
+#
+# CONFIG_ICST is not set
+# CONFIG_CLK_SP810 is not set
+# end of Clock driver for ARM Reference designs
+
 # CONFIG_COMMON_CLK_AXI_CLKGEN is not set
 # CONFIG_COMMON_CLK_XGENE is not set
 # CONFIG_COMMON_CLK_FIXED_MMIO is not set
@@ -2104,9 +2124,6 @@ CONFIG_IOMMU_DMA=y
 # CONFIG_ARM_SMMU is not set
 # CONFIG_ARM_SMMU_V3 is not set
 CONFIG_VIRTIO_IOMMU=y
-CONFIG_VIRTIO_IOMMU_TOPOLOGY_HELPERS=y
-CONFIG_VIRTIO_IOMMU_TOPOLOGY=y
-CONFIG_ACPI_VIOT=y
 
 #
 # Remoteproc drivers
@@ -2199,6 +2216,7 @@ CONFIG_PARTITION_PERCPU=y
 #
 # CONFIG_GENERIC_PHY is not set
 # CONFIG_PHY_XGENE is not set
+# CONFIG_PHY_CAN_TRANSCEIVER is not set
 # CONFIG_BCM_KONA_USB2_PHY is not set
 # CONFIG_PHY_CADENCE_TORRENT is not set
 # CONFIG_PHY_CADENCE_DPHY is not set
@@ -2327,6 +2345,8 @@ CONFIG_OVERLAY_FS_REDIRECT_ALWAYS_FOLLOW=y
 #
 # Caches
 #
+CONFIG_NETFS_SUPPORT=y
+# CONFIG_NETFS_STATS is not set
 CONFIG_FSCACHE=y
 # CONFIG_FSCACHE_STATS is not set
 # CONFIG_FSCACHE_HISTOGRAM is not set
@@ -2371,6 +2391,7 @@ CONFIG_TMPFS=y
 CONFIG_TMPFS_POSIX_ACL=y
 CONFIG_TMPFS_XATTR=y
 # CONFIG_TMPFS_INODE64 is not set
+CONFIG_ARCH_SUPPORTS_HUGETLBFS=y
 CONFIG_HUGETLBFS=y
 CONFIG_HUGETLB_PAGE=y
 CONFIG_MEMFD_CREATE=y
@@ -2545,6 +2566,7 @@ CONFIG_CRYPTO_CRYPTD=y
 # CONFIG_CRYPTO_RSA is not set
 # CONFIG_CRYPTO_DH is not set
 # CONFIG_CRYPTO_ECDH is not set
+# CONFIG_CRYPTO_ECDSA is not set
 # CONFIG_CRYPTO_ECRDSA is not set
 # CONFIG_CRYPTO_SM2 is not set
 # CONFIG_CRYPTO_CURVE25519 is not set
@@ -2673,6 +2695,8 @@ CONFIG_SYSTEM_BLACKLIST_KEYRING=y
 CONFIG_SYSTEM_BLACKLIST_HASH_LIST=""
 # end of Certificates for signature checking
 
+CONFIG_BINARY_PRINTF=y
+
 #
 # Library routines
 #
@@ -2682,6 +2706,7 @@ CONFIG_HAVE_ARCH_BITREVERSE=y
 CONFIG_GENERIC_STRNCPY_FROM_USER=y
 CONFIG_GENERIC_STRNLEN_USER=y
 CONFIG_GENERIC_NET_UTILS=y
+CONFIG_GENERIC_FIND_FIRST_BIT=y
 # CONFIG_CORDIC is not set
 # CONFIG_PRIME_NUMBERS is not set
 CONFIG_RATIONAL=y
@@ -2770,7 +2795,6 @@ CONFIG_FONT_8x16=y
 CONFIG_MEMREGION=y
 CONFIG_ARCH_STACKWALK=y
 CONFIG_SBITMAP=y
-# CONFIG_STRING_SELFTEST is not set
 # end of Library routines
 
 CONFIG_GENERIC_LIB_DEVMEM_IS_ALLOWED=y
@@ -2784,6 +2808,7 @@ CONFIG_GENERIC_LIB_DEVMEM_IS_ALLOWED=y
 #
 CONFIG_PRINTK_TIME=y
 # CONFIG_PRINTK_CALLER is not set
+# CONFIG_STACKTRACE_BUILD_ID is not set
 CONFIG_CONSOLE_LOGLEVEL_DEFAULT=7
 CONFIG_CONSOLE_LOGLEVEL_QUIET=4
 CONFIG_MESSAGE_LOGLEVEL_DEFAULT=4
@@ -2804,9 +2829,10 @@ CONFIG_STRIP_ASM_SYMS=y
 # CONFIG_HEADERS_INSTALL is not set
 CONFIG_DEBUG_SECTION_MISMATCH=y
 CONFIG_SECTION_MISMATCH_WARN_ONLY=y
-# CONFIG_DEBUG_FORCE_FUNCTION_ALIGN_32B is not set
+# CONFIG_DEBUG_FORCE_FUNCTION_ALIGN_64B is not set
 CONFIG_ARCH_WANT_FRAME_POINTERS=y
 CONFIG_FRAME_POINTER=y
+# CONFIG_VMLINUX_MAP is not set
 # CONFIG_DEBUG_FORCE_WEAK_PER_CPU is not set
 # end of Compile-time checks and compiler options
 
@@ -2858,10 +2884,8 @@ CONFIG_DEBUG_MEMORY_INIT=y
 # CONFIG_DEBUG_PER_CPU_MAPS is not set
 CONFIG_HAVE_ARCH_KASAN=y
 CONFIG_HAVE_ARCH_KASAN_SW_TAGS=y
-CONFIG_HAVE_ARCH_KASAN_HW_TAGS=y
+CONFIG_HAVE_ARCH_KASAN_VMALLOC=y
 CONFIG_CC_HAS_KASAN_GENERIC=y
-CONFIG_CC_HAS_WORKING_NOSANITIZE_ADDRESS=y
-# CONFIG_KASAN is not set
 CONFIG_HAVE_ARCH_KFENCE=y
 # CONFIG_KFENCE is not set
 # end of Memory Debugging
@@ -2913,7 +2937,6 @@ CONFIG_LOCK_DEBUGGING_SUPPORT=y
 CONFIG_STACKTRACE=y
 # CONFIG_WARN_ALL_UNSEEDED_RANDOM is not set
 # CONFIG_DEBUG_KOBJECT is not set
-CONFIG_HAVE_DEBUG_BUGVERBOSE=y
 
 #
 # Debug kernel data structures
@@ -2945,7 +2968,6 @@ CONFIG_RCU_CPU_STALL_TIMEOUT=59
 CONFIG_HAVE_FUNCTION_TRACER=y
 CONFIG_HAVE_FUNCTION_GRAPH_TRACER=y
 CONFIG_HAVE_DYNAMIC_FTRACE=y
-CONFIG_HAVE_DYNAMIC_FTRACE_WITH_REGS=y
 CONFIG_HAVE_FTRACE_MCOUNT_RECORD=y
 CONFIG_HAVE_SYSCALL_TRACEPOINTS=y
 CONFIG_HAVE_C_RECORDMCOUNT=y
@@ -2973,19 +2995,21 @@ CONFIG_CC_HAS_SANCOV_TRACE_PC=y
 # CONFIG_KCOV is not set
 CONFIG_RUNTIME_TESTING_MENU=y
 # CONFIG_LKDTM is not set
-# CONFIG_TEST_LIST_SORT is not set
 # CONFIG_TEST_MIN_HEAP is not set
 # CONFIG_TEST_SORT is not set
+# CONFIG_TEST_DIV64 is not set
 # CONFIG_BACKTRACE_SELF_TEST is not set
 # CONFIG_RBTREE_TEST is not set
 # CONFIG_REED_SOLOMON_TEST is not set
 # CONFIG_INTERVAL_TREE_TEST is not set
 # CONFIG_ATOMIC64_SELFTEST is not set
 # CONFIG_TEST_HEXDUMP is not set
+# CONFIG_STRING_SELFTEST is not set
 # CONFIG_TEST_STRING_HELPERS is not set
 # CONFIG_TEST_STRSCPY is not set
 # CONFIG_TEST_KSTRTOX is not set
 # CONFIG_TEST_PRINTF is not set
+# CONFIG_TEST_SCANF is not set
 # CONFIG_TEST_BITMAP is not set
 # CONFIG_TEST_UUID is not set
 # CONFIG_TEST_XARRAY is not set
@@ -3001,6 +3025,7 @@ CONFIG_RUNTIME_TESTING_MENU=y
 # CONFIG_TEST_STACKINIT is not set
 # CONFIG_TEST_MEMINIT is not set
 # CONFIG_TEST_FREE_PAGES is not set
+CONFIG_ARCH_USE_MEMTEST=y
 # CONFIG_MEMTEST is not set
 # end of Kernel Testing and Coverage
 # end of Kernel hacking

resources/linux-config-x86_64

@@ -1,10 +1,10 @@
 #
 # Automatically generated file; DO NOT EDIT.
-# Linux/x86 5.13.0-rc5 Kernel Configuration
+# Linux/x86 5.14.0 Kernel Configuration
 #
-CONFIG_CC_VERSION_TEXT="gcc (Ubuntu 10.2.0-13ubuntu1) 10.2.0"
+CONFIG_CC_VERSION_TEXT="gcc (Ubuntu 10.3.0-1ubuntu1) 10.3.0"
 CONFIG_CC_IS_GCC=y
-CONFIG_GCC_VERSION=100200
+CONFIG_GCC_VERSION=100300
 CONFIG_CLANG_VERSION=0
 CONFIG_AS_IS_GNU=y
 CONFIG_AS_VERSION=23500
@@ -15,6 +15,7 @@ CONFIG_CC_CAN_LINK=y
 CONFIG_CC_CAN_LINK_STATIC=y
 CONFIG_CC_HAS_ASM_GOTO=y
 CONFIG_CC_HAS_ASM_INLINE=y
+CONFIG_CC_HAS_NO_PROFILE_FN_ATTR=y
 CONFIG_IRQ_WORK=y
 CONFIG_BUILDTIME_TABLE_SORT=y
 CONFIG_THREAD_INFO_IN_TASK=y
@@ -114,6 +115,7 @@ CONFIG_BPF_SYSCALL=y
 CONFIG_PREEMPT_NONE=y
 # CONFIG_PREEMPT_VOLUNTARY is not set
 # CONFIG_PREEMPT is not set
+# CONFIG_SCHED_CORE is not set
 
 #
 # CPU/Task time and stats accounting
@@ -299,7 +301,6 @@ CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK=y
 CONFIG_ARCH_HIBERNATION_POSSIBLE=y
 CONFIG_ARCH_SUSPEND_POSSIBLE=y
 CONFIG_ARCH_WANT_GENERAL_HUGETLB=y
-CONFIG_ZONE_DMA32=y
 CONFIG_AUDIT_ARCH=y
 CONFIG_X86_64_SMP=y
 CONFIG_ARCH_SUPPORTS_UPROBES=y
@@ -311,7 +312,6 @@ CONFIG_CC_HAS_SANE_STACKPROTECTOR=y
 #
 # Processor type and features
 #
-CONFIG_ZONE_DMA=y
 CONFIG_SMP=y
 CONFIG_X86_FEATURE_NAMES=y
 CONFIG_X86_X2APIC=y
@@ -409,7 +409,6 @@ CONFIG_ILLEGAL_POINTER_VALUE=0xdead000000000000
 # CONFIG_X86_PMEM_LEGACY is not set
 CONFIG_X86_CHECK_BIOS_CORRUPTION=y
 CONFIG_X86_BOOTPARAM_MEMORY_CORRUPTION_CHECK=y
-CONFIG_X86_RESERVE_LOW=64
 CONFIG_MTRR=y
 CONFIG_MTRR_SANITIZER=y
 CONFIG_MTRR_SANITIZER_ENABLE_DEFAULT=0
@@ -529,6 +528,7 @@ CONFIG_HAVE_ACPI_APEI_NMI=y
 # CONFIG_PMIC_OPREGION is not set
 CONFIG_ACPI_VIOT=y
 CONFIG_X86_PM_TIMER=y
+# CONFIG_ACPI_PRMT is not set
 
 #
 # CPU Frequency scaling
@@ -647,6 +647,7 @@ CONFIG_KVM_GENERIC_DIRTYLOG_READ_PROTECT=y
 CONFIG_HAVE_KVM_IRQ_BYPASS=y
 CONFIG_HAVE_KVM_NO_POLL=y
 CONFIG_KVM_XFER_TO_GUEST_WORK=y
+CONFIG_HAVE_KVM_PM_NOTIFIER=y
 CONFIG_VIRTUALIZATION=y
 CONFIG_KVM=y
 # CONFIG_KVM_WERROR is not set
@@ -686,6 +687,7 @@ CONFIG_ARCH_HAS_SET_MEMORY=y
 CONFIG_ARCH_HAS_SET_DIRECT_MAP=y
 CONFIG_HAVE_ARCH_THREAD_STRUCT_WHITELIST=y
 CONFIG_ARCH_WANTS_DYNAMIC_TASK_STRUCT=y
+CONFIG_ARCH_WANTS_NO_INSTR=y
 CONFIG_HAVE_ASM_MODVERSIONS=y
 CONFIG_HAVE_REGS_AND_STACK_ACCESS_API=y
 CONFIG_HAVE_RSEQ=y
@@ -778,14 +780,16 @@ CONFIG_BLK_CGROUP_RWSTAT=y
 CONFIG_BLK_DEV_BSG=y
 CONFIG_BLK_DEV_BSGLIB=y
 CONFIG_BLK_DEV_INTEGRITY=y
+CONFIG_BLK_DEV_INTEGRITY_T10=y
 # CONFIG_BLK_DEV_ZONED is not set
 CONFIG_BLK_DEV_THROTTLING=y
 # CONFIG_BLK_DEV_THROTTLING_LOW is not set
 CONFIG_BLK_CMDLINE_PARSER=y
 CONFIG_BLK_WBT=y
+CONFIG_BLK_WBT_MQ=y
 # CONFIG_BLK_CGROUP_IOLATENCY is not set
 # CONFIG_BLK_CGROUP_IOCOST is not set
-CONFIG_BLK_WBT_MQ=y
+# CONFIG_BLK_CGROUP_IOPRIO is not set
 CONFIG_BLK_DEBUG_FS=y
 # CONFIG_BLK_SED_OPAL is not set
 # CONFIG_BLK_INLINE_ENCRYPTION is not set
@@ -859,7 +863,6 @@ CONFIG_COREDUMP=y
 CONFIG_SELECT_MEMORY_MODEL=y
 CONFIG_SPARSEMEM_MANUAL=y
 CONFIG_SPARSEMEM=y
-CONFIG_NEED_MULTIPLE_NODES=y
 CONFIG_SPARSEMEM_EXTREME=y
 CONFIG_SPARSEMEM_VMEMMAP_ENABLE=y
 CONFIG_SPARSEMEM_VMEMMAP=y
@@ -920,6 +923,9 @@ CONFIG_GENERIC_EARLY_IOREMAP=y
 # CONFIG_IDLE_PAGE_TRACKING is not set
 CONFIG_ARCH_HAS_CACHE_LINE_SIZE=y
 CONFIG_ARCH_HAS_PTE_DEVMAP=y
+CONFIG_ARCH_HAS_ZONE_DMA_SET=y
+CONFIG_ZONE_DMA=y
+CONFIG_ZONE_DMA32=y
 CONFIG_ZONE_DEVICE=y
 CONFIG_DEV_PAGEMAP_OPS=y
 # CONFIG_DEVICE_PRIVATE is not set
@@ -927,6 +933,7 @@ CONFIG_PERCPU_STATS=y
 # CONFIG_GUP_TEST is not set
 # CONFIG_READ_ONLY_THP_FOR_FS is not set
 CONFIG_ARCH_HAS_PTE_SPECIAL=y
+CONFIG_SECRETMEM=y
 # end of Memory Management options
 
 CONFIG_NET=y
@@ -1218,7 +1225,9 @@ CONFIG_VIRTIO_BLK=y
 #
 # NVME Support
 #
-# CONFIG_BLK_DEV_NVME is not set
+CONFIG_NVME_CORE=y
+CONFIG_BLK_DEV_NVME=y
+# CONFIG_NVME_MULTIPATH is not set
 # CONFIG_NVME_FC is not set
 # CONFIG_NVME_TCP is not set
 # CONFIG_NVME_TARGET is not set
@@ -1250,6 +1259,7 @@ CONFIG_VIRTIO_BLK=y
 #
 # Texas Instruments shared transport line discipline
 #
+# CONFIG_TI_ST is not set
 # end of Texas Instruments shared transport line discipline
 
 #
@@ -1269,9 +1279,6 @@ CONFIG_VIRTIO_BLK=y
 # CONFIG_PVPANIC is not set
 # end of Misc devices
 
-CONFIG_HAVE_IDE=y
-# CONFIG_IDE is not set
-
 #
 # SCSI device support
 #
@@ -1334,7 +1341,13 @@ CONFIG_VIRTIO_NET=y
 #
 # CONFIG_WLAN is not set
 # CONFIG_WAN is not set
+
+#
+# Wireless WAN
+#
 # CONFIG_WWAN is not set
+# end of Wireless WAN
+
 # CONFIG_VMXNET3 is not set
 # CONFIG_FUJITSU_ES is not set
 # CONFIG_HYPERV_NET is not set
@@ -1368,6 +1381,9 @@ CONFIG_INPUT_EVDEV=y
 CONFIG_INPUT_KEYBOARD=y
 # CONFIG_KEYBOARD_ATKBD is not set
 # CONFIG_KEYBOARD_LKKBD is not set
+# CONFIG_KEYBOARD_GPIO is not set
+# CONFIG_KEYBOARD_GPIO_POLLED is not set
+# CONFIG_KEYBOARD_MATRIX is not set
 # CONFIG_KEYBOARD_NEWTON is not set
 # CONFIG_KEYBOARD_OPENCORES is not set
 # CONFIG_KEYBOARD_SAMSUNG is not set
@@ -1382,8 +1398,12 @@ CONFIG_INPUT_MISC=y
 # CONFIG_INPUT_AD714X is not set
 # CONFIG_INPUT_E3X0_BUTTON is not set
 # CONFIG_INPUT_PCSPKR is not set
+# CONFIG_INPUT_GPIO_BEEPER is not set
+# CONFIG_INPUT_GPIO_DECODER is not set
+# CONFIG_INPUT_GPIO_VIBRA is not set
 # CONFIG_INPUT_ATLAS_BTNS is not set
 CONFIG_INPUT_UINPUT=y
+# CONFIG_INPUT_GPIO_ROTARY_ENCODER is not set
 # CONFIG_INPUT_ADXL34X is not set
 # CONFIG_INPUT_CMA3000 is not set
 # CONFIG_RMI4_CORE is not set
@@ -1454,6 +1474,7 @@ CONFIG_SERIAL_ARC_NR_PORTS=1
 # CONFIG_SERIAL_SPRD is not set
 # end of Serial drivers
 
+CONFIG_SERIAL_MCTRL_GPIO=y
 # CONFIG_SERIAL_NONSTANDARD is not set
 # CONFIG_N_GSM is not set
 # CONFIG_NOZOMI is not set
@@ -1476,8 +1497,6 @@ CONFIG_HW_RANDOM_VIRTIO=y
 # CONFIG_MWAVE is not set
 CONFIG_DEVMEM=y
 CONFIG_NVRAM=y
-CONFIG_RAW_DRIVER=y
-CONFIG_MAX_RAW_DEVS=256
 CONFIG_DEVPORT=y
 # CONFIG_HPET is not set
 CONFIG_HANGCHECK_TIMER=y
@@ -1527,7 +1546,60 @@ CONFIG_PTP_1588_CLOCK_KVM=y
 # end of PTP clock support
 
 # CONFIG_PINCTRL is not set
-# CONFIG_GPIOLIB is not set
+CONFIG_GPIOLIB=y
+CONFIG_GPIOLIB_FASTPATH_LIMIT=512
+CONFIG_GPIO_ACPI=y
+CONFIG_GPIOLIB_IRQCHIP=y
+# CONFIG_DEBUG_GPIO is not set
+CONFIG_GPIO_SYSFS=y
+CONFIG_GPIO_CDEV=y
+CONFIG_GPIO_CDEV_V1=y
+
+#
+# Memory mapped GPIO drivers
+#
+# CONFIG_GPIO_AMDPT is not set
+# CONFIG_GPIO_DWAPB is not set
+# CONFIG_GPIO_EXAR is not set
+# CONFIG_GPIO_GENERIC_PLATFORM is not set
+# CONFIG_GPIO_MB86S7X is not set
+# CONFIG_GPIO_VX855 is not set
+# CONFIG_GPIO_AMD_FCH is not set
+# end of Memory mapped GPIO drivers
+
+#
+# Port-mapped I/O GPIO drivers
+#
+# CONFIG_GPIO_F7188X is not set
+# CONFIG_GPIO_IT87 is not set
+# CONFIG_GPIO_SCH311X is not set
+# CONFIG_GPIO_WINBOND is not set
+# CONFIG_GPIO_WS16C48 is not set
+# end of Port-mapped I/O GPIO drivers
+
+#
+# MFD GPIO expanders
+#
+# end of MFD GPIO expanders
+
+#
+# PCI GPIO expanders
+#
+# CONFIG_GPIO_AMD8111 is not set
+# CONFIG_GPIO_BT8XX is not set
+# CONFIG_GPIO_ML_IOH is not set
+CONFIG_GPIO_PCI_IDIO_16=y
+# CONFIG_GPIO_PCIE_IDIO_24 is not set
+# CONFIG_GPIO_RDC321X is not set
+# end of PCI GPIO expanders
+
+#
+# Virtual GPIO drivers
+#
+# CONFIG_GPIO_AGGREGATOR is not set
+# CONFIG_GPIO_MOCKUP is not set
+# end of Virtual GPIO drivers
+
 # CONFIG_W1 is not set
 # CONFIG_POWER_RESET is not set
 # CONFIG_POWER_SUPPLY is not set
@@ -1570,6 +1642,7 @@ CONFIG_WATCHDOG_CORE=y
 CONFIG_WATCHDOG_HANDLE_BOOT_ENABLED=y
 CONFIG_WATCHDOG_OPEN_TIMEOUT=0
 # CONFIG_WATCHDOG_SYSFS is not set
+# CONFIG_WATCHDOG_HRTIMER_PRETIMEOUT is not set
 
 #
 # Watchdog Pretimeout Governors
@@ -1619,6 +1692,7 @@ CONFIG_WATCHDOG_OPEN_TIMEOUT=0
 # CONFIG_SBC_EPX_C3_WATCHDOG is not set
 # CONFIG_NI903X_WDT is not set
 # CONFIG_NIC7018_WDT is not set
+# CONFIG_MEN_A21_WDT is not set
 CONFIG_VIRTIO_WDT=y
 
 #
@@ -1736,9 +1810,11 @@ CONFIG_FB=y
 CONFIG_LCD_CLASS_DEVICE=y
 # CONFIG_LCD_PLATFORM is not set
 CONFIG_BACKLIGHT_CLASS_DEVICE=y
+# CONFIG_BACKLIGHT_KTD253 is not set
 # CONFIG_BACKLIGHT_APPLE is not set
 # CONFIG_BACKLIGHT_QCOM_WLED is not set
 # CONFIG_BACKLIGHT_SAHARA is not set
+# CONFIG_BACKLIGHT_GPIO is not set
 # end of Backlight & LCD device support
 
 #
@@ -1878,7 +1954,6 @@ CONFIG_DMA_ACPI=y
 # CONFIG_INTEL_IDXD is not set
 # CONFIG_INTEL_IOATDMA is not set
 # CONFIG_PLX_DMA is not set
-# CONFIG_XILINX_ZYNQMP_DPDMA is not set
 # CONFIG_QCOM_HIDMA_MGMT is not set
 # CONFIG_QCOM_HIDMA is not set
 CONFIG_DW_DMAC_CORE=y
@@ -1962,9 +2037,16 @@ CONFIG_PMC_ATOM=y
 # CONFIG_MELLANOX_PLATFORM is not set
 # CONFIG_SURFACE_PLATFORMS is not set
 CONFIG_HAVE_CLK=y
-CONFIG_CLKDEV_LOOKUP=y
 CONFIG_HAVE_CLK_PREPARE=y
 CONFIG_COMMON_CLK=y
+
+#
+# Clock driver for ARM Reference designs
+#
+# CONFIG_ICST is not set
+# CONFIG_CLK_SP810 is not set
+# end of Clock driver for ARM Reference designs
+
 # CONFIG_XILINX_VCU is not set
 # CONFIG_HWSPINLOCK is not set
 
@@ -2074,6 +2156,7 @@ CONFIG_VIRTIO_IOMMU=y
 # PHY Subsystem
 #
 # CONFIG_GENERIC_PHY is not set
+# CONFIG_PHY_CAN_TRANSCEIVER is not set
 # CONFIG_BCM_KONA_USB2_PHY is not set
 # CONFIG_PHY_PXA_28NM_HSIC is not set
 # CONFIG_PHY_PXA_28NM_USB2 is not set
@@ -2232,6 +2315,8 @@ CONFIG_TMPFS_XATTR=y
 # CONFIG_TMPFS_INODE64 is not set
 CONFIG_HUGETLBFS=y
 CONFIG_HUGETLB_PAGE=y
+CONFIG_HUGETLB_PAGE_FREE_VMEMMAP=y
+# CONFIG_HUGETLB_PAGE_FREE_VMEMMAP_DEFAULT_ON is not set
 CONFIG_MEMFD_CREATE=y
 CONFIG_ARCH_HAS_GIGANTIC_PAGE=y
 CONFIG_CONFIGFS_FS=y
@@ -2621,7 +2706,6 @@ CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE=y
 CONFIG_ARCH_HAS_COPY_MC=y
 CONFIG_ARCH_STACKWALK=y
 CONFIG_SBITMAP=y
-# CONFIG_STRING_SELFTEST is not set
 # end of Library routines
 
 #
@@ -2633,6 +2717,7 @@ CONFIG_SBITMAP=y
 #
 CONFIG_PRINTK_TIME=y
 # CONFIG_PRINTK_CALLER is not set
+# CONFIG_STACKTRACE_BUILD_ID is not set
 CONFIG_CONSOLE_LOGLEVEL_DEFAULT=7
 CONFIG_CONSOLE_LOGLEVEL_QUIET=4
 CONFIG_MESSAGE_LOGLEVEL_DEFAULT=4
@@ -2653,7 +2738,7 @@ CONFIG_STRIP_ASM_SYMS=y
 # CONFIG_HEADERS_INSTALL is not set
 CONFIG_DEBUG_SECTION_MISMATCH=y
 CONFIG_SECTION_MISMATCH_WARN_ONLY=y
-# CONFIG_DEBUG_FORCE_FUNCTION_ALIGN_32B is not set
+# CONFIG_DEBUG_FORCE_FUNCTION_ALIGN_64B is not set
 CONFIG_FRAME_POINTER=y
 CONFIG_STACK_VALIDATION=y
 # CONFIG_VMLINUX_MAP is not set
@@ -2854,7 +2939,6 @@ CONFIG_CC_HAS_SANCOV_TRACE_PC=y
 # CONFIG_KCOV is not set
 CONFIG_RUNTIME_TESTING_MENU=y
 # CONFIG_LKDTM is not set
-# CONFIG_TEST_LIST_SORT is not set
 # CONFIG_TEST_MIN_HEAP is not set
 # CONFIG_TEST_SORT is not set
 # CONFIG_TEST_DIV64 is not set
@@ -2864,10 +2948,12 @@ CONFIG_RUNTIME_TESTING_MENU=y
 # CONFIG_INTERVAL_TREE_TEST is not set
 # CONFIG_ATOMIC64_SELFTEST is not set
 # CONFIG_TEST_HEXDUMP is not set
+# CONFIG_STRING_SELFTEST is not set
 # CONFIG_TEST_STRING_HELPERS is not set
 # CONFIG_TEST_STRSCPY is not set
 # CONFIG_TEST_KSTRTOX is not set
 # CONFIG_TEST_PRINTF is not set
+# CONFIG_TEST_SCANF is not set
 # CONFIG_TEST_BITMAP is not set
 # CONFIG_TEST_UUID is not set
 # CONFIG_TEST_XARRAY is not set
@@ -2884,6 +2970,7 @@ CONFIG_RUNTIME_TESTING_MENU=y
 # CONFIG_TEST_MEMINIT is not set
 # CONFIG_TEST_FREE_PAGES is not set
 # CONFIG_TEST_FPU is not set
+# CONFIG_TEST_CLOCKSOURCE_WATCHDOG is not set
 CONFIG_ARCH_USE_MEMTEST=y
 # CONFIG_MEMTEST is not set
 # CONFIG_HYPERV_TESTING is not set

rpm/cloud-hypervisor.spec

@@ -1,6 +1,6 @@
 Name:           cloud-hypervisor
 Summary:        Cloud Hypervisor is an open source Virtual Machine Monitor (VMM) that runs on top of KVM.
-Version:        15.0
+Version:        18.0
 Release:        0%{?dist}
 License:        ASL 2.0 or BSD-3-clause
 Group:          Applications/System

scripts/dev_cli.sh

@@ -178,16 +178,17 @@ cmd_help() {
     echo ""
     echo "    tests [--unit|--cargo|--all] [--libc musl|gnu] [-- [<cargo test args>]]"
     echo "        Run the Cloud Hypervisor tests."
-    echo "        --unit                Run the unit tests."
-    echo "        --cargo               Run the cargo tests."
-    echo "        --integration         Run the integration tests."
-    echo "        --integration-sgx     Run the SGX integration tests."
-    echo "        --integration-vfio    Run the VFIO integration tests."
-    echo "        --integration-windows Run the Windows guest integration tests."
-    echo "        --libc                Select the C library Cloud Hypervisor will be built against. Default is gnu"
-    echo "        --volumes             Hash separated volumes to be exported. Example --volumes /mnt:/mnt#/myvol:/myvol"
-    echo "        --hypervisor          Underlying hypervisor. Options kvm, mshv"
-    echo "        --all                 Run all tests."
+    echo "        --unit                       Run the unit tests."
+    echo "        --cargo                      Run the cargo tests."
+    echo "        --integration                Run the integration tests."
+    echo "        --integration-sgx            Run the SGX integration tests."
+    echo "        --integration-vfio           Run the VFIO integration tests."
+    echo "        --integration-windows        Run the Windows guest integration tests."
+    echo "        --integration-live-migration Run the live-migration integration tests."
+    echo "        --libc                       Select the C library Cloud Hypervisor will be built against. Default is gnu"
+    echo "        --volumes                    Hash separated volumes to be exported. Example --volumes /mnt:/mnt#/myvol:/myvol"
+    echo "        --hypervisor                 Underlying hypervisor. Options kvm, mshv"
+    echo "        --all                        Run all tests."
     echo ""
     echo "    build-container [--type]"
     echo "        Build the Cloud Hypervisor container."
@@ -239,9 +240,7 @@ cmd_build() {
     done
 
     ensure_build_dir
-    if [ $(uname -m) = "x86_64" ]; then
-	ensure_latest_ctr
-    fi
+    ensure_latest_ctr
 
     process_volumes_args
     if [[ "$hypervisor" != "kvm" ]]; then
@@ -280,9 +279,7 @@ cmd_clean() {
     cargo_args=("$@")
 
     ensure_build_dir
-    if [ $(uname -m) = "x86_64" ]; then
-	ensure_latest_ctr
-    fi
+    ensure_latest_ctr
 
     $DOCKER_RUNTIME run \
 	   --user "$(id -u):$(id -g)" \
@@ -302,19 +299,21 @@ cmd_tests() {
     integration_sgx=false
     integration_vfio=false
     integration_windows=false
+    integration_live_migration=false
     libc="gnu"
     arg_vols=""
     hypervisor="kvm"
     exported_device="/dev/kvm"
     while [ $# -gt 0 ]; do
 	case "$1" in
-            "-h"|"--help")           { cmd_help; exit 1; } ;;
-            "--unit")                { unit=true; } ;;
-            "--cargo")               { cargo=true; } ;;
-            "--integration")         { integration=true; } ;;
-            "--integration-sgx")     { integration_sgx=true; } ;;
-            "--integration-vfio")    { integration_vfio=true; } ;;
-            "--integration-windows") { integration_windows=true; } ;;
+            "-h"|"--help")                  { cmd_help; exit 1; } ;;
+            "--unit")                       { unit=true; } ;;
+            "--cargo")                      { cargo=true; } ;;
+            "--integration")                { integration=true; } ;;
+            "--integration-sgx")            { integration_sgx=true; } ;;
+            "--integration-vfio")           { integration_vfio=true; } ;;
+            "--integration-windows")        { integration_windows=true; } ;;
+            "--integration-live-migration") { integration_live_migration=true; } ;;
             "--libc")
                 shift
                 [[ "$1" =~ ^(musl|gnu)$ ]] || \
@@ -346,9 +345,7 @@ cmd_tests() {
     set -- "$@" '--hypervisor' $hypervisor
 
     ensure_build_dir
-    if [ $(uname -m) = "x86_64" ]; then
-	ensure_latest_ctr
-    fi
+    ensure_latest_ctr
 
     process_volumes_args
     target="$(uname -m)-unknown-linux-${libc}"
@@ -460,6 +457,25 @@ cmd_tests() {
 	       "$CTR_IMAGE" \
 	       ./scripts/run_integration_tests_windows.sh "$@" || fix_dir_perms $? || exit $?
     fi
+
+    if [ "$integration_live_migration" = true ] ;  then
+	say "Running 'live migration' integration tests for $target..."
+	$DOCKER_RUNTIME run \
+	       --workdir "$CTR_CLH_ROOT_DIR" \
+	       --rm \
+	       --privileged \
+	       --security-opt seccomp=unconfined \
+	       --ipc=host \
+	       --net="$CTR_CLH_NET" \
+	       --mount type=tmpfs,destination=/tmp \
+	       --volume /dev:/dev \
+	       --volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR"  $exported_volumes \
+	       --volume "$CLH_INTEGRATION_WORKLOADS:$CTR_CLH_INTEGRATION_WORKLOADS" \
+	       --env USER="root" \
+	       --env CH_LIBC="${libc}" \
+	       "$CTR_IMAGE" \
+	       ./scripts/run_integration_tests_live_migration.sh "$@" || fix_dir_perms $? || exit $?
+    fi
     fix_dir_perms $?
 }
 
@@ -479,9 +495,7 @@ cmd_build-container() {
     done
 
     ensure_build_dir
-    if [ $(uname -m) = "x86_64" ]; then
-	ensure_latest_ctr
-    fi
+    ensure_latest_ctr
 
     BUILD_DIR=/tmp/cloud-hypervisor/container/
 
@@ -514,9 +528,7 @@ cmd_shell() {
 	shift
     done
     ensure_build_dir
-    if [ $(uname -m) = "x86_64" ]; then
-	ensure_latest_ctr
-    fi
+    ensure_latest_ctr
     process_volumes_args
     say_warn "Starting a privileged shell prompt as root ..."
     say_warn "WARNING: Your $CLH_ROOT_DIR folder will be bind-mounted in the container under $CTR_CLH_ROOT_DIR"

scripts/run_integration_tests_aarch64.sh

@@ -171,7 +171,7 @@ update_workloads() {
     }
 
     SRCDIR=$PWD
-    LINUX_CUSTOM_BRANCH="ch-5.12"
+    LINUX_CUSTOM_BRANCH="ch-5.14"
 
     # Check whether the local HEAD commit same as the remote HEAD or not. Remove the folder if they are different.
     if [ -d "$LINUX_CUSTOM_DIR" ]; then
@@ -249,7 +249,7 @@ process_common_args "$@"
 
 # aarch64 not supported for MSHV
 if [[ "$hypervisor" = "mshv" ]]; then
-    echo "Aarch64 is not supported in Microsoft Hypervisor"
+    echo "AArch64 is not supported in Microsoft Hypervisor"
     exit 1
 fi
 
@@ -299,7 +299,33 @@ ovs-vsctl init
 ovs-vsctl set Open_vSwitch . other_config:dpdk-init=true
 service openvswitch-switch restart
 
+# Run all direct kernel boot (Device Tree) test cases in mod `parallel`
 time cargo test $features_test "tests::parallel::$test_filter"
 RES=$?
 
+# Run some tests in sequence since the result could be affected by other tests
+# running in parallel.
+if [ $RES -eq 0 ]; then
+    time cargo test $features_test "tests::sequential::$test_filter" -- --test-threads=1
+    RES=$?
+else
+    exit $RES
+fi
+
+# Run all ACPI test cases
+if [ $RES -eq 0 ]; then
+    time cargo test $features_test "tests::aarch64_acpi::$test_filter"
+    RES=$?
+else
+    exit $RES
+fi
+
+# Run all test cases related to live migration
+if [ $RES -eq 0 ]; then
+    time cargo test $features_test "tests::live_migration::$test_filter" -- --test-threads=1
+    RES=$?
+else
+    exit $RES
+fi
+
 exit $RES

scripts/run_integration_tests_live_migration.sh

@@ -0,0 +1,87 @@
+#!/bin/bash
+set -x
+
+source $HOME/.cargo/env
+source $(dirname "$0")/test-util.sh
+
+export BUILD_TARGET=${BUILD_TARGET-x86_64-unknown-linux-gnu}
+
+WORKLOADS_DIR="$HOME/workloads"
+mkdir -p "$WORKLOADS_DIR"
+
+process_common_args "$@"
+
+# For now these values are deafult for kvm
+features_build=""
+features_test="--features integration_tests"
+
+cp scripts/sha1sums-x86_64 $WORKLOADS_DIR
+
+FOCAL_OS_IMAGE_NAME="focal-server-cloudimg-amd64-custom-20210609-0.qcow2"
+FOCAL_OS_IMAGE_URL="https://cloud-hypervisor.azureedge.net/$FOCAL_OS_IMAGE_NAME"
+FOCAL_OS_IMAGE="$WORKLOADS_DIR/$FOCAL_OS_IMAGE_NAME"
+if [ ! -f "$FOCAL_OS_IMAGE" ]; then
+    pushd $WORKLOADS_DIR
+    time wget --quiet $FOCAL_OS_IMAGE_URL || exit 1
+    popd
+fi
+
+FOCAL_OS_RAW_IMAGE_NAME="focal-server-cloudimg-amd64-custom-20210609-0.raw"
+FOCAL_OS_RAW_IMAGE="$WORKLOADS_DIR/$FOCAL_OS_RAW_IMAGE_NAME"
+if [ ! -f "$FOCAL_OS_RAW_IMAGE" ]; then
+    pushd $WORKLOADS_DIR
+    time qemu-img convert -p -f qcow2 -O raw $FOCAL_OS_IMAGE_NAME $FOCAL_OS_RAW_IMAGE_NAME || exit 1
+    popd
+fi
+
+pushd $WORKLOADS_DIR
+grep focal sha1sums-x86_64 | sha1sum --check
+if [ $? -ne 0 ]; then
+    echo "sha1sum validation of images failed, remove invalid images to fix the issue."
+    exit 1
+fi
+popd
+
+# Build custom kernel based on virtio-pmem and virtio-fs upstream patches
+VMLINUX_IMAGE="$WORKLOADS_DIR/vmlinux"
+
+LINUX_CUSTOM_DIR="$WORKLOADS_DIR/linux-custom"
+
+if [ ! -f "$VMLINUX_IMAGE" ]; then
+    SRCDIR=$PWD
+    pushd $WORKLOADS_DIR
+    time git clone --depth 1 "https://github.com/cloud-hypervisor/linux.git" -b "ch-5.14" $LINUX_CUSTOM_DIR
+    cp $SRCDIR/resources/linux-config-x86_64 $LINUX_CUSTOM_DIR/.config
+    popd
+fi
+
+if [ ! -f "$VMLINUX_IMAGE" ]; then
+    pushd $LINUX_CUSTOM_DIR
+    time make bzImage -j `nproc`
+    cp vmlinux $VMLINUX_IMAGE || exit 1
+    popd
+fi
+
+if [ -d "$LINUX_CUSTOM_DIR" ]; then
+    rm -rf $LINUX_CUSTOM_DIR
+fi
+
+BUILD_TARGET="$(uname -m)-unknown-linux-${CH_LIBC}"
+CFLAGS=""
+TARGET_CC=""
+if [[ "${BUILD_TARGET}" == "x86_64-unknown-linux-musl" ]]; then
+    TARGET_CC="musl-gcc"
+    CFLAGS="-I /usr/include/x86_64-linux-musl/ -idirafter /usr/include/"
+fi
+
+cargo build --all --release $features_build --target $BUILD_TARGET
+strip target/$BUILD_TARGET/release/cloud-hypervisor
+strip target/$BUILD_TARGET/release/vhost_user_net
+strip target/$BUILD_TARGET/release/ch-remote
+
+export RUST_BACKTRACE=1
+
+time cargo test $features_test "tests::live_migration::$test_filter" -- --test-threads=1
+RES=$?
+
+exit $RES

scripts/run_integration_tests_x86_64.sh

@@ -102,7 +102,7 @@ LINUX_CUSTOM_DIR="$WORKLOADS_DIR/linux-custom"
 if [ ! -f "$VMLINUX_IMAGE" ]; then
     SRCDIR=$PWD
     pushd $WORKLOADS_DIR
-    time git clone --depth 1 "https://github.com/cloud-hypervisor/linux.git" -b "ch-5.13-rc5" $LINUX_CUSTOM_DIR
+    time git clone --depth 1 "https://github.com/cloud-hypervisor/linux.git" -b "ch-5.14" $LINUX_CUSTOM_DIR
     cp $SRCDIR/resources/linux-config-x86_64 $LINUX_CUSTOM_DIR/.config
     popd
 fi

src/bin/ch-remote.rs

@@ -27,6 +27,7 @@ enum Error {
     AddFsConfig(vmm::config::Error),
     AddPmemConfig(vmm::config::Error),
     AddNetConfig(vmm::config::Error),
+    AddUserDeviceConfig(vmm::config::Error),
     AddVsockConfig(vmm::config::Error),
     Restore(vmm::config::Error),
 }
@@ -45,6 +46,7 @@ impl fmt::Display for Error {
             AddFsConfig(e) => write!(f, "Error parsing filesystem syntax: {}", e),
             AddPmemConfig(e) => write!(f, "Error parsing persistent memory syntax: {}", e),
             AddNetConfig(e) => write!(f, "Error parsing network syntax: {}", e),
+            AddUserDeviceConfig(e) => write!(f, "Error parsing user device syntax: {}", e),
             AddVsockConfig(e) => write!(f, "Error parsing vsock syntax: {}", e),
             Restore(e) => write!(f, "Error parsing restore syntax: {}", e),
         }
@@ -130,6 +132,19 @@ fn add_device_api_command(socket: &mut UnixStream, config: &str) -> Result<(), E
     .map_err(Error::ApiClient)
 }
 
+fn add_user_device_api_command(socket: &mut UnixStream, config: &str) -> Result<(), Error> {
+    let device_config =
+        vmm::config::UserDeviceConfig::parse(config).map_err(Error::AddUserDeviceConfig)?;
+
+    simple_api_command(
+        socket,
+        "PUT",
+        "add-user-device",
+        Some(&serde_json::to_string(&device_config).unwrap()),
+    )
+    .map_err(Error::ApiClient)
+}
+
 fn remove_device_api_command(socket: &mut UnixStream, id: &str) -> Result<(), Error> {
     let remove_device_data = vmm::api::VmRemoveDeviceData { id: id.to_owned() };
 
@@ -348,6 +363,14 @@ fn do_command(matches: &ArgMatches) -> Result<(), Error> {
                 .value_of("net_config")
                 .unwrap(),
         ),
+        Some("add-user-device") => add_user_device_api_command(
+            &mut socket,
+            matches
+                .subcommand_matches("add-user-device")
+                .unwrap()
+                .value_of("device_config")
+                .unwrap(),
+        ),
         Some("add-vsock") => add_vsock_api_command(
             &mut socket,
             matches
@@ -451,6 +474,15 @@ fn main() {
                         .help(vmm::config::NetConfig::SYNTAX),
                 ),
         )
+        .subcommand(
+            SubCommand::with_name("add-user-device")
+                .about("Add userspace device")
+                .arg(
+                    Arg::with_name("device_config")
+                        .index(1)
+                        .help(vmm::config::UserDeviceConfig::SYNTAX),
+                ),
+        )
         .subcommand(
             SubCommand::with_name("add-vsock")
                 .about("Add vsock device")

src/main.rs

@@ -12,7 +12,7 @@ use clap::{App, Arg, ArgGroup, ArgMatches};
 use libc::EFD_NONBLOCK;
 use log::LevelFilter;
 use option_parser::OptionParser;
-use seccomp::SeccompAction;
+use seccompiler::SeccompAction;
 use signal_hook::{
     consts::SIGSYS,
     iterator::{exfiltrator::WithRawSiginfo, SignalsInfo},
@@ -26,6 +26,7 @@ use std::thread;
 use thiserror::Error;
 use vmm::config;
 use vmm_sys_util::eventfd::EventFd;
+use vmm_sys_util::signal::block_signal;
 
 #[derive(Error, Debug)]
 enum Error {
@@ -279,6 +280,14 @@ fn create_app<'a, 'b>(
                 .min_values(1)
                 .group("vm-config"),
         )
+        .arg(
+            Arg::with_name("user-device")
+                .long("user-device")
+                .help(config::UserDeviceConfig::SYNTAX)
+                .takes_value(true)
+                .min_values(1)
+                .group("vm-config"),
+        )
         .arg(
             Arg::with_name("vsock")
                 .long("vsock")
@@ -488,6 +497,13 @@ fn start_vmm(cmd_arguments: ArgMatches) -> Result<Option<String>, Error> {
             .unwrap();
     }
 
+    // Before we start any threads, mask the signals we'll be
+    // installing handlers for, to make sure they only ever run on the
+    // dedicated signal handling thread we'll start in a bit.
+    for sig in vmm::vm::HANDLED_SIGNALS {
+        block_signal(sig).unwrap();
+    }
+
     event!("vmm", "starting");
 
     let hypervisor = hypervisor::new().map_err(Error::CreateHypervisor)?;
@@ -656,6 +672,7 @@ mod unit_tests {
                     iommu: false,
                 },
                 devices: None,
+                user_devices: None,
                 vsock: None,
                 iommu: false,
                 #[cfg(target_arch = "x86_64")]

test_infra/src/lib.rs

@@ -226,7 +226,7 @@ impl DiskConfig for UbuntuDiskConfig {
             .expect("Expected reading user-data file in to succeed");
         user_data_string = user_data_string.replace(
             "@DEFAULT_TCP_LISTENER_MESSAGE",
-            &DEFAULT_TCP_LISTENER_MESSAGE,
+            DEFAULT_TCP_LISTENER_MESSAGE,
         );
         user_data_string = user_data_string.replace("@HOST_IP", &network.host_ip);
         user_data_string =
@@ -234,7 +234,7 @@ impl DiskConfig for UbuntuDiskConfig {
 
         fs::File::create(cloud_init_directory.join("user-data"))
             .unwrap()
-            .write_all(&user_data_string.as_bytes())
+            .write_all(user_data_string.as_bytes())
             .expect("Expected writing out user-data to succeed");
 
         let mut network_config_string = String::new();
@@ -260,7 +260,7 @@ impl DiskConfig for UbuntuDiskConfig {
 
         fs::File::create(cloud_init_directory.join("network-config"))
             .unwrap()
-            .write_all(&network_config_string.as_bytes())
+            .write_all(network_config_string.as_bytes())
             .expect("Expected writing out network-config to succeed");
 
         std::process::Command::new("mkdosfs")

vfio_user/Cargo.toml

@@ -0,0 +1,20 @@
+[package]
+name = "vfio_user"
+version = "0.1.0"
+authors = ["The Cloud Hypervisor Authors"]
+edition = "2018"
+
+[dependencies]
+anyhow = "1.0.43"
+libc = "0.2.101"
+log = "0.4.14"
+serde = {version = ">=1.0.27", features = ["rc"] }
+serde_derive = ">=1.0.27"
+serde_json = ">=1.0.9"
+thiserror = "1.0.29"
+vm-memory = { version = "0.6.0", features = ["backend-mmap", "backend-atomic"] }
+vmm-sys-util = ">=0.3.1"
+
+[dependencies.vfio-bindings]
+version = "0.2.0"
+features = ["fam-wrappers"]

vfio_user/src/lib.rs

@@ -0,0 +1,675 @@
+// Copyright © 2021 Intel Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+use std::ffi::CString;
+use std::io::{IoSlice, Read, Write};
+use std::num::Wrapping;
+use std::os::unix::net::UnixStream;
+use std::os::unix::prelude::RawFd;
+use std::path::Path;
+use thiserror::Error;
+use vfio_bindings::bindings::vfio::*;
+use vm_memory::{ByteValued, FileOffset};
+use vmm_sys_util::sock_ctrl_msg::ScmSocket;
+
+#[macro_use]
+extern crate serde_derive;
+
+#[macro_use]
+extern crate log;
+
+#[allow(dead_code)]
+#[repr(u16)]
+#[derive(Clone, Copy, Debug)]
+enum Command {
+    Unknown = 0,
+    Version = 1,
+    DmaMap = 2,
+    DmaUnmap = 3,
+    DeviceGetInfo = 4,
+    DeviceGetRegionInfo = 5,
+    GetRegionIoFds = 6,
+    GetIrqInfo = 7,
+    SetIrqs = 8,
+    RegionRead = 9,
+    RegionWrite = 10,
+    DmaRead = 11,
+    DmaWrite = 12,
+    DeviceReset = 13,
+    UserDirtyPages = 14,
+}
+
+impl Default for Command {
+    fn default() -> Self {
+        Command::Unknown
+    }
+}
+
+#[allow(dead_code)]
+#[repr(u32)]
+#[derive(Clone, Copy, Debug, PartialEq)]
+enum HeaderFlags {
+    Command = 0,
+    Reply = 1,
+    NoReply = 1 << 4,
+    Error = 1 << 5,
+}
+
+impl Default for HeaderFlags {
+    fn default() -> Self {
+        HeaderFlags::Command
+    }
+}
+
+#[repr(C)]
+#[derive(Default, Clone, Copy, Debug)]
+struct Header {
+    message_id: u16,
+    command: Command,
+    message_size: u32,
+    flags: u32,
+    error: u32,
+}
+
+unsafe impl ByteValued for Header {}
+
+#[repr(C)]
+#[derive(Default, Clone, Copy, Debug)]
+struct Version {
+    header: Header,
+    major: u16,
+    minor: u16,
+}
+unsafe impl ByteValued for Version {}
+
+#[derive(Serialize, Deserialize, Debug)]
+struct MigrationCapabilities {
+    pgsize: u32,
+}
+
+const fn default_max_msg_fds() -> u32 {
+    1
+}
+
+const fn default_max_data_xfer_size() -> u32 {
+    1048576
+}
+
+const fn default_migration_capabilities() -> MigrationCapabilities {
+    MigrationCapabilities { pgsize: 4096 }
+}
+
+#[repr(u32)]
+#[derive(Clone, Copy, Debug)]
+#[allow(dead_code)]
+enum DmaMapFlags {
+    Unknown = 0,
+    ReadOnly = 1,
+    WriteOnly = 2,
+    ReadWrite = 3,
+}
+
+impl Default for DmaMapFlags {
+    fn default() -> Self {
+        Self::Unknown
+    }
+}
+
+#[repr(C)]
+#[derive(Default, Clone, Copy, Debug)]
+struct DmaMap {
+    header: Header,
+    argsz: u32,
+    flags: DmaMapFlags,
+    offset: u64,
+    address: u64,
+    size: u64,
+}
+
+unsafe impl ByteValued for DmaMap {}
+
+#[repr(C)]
+#[derive(Default, Clone, Copy, Debug)]
+struct DmaUnmap {
+    header: Header,
+    argsz: u32,
+    flags: u32,
+    address: u64,
+    size: u64,
+}
+
+unsafe impl ByteValued for DmaUnmap {}
+
+#[repr(C)]
+#[derive(Default, Clone, Copy, Debug)]
+struct DeviceGetInfo {
+    header: Header,
+    argsz: u32,
+    flags: u32,
+    num_regions: u32,
+    num_irqs: u32,
+}
+
+unsafe impl ByteValued for DeviceGetInfo {}
+
+#[repr(C)]
+#[derive(Default, Clone, Copy, Debug)]
+struct DeviceGetRegionInfo {
+    header: Header,
+    region_info: vfio_region_info,
+}
+
+unsafe impl ByteValued for DeviceGetRegionInfo {}
+
+#[repr(C)]
+#[derive(Default, Clone, Copy, Debug)]
+struct RegionAccess {
+    header: Header,
+    offset: u64,
+    region: u32,
+    count: u32,
+}
+
+unsafe impl ByteValued for RegionAccess {}
+
+#[repr(C)]
+#[derive(Default, Clone, Copy, Debug)]
+struct GetIrqInfo {
+    header: Header,
+    argsz: u32,
+    flags: u32,
+    index: u32,
+    count: u32,
+}
+
+unsafe impl ByteValued for GetIrqInfo {}
+
+#[repr(C)]
+#[derive(Default, Clone, Copy, Debug)]
+struct SetIrqs {
+    header: Header,
+    argsz: u32,
+    flags: u32,
+    index: u32,
+    start: u32,
+    count: u32,
+}
+
+unsafe impl ByteValued for SetIrqs {}
+
+#[repr(C)]
+#[derive(Default, Clone, Copy, Debug)]
+struct DeviceReset {
+    header: Header,
+}
+
+unsafe impl ByteValued for DeviceReset {}
+
+#[derive(Serialize, Deserialize, Debug)]
+struct Capabilities {
+    #[serde(default = "default_max_msg_fds")]
+    max_msg_fds: u32,
+    #[serde(default = "default_max_data_xfer_size")]
+    max_data_xfer_size: u32,
+    #[serde(default = "default_migration_capabilities")]
+    migration: MigrationCapabilities,
+}
+
+impl Default for Capabilities {
+    fn default() -> Self {
+        Self {
+            max_msg_fds: default_max_msg_fds(),
+            max_data_xfer_size: default_max_data_xfer_size(),
+            migration: default_migration_capabilities(),
+        }
+    }
+}
+
+pub struct Client {
+    stream: UnixStream,
+    next_message_id: Wrapping<u16>,
+    num_irqs: u32,
+    resettable: bool,
+    regions: Vec<Region>,
+}
+
+#[derive(Debug)]
+pub struct Region {
+    pub flags: u32,
+    pub index: u32,
+    pub size: u64,
+    pub file_offset: Option<FileOffset>,
+}
+
+#[derive(Debug)]
+pub struct IrqInfo {
+    pub index: u32,
+    pub flags: u32,
+    pub count: u32,
+}
+
+#[derive(Error, Debug)]
+pub enum Error {
+    #[error("Error connecting: {0}")]
+    Connect(#[source] std::io::Error),
+    #[error("Error serializing capabilities: {0}")]
+    SerializeCapabilites(#[source] serde_json::Error),
+    #[error("Error deserializing capabilities: {0}")]
+    DeserializeCapabilites(#[source] serde_json::Error),
+    #[error("Error writing to stream: {0}")]
+    StreamWrite(#[source] std::io::Error),
+    #[error("Error reading from stream: {0}")]
+    StreamRead(#[source] std::io::Error),
+    #[error("Error writing with file descriptors: {0}")]
+    SendWithFd(#[source] vmm_sys_util::errno::Error),
+    #[error("Error reading with file descriptors: {0}")]
+    ReceiveWithFd(#[source] vmm_sys_util::errno::Error),
+    #[error("Not a PCI device")]
+    NotPciDevice,
+}
+
+impl Client {
+    pub fn new(path: &Path) -> Result<Client, Error> {
+        let stream = UnixStream::connect(path).map_err(Error::Connect)?;
+
+        let mut client = Client {
+            next_message_id: Wrapping(0),
+            stream,
+            num_irqs: 0,
+            resettable: false,
+            regions: Vec::new(),
+        };
+
+        client.negotiate_version()?;
+
+        client.regions = client.get_regions()?;
+
+        Ok(client)
+    }
+
+    fn negotiate_version(&mut self) -> Result<(), Error> {
+        let caps = Capabilities::default();
+
+        let version_data = serde_json::to_string(&caps).map_err(Error::SerializeCapabilites)?;
+
+        let version = Version {
+            header: Header {
+                message_id: self.next_message_id.0,
+                command: Command::Version,
+                flags: HeaderFlags::Command as u32,
+                message_size: (std::mem::size_of::<Version>() + version_data.len() + 1) as u32,
+                ..Default::default()
+            },
+            major: 0,
+            minor: 1,
+        };
+        info!("Command: {:?}", version);
+
+        let version_data = CString::new(version_data.as_bytes()).unwrap();
+        let bufs = vec![
+            IoSlice::new(version.as_slice()),
+            IoSlice::new(version_data.as_bytes_with_nul()),
+        ];
+
+        // TODO: Use write_all_vectored() when ready
+        let _ = self
+            .stream
+            .write_vectored(&bufs)
+            .map_err(Error::StreamWrite)?;
+
+        info!(
+            "Sent client version information: major = {} minor = {} capabilities = {:?}",
+            version.major, version.minor, &caps
+        );
+
+        self.next_message_id += Wrapping(1);
+
+        let mut server_version: Version = Version::default();
+        self.stream
+            .read_exact(server_version.as_mut_slice())
+            .map_err(Error::StreamRead)?;
+
+        info!("Reply: {:?}", server_version);
+
+        let mut server_version_data = Vec::new();
+        server_version_data.resize(
+            server_version.header.message_size as usize - std::mem::size_of::<Version>(),
+            0,
+        );
+        self.stream
+            .read_exact(server_version_data.as_mut_slice())
+            .map_err(Error::StreamRead)?;
+
+        let server_caps: Capabilities =
+            serde_json::from_slice(&server_version_data[0..server_version_data.len() - 1])
+                .map_err(Error::DeserializeCapabilites)?;
+
+        info!(
+            "Received server version information: major = {} minor = {} capabilities = {:?}",
+            server_version.major, server_version.minor, &server_caps
+        );
+
+        Ok(())
+    }
+
+    pub fn dma_map(
+        &mut self,
+        offset: u64,
+        address: u64,
+        size: u64,
+        fd: RawFd,
+    ) -> Result<(), Error> {
+        let dma_map = DmaMap {
+            header: Header {
+                message_id: self.next_message_id.0,
+                command: Command::DmaMap,
+                flags: HeaderFlags::Command as u32,
+                message_size: std::mem::size_of::<DmaMap>() as u32,
+                ..Default::default()
+            },
+            argsz: (std::mem::size_of::<DmaMap>() - std::mem::size_of::<Header>()) as u32,
+            flags: DmaMapFlags::ReadWrite,
+            offset,
+            address,
+            size,
+        };
+        info!("Command: {:?}", dma_map);
+        self.next_message_id += Wrapping(1);
+        self.stream
+            .send_with_fd(dma_map.as_slice(), fd)
+            .map_err(Error::SendWithFd)?;
+
+        let mut reply = Header::default();
+        self.stream
+            .read_exact(reply.as_mut_slice())
+            .map_err(Error::StreamRead)?;
+        info!("Reply: {:?}", reply);
+
+        Ok(())
+    }
+
+    pub fn dma_unmap(&mut self, address: u64, size: u64) -> Result<(), Error> {
+        let dma_unmap = DmaUnmap {
+            header: Header {
+                message_id: self.next_message_id.0,
+                command: Command::DmaUnmap,
+                flags: HeaderFlags::Command as u32,
+                message_size: std::mem::size_of::<DmaUnmap>() as u32,
+                ..Default::default()
+            },
+            argsz: (std::mem::size_of::<DmaUnmap>() - std::mem::size_of::<Header>()) as u32,
+            flags: 0,
+            address,
+            size,
+        };
+        info!("Command: {:?}", dma_unmap);
+        self.next_message_id += Wrapping(1);
+        self.stream
+            .write_all(dma_unmap.as_slice())
+            .map_err(Error::StreamWrite)?;
+
+        let mut reply = DmaUnmap::default();
+        self.stream
+            .read_exact(reply.as_mut_slice())
+            .map_err(Error::StreamRead)?;
+        info!("Reply: {:?}", reply);
+
+        Ok(())
+    }
+
+    pub fn reset(&mut self) -> Result<(), Error> {
+        let reset = DeviceReset {
+            header: Header {
+                message_id: self.next_message_id.0,
+                command: Command::DeviceReset,
+                flags: HeaderFlags::Command as u32,
+                message_size: std::mem::size_of::<DeviceReset>() as u32,
+                ..Default::default()
+            },
+        };
+        info!("Command: {:?}", reset);
+        self.next_message_id += Wrapping(1);
+        self.stream
+            .write_all(reset.as_slice())
+            .map_err(Error::StreamWrite)?;
+
+        let mut reply = Header::default();
+        self.stream
+            .read_exact(reply.as_mut_slice())
+            .map_err(Error::StreamRead)?;
+        info!("Reply: {:?}", reply);
+
+        Ok(())
+    }
+
+    fn get_regions(&mut self) -> Result<Vec<Region>, Error> {
+        let get_info = DeviceGetInfo {
+            header: Header {
+                message_id: self.next_message_id.0,
+                command: Command::DeviceGetInfo,
+                flags: HeaderFlags::Command as u32,
+                message_size: std::mem::size_of::<DeviceGetInfo>() as u32,
+                ..Default::default()
+            },
+            argsz: std::mem::size_of::<DeviceGetInfo>() as u32,
+            ..Default::default()
+        };
+        info!("Command: {:?}", get_info);
+        self.next_message_id += Wrapping(1);
+
+        self.stream
+            .write_all(get_info.as_slice())
+            .map_err(Error::StreamWrite)?;
+
+        let mut reply = DeviceGetInfo::default();
+        self.stream
+            .read_exact(reply.as_mut_slice())
+            .map_err(Error::StreamRead)?;
+        info!("Reply: {:?}", reply);
+        self.num_irqs = reply.num_irqs;
+
+        if reply.flags & VFIO_DEVICE_FLAGS_PCI != VFIO_DEVICE_FLAGS_PCI {
+            return Err(Error::NotPciDevice);
+        }
+
+        self.resettable = reply.flags & VFIO_DEVICE_FLAGS_RESET != VFIO_DEVICE_FLAGS_RESET;
+
+        let num_regions = reply.num_regions;
+        let mut regions = Vec::new();
+        for index in 0..num_regions {
+            let get_region_info = DeviceGetRegionInfo {
+                header: Header {
+                    message_id: self.next_message_id.0,
+                    command: Command::DeviceGetRegionInfo,
+                    flags: HeaderFlags::Command as u32,
+                    message_size: std::mem::size_of::<DeviceGetRegionInfo>() as u32,
+                    ..Default::default()
+                },
+                region_info: vfio_region_info {
+                    argsz: 1024, // Arbitrary max size
+                    index,
+                    ..Default::default()
+                },
+            };
+            info!("Command: {:?}", get_region_info);
+            self.next_message_id += Wrapping(1);
+
+            self.stream
+                .write_all(get_region_info.as_slice())
+                .map_err(Error::StreamWrite)?;
+
+            let mut reply = DeviceGetRegionInfo::default();
+            let (_, fd) = self
+                .stream
+                .recv_with_fd(reply.as_mut_slice())
+                .map_err(Error::ReceiveWithFd)?;
+            info!("Reply: {:?}", reply);
+
+            regions.push(Region {
+                flags: reply.region_info.flags,
+                index: reply.region_info.index,
+                size: reply.region_info.size,
+                file_offset: fd.map(|fd| FileOffset::new(fd, reply.region_info.offset)),
+            });
+
+            // TODO: Handle region with capabilities
+            let mut _cap_data = Vec::with_capacity(
+                reply.header.message_size as usize - std::mem::size_of::<DeviceGetRegionInfo>(),
+            );
+            _cap_data.resize(_cap_data.capacity(), 0u8);
+            self.stream
+                .read_exact(_cap_data.as_mut_slice())
+                .map_err(Error::StreamRead)?;
+        }
+
+        Ok(regions)
+    }
+
+    pub fn region_read(&mut self, region: u32, offset: u64, data: &mut [u8]) -> Result<(), Error> {
+        let region_read = RegionAccess {
+            header: Header {
+                message_id: self.next_message_id.0,
+                command: Command::RegionRead,
+                flags: HeaderFlags::Command as u32,
+                message_size: std::mem::size_of::<RegionAccess>() as u32,
+                ..Default::default()
+            },
+            offset,
+            count: data.len() as u32,
+            region,
+        };
+        info!("Command: {:?}", region_read);
+        self.next_message_id += Wrapping(1);
+        self.stream
+            .write_all(region_read.as_slice())
+            .map_err(Error::StreamWrite)?;
+
+        let mut reply = RegionAccess::default();
+        self.stream
+            .read_exact(reply.as_mut_slice())
+            .map_err(Error::StreamRead)?;
+        info!("Reply: {:?}", reply);
+        self.stream.read_exact(data).map_err(Error::StreamRead)?;
+        Ok(())
+    }
+
+    pub fn region_write(&mut self, region: u32, offset: u64, data: &[u8]) -> Result<(), Error> {
+        let region_write = RegionAccess {
+            header: Header {
+                message_id: self.next_message_id.0,
+                command: Command::RegionWrite,
+                flags: HeaderFlags::Command as u32,
+                message_size: (std::mem::size_of::<RegionAccess>() + data.len()) as u32,
+                ..Default::default()
+            },
+            offset,
+            count: data.len() as u32,
+            region,
+        };
+        info!("Command: {:?}", region_write);
+        self.next_message_id += Wrapping(1);
+
+        let bufs = vec![IoSlice::new(region_write.as_slice()), IoSlice::new(data)];
+
+        // TODO: Use write_all_vectored() when ready
+        let _ = self
+            .stream
+            .write_vectored(&bufs)
+            .map_err(Error::StreamWrite)?;
+
+        let mut reply = RegionAccess::default();
+        self.stream
+            .read_exact(reply.as_mut_slice())
+            .map_err(Error::StreamRead)?;
+        info!("Reply: {:?}", reply);
+        Ok(())
+    }
+
+    pub fn get_irq_info(&mut self, index: u32) -> Result<IrqInfo, Error> {
+        let get_irq_info = GetIrqInfo {
+            header: Header {
+                message_id: self.next_message_id.0,
+                command: Command::GetIrqInfo,
+                flags: HeaderFlags::Command as u32,
+                message_size: std::mem::size_of::<GetIrqInfo>() as u32,
+                ..Default::default()
+            },
+            argsz: (std::mem::size_of::<GetIrqInfo>() - std::mem::size_of::<Header>()) as u32,
+            flags: 0,
+            index,
+            count: 0,
+        };
+        info!("Command: {:?}", get_irq_info);
+        self.next_message_id += Wrapping(1);
+
+        self.stream
+            .write_all(get_irq_info.as_slice())
+            .map_err(Error::StreamWrite)?;
+
+        let mut reply = GetIrqInfo::default();
+        self.stream
+            .read_exact(reply.as_mut_slice())
+            .map_err(Error::StreamRead)?;
+        info!("Reply: {:?}", reply);
+
+        Ok(IrqInfo {
+            index: reply.index,
+            flags: reply.flags,
+            count: reply.count,
+        })
+    }
+
+    pub fn set_irqs(
+        &mut self,
+        index: u32,
+        flags: u32,
+        start: u32,
+        count: u32,
+        fds: &[RawFd],
+    ) -> Result<(), Error> {
+        let set_irqs = SetIrqs {
+            header: Header {
+                message_id: self.next_message_id.0,
+                command: Command::SetIrqs,
+                flags: HeaderFlags::Command as u32,
+                message_size: std::mem::size_of::<SetIrqs>() as u32,
+                ..Default::default()
+            },
+            argsz: (std::mem::size_of::<SetIrqs>() - std::mem::size_of::<Header>()) as u32,
+            flags,
+            start,
+            index,
+            count,
+        };
+        info!("Command: {:?}", set_irqs);
+        self.next_message_id += Wrapping(1);
+
+        self.stream
+            .send_with_fds(&[set_irqs.as_slice()], fds)
+            .map_err(Error::SendWithFd)?;
+
+        let mut reply = Header::default();
+        self.stream
+            .read_exact(reply.as_mut_slice())
+            .map_err(Error::StreamRead)?;
+        info!("Reply: {:?}", reply);
+
+        Ok(())
+    }
+
+    pub fn region(&self, region_index: u32) -> Option<&Region> {
+        for region in &self.regions {
+            if region.index == region_index {
+                return Some(region);
+            }
+        }
+
+        None
+    }
+
+    pub fn resettable(&self) -> bool {
+        self.resettable
+    }
+}

vhdx/Cargo.toml

@@ -0,0 +1,16 @@
+[package]
+name = "vhdx"
+version = "0.1.0"
+authors = ["The Cloud Hypervisor Authors"]
+edition = "2018"
+license = "Apache-2.0"
+
+[dependencies]
+byteorder = "1.4.3"
+crc32c = "0.6.0"
+libc = "0.2.101"
+log = "0.4.14"
+remain = "0.2.2"
+thiserror = "1.0"
+uuid = { version = "0.8.2", features = ["v4"] }
+vmm-sys-util = ">=0.3.1"

vhdx/src/lib.rs

@@ -0,0 +1,31 @@
+// Copyright © 2021 Intel Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+
+use byteorder::{BigEndian, ByteOrder};
+use std::result::Result;
+use uuid::Uuid;
+
+macro_rules! div_round_up {
+    ($n:expr,$d:expr) => {
+        ($n + $d - 1) / $d
+    };
+}
+
+pub mod vhdx;
+mod vhdx_bat;
+mod vhdx_header;
+mod vhdx_io;
+mod vhdx_metadata;
+
+pub(crate) fn uuid_from_guid(buf: &[u8]) -> Result<Uuid, uuid::Error> {
+    // The first 3 fields of UUID are stored in Big Endian format, and
+    // the last 8 bytes are stored as byte array. Therefore, we read the
+    // first 3 fields in Big Endian format instead of Little Endian.
+    Uuid::from_fields_le(
+        BigEndian::read_u32(&buf[0..4]),
+        BigEndian::read_u16(&buf[4..6]),
+        BigEndian::read_u16(&buf[6..8]),
+        &buf[8..16],
+    )
+}

vhdx/src/vhdx.rs

@@ -0,0 +1,212 @@
+// Copyright © 2021 Intel Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+
+use crate::vhdx_bat::{BatEntry, VhdxBatError};
+use crate::vhdx_header::{self, RegionInfo, RegionTableEntry, VhdxHeader, VhdxHeaderError};
+use crate::vhdx_io::{self, VhdxIoError};
+use crate::vhdx_metadata::{DiskSpec, VhdxMetadataError};
+use remain::sorted;
+use std::collections::btree_map::BTreeMap;
+use std::fs::File;
+use std::io::{Read, Seek, SeekFrom, Write};
+use thiserror::Error;
+
+#[sorted]
+#[derive(Error, Debug)]
+pub enum VhdxError {
+    #[error("Not a VHDx file {0}")]
+    NotVhdx(#[source] VhdxHeaderError),
+    #[error("Failed to parse VHDx header {0}")]
+    ParseVhdxHeader(#[source] VhdxHeaderError),
+    #[error("Failed to parse VHDx metadata {0}")]
+    ParseVhdxMetadata(#[source] VhdxMetadataError),
+    #[error("Failed to parse VHDx region entries {0}")]
+    ParseVhdxRegionEntry(#[source] VhdxHeaderError),
+    #[error("Failed reading metadata {0}")]
+    ReadBatEntry(#[source] VhdxBatError),
+    #[error("Failed reading sector from disk {0}")]
+    ReadFailed(#[source] VhdxIoError),
+    #[error("Failed writing to sector on disk {0}")]
+    WriteFailed(#[source] VhdxIoError),
+}
+
+pub type Result<T> = std::result::Result<T, VhdxError>;
+
+#[derive(Debug)]
+pub struct Vhdx {
+    file: File,
+    vhdx_header: VhdxHeader,
+    region_entries: BTreeMap<u64, u64>,
+    bat_entry: RegionTableEntry,
+    mdr_entry: RegionTableEntry,
+    disk_spec: DiskSpec,
+    bat_entries: Vec<BatEntry>,
+    current_offset: u64,
+    first_write: bool,
+}
+
+impl Vhdx {
+    /// Parse the Vhdx header, BAT, and metadata from a file and store info
+    // in Vhdx structure.
+    pub fn new(mut file: File) -> Result<Vhdx> {
+        let vhdx_header = VhdxHeader::new(&mut file).map_err(VhdxError::ParseVhdxHeader)?;
+
+        let collected_entries = RegionInfo::new(
+            &mut file,
+            vhdx_header::REGION_TABLE_1_START,
+            vhdx_header.region_entry_count(),
+        )
+        .map_err(VhdxError::ParseVhdxRegionEntry)?;
+
+        let bat_entry = collected_entries.bat_entry;
+        let mdr_entry = collected_entries.mdr_entry;
+
+        let disk_spec =
+            DiskSpec::new(&mut file, &mdr_entry).map_err(VhdxError::ParseVhdxMetadata)?;
+        let bat_entries = BatEntry::collect_bat_entries(&mut file, &disk_spec, &bat_entry)
+            .map_err(VhdxError::ReadBatEntry)?;
+
+        Ok(Vhdx {
+            file,
+            vhdx_header,
+            region_entries: collected_entries.region_entries,
+            bat_entry,
+            mdr_entry,
+            disk_spec,
+            bat_entries,
+            current_offset: 0,
+            first_write: true,
+        })
+    }
+
+    pub fn virtual_disk_size(&self) -> u64 {
+        self.disk_spec.virtual_disk_size
+    }
+}
+
+impl Read for Vhdx {
+    /// Wrapper function to satisfy Read trait implementation for VHDx disk.
+    /// Convert the offset to sector index and buffer length to sector count.
+    fn read(&mut self, buf: &mut [u8]) -> std::result::Result<usize, std::io::Error> {
+        let sector_count =
+            div_round_up!(buf.len() as u64, self.disk_spec.logical_sector_size as u64);
+        let sector_index = self.current_offset / self.disk_spec.logical_sector_size as u64;
+
+        vhdx_io::read(
+            &mut self.file,
+            buf,
+            &self.disk_spec,
+            &self.bat_entries,
+            sector_index,
+            sector_count,
+        )
+        .map_err(|e| {
+            std::io::Error::new(
+                std::io::ErrorKind::Other,
+                format!(
+                    "Failed reading {} sectors from VHDx at index {}: {}",
+                    sector_count, sector_index, e
+                ),
+            )
+        })
+    }
+}
+
+impl Write for Vhdx {
+    fn flush(&mut self) -> std::result::Result<(), std::io::Error> {
+        self.file.flush()
+    }
+
+    /// Wrapper function to satisfy Write trait implementation for VHDx disk.
+    /// Convert the offset to sector index and buffer length to sector count.
+    fn write(&mut self, buf: &[u8]) -> std::result::Result<usize, std::io::Error> {
+        let sector_count =
+            div_round_up!(buf.len() as u64, self.disk_spec.logical_sector_size as u64);
+        let sector_index = self.current_offset / self.disk_spec.logical_sector_size as u64;
+
+        if self.first_write {
+            self.first_write = false;
+            self.vhdx_header.update(&mut self.file).map_err(|e| {
+                std::io::Error::new(
+                    std::io::ErrorKind::Other,
+                    format!("Failed to update VHDx header: {}", e),
+                )
+            })?;
+        }
+
+        vhdx_io::write(
+            &mut self.file,
+            buf,
+            &mut self.disk_spec,
+            self.bat_entry.file_offset,
+            &mut self.bat_entries,
+            sector_index,
+            sector_count,
+        )
+        .map_err(|e| {
+            std::io::Error::new(
+                std::io::ErrorKind::Other,
+                format!(
+                    "Failed writing {} sectors on VHDx at index {}: {}",
+                    sector_count, sector_index, e
+                ),
+            )
+        })
+    }
+}
+
+impl Seek for Vhdx {
+    /// Wrapper function to satisfy Seek trait implementation for VHDx disk.
+    /// Updates the offset field in the Vhdx struct.
+    fn seek(&mut self, pos: SeekFrom) -> std::io::Result<u64> {
+        let new_offset: Option<u64> = match pos {
+            SeekFrom::Start(off) => Some(off),
+            SeekFrom::End(off) => {
+                if off < 0 {
+                    0i64.checked_sub(off).and_then(|increment| {
+                        self.virtual_disk_size().checked_sub(increment as u64)
+                    })
+                } else {
+                    self.virtual_disk_size().checked_add(off as u64)
+                }
+            }
+            SeekFrom::Current(off) => {
+                if off < 0 {
+                    0i64.checked_sub(off)
+                        .and_then(|increment| self.current_offset.checked_sub(increment as u64))
+                } else {
+                    self.current_offset.checked_add(off as u64)
+                }
+            }
+        };
+
+        if let Some(o) = new_offset {
+            if o <= self.virtual_disk_size() {
+                self.current_offset = o;
+                return Ok(o);
+            }
+        }
+
+        Err(std::io::Error::new(
+            std::io::ErrorKind::InvalidData,
+            "Failed seek operation",
+        ))
+    }
+}
+
+impl Clone for Vhdx {
+    fn clone(&self) -> Self {
+        Vhdx {
+            file: self.file.try_clone().unwrap(),
+            vhdx_header: self.vhdx_header.clone(),
+            region_entries: self.region_entries.clone(),
+            bat_entry: self.bat_entry,
+            mdr_entry: self.mdr_entry,
+            disk_spec: self.disk_spec.clone(),
+            bat_entries: self.bat_entries.clone(),
+            current_offset: self.current_offset,
+            first_write: self.first_write,
+        }
+    }
+}

vhdx/src/vhdx_bat.rs

@@ -0,0 +1,104 @@
+// Copyright © 2021 Intel Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+
+use crate::vhdx_header::RegionTableEntry;
+use crate::vhdx_metadata::DiskSpec;
+use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt};
+use remain::sorted;
+use std::fs::File;
+use std::io::{self, Seek, SeekFrom};
+use std::mem::size_of;
+use thiserror::Error;
+
+// Payload BAT Entry States
+pub const PAYLOAD_BLOCK_NOT_PRESENT: u64 = 0;
+pub const PAYLOAD_BLOCK_UNDEFINED: u64 = 1;
+pub const PAYLOAD_BLOCK_ZERO: u64 = 2;
+pub const PAYLOAD_BLOCK_UNMAPPED: u64 = 3;
+pub const PAYLOAD_BLOCK_FULLY_PRESENT: u64 = 6;
+pub const PAYLOAD_BLOCK_PARTIALLY_PRESENT: u64 = 7;
+
+// Mask for the BAT state
+pub const BAT_STATE_BIT_MASK: u64 = 0x07;
+// Mask for the offset within the file in units of 1 MB
+pub const BAT_FILE_OFF_MASK: u64 = 0xFFFFFFFFFFF00000;
+
+#[sorted]
+#[derive(Error, Debug)]
+pub enum VhdxBatError {
+    #[error("Invalid BAT entry")]
+    InvalidBatEntry,
+    #[error("Invalid BAT entry count")]
+    InvalidEntryCount,
+    #[error("Failed to read BAT entry {0}")]
+    ReadBat(#[source] io::Error),
+    #[error("Failed to write BAT entry {0}")]
+    WriteBat(#[source] io::Error),
+}
+
+pub type Result<T> = std::result::Result<T, VhdxBatError>;
+
+#[derive(Default, Clone, Debug)]
+pub struct BatEntry(pub u64);
+
+impl BatEntry {
+    // Read all BAT entries presented on the disk and insert them to a vector
+    pub fn collect_bat_entries(
+        f: &mut File,
+        disk_spec: &DiskSpec,
+        bat_entry: &RegionTableEntry,
+    ) -> Result<Vec<BatEntry>> {
+        let entry_count = BatEntry::calculate_entries(
+            disk_spec.block_size,
+            disk_spec.virtual_disk_size,
+            disk_spec.chunk_ratio,
+        );
+        if entry_count as usize > (bat_entry.length as usize / size_of::<BatEntry>()) {
+            return Err(VhdxBatError::InvalidEntryCount);
+        }
+
+        let mut bat: Vec<BatEntry> = Vec::with_capacity(bat_entry.length as usize);
+        let offset = bat_entry.file_offset;
+        for i in 0..entry_count {
+            f.seek(SeekFrom::Start(offset + i * size_of::<u64>() as u64))
+                .map_err(VhdxBatError::ReadBat)?;
+
+            let bat_entry = BatEntry(
+                f.read_u64::<LittleEndian>()
+                    .map_err(VhdxBatError::ReadBat)?,
+            );
+            bat.insert(i as usize, bat_entry);
+        }
+
+        Ok(bat)
+    }
+
+    // Calculate the number of entries in the BAT
+    fn calculate_entries(block_size: u32, virtual_disk_size: u64, chunk_ratio: u64) -> u64 {
+        let data_blocks_count = div_round_up!(virtual_disk_size, block_size as u64);
+        data_blocks_count + (data_blocks_count - 1) / chunk_ratio
+    }
+
+    // Routine for writing BAT entries to the disk
+    pub fn write_bat_entries(
+        f: &mut File,
+        bat_offset: u64,
+        bat_entries: &[BatEntry],
+    ) -> Result<()> {
+        for i in 0..bat_entries.len() as u64 {
+            f.seek(SeekFrom::Start(bat_offset + i * size_of::<u64>() as u64))
+                .map_err(VhdxBatError::WriteBat)?;
+            let bat_entry = match bat_entries.get(i as usize) {
+                Some(entry) => entry.0,
+                None => {
+                    return Err(VhdxBatError::InvalidBatEntry);
+                }
+            };
+
+            f.write_u64::<LittleEndian>(bat_entry)
+                .map_err(VhdxBatError::WriteBat)?;
+        }
+        Ok(())
+    }
+}

vhdx/src/vhdx_header.rs

@@ -0,0 +1,489 @@
+// Copyright © 2021 Intel Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+
+extern crate log;
+
+use byteorder::{ByteOrder, LittleEndian, ReadBytesExt};
+use remain::sorted;
+use std::collections::btree_map::BTreeMap;
+use std::convert::TryInto;
+use std::fs::File;
+use std::io::{self, Read, Seek, SeekFrom, Write};
+use std::mem::size_of;
+use thiserror::Error;
+use uuid::Uuid;
+
+const VHDX_SIGN: u64 = 0x656C_6966_7864_6876; // "vhdxfile"
+const HEADER_SIGN: u32 = 0x6461_6568; // "head"
+const REGION_SIGN: u32 = 0x6967_6572; // "regi"
+
+const FILE_START: u64 = 0; // The first element
+const HEADER_1_START: u64 = 64 * 1024; // Header 1 start in Bytes
+const HEADER_2_START: u64 = 128 * 1024; // Header 2 start in Bytes
+pub const REGION_TABLE_1_START: u64 = 192 * 1024; // Region 1 start in Bytes
+const REGION_TABLE_2_START: u64 = 256 * 1024; // Region 2 start in Bytes
+
+const HEADER_SIZE: u64 = 4 * 1024; // Each header is 64 KiB, but only first 4 kiB contains info
+const REGION_SIZE: u64 = 64 * 1024; // Each region size is 64 KiB
+
+const REGION_ENTRY_REQUIRED: u32 = 1;
+
+const BAT_GUID: &str = "2DC27766-F623-4200-9D64-115E9BFD4A08"; // BAT GUID
+const MDR_GUID: &str = "8B7CA206-4790-4B9A-B8FE-575F050F886E"; // Metadata GUID
+
+#[sorted]
+#[derive(Error, Debug)]
+pub enum VhdxHeaderError {
+    #[error("Failed to calculate checksum")]
+    CalculateChecksum,
+    #[error("BAT entry is not unique")]
+    DuplicateBATEntry,
+    #[error("Metadata region entry is not unique")]
+    DuplicateMDREntry,
+    #[error("Checksum doesn't match for")]
+    InvalidChecksum(String),
+    #[error("Invalid entry count")]
+    InvalidEntryCount,
+    #[error("Not a valid VHDx header")]
+    InvalidHeaderSign,
+    #[error("Not a valid VHDx region")]
+    InvalidRegionSign,
+    #[error("Couldn't parse Uuid for region entry {0}")]
+    InvalidUuid(#[source] uuid::Error),
+    #[error("Not a VHDx file")]
+    InvalidVHDXSign,
+    #[error("No valid header found")]
+    NoValidHeader,
+    #[error("Cannot read checksum")]
+    ReadChecksum,
+    #[error("Failed to read File Type Identifier {0}")]
+    ReadFileTypeIdentifier(#[source] io::Error),
+    #[error("Failed to read headers {0}")]
+    ReadHeader(#[source] io::Error),
+    #[error("Failed to read metadata {0}")]
+    ReadMetadata(#[source] std::io::Error),
+    #[error("Failed to read region table entries {0}")]
+    ReadRegionTableEntries(#[source] io::Error),
+    #[error("Failed to read region table header {0}")]
+    ReadRegionTableHeader(#[source] io::Error),
+    #[error("Failed to read region entries")]
+    RegionEntryCollectionFailed,
+    #[error("Overlapping regions found")]
+    RegionOverlap,
+    #[error("Reserved region has non-zero value")]
+    ReservedIsNonZero,
+    #[error("Failed to seek in File Type Identifier {0}")]
+    SeekFileTypeIdentifier(#[source] io::Error),
+    #[error("Failed to seek in headers {0}")]
+    SeekHeader(#[source] io::Error),
+    #[error("Failed to seek in region table entries {0}")]
+    SeekRegionTableEntries(#[source] io::Error),
+    #[error("Failed to seek in region table header {0}")]
+    SeekRegionTableHeader(#[source] io::Error),
+    #[error("We do not recongize this entry")]
+    UnrecognizedRegionEntry,
+    #[error("Failed to write header {0}")]
+    WriteHeader(#[source] io::Error),
+}
+
+pub type Result<T> = std::result::Result<T, VhdxHeaderError>;
+
+#[derive(Clone, Debug)]
+pub struct FileTypeIdentifier {
+    pub signature: u64,
+}
+
+impl FileTypeIdentifier {
+    /// Reads the File Type Identifier structure from a reference VHDx file
+    pub fn new(f: &mut File) -> Result<FileTypeIdentifier> {
+        f.seek(SeekFrom::Start(FILE_START))
+            .map_err(VhdxHeaderError::SeekFileTypeIdentifier)?;
+        let signature = f
+            .read_u64::<LittleEndian>()
+            .map_err(VhdxHeaderError::ReadFileTypeIdentifier)?;
+        if signature != VHDX_SIGN {
+            return Err(VhdxHeaderError::InvalidVHDXSign);
+        }
+
+        Ok(FileTypeIdentifier { signature })
+    }
+}
+
+#[repr(packed)]
+#[derive(Clone, Copy, Debug)]
+pub struct Header {
+    pub signature: u32,
+    pub checksum: u32,
+    pub sequence_number: u64,
+    pub file_write_guid: u128,
+    pub data_write_guid: u128,
+    pub log_guid: u128,
+    pub log_version: u16,
+    pub version: u16,
+    pub log_length: u32,
+    pub log_offset: u64,
+}
+
+impl Header {
+    /// Reads the Header structure from a reference VHDx file
+    pub fn new(f: &mut File, start: u64) -> Result<Header> {
+        // Read the whole header in to a buffer. We will need it for
+        // calculating checksum.
+        let mut buffer = [0; HEADER_SIZE as usize];
+        f.seek(SeekFrom::Start(start))
+            .map_err(VhdxHeaderError::SeekHeader)?;
+        f.read_exact(&mut buffer)
+            .map_err(VhdxHeaderError::ReadHeader)?;
+
+        let header = unsafe { *(buffer.as_ptr() as *mut Header) };
+        if header.signature != HEADER_SIGN {
+            return Err(VhdxHeaderError::InvalidHeaderSign);
+        }
+
+        let new_checksum = calculate_checksum(&mut buffer, size_of::<u32>())?;
+        if header.checksum != new_checksum {
+            return Err(VhdxHeaderError::InvalidChecksum(String::from("Header")));
+        }
+
+        Ok(header)
+    }
+
+    /// Converts the header structure into a buffer
+    fn get_header_as_buffer(&self, buffer: &mut [u8; HEADER_SIZE as usize]) {
+        let reference = unsafe {
+            std::slice::from_raw_parts(self as *const Header as *const u8, HEADER_SIZE as usize)
+        };
+        *buffer = reference.try_into().unwrap();
+    }
+
+    /// Creates and returns new updated header from the provided current header
+    pub fn update_header(
+        f: &mut File,
+        current_header: &Header,
+        change_data_guid: bool,
+        mut file_write_guid: u128,
+        start: u64,
+    ) -> Result<Header> {
+        let mut buffer = [0u8; HEADER_SIZE as usize];
+        let mut data_write_guid = current_header.data_write_guid;
+
+        if change_data_guid {
+            data_write_guid = Uuid::new_v4().as_u128();
+        }
+
+        if file_write_guid == 0 {
+            file_write_guid = current_header.file_write_guid;
+        }
+
+        let mut new_header = Header {
+            signature: current_header.signature,
+            checksum: 0,
+            sequence_number: current_header.sequence_number + 1,
+            file_write_guid,
+            data_write_guid,
+            log_guid: current_header.log_guid,
+            log_version: current_header.log_version,
+            version: current_header.version,
+            log_length: current_header.log_length,
+            log_offset: current_header.log_offset,
+        };
+
+        new_header.get_header_as_buffer(&mut buffer);
+        new_header.checksum = crc32c::crc32c(&buffer);
+        new_header.get_header_as_buffer(&mut buffer);
+
+        f.seek(SeekFrom::Start(start))
+            .map_err(VhdxHeaderError::SeekHeader)?;
+        f.write(&buffer).map_err(VhdxHeaderError::WriteHeader)?;
+
+        Ok(new_header)
+    }
+}
+
+#[repr(packed)]
+#[derive(Clone, Copy, Debug)]
+struct RegionTableHeader {
+    pub signature: u32,
+    pub checksum: u32,
+    pub entry_count: u32,
+    pub reserved: u32,
+}
+
+impl RegionTableHeader {
+    /// Reads the Region Table Header structure from a reference VHDx file
+    pub fn new(f: &mut File, start: u64) -> Result<RegionTableHeader> {
+        // Read the whole header into a buffer. We will need it for calculating
+        // checksum.
+        let mut buffer = [0u8; REGION_SIZE as usize];
+        f.seek(SeekFrom::Start(start))
+            .map_err(VhdxHeaderError::SeekRegionTableHeader)?;
+        f.read_exact(&mut buffer)
+            .map_err(VhdxHeaderError::ReadRegionTableHeader)?;
+
+        let region_table_header = unsafe { *(buffer.as_ptr() as *mut RegionTableHeader) };
+        if region_table_header.signature != REGION_SIGN {
+            return Err(VhdxHeaderError::InvalidRegionSign);
+        }
+
+        let new_checksum = calculate_checksum(&mut buffer, size_of::<u32>())?;
+        if region_table_header.checksum != new_checksum {
+            return Err(VhdxHeaderError::InvalidChecksum(String::from("Region")));
+        }
+
+        if region_table_header.entry_count > 2047 {
+            return Err(VhdxHeaderError::InvalidEntryCount);
+        }
+
+        if region_table_header.reserved != 0 {
+            return Err(VhdxHeaderError::ReservedIsNonZero);
+        }
+
+        Ok(region_table_header)
+    }
+}
+
+pub struct RegionInfo {
+    pub bat_entry: RegionTableEntry,
+    pub mdr_entry: RegionTableEntry,
+    pub region_entries: BTreeMap<u64, u64>,
+}
+
+impl RegionInfo {
+    /// Collect all entries in a BTreeMap from the Region Table and identifies
+    /// BAT and metadata regions
+    pub fn new(f: &mut File, region_start: u64, entry_count: u32) -> Result<RegionInfo> {
+        let mut bat_entry: Option<RegionTableEntry> = None;
+        let mut mdr_entry: Option<RegionTableEntry> = None;
+
+        let mut offset = 0;
+        let mut region_entries = BTreeMap::new();
+
+        let mut buffer = [0; REGION_SIZE as usize];
+        // Seek after the Region Table Header
+        f.seek(SeekFrom::Start(
+            region_start + size_of::<RegionTableHeader>() as u64,
+        ))
+        .map_err(VhdxHeaderError::SeekRegionTableEntries)?;
+        f.read_exact(&mut buffer)
+            .map_err(VhdxHeaderError::ReadRegionTableEntries)?;
+
+        for _ in 0..entry_count {
+            let entry =
+                RegionTableEntry::new(&buffer[offset..offset + size_of::<RegionTableEntry>()])?;
+
+            offset += size_of::<RegionTableEntry>();
+            let start = entry.file_offset;
+            let end = start + entry.length as u64;
+
+            for (region_ent_start, region_ent_end) in region_entries.iter() {
+                if !((start >= *region_ent_start) || (end <= *region_ent_end)) {
+                    return Err(VhdxHeaderError::RegionOverlap);
+                }
+            }
+
+            region_entries.insert(entry.file_offset, entry.file_offset + entry.length as u64);
+
+            if entry.guid == Uuid::parse_str(BAT_GUID).map_err(VhdxHeaderError::InvalidUuid)? {
+                if bat_entry.is_none() {
+                    bat_entry = Some(entry);
+                    continue;
+                }
+                return Err(VhdxHeaderError::DuplicateBATEntry);
+            }
+
+            if entry.guid == Uuid::parse_str(MDR_GUID).map_err(VhdxHeaderError::InvalidUuid)? {
+                if mdr_entry.is_none() {
+                    mdr_entry = Some(entry);
+                    continue;
+                }
+                return Err(VhdxHeaderError::DuplicateMDREntry);
+            }
+
+            if (entry.required & REGION_ENTRY_REQUIRED) == 1 {
+                // This implementation doesn't recognize this field.
+                // Therefore, accoding to the spec, we are throwing an error.
+                return Err(VhdxHeaderError::UnrecognizedRegionEntry);
+            }
+        }
+
+        if bat_entry.is_none() || mdr_entry.is_none() {
+            region_entries.clear();
+            return Err(VhdxHeaderError::RegionEntryCollectionFailed);
+        }
+
+        // It's safe to unwrap as we checked both entries have been filled.
+        // Otherwise, an error is already returned.
+        let bat_entry = bat_entry.unwrap();
+        let mdr_entry = mdr_entry.unwrap();
+
+        Ok(RegionInfo {
+            bat_entry,
+            mdr_entry,
+            region_entries,
+        })
+    }
+}
+
+#[repr(packed)]
+#[derive(Clone, Copy, Debug)]
+pub struct RegionTableEntry {
+    pub guid: Uuid,
+    pub file_offset: u64,
+    pub length: u32,
+    pub required: u32,
+}
+
+impl RegionTableEntry {
+    /// Reads one Region Entry from a Region Table index that starts from 0
+    pub fn new(buffer: &[u8]) -> Result<RegionTableEntry> {
+        let mut region_table_entry = unsafe { *(buffer.as_ptr() as *mut RegionTableEntry) };
+
+        let uuid = crate::uuid_from_guid(buffer).map_err(VhdxHeaderError::InvalidUuid)?;
+        region_table_entry.guid = uuid;
+
+        Ok(region_table_entry)
+    }
+}
+
+#[derive(Clone, Debug)]
+struct RegionEntry {
+    start: u64,
+    end: u64,
+}
+
+enum HeaderNo {
+    First,
+    Second,
+}
+
+/// Contains the information from the header of a VHDx file
+#[derive(Clone, Debug)]
+pub struct VhdxHeader {
+    file_type_identifier: FileTypeIdentifier,
+    header_1: Header,
+    header_2: Header,
+    region_table_1: RegionTableHeader,
+    region_table_2: RegionTableHeader,
+}
+
+impl VhdxHeader {
+    /// Creates a VhdxHeader from a reference to a file
+    pub fn new(f: &mut File) -> Result<VhdxHeader> {
+        let file_type_identifier: FileTypeIdentifier = FileTypeIdentifier::new(f)?;
+        let header_1 = Header::new(f, HEADER_1_START);
+        let header_2 = Header::new(f, HEADER_2_START);
+
+        let mut file_write_guid: u128 = 0;
+        let metadata = f.metadata().map_err(VhdxHeaderError::ReadMetadata)?;
+        if !metadata.permissions().readonly() {
+            file_write_guid = Uuid::new_v4().as_u128();
+        }
+
+        let (header_1, header_2) =
+            VhdxHeader::update_headers(f, header_1, header_2, file_write_guid)?;
+        Ok(VhdxHeader {
+            file_type_identifier,
+            header_1,
+            header_2,
+            region_table_1: RegionTableHeader::new(f, REGION_TABLE_1_START)?,
+            region_table_2: RegionTableHeader::new(f, REGION_TABLE_2_START)?,
+        })
+    }
+
+    /// Identify the current header and return both headers along with an
+    /// integer indicating the current header.
+    fn current_header(
+        header_1: Result<Header>,
+        header_2: Result<Header>,
+    ) -> Result<(HeaderNo, Header)> {
+        let mut header1_seq_num: u64 = 0;
+        let mut header2_seq_num: u64 = 0;
+        let mut valid_hdr_found: bool = false;
+
+        if let Ok(ref header_1) = header_1 {
+            valid_hdr_found = true;
+            header1_seq_num = header_1.sequence_number;
+        }
+
+        if let Ok(ref header_2) = header_2 {
+            valid_hdr_found = true;
+            header2_seq_num = header_2.sequence_number;
+        }
+
+        if !valid_hdr_found {
+            Err(VhdxHeaderError::NoValidHeader)
+        } else if header1_seq_num >= header2_seq_num {
+            Ok((HeaderNo::First, header_1.unwrap()))
+        } else {
+            Ok((HeaderNo::Second, header_2.unwrap()))
+        }
+    }
+
+    /// This takes two headers and update the noncurrent header with the
+    /// current one. Returns both headers as a tuple sequenced the way it was
+    /// received from the parameter list.
+    fn update_header(
+        f: &mut File,
+        header_1: Result<Header>,
+        header_2: Result<Header>,
+        guid: u128,
+    ) -> Result<(Header, Header)> {
+        let (header_no, current_header) = VhdxHeader::current_header(header_1, header_2)?;
+
+        match header_no {
+            HeaderNo::First => {
+                let other_header =
+                    Header::update_header(f, &current_header, true, guid, HEADER_2_START)?;
+                Ok((current_header, other_header))
+            }
+            HeaderNo::Second => {
+                let other_header =
+                    Header::update_header(f, &current_header, true, guid, HEADER_1_START)?;
+                Ok((other_header, current_header))
+            }
+        }
+    }
+
+    // Update the provided headers accoding to the spec
+    fn update_headers(
+        f: &mut File,
+        header_1: Result<Header>,
+        header_2: Result<Header>,
+        guid: u128,
+    ) -> Result<(Header, Header)> {
+        // According to the spec, update twice
+        let (header_1, header_2) = VhdxHeader::update_header(f, header_1, header_2, guid)?;
+        VhdxHeader::update_header(f, Ok(header_1), Ok(header_2), guid)
+    }
+
+    pub fn update(&mut self, f: &mut File) -> Result<()> {
+        let headers = VhdxHeader::update_headers(f, Ok(self.header_1), Ok(self.header_2), 0)?;
+        self.header_1 = headers.0;
+        self.header_2 = headers.1;
+        Ok(())
+    }
+
+    pub fn region_entry_count(&self) -> u32 {
+        self.region_table_1.entry_count
+    }
+}
+
+/// Calculates the checksum of a buffer that itself containts its checksum
+/// Therefore, before calculating, the existing checksum is retrieved and the
+/// corresponding field is made zero. After the calculation, the existing checksum
+/// is put back to the buffer.
+pub fn calculate_checksum(buffer: &mut [u8], csum_offset: usize) -> Result<u32> {
+    // Read the checksum into a mutable slice
+    let csum_buf = &mut buffer[csum_offset..csum_offset + 4];
+    // Convert the checksum chunk in to a u32 integer
+    let orig_csum = LittleEndian::read_u32(csum_buf);
+    // Zero the checksum in the buffer
+    LittleEndian::write_u32(csum_buf, 0);
+    // Calculate the checksum on the resulting buffer
+    let new_csum = crc32c::crc32c(buffer);
+    // Put back the original checksum in the buffer
+    LittleEndian::write_u32(&mut buffer[csum_offset..csum_offset + 4], orig_csum);
+
+    Ok(new_csum)
+}

vhdx/src/vhdx_io.rs

@@ -0,0 +1,229 @@
+// Copyright © 2021 Intel Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+
+use crate::vhdx_bat::{self, BatEntry, VhdxBatError};
+use crate::vhdx_metadata::{self, DiskSpec};
+use remain::sorted;
+use std::fs::File;
+use std::io::{self, Read, Seek, SeekFrom, Write};
+use thiserror::Error;
+
+const SECTOR_SIZE: u64 = 512;
+
+#[sorted]
+#[derive(Error, Debug)]
+pub enum VhdxIoError {
+    #[error("Invalid BAT entry state")]
+    InvalidBatEntryState,
+    #[error("Invalid BAT entry count")]
+    InvalidBatIndex,
+    #[error("Invalid disk size")]
+    InvalidDiskSize,
+    #[error("Failed reading sector blocks from file {0}")]
+    ReadSectorBlock(#[source] io::Error),
+    #[error("Failed changing file length {0}")]
+    ResizeFile(#[source] io::Error),
+    #[error("Differencing mode is not supported yet")]
+    UnsupportedMode,
+    #[error("Failed writing BAT to file {0}")]
+    WriteBat(#[source] VhdxBatError),
+}
+
+pub type Result<T> = std::result::Result<T, VhdxIoError>;
+
+macro_rules! align {
+    ($n:expr, $align:expr) => {{
+        if $align > $n {
+            $align
+        } else {
+            let rem = $n % $align;
+            (($n / $align) + rem) * $align
+        }
+    }};
+}
+
+#[derive(Default)]
+struct Sector {
+    bat_index: u64,
+    free_sectors: u64,
+    free_bytes: u64,
+    file_offset: u64,
+    block_offset: u64,
+}
+
+impl Sector {
+    /// Translate sector index and count of data in file to actual offsets and
+    /// BAT index.
+    pub fn new(
+        disk_spec: &DiskSpec,
+        bat: &[BatEntry],
+        sector_index: u64,
+        sector_count: u64,
+    ) -> Result<Sector> {
+        let mut sector = Sector::default();
+
+        sector.bat_index = sector_index / disk_spec.sectors_per_block as u64;
+        sector.block_offset = sector_index % disk_spec.sectors_per_block as u64;
+        sector.free_sectors = disk_spec.sectors_per_block as u64 - sector.block_offset;
+        if sector.free_sectors > sector_count {
+            sector.free_sectors = sector_count;
+        }
+
+        sector.free_bytes = sector.free_sectors * disk_spec.logical_sector_size as u64;
+        sector.block_offset *= disk_spec.logical_sector_size as u64;
+
+        let bat_entry = match bat.get(sector.bat_index as usize) {
+            Some(entry) => entry.0,
+            None => {
+                return Err(VhdxIoError::InvalidBatIndex);
+            }
+        };
+        sector.file_offset = bat_entry & vhdx_bat::BAT_FILE_OFF_MASK;
+        if sector.file_offset != 0 {
+            sector.file_offset += sector.block_offset;
+        }
+
+        Ok(sector)
+    }
+}
+
+/// VHDx IO read routine: requires relative sector index and count for the
+/// requested data.
+pub fn read(
+    f: &mut File,
+    buf: &mut [u8],
+    disk_spec: &DiskSpec,
+    bat: &[BatEntry],
+    mut sector_index: u64,
+    mut sector_count: u64,
+) -> Result<usize> {
+    let mut read_count: usize = 0;
+
+    while sector_count > 0 {
+        if disk_spec.has_parent {
+            return Err(VhdxIoError::UnsupportedMode);
+        } else {
+            let sector = Sector::new(disk_spec, bat, sector_index, sector_count)?;
+
+            let bat_entry = match bat.get(sector.bat_index as usize) {
+                Some(entry) => entry.0,
+                None => {
+                    return Err(VhdxIoError::InvalidBatIndex);
+                }
+            };
+
+            match bat_entry & vhdx_bat::BAT_STATE_BIT_MASK {
+                vhdx_bat::PAYLOAD_BLOCK_NOT_PRESENT
+                | vhdx_bat::PAYLOAD_BLOCK_UNDEFINED
+                | vhdx_bat::PAYLOAD_BLOCK_UNMAPPED
+                | vhdx_bat::PAYLOAD_BLOCK_ZERO => {}
+                vhdx_bat::PAYLOAD_BLOCK_FULLY_PRESENT => {
+                    f.seek(SeekFrom::Start(sector.file_offset))
+                        .map_err(VhdxIoError::ReadSectorBlock)?;
+                    f.read_exact(
+                        &mut buf[read_count
+                            ..(read_count + (sector.free_sectors * SECTOR_SIZE) as usize)],
+                    )
+                    .map_err(VhdxIoError::ReadSectorBlock)?;
+                }
+                vhdx_bat::PAYLOAD_BLOCK_PARTIALLY_PRESENT => {
+                    return Err(VhdxIoError::UnsupportedMode);
+                }
+                _ => {
+                    return Err(VhdxIoError::InvalidBatEntryState);
+                }
+            };
+            sector_count -= sector.free_sectors;
+            sector_index += sector.free_sectors;
+            read_count = sector.free_bytes as usize;
+        };
+    }
+    Ok(read_count)
+}
+
+/// VHDx IO write routine: requires relative sector index and count for the
+/// requested data.
+pub fn write(
+    f: &mut File,
+    buf: &[u8],
+    disk_spec: &mut DiskSpec,
+    bat_offset: u64,
+    bat: &mut [BatEntry],
+    mut sector_index: u64,
+    mut sector_count: u64,
+) -> Result<usize> {
+    let mut write_count: usize = 0;
+
+    while sector_count > 0 {
+        if disk_spec.has_parent {
+            return Err(VhdxIoError::UnsupportedMode);
+        } else {
+            let sector = Sector::new(disk_spec, bat, sector_index, sector_count)?;
+
+            let bat_entry = match bat.get(sector.bat_index as usize) {
+                Some(entry) => entry.0,
+                None => {
+                    return Err(VhdxIoError::InvalidBatIndex);
+                }
+            };
+
+            match bat_entry & vhdx_bat::BAT_STATE_BIT_MASK {
+                vhdx_bat::PAYLOAD_BLOCK_NOT_PRESENT
+                | vhdx_bat::PAYLOAD_BLOCK_UNDEFINED
+                | vhdx_bat::PAYLOAD_BLOCK_UNMAPPED
+                | vhdx_bat::PAYLOAD_BLOCK_ZERO => {
+                    let file_offset =
+                        align!(disk_spec.image_size, vhdx_metadata::BLOCK_SIZE_MIN as u64);
+                    let new_size = file_offset
+                        .checked_add(disk_spec.block_size as u64)
+                        .ok_or(VhdxIoError::InvalidDiskSize)?;
+
+                    f.set_len(new_size).map_err(VhdxIoError::ResizeFile)?;
+                    disk_spec.image_size = new_size;
+
+                    let new_bat_entry = file_offset
+                        | (vhdx_bat::PAYLOAD_BLOCK_FULLY_PRESENT & vhdx_bat::BAT_STATE_BIT_MASK);
+                    bat[sector.bat_index as usize] = BatEntry(new_bat_entry);
+                    BatEntry::write_bat_entries(f, bat_offset, bat)
+                        .map_err(VhdxIoError::WriteBat)?;
+
+                    if file_offset < vhdx_metadata::BLOCK_SIZE_MIN as u64 {
+                        break;
+                    }
+
+                    f.seek(SeekFrom::Start(file_offset))
+                        .map_err(VhdxIoError::ReadSectorBlock)?;
+                    f.write_all(
+                        &buf[write_count
+                            ..(write_count + (sector.free_sectors * SECTOR_SIZE) as usize)],
+                    )
+                    .map_err(VhdxIoError::ReadSectorBlock)?;
+                }
+                vhdx_bat::PAYLOAD_BLOCK_FULLY_PRESENT => {
+                    if sector.file_offset < vhdx_metadata::BLOCK_SIZE_MIN as u64 {
+                        break;
+                    }
+
+                    f.seek(SeekFrom::Start(sector.file_offset))
+                        .map_err(VhdxIoError::ReadSectorBlock)?;
+                    f.write_all(
+                        &buf[write_count
+                            ..(write_count + (sector.free_sectors * SECTOR_SIZE) as usize)],
+                    )
+                    .map_err(VhdxIoError::ReadSectorBlock)?;
+                }
+                vhdx_bat::PAYLOAD_BLOCK_PARTIALLY_PRESENT => {
+                    return Err(VhdxIoError::UnsupportedMode);
+                }
+                _ => {
+                    return Err(VhdxIoError::InvalidBatEntryState);
+                }
+            };
+            sector_count -= sector.free_sectors;
+            sector_index += sector.free_sectors;
+            write_count = sector.free_bytes as usize;
+        };
+    }
+    Ok(write_count)
+}

vhdx/src/vhdx_metadata.rs

@@ -0,0 +1,323 @@
+// Copyright © 2021 Intel Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+
+use crate::vhdx_header::RegionTableEntry;
+use byteorder::{LittleEndian, ReadBytesExt};
+use remain::sorted;
+use std::fs::File;
+use std::io::{self, Read, Seek, SeekFrom};
+use std::mem::size_of;
+use thiserror::Error;
+use uuid::Uuid;
+
+const METADATA_SIGN: u64 = 0x6174_6164_6174_656D;
+const METADATA_ENTRY_SIZE: usize = 32;
+const METADATA_MAX_ENTRIES: u16 = 2047;
+// The size including the table header and entries
+const METADATA_TABLE_MAX_SIZE: usize = METADATA_ENTRY_SIZE * (METADATA_MAX_ENTRIES as usize + 1);
+
+const METADATA_FLAGS_IS_REQUIRED: u32 = 0x04;
+
+pub const BLOCK_SIZE_MIN: u32 = 1 << 20; // 1 MiB
+const BLOCK_SIZE_MAX: u32 = 256 << 20; // 256 MiB
+const MAX_SECTORS_PER_BLOCK: u64 = 1 << 23;
+
+const BLOCK_HAS_PARENT: u32 = 0x02; // Has a parent or a backing file
+
+// GUID for known metadata items
+const METADATA_FILE_PARAMETER: &str = "CAA16737-FA36-4D43-B3B6-33F0AA44E76B";
+const METADATA_VIRTUAL_DISK_SIZE: &str = "2FA54224-CD1B-4876-B211-5DBED83BF4B8";
+const METADATA_VIRTUAL_DISK_ID: &str = "BECA12AB-B2E6-4523-93EF-C309E000C746";
+const METADATA_LOGICAL_SECTOR_SIZE: &str = "8141BF1D-A96F-4709-BA47-F233A8FAAB5F";
+const METADATA_PHYSICAL_SECTOR_SIZE: &str = "CDA348C7-445D-4471-9CC9-E9885251C556";
+const METADATA_PARENT_LOCATOR: &str = "A8D35F2D-B30B-454D-ABF7-D3D84834AB0C";
+
+const METADATA_FILE_PARAMETER_PRESENT: u16 = 0x01;
+const METADATA_VIRTUAL_DISK_SIZE_PRESENT: u16 = 0x02;
+const METADATA_VIRTUAL_DISK_ID_PRESENT: u16 = 0x04;
+const METADATA_LOGICAL_SECTOR_SIZE_PRESENT: u16 = 0x08;
+const METADATA_PHYSICAL_SECTOR_SIZE_PRESENT: u16 = 0x10;
+const METADATA_PARENT_LOCATOR_PRESENT: u16 = 0x20;
+
+const METADATA_ALL_PRESENT: u16 = METADATA_FILE_PARAMETER_PRESENT
+    | METADATA_VIRTUAL_DISK_SIZE_PRESENT
+    | METADATA_VIRTUAL_DISK_ID_PRESENT
+    | METADATA_LOGICAL_SECTOR_SIZE_PRESENT
+    | METADATA_PHYSICAL_SECTOR_SIZE_PRESENT;
+
+const METADATA_LENGTH_MAX: u32 = 1 << 20; // 1 MiB
+
+#[sorted]
+#[derive(Error, Debug)]
+pub enum VhdxMetadataError {
+    #[error("Invalid block size count")]
+    InvalidBlockSize,
+    #[error("Invalid metadata entry count")]
+    InvalidEntryCount,
+    #[error("Invalid logical sector size")]
+    InvalidLogicalSectorSize,
+    #[error("Invalid metadata ID")]
+    InvalidMetadataItem,
+    #[error("Invalid metadata length")]
+    InvalidMetadataLength,
+    #[error("Metadata sign doesn't match")]
+    InvalidMetadataSign,
+    #[error("Invalid physical sector size")]
+    InvalidPhysicalSectorSize,
+    #[error("Invalid UUID")]
+    InvalidUuid(#[source] uuid::Error),
+    #[error("Invalid value")]
+    InvalidValue,
+    #[error("Not all required metadata found")]
+    MissingMetadata,
+    #[error("Failed to read metadata headers {0}")]
+    ReadMetadata(#[source] io::Error),
+    #[error("Reserved region has non-zero value")]
+    ReservedIsNonZero,
+    #[error("This implementation doesn't support this metadata flag")]
+    UnsupportedFlag,
+}
+
+pub type Result<T> = std::result::Result<T, VhdxMetadataError>;
+
+#[derive(Default, Clone, Debug)]
+pub struct DiskSpec {
+    pub disk_id: u128,
+    pub image_size: u64,
+    pub block_size: u32,
+    pub has_parent: bool,
+    pub sectors_per_block: u32,
+    pub virtual_disk_size: u64,
+    pub logical_sector_size: u32,
+    pub physical_sector_size: u32,
+    pub chunk_ratio: u64,
+    pub total_sectors: u64,
+}
+
+impl DiskSpec {
+    /// Parse all meatadata from the provided file and store info in DiskSpec
+    /// structure.
+    pub fn new(f: &mut File, metadata_region: &RegionTableEntry) -> Result<DiskSpec> {
+        let mut disk_spec = DiskSpec::default();
+        let mut metadata_presence: u16 = 0;
+        let mut offset = 0;
+        let metadata = f.metadata().map_err(VhdxMetadataError::ReadMetadata)?;
+        disk_spec.image_size = metadata.len();
+
+        let mut buffer = [0u8; METADATA_TABLE_MAX_SIZE];
+        f.seek(SeekFrom::Start(metadata_region.file_offset))
+            .map_err(VhdxMetadataError::ReadMetadata)?;
+        f.read_exact(&mut buffer)
+            .map_err(VhdxMetadataError::ReadMetadata)?;
+
+        let metadata_header =
+            MetadataTableHeader::new(&buffer[0..size_of::<MetadataTableHeader>()])?;
+
+        offset += size_of::<MetadataTableHeader>();
+        for _ in 0..metadata_header.entry_count {
+            let metadata_entry =
+                MetadataTableEntry::new(&buffer[offset..offset + size_of::<MetadataTableEntry>()])?;
+
+            f.seek(SeekFrom::Start(
+                metadata_region.file_offset + metadata_entry.offset as u64,
+            ))
+            .map_err(VhdxMetadataError::ReadMetadata)?;
+
+            if metadata_entry.item_id
+                == Uuid::parse_str(METADATA_FILE_PARAMETER)
+                    .map_err(VhdxMetadataError::InvalidUuid)?
+            {
+                disk_spec.block_size = f
+                    .read_u32::<LittleEndian>()
+                    .map_err(VhdxMetadataError::ReadMetadata)?;
+
+                // MUST be at least 1 MiB and not greater than 256 MiB
+                if disk_spec.block_size < BLOCK_SIZE_MIN && disk_spec.block_size > BLOCK_SIZE_MAX {
+                    return Err(VhdxMetadataError::InvalidBlockSize);
+                }
+
+                // MUST be power of 2
+                if !disk_spec.block_size.is_power_of_two() {
+                    return Err(VhdxMetadataError::InvalidBlockSize);
+                }
+
+                let bits = f
+                    .read_u32::<LittleEndian>()
+                    .map_err(VhdxMetadataError::ReadMetadata)?;
+                if bits & BLOCK_HAS_PARENT != 0 {
+                    disk_spec.has_parent = true;
+                } else {
+                    disk_spec.has_parent = false;
+                }
+
+                metadata_presence |= METADATA_FILE_PARAMETER_PRESENT;
+            } else if metadata_entry.item_id
+                == Uuid::parse_str(METADATA_VIRTUAL_DISK_SIZE)
+                    .map_err(VhdxMetadataError::InvalidUuid)?
+            {
+                disk_spec.virtual_disk_size = f
+                    .read_u64::<LittleEndian>()
+                    .map_err(VhdxMetadataError::ReadMetadata)?;
+
+                metadata_presence |= METADATA_VIRTUAL_DISK_SIZE_PRESENT;
+            } else if metadata_entry.item_id
+                == Uuid::parse_str(METADATA_VIRTUAL_DISK_ID)
+                    .map_err(VhdxMetadataError::InvalidUuid)?
+            {
+                disk_spec.disk_id = f
+                    .read_u128::<LittleEndian>()
+                    .map_err(VhdxMetadataError::ReadMetadata)?;
+
+                metadata_presence |= METADATA_VIRTUAL_DISK_ID_PRESENT;
+            } else if metadata_entry.item_id
+                == Uuid::parse_str(METADATA_LOGICAL_SECTOR_SIZE)
+                    .map_err(VhdxMetadataError::InvalidUuid)?
+            {
+                disk_spec.logical_sector_size = f
+                    .read_u32::<LittleEndian>()
+                    .map_err(VhdxMetadataError::ReadMetadata)?;
+                if !(disk_spec.logical_sector_size == 512 || disk_spec.logical_sector_size == 4096)
+                {
+                    return Err(VhdxMetadataError::InvalidLogicalSectorSize);
+                }
+
+                metadata_presence |= METADATA_LOGICAL_SECTOR_SIZE_PRESENT;
+            } else if metadata_entry.item_id
+                == Uuid::parse_str(METADATA_PHYSICAL_SECTOR_SIZE)
+                    .map_err(VhdxMetadataError::InvalidUuid)?
+            {
+                disk_spec.physical_sector_size = f
+                    .read_u32::<LittleEndian>()
+                    .map_err(VhdxMetadataError::ReadMetadata)?;
+                if !(disk_spec.physical_sector_size == 512
+                    || disk_spec.physical_sector_size == 4096)
+                {
+                    return Err(VhdxMetadataError::InvalidPhysicalSectorSize);
+                }
+
+                metadata_presence |= METADATA_PHYSICAL_SECTOR_SIZE_PRESENT;
+            } else if metadata_entry.item_id
+                == Uuid::parse_str(METADATA_PARENT_LOCATOR)
+                    .map_err(VhdxMetadataError::InvalidUuid)?
+            {
+                metadata_presence |= METADATA_PARENT_LOCATOR_PRESENT;
+            } else {
+                return Err(VhdxMetadataError::InvalidMetadataItem);
+            }
+
+            if (metadata_entry.flag_bits & METADATA_FLAGS_IS_REQUIRED) == 0 {
+                return Err(VhdxMetadataError::UnsupportedFlag);
+            }
+            offset += size_of::<MetadataTableEntry>();
+        }
+
+        // Check if all required metadata are present
+        if metadata_presence != METADATA_ALL_PRESENT {
+            return Err(VhdxMetadataError::MissingMetadata);
+        }
+        // Check if the virtual disk size is a multiple of the logical sector
+        // size.
+        if ((metadata_presence & METADATA_LOGICAL_SECTOR_SIZE_PRESENT) != 0)
+            && (disk_spec.virtual_disk_size % disk_spec.logical_sector_size as u64 != 0)
+        {
+            return Err(VhdxMetadataError::InvalidBlockSize);
+        }
+
+        disk_spec.sectors_per_block =
+            DiskSpec::sectors_per_block(disk_spec.block_size, disk_spec.logical_sector_size)?;
+
+        disk_spec.chunk_ratio =
+            DiskSpec::chunk_ratio(disk_spec.block_size, disk_spec.logical_sector_size)?;
+
+        disk_spec.total_sectors =
+            disk_spec.virtual_disk_size / disk_spec.logical_sector_size as u64;
+
+        Ok(disk_spec)
+    }
+
+    /// Calculates the number of sectors per block
+    fn sectors_per_block(block_size: u32, logical_sector_size: u32) -> Result<u32> {
+        let sectors_per_block = block_size / logical_sector_size;
+
+        if !sectors_per_block.is_power_of_two() {
+            return Err(VhdxMetadataError::InvalidValue);
+        }
+
+        Ok(sectors_per_block)
+    }
+
+    /// Calculate the chunk ratio
+    fn chunk_ratio(block_size: u32, logical_sector_size: u32) -> Result<u64> {
+        let chunk_ratio = (MAX_SECTORS_PER_BLOCK * logical_sector_size as u64) / block_size as u64;
+
+        if !chunk_ratio.is_power_of_two() {
+            return Err(VhdxMetadataError::InvalidValue);
+        }
+
+        Ok(chunk_ratio)
+    }
+}
+
+#[repr(packed)]
+#[derive(Default, Debug, Clone, Copy)]
+struct MetadataTableHeader {
+    signature: u64,
+    reserved: u16,
+    entry_count: u16,
+    _reserved2: [u8; 20],
+}
+
+impl MetadataTableHeader {
+    pub fn new(buffer: &[u8]) -> Result<MetadataTableHeader> {
+        let metadata_table_header = unsafe { *(buffer.as_ptr() as *mut MetadataTableHeader) };
+
+        if metadata_table_header.signature != METADATA_SIGN {
+            return Err(VhdxMetadataError::InvalidMetadataSign);
+        }
+
+        if metadata_table_header.entry_count > METADATA_MAX_ENTRIES {
+            return Err(VhdxMetadataError::InvalidEntryCount);
+        }
+
+        if metadata_table_header.reserved != 0 {
+            return Err(VhdxMetadataError::ReservedIsNonZero);
+        }
+
+        Ok(metadata_table_header)
+    }
+}
+
+#[repr(packed)]
+#[derive(Default, Debug, Clone, Copy)]
+pub struct MetadataTableEntry {
+    item_id: Uuid,
+    offset: u32,
+    length: u32,
+    flag_bits: u32,
+    reserved: u32,
+}
+
+impl MetadataTableEntry {
+    /// Parse one metadata entry from the buffer
+    fn new(buffer: &[u8]) -> Result<MetadataTableEntry> {
+        let mut metadata_table_entry = unsafe { *(buffer.as_ptr() as *mut MetadataTableEntry) };
+
+        let uuid = crate::uuid_from_guid(buffer).map_err(VhdxMetadataError::InvalidUuid)?;
+        metadata_table_entry.item_id = uuid;
+
+        if metadata_table_entry.length > METADATA_LENGTH_MAX {
+            return Err(VhdxMetadataError::InvalidMetadataLength);
+        }
+
+        if metadata_table_entry.length == 0 && metadata_table_entry.offset != 0 {
+            return Err(VhdxMetadataError::InvalidMetadataLength);
+        }
+
+        if metadata_table_entry.reserved != 0 {
+            return Err(VhdxMetadataError::ReservedIsNonZero);
+        }
+
+        Ok(metadata_table_entry)
+    }
+}

vhost_user_backend/Cargo.toml

@@ -8,11 +8,11 @@ edition = "2018"
 default = []
 
 [dependencies]
-epoll = ">=4.0.1"
-libc = "0.2.98"
+epoll = "4.3.1"
+libc = "0.2.101"
 log = "0.4.14"
 virtio-bindings = "0.1.0"
-vm-memory = { version = "0.5.0", features = ["backend-bitmap"] }
+vm-memory = { version = "0.6.0", features = ["backend-bitmap"] }
 vm-virtio = { path = "../vm-virtio" }
-vmm-sys-util = ">=0.3.1"
-vhost = { git = "https://github.com/rust-vmm/vhost", branch = "master", package = "vhost", features = ["vhost-user-slave"] }
+vmm-sys-util = "0.8.0"
+vhost = { version = "0.1.0", features = ["vhost-user-slave"] }

vhost_user_block/Cargo.toml

@@ -6,14 +6,15 @@ edition = "2018"
 
 [dependencies]
 block_util = { path = "../block_util" }
-clap = { version = "2.33.3", features=["wrap_help"] }
-epoll = ">=4.0.1"
-libc = "0.2.98"
+clap = { version = "2.33.3", features = ["wrap_help"] }
+env_logger = "0.9.0"
+epoll = "4.3.1"
+libc = "0.2.101"
 log = "0.4.14"
 option_parser = { path = "../option_parser" }
 qcow = { path = "../qcow" }
 vhost_user_backend = { path = "../vhost_user_backend" }
-vhost = { git = "https://github.com/rust-vmm/vhost", branch = "master", package = "vhost", features = ["vhost-user-slave"] }
+vhost = { version = "0.1.0", features = ["vhost-user-slave"] }
 virtio-bindings = "0.1.0"
-vm-memory = "0.5.0"
-vmm-sys-util = ">=0.3.1"
+vm-memory = "0.6.0"
+vmm-sys-util = "0.8.0"

vhost_user_block/src/main.rs

@@ -16,6 +16,8 @@ use clap::{App, Arg};
 use vhost_user_block::start_block_backend;
 
 fn main() {
+    env_logger::init();
+
     let cmd_arguments = App::new("vhost-user-blk backend")
         .version(crate_version!())
         .author(crate_authors!())
@@ -25,7 +27,7 @@ fn main() {
                 .long("block-backend")
                 .help(vhost_user_block::SYNTAX)
                 .takes_value(true)
-                .min_values(1),
+                .required(true),
         )
         .get_matches();
 

vhost_user_net/Cargo.toml

@@ -5,14 +5,15 @@ authors = ["The Cloud Hypervisor Authors"]
 edition = "2018"
 
 [dependencies]
-clap = { version = "2.33.3", features=["wrap_help"] }
-epoll = ">=4.0.1"
-libc = "0.2.98"
+clap = { version = "2.33.3", features = ["wrap_help"] }
+env_logger = "0.9.0"
+epoll = "4.3.1"
+libc = "0.2.101"
 log = "0.4.14"
 net_util = { path = "../net_util" }
 option_parser = { path = "../option_parser" }
 vhost_user_backend = { path = "../vhost_user_backend" }
-vhost = { git = "https://github.com/rust-vmm/vhost", branch = "master", package = "vhost", features = ["vhost-user-slave"] }
+vhost = { version = "0.1.0", features = ["vhost-user-slave"] }
 virtio-bindings = "0.1.0"
-vm-memory = "0.5.0"
-vmm-sys-util = ">=0.3.1"
+vm-memory = "0.6.0"
+vmm-sys-util = "0.8.0"

vhost_user_net/src/lib.rs

@@ -24,7 +24,6 @@ use vhost::vhost_user::message::*;
 use vhost::vhost_user::Listener;
 use vhost_user_backend::{GuestMemoryMmap, VhostUserBackend, VhostUserDaemon, Vring, VringWorker};
 use virtio_bindings::bindings::virtio_net::*;
-use virtio_bindings::bindings::virtio_ring::VIRTIO_RING_F_EVENT_IDX;
 use vm_memory::GuestMemoryAtomic;
 use vmm_sys_util::eventfd::EventFd;
 
@@ -174,7 +173,6 @@ impl VhostUserBackend for VhostUserNetBackend {
             | 1 << VIRTIO_NET_F_MAC
             | 1 << VIRTIO_F_NOTIFY_ON_EMPTY
             | 1 << VIRTIO_F_VERSION_1
-            | 1 << VIRTIO_RING_F_EVENT_IDX
             | VhostUserVirtioFeatures::PROTOCOL_FEATURES.bits()
     }
 

vhost_user_net/src/main.rs

@@ -13,6 +13,8 @@ use clap::{App, Arg};
 use vhost_user_net::start_net_backend;
 
 fn main() {
+    env_logger::init();
+
     let cmd_arguments = App::new("vhost-user-net backend")
         .version(crate_version!())
         .author(crate_authors!())
@@ -22,7 +24,7 @@ fn main() {
                 .long("net-backend")
                 .help(vhost_user_net::SYNTAX)
                 .takes_value(true)
-                .min_values(1),
+                .required(true),
         )
         .get_matches();
 

virtio-devices/Cargo.toml

@@ -10,30 +10,30 @@ io_uring = ["block_util/io_uring"]
 mshv = []
 
 [dependencies]
-anyhow = "1.0"
-arc-swap = ">=1.0.0"
+anyhow = "1.0.43"
+arc-swap = "1.3.2"
 block_util = { path = "../block_util" }
 byteorder = "1.4.3"
-epoll = ">=4.0.1"
+epoll = "4.3.1"
 event_monitor = { path = "../event_monitor" }
-io-uring = ">=0.4.0"
-libc = "0.2.98"
+io-uring = "0.5.2"
+libc = "0.2.101"
 log = "0.4.14"
 net_gen = { path = "../net_gen" }
 net_util = { path = "../net_util" }
 pci = { path = "../pci" }
 rate_limiter = { path = "../rate_limiter" }
-seccomp = { git = "https://github.com/firecracker-microvm/firecracker", tag = "v0.24.4" }
-serde = ">=1.0.27"
-serde_derive = ">=1.0.27"
-serde_json = ">=1.0.9"
+seccompiler = { git = "https://github.com/rust-vmm/seccompiler"}
+serde = "1.0.130"
+serde_derive = "1.0.130"
+serde_json = "1.0.67"
 versionize = "0.1.6"
 versionize_derive = "0.1.4"
-vhost = { git = "https://github.com/rust-vmm/vhost", branch = "master", package = "vhost", features = ["vhost-user-master", "vhost-user-slave"] }
-virtio-bindings = { version = "0.1", features = ["virtio-v5_0_0"]}
+vhost = { version = "0.1.0", features = ["vhost-user-master", "vhost-user-slave", "vhost-kern"] }
+virtio-bindings = { version = "0.1.0", features = ["virtio-v5_0_0"] }
 vm-allocator = { path = "../vm-allocator" }
 vm-device = { path = "../vm-device" }
-vm-memory = { version = "0.5.0", features = ["backend-mmap", "backend-atomic", "backend-bitmap"] }
+vm-memory = { version = "0.6.0", features = ["backend-mmap", "backend-atomic", "backend-bitmap"] }
 vm-migration = { path = "../vm-migration" }
 vm-virtio = { path = "../vm-virtio" }
-vmm-sys-util = ">=0.3.1"
+vmm-sys-util = "0.8.0"

virtio-devices/src/balloon.rs

@@ -16,11 +16,12 @@ use super::{
     ActivateError, ActivateResult, EpollHelper, EpollHelperError, EpollHelperHandler, Queue,
     VirtioCommon, VirtioDevice, VirtioDeviceType, EPOLL_HELPER_EVENT_LAST, VIRTIO_F_VERSION_1,
 };
-use crate::seccomp_filters::{get_seccomp_filter, Thread};
+use crate::seccomp_filters::Thread;
+use crate::thread_helper::spawn_virtio_thread;
 use crate::GuestMemoryMmap;
 use crate::{VirtioInterrupt, VirtioInterruptType};
 use libc::EFD_NONBLOCK;
-use seccomp::{SeccompAction, SeccompFilter};
+use seccompiler::SeccompAction;
 use std::io;
 use std::mem::size_of;
 use std::os::unix::io::AsRawFd;
@@ -28,7 +29,6 @@ use std::result;
 use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
 use std::sync::mpsc;
 use std::sync::{Arc, Barrier, Mutex};
-use std::thread;
 use vm_memory::GuestMemory;
 use vm_memory::{
     Address, ByteValued, Bytes, GuestAddress, GuestAddressSpace, GuestMemoryAtomic,
@@ -342,6 +342,7 @@ pub struct Balloon {
     resize: VirtioBalloonResize,
     config: Arc<Mutex<VirtioBalloonConfig>>,
     seccomp_action: SeccompAction,
+    exit_evt: EventFd,
 }
 
 impl Balloon {
@@ -351,6 +352,7 @@ impl Balloon {
         size: u64,
         deflate_on_oom: bool,
         seccomp_action: SeccompAction,
+        exit_evt: EventFd,
     ) -> io::Result<Self> {
         let mut avail_features = 1u64 << VIRTIO_F_VERSION_1;
         if deflate_on_oom {
@@ -375,6 +377,7 @@ impl Balloon {
             resize: VirtioBalloonResize::new()?,
             config: Arc::new(Mutex::new(config)),
             seccomp_action,
+            exit_evt,
         })
     }
 
@@ -460,23 +463,19 @@ impl VirtioDevice for Balloon {
         let paused = self.common.paused.clone();
         let paused_sync = self.common.paused_sync.clone();
         let mut epoll_threads = Vec::new();
-        let virtio_balloon_seccomp_filter =
-            get_seccomp_filter(&self.seccomp_action, Thread::VirtioBalloon)
-                .map_err(ActivateError::CreateSeccompFilter)?;
-        thread::Builder::new()
-            .name(self.id.clone())
-            .spawn(move || {
-                if let Err(e) = SeccompFilter::apply(virtio_balloon_seccomp_filter) {
-                    error!("Error applying seccomp filter: {:?}", e);
-                } else if let Err(e) = handler.run(paused, paused_sync.unwrap()) {
+
+        spawn_virtio_thread(
+            &self.id,
+            &self.seccomp_action,
+            Thread::VirtioBalloon,
+            &mut epoll_threads,
+            &self.exit_evt,
+            move || {
+                if let Err(e) = handler.run(paused, paused_sync.unwrap()) {
                     error!("Error running worker: {:?}", e);
                 }
-            })
-            .map(|thread| epoll_threads.push(thread))
-            .map_err(|e| {
-                error!("failed to clone virtio-balloon epoll thread: {}", e);
-                ActivateError::BadActivate
-            })?;
+            },
+        )?;
         self.common.epoll_threads = Some(epoll_threads);
 
         event!("virtio-device", "activated", "id", &self.id);

virtio-devices/src/block.rs

@@ -14,7 +14,8 @@ use super::{
     RateLimiterConfig, VirtioCommon, VirtioDevice, VirtioDeviceType, VirtioInterruptType,
     EPOLL_HELPER_EVENT_LAST,
 };
-use crate::seccomp_filters::{get_seccomp_filter, Thread};
+use crate::seccomp_filters::Thread;
+use crate::thread_helper::spawn_virtio_thread;
 use crate::GuestMemoryMmap;
 use crate::VirtioInterrupt;
 use block_util::{
@@ -22,7 +23,7 @@ use block_util::{
     RequestType, VirtioBlockConfig,
 };
 use rate_limiter::{RateLimiter, TokenType};
-use seccomp::{SeccompAction, SeccompFilter};
+use seccompiler::SeccompAction;
 use std::io;
 use std::num::Wrapping;
 use std::os::unix::io::AsRawFd;
@@ -30,7 +31,6 @@ use std::path::PathBuf;
 use std::result;
 use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
 use std::sync::{Arc, Barrier};
-use std::thread;
 use std::{collections::HashMap, convert::TryInto};
 use versionize::{VersionMap, Versionize, VersionizeResult};
 use versionize_derive::Versionize;
@@ -365,6 +365,7 @@ pub struct Block {
     counters: BlockCounters,
     seccomp_action: SeccompAction,
     rate_limiter_config: Option<RateLimiterConfig>,
+    exit_evt: EventFd,
 }
 
 #[derive(Versionize)]
@@ -391,6 +392,7 @@ impl Block {
         queue_size: u16,
         seccomp_action: SeccompAction,
         rate_limiter_config: Option<RateLimiterConfig>,
+        exit_evt: EventFd,
     ) -> io::Result<Self> {
         let disk_size = disk_image.size().map_err(|e| {
             io::Error::new(
@@ -448,6 +450,7 @@ impl Block {
             counters: BlockCounters::default(),
             seccomp_action,
             rate_limiter_config,
+            exit_evt,
         })
     }
 
@@ -588,25 +591,18 @@ impl VirtioDevice for Block {
             let paused = self.common.paused.clone();
             let paused_sync = self.common.paused_sync.clone();
 
-            // Retrieve seccomp filter for virtio_block thread
-            let virtio_block_seccomp_filter =
-                get_seccomp_filter(&self.seccomp_action, Thread::VirtioBlock)
-                    .map_err(ActivateError::CreateSeccompFilter)?;
-
-            thread::Builder::new()
-                .name(format!("{}_q{}", self.id.clone(), i))
-                .spawn(move || {
-                    if let Err(e) = SeccompFilter::apply(virtio_block_seccomp_filter) {
-                        error!("Error applying seccomp filter: {:?}", e);
-                    } else if let Err(e) = handler.run(paused, paused_sync.unwrap()) {
+            spawn_virtio_thread(
+                &format!("{}_q{}", self.id.clone(), i),
+                &self.seccomp_action,
+                Thread::VirtioBlock,
+                &mut epoll_threads,
+                &self.exit_evt,
+                move || {
+                    if let Err(e) = handler.run(paused, paused_sync.unwrap()) {
                         error!("Error running worker: {:?}", e);
                     }
-                })
-                .map(|thread| epoll_threads.push(thread))
-                .map_err(|e| {
-                    error!("failed to clone the virtio-block epoll thread: {}", e);
-                    ActivateError::BadActivate
-                })?;
+                },
+            )?;
         }
 
         self.common.epoll_threads = Some(epoll_threads);

virtio-devices/src/console.rs

@@ -3,25 +3,25 @@
 
 use super::Error as DeviceError;
 use super::{
-    ActivateError, ActivateResult, EpollHelper, EpollHelperError, EpollHelperHandler, Queue,
-    VirtioCommon, VirtioDevice, VirtioDeviceType, VirtioInterruptType, EPOLL_HELPER_EVENT_LAST,
+    ActivateResult, EpollHelper, EpollHelperError, EpollHelperHandler, Queue, VirtioCommon,
+    VirtioDevice, VirtioDeviceType, VirtioInterruptType, EPOLL_HELPER_EVENT_LAST,
     VIRTIO_F_IOMMU_PLATFORM, VIRTIO_F_VERSION_1,
 };
-use crate::seccomp_filters::{get_seccomp_filter, Thread};
+use crate::seccomp_filters::Thread;
+use crate::thread_helper::spawn_virtio_thread;
 use crate::GuestMemoryMmap;
 use crate::VirtioInterrupt;
 use libc::EFD_NONBLOCK;
-use seccomp::{SeccompAction, SeccompFilter};
+use seccompiler::SeccompAction;
 use std::cmp;
 use std::collections::VecDeque;
+use std::fs::File;
 use std::io;
-use std::io::Write;
-use std::ops::DerefMut;
+use std::io::{Read, Write};
 use std::os::unix::io::AsRawFd;
 use std::result;
 use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
 use std::sync::{Arc, Barrier, Mutex};
-use std::thread;
 use versionize::{VersionMap, Versionize, VersionizeResult};
 use versionize_derive::Versionize;
 use vm_memory::{ByteValued, Bytes, GuestAddressSpace, GuestMemoryAtomic};
@@ -40,6 +40,8 @@ const OUTPUT_QUEUE_EVENT: u16 = EPOLL_HELPER_EVENT_LAST + 2;
 const INPUT_EVENT: u16 = EPOLL_HELPER_EVENT_LAST + 3;
 // Console configuration change event is triggered.
 const CONFIG_EVENT: u16 = EPOLL_HELPER_EVENT_LAST + 4;
+// File written to (input ready)
+const FILE_EVENT: u16 = EPOLL_HELPER_EVENT_LAST + 5;
 
 //Console size feature bit
 const VIRTIO_CONSOLE_F_SIZE: u64 = 0;
@@ -61,7 +63,7 @@ struct ConsoleEpollHandler {
     mem: GuestMemoryAtomic<GuestMemoryMmap>,
     interrupt_cb: Arc<dyn VirtioInterrupt>,
     in_buffer: Arc<Mutex<VecDeque<u8>>>,
-    out: Arc<Mutex<Box<dyn io::Write + Send + Sync + 'static>>>,
+    endpoint: Endpoint,
     input_queue_evt: EventFd,
     output_queue_evt: EventFd,
     input_evt: EventFd,
@@ -70,6 +72,42 @@ struct ConsoleEpollHandler {
     pause_evt: EventFd,
 }
 
+pub enum Endpoint {
+    File(File),
+    FilePair(File, File),
+    Null,
+}
+
+impl Endpoint {
+    fn out_file(&self) -> Option<&File> {
+        match self {
+            Self::File(f) => Some(f),
+            Self::FilePair(f, _) => Some(f),
+            Self::Null => None,
+        }
+    }
+
+    fn in_file(&self) -> Option<&File> {
+        match self {
+            Self::File(_) => None,
+            Self::FilePair(_, f) => Some(f),
+            Self::Null => None,
+        }
+    }
+}
+
+impl Clone for Endpoint {
+    fn clone(&self) -> Self {
+        match self {
+            Self::File(f) => Self::File(f.try_clone().unwrap()),
+            Self::FilePair(f_out, f_in) => {
+                Self::FilePair(f_out.try_clone().unwrap(), f_in.try_clone().unwrap())
+            }
+            Self::Null => Self::Null,
+        }
+    }
+}
+
 impl ConsoleEpollHandler {
     /*
      * Each port of virtio console device has one receive
@@ -127,14 +165,10 @@ impl ConsoleEpollHandler {
         let mem = self.mem.memory();
         for avail_desc in trans_queue.iter(&mem) {
             let len;
-            let mut out = self.out.lock().unwrap();
-            let _ = mem.write_to(
-                avail_desc.addr,
-                &mut out.deref_mut(),
-                avail_desc.len as usize,
-            );
-            let _ = out.flush();
-
+            if let Some(ref mut out) = self.endpoint.out_file() {
+                let _ = mem.write_to(avail_desc.addr, out, avail_desc.len as usize);
+                let _ = out.flush();
+            }
             len = avail_desc.len;
             used_desc_heads[used_count] = (avail_desc.index, len);
             used_count += 1;
@@ -165,6 +199,9 @@ impl ConsoleEpollHandler {
         helper.add_event(self.output_queue_evt.as_raw_fd(), OUTPUT_QUEUE_EVENT)?;
         helper.add_event(self.input_evt.as_raw_fd(), INPUT_EVENT)?;
         helper.add_event(self.config_evt.as_raw_fd(), CONFIG_EVENT)?;
+        if let Some(in_file) = self.endpoint.in_file() {
+            helper.add_event(in_file.as_raw_fd(), FILE_EVENT)?;
+        }
         helper.run(paused, paused_sync, self)?;
 
         Ok(())
@@ -217,6 +254,22 @@ impl EpollHelperHandler for ConsoleEpollHandler {
                     return true;
                 }
             }
+            FILE_EVENT => {
+                let mut input = [0u8; 64];
+                if let Some(ref mut in_file) = self.endpoint.in_file() {
+                    if let Ok(count) = in_file.read(&mut input) {
+                        let mut in_buffer = self.in_buffer.lock().unwrap();
+                        in_buffer.extend(&input[..count]);
+                    }
+
+                    if self.process_input_queue() {
+                        if let Err(e) = self.signal_used_queue() {
+                            error!("Failed to signal used queue: {:?}", e);
+                            return true;
+                        }
+                    }
+                }
+            }
             _ => {
                 error!("Unknown event for virtio-console");
                 return true;
@@ -226,22 +279,14 @@ impl EpollHelperHandler for ConsoleEpollHandler {
     }
 }
 
-/// Input device.
-pub struct ConsoleInput {
-    input_evt: EventFd,
+/// Resize handler
+pub struct ConsoleResizer {
     config_evt: EventFd,
-    in_buffer: Arc<Mutex<VecDeque<u8>>>,
     config: Arc<Mutex<VirtioConsoleConfig>>,
     acked_features: AtomicU64,
 }
 
-impl ConsoleInput {
-    pub fn queue_input_bytes(&self, input: &[u8]) {
-        let mut in_buffer = self.in_buffer.lock().unwrap();
-        in_buffer.extend(input);
-        let _ = self.input_evt.write(1);
-    }
-
+impl ConsoleResizer {
     pub fn update_console_size(&self, cols: u16, rows: u16) {
         if self
             .acked_features
@@ -276,9 +321,11 @@ pub struct Console {
     common: VirtioCommon,
     id: String,
     config: Arc<Mutex<VirtioConsoleConfig>>,
-    input: Arc<ConsoleInput>,
-    out: Arc<Mutex<Box<dyn io::Write + Send + Sync + 'static>>>,
+    resizer: Arc<ConsoleResizer>,
+    endpoint: Endpoint,
     seccomp_action: SeccompAction,
+    in_buffer: Arc<Mutex<VecDeque<u8>>>,
+    exit_evt: EventFd,
 }
 
 #[derive(Versionize)]
@@ -295,25 +342,23 @@ impl Console {
     /// Create a new virtio console device that gets random data from /dev/urandom.
     pub fn new(
         id: String,
-        out: Box<dyn io::Write + Send + Sync + 'static>,
+        endpoint: Endpoint,
         cols: u16,
         rows: u16,
         iommu: bool,
         seccomp_action: SeccompAction,
-    ) -> io::Result<(Console, Arc<ConsoleInput>)> {
+        exit_evt: EventFd,
+    ) -> io::Result<(Console, Arc<ConsoleResizer>)> {
         let mut avail_features = 1u64 << VIRTIO_F_VERSION_1 | 1u64 << VIRTIO_CONSOLE_F_SIZE;
 
         if iommu {
             avail_features |= 1u64 << VIRTIO_F_IOMMU_PLATFORM;
         }
 
-        let input_evt = EventFd::new(EFD_NONBLOCK).unwrap();
         let config_evt = EventFd::new(EFD_NONBLOCK).unwrap();
         let console_config = Arc::new(Mutex::new(VirtioConsoleConfig::new(cols, rows)));
-        let console_input = Arc::new(ConsoleInput {
-            input_evt,
+        let resizer = Arc::new(ConsoleResizer {
             config_evt,
-            in_buffer: Arc::new(Mutex::new(VecDeque::new())),
             config: console_config.clone(),
             acked_features: AtomicU64::new(0),
         });
@@ -330,11 +375,13 @@ impl Console {
                 },
                 id,
                 config: console_config,
-                input: console_input.clone(),
-                out: Arc::new(Mutex::new(out)),
+                resizer: resizer.clone(),
+                endpoint,
                 seccomp_action,
+                in_buffer: Arc::new(Mutex::new(VecDeque::new())),
+                exit_evt,
             },
-            console_input,
+            resizer,
         ))
     }
 
@@ -343,7 +390,7 @@ impl Console {
             avail_features: self.common.avail_features,
             acked_features: self.common.acked_features,
             config: *(self.config.lock().unwrap()),
-            in_buffer: self.input.in_buffer.lock().unwrap().clone().into(),
+            in_buffer: self.in_buffer.lock().unwrap().clone().into(),
         }
     }
 
@@ -351,7 +398,7 @@ impl Console {
         self.common.avail_features = state.avail_features;
         self.common.acked_features = state.acked_features;
         *(self.config.lock().unwrap()) = state.config;
-        *(self.input.in_buffer.lock().unwrap()) = state.in_buffer.clone().into();
+        *(self.in_buffer.lock().unwrap()) = state.in_buffer.clone().into();
     }
 }
 
@@ -393,7 +440,7 @@ impl VirtioDevice for Console {
         mut queue_evts: Vec<EventFd>,
     ) -> ActivateResult {
         self.common.activate(&queues, &queue_evts, &interrupt_cb)?;
-        self.input
+        self.resizer
             .acked_features
             .store(self.common.acked_features, Ordering::Relaxed);
 
@@ -404,17 +451,18 @@ impl VirtioDevice for Console {
         }
 
         let (kill_evt, pause_evt) = self.common.dup_eventfds();
+        let input_evt = EventFd::new(EFD_NONBLOCK).unwrap();
 
         let mut handler = ConsoleEpollHandler {
             queues,
             mem,
             interrupt_cb,
-            in_buffer: self.input.in_buffer.clone(),
-            out: self.out.clone(),
+            in_buffer: self.in_buffer.clone(),
+            endpoint: self.endpoint.clone(),
             input_queue_evt: queue_evts.remove(0),
             output_queue_evt: queue_evts.remove(0),
-            input_evt: self.input.input_evt.try_clone().unwrap(),
-            config_evt: self.input.config_evt.try_clone().unwrap(),
+            input_evt,
+            config_evt: self.resizer.config_evt.try_clone().unwrap(),
             kill_evt,
             pause_evt,
         };
@@ -422,24 +470,19 @@ impl VirtioDevice for Console {
         let paused = self.common.paused.clone();
         let paused_sync = self.common.paused_sync.clone();
         let mut epoll_threads = Vec::new();
-        // Retrieve seccomp filter for virtio_console thread
-        let virtio_console_seccomp_filter =
-            get_seccomp_filter(&self.seccomp_action, Thread::VirtioConsole)
-                .map_err(ActivateError::CreateSeccompFilter)?;
-        thread::Builder::new()
-            .name(self.id.clone())
-            .spawn(move || {
-                if let Err(e) = SeccompFilter::apply(virtio_console_seccomp_filter) {
-                    error!("Error applying seccomp filter: {:?}", e);
-                } else if let Err(e) = handler.run(paused, paused_sync.unwrap()) {
+
+        spawn_virtio_thread(
+            &self.id,
+            &self.seccomp_action,
+            Thread::VirtioConsole,
+            &mut epoll_threads,
+            &self.exit_evt,
+            move || {
+                if let Err(e) = handler.run(paused, paused_sync.unwrap()) {
                     error!("Error running worker: {:?}", e);
                 }
-            })
-            .map(|thread| epoll_threads.push(thread))
-            .map_err(|e| {
-                error!("failed to clone the virtio-console epoll thread: {}", e);
-                ActivateError::BadActivate
-            })?;
+            },
+        )?;
 
         self.common.epoll_threads = Some(epoll_threads);
 

virtio-devices/src/device.rs

@@ -331,7 +331,7 @@ impl VirtioCommon {
 
 impl Pausable for VirtioCommon {
     fn pause(&mut self) -> std::result::Result<(), MigratableError> {
-        debug!(
+        info!(
             "Pausing virtio-{}",
             VirtioDeviceType::from(self.device_type)
         );
@@ -353,7 +353,7 @@ impl Pausable for VirtioCommon {
     }
 
     fn resume(&mut self) -> std::result::Result<(), MigratableError> {
-        debug!(
+        info!(
             "Resuming virtio-{}",
             VirtioDeviceType::from(self.device_type)
         );

virtio-devices/src/epoll_helper.rs

@@ -131,11 +131,11 @@ impl EpollHelper {
 
                 match ev_type {
                     EPOLL_HELPER_EVENT_KILL => {
-                        debug!("KILL_EVENT received, stopping epoll loop");
+                        info!("KILL_EVENT received, stopping epoll loop");
                         return Ok(());
                     }
                     EPOLL_HELPER_EVENT_PAUSE => {
-                        debug!("PAUSE_EVENT received, pausing epoll loop");
+                        info!("PAUSE_EVENT received, pausing epoll loop");
 
                         // Acknowledge the pause is effective by using the
                         // paused_sync barrier.

virtio-devices/src/iommu.rs

@@ -4,14 +4,14 @@
 
 use super::Error as DeviceError;
 use super::{
-    ActivateError, ActivateResult, DescriptorChain, EpollHelper, EpollHelperError,
-    EpollHelperHandler, Queue, VirtioCommon, VirtioDevice, VirtioDeviceType,
-    EPOLL_HELPER_EVENT_LAST, VIRTIO_F_VERSION_1,
+    ActivateResult, DescriptorChain, EpollHelper, EpollHelperError, EpollHelperHandler, Queue,
+    VirtioCommon, VirtioDevice, VirtioDeviceType, EPOLL_HELPER_EVENT_LAST, VIRTIO_F_VERSION_1,
 };
-use crate::seccomp_filters::{get_seccomp_filter, Thread};
+use crate::seccomp_filters::Thread;
+use crate::thread_helper::spawn_virtio_thread;
 use crate::GuestMemoryMmap;
 use crate::{DmaRemapping, VirtioInterrupt, VirtioInterruptType};
-use seccomp::{SeccompAction, SeccompFilter};
+use seccompiler::SeccompAction;
 use std::collections::BTreeMap;
 use std::fmt::{self, Display};
 use std::io;
@@ -21,7 +21,6 @@ use std::os::unix::io::AsRawFd;
 use std::result;
 use std::sync::atomic::AtomicBool;
 use std::sync::{Arc, Barrier, RwLock};
-use std::thread;
 use versionize::{VersionMap, Versionize, VersionizeResult};
 use versionize_derive::Versionize;
 use vm_device::dma_mapping::ExternalDmaMapping;
@@ -702,6 +701,7 @@ pub struct Iommu {
     mapping: Arc<IommuMapping>,
     ext_mapping: BTreeMap<u32, Arc<dyn ExternalDmaMapping>>,
     seccomp_action: SeccompAction,
+    exit_evt: EventFd,
 }
 
 #[derive(Versionize)]
@@ -715,7 +715,11 @@ struct IommuState {
 impl VersionMapped for IommuState {}
 
 impl Iommu {
-    pub fn new(id: String, seccomp_action: SeccompAction) -> io::Result<(Self, Arc<IommuMapping>)> {
+    pub fn new(
+        id: String,
+        seccomp_action: SeccompAction,
+        exit_evt: EventFd,
+    ) -> io::Result<(Self, Arc<IommuMapping>)> {
         let config = VirtioIommuConfig {
             page_size_mask: VIRTIO_IOMMU_PAGE_SIZE_MASK,
             probe_size: PROBE_PROP_SIZE,
@@ -743,6 +747,7 @@ impl Iommu {
                 mapping: mapping.clone(),
                 ext_mapping: BTreeMap::new(),
                 seccomp_action,
+                exit_evt,
             },
             mapping,
         ))
@@ -843,24 +848,18 @@ impl VirtioDevice for Iommu {
         let paused = self.common.paused.clone();
         let paused_sync = self.common.paused_sync.clone();
         let mut epoll_threads = Vec::new();
-        // Retrieve seccomp filter for virtio_iommu thread
-        let virtio_iommu_seccomp_filter =
-            get_seccomp_filter(&self.seccomp_action, Thread::VirtioIommu)
-                .map_err(ActivateError::CreateSeccompFilter)?;
-        thread::Builder::new()
-            .name(self.id.clone())
-            .spawn(move || {
-                if let Err(e) = SeccompFilter::apply(virtio_iommu_seccomp_filter) {
-                    error!("Error applying seccomp filter: {:?}", e);
-                } else if let Err(e) = handler.run(paused, paused_sync.unwrap()) {
+        spawn_virtio_thread(
+            &self.id,
+            &self.seccomp_action,
+            Thread::VirtioIommu,
+            &mut epoll_threads,
+            &self.exit_evt,
+            move || {
+                if let Err(e) = handler.run(paused, paused_sync.unwrap()) {
                     error!("Error running worker: {:?}", e);
                 }
-            })
-            .map(|thread| epoll_threads.push(thread))
-            .map_err(|e| {
-                error!("failed to clone the virtio-iommu epoll thread: {}", e);
-                ActivateError::BadActivate
-            })?;
+            },
+        )?;
 
         self.common.epoll_threads = Some(epoll_threads);
 

virtio-devices/src/lib.rs

@@ -32,6 +32,7 @@ pub mod net;
 mod pmem;
 mod rng;
 pub mod seccomp_filters;
+mod thread_helper;
 pub mod transport;
 pub mod vhost_user;
 pub mod vsock;
@@ -79,8 +80,12 @@ pub enum ActivateError {
     BadActivate,
     /// Queue number is not correct
     BadQueueNum,
-    /// Failed to clone Kill event
+    /// Failed to clone Kill event fd
     CloneKillEventFd,
+    /// Failed to clone exit event fd
+    CloneExitEventFd(std::io::Error),
+    // Failed to spawn thread
+    ThreadSpawn(std::io::Error),
     /// Failed to create Vhost-user interrupt eventfd
     VhostIrqCreate,
     /// Failed to setup vhost-user-fs daemon.
@@ -92,7 +97,7 @@ pub enum ActivateError {
     /// Failed to reset vhost-user daemon.
     VhostUserReset(vhost_user::Error),
     /// Cannot create seccomp filter
-    CreateSeccompFilter(seccomp::SeccompError),
+    CreateSeccompFilter(seccompiler::Error),
     /// Cannot create rate limiter
     CreateRateLimiter(std::io::Error),
 }
@@ -109,7 +114,7 @@ pub enum Error {
     VhostUserAddMemoryRegion(vhost_user::Error),
     SetShmRegionsNotSupported,
     NetQueuePair(::net_util::NetQueuePairError),
-    ApplySeccompFilter(seccomp::Error),
+    ApplySeccompFilter(seccompiler::Error),
 }
 
 #[derive(Clone, Copy, Debug, Default, Deserialize, Serialize, PartialEq)]

virtio-devices/src/mem.rs

@@ -18,12 +18,13 @@ use super::{
     EpollHelperHandler, Queue, VirtioCommon, VirtioDevice, VirtioDeviceType,
     EPOLL_HELPER_EVENT_LAST, VIRTIO_F_VERSION_1,
 };
-use crate::seccomp_filters::{get_seccomp_filter, Thread};
+use crate::seccomp_filters::Thread;
+use crate::thread_helper::spawn_virtio_thread;
 use crate::{GuestMemoryMmap, GuestRegionMmap};
 use crate::{VirtioInterrupt, VirtioInterruptType};
 use anyhow::anyhow;
 use libc::EFD_NONBLOCK;
-use seccomp::{SeccompAction, SeccompFilter};
+use seccompiler::SeccompAction;
 use std::collections::BTreeMap;
 use std::io;
 use std::mem::size_of;
@@ -32,7 +33,6 @@ use std::result;
 use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
 use std::sync::mpsc;
 use std::sync::{Arc, Barrier, Mutex};
-use std::thread;
 use vm_device::dma_mapping::ExternalDmaMapping;
 use vm_memory::{
     Address, ByteValued, Bytes, GuestAddress, GuestAddressSpace, GuestMemoryAtomic,
@@ -746,10 +746,12 @@ pub struct Mem {
     hugepages: bool,
     dma_mapping_handlers: Arc<Mutex<BTreeMap<u32, Arc<dyn ExternalDmaMapping>>>>,
     blocks_state: Arc<Mutex<BlocksState>>,
+    exit_evt: EventFd,
 }
 
 impl Mem {
     // Create a new virtio-mem device.
+    #[allow(clippy::too_many_arguments)]
     pub fn new(
         id: String,
         region: &Arc<GuestRegionMmap>,
@@ -758,6 +760,7 @@ impl Mem {
         numa_node_id: Option<u16>,
         initial_size: u64,
         hugepages: bool,
+        exit_evt: EventFd,
     ) -> io::Result<Mem> {
         let region_len = region.len();
 
@@ -835,6 +838,7 @@ impl Mem {
                 (config.region_size / config.block_size)
                     as usize
             ]))),
+            exit_evt,
         })
     }
 
@@ -955,23 +959,19 @@ impl VirtioDevice for Mem {
         let paused = self.common.paused.clone();
         let paused_sync = self.common.paused_sync.clone();
         let mut epoll_threads = Vec::new();
-        // Retrieve seccomp filter for virtio_mem thread
-        let virtio_mem_seccomp_filter = get_seccomp_filter(&self.seccomp_action, Thread::VirtioMem)
-            .map_err(ActivateError::CreateSeccompFilter)?;
-        thread::Builder::new()
-            .name(self.id.clone())
-            .spawn(move || {
-                if let Err(e) = SeccompFilter::apply(virtio_mem_seccomp_filter) {
-                    error!("Error applying seccomp filter: {:?}", e);
-                } else if let Err(e) = handler.run(paused, paused_sync.unwrap()) {
+
+        spawn_virtio_thread(
+            &self.id,
+            &self.seccomp_action,
+            Thread::VirtioMem,
+            &mut epoll_threads,
+            &self.exit_evt,
+            move || {
+                if let Err(e) = handler.run(paused, paused_sync.unwrap()) {
                     error!("Error running worker: {:?}", e);
                 }
-            })
-            .map(|thread| epoll_threads.push(thread))
-            .map_err(|e| {
-                error!("failed to clone virtio-mem epoll thread: {}", e);
-                ActivateError::BadActivate
-            })?;
+            },
+        )?;
         self.common.epoll_threads = Some(epoll_threads);
 
         event!("virtio-device", "activated", "id", &self.id);

virtio-devices/src/net.rs

@@ -11,7 +11,8 @@ use super::{
     RateLimiterConfig, VirtioCommon, VirtioDevice, VirtioDeviceType, VirtioInterruptType,
     EPOLL_HELPER_EVENT_LAST,
 };
-use crate::seccomp_filters::{get_seccomp_filter, Thread};
+use crate::seccomp_filters::Thread;
+use crate::thread_helper::spawn_virtio_thread;
 use crate::GuestMemoryMmap;
 use crate::VirtioInterrupt;
 use net_util::CtrlQueue;
@@ -20,7 +21,7 @@ use net_util::{
     virtio_features_to_tap_offload, MacAddr, NetCounters, NetQueuePair, OpenTapError, RxVirtio,
     Tap, TapError, TxVirtio, VirtioNetConfig,
 };
-use seccomp::{SeccompAction, SeccompFilter};
+use seccompiler::SeccompAction;
 use std::net::Ipv4Addr;
 use std::num::Wrapping;
 use std::os::unix::io::{AsRawFd, RawFd};
@@ -354,6 +355,7 @@ pub struct Net {
     counters: NetCounters,
     seccomp_action: SeccompAction,
     rate_limiter_config: Option<RateLimiterConfig>,
+    exit_evt: EventFd,
 }
 
 #[derive(Versionize)]
@@ -378,6 +380,7 @@ impl Net {
         queue_size: u16,
         seccomp_action: SeccompAction,
         rate_limiter_config: Option<RateLimiterConfig>,
+        exit_evt: EventFd,
     ) -> Result<Self> {
         let mut avail_features = 1 << VIRTIO_NET_F_CSUM
             | 1 << VIRTIO_NET_F_CTRL_GUEST_OFFLOADS
@@ -423,6 +426,7 @@ impl Net {
             counters: NetCounters::default(),
             seccomp_action,
             rate_limiter_config,
+            exit_evt,
         })
     }
 
@@ -441,6 +445,7 @@ impl Net {
         queue_size: u16,
         seccomp_action: SeccompAction,
         rate_limiter_config: Option<RateLimiterConfig>,
+        exit_evt: EventFd,
     ) -> Result<Self> {
         let taps = open_tap(if_name, ip_addr, netmask, host_mac, num_queues / 2, None)
             .map_err(Error::OpenTap)?;
@@ -454,9 +459,11 @@ impl Net {
             queue_size,
             seccomp_action,
             rate_limiter_config,
+            exit_evt,
         )
     }
 
+    #[allow(clippy::too_many_arguments)]
     pub fn from_tap_fds(
         id: String,
         fds: &[RawFd],
@@ -465,6 +472,7 @@ impl Net {
         queue_size: u16,
         seccomp_action: SeccompAction,
         rate_limiter_config: Option<RateLimiterConfig>,
+        exit_evt: EventFd,
     ) -> Result<Self> {
         let mut taps: Vec<Tap> = Vec::new();
         let num_queue_pairs = fds.len();
@@ -488,6 +496,7 @@ impl Net {
             queue_size,
             seccomp_action,
             rate_limiter_config,
+            exit_evt,
         )
     }
 
@@ -569,24 +578,20 @@ impl VirtioDevice for Net {
             self.common.paused_sync = Some(Arc::new(Barrier::new(self.taps.len() + 2)));
             let paused_sync = self.common.paused_sync.clone();
 
-            // Retrieve seccomp filter for virtio_net_ctl thread
-            let virtio_net_ctl_seccomp_filter =
-                get_seccomp_filter(&self.seccomp_action, Thread::VirtioNetCtl)
-                    .map_err(ActivateError::CreateSeccompFilter)?;
-            thread::Builder::new()
-                .name(format!("{}_ctrl", self.id))
-                .spawn(move || {
-                    if let Err(e) = SeccompFilter::apply(virtio_net_ctl_seccomp_filter) {
-                        error!("Error applying seccomp filter: {:?}", e);
-                    } else if let Err(e) = ctrl_handler.run_ctrl(paused, paused_sync.unwrap()) {
+            let mut epoll_threads = Vec::new();
+            spawn_virtio_thread(
+                &format!("{}_ctrl", &self.id),
+                &self.seccomp_action,
+                Thread::VirtioNetCtl,
+                &mut epoll_threads,
+                &self.exit_evt,
+                move || {
+                    if let Err(e) = ctrl_handler.run_ctrl(paused, paused_sync.unwrap()) {
                         error!("Error running worker: {:?}", e);
                     }
-                })
-                .map(|thread| self.ctrl_queue_epoll_thread = Some(thread))
-                .map_err(|e| {
-                    error!("failed to clone queue EventFd: {}", e);
-                    ActivateError::BadActivate
-                })?;
+                },
+            )?;
+            self.ctrl_queue_epoll_thread = Some(epoll_threads.remove(0));
         }
 
         let event_idx = self.common.feature_acked(VIRTIO_RING_F_EVENT_IDX.into());
@@ -652,24 +657,19 @@ impl VirtioDevice for Net {
 
             let paused = self.common.paused.clone();
             let paused_sync = self.common.paused_sync.clone();
-            // Retrieve seccomp filter for virtio_net thread
-            let virtio_net_seccomp_filter =
-                get_seccomp_filter(&self.seccomp_action, Thread::VirtioNet)
-                    .map_err(ActivateError::CreateSeccompFilter)?;
-            thread::Builder::new()
-                .name(format!("{}_qp{}", self.id.clone(), i))
-                .spawn(move || {
-                    if let Err(e) = SeccompFilter::apply(virtio_net_seccomp_filter) {
-                        error!("Error applying seccomp filter: {:?}", e);
-                    } else if let Err(e) = handler.run(paused, paused_sync.unwrap()) {
+
+            spawn_virtio_thread(
+                &format!("{}_qp{}", self.id.clone(), i),
+                &self.seccomp_action,
+                Thread::VirtioNet,
+                &mut epoll_threads,
+                &self.exit_evt,
+                move || {
+                    if let Err(e) = handler.run(paused, paused_sync.unwrap()) {
                         error!("Error running worker: {:?}", e);
                     }
-                })
-                .map(|thread| epoll_threads.push(thread))
-                .map_err(|e| {
-                    error!("failed to clone queue EventFd: {}", e);
-                    ActivateError::BadActivate
-                })?;
+                },
+            )?;
         }
 
         self.common.epoll_threads = Some(epoll_threads);

virtio-devices/src/pmem.rs

@@ -12,10 +12,11 @@ use super::{
     EpollHelperHandler, Queue, UserspaceMapping, VirtioCommon, VirtioDevice, VirtioDeviceType,
     EPOLL_HELPER_EVENT_LAST, VIRTIO_F_IOMMU_PLATFORM, VIRTIO_F_VERSION_1,
 };
-use crate::seccomp_filters::{get_seccomp_filter, Thread};
+use crate::seccomp_filters::Thread;
+use crate::thread_helper::spawn_virtio_thread;
 use crate::{GuestMemoryMmap, MmapRegion};
 use crate::{VirtioInterrupt, VirtioInterruptType};
-use seccomp::{SeccompAction, SeccompFilter};
+use seccompiler::SeccompAction;
 use std::fmt::{self, Display};
 use std::fs::File;
 use std::io;
@@ -24,7 +25,6 @@ use std::os::unix::io::AsRawFd;
 use std::result;
 use std::sync::atomic::AtomicBool;
 use std::sync::{Arc, Barrier};
-use std::thread;
 use versionize::{VersionMap, Versionize, VersionizeResult};
 use versionize_derive::Versionize;
 use vm_memory::{
@@ -264,6 +264,7 @@ pub struct Pmem {
     config: VirtioPmemConfig,
     mapping: UserspaceMapping,
     seccomp_action: SeccompAction,
+    exit_evt: EventFd,
 
     // Hold ownership of the memory that is allocated for the device
     // which will be automatically dropped when the device is dropped
@@ -280,6 +281,7 @@ pub struct PmemState {
 impl VersionMapped for PmemState {}
 
 impl Pmem {
+    #[allow(clippy::too_many_arguments)]
     pub fn new(
         id: String,
         disk: File,
@@ -288,6 +290,7 @@ impl Pmem {
         _region: MmapRegion,
         iommu: bool,
         seccomp_action: SeccompAction,
+        exit_evt: EventFd,
     ) -> io::Result<Pmem> {
         let config = VirtioPmemConfig {
             start: addr.raw_value().to_le(),
@@ -315,6 +318,7 @@ impl Pmem {
             mapping,
             seccomp_action,
             _region,
+            exit_evt,
         })
     }
 
@@ -390,24 +394,19 @@ impl VirtioDevice for Pmem {
             let paused = self.common.paused.clone();
             let paused_sync = self.common.paused_sync.clone();
             let mut epoll_threads = Vec::new();
-            // Retrieve seccomp filter for virtio_pmem thread
-            let virtio_pmem_seccomp_filter =
-                get_seccomp_filter(&self.seccomp_action, Thread::VirtioPmem)
-                    .map_err(ActivateError::CreateSeccompFilter)?;
-            thread::Builder::new()
-                .name(self.id.clone())
-                .spawn(move || {
-                    if let Err(e) = SeccompFilter::apply(virtio_pmem_seccomp_filter) {
-                        error!("Error applying seccomp filter: {:?}", e);
-                    } else if let Err(e) = handler.run(paused, paused_sync.unwrap()) {
+
+            spawn_virtio_thread(
+                &self.id,
+                &self.seccomp_action,
+                Thread::VirtioPmem,
+                &mut epoll_threads,
+                &self.exit_evt,
+                move || {
+                    if let Err(e) = handler.run(paused, paused_sync.unwrap()) {
                         error!("Error running worker: {:?}", e);
                     }
-                })
-                .map(|thread| epoll_threads.push(thread))
-                .map_err(|e| {
-                    error!("failed to clone virtio-pmem epoll thread: {}", e);
-                    ActivateError::BadActivate
-                })?;
+                },
+            )?;
 
             self.common.epoll_threads = Some(epoll_threads);
 

virtio-devices/src/rng.rs

@@ -8,17 +8,17 @@ use super::{
     VirtioCommon, VirtioDevice, VirtioDeviceType, EPOLL_HELPER_EVENT_LAST, VIRTIO_F_IOMMU_PLATFORM,
     VIRTIO_F_VERSION_1,
 };
-use crate::seccomp_filters::{get_seccomp_filter, Thread};
+use crate::seccomp_filters::Thread;
+use crate::thread_helper::spawn_virtio_thread;
 use crate::GuestMemoryMmap;
 use crate::{VirtioInterrupt, VirtioInterruptType};
-use seccomp::{SeccompAction, SeccompFilter};
+use seccompiler::SeccompAction;
 use std::fs::File;
 use std::io;
 use std::os::unix::io::AsRawFd;
 use std::result;
 use std::sync::atomic::AtomicBool;
 use std::sync::{Arc, Barrier};
-use std::thread;
 use versionize::{VersionMap, Versionize, VersionizeResult};
 use versionize_derive::Versionize;
 use vm_memory::{Bytes, GuestAddressSpace, GuestMemoryAtomic};
@@ -129,6 +129,7 @@ pub struct Rng {
     id: String,
     random_file: Option<File>,
     seccomp_action: SeccompAction,
+    exit_evt: EventFd,
 }
 
 #[derive(Versionize)]
@@ -146,6 +147,7 @@ impl Rng {
         path: &str,
         iommu: bool,
         seccomp_action: SeccompAction,
+        exit_evt: EventFd,
     ) -> io::Result<Rng> {
         let random_file = File::open(path)?;
         let mut avail_features = 1u64 << VIRTIO_F_VERSION_1;
@@ -166,6 +168,7 @@ impl Rng {
             id,
             random_file: Some(random_file),
             seccomp_action,
+            exit_evt,
         })
     }
 
@@ -236,24 +239,18 @@ impl VirtioDevice for Rng {
             let paused = self.common.paused.clone();
             let paused_sync = self.common.paused_sync.clone();
             let mut epoll_threads = Vec::new();
-            // Retrieve seccomp filter for virtio_rng thread
-            let virtio_rng_seccomp_filter =
-                get_seccomp_filter(&self.seccomp_action, Thread::VirtioRng)
-                    .map_err(ActivateError::CreateSeccompFilter)?;
-            thread::Builder::new()
-                .name(self.id.clone())
-                .spawn(move || {
-                    if let Err(e) = SeccompFilter::apply(virtio_rng_seccomp_filter) {
-                        error!("Error applying seccomp filter: {:?}", e);
-                    } else if let Err(e) = handler.run(paused, paused_sync.unwrap()) {
+            spawn_virtio_thread(
+                &self.id,
+                &self.seccomp_action,
+                Thread::VirtioRng,
+                &mut epoll_threads,
+                &self.exit_evt,
+                move || {
+                    if let Err(e) = handler.run(paused, paused_sync.unwrap()) {
                         error!("Error running worker: {:?}", e);
                     }
-                })
-                .map(|thread| epoll_threads.push(thread))
-                .map_err(|e| {
-                    error!("failed to clone the virtio-rng epoll thread: {}", e);
-                    ActivateError::BadActivate
-                })?;
+                },
+            )?;
 
             self.common.epoll_threads = Some(epoll_threads);