build: Release v53.0

Signed-off-by: Rob Bradford <rbradford@meta.com>

.config/nextest.toml

@@ -1,3 +1,38 @@
 [profile.default]
 # Don't let one individual test run for more than 10 minutes
 slow-timeout = { period = "60s", terminate-after = 10 }
+
+[profile.integration]
+fail-fast = false
+retries = 3
+
+[profile.common_tests]
+inherits = "integration"
+default-filter = 'test(common_parallel::) | test(common_sequential::) | test(aarch64_acpi::)'
+junit.path = "/root/workloads/junit/common.xml"
+
+[[profile.common_tests.overrides]]
+filter = 'test(common_sequential::)'
+# use up all the available test threads for each of the sequential tests
+# i.e. no other test can be running while a sequential test is running.
+threads-required = 'num-test-threads'
+
+[profile.dbus]
+inherits = "integration"
+default-filter = 'test(dbus_api::)'
+junit.path = "/root/workloads/junit/dbus.xml"
+
+[profile.fw_cfg]
+inherits = "integration"
+default-filter = 'test(fw_cfg::)'
+junit.path = "/root/workloads/junit/fw_cfg.xml"
+
+[profile.ivshmem]
+inherits = "integration"
+default-filter = 'test(ivshmem::)'
+junit.path = "/root/workloads/junit/ivshmem.xml"
+
+[profile.common_cvm]
+inherits = "integration"
+default-filter = 'test(common_cvm::)'
+junit.path = "/root/workloads/junit/cvm.xml"

.github/dependabot.yml

@@ -8,6 +8,7 @@ updates:
     interval: weekly
   allow:
     - dependency-name: "acpi_tables"
+    - dependency-name: "iommufd-ioctls"
     - dependency-name: "kvm-bindings"
     - dependency-name: "kvm-ioctls"
     - dependency-name: "linux-loader"
@@ -44,6 +45,7 @@ updates:
     semver-patch-days: 3
   ignore:
     - dependency-name: "acpi_tables"
+    - dependency-name: "iommufd-ioctls"
     - dependency-name: "kvm-bindings"
     - dependency-name: "kvm-ioctls"
     - dependency-name: "linux-loader"

.github/workflows/ci.yaml

@@ -20,11 +20,11 @@ jobs:
       ci: ${{ steps.changes.outputs.ci }}
       docs: ${{ steps.changes.outputs.docs }}
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
         with:
           fetch-depth: 0
       - id: changes
-        uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d # v4.0.1
+        uses: dorny/paths-filter@7b450fff21473bca461d4b92ce414b9d0420d706 # v4.0.2
         with:
           filters: |
             rust:
@@ -69,7 +69,7 @@ jobs:
     needs: [preflight]
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
       - name: Set up Python 3.x
         uses: actions/setup-python@v6
         with:
@@ -90,7 +90,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
         with:
           # PR head, not the merge ref, so gitlint sees the PR's commits.
           ref: ${{ github.event.pull_request.head.sha }}
@@ -113,7 +113,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Code checkout
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
         with:
           fetch-depth: 0
       - name: Get changed files in PR
@@ -139,7 +139,7 @@ jobs:
           fi
       - name: Link Availability Check (Diff Only)
         if: ${{ steps.changed-files.outputs.all_changed_files != '' }}
-        uses: lycheeverse/lychee-action@8646ba30535128ac92d33dfc9133794bfdd9b411 # v2.8.0
+        uses: lycheeverse/lychee-action@e7477775783ea5526144ba13e8db5eec57747ce8 # v2.9.0
         with:
           args: --verbose --config .lychee.toml ${{ steps.changed-files.outputs.all_changed_files }}
           failIfEmpty: false
@@ -151,7 +151,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Code checkout
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
       - name: Install Rust toolchain
         uses: dtolnay/rust-toolchain@stable
       - name: Install build dependencies
@@ -166,7 +166,7 @@ jobs:
     if: needs.preflight.outputs.cargo == 'true'
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
       - uses: actions-rust-lang/audit@v1
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
@@ -177,7 +177,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
       - name: Run the shell script checkers
         uses: luizm/action-sh-checker@883217215b11c1fabbf00eb1a9a041f62d74c744 # v0.10.0
         env:
@@ -190,7 +190,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
       - name: Lint Dockerfile
         uses: hadolint/hadolint-action@2332a7b74a6de0dda2e2221d575162eba76ba5e5 # v3.3.0
         with:
@@ -205,7 +205,7 @@ jobs:
     if: needs.preflight.outputs.full == 'true' || needs.preflight.outputs.cargo == 'true'
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
       - name: REUSE Compliance Check
         uses: fsfe/reuse-action@v6
   formatting:
@@ -223,7 +223,7 @@ jobs:
       RUSTFLAGS: -D warnings
     steps:
       - name: Code checkout
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
       - name: Install Rust toolchain (${{ matrix.rust }})
         uses: dtolnay/rust-toolchain@stable
         with:
@@ -241,7 +241,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Code checkout
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
         with:
           fetch-depth: 0
       - name: Install dependencies
@@ -271,7 +271,7 @@ jobs:
       RUSTFLAGS: -D warnings
     steps:
       - name: Code checkout
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
       - name: Install Rust toolchain (${{ matrix.rust }})
         uses: dtolnay/rust-toolchain@stable
         with:
@@ -290,7 +290,7 @@ jobs:
     runs-on: ubuntu-latest
     container: openapitools/openapi-generator-cli
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
       - name: Validate OpenAPI
         run: |
           /usr/local/bin/docker-entrypoint.sh validate -i vmm/src/api/openapi/cloud-hypervisor.yaml
@@ -300,8 +300,8 @@ jobs:
     if: github.event_name == 'pull_request'
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v6
-      - uses: crate-ci/typos@5374cbf686e897b15713110e233094e2874de7ef # v1.46.1
+      - uses: actions/checkout@v7
+      - uses: crate-ci/typos@bee27e3a4fd1ea2111cf90ab89cd076c870fce14 # v1.48.0
   quality:
     name: quality
     needs: [preflight]
@@ -328,7 +328,7 @@ jobs:
             experimental: false
     steps:
       - name: Code checkout
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
         with:
           fetch-depth: 0
       - name: Install Rust toolchain (${{ matrix.rust }})
@@ -408,30 +408,46 @@ jobs:
           toolchain: ${{ matrix.rust }}
           target: ${{ matrix.target }}
           args: --locked --all --all-targets --tests --examples --features "ivshmem" -- -D warnings
-      - name: Clippy (sev_snp)
+      - name: Clippy (kvm + sev_snp)
         if: ${{ matrix.target == 'x86_64-unknown-linux-gnu' }}
         uses: houseabsolute/actions-rust-cross@v1
         with:
           command: clippy
           toolchain: ${{ matrix.rust }}
           target: ${{ matrix.target }}
-          args: --locked --all --all-targets --no-default-features --tests --examples --features "sev_snp" -- -D warnings
-      - name: Clippy (igvm)
+          args: --locked --all --all-targets --no-default-features --tests --examples --features "kvm,sev_snp" -- -D warnings
+      - name: Clippy (mshv + sev_snp)
         if: ${{ matrix.target == 'x86_64-unknown-linux-gnu' }}
         uses: houseabsolute/actions-rust-cross@v1
         with:
           command: clippy
           toolchain: ${{ matrix.rust }}
           target: ${{ matrix.target }}
-          args: --locked --all --all-targets --no-default-features --tests --examples --features "igvm" -- -D warnings
-      - name: Clippy (kvm + tdx)
+          args: --locked --all --all-targets --no-default-features --tests --examples --features "mshv,sev_snp" -- -D warnings
+      - name: Clippy (mshv + igvm + sev_snp)
         if: ${{ matrix.target == 'x86_64-unknown-linux-gnu' }}
         uses: houseabsolute/actions-rust-cross@v1
         with:
           command: clippy
           toolchain: ${{ matrix.rust }}
           target: ${{ matrix.target }}
-          args: --locked --all --all-targets --no-default-features --tests --examples --features "tdx,kvm" -- -D warnings
+          args: --locked --all --all-targets --no-default-features --tests --examples --features "mshv,igvm,sev_snp" -- -D warnings
+      - name: Clippy (kvm + igvm)
+        if: ${{ matrix.target == 'x86_64-unknown-linux-gnu' }}
+        uses: houseabsolute/actions-rust-cross@v1
+        with:
+          command: clippy
+          toolchain: ${{ matrix.rust }}
+          target: ${{ matrix.target }}
+          args: --locked --all --all-targets --no-default-features --tests --examples --features "kvm,igvm" -- -D warnings
+      - name: Clippy (mshv + igvm)
+        if: ${{ matrix.target == 'x86_64-unknown-linux-gnu' }}
+        uses: houseabsolute/actions-rust-cross@v1
+        with:
+          command: clippy
+          toolchain: ${{ matrix.rust }}
+          target: ${{ matrix.target }}
+          args: --locked --all --all-targets --no-default-features --tests --examples --features "mshv,igvm" -- -D warnings
       - name: Clippy (kvm + igvm + sev_snp + fw_cfg)
         if: ${{ matrix.target == 'x86_64-unknown-linux-gnu' }}
         uses: houseabsolute/actions-rust-cross@v1
@@ -470,7 +486,7 @@ jobs:
           - x86_64-unknown-linux-musl
     steps:
       - name: Code checkout
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
         with:
           fetch-depth: 0
       - name: Install musl-gcc
@@ -484,8 +500,6 @@ jobs:
         run: cargo build --locked --bin cloud-hypervisor
       - name: Build (kvm)
         run: cargo build --locked --bin cloud-hypervisor --no-default-features --features "kvm"
-      - name: Build (default features + tdx)
-        run: cargo build --locked --bin cloud-hypervisor --features "tdx"
       - name: Build (default features + dbus_api)
         run: cargo build --locked --bin cloud-hypervisor --features "dbus_api"
       - name: Build (default features + guest_debug)
@@ -498,18 +512,53 @@ jobs:
         run: cargo build --locked --bin cloud-hypervisor --features "ivshmem"
       - name: Build (mshv)
         run: cargo build --locked --bin cloud-hypervisor --no-default-features --features "mshv"
-      - name: Build (sev_snp)
-        run: cargo build --locked --bin cloud-hypervisor --no-default-features --features "sev_snp"
+      - name: Build (mshv + igvm)
+        run: cargo build --locked --bin cloud-hypervisor --no-default-features --features "mshv,igvm"
+      - name: Build (mshv + sev_snp)
+        run: cargo build --locked --bin cloud-hypervisor --no-default-features --features "mshv,sev_snp"
+      - name: Build (mshv + igvm + sev_snp)
+        run: cargo build --locked --bin cloud-hypervisor --no-default-features --features "mshv,igvm,sev_snp"
+      - name: Build (kvm + sev_snp)
+        run: cargo build --locked --bin cloud-hypervisor --no-default-features --features "kvm,sev_snp"
       - name: Build (kvm + igvm + sev_snp + fw_cfg)
         run: cargo build --locked --bin cloud-hypervisor --no-default-features --features "kvm,igvm,sev_snp,fw_cfg"
-      - name: Build (igvm)
-        run: cargo build --locked --bin cloud-hypervisor --no-default-features --features "igvm"
+      - name: Build (kvm + igvm)
+        run: cargo build --locked --bin cloud-hypervisor --no-default-features --features "kvm,igvm"
       - name: Build (mshv + kvm)
         run: cargo build --locked --bin cloud-hypervisor --no-default-features --features "mshv,kvm"
       - name: Release Build (default features)
         run: cargo build --locked --all --release --target=${{ matrix.target }}
       - name: Check build did not modify any files
         run: test -z "$(git status --porcelain)"
+  build-riscv64:
+    name: build-riscv64
+    needs: [preflight]
+    if: needs.preflight.outputs.full == 'true'
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        rust:
+          - stable
+          - "1.89.0" # MSRV — keep quoted.
+    env:
+      CARGO_TARGET_RISCV64GC_UNKNOWN_LINUX_GNU_LINKER: riscv64-linux-gnu-gcc
+    steps:
+      - name: Code checkout
+        uses: actions/checkout@v7
+        with:
+          fetch-depth: 0
+      - name: Install riscv64 cross linker
+        run: sudo apt-get update && sudo apt-get install -y gcc-riscv64-linux-gnu
+      - name: Install Rust toolchain (${{ matrix.rust }})
+        uses: dtolnay/rust-toolchain@stable
+        with:
+          toolchain: ${{ matrix.rust }}
+          target: riscv64gc-unknown-linux-gnu
+      - name: Build (kvm)
+        run: cargo build --locked --package cloud-hypervisor --no-default-features --features "kvm" --target riscv64gc-unknown-linux-gnu
+      - name: Check build did not modify any files
+        run: test -z "$(git status --porcelain)"
   # garm-jammy + gnu: runs on PR and MQ. Other 3 matrix entries are in
   # integration-x86-64-mq (sibling, MQ-only, runs in parallel).
   integration-x86-64-pr:
@@ -526,7 +575,7 @@ jobs:
     runs-on: garm-jammy-16
     steps:
       - name: Code checkout
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
         with:
           fetch-depth: 0
       - name: Install Docker
@@ -571,7 +620,7 @@ jobs:
     runs-on: ${{ format('{0}-16', matrix.runner) }}
     steps:
       - name: Code checkout
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
         with:
           fetch-depth: 0
       - name: Install Docker
@@ -610,7 +659,7 @@ jobs:
       - name: Fix workspace permissions
         run: sudo chown -R runner:runner ${GITHUB_WORKSPACE}
       - name: Code checkout
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
         with:
           fetch-depth: 0
       - name: Run unit tests (musl)
@@ -635,7 +684,7 @@ jobs:
           IMG_BASENAME=windows-11-iot-enterprise-aarch64.raw
           IMG_PATH=$HOME/workloads/$IMG_BASENAME
           IMG_GZ_PATH=$HOME/workloads/$IMG_BASENAME.gz
-          IMG_GZ_BLOB_NAME=windows-11-iot-enterprise-aarch64-9-min.raw.gz
+          IMG_GZ_BLOB_NAME=windows-11-iot-enterprise-aarch64-25h2-6.raw.gz
           cp "scripts/$IMG_BASENAME.sha1" "$HOME/workloads/"
           pushd "$HOME/workloads"
           if sha1sum "$IMG_BASENAME.sha1" --check; then
@@ -643,6 +692,7 @@ jobs:
           fi
           popd
           mkdir -p "$HOME/workloads"
+          rm -f "$IMG_PATH" "$IMG_GZ_PATH"
           az storage blob download --container-name private-images --file "$IMG_GZ_PATH" --name "$IMG_GZ_BLOB_NAME" --connection-string "${{ secrets.CH_PRIVATE_IMAGES }}"
           gzip -d "$IMG_GZ_PATH"
       - name: Run Windows guest integration tests
@@ -661,7 +711,7 @@ jobs:
       - name: Fix workspace permissions
         run: sudo chown -R github-runner:github-runner "${GITHUB_WORKSPACE}"
       - name: Code checkout
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
         with:
           fetch-depth: 0
       - name: Run VFIO integration tests
@@ -679,7 +729,7 @@ jobs:
     runs-on: garm-jammy-16
     steps:
       - name: Code checkout
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
         with:
           fetch-depth: 0
       - name: Install Docker
@@ -711,38 +761,108 @@ jobs:
       - name: Run Windows guest integration tests for musl
         timeout-minutes: 15
         run: scripts/dev_cli.sh tests --integration-windows --libc musl
-  integration-rate-limiter:
-    name: integration-rate-limiter
+  integration-mshv-x86-64:
+    name: integration-mshv-x86-64
     needs: [preflight, dco, quality, build]
     if: >-
       github.event_name == 'merge_group' && needs.preflight.outputs.full == 'true' && needs.dco.result == 'success' && needs.quality.result == 'success' && needs.build.result == 'success'
-    runs-on: bare-metal-9950x
-    env:
-      AUTH_DOWNLOAD_TOKEN: ${{ secrets.AUTH_DOWNLOAD_TOKEN }}
+    timeout-minutes: 35
+    runs-on: mshv
     steps:
+      # mshv runner user is "lsgunner"
+      - name: Fix workspace and Docker socket permissions
+        run: |
+          sudo chown -R lsgrunner:lsgrunner ${GITHUB_WORKSPACE}
+          sudo chmod 666 /var/run/docker.sock
       - name: Code checkout
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
         with:
           fetch-depth: 0
-      - name: Run rate-limiter integration tests
+      - name: Prepare for VDPA
+        run: scripts/prepare_vdpa.sh
+      - name: Run integration tests
+        timeout-minutes: 30
+        run: scripts/dev_cli.sh tests --integration
+  # Rate-limiter host is not available
+  # integration-rate-limiter:
+  #   name: integration-rate-limiter
+  #   needs: [preflight, dco, quality, build]
+  #   if: >-
+  #     github.event_name == 'merge_group' && needs.preflight.outputs.full == 'true' && needs.dco.result == 'success' && needs.quality.result == 'success' && needs.build.result == 'success'
+  #   runs-on: bare-metal-9950x
+  #   env:
+  #     AUTH_DOWNLOAD_TOKEN: ${{ secrets.AUTH_DOWNLOAD_TOKEN }}
+  #   steps:
+  #     - name: Code checkout
+  #       uses: actions/checkout@v7
+  #       with:
+  #         fetch-depth: 0
+  #     - name: Run rate-limiter integration tests
+  #       timeout-minutes: 20
+  #       run: scripts/dev_cli.sh tests --integration-rate-limiter
+  integration-sev-snp:
+    name: integration-sev-snp
+    needs: [preflight, dco, quality, build]
+    if: >-
+      github.event_name == 'merge_group' && needs.preflight.outputs.full == 'true' && needs.dco.result == 'success' && needs.quality.result == 'success' && needs.build.result == 'success'
+    timeout-minutes: 30
+    runs-on: noble-sevsnp
+    steps:
+      # Self-hosted runners reuse their workdir; a previous privileged
+      # container run can leave root-owned files behind.
+      - name: Fix workspace permissions
+        run: sudo chown -R "$(id -un):$(id -gn)" "${GITHUB_WORKSPACE}"
+      - name: Code checkout
+        uses: actions/checkout@v7
+        with:
+          fetch-depth: 0
+      - name: Sanity-check SEV-SNP prerequisites
+        run: |
+          set -eufo pipefail
+          echo "Checking hypervisor device nodes..."
+          test -e /dev/kvm || { echo "::error::/dev/kvm missing"; exit 1; }
+          test -e /dev/sev || { echo "::error::/dev/sev missing"; exit 1; }
+          echo "Checking staged IGVM/kernel artifacts..."
+          test -d /usr/share/cloud-hypervisor/cvm \
+            || { echo "::error::/usr/share/cloud-hypervisor/cvm missing"; exit 1; }
+          ls -l /usr/share/cloud-hypervisor/cvm
+      - name: Run CVM (SEV-SNP) integration tests
         timeout-minutes: 20
-        run: scripts/dev_cli.sh tests --integration-rate-limiter
+        run: scripts/dev_cli.sh tests --integration-cvm --hypervisor kvm
+  # Rate-limiter host is not available
+  # integration-rate-limiter:
+  #   name: integration-rate-limiter
+  #   needs: [preflight, dco, quality, build]
+  #   if: >-
+  #     github.event_name == 'merge_group' && needs.preflight.outputs.full == 'true' && needs.dco.result == 'success' && needs.quality.result == 'success' && needs.build.result == 'success'
+  #   runs-on: bare-metal-9950x
+  #   env:
+  #     AUTH_DOWNLOAD_TOKEN: ${{ secrets.AUTH_DOWNLOAD_TOKEN }}
+  #   steps:
+  #     - name: Code checkout
+  #       uses: actions/checkout@v7
+  #       with:
+  #         fetch-depth: 0
+  #     - name: Run rate-limiter integration tests
+  #       timeout-minutes: 20
+  #       run: scripts/dev_cli.sh tests --integration-rate-limiter
   # The single required-status check. Branch protection requires this one job.
   all-green:
     name: all-green
     needs:
       - audit
       - build
+      - build-riscv64
       - dco
       - formatting
       - fuzz-build
       - gitlint
       - hadolint
       - integration-arm64
-      # VFIO worker is failing #8160
-      # - integration-vfio
-      # See: #8211
-      # - integration-windows
+      - integration-sev-snp
+      - integration-vfio
+      - integration-mshv-x86-64
+      - integration-windows
       - integration-x86-64-mq
       - integration-x86-64-pr
       - openapi

.github/workflows/docker-image.yaml

@@ -14,11 +14,28 @@ env:
   IMAGE_NAME: ${{ github.repository }}
 
 jobs:
-  main:
+  build:
+    strategy:
+      fail-fast: false
+      matrix:
+        platform:
+          - linux/amd64
+          - linux/arm64
     runs-on: ubuntu-latest
     steps:
+      - name: Prepare
+        run: |
+          platform=${{ matrix.platform }}
+          echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV
+
       - name: Code checkout
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
+
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v6
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
 
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v4
@@ -27,12 +44,52 @@ jobs:
         uses: docker/setup-buildx-action@v4
 
       - name: Login to ghcr
+        if: ${{ github.event_name == 'push' }}
         uses: docker/login-action@v4
         with:
           registry: ${{ env.REGISTRY }}
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
+      - name: Build and push by digest
+        id: build
+        uses: docker/build-push-action@v7
+        with:
+          file: ./resources/Dockerfile
+          platforms: ${{ matrix.platform }}
+          labels: ${{ steps.meta.outputs.labels }}
+          outputs: type=image,name=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }},push-by-digest=true,name-canonical=true,push=${{ github.event_name == 'push' }}
+
+      - name: Export digest
+        if: ${{ github.event_name == 'push' }}
+        run: |
+          mkdir -p /tmp/digests
+          digest="${{ steps.build.outputs.digest }}"
+          touch "/tmp/digests/${digest#sha256:}"
+
+      - name: Upload digest
+        if: ${{ github.event_name == 'push' }}
+        uses: actions/upload-artifact@v7
+        with:
+          name: digests-${{ env.PLATFORM_PAIR }}
+          path: /tmp/digests/*
+          if-no-files-found: error
+          retention-days: 1
+
+  merge:
+    runs-on: ubuntu-latest
+    needs: build
+    if: ${{ github.event_name == 'push' }}
+    steps:
+      - name: Download digests
+        uses: actions/download-artifact@v8
+        with:
+          path: /tmp/digests
+          pattern: digests-*
+          merge-multiple: true
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v4
 
       - name: Docker meta
         id: meta
@@ -41,25 +98,22 @@ jobs:
           images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
           # generate Docker tags based on the following events/attributes
           tags: |
-            type=raw,value=20251114-0
+            type=raw,value=20260522-0
             type=sha
 
-      - name: Build and push
-        if: ${{ github.event_name == 'push' }}
-        uses: docker/build-push-action@v7
+      - name: Login to ghcr
+        uses: docker/login-action@v4
         with:
-          file: ./resources/Dockerfile
-          platforms: linux/amd64,linux/arm64
-          push: true
-          tags: ${{ steps.meta.outputs.tags }}
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Build only
-        if: ${{ github.event_name == 'pull_request' }}
-        uses: docker/build-push-action@v7
-        with:
-          file: ./resources/Dockerfile
-          platforms: linux/amd64,linux/arm64
-          tags: ${{ steps.meta.outputs.tags }}
+      - name: Create manifest list and push
+        working-directory: /tmp/digests
+        run: |
+          docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
+            $(printf '${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}@sha256:%s ' *)
 
-      - name: Image digest
-        run: echo ${{ steps.docker_build.outputs.digest }}
+      - name: Inspect image
+        run: |
+          docker buildx imagetools inspect ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.meta.outputs.version }}

.github/workflows/integration-metrics.yaml

@@ -7,16 +7,26 @@ on:
 jobs:
   build:
     name: Tests (Metrics)
-    runs-on: bare-metal-9950x
+    runs-on: garm-jammy-16
     env:
       METRICS_PUBLISH_KEY: ${{ secrets.METRICS_PUBLISH_KEY }}
     steps:
       - name: Code checkout
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
         with:
           fetch-depth: 0
+      - name: Install Docker
+        run: |
+          set -eufo pipefail
+          sudo apt-get update
+          sudo apt-get -y install ca-certificates curl gnupg
+          curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg
+          sudo chmod a+r /usr/share/keyrings/docker-archive-keyring.gpg
+          echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
+          sudo apt-get update
+          sudo apt install -y docker-ce docker-ce-cli
       - name: Run metrics tests
         timeout-minutes: 60
-        run: scripts/dev_cli.sh tests --metrics -- --test-exclude micro_ -- --report-file /root/workloads/metrics.json
+        run: scripts/dev_cli.sh tests --metrics -- --test-exclude micro_,block_qcow2 -- --report-file /root/workloads/metrics.json
       - name: Upload metrics report
         run: 'curl -X PUT https://ch-metrics.azurewebsites.net/api/publishmetrics -H "x-functions-key: $METRICS_PUBLISH_KEY" -T ~/workloads/metrics.json'

.github/workflows/mshv-infra.yaml

@@ -1,246 +0,0 @@
-name: MSHV Infra Setup
-on:
-  workflow_call:
-    inputs:
-      ARCH:
-        description: 'Architecture for the VM'
-        required: true
-        type: string
-      KEY:
-        description: 'SSH Key Name'
-        required: true
-        type: string
-      OS_DISK_SIZE:
-        description: 'OS Disk Size in GB'
-        required: true
-        type: number
-      RG:
-        description: 'Resource Group Name'
-        required: true
-        type: string
-      VM_SKU:
-        description: 'VM SKU'
-        required: true
-        type: string
-    secrets:
-      MI_CLIENT_ID:
-        required: true
-      RUNNER_RG:
-        required: true
-      STORAGE_ACCOUNT_PATHS:
-        required: true
-      ARCH_SOURCE_PATH:
-        required: true
-      USERNAME:
-        required: true
-    outputs:
-      RG_NAME:
-        description: 'Resource group of the VM'
-        value: ${{ jobs.infra-setup.outputs.RG_NAME }}
-      VM_NAME:
-        description: 'Name of the VM'
-        value: ${{ jobs.infra-setup.outputs.VM_NAME }}
-      PRIVATE_IP:
-        description: 'Private IP of the VM'
-        value: ${{ jobs.infra-setup.outputs.PRIVATE_IP }}
-concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}-${{ github.event_name }}
-  cancel-in-progress: true
-jobs:
-  infra-setup:
-    name: ${{ inputs.ARCH }} VM Provision
-    runs-on: mshv
-    outputs:
-      RG_NAME: ${{ steps.rg-setup.outputs.RG_NAME }}
-      VM_NAME: ${{ steps.vm-setup.outputs.VM_NAME }}
-      PRIVATE_IP: ${{ steps.get-vm-ip.outputs.PRIVATE_IP }}
-    steps:
-      - name: Install & login to AZ CLI
-        env:
-          MI_CLIENT_ID: ${{ secrets.MI_CLIENT_ID }}
-        run: |
-          set -eufo pipefail
-          echo "Installing Azure CLI if not already installed"
-          if ! command -v az &>/dev/null; then
-            curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash
-          else
-            echo "Azure CLI already installed"
-          fi
-          az --version
-          echo "Logging into Azure CLI using Managed Identity"
-          az login --identity --client-id "${MI_CLIENT_ID}"
-
-      - name: Get Location
-        id: get-location
-        env:
-          SKU: ${{ inputs.VM_SKU }}
-          STORAGE_ACCOUNT_PATHS: ${{ secrets.STORAGE_ACCOUNT_PATHS }}
-        run: |
-          set -eufo pipefail
-          # Extract vCPU count from SKU (e.g., "Standard_D2s_v3" => 2)
-          if ! [[ "$SKU" =~ ^Standard_[A-Za-z]+([1-9][0-9]*) ]]; then
-            printf 'Cannot extract vCPU count from SKU: %q\n' "$SKU"
-            exit 1
-          fi
-          vcpu=${BASH_REMATCH[1]}
-
-          SUPPORTED_LOCATIONS=$(echo "$STORAGE_ACCOUNT_PATHS" | jq -r 'to_entries[] | .key')
-
-          for location in $SUPPORTED_LOCATIONS; do
-            family=$(az vm list-skus --size "$SKU" --location "$location" --resource-type "virtualMachines" --query '[0].family' -o tsv)
-            if [[ -z "$family" ]]; then
-              echo "Cannot determine VM family for SKU: $SKU in $location"
-              continue
-            fi
-
-            remaining=$(az vm list-usage --location "$location" --query "[?name.value=='$family'] | [0]" -o json |
-                        jq '(.limit | tonumber) - (.currentValue | tonumber) >= ($ARGS.positional[0] | tonumber)' --jsonargs "$vcpu")
-            if [[ "$remaining" = true ]]; then
-              echo "Sufficient quota found in $location"
-              echo "location=$location" >> "$GITHUB_OUTPUT"
-              exit 0
-            fi
-          done
-
-          echo "No location found with sufficient vCPU quota for SKU: $SKU"
-          exit 1
-
-      - name: Create Resource Group
-        id: rg-setup
-        env:
-          LOCATION: ${{ steps.get-location.outputs.location }}
-          RG: ${{ inputs.RG }}
-          STORAGE_ACCOUNT_PATHS: ${{ secrets.STORAGE_ACCOUNT_PATHS }}
-        run: |
-          set -eufo pipefail
-          echo "Creating Resource Group: $RG"
-          # Create the resource group
-          echo "Creating resource group in location: ${LOCATION}"
-          az group create --name "${RG}" --location "${LOCATION}"
-          echo "RG_NAME=${RG}" >> $GITHUB_OUTPUT
-          echo "Resource group created successfully."
-
-      - name: Generate SSH Key
-        id: generate-ssh-key
-        env:
-          KEY: ${{ inputs.KEY }}
-        run: |
-          set -eufo pipefail
-          echo "Generating SSH key: $KEY"
-          mkdir -p ~/.ssh
-          ssh-keygen -t rsa -b 4096 -f ~/.ssh/"${KEY}" -N ""
-
-      - name: Create VM
-        id: vm-setup
-        env:
-          KEY: ${{ inputs.KEY }}
-          LOCATION: ${{ steps.get-location.outputs.location }}
-          OS_DISK_SIZE: ${{ inputs.OS_DISK_SIZE }}
-          RG: ${{ inputs.RG }}
-          RUNNER_RG: ${{ secrets.RUNNER_RG }}
-          USERNAME: ${{ secrets.USERNAME }}
-          VM_SKU: ${{ inputs.VM_SKU }}
-          VM_IMAGE_NAME: ${{ inputs.ARCH }}_${{ steps.get-location.outputs.location }}_image
-          VM_NAME: ${{ inputs.ARCH }}_${{ steps.get-location.outputs.location }}_${{ github.run_id }}
-        run: |
-          set -eufo pipefail
-          echo "Creating $VM_SKU VM: $VM_NAME"
-
-          # Extract subnet ID from the runner VM
-          echo "Retrieving subnet ID..."
-          SUBNET_ID=$(az network vnet list --resource-group "$RUNNER_RG" --query "[?contains(location, '${LOCATION}')].{SUBNETS:subnets}" | jq -r ".[0].SUBNETS[0].id")
-          if [[ -z "${SUBNET_ID}" ]]; then
-            echo "ERROR: Failed to retrieve Subnet ID."
-            exit 1
-          fi
-
-          # Extract image ID from the runner VM
-          echo "Retrieving image ID..."
-          IMAGE_ID=$(az image show --resource-group "$RUNNER_RG" --name "$VM_IMAGE_NAME" --query "id" -o tsv)
-          if [[ -z "${IMAGE_ID}" ]]; then
-            echo "ERROR: Failed to retrieve Image ID."
-            exit 1
-          fi
-
-          # Create VM
-          az vm create \
-            --resource-group "${RG}" \
-            --name "${VM_NAME}" \
-            --subnet "${SUBNET_ID}" \
-            --size "${VM_SKU}" \
-            --location "${LOCATION}" \
-            --image "${IMAGE_ID}" \
-            --os-disk-size-gb "${OS_DISK_SIZE}" \
-            --public-ip-sku Standard \
-            --storage-sku Premium_LRS \
-            --public-ip-address "" \
-            --admin-username "${USERNAME}" \
-            --ssh-key-value  ~/.ssh/"${KEY}".pub \
-            --security-type Standard \
-            --output json
-
-          az vm boot-diagnostics enable --name "${VM_NAME}" --resource-group "${RG}"
-
-          echo "VM_NAME=${VM_NAME}" >> "$GITHUB_OUTPUT"
-          echo "VM creation process completed successfully."
-
-      - name: Get VM Private IP
-        id: get-vm-ip
-        env:
-          RG: ${{ inputs.RG }}
-          VM_NAME: ${{ inputs.ARCH }}_${{ steps.get-location.outputs.location }}_${{ github.run_id }}
-        run: |
-          set -eufo pipefail
-          echo "Retrieving VM Private IP address..."
-          # Retrieve VM Private IP address
-          PRIVATE_IP=$(az vm show -g "${RG}" -n "${VM_NAME}" -d --query privateIps -o tsv)
-          if [[ -z "$PRIVATE_IP" ]]; then
-            echo "ERROR: Failed to retrieve private IP address."
-            exit 1
-          fi
-          echo "PRIVATE_IP=$PRIVATE_IP" >> "$GITHUB_OUTPUT"
-
-      - name: Wait for SSH availability
-        env:
-          KEY: ${{ inputs.KEY }}
-          PRIVATE_IP: ${{ steps.get-vm-ip.outputs.PRIVATE_IP }}
-          USERNAME: ${{ secrets.USERNAME }}
-        run: |
-          echo "Waiting for SSH to be accessible..."
-          timeout 120 bash -c 'until ssh -o StrictHostKeyChecking=no -i ~/.ssh/"${KEY}" -- "${USERNAME}@${PRIVATE_IP}" "exit" 2>/dev/null; do sleep 5; done'
-          echo "VM is accessible!"
-
-      - name: Remove Old Host Key
-        env:
-          PRIVATE_IP: ${{ steps.get-vm-ip.outputs.PRIVATE_IP }}
-        run: |
-          set -eufo pipefail
-          echo "Removing the old host key"
-          ssh-keygen -R "$PRIVATE_IP"
-
-      - name: SSH into VM and Install Dependencies
-        env:
-          KEY: ${{ inputs.KEY }}
-          PRIVATE_IP: ${{ steps.get-vm-ip.outputs.PRIVATE_IP }}
-          USERNAME: ${{ secrets.USERNAME }}
-        run: |
-          set -eufo pipefail
-          ssh -i ~/.ssh/"${KEY}" -o StrictHostKeyChecking=no -- "${USERNAME}@${PRIVATE_IP}" << EOF
-            set -eufo pipefail
-            echo "Logged in successfully."
-            echo "Installing dependencies..."
-            sudo tdnf install -y git moby-engine moby-cli clang llvm pkg-config make gcc glibc-devel
-            echo "Installing Rust..."
-            curl -sSf https://sh.rustup.rs | sh -s -- --default-toolchain stable --profile default -y
-            export PATH="\$HOME/.cargo/bin:\$PATH"
-            cargo --version
-            sudo mkdir -p /etc/docker/
-            echo '{"default-ulimits":{"nofile":{"Hard":65535,"Name":"nofile","Soft":65535}}}' | sudo tee /etc/docker/daemon.json
-            sudo systemctl stop docker
-            sudo systemctl enable docker.service
-            sudo systemctl enable containerd.service
-            sudo systemctl start docker
-            sudo groupadd -f docker
-            sudo usermod -a -G docker "${USERNAME}"
-            sudo systemctl restart docker
-          EOF

.github/workflows/mshv-integration.yaml

@@ -1,129 +0,0 @@
-name: Cloud Hypervisor Tests (MSHV) (x86_64)
-on: [pull_request_target, merge_group]
-permissions: {}
-
-jobs:
-  infra-setup:
-    name: MSHV Infra Setup (x86_64)
-    uses: ./.github/workflows/mshv-infra.yaml
-    with:
-      ARCH: x86_64
-      KEY: azure_key_${{ github.run_id }}
-      OS_DISK_SIZE: 512
-      RG: MSHV-INTEGRATION-${{ github.run_id }}
-      VM_SKU: Standard_D16s_v5
-    secrets:
-      MI_CLIENT_ID: ${{ secrets.MSHV_MI_CLIENT_ID }}
-      RUNNER_RG: ${{ secrets.MSHV_RUNNER_RG }}
-      STORAGE_ACCOUNT_PATHS: ${{ secrets.MSHV_STORAGE_ACCOUNT_PATHS }}
-      ARCH_SOURCE_PATH: ${{ secrets.MSHV_X86_SOURCE_PATH }}
-      USERNAME: ${{ secrets.MSHV_USERNAME }}
-
-  run-tests:
-    name: Integration Tests (x86_64)
-    needs: infra-setup
-    if: ${{ always() && needs.infra-setup.result == 'success' }}
-    runs-on: mshv
-    steps:
-      - name: Run integration tests
-        timeout-minutes: 60
-        env:
-          KEY: azure_key_${{ github.run_id }}
-          PR_NUMBER: ${{ github.event.pull_request.number }}
-          REPO_URL: https://github.com/cloud-hypervisor/cloud-hypervisor.git
-          REPO_DIR: cloud-hypervisor
-          PRIVATE_IP: ${{ needs.infra-setup.outputs.PRIVATE_IP }}
-          RG: MSHV-${{ github.run_id }}
-          USERNAME: ${{ secrets.MSHV_USERNAME }}
-        run: |
-          set -eufo pipefail
-          echo "Connecting to the VM via SSH..."
-          ssh -i ~/.ssh/"${KEY}" -o StrictHostKeyChecking=no -- "${USERNAME}@${PRIVATE_IP}" << EOF
-            set -e
-            echo "Logged in successfully."
-            export PATH="\$HOME/.cargo/bin:\$PATH"
-
-            if [[ "${{ github.event_name }}" == "pull_request_target" ]]; then
-              git clone --depth 1 "$REPO_URL" "$REPO_DIR"
-              cd "$REPO_DIR"
-              git fetch origin pull/${{ github.event.pull_request.number }}/merge
-              git checkout FETCH_HEAD
-            else
-              git clone --depth 1 --single-branch --branch "${{ github.ref_name }}" "$REPO_URL" "$REPO_DIR"
-              cd "$REPO_DIR"
-            fi
-
-            echo "Loading VDPA kernel modules..."
-            sudo modprobe vdpa
-            sudo modprobe vhost_vdpa
-            sudo modprobe vdpa_sim
-            sudo modprobe vdpa_sim_blk
-            sudo modprobe vdpa_sim_net
-
-            echo "Creating VDPA devices..."
-            sudo vdpa dev add name vdpa-blk0 mgmtdev vdpasim_blk
-            sudo vdpa dev add name vdpa-blk1 mgmtdev vdpasim_blk
-            sudo vdpa dev add name vdpa-blk2 mgmtdev vdpasim_net
-
-            echo "Setting permissions..."
-            for i in 0 1 2; do
-              dev="/dev/vhost-vdpa-\$i"
-              if [ -e "\$dev" ]; then
-                sudo chown \$USER:\$USER "\$dev"
-                sudo chmod 660 "\$dev"
-              else
-                echo "Warning: Device \$dev not found"
-              fi
-            done
-
-            sudo ./scripts/dev_cli.sh tests --hypervisor mshv --integration
-          EOF
-
-      - name: Dump dmesg
-        if: always()
-        continue-on-error: true
-        env:
-          KEY: azure_key_${{ github.run_id }}
-          PRIVATE_IP: ${{ needs.infra-setup.outputs.PRIVATE_IP }}
-          USERNAME: ${{ secrets.MSHV_USERNAME }}
-        run: |
-          ssh -i ~/.ssh/"${KEY}" -o StrictHostKeyChecking=no -- "${USERNAME}@${PRIVATE_IP}" sudo dmesg
-
-      - name: Dump serial console logs
-        if: always()
-        continue-on-error: true
-        env:
-          RG_NAME: ${{ needs.infra-setup.outputs.RG_NAME }}
-          VM_NAME: ${{ needs.infra-setup.outputs.VM_NAME }}
-        run: |
-          set -eufo pipefail
-          az vm boot-diagnostics get-boot-log --name "${VM_NAME}" --resource-group "${RG_NAME}" | jq -r
-
-  cleanup:
-    name: Cleanup
-    needs: run-tests
-    if: always()
-    runs-on: mshv
-    steps:
-      - name: Delete RG
-        env:
-          RG: MSHV-INTEGRATION-${{ github.run_id }}
-        run: |
-          if az group exists --name "${RG}"; then
-            az group delete --name "${RG}" --yes --no-wait
-          else
-            echo "Resource Group ${RG} does not exist. Skipping deletion."
-          fi
-          echo "Cleanup process completed."
-
-      - name: Delete SSH Key
-        env:
-          KEY: azure_key_${{ github.run_id }}
-        run: |
-          if [ -f ~/.ssh/"${KEY}" ]; then
-            rm -f ~/.ssh/"${KEY}" ~/.ssh/"${KEY}.pub"
-            echo "SSH key deleted successfully."
-          else
-            echo "SSH key does not exist. Skipping deletion."
-          fi
-          echo "Cleanup process completed."

.github/workflows/release.yaml

@@ -29,7 +29,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Code checkout
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
       - name: Install musl-gcc
         if: contains(matrix.platform.target, 'musl')
         run: sudo apt install -y musl-tools

.lychee.toml

@@ -23,6 +23,9 @@ exclude = [
   '^tcp://192\.168\.1\.10',
   # Slack invite endpoints reject automated GETs and return 403.
   '^https://join\.slack\.com/t/',
+  # Metrics publish endpoint only answers authenticated PUTs; a plain GET
+  # returns 404.
+  '^https://ch-metrics\.azurewebsites\.net/api/publishmetrics',
 ]
 
 # Exclude loopback addresses

AGENTS.md

@@ -11,9 +11,8 @@ reviewable, and compatible with the project's normal engineering constraints.
 - Start with `README.md` for the project shape and `CONTRIBUTING.md` for the
   contribution rules, coding style, commit message guidance, and LLM assistance
   disclosure policy. Following `CONTRIBUTING.md` is crucial!
-- The main supported architectures are `x86_64` and `aarch64`; the main
-  hypervisor backends are KVM and MSHV. `x86_64` with KVM gets the most regular
-  exercise, but changes must not make the other first-class targets worse.
+- Respect `.editorconfig` when editing files, in addition to any
+  language-specific formatter required by `CONTRIBUTING.md`.
 
 ### Change Guidelines
 
@@ -32,6 +31,11 @@ reviewable, and compatible with the project's normal engineering constraints.
   migration implications.
 - Do not invent APIs, behavior, or requirements. If something is uncertain,
   state the uncertainty and proceed only with minimal, explicit assumptions.
+- For `thiserror`-style errors, start messages with a capital letter and keep
+  the outer `Display` text short. Put all non-`#[source]` attributes in the
+  message to improve helpfulness, but do not repeat a `#[source]` value
+  inline: Cloud Hypervisor prints the full error chain, so only include the
+  concrete failure text directly when there is no source to report.
 
 ### Safety and Domain Notes
 
@@ -39,11 +43,6 @@ reviewable, and compatible with the project's normal engineering constraints.
   comment with the invariants, and make sure the surrounding code upholds them.
 - Assume concurrency matters. Avoid races, unsynchronized shared state, and
   implicit ordering assumptions; prefer clear ownership and synchronization.
-- Keep docs and comments short and useful. Document non-trivial invariants at
-  struct definitions and critical state transitions.
-- Logging should be minimal and high signal. Use `info!` for important normal
-  state changes that matter in production; use `warn!` or `error!` only for
-  abnormal conditions. Keep `debug!` for focused diagnostics.
 
 ### Build and Test Notes
 
@@ -61,9 +60,7 @@ reviewable, and compatible with the project's normal engineering constraints.
   host privileges, workloads, and container setup. To build the integration-test
   code directly without the infrastructure from `./scripts`, set the Rust cfg
   `devcli_testenv` or simply build through `clippy` which automatically includes
-  these code paths; otherwise the integration-test code is not included. Do not
-  assume the tests can be run directly in a restricted agent environment; ask
-  the developer to run them when real integration coverage is needed.
+  these code paths; otherwise the integration-test code is not included.
 
 ### Commit and Patch Formatting
 
@@ -74,8 +71,9 @@ reviewable, and compatible with the project's normal engineering constraints.
   specified in `./scripts/gitlint/rules`.
 - For LLM-assisted changes, follow the disclosure guidance in `CONTRIBUTING.md`:
   use the project's `Assisted-by:` trailer when disclosure is needed, and do not
-  add `Co-authored-by` or similar trailers unless that policy changes.
+  add `Co-authored-by` or similar trailers unless that policy changes. Prefer
+  explicit version numbers, such as `Assisted-by: Claude:Opus-4.7`, rather than
+  `Assisted-by: Claude:Opus-4`.
 - Temporary allowances such as `#[allow(unused)]` or ignored tests are only
   acceptable if resolved within the same commit series or paired with a clear
   TODO referencing a ticket. Ask the developer if in doubt.
-

CONTRIBUTING.md

@@ -11,6 +11,11 @@ license of those projects.
 New code should be under the [Apache v2
 License](https://opensource.org/licenses/Apache-2.0).
 
+Cloud Hypervisor's main supported architectures are `x86_64` and `aarch64`,
+and the main hypervisor backends are KVM and MSHV. `x86_64` with KVM gets the
+most regular exercise, but changes should not make the other supported
+architecture and backend combinations worse.
+
 ## Coding Style & Code Comments
 
 We use the [Rust Style] guide and enforce formatting and linting in CI,
@@ -34,6 +39,17 @@ For new code, add documentation and comments where they **provide additional val
 
 Comments should be concise and add additional context or information to the code.
 
+Logging should be minimal and high signal. Use `info!` for important normal
+state changes that matter in production; use `warn!` or `error!` only for
+abnormal conditions. Keep `debug!` for focused diagnostics. Please find more
+information in [`docs/logging.md`](docs/logging.md).
+
+Error messages should be sentence-style: start with a capital letter and stay
+concise. For `thiserror`-style errors, put all non-`#[source]` attributes
+(if they provide clear value) in the outer `Display` text to improve helpfulness,
+but do not repeat a `#[source]` value there because Cloud Hypervisor prints the
+full chain elsewhere.
+
 [Rust Style]: https://github.com/rust-lang/rust/tree/HEAD/src/doc/style-guide/src
 
 ## Basic Checks

Cargo.toml

@@ -23,6 +23,7 @@ members = [
   "event_monitor",
   "hypervisor",
   "net_util",
+  "offload_daemon",
   "option_parser",
   "pci",
   "performance-metrics",
@@ -52,16 +53,16 @@ resolver = "3"
 
 [workspace.dependencies]
 # rust-vmm crates
-acpi_tables = "0.2.0"
-iommufd-ioctls = "0.1.0"
-kvm-bindings = "0.14.0"
-kvm-ioctls = "0.24.0"
+acpi_tables = "0.2.1"
+iommufd-ioctls = "0.2.0"
+kvm-bindings = "0.14.1"
+kvm-ioctls = "0.25.0"
 linux-loader = "0.13.2"
 mshv-bindings = "0.6.9"
 mshv-ioctls = "0.6.9"
 seccompiler = "0.5.0"
 vfio-bindings = { version = "0.6.2", default-features = false }
-vfio-ioctls = { version = "0.6.0", default-features = false }
+vfio-ioctls = { version = "0.7.0", default-features = false }
 vfio_user = { version = "0.1.3", default-features = false }
 vhost = { version = "0.16.0", default-features = false }
 vhost-user-backend = { version = "0.22.0", default-features = false }
@@ -77,11 +78,12 @@ igvm_defs = "0.4.0"
 
 # serde crates
 serde = "1.0.228"
-serde_json = "1.0.149"
-serde_with = { version = "3.18.0", default-features = false }
+serde_json = "1.0.150"
+serde_with = { version = "3.19.0", default-features = false }
 
 # other crates
 anyhow = "1.0.102"
+base64 = "0.22.1"
 bitflags = "2.11.1"
 byteorder = "1.5.0"
 cfg-if = "1.0.4"
@@ -91,16 +93,23 @@ dirs = "6.0.0"
 env_logger = "0.11.10"
 epoll = "4.4.0"
 flume = "0.12.0"
-itertools = "0.14.0"
+itertools = "0.15.0"
 jiff = { version = "0.2", default-features = false, features = ["std"] }
 libc = "0.2.186"
-log = "0.4.29"
+log = "0.4.30"
+rustls = { version = "0.23.40", default-features = false, features = [
+  "logging",
+  "ring",
+  "std",
+] }
 sha2 = "0.11.0"
 signal-hook = "0.4.4"
+signal-hook-registry = "1.4.8"
+smallvec = "1.15.1"
 thiserror = "2.0.18"
-uuid = { version = "1.23.1" }
+uuid = { version = "1.23.2" }
 wait-timeout = "0.2.1"
-zerocopy = { version = "0.8.48", default-features = false }
+zerocopy = { version = "0.8.50", default-features = false }
 
 [workspace.lints.clippy]
 # Any clippy lint (group) in alphabetical order:
@@ -115,6 +124,7 @@ style = "deny"
 suspicious = "deny"
 
 # Individual Lints
+absolute_paths = "deny"
 assertions_on_result_states = "deny"
 if_not_else = "deny"
 manual_string_new = "deny"

README.md

@@ -59,10 +59,10 @@ based on the [Rust VMM](https://github.com/rust-vmm) crates.
 
 ### Architectures
 
-Cloud Hypervisor supports the `x86-64`, `AArch64` and `riscv64`
-architectures, with functionality varying across these platforms. The
-functionality differences between `x86-64` and `AArch64` are documented
-in [#1125](https://github.com/cloud-hypervisor/cloud-hypervisor/issues/1125).
+Cloud Hypervisor's main supported architectures are `x86-64` and `AArch64`,
+with functionality varying across these platforms. The functionality
+differences between `x86-64` and `AArch64` are documented in
+[#1125](https://github.com/cloud-hypervisor/cloud-hypervisor/issues/1125).
 The `riscv64` architecture support is experimental and offers limited
 functionality. For more details and instructions, please refer to [riscv
 documentation](docs/riscv.md).
@@ -203,7 +203,7 @@ To build the kernel:
 
 ```shell
 # Clone the Cloud Hypervisor Linux branch
-$ git clone --depth 1 https://github.com/cloud-hypervisor/linux.git -b ch-6.12.8 linux-cloud-hypervisor
+$ git clone --depth 1 https://github.com/cloud-hypervisor/linux.git -b ch-6.16.9 linux-cloud-hypervisor
 $ pushd linux-cloud-hypervisor
 $ make ch_defconfig
 # Do native build of the x86-64 kernel

api_client/src/lib.rs

@@ -3,24 +3,26 @@
 // SPDX-License-Identifier: Apache-2.0
 //
 
-use std::io::{Read, Write};
+use std::io::{self, Read, Write};
 use std::os::unix::io::RawFd;
+use std::{num, str};
 
 use thiserror::Error;
+use vmm_sys_util::errno;
 use vmm_sys_util::sock_ctrl_msg::ScmSocket;
 
 #[derive(Debug, Error)]
 pub enum Error {
     #[error("Error writing to or reading from HTTP socket")]
-    Socket(#[source] std::io::Error),
+    Socket(#[source] io::Error),
     #[error("Error sending file descriptors")]
-    SocketSendFds(#[source] vmm_sys_util::errno::Error),
+    SocketSendFds(#[source] errno::Error),
     #[error("Error parsing HTTP status code")]
-    StatusCodeParsing(#[source] std::num::ParseIntError),
+    StatusCodeParsing(#[source] num::ParseIntError),
     #[error("HTTP output is missing protocol statement")]
     MissingProtocol,
     #[error("Error parsing HTTP Content-Length field")]
-    ContentLengthParsing(#[source] std::num::ParseIntError),
+    ContentLengthParsing(#[source] num::ParseIntError),
     #[error("Server responded with error {0:?}: {1:?}")]
     ServerResponse(
         StatusCode,
@@ -100,7 +102,7 @@ fn parse_http_response(socket: &mut dyn Read) -> Result<Option<String>, Error> {
         if count == 0 {
             break;
         }
-        res.push_str(std::str::from_utf8(&bytes[0..count]).unwrap());
+        res.push_str(str::from_utf8(&bytes[0..count]).unwrap());
 
         // End of headers
         if let Some(o) = res.find("\r\n\r\n") {

arch/Cargo.toml

@@ -25,6 +25,10 @@ uuid = { workspace = true }
 vm-memory = { workspace = true, features = ["backend-bitmap", "backend-mmap"] }
 vmm-sys-util = { workspace = true, features = ["with-serde"] }
 
+[dev-dependencies]
+proptest = "1.0.0"
+serde_json = { workspace = true }
+
 [target.'cfg(any(target_arch = "aarch64", target_arch = "riscv64"))'.dependencies]
 fdt_parser = { version = "0.1.5", package = "fdt" }
 vm-fdt = { workspace = true }

arch/src/aarch64/cache.rs

@@ -0,0 +1,188 @@
+// Copyright 2020 Arm Limited (or its affiliates). All rights reserved.
+// Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+//
+// Portions Copyright 2017 The Chromium OS Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the THIRD-PARTY file.
+
+use std::fs;
+use std::path::Path;
+
+use log::warn;
+
+#[derive(Copy, Clone)]
+pub enum CacheLevel {
+    /// L1 data cache
+    L1D = 0,
+    /// L1 instruction cache
+    L1I = 1,
+    /// L2 cache
+    L2 = 2,
+    /// L3 cache
+    L3 = 3,
+}
+
+/// NOTE: cache size file directory example,
+/// "/sys/devices/system/cpu/cpu0/cache/index0/size".
+pub fn get_cache_size(cache_level: CacheLevel) -> u32 {
+    let mut file_directory: String = "/sys/devices/system/cpu/cpu0/cache".to_string();
+    match cache_level {
+        CacheLevel::L1D => file_directory += "/index0/size",
+        CacheLevel::L1I => file_directory += "/index1/size",
+        CacheLevel::L2 => file_directory += "/index2/size",
+        CacheLevel::L3 => file_directory += "/index3/size",
+    }
+
+    let file_path = Path::new(&file_directory);
+    if file_path.exists() {
+        let src = fs::read_to_string(file_directory).expect("File not exists or file corrupted.");
+        // The content of the file is as simple as a size, like: "32K"
+        let src = src.trim();
+        let src_digits: u32 = src[0..src.len() - 1].parse().unwrap();
+        let src_unit = &src[src.len() - 1..];
+
+        src_digits
+            * match src_unit {
+                "K" => 1u32 << 10,
+                "M" => 1u32 << 20,
+                "G" => 1u32 << 30,
+                _ => 1,
+            }
+    } else {
+        0
+    }
+}
+
+/// NOTE: coherency_line_size file directory example,
+/// "/sys/devices/system/cpu/cpu0/cache/index0/coherency_line_size".
+pub fn get_cache_coherency_line_size(cache_level: CacheLevel) -> u32 {
+    let mut file_directory: String = "/sys/devices/system/cpu/cpu0/cache".to_string();
+    match cache_level {
+        CacheLevel::L1D => file_directory += "/index0/coherency_line_size",
+        CacheLevel::L1I => file_directory += "/index1/coherency_line_size",
+        CacheLevel::L2 => file_directory += "/index2/coherency_line_size",
+        CacheLevel::L3 => file_directory += "/index3/coherency_line_size",
+    }
+
+    let file_path = Path::new(&file_directory);
+    if file_path.exists() {
+        let src = fs::read_to_string(file_directory).expect("File not exists or file corrupted.");
+        src.trim().parse::<u32>().unwrap()
+    } else {
+        0
+    }
+}
+
+/// NOTE: number_of_sets file directory example,
+/// "/sys/devices/system/cpu/cpu0/cache/index0/number_of_sets".
+pub fn get_cache_number_of_sets(cache_level: CacheLevel) -> u32 {
+    let mut file_directory: String = "/sys/devices/system/cpu/cpu0/cache".to_string();
+    match cache_level {
+        CacheLevel::L1D => file_directory += "/index0/number_of_sets",
+        CacheLevel::L1I => file_directory += "/index1/number_of_sets",
+        CacheLevel::L2 => file_directory += "/index2/number_of_sets",
+        CacheLevel::L3 => file_directory += "/index3/number_of_sets",
+    }
+
+    let file_path = Path::new(&file_directory);
+    if file_path.exists() {
+        let src = fs::read_to_string(file_directory).expect("File not exists or file corrupted.");
+        src.trim().parse::<u32>().unwrap()
+    } else {
+        0
+    }
+}
+
+/// NOTE: shared_cpu_list file directory example,
+/// "/sys/devices/system/cpu/cpu0/cache/index0/shared_cpu_list".
+pub fn get_cache_shared(cache_level: CacheLevel) -> bool {
+    let mut file_directory: String = "/sys/devices/system/cpu/cpu0/cache".to_string();
+    let mut result = true;
+
+    match cache_level {
+        CacheLevel::L1D | CacheLevel::L1I => result = false,
+        CacheLevel::L2 => file_directory += "/index2/shared_cpu_list",
+        CacheLevel::L3 => file_directory += "/index3/shared_cpu_list",
+    }
+
+    if !result {
+        return false;
+    }
+
+    let file_path = Path::new(&file_directory);
+    if file_path.exists() {
+        let src = fs::read_to_string(file_directory).expect("File not exists or file corrupted.");
+        let src = src.trim();
+        if src.is_empty() {
+            result = false;
+        } else {
+            result = src.contains('-') || src.contains(',');
+        }
+    } else {
+        result = false;
+    }
+
+    result
+}
+
+#[derive(Default, Copy, Clone, Debug)]
+pub struct CacheTopologyInfo {
+    pub l1_d_cache_size: u32,
+    pub l1_d_cache_line_size: u32,
+    pub l1_d_cache_sets: u32,
+
+    pub l1_i_cache_size: u32,
+    pub l1_i_cache_line_size: u32,
+    pub l1_i_cache_sets: u32,
+
+    pub l2_cache_size: u32,
+    pub l2_cache_line_size: u32,
+    pub l2_cache_sets: u32,
+
+    pub l3_cache_size: u32,
+    pub l3_cache_line_size: u32,
+    pub l3_cache_sets: u32,
+
+    pub l2_cache_shared: bool,
+    pub l3_cache_shared: bool,
+}
+
+/// Reads cache topology information from sysfs for cpu0.
+pub fn read_cache_topology() -> Option<CacheTopologyInfo> {
+    let cache_path = Path::new("/sys/devices/system/cpu/cpu0/cache");
+    if !cache_path.exists() {
+        warn!("Cache topology information is not available in sysfs.");
+        return None;
+    }
+
+    let mut info = CacheTopologyInfo {
+        l1_d_cache_size: get_cache_size(CacheLevel::L1D),
+        l1_d_cache_line_size: get_cache_coherency_line_size(CacheLevel::L1D),
+        l1_d_cache_sets: get_cache_number_of_sets(CacheLevel::L1D),
+
+        l1_i_cache_size: get_cache_size(CacheLevel::L1I),
+        l1_i_cache_line_size: get_cache_coherency_line_size(CacheLevel::L1I),
+        l1_i_cache_sets: get_cache_number_of_sets(CacheLevel::L1I),
+
+        l2_cache_size: get_cache_size(CacheLevel::L2),
+        l2_cache_line_size: get_cache_coherency_line_size(CacheLevel::L2),
+        l2_cache_sets: get_cache_number_of_sets(CacheLevel::L2),
+
+        l3_cache_size: get_cache_size(CacheLevel::L3),
+        l3_cache_line_size: get_cache_coherency_line_size(CacheLevel::L3),
+        l3_cache_sets: get_cache_number_of_sets(CacheLevel::L3),
+
+        l2_cache_shared: false,
+        l3_cache_shared: false,
+    };
+
+    if info.l2_cache_size != 0 {
+        info.l2_cache_shared = get_cache_shared(CacheLevel::L2);
+    }
+    if info.l3_cache_size != 0 {
+        info.l3_cache_shared = get_cache_shared(CacheLevel::L3);
+    }
+
+    Some(info)
+}

arch/src/aarch64/fdt.rs

@@ -9,22 +9,24 @@
 use std::collections::HashMap;
 use std::ffi::CStr;
 use std::fmt::Debug;
-use std::path::Path;
+use std::hash::BuildHasher;
 use std::sync::{Arc, Mutex};
-use std::{cmp, fs, result, str};
+use std::{cmp, result, str};
 
 use byteorder::{BigEndian, ByteOrder};
+use fdt_parser::node::FdtNode;
 use hypervisor::arch::aarch64::gic::Vgic;
 use hypervisor::arch::aarch64::regs::{
     AARCH64_ARCH_TIMER_HYP_IRQ, AARCH64_ARCH_TIMER_PHYS_NONSECURE_IRQ,
     AARCH64_ARCH_TIMER_PHYS_SECURE_IRQ, AARCH64_ARCH_TIMER_VIRT_IRQ, AARCH64_PMU_IRQ,
 };
-use log::{debug, info, warn};
+use log::{debug, info};
 use thiserror::Error;
 use vm_fdt::{FdtWriter, FdtWriterResult};
 use vm_memory::{Address, Bytes, GuestMemory, GuestMemoryError, GuestMemoryRegion};
 
 use super::super::{DeviceType, GuestMemoryMmap, InitramfsConfig};
+use super::cache::{CacheTopologyInfo, read_cache_topology};
 use super::layout::{
     GIC_V2M_COMPATIBLE, GICV2M_SPI_BASE, GICV2M_SPI_NUM, IRQ_BASE, MEM_32BIT_DEVICES_SIZE,
     MEM_32BIT_DEVICES_START, MEM_PCI_IO_SIZE, MEM_PCI_IO_START, PCI_HIGH_BASE,
@@ -88,124 +90,9 @@ pub enum Error {
 }
 type Result<T> = result::Result<T, Error>;
 
-#[derive(Copy, Clone)]
-pub enum CacheLevel {
-    /// L1 data cache
-    L1D = 0,
-    /// L1 instruction cache
-    L1I = 1,
-    /// L2 cache
-    L2 = 2,
-    /// L3 cache
-    L3 = 3,
-}
-
-/// NOTE: cache size file directory example,
-/// "/sys/devices/system/cpu/cpu0/cache/index0/size".
-pub fn get_cache_size(cache_level: CacheLevel) -> u32 {
-    let mut file_directory: String = "/sys/devices/system/cpu/cpu0/cache".to_string();
-    match cache_level {
-        CacheLevel::L1D => file_directory += "/index0/size",
-        CacheLevel::L1I => file_directory += "/index1/size",
-        CacheLevel::L2 => file_directory += "/index2/size",
-        CacheLevel::L3 => file_directory += "/index3/size",
-    }
-
-    let file_path = Path::new(&file_directory);
-    if file_path.exists() {
-        let src = fs::read_to_string(file_directory).expect("File not exists or file corrupted.");
-        // The content of the file is as simple as a size, like: "32K"
-        let src = src.trim();
-        let src_digits: u32 = src[0..src.len() - 1].parse().unwrap();
-        let src_unit = &src[src.len() - 1..];
-
-        src_digits
-            * match src_unit {
-                "K" => 1024,
-                "M" => 1024u32.pow(2),
-                "G" => 1024u32.pow(3),
-                _ => 1,
-            }
-    } else {
-        0
-    }
-}
-
-/// NOTE: coherency_line_size file directory example,
-/// "/sys/devices/system/cpu/cpu0/cache/index0/coherency_line_size".
-pub fn get_cache_coherency_line_size(cache_level: CacheLevel) -> u32 {
-    let mut file_directory: String = "/sys/devices/system/cpu/cpu0/cache".to_string();
-    match cache_level {
-        CacheLevel::L1D => file_directory += "/index0/coherency_line_size",
-        CacheLevel::L1I => file_directory += "/index1/coherency_line_size",
-        CacheLevel::L2 => file_directory += "/index2/coherency_line_size",
-        CacheLevel::L3 => file_directory += "/index3/coherency_line_size",
-    }
-
-    let file_path = Path::new(&file_directory);
-    if file_path.exists() {
-        let src = fs::read_to_string(file_directory).expect("File not exists or file corrupted.");
-        src.trim().parse::<u32>().unwrap()
-    } else {
-        0
-    }
-}
-
-/// NOTE: number_of_sets file directory example,
-/// "/sys/devices/system/cpu/cpu0/cache/index0/number_of_sets".
-pub fn get_cache_number_of_sets(cache_level: CacheLevel) -> u32 {
-    let mut file_directory: String = "/sys/devices/system/cpu/cpu0/cache".to_string();
-    match cache_level {
-        CacheLevel::L1D => file_directory += "/index0/number_of_sets",
-        CacheLevel::L1I => file_directory += "/index1/number_of_sets",
-        CacheLevel::L2 => file_directory += "/index2/number_of_sets",
-        CacheLevel::L3 => file_directory += "/index3/number_of_sets",
-    }
-
-    let file_path = Path::new(&file_directory);
-    if file_path.exists() {
-        let src = fs::read_to_string(file_directory).expect("File not exists or file corrupted.");
-        src.trim().parse::<u32>().unwrap()
-    } else {
-        0
-    }
-}
-
-/// NOTE: shared_cpu_list file directory example,
-/// "/sys/devices/system/cpu/cpu0/cache/index0/shared_cpu_list".
-pub fn get_cache_shared(cache_level: CacheLevel) -> bool {
-    let mut file_directory: String = "/sys/devices/system/cpu/cpu0/cache".to_string();
-    let mut result = true;
-
-    match cache_level {
-        CacheLevel::L1D | CacheLevel::L1I => result = false,
-        CacheLevel::L2 => file_directory += "/index2/shared_cpu_list",
-        CacheLevel::L3 => file_directory += "/index3/shared_cpu_list",
-    }
-
-    if !result {
-        return false;
-    }
-
-    let file_path = Path::new(&file_directory);
-    if file_path.exists() {
-        let src = fs::read_to_string(file_directory).expect("File not exists or file corrupted.");
-        let src = src.trim();
-        if src.is_empty() {
-            result = false;
-        } else {
-            result = src.contains('-') || src.contains(',');
-        }
-    } else {
-        result = false;
-    }
-
-    result
-}
-
 /// Creates the flattened device tree for this aarch64 VM.
-#[allow(clippy::too_many_arguments)]
-pub fn create_fdt<T: DeviceInfoForFdt + Clone + Debug, S: ::std::hash::BuildHasher>(
+#[expect(clippy::too_many_arguments)]
+pub fn create_fdt<T: DeviceInfoForFdt + Clone + Debug, S: BuildHasher>(
     guest_mem: &GuestMemoryMmap,
     cmdline: &str,
     vcpu_mpidr: &[u64],
@@ -287,63 +174,24 @@ fn create_cpu_nodes(
         threads_per_core as u32 * cores_per_die as u32 * dies_per_package as u32 * packages as u32;
 
     // Add cache info.
-    // L1 Data Cache Info.
-    let mut l1_d_cache_size: u32 = 0;
-    let mut l1_d_cache_line_size: u32 = 0;
-    let mut l1_d_cache_sets: u32 = 0;
-
-    // L1 Instruction Cache Info.
-    let mut l1_i_cache_size: u32 = 0;
-    let mut l1_i_cache_line_size: u32 = 0;
-    let mut l1_i_cache_sets: u32 = 0;
-
-    // L2 Cache Info.
-    let mut l2_cache_size: u32 = 0;
-    let mut l2_cache_line_size: u32 = 0;
-    let mut l2_cache_sets: u32 = 0;
-
-    // L3 Cache Info.
-    let mut l3_cache_size: u32 = 0;
-    let mut l3_cache_line_size: u32 = 0;
-    let mut l3_cache_sets: u32 = 0;
-
-    // Cache Shared Info.
-    let mut l2_cache_shared: bool = false;
-    let mut l3_cache_shared: bool = false;
-
-    let cache_path = Path::new("/sys/devices/system/cpu/cpu0/cache");
-    let cache_exist: bool = cache_path.exists();
-    if cache_exist {
-        // L1 Data Cache Info.
-        l1_d_cache_size = get_cache_size(CacheLevel::L1D);
-        l1_d_cache_line_size = get_cache_coherency_line_size(CacheLevel::L1D);
-        l1_d_cache_sets = get_cache_number_of_sets(CacheLevel::L1D);
-
-        // L1 Instruction Cache Info.
-        l1_i_cache_size = get_cache_size(CacheLevel::L1I);
-        l1_i_cache_line_size = get_cache_coherency_line_size(CacheLevel::L1I);
-        l1_i_cache_sets = get_cache_number_of_sets(CacheLevel::L1I);
-
-        // L2 Cache Info.
-        l2_cache_size = get_cache_size(CacheLevel::L2);
-        l2_cache_line_size = get_cache_coherency_line_size(CacheLevel::L2);
-        l2_cache_sets = get_cache_number_of_sets(CacheLevel::L2);
-
-        // L3 Cache Info.
-        l3_cache_size = get_cache_size(CacheLevel::L3);
-        l3_cache_line_size = get_cache_coherency_line_size(CacheLevel::L3);
-        l3_cache_sets = get_cache_number_of_sets(CacheLevel::L3);
-
-        // Cache Shared Info.
-        if l2_cache_size != 0 {
-            l2_cache_shared = get_cache_shared(CacheLevel::L2);
-        }
-        if l3_cache_size != 0 {
-            l3_cache_shared = get_cache_shared(CacheLevel::L3);
-        }
-    } else {
-        warn!("cache sysfs system does not exist.");
-    }
+    let cache_info = read_cache_topology();
+    let cache_exist = cache_info.is_some();
+    let CacheTopologyInfo {
+        l1_d_cache_size,
+        l1_d_cache_line_size,
+        l1_d_cache_sets,
+        l1_i_cache_size,
+        l1_i_cache_line_size,
+        l1_i_cache_sets,
+        l2_cache_size,
+        l2_cache_line_size,
+        l2_cache_sets,
+        l3_cache_size,
+        l3_cache_line_size,
+        l3_cache_sets,
+        l2_cache_shared,
+        l3_cache_shared,
+    } = cache_info.unwrap_or_default();
 
     // Arm boot protocol requires a minimal Device Tree
     // https://docs.kernel.org/arch/arm64/booting.html
@@ -879,7 +727,7 @@ fn create_fw_cfg_node<T: DeviceInfoForFdt + Clone + Debug>(
     Ok(())
 }
 
-fn create_devices_node<T: DeviceInfoForFdt + Clone + Debug, S: ::std::hash::BuildHasher>(
+fn create_devices_node<T: DeviceInfoForFdt + Clone + Debug, S: BuildHasher>(
     fdt: &mut FdtWriter,
     dev_info: &HashMap<(DeviceType, String), T, S>,
 ) -> FdtWriterResult<()> {
@@ -1145,7 +993,7 @@ pub fn print_fdt(dtb: &[u8]) {
     }
 }
 
-fn print_node(node: fdt_parser::node::FdtNode<'_, '_>, n_spaces: usize) {
+fn print_node(node: FdtNode<'_, '_>, n_spaces: usize) {
     debug!("{:indent$}{}/", "", node.name, indent = n_spaces);
     for property in node.properties() {
         let name = property.name;

arch/src/aarch64/mod.rs

@@ -2,6 +2,8 @@
 // Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
 // SPDX-License-Identifier: Apache-2.0
 
+/// Module for cache info.
+pub mod cache;
 /// Module for the flattened device tree.
 pub mod fdt;
 /// Layout for this aarch64 system.
@@ -11,6 +13,7 @@ pub mod uefi;
 
 use std::collections::HashMap;
 use std::fmt::Debug;
+use std::hash::BuildHasher;
 use std::sync::{Arc, Mutex};
 
 use hypervisor::arch::aarch64::gic::Vgic;
@@ -121,8 +124,8 @@ pub fn arch_memory_regions() -> Vec<(GuestAddress, usize, RegionType)> {
 }
 
 /// Configures the system and should be called once per vm before starting vcpu threads.
-#[allow(clippy::too_many_arguments)]
-pub fn configure_system<T: DeviceInfoForFdt + Clone + Debug, S: ::std::hash::BuildHasher>(
+#[expect(clippy::too_many_arguments)]
+pub fn configure_system<T: DeviceInfoForFdt + Clone + Debug, S: BuildHasher>(
     guest_mem: &GuestMemoryMmap,
     cmdline: &str,
     vcpu_mpidr: &[u64],

arch/src/lib.rs

@@ -9,14 +9,17 @@
 //! Supported platforms: x86_64, aarch64, riscv64.
 
 use std::collections::BTreeMap;
+use std::str::FromStr;
 use std::sync::Arc;
 use std::{fmt, result};
 
+use serde::de::{IntoDeserializer, value};
 use serde::{Deserialize, Serialize};
 use thiserror::Error;
+use vm_memory::bitmap::AtomicBitmap;
 
-type GuestMemoryMmap = vm_memory::GuestMemoryMmap<vm_memory::bitmap::AtomicBitmap>;
-type GuestRegionMmap = vm_memory::GuestRegionMmap<vm_memory::bitmap::AtomicBitmap>;
+type GuestMemoryMmap = vm_memory::GuestMemoryMmap<AtomicBitmap>;
+type GuestRegionMmap = vm_memory::GuestRegionMmap<AtomicBitmap>;
 
 /// Type for returning error code.
 #[derive(Debug, Error)]
@@ -33,11 +36,13 @@ pub enum Error {
     #[error("The memory map table extends past the end of guest memory")]
     MemmapTablePastRamEnd,
     #[error("Error writing memory map table to guest memory")]
-    MemmapTableSetup,
+    MemmapTableSetup(#[source] vm_memory::GuestMemoryError),
+    #[error("Error generating memory map table")]
+    MemmapTableGeneration,
     #[error("The hvm_start_info structure extends past the end of guest memory")]
     StartInfoPastRamEnd,
     #[error("Error writing hvm_start_info to guest memory")]
-    StartInfoSetup,
+    StartInfoSetup(#[source] vm_memory::GuestMemoryError),
     #[error("Failed to compute initramfs address")]
     InitramfsAddress,
     #[error("Error writing module entry to guest memory")]
@@ -53,6 +58,26 @@ pub enum Error {
 /// Type for returning public functions outcome.
 pub type Result<T> = result::Result<T, Error>;
 
+// If the target_arch is x86_64 we import CpuProfile from the x86_64 module, otherwise we
+// declare it here with only "host" as a selectable CPU profile. This trick is useful to prevent
+// excessive conditional compilation throughout the codebase.
+#[cfg(not(target_arch = "x86_64"))]
+#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
+/// A [`CpuProfile`] is a mechanism for ensuring live migration compatibility
+/// between host's with potentially different CPU models.
+pub enum CpuProfile {
+    #[default]
+    Host,
+}
+
+// Note that this trait impl is architecture agnostic and may thus reside here.
+impl FromStr for CpuProfile {
+    type Err = value::Error;
+    fn from_str(s: &str) -> result::Result<Self, Self::Err> {
+        Self::deserialize(s.into_deserializer())
+    }
+}
+
 /// Type for memory region types.
 #[derive(Clone, Copy, PartialEq, Eq, Debug, Serialize, Deserialize)]
 pub enum RegionType {
@@ -100,8 +125,9 @@ pub mod x86_64;
 #[cfg(target_arch = "x86_64")]
 pub use x86_64::{
     _NSIG, CpuidConfig, CpuidFeatureEntry, EntryPoint, arch_memory_regions, configure_system,
-    configure_vcpu, generate_common_cpuid, generate_ram_ranges, get_host_cpu_phys_bits,
-    initramfs_load_addr, layout, layout::CMDLINE_MAX_SIZE, layout::CMDLINE_START, regs,
+    configure_vcpu, cpu_profile::CpuProfile, generate_common_cpuid, generate_ram_ranges,
+    get_host_cpu_phys_bits, initramfs_load_addr, layout, layout::CMDLINE_MAX_SIZE,
+    layout::CMDLINE_START, regs,
 };
 
 /// Safe wrapper for `sysconf(_SC_PAGESIZE)`.

arch/src/riscv64/fdt.rs

@@ -61,7 +61,7 @@ pub enum Error {
 type Result<T> = result::Result<T, Error>;
 
 /// Creates the flattened device tree for this riscv64 VM.
-#[allow(clippy::too_many_arguments)]
+#[expect(clippy::too_many_arguments)]
 pub fn create_fdt<T: DeviceInfoForFdt + Clone + Debug, S: ::std::hash::BuildHasher>(
     guest_mem: &GuestMemoryMmap,
     cmdline: &str,
@@ -71,6 +71,7 @@ pub fn create_fdt<T: DeviceInfoForFdt + Clone + Debug, S: ::std::hash::BuildHash
     aia_device: &Arc<Mutex<dyn Vaia>>,
     initrd: &Option<InitramfsConfig>,
     pci_space_info: &[PciSpaceInfo],
+    timebase_frequency: u32,
 ) -> FdtWriterResult<Vec<u8>> {
     // Allocate stuff necessary for the holding the blob.
     let mut fdt = FdtWriter::new()?;
@@ -86,7 +87,7 @@ pub fn create_fdt<T: DeviceInfoForFdt + Clone + Debug, S: ::std::hash::BuildHash
     // Properties
     fdt.property_u32("#address-cells", ADDRESS_CELLS)?;
     fdt.property_u32("#size-cells", SIZE_CELLS)?;
-    create_cpu_nodes(&mut fdt, num_vcpu, isa_string)?;
+    create_cpu_nodes(&mut fdt, num_vcpu, isa_string, timebase_frequency)?;
     create_memory_node(&mut fdt, guest_mem)?;
     create_chosen_node(&mut fdt, cmdline, initrd)?;
     create_aia_node(&mut fdt, aia_device)?;
@@ -110,14 +111,17 @@ pub fn write_fdt_to_memory(fdt_final: &[u8], guest_mem: &GuestMemoryMmap) -> Res
 }
 
 // Following are the auxiliary function for creating the different nodes that we append to our FDT.
-fn create_cpu_nodes(fdt: &mut FdtWriter, num_cpus: u32, isa_string: &str) -> FdtWriterResult<()> {
+fn create_cpu_nodes(
+    fdt: &mut FdtWriter,
+    num_cpus: u32,
+    isa_string: &str,
+    timebase_frequency: u32,
+) -> FdtWriterResult<()> {
     // See https://elixir.bootlin.com/linux/v6.10/source/Documentation/devicetree/bindings/riscv/cpus.yaml
     let cpus = fdt.begin_node("cpus")?;
     // As per documentation, on RISC-V 64-bit systems value should be set to 1.
     fdt.property_u32("#address-cells", 0x01)?;
     fdt.property_u32("#size-cells", 0x0)?;
-    // TODO: Retrieve CPU frequency from cpu timer regs
-    let timebase_frequency: u32 = 0x989680;
     fdt.property_u32("timebase-frequency", timebase_frequency)?;
 
     for cpu_index in 0..num_cpus {
@@ -231,8 +235,8 @@ fn create_aia_node(fdt: &mut FdtWriter, aia_device: &Arc<Mutex<dyn Vaia>>) -> Fd
         fdt.property_u32("#interrupt-cells", 0u32)?;
         fdt.property_null("interrupt-controller")?;
         fdt.property_null("msi-controller")?;
-        // TODO complete num-ids
-        fdt.property_u32("riscv,num-ids", 2047u32)?;
+        let imsic_num_ids = aia_device.lock().unwrap().imsic_num_ids();
+        fdt.property_u32("riscv,num-ids", imsic_num_ids)?;
         fdt.property_u32("phandle", AIA_IMSIC_PHANDLE)?;
 
         let mut irq_cells = Vec::new();

arch/src/riscv64/mod.rs

@@ -160,7 +160,6 @@ fn isa_string_from_host() -> Result<String, Error> {
 }
 
 /// Configures the system and should be called once per vm before starting vcpu threads.
-#[allow(clippy::too_many_arguments)]
 pub fn configure_system<T: DeviceInfoForFdt + Clone + Debug, S: ::std::hash::BuildHasher>(
     guest_mem: &GuestMemoryMmap,
     cmdline: &str,
@@ -169,6 +168,7 @@ pub fn configure_system<T: DeviceInfoForFdt + Clone + Debug, S: ::std::hash::Bui
     initrd: &Option<super::InitramfsConfig>,
     pci_space_info: &[PciSpaceInfo],
     aia_device: &Arc<Mutex<dyn Vaia>>,
+    timebase_frequency: u32,
 ) -> super::Result<()> {
     let isa_string = isa_string_from_host()?;
     let fdt_final = fdt::create_fdt(
@@ -180,6 +180,7 @@ pub fn configure_system<T: DeviceInfoForFdt + Clone + Debug, S: ::std::hash::Bui
         aia_device,
         initrd,
         pci_space_info,
+        timebase_frequency,
     )
     .map_err(|_| Error::SetupFdt)?;
 

arch/src/riscv64/uefi.rs

@@ -7,7 +7,7 @@ use std::os::fd::AsFd;
 use std::result;
 
 use thiserror::Error;
-use vm_memory::{GuestAddress, GuestMemory};
+use vm_memory::{Bytes, GuestAddress, GuestMemory};
 
 /// Errors thrown while loading UEFI binary
 #[derive(Debug, Error)]

arch/src/x86_64/cpu_profile/cpuid_adjustments.rs

@@ -0,0 +1,190 @@
+// Copyright © 2026 Cyberus Technology GmbH
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+//! This module contains types associated with adjusting CPUID entries according
+//! to a selected CPU profile.
+
+use std::ops::RangeInclusive;
+
+use hypervisor::arch::x86::CpuIdEntry;
+use log::error;
+use serde::{Deserialize, Serialize};
+use thiserror::Error;
+
+use crate::x86_64::{CpuidReg, deserialize_u32_hex, serialize_u32_hex};
+
+/// Parameters for inspecting CPUID definitions.
+#[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize)]
+pub struct CpuidParameters {
+    /// The leaf (EAX) parameter used with the CPUID instruction
+    #[serde(
+        serialize_with = "serialize_u32_hex",
+        deserialize_with = "deserialize_u32_hex"
+    )]
+    pub leaf: u32,
+    /// The sub-leaf (ECX) parameter used with the CPUID instruction
+    #[serde(
+        serialize_with = "serialize_range_hex",
+        deserialize_with = "deserialize_range_hex"
+    )]
+    pub sub_leaf: RangeInclusive<u32>,
+    /// The register we are interested in inspecting which gets filled by the CPUID instruction
+    pub register: CpuidReg,
+}
+
+// Only used for (de-)serialization
+#[derive(Debug, Serialize, Deserialize)]
+struct ProvisionalRangeInclusive {
+    #[serde(
+        serialize_with = "serialize_u32_hex",
+        deserialize_with = "deserialize_u32_hex"
+    )]
+    start: u32,
+    #[serde(
+        serialize_with = "serialize_u32_hex",
+        deserialize_with = "deserialize_u32_hex"
+    )]
+    end: u32,
+}
+
+fn serialize_range_hex<S: serde::Serializer>(
+    input: &RangeInclusive<u32>,
+    serializer: S,
+) -> Result<S::Ok, S::Error> {
+    let provisional = ProvisionalRangeInclusive {
+        start: *input.start(),
+        end: *input.end(),
+    };
+    provisional.serialize(serializer)
+}
+
+fn deserialize_range_hex<'de, D: serde::Deserializer<'de>>(
+    deserializer: D,
+) -> Result<RangeInclusive<u32>, D::Error> {
+    let ProvisionalRangeInclusive { start, end } =
+        ProvisionalRangeInclusive::deserialize(deserializer)?;
+    Ok(start..=end)
+}
+
+/// Used for adjusting an entire cpuid output register (EAX, EBX, ECX or EDX).
+///
+/// Instances of this struct typically adjust CPUID according to the following
+/// formula: `cpuid_reg_value = (self.mask & cpuid_reg_value) | self.replacements`.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
+pub struct CpuidOutputRegisterAdjustments {
+    /// Packs values to be placed into the given CPUID output register.
+    #[serde(
+        serialize_with = "serialize_u32_hex",
+        deserialize_with = "deserialize_u32_hex"
+    )]
+    pub replacements: u32,
+    /// Used to zero out the area `replacements` occupy. This mask is not necessarily !replacements, as replacements
+    /// may pack values of different types that occupy varying ranges of bits.
+    ///
+    /// Bit ranges within a CPUID output register that are **not** supposed to be replaced/overwritten should be set in
+    /// this mask.
+    #[serde(
+        serialize_with = "serialize_u32_hex",
+        deserialize_with = "deserialize_u32_hex"
+    )]
+    pub mask: u32,
+}
+
+/// Error type indicating that expected CPUID entries could not be found.
+///
+/// This type does not record which entries could not be found as we do not
+/// expect this to be actionable at runtime. Instead we encourage logging such
+/// violations when and where they are detected.
+#[derive(Debug, Error)]
+#[error("Required CPUID entries not found")]
+pub struct MissingCpuidEntriesError;
+
+impl CpuidOutputRegisterAdjustments {
+    /// Adjust the given `cpuid_output_register` by retaining and replacing values according to `self`.
+    fn adjust(self, cpuid_output_register: &mut u32) {
+        *cpuid_output_register &= self.mask;
+        *cpuid_output_register |= self.replacements;
+    }
+
+    /// Adjust `cpuid` according to the given `adjustments`.
+    ///
+    /// The returned vector of cpuid entries covers the same CPUID (sub-) leaves as the given `cpuid` input,
+    /// but values without matching [`CpuidParameters`] are zeroed out.
+    ///
+    /// # Errors
+    ///
+    /// An error is returned if an entry cannot be found for an adjustment describing non-zero replacements.
+    pub(super) fn adjust_cpuid_entries(
+        mut cpuid: Vec<CpuIdEntry>,
+        adjustments: &[(CpuidParameters, Self)],
+    ) -> Result<Vec<CpuIdEntry>, MissingCpuidEntriesError> {
+        for entry in &mut cpuid {
+            for (reg, reg_value) in [
+                (CpuidReg::EAX, &mut entry.eax),
+                (CpuidReg::EBX, &mut entry.ebx),
+                (CpuidReg::ECX, &mut entry.ecx),
+                (CpuidReg::EDX, &mut entry.edx),
+            ] {
+                // Lookup the adjustment corresponding to the entry's function/leaf and index/sub-leaf for each of the register.
+                let register_adjustments: Option<CpuidOutputRegisterAdjustments> =
+                    adjustments.iter().find_map(|(param, adjustment)| {
+                        ((param.leaf == entry.function)
+                            && param.sub_leaf.contains(&entry.index)
+                            && (param.register == reg))
+                            .then_some(*adjustment)
+                    });
+
+                match register_adjustments {
+                    Some(adjustment) => adjustment.adjust(reg_value),
+                    None => {
+                        // No matching cpuid parameters were found. We thus set the value of the register to 0.
+                        *reg_value = 0;
+                    }
+                }
+            }
+        }
+
+        Self::expected_entries_found(&cpuid, adjustments)?;
+        Ok(cpuid)
+    }
+
+    /// Check that we found every value that was supposed to be replaced with something else than 0
+    ///
+    /// IMPORTANT: This function assumes that the given `cpuid` has already been adjusted with the
+    /// provided `adjustments`.
+    fn expected_entries_found(
+        cpuid: &[CpuIdEntry],
+        adjustments: &[(CpuidParameters, Self)],
+    ) -> Result<(), MissingCpuidEntriesError> {
+        let mut missing_entry = false;
+
+        for (param, adjustment) in adjustments {
+            if adjustment.replacements == 0 {
+                continue;
+            }
+
+            if !cpuid.iter().any(|entry| {
+                (entry.function == param.leaf) && (param.sub_leaf.contains(&entry.index))
+            }) {
+                error!(
+                    "cannot adjust CPU profile. No entry found matching the required parameters: {param:?}"
+                );
+                missing_entry = true;
+            }
+        }
+        if missing_entry {
+            Err(MissingCpuidEntriesError)
+        } else {
+            Ok(())
+        }
+    }
+}
+
+/// Data describing CPUID adjustments related to a CPU Profile.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct CpuidProfileData {
+    /// Adjustments necessary to become compatible with the desired target.
+    pub adjustments: Vec<(CpuidParameters, CpuidOutputRegisterAdjustments)>,
+}

arch/src/x86_64/cpu_profile/mod.rs

@@ -0,0 +1,880 @@
+// Copyright © 2026 Cyberus Technology GmbH
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+use hypervisor::CpuVendor;
+use hypervisor::arch::x86::CpuIdEntry;
+
+use crate::x86_64::cpu_profile::cpuid_adjustments::{
+    CpuidOutputRegisterAdjustments, CpuidProfileData, MissingCpuidEntriesError,
+};
+use crate::x86_64::{AMX_TILECFG_BIT, AMX_TILEDATA_BIT, CpuidReg};
+
+/// Mask indicating availability of the AMX TILECFG state component
+const TILECFG_MASK: u32 = 1_u32 << AMX_TILECFG_BIT;
+/// Mask indicating availability of the AMX TILEDATA state component
+const TILEDATA_MASK: u32 = 1_u32 << AMX_TILEDATA_BIT;
+
+pub mod cpuid_adjustments;
+
+// TODO: Auto generate the CpuProfile enum with a build script once we introduce user facing CPU profiles.
+
+/// A [`CpuProfile`] is a mechanism for ensuring live migration compatibility
+/// between hosts with potentially different CPU models.
+#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
+pub enum CpuProfile {
+    #[default]
+    Host,
+}
+
+impl CpuProfile {
+    /// Adjust `cpuid` to the chosen CPU profile.
+    ///
+    /// The CPUID data obtained from the hypervisor is thus downgraded to the selected profile.
+    ///
+    /// This method does **not** perform any compatibility checks beyond
+    /// ensuring that all expected (sub) leaves required by the CPU profile are present.
+    ///
+    /// The caller is responsible for ensuring compatibility of `cpuid` by the time it is
+    /// utilized.
+    ///
+    /// If Intel AMX is not desired, then passing `amx = false` will permit missing (sub)-leaves
+    /// that are **purely AMX related**.
+    ///
+    /// The Host profile guarantees that `cpuid` is returned without any modifications.
+    pub(in crate::x86_64) fn adjust_cpuid(
+        &self,
+        cpuid: Vec<CpuIdEntry>,
+        amx: bool,
+        cpu_vendor: CpuVendor,
+    ) -> Result<Vec<CpuIdEntry>, MissingCpuidEntriesError> {
+        let Some(cpuid_profile_data) = self.cpuid_data() else {
+            return Ok(cpuid);
+        };
+        adjust_cpuid(cpuid_profile_data, cpuid, amx, cpu_vendor)
+    }
+
+    /// Obtain CPUID adjustment data related to the CPU profile.
+    fn cpuid_data(&self) -> Option<CpuidProfileData> {
+        // TODO: Auto generate this through a build script once
+        // we introduce actual CPU profiles.
+        match self {
+            CpuProfile::Host => None,
+        }
+    }
+}
+
+/// See [`CpuProfile::adjust_cpuid`](CpuProfile::adjust_cpuid)
+fn adjust_cpuid(
+    CpuidProfileData { mut adjustments }: CpuidProfileData,
+    cpuid: Vec<CpuIdEntry>,
+    amx: bool,
+    cpu_vendor: CpuVendor,
+) -> Result<Vec<CpuIdEntry>, MissingCpuidEntriesError> {
+    if (!amx) && matches!(cpu_vendor, CpuVendor::Intel) {
+        let amx_tilecfg_leaf = u32::from(AMX_TILECFG_BIT);
+        let amx_tiledata_leaf = u32::from(AMX_TILEDATA_BIT);
+
+        // In this case we invalidate tile state components and zero out all other purely AMX related leaves
+        // in order to maximize our chances of finding all required (sub) leaves.
+        for adj in adjustments.iter_mut() {
+            if adj.0.sub_leaf.start() != adj.0.sub_leaf.end() {
+                continue;
+            }
+            let sub_leaf = *adj.0.sub_leaf.start();
+            let leaf = adj.0.leaf;
+            if (leaf == 0xd) && (sub_leaf == 0) && (adj.0.register == CpuidReg::EAX) {
+                adj.1.mask &= !(TILECFG_MASK | TILEDATA_MASK);
+                adj.1.replacements &= !(TILECFG_MASK | TILEDATA_MASK);
+            }
+
+            if (leaf == 0xd) && (sub_leaf == 1) && (adj.0.register == CpuidReg::ECX) {
+                adj.1.mask &= !(TILECFG_MASK | TILEDATA_MASK);
+                adj.1.replacements &= !(TILECFG_MASK | TILEDATA_MASK);
+            }
+
+            if (leaf == 0xd) && ((sub_leaf == amx_tilecfg_leaf) || (sub_leaf == amx_tiledata_leaf))
+            {
+                adj.1.mask = 0;
+                adj.1.replacements = 0;
+            }
+
+            // Tile Information (purely AMX related).
+            if leaf == 0x1d {
+                adj.1.mask = 0;
+                adj.1.replacements = 0;
+            }
+
+            // TMUL information (purely AMX related)
+            if leaf == 0x1e {
+                adj.1.mask = 0;
+                adj.1.replacements = 0;
+            }
+        }
+    }
+
+    CpuidOutputRegisterAdjustments::adjust_cpuid_entries(cpuid, &adjustments)
+}
+
+#[cfg(test)]
+mod unit_tests {
+    use proptest::prelude::*;
+
+    use super::{CpuIdEntry, CpuVendor, CpuidProfileData, CpuidReg, adjust_cpuid};
+    use crate::x86_64::cpu_profile::cpuid_adjustments::{
+        CpuidOutputRegisterAdjustments, CpuidParameters,
+    };
+    use crate::x86_64::cpu_profile::{TILECFG_MASK, TILEDATA_MASK};
+
+    // Note that the tests for adjust_cpuid within this module tend to use much simpler inputs
+    // than what it will be called with at runtime within Cloud hypervisor. We do this here in order
+    // to keep each test focused on the behavioral aspect under test.
+
+    /// Helper function that returns adjustments tied to purely AMX related leaves.
+    fn amx_related_adjustments() -> Vec<(CpuidParameters, CpuidOutputRegisterAdjustments)> {
+        let amx_adjustments_json = r#"
+            [
+                [
+                  {
+                    "leaf": "0xd",
+                    "sub_leaf": {
+                      "start": "0x11",
+                      "end": "0x11"
+                    },
+                    "register": "EAX"
+                  },
+                  {
+                    "replacements": "0x40",
+                    "mask": "0x0"
+                  }
+                ],
+                [
+                  {
+                    "leaf": "0xd",
+                    "sub_leaf": {
+                      "start": "0x11",
+                      "end": "0x11"
+                    },
+                    "register": "EBX"
+                  },
+                  {
+                    "replacements": "0xac0",
+                    "mask": "0x0"
+                  }
+                ],
+                [
+                  {
+                    "leaf": "0xd",
+                    "sub_leaf": {
+                      "start": "0x11",
+                      "end": "0x11"
+                    },
+                    "register": "ECX"
+                  },
+                  {
+                    "replacements": "0x2",
+                    "mask": "0x0"
+                  }
+                ],
+                [
+                  {
+                    "leaf": "0xd",
+                    "sub_leaf": {
+                      "start": "0x12",
+                      "end": "0x12"
+                    },
+                    "register": "EAX"
+                  },
+                  {
+                    "replacements": "0x2000",
+                    "mask": "0x0"
+                  }
+                ],
+                [
+                  {
+                    "leaf": "0xd",
+                    "sub_leaf": {
+                      "start": "0x12",
+                      "end": "0x12"
+                    },
+                    "register": "EBX"
+                  },
+                  {
+                    "replacements": "0xb00",
+                    "mask": "0x0"
+                  }
+                ],
+                [
+                  {
+                    "leaf": "0xd",
+                    "sub_leaf": {
+                      "start": "0x12",
+                      "end": "0x12"
+                    },
+                    "register": "ECX"
+                  },
+                  {
+                    "replacements": "0x6",
+                    "mask": "0x0"
+                  }
+                ],
+                [
+                  {
+                    "leaf": "0x1d",
+                    "sub_leaf": {
+                      "start": "0x0",
+                      "end": "0x0"
+                    },
+                    "register": "EAX"
+                  },
+                  {
+                    "replacements": "0x1",
+                    "mask": "0x0"
+                  }
+                ],
+                [
+                  {
+                    "leaf": "0x1d",
+                    "sub_leaf": {
+                      "start": "0x1",
+                      "end": "0x1"
+                    },
+                    "register": "EAX"
+                  },
+                  {
+                    "replacements": "0x4002000",
+                    "mask": "0x0"
+                  }
+                ],
+                [
+                  {
+                    "leaf": "0x1d",
+                    "sub_leaf": {
+                      "start": "0x1",
+                      "end": "0x1"
+                    },
+                    "register": "EBX"
+                  },
+                  {
+                    "replacements": "0x80040",
+                    "mask": "0x0"
+                  }
+                ],
+                [
+                  {
+                    "leaf": "0x1d",
+                    "sub_leaf": {
+                      "start": "0x1",
+                      "end": "0x1"
+                    },
+                    "register": "ECX"
+                  },
+                  {
+                    "replacements": "0x10",
+                    "mask": "0x0"
+                  }
+                ],
+                [
+                  {
+                    "leaf": "0x1e",
+                    "sub_leaf": {
+                      "start": "0x0",
+                      "end": "0x0"
+                    },
+                    "register": "EAX"
+                  },
+                  {
+                    "replacements": "0x0",
+                    "mask": "0x0"
+                  }
+                ],
+                [
+                  {
+                    "leaf": "0x1e",
+                    "sub_leaf": {
+                      "start": "0x0",
+                      "end": "0x0"
+                    },
+                    "register": "EBX"
+                  },
+                  {
+                    "replacements": "0x4010",
+                    "mask": "0x0"
+                  }
+                ]
+            ]"#;
+        serde_json::from_str(amx_adjustments_json).unwrap()
+    }
+
+    // Randonly generate three CPUID entries and construct some simple adjustments which we apply
+    // through the `adjust_cpuid` function and assert that our expectations are met.
+    proptest! {
+        #[test]
+        fn adjust_cpuid_simple_adjustments(
+            leaf0 in any::<u32>(),
+            leaf1 in any::<u32>(),
+            leaf2 in any::<u32>(),
+            a in any::<u32>(),
+            b in any::<u32>(),
+            c in any::<u32>(),
+            d in any::<u32>(),
+        ) {
+            // Ensure that we have distinct leaves in this test
+            let mut leaves = [leaf0, leaf1, leaf2];
+            leaves.sort_unstable();
+            for (l, i) in leaves.iter_mut().zip([0, 1, 2]) {
+                *l = l.wrapping_add(i);
+            }
+
+            let [leaf0, leaf1, leaf2] = leaves;
+
+            // The following leaves have some special handling that we test in later
+            // more specialized tests. In this simple test we want to avoid them.
+            let leaves_with_special_handling = { [0xd, 0x1d, 0x1e] };
+
+            let transform_leaf = |leaf: u32| {
+                if leaves_with_special_handling.contains(&leaf) {
+                    // Ensures that we get a leaf different from any of the leaves that have special handling
+                    leaf | 0x1000
+                } else {
+                    leaf
+                }
+            };
+
+            let leaf0 = transform_leaf(leaf0);
+
+            let leaf1 = transform_leaf(leaf1);
+
+            let leaf2 = transform_leaf(leaf2);
+
+            // The leaves should still be distinct
+            assert!(leaf0 != leaf1);
+            assert!(leaf0 != leaf2);
+            assert!(leaf1 != leaf2);
+            // We have now setup leaves to be used in this test
+
+            // Let's now construct some simple adjustments
+
+            // mask retaining bits 0,1,2 and 3
+            let first_four_bits_mask = 15;
+
+            // Retain the first four bits of the register and overwrite the remaining bits with the value "42"
+            let adjustment_u = CpuidOutputRegisterAdjustments {
+                mask: first_four_bits_mask,
+                replacements: 42 << 4,
+            };
+
+            let assert_adjustment_u = |new_value: u32, old_value: u32| {
+                assert_eq!(new_value & first_four_bits_mask, (old_value & first_four_bits_mask));
+                // Recall that we placed the value 42 into bits 31:4
+                assert_eq!(new_value >> 4, 42);
+            };
+
+            // Set bits 0 and 28 and zero out the rest
+            let adjustment_v = CpuidOutputRegisterAdjustments {
+                replacements: 1 | (1 << 28),
+                mask: 0
+            };
+
+            let assert_adjustment_v = |new_value: u32| {
+                assert_eq!(new_value, 1 | (1 << 28));
+            };
+
+            // Make adjustment_u apply to EAX of leaf0 and EBX of leaf1.
+            //
+            // Make adjustment_v apply to EDX of leaf0 and ECX of leaf1.
+            //
+            // We do not specify any adjustment for leaf2.
+            let cpuid_profile_data = CpuidProfileData {
+                adjustments: vec![
+                    (
+                        CpuidParameters {
+                            leaf: leaf0,
+                            sub_leaf: 0..=0,
+                            register: CpuidReg::EAX,
+                        },
+                        adjustment_u,
+                    ),
+                    (
+                        CpuidParameters {
+                            leaf: leaf0,
+                            sub_leaf: 0..=0,
+                            register: CpuidReg::EDX,
+                        },
+                        adjustment_v,
+                    ),
+                    (
+                        CpuidParameters {
+                            leaf: leaf1,
+                            sub_leaf: 0..=0,
+                            register: CpuidReg::EBX,
+                        },
+                        adjustment_u,
+                    ),
+                    (
+                        CpuidParameters {
+                            leaf: leaf1,
+                            sub_leaf: 0..=0,
+                            register: CpuidReg::ECX,
+                        },
+                        adjustment_v,
+                    ),
+                ],
+            };
+
+            // Construct cpuid entries consisting of leaves leaf0, leaf1 and leaf2.
+            // The registers eax, ebx, ecx, edx are populated with the randomly generated values `a`, `b`, `c` and `d`
+            // and we do not consider sub-leaves in this test.
+            let cpuid = vec![
+                CpuIdEntry {
+                    function: leaf0,
+                    index: 0,
+                    flags: 0,
+                    eax: a,
+                    ebx: b,
+                    ecx: c,
+                    edx: d,
+                },
+                CpuIdEntry {
+                    function: leaf1,
+                    index: 0,
+                    flags: 0,
+                    eax: a,
+                    ebx: b,
+                    ecx: c,
+                    edx: d,
+                },
+                CpuIdEntry {
+                    function: leaf2,
+                    index: 0,
+                    flags: 0,
+                    eax: a,
+                    ebx: b,
+                    ecx: c,
+                    edx: d,
+                },
+            ];
+
+            // Check that the output of `adjust_cpuid` contains the same CPUID leaves as the
+            // `cpuid` vector we started with.
+            let expected_num_entries = cpuid.len();
+            let mut found_entry_count = 0;
+
+            let adjusted_cpuid =
+                adjust_cpuid(cpuid_profile_data, cpuid, false, CpuVendor::Intel).unwrap();
+
+            // Iterate through our adjusted entries and assert that our expectations are met.
+            for entry in adjusted_cpuid {
+                let CpuIdEntry {
+                    function,
+                    index,
+                    flags,
+                    eax,
+                    ebx,
+                    ecx,
+                    edx,
+                } = entry;
+                if function == leaf0 {
+                    found_entry_count += 1;
+
+                    assert_adjustment_u(eax, a);
+                    assert_adjustment_v(edx);
+
+                    // ebx and ecx should be zeroed out
+                    assert_eq!(ebx, 0);
+                    assert_eq!(ecx, 0);
+                }
+                if function == leaf1 {
+                    found_entry_count += 1;
+                    assert_adjustment_u(ebx, b);
+                    assert_adjustment_v(ecx);
+
+                    // eax and edx should be zeroed out
+                    assert_eq!(eax, 0);
+                    assert_eq!(edx, 0);
+                }
+
+                if function == leaf2 {
+                    found_entry_count += 1;
+
+                    // All registers should be zeroed out
+                    assert_eq!(eax, ebx);
+                    assert_eq!(ebx, ecx);
+                    assert_eq!(ecx, edx);
+                    assert_eq!(edx, 0);
+                }
+
+                // Index and flags should not be altered. Since these were both
+                // always 0 for all leaves in the original `cpuid` that should
+                // remain the case.
+                assert_eq!(index, 0);
+                assert_eq!(flags, 0);
+            }
+
+            assert_eq!(expected_num_entries, found_entry_count);
+        }
+    }
+
+    // Check that adjust_cpuid follows the prescribed adjustments on
+    // specified subleaf ranges
+    #[test]
+    fn adjust_cpuid_works_with_subleaf_ranges() {
+        // As in the real runtime case the Topology enumeration leaves should not be altered
+        // by CPU profiles. In this test we thus define adjustment's that do not alter non-reserved
+        // bits for the 0x1f leaf and its sub-leaves.
+        let adjustments_json = r#"
+        [
+            [
+              {
+                "leaf": "0x1f",
+                "sub_leaf": {
+                  "start": "0x0",
+                  "end": "0xffffffff"
+                },
+                "register": "EAX"
+              },
+              {
+                "replacements": "0x0",
+                "mask": "0x1f"
+              }
+            ],
+            [
+              {
+                "leaf": "0x1f",
+                "sub_leaf": {
+                  "start": "0x0",
+                  "end": "0xffffffff"
+                },
+                "register": "EBX"
+              },
+              {
+                "replacements": "0x0",
+                "mask": "0xffff"
+              }
+            ],
+            [
+              {
+                "leaf": "0x1f",
+                "sub_leaf": {
+                  "start": "0x0",
+                  "end": "0xffffffff"
+                },
+                "register": "ECX"
+              },
+              {
+                "replacements": "0x0",
+                "mask": "0xffff"
+              }
+            ],
+            [
+              {
+                "leaf": "0x1f",
+                "sub_leaf": {
+                  "start": "0x0",
+                  "end": "0xffffffff"
+                },
+                "register": "EDX"
+              },
+              {
+                "replacements": "0x0",
+                "mask": "0xffffffff"
+              }
+            ]
+        ]"#;
+        let cpuid_profile_data = CpuidProfileData {
+            adjustments: serde_json::from_str(adjustments_json).unwrap(),
+        };
+
+        let cpuid = vec![
+            CpuIdEntry {
+                function: 0x1f,
+                index: 0,
+                flags: 1,
+                eax: 0x00000001,
+                ebx: 0x00000002,
+                ecx: 0x00000100,
+                edx: 0x00000000,
+            },
+            CpuIdEntry {
+                function: 0x1f,
+                index: 1,
+                flags: 1,
+                eax: 0x00000004,
+                ebx: 0x00000008,
+                ecx: 0x00000201,
+                edx: 0x00000006,
+            },
+        ];
+
+        let adjusted_cpuid =
+            adjust_cpuid(cpuid_profile_data, cpuid.clone(), false, CpuVendor::Intel).unwrap();
+
+        // Check that cpuid was indeed not altered
+        for entry in cpuid {
+            let adjusted_entry = adjusted_cpuid
+                .iter()
+                .find(|e| (e.function == entry.function) && (e.index == entry.index))
+                .unwrap();
+            assert_eq!(entry.eax, adjusted_entry.eax);
+            assert_eq!(entry.ebx, adjusted_entry.ebx);
+            assert_eq!(entry.ecx, adjusted_entry.ecx);
+            assert_eq!(entry.edx, adjusted_entry.edx);
+        }
+    }
+
+    #[test]
+    fn adjust_cpuid_fails_on_missing_entries() {
+        let cpuid = vec![CpuIdEntry {
+            function: 0x0,
+            index: 0x0,
+            eax: 0x20,
+            ebx: 0x756e6547,
+            ecx: 0x6c65746e,
+            edx: 0x49656e69,
+            flags: 0,
+        }];
+
+        let cpuid_profile_data = CpuidProfileData {
+            adjustments: vec![(
+                CpuidParameters {
+                    leaf: 0x1,
+                    sub_leaf: 0x0..=0x0,
+                    register: CpuidReg::EAX,
+                },
+                CpuidOutputRegisterAdjustments {
+                    replacements: 0x000806f8,
+                    mask: 0,
+                },
+            )],
+        };
+        let _ =
+            adjust_cpuid(cpuid_profile_data, cpuid.clone(), false, CpuVendor::Intel).unwrap_err();
+
+        // Also check this for a purely AMX related leaves which has special handling
+        let _ = adjust_cpuid(
+            CpuidProfileData {
+                adjustments: amx_related_adjustments(),
+            },
+            cpuid,
+            true,
+            CpuVendor::Intel,
+        )
+        .unwrap_err();
+    }
+
+    // Check that if `amx = false` then AMX related leaves are zeroed out
+    #[test]
+    fn adjust_cpuid_no_amx_zeros_amx_leaves() {
+        // Not AMX related
+        let leaf_zero = CpuIdEntry {
+            function: 0x0,
+            index: 0x0,
+            eax: 0x20,
+            ebx: 0x756e6547,
+            ecx: 0x6c65746e,
+            edx: 0x49656e69,
+            flags: 0,
+        };
+
+        let cpuid = vec![
+            leaf_zero,
+            // State components base leaf extracted from granite rapids the AMX related state component bits are set
+            CpuIdEntry {
+                function: 0xd,
+                index: 0x0,
+                flags: 1,
+                eax: 0x000602e7,
+                ebx: 0x00002b00,
+                ecx: 0x00002b00,
+                edx: 0x00000000,
+            },
+            // TILECFG state
+            CpuIdEntry {
+                function: 0xd,
+                index: 0x11,
+                flags: 1,
+                eax: 0x00000040,
+                ebx: 0x00000ac0,
+                ecx: 0x00000002,
+                edx: 0x00000000,
+            },
+            // TILEDATA state
+            CpuIdEntry {
+                function: 0xd,
+                index: 0x12,
+                flags: 1,
+                eax: 0x00002000,
+                ebx: 0x00000b00,
+                ecx: 0x00000006,
+                edx: 0x00000000,
+            },
+            // Tile information base leaf
+            CpuIdEntry {
+                function: 0x1d,
+                index: 0x0,
+                flags: 1,
+                eax: 0x00000001,
+                ebx: 0x00000000,
+                ecx: 0x00000000,
+                edx: 0x00000000,
+            },
+            // Tile Palette 1
+            CpuIdEntry {
+                function: 0x1d,
+                index: 0x1,
+                flags: 1,
+                eax: 0x04002000,
+                ebx: 0x00080040,
+                ecx: 0x00000010,
+                edx: 0x00000000,
+            },
+            // TMUL information base leaf
+            CpuIdEntry {
+                function: 0x1e,
+                index: 0x0,
+                flags: 1,
+                eax: 0x00000000,
+                ebx: 0x00004010,
+                ecx: 0x00000000,
+                edx: 0x00000000,
+            },
+        ];
+
+        let adjustments: Vec<(CpuidParameters, CpuidOutputRegisterAdjustments)> =
+            amx_related_adjustments()
+                .into_iter()
+                // leave leaf 0 untouched
+                .chain([
+                    (
+                        CpuidParameters {
+                            leaf: 0x0,
+                            sub_leaf: 0x0..=0x0,
+                            register: CpuidReg::EAX,
+                        },
+                        CpuidOutputRegisterAdjustments {
+                            replacements: 0,
+                            mask: u32::MAX,
+                        },
+                    ),
+                    (
+                        CpuidParameters {
+                            leaf: 0x0,
+                            sub_leaf: 0x0..=0x0,
+                            register: CpuidReg::EBX,
+                        },
+                        CpuidOutputRegisterAdjustments {
+                            replacements: 0,
+                            mask: u32::MAX,
+                        },
+                    ),
+                    (
+                        CpuidParameters {
+                            leaf: 0x0,
+                            sub_leaf: 0x0..=0x0,
+                            register: CpuidReg::ECX,
+                        },
+                        CpuidOutputRegisterAdjustments {
+                            replacements: 0,
+                            mask: u32::MAX,
+                        },
+                    ),
+                    (
+                        CpuidParameters {
+                            leaf: 0x0,
+                            sub_leaf: 0x0..=0x0,
+                            register: CpuidReg::EDX,
+                        },
+                        CpuidOutputRegisterAdjustments {
+                            replacements: 0,
+                            mask: u32::MAX,
+                        },
+                    ),
+                ])
+                // Keep EAX of leaf 0xd so we see that the AMX-related state component bits get unset, regardless of what
+                // the adjustment says
+                .chain([(
+                    CpuidParameters {
+                        leaf: 0xd,
+                        sub_leaf: 0x0..=0x0,
+                        register: CpuidReg::EAX,
+                    },
+                    CpuidOutputRegisterAdjustments {
+                        mask: u32::MAX,
+                        replacements: 0,
+                    },
+                )])
+                .collect();
+
+        let adjusted_cpuid = adjust_cpuid(
+            CpuidProfileData { adjustments },
+            cpuid.clone(),
+            false,
+            CpuVendor::Intel,
+        )
+        .unwrap();
+
+        // Check that leaf zero is left untouched as expected
+        {
+            let adjusted_leaf_zero = adjusted_cpuid
+                .iter()
+                .find(|entry| entry.function == 0x0)
+                .unwrap();
+            assert_eq!(adjusted_leaf_zero.eax, leaf_zero.eax);
+            assert_eq!(adjusted_leaf_zero.ebx, leaf_zero.ebx);
+            assert_eq!(adjusted_leaf_zero.ecx, leaf_zero.ecx);
+            assert_eq!(adjusted_leaf_zero.edx, leaf_zero.edx);
+        }
+
+        // Check that the TILECFG and TILEDATA state bits are now zeroed ut
+        {
+            let state_cmp_base_leaf = adjusted_cpuid
+                .iter()
+                .find(|entry| (entry.function == 0xd) && (entry.index == 0x0))
+                .unwrap();
+            // EAX should not have been zeroed out in its entirety
+            assert!(state_cmp_base_leaf.eax != 0);
+            // The TILECFG state bit should be unset
+            assert_eq!(state_cmp_base_leaf.eax & TILECFG_MASK, 0);
+            // The TILEDATA state bit should be unset
+            assert_eq!(state_cmp_base_leaf.eax & TILEDATA_MASK, 0);
+        }
+
+        // Since all remaining entries we placed in `cpuid` are purely AMX related we now
+        // expect them to be zeroed out
+        for entry in adjusted_cpuid {
+            if entry.function == 0 || (entry.function == 0xd && entry.index == 0x0) {
+                continue;
+            }
+            assert_eq!(entry.eax, 0);
+            assert_eq!(entry.ebx, 0);
+            assert_eq!(entry.ecx, 0);
+            assert_eq!(entry.edx, 0);
+        }
+    }
+
+    // Check that if `amx = false` then missing purely AMX related leaves
+    // do not lead to failure
+    #[test]
+    fn adjust_cpuid_no_amx_missing_amx_leaves_accepted() {
+        let cpuid = vec![CpuIdEntry {
+            function: 0x0,
+            index: 0x0,
+            eax: 0x20,
+            ebx: 0x756e6547,
+            ecx: 0x6c65746e,
+            edx: 0x49656e69,
+            flags: 0,
+        }];
+
+        let _ = adjust_cpuid(
+            CpuidProfileData {
+                adjustments: amx_related_adjustments(),
+            },
+            cpuid,
+            false,
+            CpuVendor::Intel,
+        )
+        .unwrap();
+    }
+}

arch/src/x86_64/helpers.rs

@@ -0,0 +1,82 @@
+// Copyright © 2026 Cyberus Technology GmbH
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+use std::result;
+
+use serde::{Deserialize, Deserializer, Serializer, de};
+
+/// Serializes the given `input` as a hex string (starting with "0x").
+///
+/// As an example if `input:=5` then this function will feed the given
+/// `serializer` the string "0x5".
+pub(crate) fn serialize_u32_hex<S: Serializer>(
+    input: &u32,
+    serializer: S,
+) -> result::Result<S::Ok, S::Error> {
+    serializer.serialize_str(&format!("{input:#x}"))
+}
+
+/// Deserializes a u32 from a hex string representation.
+pub(crate) fn deserialize_u32_hex<'de, D: Deserializer<'de>>(
+    deserializer: D,
+) -> result::Result<u32, D::Error> {
+    let hex: &str = <&str>::deserialize(deserializer)?;
+    u32::from_str_radix(hex.strip_prefix("0x").unwrap_or(""), 16).map_err(|_| {
+        <D::Error as de::Error>::custom(format!("{hex} is not a hex encoded 32 bit integer"))
+    })
+}
+
+#[cfg(test)]
+mod unit_tests {
+    use proptest::prelude::*;
+    use serde::{Deserialize, Serialize};
+
+    use super::*;
+
+    #[derive(Debug, Serialize, Deserialize, Clone, Copy, PartialEq, Eq)]
+    struct TestStruct {
+        #[serde(
+            serialize_with = "serialize_u32_hex",
+            deserialize_with = "deserialize_u32_hex"
+        )]
+        foo: u32,
+        #[serde(
+            serialize_with = "serialize_u32_hex",
+            deserialize_with = "deserialize_u32_hex"
+        )]
+        bar: u32,
+    }
+
+    // Check that our hex serializers satisfy the two following invariants
+    // 1. Serialization followed by deserialization is the identity.
+    // 2. Values of type u32 are serialized to strings starting with "0x" and then
+    // a sub-string where all characters are ascii hex digits (with the letters [a-f] always in lowercase).
+    proptest! {
+        #[test]
+        fn hex_serialization_works(foo in any::<u32>(), bar in any::<u32>()) {
+            let t = TestStruct { foo , bar };
+
+            let t_string = serde_json::to_string(&t).unwrap();
+            let t_deserialized = serde_json::from_str(&t_string).unwrap();
+            prop_assert_eq!(t, t_deserialized);
+
+            let t_json = serde_json::to_value(t).unwrap();
+
+            let check_str_invariants = |value: &str| {
+                prop_assert!(value.starts_with("0x"));
+                prop_assert!(value.as_bytes()[2..].iter().all(u8::is_ascii_hexdigit));
+                prop_assert!(!value.as_bytes()[2..].iter().any(u8::is_ascii_uppercase));
+                Ok(())
+            };
+
+            let foo_str = t_json.get("foo").unwrap().as_str().unwrap();
+            let bar_str = t_json.get("bar").unwrap().as_str().unwrap();
+
+            check_str_invariants(foo_str)?;
+            check_str_invariants(bar_str)?;
+
+        }
+    }
+}

arch/src/x86_64/mod.rs

@@ -7,6 +7,7 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE-BSD-3-Clause file.
 
+pub mod cpu_profile;
 pub mod interrupts;
 pub mod layout;
 pub mod regs;
@@ -14,13 +15,14 @@ pub mod regs;
 #[cfg(feature = "tdx")]
 pub mod tdx;
 
+mod helpers;
 mod mpspec;
 mod mptable;
 mod smbios;
 
 use std::arch::x86_64;
-use std::mem;
 
+use helpers::{deserialize_u32_hex, serialize_u32_hex};
 use hypervisor::arch::x86::{CPUID_FLAG_VALID_INDEX, CpuIdEntry};
 use hypervisor::{CpuVendor, HypervisorCpuError, HypervisorError};
 use linux_loader::loader::bootparam::{boot_params, setup_header};
@@ -28,13 +30,16 @@ use linux_loader::loader::elf::start_info::{
     hvm_memmap_table_entry, hvm_modlist_entry, hvm_start_info,
 };
 use log::{debug, error, info};
+pub use smbios::{SmbiosChassisConfig, SmbiosConfig, SmbiosSystem};
 use thiserror::Error;
 use vm_memory::{
     Address, Bytes, GuestAddress, GuestAddressSpace, GuestMemory, GuestMemoryAtomic,
     GuestMemoryRegion,
 };
+use vmm_sys_util::fam;
 
-use crate::{GuestMemoryMmap, InitramfsConfig, RegionType};
+use crate::x86_64::cpu_profile::cpuid_adjustments::MissingCpuidEntriesError;
+use crate::{CpuProfile, GuestMemoryMmap, InitramfsConfig, RegionType};
 
 // While modern architectures support more than 255 CPUs via x2APIC,
 // legacy devices such as mptable support at most 254 CPUs.
@@ -55,6 +60,9 @@ const AMX_INT8: u8 = 25; // AMX tile computation on 8-bit integers
 const AMX_FP16: u8 = 21; // AMX tile computation on fp16 numbers
 const AMX_COMPLEX: u8 = 8; // AMX tile computation on complex numbers
 
+const AMX_TILECFG_BIT: u8 = 17; // AMX tile cfg state component bit
+const AMX_TILEDATA_BIT: u8 = 18; // AMX tile data state component bit
+
 // KVM feature bits
 #[cfg(feature = "tdx")]
 const KVM_FEATURE_CLOCKSOURCE_BIT: u8 = 0;
@@ -93,6 +101,7 @@ pub struct CpuidConfig {
     #[cfg(feature = "tdx")]
     pub tdx: bool,
     pub amx: bool,
+    pub profile: CpuProfile,
 }
 
 #[derive(Debug, Error)]
@@ -135,16 +144,32 @@ pub enum Error {
 
     /// Error populating CPUID with KVM HyperV emulation details
     #[error("Error populating CPUID with KVM HyperV emulation details")]
-    CpuidKvmHyperV(#[source] vmm_sys_util::fam::Error),
+    CpuidKvmHyperV(#[source] fam::Error),
 
     /// Error populating CPUID with CPU identification
     #[error("Error populating CPUID with CPU identification")]
-    CpuidIdentification(#[source] vmm_sys_util::fam::Error),
+    CpuidIdentification(#[source] fam::Error),
 
     /// Error checking CPUID compatibility
     #[error("Error checking CPUID compatibility")]
     CpuidCheckCompatibility,
 
+    /// Error checking if CPUID is compatible with profile
+    #[error(
+        "The selected CPU profile cannot be utilized because the host's CPUID entries are not compatible with the profile"
+    )]
+    CpuProfileCpuidIncompatibility,
+
+    /// Error because TDX cannot be enabled when a custom (non host) CPU profile has been selected
+    #[error("TDX cannot be enabled when a custom CPU profile has been selected")]
+    CpuProfileTdxIncompatibility,
+
+    /// Error when trying to apply a CPU profile because a necessary CPUID entry was not found
+    #[error(
+        "The selected CPU profile cannot be utilized because a necessary CPUID entry was not found"
+    )]
+    MissingExpectedCpuidEntry(#[source] MissingCpuidEntriesError),
+
     // Error writing EBDA address
     #[error("Error writing EBDA address")]
     EbdaSetup(#[source] vm_memory::GuestMemoryError),
@@ -190,7 +215,7 @@ pub fn get_max_x2apic_id(topology: (u16, u16, u16, u16)) -> u32 {
     )
 }
 
-#[derive(Copy, Clone, Debug)]
+#[derive(Copy, Clone, Debug, PartialEq, Eq, serde::Deserialize, serde::Serialize)]
 pub enum CpuidReg {
     EAX,
     EBX,
@@ -502,11 +527,27 @@ impl CpuidFeatureEntry {
         features
     }
 
-    // The function returns `Error` (a.k.a. "incompatible"), when the CPUID features from `src_vm_cpuid`
-    // is not a subset of those of the `dest_vm_cpuid`.
+    /// The function returns `Error` (a.k.a. "incompatible"), when the CPUID features from `src_vm_cpuid`
+    /// is not a subset of those of the `dest_vm_cpuid`.
     pub fn check_cpuid_compatibility(
         src_vm_cpuid: &[CpuIdEntry],
         dest_vm_cpuid: &[CpuIdEntry],
+    ) -> Result<(), Error> {
+        Self::check_cpuid_compatibility_with_descriptions(
+            src_vm_cpuid,
+            "source VM",
+            dest_vm_cpuid,
+            "destination VM",
+        )
+    }
+
+    /// Similar to `check_cpuid_compatibility`, but with the possibility to change
+    /// the description of the source and destination for logging purposes.
+    fn check_cpuid_compatibility_with_descriptions(
+        src_vm_cpuid: &[CpuIdEntry],
+        src_description: &str,
+        dest_vm_cpuid: &[CpuIdEntry],
+        dest_description: &str,
     ) -> Result<(), Error> {
         let feature_entry_list = &Self::checked_feature_entry_list();
         let src_vm_features = Self::get_features_from_cpuid(src_vm_cpuid, feature_entry_list);
@@ -532,8 +573,8 @@ impl CpuidFeatureEntry {
             };
             if !entry_compatible {
                 error!(
-                    "Detected incompatible CPUID entry: leaf={:#02x} (subleaf={:#02x}), register='{:?}', \
-                    compatible_check='{:?}', source VM feature='{:#04x}', destination VM feature'{:#04x}'.",
+                    "Detected incompatible CPUID entry: leaf={:#04x} (subleaf={:#04x}), register='{:?}', \
+                    compatible_check='{:?}', {src_description} feature='{:#04x}', {dest_description} feature='{:#04x}'.",
                     entry.function,
                     entry.index,
                     entry.feature_reg,
@@ -555,6 +596,15 @@ impl CpuidFeatureEntry {
     }
 }
 
+/// Generate the CPUID entries intended for every vCPU.
+///
+/// ## CPU profiles
+///
+/// This function takes the CPU profile given in `config` into account and returns compatible CPUID entries
+/// if possible.
+///
+/// An error is returned when the CPUID entries obtained from the hypervisor do not satisfy the requirements
+/// to apply the selected CPU profile.
 pub fn generate_common_cpuid(
     hypervisor: &dyn hypervisor::Hypervisor,
     config: &CpuidConfig,
@@ -577,9 +627,86 @@ pub fn generate_common_cpuid(
     }
 
     info!(
-        "Generating guest CPUID for with physical address size: {}",
+        "Generating guest CPUID with physical address size: {}",
         config.phys_bits
     );
+
+    // Supported CPUID
+    let mut cpuid = hypervisor
+        .get_supported_cpuid()
+        .map_err(Error::CpuidGetSupported)?;
+
+    let is_non_host_profile = !matches!(config.profile, CpuProfile::Host);
+    #[cfg(feature = "tdx")]
+    if config.tdx {
+        if is_non_host_profile {
+            // TDX is not supported by CPU profiles other than host for the time being.
+            return Err(Error::CpuProfileTdxIncompatibility.into());
+        }
+        common_cpuid_tdx_configuration(&mut cpuid, hypervisor)?;
+    }
+
+    // Copy CPU identification string
+    for i in 0x8000_0002..=0x8000_0004 {
+        cpuid.retain(|c| c.function != i);
+        // SAFETY: call cpuid with valid leaves
+        #[allow(unused_unsafe)]
+        let leaf = unsafe { x86_64::__cpuid(i) };
+        cpuid.push(CpuIdEntry {
+            function: i,
+            eax: leaf.eax,
+            ebx: leaf.ebx,
+            ecx: leaf.ecx,
+            edx: leaf.edx,
+            ..Default::default()
+        });
+    }
+
+    let cpuid_profile = if is_non_host_profile {
+        let cpuid_profile = config
+            .profile
+            .adjust_cpuid(cpuid.clone(), config.amx, hypervisor.get_cpu_vendor())
+            .map_err(Error::MissingExpectedCpuidEntry)?;
+
+        required_common_cpuid_updates(
+            cpuid_profile,
+            config,
+            #[cfg(feature = "kvm")]
+            hypervisor.hypervisor_type(),
+        )
+    } else {
+        Vec::new()
+    };
+
+    let cpuid_host = required_common_cpuid_updates(
+        cpuid,
+        config,
+        #[cfg(feature = "kvm")]
+        hypervisor.hypervisor_type(),
+    );
+
+    // If we want to apply a CPU profile we need to check that it remains compatible with `cpuid_host`
+    if is_non_host_profile {
+        CpuidFeatureEntry::check_cpuid_compatibility_with_descriptions(
+            &cpuid_profile,
+            "CPU Profile",
+            &cpuid_host,
+            "Host VM",
+        )
+        .map_err(|_| Error::CpuProfileCpuidIncompatibility)?;
+        Ok(cpuid_profile)
+    } else {
+        Ok(cpuid_host)
+    }
+}
+
+/// Apply updates to common CPUID (not vCPU specific) that are necessary regardless of
+/// the chosen CPU profile.
+fn required_common_cpuid_updates(
+    mut cpuid: Vec<CpuIdEntry>,
+    config: &CpuidConfig,
+    #[cfg(feature = "kvm")] hypervisor_type: hypervisor::HypervisorType,
+) -> Vec<CpuIdEntry> {
     #[allow(unused_mut)]
     let mut cpuid_patches = vec![
         // Patch hypervisor bit
@@ -605,10 +732,7 @@ pub fn generate_common_cpuid(
     ];
 
     #[cfg(feature = "kvm")]
-    if matches!(
-        hypervisor.hypervisor_type(),
-        hypervisor::HypervisorType::Kvm
-    ) {
+    if matches!(hypervisor_type, hypervisor::HypervisorType::Kvm) {
         // Patch tsc deadline timer bit
         cpuid_patches.push(CpuidPatch {
             function: 1,
@@ -621,24 +745,8 @@ pub fn generate_common_cpuid(
         });
     }
 
-    // Supported CPUID
-    let mut cpuid = hypervisor
-        .get_supported_cpuid()
-        .map_err(Error::CpuidGetSupported)?;
-
     CpuidPatch::patch_cpuid(&mut cpuid, &cpuid_patches);
 
-    #[cfg(feature = "tdx")]
-    let tdx_capabilities = if config.tdx {
-        let caps = hypervisor
-            .tdx_capabilities()
-            .map_err(Error::TdxCapabilities)?;
-        info!("TDX capabilities {caps:#?}");
-        Some(caps)
-    } else {
-        None
-    };
-
     // Update some existing CPUID
     for entry in cpuid.as_mut_slice().iter_mut() {
         #[allow(unused_unsafe)]
@@ -653,25 +761,6 @@ pub fn generate_common_cpuid(
                     entry.edx &= !(1 << AMX_COMPLEX);
                 }
             }
-            0xd =>
-            {
-                #[cfg(feature = "tdx")]
-                if let Some(caps) = &tdx_capabilities {
-                    let xcr0_mask: u64 = 0x82ff;
-                    let xss_mask: u64 = !xcr0_mask;
-                    if entry.index == 0 {
-                        entry.eax &= (caps.xfam_fixed0 as u32) & (xcr0_mask as u32);
-                        entry.eax |= (caps.xfam_fixed1 as u32) & (xcr0_mask as u32);
-                        entry.edx &= ((caps.xfam_fixed0 & xcr0_mask) >> 32) as u32;
-                        entry.edx |= ((caps.xfam_fixed1 & xcr0_mask) >> 32) as u32;
-                    } else if entry.index == 1 {
-                        entry.ecx &= (caps.xfam_fixed0 as u32) & (xss_mask as u32);
-                        entry.ecx |= (caps.xfam_fixed1 as u32) & (xss_mask as u32);
-                        entry.edx &= ((caps.xfam_fixed0 & xss_mask) >> 32) as u32;
-                        entry.edx |= ((caps.xfam_fixed1 & xss_mask) >> 32) as u32;
-                    }
-                }
-            }
             // Tile Information (purely AMX related).
             0x1d if !config.amx => {
                 entry.eax = 0;
@@ -694,10 +783,10 @@ pub fn generate_common_cpuid(
                     && entry.ecx == 0
                     && entry.edx == 0
                     // SAFETY: cpuid called with valid leaves
-                    && unsafe { std::arch::x86_64::__cpuid(0x8000_0000).eax } >= 0x8000_0005 =>
+                    && unsafe { x86_64::__cpuid(0x8000_0000).eax } >= 0x8000_0005 =>
             {
                 // SAFETY: cpuid called with valid leaves
-                let leaf = unsafe { std::arch::x86_64::__cpuid(0x8000_0005) };
+                let leaf = unsafe { x86_64::__cpuid(0x8000_0005) };
                 entry.eax = leaf.eax;
                 entry.ebx = leaf.ebx;
                 entry.ecx = leaf.ecx;
@@ -710,18 +799,20 @@ pub fn generate_common_cpuid(
                     && entry.ecx == 0
                     && entry.edx == 0
                     // SAFETY: cpuid called with valid leaves
-                    && unsafe { std::arch::x86_64::__cpuid(0x8000_0000).eax } >= 0x8000_0006 =>
+                    && unsafe { x86_64::__cpuid(0x8000_0000).eax } >= 0x8000_0006 =>
             {
                 // SAFETY: cpuid called with valid leaves
-                let leaf = unsafe { std::arch::x86_64::__cpuid(0x8000_0006) };
+                let leaf = unsafe { x86_64::__cpuid(0x8000_0006) };
                 entry.eax = leaf.eax;
                 entry.ebx = leaf.ebx;
                 entry.ecx = leaf.ecx;
                 entry.edx = leaf.edx;
             }
-            // Set CPU physical bits
+            // Set CPU physical bits and guest physical bits
             0x8000_0008 => {
-                entry.eax = (entry.eax & 0xffff_ff00) | (config.phys_bits as u32 & 0xff);
+                entry.eax = (entry.eax & 0xff00_ff00)
+                    | (config.phys_bits as u32 & 0xff)
+                    | ((config.phys_bits as u32 & 0xff) << 16);
             }
             0x4000_0001 => {
                 // Enable KVM_FEATURE_MSI_EXT_DEST_ID. This allows the guest to target
@@ -743,22 +834,6 @@ pub fn generate_common_cpuid(
         }
     }
 
-    // Copy CPU identification string
-    for i in 0x8000_0002..=0x8000_0004 {
-        cpuid.retain(|c| c.function != i);
-        // SAFETY: call cpuid with valid leaves
-        #[allow(unused_unsafe)]
-        let leaf = unsafe { std::arch::x86_64::__cpuid(i) };
-        cpuid.push(CpuIdEntry {
-            function: i,
-            eax: leaf.eax,
-            ebx: leaf.ebx,
-            ecx: leaf.ecx,
-            edx: leaf.edx,
-            ..Default::default()
-        });
-    }
-
     if config.kvm_hyperv {
         // Remove conflicting entries
         cpuid.retain(|c| c.function != 0x4000_0000);
@@ -786,16 +861,31 @@ pub fn generate_common_cpuid(
         });
         cpuid.push(CpuIdEntry {
             function: 0x4000_0003,
-            eax: (1 << 1) // AccessPartitionReferenceCounter
+            eax: (1 << 0) // AccessVpRunTimeReg
+                   | (1 << 1) // AccessPartitionReferenceCounter
                    | (1 << 2) // AccessSynicRegs
                    | (1 << 3) // AccessSyntheticTimerRegs
-                   | (1 << 9), // AccessPartitionReferenceTsc
-            edx: 1 << 3, // CPU dynamic partitioning
+                   | (1 << 4) // AccessIntrCtrlRegs (APIC access MSRs / VP Assist EOI)
+                   | (1 << 5) // AccessHypercallMsrs
+                   | (1 << 6) // AccessVpIndex
+                   | (1 << 9) // AccessPartitionReferenceTsc
+                   | (1 << 11), // AccessFrequencyMsrs (TSC/APIC frequency MSRs)
+            edx: (1 << 3) // CPU dynamic partitioning
+                   | (1 << 4) // FastHypercall (XMM register hypercall input)
+                   | (1 << 8), // ExtendedGvaRangesForFlushVirtualAddressList
             ..Default::default()
         });
         cpuid.push(CpuIdEntry {
             function: 0x4000_0004,
-            eax: 1 << 5, // Recommend relaxed timing
+            // Recommendation hints to Hyper-V-aware guests. Bit semantics per
+            // Microsoft Hypervisor Top-Level Functional Specification 7.4.5.
+            eax: (1 << 1) // LocalTlbFlushRecommended
+                   | (1 << 2) // RemoteTlbFlushRecommended
+                   | (1 << 3) // ApicAccessRecommended (VP Assist page MSR EOI/ICR/TPR)
+                   | (1 << 5) // RelaxedTimingRecommended
+                   | (1 << 9) // DeprecatingAeoiRecommended (keeps APICv on with SynIC)
+                   | (1 << 10), // ClusterIpiRecommended (HvCallSendSyntheticClusterIpi)
+            ebx: 0xfff, // Suggested spinlock retry attempts before trapping to host
             ..Default::default()
         });
         for i in 0x4000_0005..=0x4000_000a {
@@ -806,10 +896,39 @@ pub fn generate_common_cpuid(
         }
     }
 
-    Ok(cpuid)
+    cpuid
 }
 
-#[allow(clippy::too_many_arguments)]
+#[cfg(feature = "tdx")]
+fn common_cpuid_tdx_configuration(
+    cpuid: &mut [CpuIdEntry],
+    hypervisor: &dyn hypervisor::Hypervisor,
+) -> super::Result<()> {
+    let caps = hypervisor
+        .tdx_capabilities()
+        .map_err(Error::TdxCapabilities)?;
+    info!("TDX capabilities {caps:#?}");
+
+    for entry in cpuid.iter_mut().filter(|entry| entry.function == 0xd) {
+        let xcr0_mask: u64 = 0x82ff;
+        let xss_mask: u64 = !xcr0_mask;
+        if entry.index == 0 {
+            entry.eax &= (caps.xfam_fixed0 as u32) & (xcr0_mask as u32);
+            entry.eax |= (caps.xfam_fixed1 as u32) & (xcr0_mask as u32);
+            entry.edx &= ((caps.xfam_fixed0 & xcr0_mask) >> 32) as u32;
+            entry.edx |= ((caps.xfam_fixed1 & xcr0_mask) >> 32) as u32;
+        } else if entry.index == 1 {
+            entry.ecx &= (caps.xfam_fixed0 as u32) & (xss_mask as u32);
+            entry.ecx |= (caps.xfam_fixed1 as u32) & (xss_mask as u32);
+            entry.edx &= ((caps.xfam_fixed0 & xss_mask) >> 32) as u32;
+            entry.edx |= ((caps.xfam_fixed1 & xss_mask) >> 32) as u32;
+        }
+    }
+
+    Ok(())
+}
+
+#[expect(clippy::too_many_arguments)]
 pub fn configure_vcpu(
     vcpu: &dyn hypervisor::Vcpu,
     id: u32,
@@ -865,9 +984,7 @@ pub fn configure_vcpu(
         // Need to check that the TSC doesn't vary with dynamic frequency
         #[allow(unused_unsafe)]
         // SAFETY: cpuid called with valid leaves
-        if unsafe { std::arch::x86_64::__cpuid(0x8000_0007) }.edx & (1u32 << INVARIANT_TSC_EDX_BIT)
-            > 0
-        {
+        if unsafe { x86_64::__cpuid(0x8000_0007) }.edx & (1u32 << INVARIANT_TSC_EDX_BIT) > 0 {
             CpuidPatch::set_cpuid_reg(&mut cpuid, 0x4000_0000, None, CpuidReg::EAX, 0x4000_0010);
             cpuid.retain(|c| c.function != 0x4000_0010);
             cpuid.push(CpuIdEntry {
@@ -949,7 +1066,7 @@ pub fn arch_memory_regions() -> Vec<(GuestAddress, usize, RegionType)> {
 /// * `cmdline_addr` - Address in `guest_mem` where the kernel command line was loaded.
 /// * `cmdline_size` - Size of the kernel command line in bytes including the null terminator.
 /// * `num_cpus` - Number of virtual CPUs the guest will have.
-#[allow(clippy::too_many_arguments)]
+#[expect(clippy::too_many_arguments)]
 pub fn configure_system(
     guest_mem: &GuestMemoryMmap,
     cmdline_addr: GuestAddress,
@@ -958,9 +1075,7 @@ pub fn configure_system(
     _num_cpus: u32,
     setup_header: Option<setup_header>,
     rsdp_addr: Option<GuestAddress>,
-    serial_number: Option<&str>,
-    uuid: Option<&str>,
-    oem_strings: Option<&[&str]>,
+    smbios: Option<&SmbiosConfig>,
     topology: Option<(u16, u16, u16, u16)>,
 ) -> super::Result<()> {
     // Write EBDA address to location where ACPICA expects to find it
@@ -968,8 +1083,7 @@ pub fn configure_system(
         .write_obj((layout::EBDA_START.0 >> 4) as u16, layout::EBDA_POINTER)
         .map_err(Error::EbdaSetup)?;
 
-    let size = smbios::setup_smbios(guest_mem, serial_number, uuid, oem_strings)
-        .map_err(Error::SmbiosSetup)?;
+    let size = smbios::setup_smbios(guest_mem, smbios).map_err(Error::SmbiosSetup)?;
 
     // Place the MP table after the SMIOS table aligned to 16 bytes
     let offset = GuestAddress(layout::SMBIOS_START).unchecked_add(size);
@@ -1039,8 +1153,9 @@ pub fn generate_ram_ranges(guest_mem: &GuestMemoryMmap) -> super::Result<Vec<Ram
     // Generate the first usable physical memory range before the gap. The e820 map
     // should only report memory above 1MiB.
     let first_ram_range = {
-        let (first_region_start, first_region_end) =
-            ram_regions.first().ok_or(super::Error::MemmapTableSetup)?;
+        let (first_region_start, first_region_end) = ram_regions
+            .first()
+            .ok_or(super::Error::MemmapTableGeneration)?;
         let high_ram_start = layout::HIGH_RAM_START.raw_value();
         let mem_32bit_reserved_start = layout::MEM_32BIT_RESERVED_START.raw_value();
 
@@ -1053,7 +1168,7 @@ pub fn generate_ram_ranges(guest_mem: &GuestMemoryMmap) -> super::Result<Vec<Ram
                 high_ram_start: 0x{high_ram_start:08x}, mem_32bit_reserved_start: 0x{mem_32bit_reserved_start:08x}"
             );
 
-            return Err(super::Error::MemmapTableSetup);
+            return Err(super::Error::MemmapTableGeneration);
         }
 
         info!(
@@ -1156,7 +1271,7 @@ fn configure_pvh(
     guest_mem
         .checked_offset(
             memmap_start_addr,
-            mem::size_of::<hvm_memmap_table_entry>() * start_info.memmap_entries as usize,
+            size_of::<hvm_memmap_table_entry>() * start_info.memmap_entries as usize,
         )
         .ok_or(super::Error::MemmapTablePastRamEnd)?;
 
@@ -1164,9 +1279,9 @@ fn configure_pvh(
     for memmap_entry in memmap {
         guest_mem
             .write_obj(memmap_entry, memmap_start_addr)
-            .map_err(|_| super::Error::MemmapTableSetup)?;
+            .map_err(super::Error::MemmapTableSetup)?;
         memmap_start_addr =
-            memmap_start_addr.unchecked_add(mem::size_of::<hvm_memmap_table_entry>() as u64);
+            memmap_start_addr.unchecked_add(size_of::<hvm_memmap_table_entry>() as u64);
     }
 
     // The hvm_start_info struct itself must be stored at PVH_START_INFO
@@ -1175,13 +1290,13 @@ fn configure_pvh(
     let start_info_addr = layout::PVH_INFO_START;
 
     guest_mem
-        .checked_offset(start_info_addr, mem::size_of::<hvm_start_info>())
+        .checked_offset(start_info_addr, size_of::<hvm_start_info>())
         .ok_or(super::Error::StartInfoPastRamEnd)?;
 
     // Write the start_info struct to guest memory.
     guest_mem
         .write_obj(start_info, start_info_addr)
-        .map_err(|_| super::Error::StartInfoSetup)?;
+        .map_err(super::Error::StartInfoSetup)?;
 
     Ok(())
 }
@@ -1254,7 +1369,7 @@ fn configure_32bit_entry(
 
     let zero_page_addr = layout::ZERO_PAGE_START;
     guest_mem
-        .checked_offset(zero_page_addr, mem::size_of::<boot_params>())
+        .checked_offset(zero_page_addr, size_of::<boot_params>())
         .ok_or(super::Error::ZeroPagePastRamEnd)?;
     guest_mem
         .write_obj(params, zero_page_addr)
@@ -1312,27 +1427,15 @@ pub fn initramfs_load_addr(
     Ok(aligned_addr)
 }
 
-pub fn get_host_cpu_phys_bits(hypervisor: &dyn hypervisor::Hypervisor) -> u8 {
+pub fn get_host_cpu_phys_bits(_hypervisor: &dyn hypervisor::Hypervisor) -> u8 {
     // SAFETY: call cpuid with valid leaves
     #[allow(unused_unsafe)]
     unsafe {
         let leaf = x86_64::__cpuid(0x8000_0000);
 
-        // Detect and handle AMD SME (Secure Memory Encryption) properly.
-        // Some physical address bits may become reserved when the feature is enabled.
-        // See AMD64 Architecture Programmer's Manual Volume 2, Section 7.10.1
-        let reduced = if leaf.eax >= 0x8000_001f
-            && matches!(hypervisor.get_cpu_vendor(), CpuVendor::AMD)
-            && x86_64::__cpuid(0x8000_001f).eax & 0x1 != 0
-        {
-            (x86_64::__cpuid(0x8000_001f).ebx >> 6) & 0x3f
-        } else {
-            0
-        };
-
         if leaf.eax >= 0x8000_0008 {
             let leaf = x86_64::__cpuid(0x8000_0008);
-            ((leaf.eax & 0xff) - reduced) as u8
+            (leaf.eax & 0xff) as u8
         } else {
             36
         }
@@ -1514,8 +1617,6 @@ mod unit_tests {
             Some(layout::RSDP_POINTER),
             None,
             None,
-            None,
-            None,
         );
         config_err.unwrap_err();
 
@@ -1538,8 +1639,6 @@ mod unit_tests {
             None,
             None,
             None,
-            None,
-            None,
         )
         .unwrap();
 
@@ -1567,8 +1666,6 @@ mod unit_tests {
             None,
             None,
             None,
-            None,
-            None,
         )
         .unwrap();
 
@@ -1582,8 +1679,6 @@ mod unit_tests {
             None,
             None,
             None,
-            None,
-            None,
         )
         .unwrap();
     }

arch/src/x86_64/mpspec.rs

@@ -3,35 +3,36 @@
 // SPDX-License-Identifier: Apache-2.0 AND BSD-3-Clause
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE-BSD-3-Clause file.
-#![allow(non_camel_case_types)]
+use std::os::raw;
+
 use vm_memory::ByteValued;
 
-pub const MP_PROCESSOR: ::std::os::raw::c_uint = 0;
-pub const MP_BUS: ::std::os::raw::c_uint = 1;
-pub const MP_IOAPIC: ::std::os::raw::c_uint = 2;
-pub const MP_INTSRC: ::std::os::raw::c_uint = 3;
-pub const MP_LINTSRC: ::std::os::raw::c_uint = 4;
-pub const CPU_ENABLED: ::std::os::raw::c_uint = 1;
-pub const CPU_BOOTPROCESSOR: ::std::os::raw::c_uint = 2;
-pub const MPC_APIC_USABLE: ::std::os::raw::c_uint = 1;
-pub const MP_IRQDIR_DEFAULT: ::std::os::raw::c_uint = 0;
+pub const MP_PROCESSOR: raw::c_uint = 0;
+pub const MP_BUS: raw::c_uint = 1;
+pub const MP_IOAPIC: raw::c_uint = 2;
+pub const MP_INTSRC: raw::c_uint = 3;
+pub const MP_LINTSRC: raw::c_uint = 4;
+pub const CPU_ENABLED: raw::c_uint = 1;
+pub const CPU_BOOTPROCESSOR: raw::c_uint = 2;
+pub const MPC_APIC_USABLE: raw::c_uint = 1;
+pub const MP_IRQDIR_DEFAULT: raw::c_uint = 0;
 
 #[repr(C)]
 #[derive(Debug, Default, Copy, Clone)]
 pub struct mpf_intel {
-    pub signature: [::std::os::raw::c_uchar; 4usize],
-    pub physptr: ::std::os::raw::c_uint,
-    pub length: ::std::os::raw::c_uchar,
-    pub specification: ::std::os::raw::c_uchar,
-    pub checksum: ::std::os::raw::c_uchar,
-    pub feature1: ::std::os::raw::c_uchar,
-    pub feature2: ::std::os::raw::c_uchar,
-    pub feature3: ::std::os::raw::c_uchar,
-    pub feature4: ::std::os::raw::c_uchar,
-    pub feature5: ::std::os::raw::c_uchar,
+    pub signature: [raw::c_uchar; 4usize],
+    pub physptr: raw::c_uint,
+    pub length: raw::c_uchar,
+    pub specification: raw::c_uchar,
+    pub checksum: raw::c_uchar,
+    pub feature1: raw::c_uchar,
+    pub feature2: raw::c_uchar,
+    pub feature3: raw::c_uchar,
+    pub feature4: raw::c_uchar,
+    pub feature5: raw::c_uchar,
 }
 
-const _: () = assert!(::core::mem::size_of::<mpf_intel>() == 16);
+const _: () = assert!(size_of::<mpf_intel>() == 16);
 // SAFETY: all members of this struct are plain integers
 // and the sum of their sizes is the size of the struct, so
 // padding and reserved values are not possible as there
@@ -41,24 +42,24 @@ unsafe impl ByteValued for mpf_intel {}
 #[repr(C)]
 #[derive(Debug, Default, Copy, Clone)]
 pub struct mpc_table {
-    pub signature: [::std::os::raw::c_uchar; 4usize],
-    pub length: ::std::os::raw::c_ushort,
-    pub spec: ::std::os::raw::c_uchar,
-    pub checksum: ::std::os::raw::c_uchar,
-    pub oem: [::std::os::raw::c_uchar; 8usize],
-    pub productid: [::std::os::raw::c_uchar; 12usize],
-    pub oemptr: ::std::os::raw::c_uint,
-    pub oemsize: ::std::os::raw::c_ushort,
-    pub oemcount: ::std::os::raw::c_ushort,
-    pub lapic: ::std::os::raw::c_uint,
-    pub reserved: ::std::os::raw::c_uint,
+    pub signature: [raw::c_uchar; 4usize],
+    pub length: raw::c_ushort,
+    pub spec: raw::c_uchar,
+    pub checksum: raw::c_uchar,
+    pub oem: [raw::c_uchar; 8usize],
+    pub productid: [raw::c_uchar; 12usize],
+    pub oemptr: raw::c_uint,
+    pub oemsize: raw::c_ushort,
+    pub oemcount: raw::c_ushort,
+    pub lapic: raw::c_uint,
+    pub reserved: raw::c_uint,
 }
 
 const _: () = {
-    assert!(::core::mem::size_of::<mpc_table>() == 4 + 2 + 1 + 1 + 8 + 12 + 4 + 2 + 2 + 4 + 4);
-    assert!(::core::mem::size_of::<::std::os::raw::c_uint>() == 4);
-    assert!(::core::mem::size_of::<::std::os::raw::c_ushort>() == 2);
-    assert!(::core::mem::size_of::<::std::os::raw::c_uchar>() == 1);
+    assert!(size_of::<mpc_table>() == 4 + 2 + 1 + 1 + 8 + 12 + 4 + 2 + 2 + 4 + 4);
+    assert!(size_of::<raw::c_uint>() == 4);
+    assert!(size_of::<raw::c_ushort>() == 2);
+    assert!(size_of::<raw::c_uchar>() == 1);
 };
 
 // SAFETY: all members of this struct are plain integers
@@ -70,16 +71,16 @@ unsafe impl ByteValued for mpc_table {}
 #[repr(C)]
 #[derive(Debug, Default, Copy, Clone)]
 pub struct mpc_cpu {
-    pub type_: ::std::os::raw::c_uchar,
-    pub apicid: ::std::os::raw::c_uchar,
-    pub apicver: ::std::os::raw::c_uchar,
-    pub cpuflag: ::std::os::raw::c_uchar,
-    pub cpufeature: ::std::os::raw::c_uint,
-    pub featureflag: ::std::os::raw::c_uint,
-    pub reserved: [::std::os::raw::c_uint; 2usize],
+    pub type_: raw::c_uchar,
+    pub apicid: raw::c_uchar,
+    pub apicver: raw::c_uchar,
+    pub cpuflag: raw::c_uchar,
+    pub cpufeature: raw::c_uint,
+    pub featureflag: raw::c_uint,
+    pub reserved: [raw::c_uint; 2usize],
 }
 
-const _: () = assert!(::core::mem::size_of::<mpc_cpu>() == 20);
+const _: () = assert!(size_of::<mpc_cpu>() == 20);
 // SAFETY: all members of this struct are plain integers
 // and the sum of their sizes is the size of the struct, so
 // padding and reserved values are not possible as there
@@ -89,12 +90,12 @@ unsafe impl ByteValued for mpc_cpu {}
 #[repr(C)]
 #[derive(Debug, Default, Copy, Clone)]
 pub struct mpc_bus {
-    pub type_: ::std::os::raw::c_uchar,
-    pub busid: ::std::os::raw::c_uchar,
-    pub bustype: [::std::os::raw::c_uchar; 6usize],
+    pub type_: raw::c_uchar,
+    pub busid: raw::c_uchar,
+    pub bustype: [raw::c_uchar; 6usize],
 }
 
-const _: () = assert!(::core::mem::size_of::<mpc_bus>() == 8);
+const _: () = assert!(size_of::<mpc_bus>() == 8);
 // SAFETY: all members of this struct are plain integers
 // and the sum of their sizes is the size of the struct, so
 // padding and reserved values are not possible as there
@@ -104,14 +105,14 @@ unsafe impl ByteValued for mpc_bus {}
 #[repr(C)]
 #[derive(Debug, Default, Copy, Clone)]
 pub struct mpc_ioapic {
-    pub type_: ::std::os::raw::c_uchar,
-    pub apicid: ::std::os::raw::c_uchar,
-    pub apicver: ::std::os::raw::c_uchar,
-    pub flags: ::std::os::raw::c_uchar,
-    pub apicaddr: ::std::os::raw::c_uint,
+    pub type_: raw::c_uchar,
+    pub apicid: raw::c_uchar,
+    pub apicver: raw::c_uchar,
+    pub flags: raw::c_uchar,
+    pub apicaddr: raw::c_uint,
 }
 
-const _: () = assert!(::core::mem::size_of::<mpc_ioapic>() == 8);
+const _: () = assert!(size_of::<mpc_ioapic>() == 8);
 // SAFETY: all members of this struct are plain integers
 // and the sum of their sizes is the size of the struct, so
 // padding and reserved values are not possible as there
@@ -121,39 +122,39 @@ unsafe impl ByteValued for mpc_ioapic {}
 #[repr(C)]
 #[derive(Debug, Default, Copy, Clone)]
 pub struct mpc_intsrc {
-    pub type_: ::std::os::raw::c_uchar,
-    pub irqtype: ::std::os::raw::c_uchar,
-    pub irqflag: ::std::os::raw::c_ushort,
-    pub srcbus: ::std::os::raw::c_uchar,
-    pub srcbusirq: ::std::os::raw::c_uchar,
-    pub dstapic: ::std::os::raw::c_uchar,
-    pub dstirq: ::std::os::raw::c_uchar,
+    pub type_: raw::c_uchar,
+    pub irqtype: raw::c_uchar,
+    pub irqflag: raw::c_ushort,
+    pub srcbus: raw::c_uchar,
+    pub srcbusirq: raw::c_uchar,
+    pub dstapic: raw::c_uchar,
+    pub dstirq: raw::c_uchar,
 }
 
-const _: () = assert!(::core::mem::size_of::<mpc_intsrc>() == 8);
+const _: () = assert!(size_of::<mpc_intsrc>() == 8);
 // SAFETY: all members of this struct are plain integers
 // and the sum of their sizes is the size of the struct, so
 // padding and reserved values are not possible as there
 // would be nowhere for them to exist.
 unsafe impl ByteValued for mpc_intsrc {}
 
-pub const MP_IRQ_SOURCE_TYPES_MP_INT: ::std::os::raw::c_uint = 0;
-pub const MP_IRQ_SOURCE_TYPES_MP_NMI: ::std::os::raw::c_uint = 1;
-pub const MP_IRQ_SOURCE_TYPES_MP_EXT_INT: ::std::os::raw::c_uint = 3;
+pub const MP_IRQ_SOURCE_TYPES_MP_INT: raw::c_uint = 0;
+pub const MP_IRQ_SOURCE_TYPES_MP_NMI: raw::c_uint = 1;
+pub const MP_IRQ_SOURCE_TYPES_MP_EXT_INT: raw::c_uint = 3;
 
 #[repr(C)]
 #[derive(Debug, Default, Copy, Clone)]
 pub struct mpc_lintsrc {
-    pub type_: ::std::os::raw::c_uchar,
-    pub irqtype: ::std::os::raw::c_uchar,
-    pub irqflag: ::std::os::raw::c_ushort,
-    pub srcbusid: ::std::os::raw::c_uchar,
-    pub srcbusirq: ::std::os::raw::c_uchar,
-    pub destapic: ::std::os::raw::c_uchar,
-    pub destapiclint: ::std::os::raw::c_uchar,
+    pub type_: raw::c_uchar,
+    pub irqtype: raw::c_uchar,
+    pub irqflag: raw::c_ushort,
+    pub srcbusid: raw::c_uchar,
+    pub srcbusirq: raw::c_uchar,
+    pub destapic: raw::c_uchar,
+    pub destapiclint: raw::c_uchar,
 }
 
-const _: () = assert!(::core::mem::size_of::<mpc_lintsrc>() == 8);
+const _: () = assert!(size_of::<mpc_lintsrc>() == 8);
 // SAFETY: all members of this struct are plain integers
 // and the sum of their sizes is the size of the struct, so
 // padding and reserved values are not possible as there
@@ -163,14 +164,14 @@ unsafe impl ByteValued for mpc_lintsrc {}
 #[repr(C)]
 #[derive(Debug, Default, Copy, Clone)]
 pub struct mpc_oemtable {
-    pub signature: [::std::os::raw::c_uchar; 4usize],
-    pub length: ::std::os::raw::c_ushort,
-    pub rev: ::std::os::raw::c_uchar,
-    pub checksum: ::std::os::raw::c_uchar,
-    pub mpc: [::std::os::raw::c_uchar; 8usize],
+    pub signature: [raw::c_uchar; 4usize],
+    pub length: raw::c_ushort,
+    pub rev: raw::c_uchar,
+    pub checksum: raw::c_uchar,
+    pub mpc: [raw::c_uchar; 8usize],
 }
 
-const _: () = assert!(::core::mem::size_of::<mpc_oemtable>() == 16);
+const _: () = assert!(size_of::<mpc_oemtable>() == 16);
 // SAFETY: all members of this struct are plain integers
 // and the sum of their sizes is the size of the struct, so
 // padding and reserved values are not possible as there

arch/src/x86_64/mptable.rs

@@ -5,7 +5,7 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE-BSD-3-Clause file.
 
-use std::{mem, result, slice};
+use std::result;
 
 use libc::c_uchar;
 use log::{info, warn};
@@ -101,11 +101,8 @@ const CPU_FEATURE_APIC: u32 = 0x200;
 const CPU_FEATURE_FPU: u32 = 0x001;
 
 fn compute_checksum<T: Copy + ByteValued>(v: &T) -> u8 {
-    let v: *const T = v;
-    // SAFETY: we are only reading the bytes within the size of the `T` reference `v`.
-    let v_slice = unsafe { slice::from_raw_parts(v.cast(), mem::size_of::<T>()) };
     let mut checksum: u8 = 0;
-    for i in v_slice.iter() {
+    for i in v.as_slice().iter() {
         checksum = checksum.wrapping_add(*i);
     }
     checksum
@@ -117,13 +114,13 @@ fn mpf_intel_compute_checksum(v: &mpspec::mpf_intel) -> u8 {
 }
 
 fn compute_mp_size(num_cpus: u32) -> usize {
-    mem::size_of::<MpfIntelWrapper>()
-        + mem::size_of::<MpcTableWrapper>()
-        + mem::size_of::<MpcCpuWrapper>() * (num_cpus as usize)
-        + mem::size_of::<MpcIoapicWrapper>()
-        + mem::size_of::<MpcBusWrapper>()
-        + mem::size_of::<MpcIntsrcWrapper>() * 16
-        + mem::size_of::<MpcLintsrcWrapper>() * 2
+    size_of::<MpfIntelWrapper>()
+        + size_of::<MpcTableWrapper>()
+        + size_of::<MpcCpuWrapper>() * (num_cpus as usize)
+        + size_of::<MpcIoapicWrapper>()
+        + size_of::<MpcBusWrapper>()
+        + size_of::<MpcIntsrcWrapper>() * 16
+        + size_of::<MpcLintsrcWrapper>() * 2
 }
 
 /// Performs setup of the MP table for the given `num_cpus`.
@@ -170,7 +167,7 @@ pub fn setup_mptable(
 
     {
         let mut mpf_intel = MpfIntelWrapper(mpspec::mpf_intel::default());
-        let size = mem::size_of::<MpfIntelWrapper>() as u64;
+        let size = size_of::<MpfIntelWrapper>() as u64;
         mpf_intel.0.signature = *SMP_MAGIC_IDENT;
         mpf_intel.0.length = 1;
         mpf_intel.0.specification = 4;
@@ -184,10 +181,10 @@ pub fn setup_mptable(
     // We set the location of the mpc_table here but we can't fill it out until we have the length
     // of the entire table later.
     let table_base = base_mp;
-    base_mp = base_mp.unchecked_add(mem::size_of::<MpcTableWrapper>() as u64);
+    base_mp = base_mp.unchecked_add(size_of::<MpcTableWrapper>() as u64);
 
     {
-        let size = mem::size_of::<MpcCpuWrapper>();
+        let size = size_of::<MpcCpuWrapper>();
         for cpu_id in 0..num_cpus {
             let mut mpc_cpu = MpcCpuWrapper(mpspec::mpc_cpu::default());
             mpc_cpu.0.type_ = mpspec::MP_PROCESSOR as u8;
@@ -208,7 +205,7 @@ pub fn setup_mptable(
         }
     }
     {
-        let size = mem::size_of::<MpcBusWrapper>();
+        let size = size_of::<MpcBusWrapper>();
         let mut mpc_bus = MpcBusWrapper(mpspec::mpc_bus::default());
         mpc_bus.0.type_ = mpspec::MP_BUS as u8;
         mpc_bus.0.busid = 0;
@@ -219,7 +216,7 @@ pub fn setup_mptable(
         checksum = checksum.wrapping_add(compute_checksum(&mpc_bus.0));
     }
     {
-        let size = mem::size_of::<MpcIoapicWrapper>();
+        let size = size_of::<MpcIoapicWrapper>();
         let mut mpc_ioapic = MpcIoapicWrapper(mpspec::mpc_ioapic::default());
         mpc_ioapic.0.type_ = mpspec::MP_IOAPIC as u8;
         mpc_ioapic.0.apicid = ioapicid;
@@ -233,7 +230,7 @@ pub fn setup_mptable(
     }
     // Per kvm_setup_default_irq_routing() in kernel
     for i in 0..16 {
-        let size = mem::size_of::<MpcIntsrcWrapper>();
+        let size = size_of::<MpcIntsrcWrapper>();
         let mut mpc_intsrc = MpcIntsrcWrapper(mpspec::mpc_intsrc::default());
         mpc_intsrc.0.type_ = mpspec::MP_INTSRC as u8;
         mpc_intsrc.0.irqtype = mpspec::MP_IRQ_SOURCE_TYPES_MP_INT as u8;
@@ -248,7 +245,7 @@ pub fn setup_mptable(
         checksum = checksum.wrapping_add(compute_checksum(&mpc_intsrc.0));
     }
     {
-        let size = mem::size_of::<MpcLintsrcWrapper>();
+        let size = size_of::<MpcLintsrcWrapper>();
         let mut mpc_lintsrc = MpcLintsrcWrapper(mpspec::mpc_lintsrc::default());
         mpc_lintsrc.0.type_ = mpspec::MP_LINTSRC as u8;
         mpc_lintsrc.0.irqtype = mpspec::MP_IRQ_SOURCE_TYPES_MP_EXT_INT as u8;
@@ -263,7 +260,7 @@ pub fn setup_mptable(
         checksum = checksum.wrapping_add(compute_checksum(&mpc_lintsrc.0));
     }
     {
-        let size = mem::size_of::<MpcLintsrcWrapper>();
+        let size = size_of::<MpcLintsrcWrapper>();
         let mut mpc_lintsrc = MpcLintsrcWrapper(mpspec::mpc_lintsrc::default());
         mpc_lintsrc.0.type_ = mpspec::MP_LINTSRC as u8;
         mpc_lintsrc.0.irqtype = mpspec::MP_IRQ_SOURCE_TYPES_MP_NMI as u8;
@@ -308,11 +305,11 @@ mod unit_tests {
 
     fn table_entry_size(type_: u8) -> usize {
         match type_ as u32 {
-            mpspec::MP_PROCESSOR => mem::size_of::<MpcCpuWrapper>(),
-            mpspec::MP_BUS => mem::size_of::<MpcBusWrapper>(),
-            mpspec::MP_IOAPIC => mem::size_of::<MpcIoapicWrapper>(),
-            mpspec::MP_INTSRC => mem::size_of::<MpcIntsrcWrapper>(),
-            mpspec::MP_LINTSRC => mem::size_of::<MpcLintsrcWrapper>(),
+            mpspec::MP_PROCESSOR => size_of::<MpcCpuWrapper>(),
+            mpspec::MP_BUS => size_of::<MpcBusWrapper>(),
+            mpspec::MP_IOAPIC => size_of::<MpcIoapicWrapper>(),
+            mpspec::MP_INTSRC => size_of::<MpcIntsrcWrapper>(),
+            mpspec::MP_LINTSRC => size_of::<MpcLintsrcWrapper>(),
             _ => panic!("unrecognized mpc table entry type: {type_}"),
         }
     }
@@ -405,7 +402,7 @@ mod unit_tests {
                 .unwrap();
 
             let mut entry_offset = mpc_offset
-                .checked_add(mem::size_of::<MpcTableWrapper>() as GuestUsize)
+                .checked_add(size_of::<MpcTableWrapper>() as GuestUsize)
                 .unwrap();
             let mut cpu_count = 0;
             while entry_offset < mpc_end {

arch/src/x86_64/regs.rs

@@ -6,7 +6,7 @@
 // Portions Copyright 2017 The Chromium OS Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE-BSD-3-Clause file.
-use std::{mem, result};
+use std::result;
 
 use hypervisor::arch::x86::gdt::{gdt_entry, segment_from_gdt};
 use hypervisor::arch::x86::regs::CR0_PE;
@@ -134,7 +134,7 @@ fn write_gdt_table(table: &[u64], guest_mem: &GuestMemoryMmap) -> Result<()> {
     let boot_gdt_addr = BOOT_GDT_START;
     for (index, entry) in table.iter().enumerate() {
         let addr = guest_mem
-            .checked_offset(boot_gdt_addr, index * mem::size_of::<u64>())
+            .checked_offset(boot_gdt_addr, index * size_of::<u64>())
             .ok_or(Error::CheckGdtAddr)?;
         guest_mem.write_obj(*entry, addr).map_err(Error::WriteGdt)?;
     }
@@ -170,11 +170,11 @@ pub fn configure_segments_and_sregs(
     // Write segments
     write_gdt_table(&gdt_table[..], mem)?;
     sregs.gdt.base = BOOT_GDT_START.raw_value();
-    sregs.gdt.limit = mem::size_of_val(&gdt_table) as u16 - 1;
+    sregs.gdt.limit = size_of_val(&gdt_table) as u16 - 1;
 
     write_idt_value(0, mem)?;
     sregs.idt.base = BOOT_IDT_START.raw_value();
-    sregs.idt.limit = mem::size_of::<u64>() as u16 - 1;
+    sregs.idt.limit = size_of::<u64>() as u16 - 1;
 
     sregs.cs = code_seg;
     sregs.ds = data_seg;

arch/src/x86_64/smbios.rs

@@ -6,7 +6,7 @@
 //
 // SPDX-License-Identifier: Apache-2.0 AND BSD-3-Clause
 
-use std::{mem, result, slice};
+use std::result;
 
 use thiserror::Error;
 use uuid::Uuid;
@@ -28,39 +28,74 @@ pub enum Error {
     Clear,
     /// Failure to write SMBIOS entrypoint structure
     #[error("Failure to write SMBIOS entrypoint structure")]
-    WriteSmbiosEp,
+    WriteSmbiosEp(#[source] vm_memory::GuestMemoryError),
     /// Failure to write additional data to memory
     #[error("Failure to write additional data to memory")]
-    WriteData,
+    WriteData(#[source] vm_memory::GuestMemoryError),
     /// Failure to parse uuid, uuid format may be error
     #[error("Failure to parse uuid: {1}")]
     ParseUuid(#[source] uuid::Error, String),
+    /// SMBIOS string index overflow (u8 limit reached).
+    #[error("SMBIOS string index overflow (u8 limit reached: {})", u8::MAX)]
+    TooManyStrings,
 }
 
 pub type Result<T> = result::Result<T, Error>;
 
-// Constants sourced from SMBIOS Spec 3.2.0.
+// Constants sourced from SMBIOS Spec 3.9.0.
 const SM3_MAGIC_IDENT: &[u8; 5usize] = b"_SM3_";
 const BIOS_INFORMATION: u8 = 0;
 const SYSTEM_INFORMATION: u8 = 1;
 const OEM_STRINGS: u8 = 11;
+const SYSTEM_ENCLOSURE: u8 = 3;
 const END_OF_TABLE: u8 = 127;
+const SYSTEM_WAKE_UP_TYPE_UNKNOWN: u8 = 0x02;
+const CHASSIS_TYPE_UNKNOWN: u8 = 0x02;
+const CHASSIS_STATE_UNKNOWN: u8 = 0x02;
+const CHASSIS_SECURITY_STATUS_NONE: u8 = 0x03;
 const PCI_SUPPORTED: u64 = 1 << 7;
 const IS_VIRTUAL_MACHINE: u8 = 1 << 4;
+pub const DEFAULT_SYSTEM_MANUFACTURER: &str = "Cloud Hypervisor";
+pub const DEFAULT_SYSTEM_PRODUCT_NAME: &str = "cloud-hypervisor";
 
-fn compute_checksum<T: Copy>(v: &T) -> u8 {
-    let v: *const T = v;
-    // SAFETY: we are only reading the bytes within the size of the `T` reference `v`.
-    let v_slice = unsafe { slice::from_raw_parts(v.cast(), mem::size_of::<T>()) };
+#[derive(Clone, Debug, Default, PartialEq, Eq)]
+pub struct SmbiosConfig {
+    pub system: Option<SmbiosSystem>,
+    pub chassis: Option<SmbiosChassisConfig>,
+    pub oem_strings: Box<[String]>,
+}
+
+#[derive(Clone, Debug, Default, PartialEq, Eq)]
+pub struct SmbiosSystem {
+    pub manufacturer: Option<String>,
+    pub product_name: Option<String>,
+    pub version: Option<String>,
+    pub serial_number: Option<String>,
+    pub uuid: Option<String>,
+    pub sku_number: Option<String>,
+    pub family: Option<String>,
+}
+
+#[derive(Clone, Debug, Default, PartialEq, Eq)]
+pub struct SmbiosChassisConfig {
+    pub asset_tag: Option<String>,
+}
+
+impl SmbiosConfig {
+    pub fn is_empty(&self) -> bool {
+        *self == Self::default()
+    }
+}
+
+fn compute_checksum<T: Copy + ByteValued>(v: &T) -> u8 {
     let mut checksum: u8 = 0;
-    for i in v_slice.iter() {
+    for i in v.as_slice().iter() {
         checksum = checksum.wrapping_add(*i);
     }
     (!checksum).wrapping_add(1)
 }
 
-#[repr(C)]
-#[repr(packed)]
+#[repr(C, packed)]
 #[derive(Default, Copy, Clone)]
 struct Smbios30Entrypoint {
     signature: [u8; 5usize],
@@ -75,8 +110,7 @@ struct Smbios30Entrypoint {
     physptr: u64,
 }
 
-#[repr(C)]
-#[repr(packed)]
+#[repr(C, packed)]
 #[derive(Default, Copy, Clone)]
 struct SmbiosBiosInfo {
     r#type: u8,
@@ -92,8 +126,7 @@ struct SmbiosBiosInfo {
     characteristics_ext2: u8,
 }
 
-#[repr(C)]
-#[repr(packed)]
+#[repr(C, packed)]
 #[derive(Default, Copy, Clone)]
 struct SmbiosSysInfo {
     r#type: u8,
@@ -109,8 +142,7 @@ struct SmbiosSysInfo {
     family: u8,
 }
 
-#[repr(C)]
-#[repr(packed)]
+#[repr(C, packed)]
 #[derive(Default, Copy, Clone)]
 struct SmbiosOemStrings {
     r#type: u8,
@@ -119,8 +151,34 @@ struct SmbiosOemStrings {
     count: u8,
 }
 
-#[repr(C)]
-#[repr(packed)]
+/// SMBIOS Chassis Table (Type 3) as defined in DMTF SMBIOS 3.9.0:
+/// https://www.dmtf.org/sites/default/files/standards/documents/DSP0134_3.9.0.pdf
+/// Note: trailing fields are omitted, so this structure is not complete.
+#[repr(C, packed)]
+#[derive(Default, Copy, Clone)]
+struct SmbiosChassis {
+    r#type: u8,
+    length: u8,
+    handle: u16,
+    manufacturer: u8,
+    chassis_type: u8,
+    version: u8,
+    serial_number: u8,
+    asset_tag: u8,
+    bootup_state: u8,
+    power_supply_state: u8,
+    thermal_state: u8,
+    security_status: u8,
+    oem_defined: u32,
+    height: u8,
+    number_of_power_cords: u8,
+    contained_element_count: u8,
+    contained_element_record_length: u8,
+    // followed by contained element records (optional, variable-length)
+    // followed by sku_number: u8, rack_type: u8, rack_height: u8
+}
+
+#[repr(C, packed)]
 #[derive(Default, Copy, Clone)]
 struct SmbiosEndOfTable {
     r#type: u8,
@@ -137,6 +195,8 @@ unsafe impl ByteValued for SmbiosSysInfo {}
 // SAFETY: data structure only contain a series of integers
 unsafe impl ByteValued for SmbiosOemStrings {}
 // SAFETY: data structure only contain a series of integers
+unsafe impl ByteValued for SmbiosChassis {}
+// SAFETY: data structure only contain a series of integers
 unsafe impl ByteValued for SmbiosEndOfTable {}
 
 fn write_and_incr<T: ByteValued>(
@@ -144,9 +204,9 @@ fn write_and_incr<T: ByteValued>(
     val: T,
     mut curptr: GuestAddress,
 ) -> Result<GuestAddress> {
-    mem.write_obj(val, curptr).map_err(|_| Error::WriteData)?;
+    mem.write_obj(val, curptr).map_err(Error::WriteData)?;
     curptr = curptr
-        .checked_add(mem::size_of::<T>() as u64)
+        .checked_add(size_of::<T>() as u64)
         .ok_or(Error::NotEnoughMemory)?;
     Ok(curptr)
 }
@@ -163,14 +223,155 @@ fn write_string(
     Ok(curptr)
 }
 
-pub fn setup_smbios(
+fn write_opt_string(
     mem: &GuestMemoryMmap,
-    serial_number: Option<&str>,
-    uuid: Option<&str>,
-    oem_strings: Option<&[&str]>,
-) -> Result<u64> {
+    s: Option<&str>,
+    cur: GuestAddress,
+) -> Result<GuestAddress> {
+    if let Some(v) = s {
+        write_string(mem, v, cur)
+    } else {
+        Ok(cur)
+    }
+}
+
+fn write_string_terminator(
+    mem: &GuestMemoryMmap,
+    cur: GuestAddress,
+    has_strings: bool,
+) -> Result<GuestAddress> {
+    // SMBIOS DSP0134 §6.1.3: if all string-reference fields are 0, follow the
+    // formatted section with two null bytes (empty string-set).
+    if has_strings {
+        write_and_incr(mem, 0u8, cur)
+    } else {
+        let cur = write_and_incr(mem, 0u8, cur)?;
+        write_and_incr(mem, 0u8, cur)
+    }
+}
+
+/// Allocate the next string index for an SMBIOS string-set.
+///
+/// Per SMBIOS DSP0134, index `0` means "no string", so valid indices run from
+/// `1` to `255`. Returns `0` when `present` is `false`. Otherwise returns the
+/// current value of `*next` and advances it by one. Fails with
+/// [`Error::TooManyStrings`] once all 255 indices have been used: `next`
+/// starts at `1`, so it can only be `0` here after wrapping past `255`.
+fn alloc_index(next: &mut u8, present: bool) -> Result<u8> {
+    if !present {
+        return Ok(0);
+    }
+
+    let idx = *next;
+    if idx == 0 {
+        return Err(Error::TooManyStrings);
+    }
+
+    *next = next.wrapping_add(1);
+    Ok(idx)
+}
+
+fn write_type1_system(
+    mem: &GuestMemoryMmap,
+    curptr: &mut GuestAddress,
+    handle: &mut u16,
+    system: Option<&SmbiosSystem>,
+) -> Result<()> {
+    *handle += 1;
+
+    let manufacturer = system
+        .and_then(|s| s.manufacturer.as_deref())
+        .unwrap_or(DEFAULT_SYSTEM_MANUFACTURER);
+    let product = system
+        .and_then(|s| s.product_name.as_deref())
+        .unwrap_or(DEFAULT_SYSTEM_PRODUCT_NAME);
+    let version = system.and_then(|s| s.version.as_deref());
+    let serial = system.and_then(|s| s.serial_number.as_deref());
+    let uuid = system.and_then(|s| s.uuid.as_deref());
+    let sku = system.and_then(|s| s.sku_number.as_deref());
+    let family = system.and_then(|s| s.family.as_deref());
+
+    let uuid_number = uuid
+        .map(Uuid::parse_str)
+        .transpose()
+        .map_err(|e| Error::ParseUuid(e, uuid.unwrap().to_string()))?
+        .unwrap_or(Uuid::nil());
+
+    let mut next = 1u8;
+    let manufacturer_idx = alloc_index(&mut next, true)?;
+    let product_idx = alloc_index(&mut next, true)?;
+    let version_idx = alloc_index(&mut next, version.is_some())?;
+    let serial_idx = alloc_index(&mut next, serial.is_some())?;
+    let sku_idx = alloc_index(&mut next, sku.is_some())?;
+    let family_idx = alloc_index(&mut next, family.is_some())?;
+
+    let sys = SmbiosSysInfo {
+        r#type: SYSTEM_INFORMATION,
+        length: size_of::<SmbiosSysInfo>() as u8,
+        handle: *handle,
+        manufacturer: manufacturer_idx,
+        product_name: product_idx,
+        version: version_idx,
+        serial_number: serial_idx,
+        uuid: uuid_number.to_bytes_le(),
+        wake_up_type: SYSTEM_WAKE_UP_TYPE_UNKNOWN,
+        sku: sku_idx,
+        family: family_idx,
+    };
+
+    *curptr = write_and_incr(mem, sys, *curptr)?;
+    *curptr = write_string(mem, manufacturer, *curptr)?;
+    *curptr = write_string(mem, product, *curptr)?;
+    *curptr = write_opt_string(mem, version, *curptr)?;
+    *curptr = write_opt_string(mem, serial, *curptr)?;
+    *curptr = write_opt_string(mem, sku, *curptr)?;
+    *curptr = write_opt_string(mem, family, *curptr)?;
+    *curptr = write_and_incr(mem, 0u8, *curptr)?;
+    Ok(())
+}
+
+fn write_type3_chassis(
+    mem: &GuestMemoryMmap,
+    curptr: &mut GuestAddress,
+    handle: &mut u16,
+    chassis: &SmbiosChassisConfig,
+) -> Result<()> {
+    *handle += 1;
+
+    let asset_tag = chassis.asset_tag.as_deref();
+    let mut next = 1u8;
+    let asset_idx = alloc_index(&mut next, asset_tag.is_some())?;
+
+    let ch = SmbiosChassis {
+        r#type: SYSTEM_ENCLOSURE,
+        length: size_of::<SmbiosChassis>() as u8,
+        handle: *handle,
+        manufacturer: 0,
+        chassis_type: CHASSIS_TYPE_UNKNOWN,
+        version: 0,
+        serial_number: 0,
+        asset_tag: asset_idx,
+        bootup_state: CHASSIS_STATE_UNKNOWN,
+        power_supply_state: CHASSIS_STATE_UNKNOWN,
+        thermal_state: CHASSIS_STATE_UNKNOWN,
+        security_status: CHASSIS_SECURITY_STATUS_NONE,
+        contained_element_count: 0,
+        contained_element_record_length: 0,
+        ..Default::default()
+    };
+
+    *curptr = write_and_incr(mem, ch, *curptr)?;
+    *curptr = write_opt_string(mem, asset_tag, *curptr)?;
+    *curptr = write_string_terminator(mem, *curptr, asset_tag.is_some())?;
+    Ok(())
+}
+
+pub fn setup_smbios(mem: &GuestMemoryMmap, smbios: Option<&SmbiosConfig>) -> Result<u64> {
+    let system = smbios.and_then(|cfg| cfg.system.as_ref());
+    let chassis = smbios.and_then(|cfg| cfg.chassis.as_ref());
+    let oem_strings: &[String] = smbios.map_or(&[], |cfg| &cfg.oem_strings);
     let physptr = GuestAddress(SMBIOS_START)
-        .checked_add(mem::size_of::<Smbios30Entrypoint>() as u64)
+        .checked_add(size_of::<Smbios30Entrypoint>() as u64)
         .ok_or(Error::NotEnoughMemory)?;
     let mut curptr = physptr;
     let mut handle = 0;
@@ -179,7 +380,7 @@ pub fn setup_smbios(
         handle += 1;
         let smbios_biosinfo = SmbiosBiosInfo {
             r#type: BIOS_INFORMATION,
-            length: mem::size_of::<SmbiosBiosInfo>() as u8,
+            length: size_of::<SmbiosBiosInfo>() as u8,
             handle,
             vendor: 1,  // First string written in this section
             version: 2, // Second string written in this section
@@ -193,39 +394,18 @@ pub fn setup_smbios(
         curptr = write_and_incr(mem, 0u8, curptr)?;
     }
 
-    {
-        handle += 1;
+    write_type1_system(mem, &mut curptr, &mut handle, system)?;
 
-        let uuid_number = uuid
-            .map(Uuid::parse_str)
-            .transpose()
-            .map_err(|e| Error::ParseUuid(e, uuid.unwrap().to_string()))?
-            .unwrap_or(Uuid::nil());
-        let smbios_sysinfo = SmbiosSysInfo {
-            r#type: SYSTEM_INFORMATION,
-            length: mem::size_of::<SmbiosSysInfo>() as u8,
-            handle,
-            manufacturer: 1, // First string written in this section
-            product_name: 2, // Second string written in this section
-            serial_number: serial_number.map(|_| 3).unwrap_or_default(), // 3rd string
-            uuid: uuid_number.to_bytes_le(), // set uuid
-            ..Default::default()
-        };
-        curptr = write_and_incr(mem, smbios_sysinfo, curptr)?;
-        curptr = write_string(mem, "Cloud Hypervisor", curptr)?;
-        curptr = write_string(mem, "cloud-hypervisor", curptr)?;
-        if let Some(serial_number) = serial_number {
-            curptr = write_string(mem, serial_number, curptr)?;
-        }
-        curptr = write_and_incr(mem, 0u8, curptr)?;
+    if let Some(chassis) = chassis {
+        write_type3_chassis(mem, &mut curptr, &mut handle, chassis)?;
     }
 
-    if let Some(oem_strings) = oem_strings {
+    if !oem_strings.is_empty() {
         handle += 1;
 
         let smbios_oemstrings = SmbiosOemStrings {
             r#type: OEM_STRINGS,
-            length: mem::size_of::<SmbiosOemStrings>() as u8,
+            length: size_of::<SmbiosOemStrings>() as u8,
             handle,
             count: oem_strings.len() as u8,
         };
@@ -236,14 +416,14 @@ pub fn setup_smbios(
             curptr = write_string(mem, s, curptr)?;
         }
 
-        curptr = write_and_incr(mem, 0u8, curptr)?;
+        curptr = write_string_terminator(mem, curptr, true)?;
     }
 
     {
         handle += 1;
         let smbios_end = SmbiosEndOfTable {
             r#type: END_OF_TABLE,
-            length: mem::size_of::<SmbiosEndOfTable>() as u8,
+            length: size_of::<SmbiosEndOfTable>() as u8,
             handle,
         };
         curptr = write_and_incr(mem, smbios_end, curptr)?;
@@ -254,7 +434,7 @@ pub fn setup_smbios(
     {
         let mut smbios_ep = Smbios30Entrypoint {
             signature: *SM3_MAGIC_IDENT,
-            length: mem::size_of::<Smbios30Entrypoint>() as u8,
+            length: size_of::<Smbios30Entrypoint>() as u8,
             // SMBIOS rev 3.2.0
             majorver: 0x03,
             minorver: 0x02,
@@ -266,43 +446,261 @@ pub fn setup_smbios(
         };
         smbios_ep.checksum = compute_checksum(&smbios_ep);
         mem.write_obj(smbios_ep, GuestAddress(SMBIOS_START))
-            .map_err(|_| Error::WriteSmbiosEp)?;
+            .map_err(Error::WriteSmbiosEp)?;
     }
 
-    Ok(curptr.unchecked_offset_from(physptr) + std::mem::size_of::<Smbios30Entrypoint>() as u64)
+    Ok(curptr.unchecked_offset_from(physptr) + size_of::<Smbios30Entrypoint>() as u64)
 }
 
 #[cfg(test)]
 mod unit_tests {
     use super::*;
 
-    #[test]
-    fn struct_size() {
-        assert_eq!(
-            mem::size_of::<Smbios30Entrypoint>(),
-            0x18usize,
-            concat!("Size of: ", stringify!(Smbios30Entrypoint))
-        );
-        assert_eq!(
-            mem::size_of::<SmbiosBiosInfo>(),
-            0x14usize,
-            concat!("Size of: ", stringify!(SmbiosBiosInfo))
-        );
-        assert_eq!(
-            mem::size_of::<SmbiosSysInfo>(),
-            0x1busize,
-            concat!("Size of: ", stringify!(SmbiosSysInfo))
-        );
+    /// Collects all strings after a SMBIOS structure, stopping at the double-NUL terminator and returns next addr.
+    fn read_string_set(mem: &GuestMemoryMmap, addr: GuestAddress) -> (Vec<String>, GuestAddress) {
+        let mut cur = addr;
+        let read_byte = |addr: GuestAddress| -> u8 { mem.read_obj(addr).unwrap() };
+
+        // SMBIOS string-set: NUL-terminated strings, terminated by an extra NUL.
+        // Empty string-set is exactly "\0\0".
+        if read_byte(cur) == 0 {
+            let next = cur.checked_add(1).unwrap();
+            assert_eq!(read_byte(next), 0);
+            return (Vec::new(), next.checked_add(1).unwrap());
+        }
+
+        let mut strings = Vec::new();
+        loop {
+            let mut bytes = Vec::new();
+            loop {
+                let b = read_byte(cur);
+                cur = cur.checked_add(1).unwrap();
+                if b == 0 {
+                    break;
+                }
+                bytes.push(b);
+            }
+            strings.push(String::from_utf8(bytes).unwrap());
+
+            // If the next byte is NUL, that's the extra terminator.
+            if read_byte(cur) == 0 {
+                cur = cur.checked_add(1).unwrap();
+                break;
+            }
+        }
+
+        (strings, cur)
     }
 
     #[test]
     fn entrypoint_checksum() {
         let mem = GuestMemoryMmap::from_ranges(&[(GuestAddress(SMBIOS_START), 4096)]).unwrap();
 
-        setup_smbios(&mem, None, None, None).unwrap();
+        setup_smbios(&mem, None).unwrap();
 
         let smbios_ep: Smbios30Entrypoint = mem.read_obj(GuestAddress(SMBIOS_START)).unwrap();
 
         assert_eq!(compute_checksum(&smbios_ep), 0);
     }
+
+    #[test]
+    fn entrypoint_struct_size() {
+        assert_eq!(
+            size_of::<Smbios30Entrypoint>(),
+            0x18usize,
+            concat!("Size of: ", stringify!(Smbios30Entrypoint))
+        );
+        assert_eq!(
+            size_of::<SmbiosBiosInfo>(),
+            0x14usize,
+            concat!("Size of: ", stringify!(SmbiosBiosInfo))
+        );
+        assert_eq!(
+            size_of::<SmbiosSysInfo>(),
+            0x1busize,
+            concat!("Size of: ", stringify!(SmbiosSysInfo))
+        );
+    }
+
+    #[test]
+    fn smbios_chassis_empty_string_set_has_double_null() {
+        let mem = GuestMemoryMmap::from_ranges(&[(GuestAddress(SMBIOS_START), 4096)]).unwrap();
+        let smbios = SmbiosConfig {
+            chassis: Some(SmbiosChassisConfig::default()),
+            ..Default::default()
+        };
+
+        setup_smbios(&mem, Some(&smbios)).unwrap();
+
+        let smbios_ep: Smbios30Entrypoint = mem.read_obj(GuestAddress(SMBIOS_START)).unwrap();
+        let mut cur = GuestAddress(smbios_ep.physptr);
+
+        let bios: SmbiosBiosInfo = mem.read_obj(cur).unwrap();
+        cur = cur.checked_add(bios.length as u64).unwrap();
+        let (_, next) = read_string_set(&mem, cur);
+        cur = next;
+
+        let sys: SmbiosSysInfo = mem.read_obj(cur).unwrap();
+        cur = cur.checked_add(sys.length as u64).unwrap();
+        let (_, next) = read_string_set(&mem, cur);
+        cur = next;
+
+        let chassis: SmbiosChassis = mem.read_obj(cur).unwrap();
+        cur = cur.checked_add(chassis.length as u64).unwrap();
+        // SMBIOS DSP0134 §6.1.3: empty string-set ends with double NUL.
+        let b0: u8 = mem.read_obj(cur).unwrap();
+        let b1: u8 = mem.read_obj(cur.checked_add(1).unwrap()).unwrap();
+        assert_eq!(b0, 0);
+        assert_eq!(b1, 0);
+        cur = cur.checked_add(2).unwrap();
+
+        let end: SmbiosEndOfTable = mem.read_obj(cur).unwrap();
+        assert_eq!(end.r#type, END_OF_TABLE);
+    }
+
+    #[test]
+    fn smbios_chassis_oem_strings_layout() {
+        let mem = GuestMemoryMmap::from_ranges(&[(GuestAddress(SMBIOS_START), 4096)]).unwrap();
+
+        let smbios = SmbiosConfig {
+            chassis: Some(SmbiosChassisConfig {
+                asset_tag: Some("rack1".to_string()),
+            }),
+            oem_strings: ["o1".to_string(), "o2".to_string()].into(),
+            ..Default::default()
+        };
+
+        setup_smbios(&mem, Some(&smbios)).unwrap();
+
+        let smbios_ep: Smbios30Entrypoint = mem.read_obj(GuestAddress(SMBIOS_START)).unwrap();
+        let mut cur = GuestAddress(smbios_ep.physptr);
+
+        let bios: SmbiosBiosInfo = mem.read_obj(cur).unwrap();
+        cur = cur.checked_add(bios.length as u64).unwrap();
+        let (_, next) = read_string_set(&mem, cur);
+        cur = next;
+
+        let sys: SmbiosSysInfo = mem.read_obj(cur).unwrap();
+        cur = cur.checked_add(sys.length as u64).unwrap();
+        let (_, next) = read_string_set(&mem, cur);
+        cur = next;
+
+        let chassis: SmbiosChassis = mem.read_obj(cur).unwrap();
+        assert_eq!(chassis.r#type, SYSTEM_ENCLOSURE);
+        assert_eq!(chassis.asset_tag, 1);
+        cur = cur.checked_add(chassis.length as u64).unwrap();
+        let (chassis_strings, next) = read_string_set(&mem, cur);
+        assert_eq!(chassis_strings, vec!["rack1"]);
+        cur = next;
+
+        let oem: SmbiosOemStrings = mem.read_obj(cur).unwrap();
+        assert_eq!(oem.r#type, OEM_STRINGS);
+        assert_eq!(oem.count, 2);
+        cur = cur.checked_add(oem.length as u64).unwrap();
+        let (oem_strings, next) = read_string_set(&mem, cur);
+        assert_eq!(oem_strings, vec!["o1", "o2"]);
+        cur = next;
+
+        let end: SmbiosEndOfTable = mem.read_obj(cur).unwrap();
+        assert_eq!(end.r#type, END_OF_TABLE);
+    }
+
+    #[test]
+    fn smbios_strings_terminators_default() {
+        let mem = GuestMemoryMmap::from_ranges(&[(GuestAddress(SMBIOS_START), 4096)]).unwrap();
+
+        setup_smbios(&mem, None).unwrap();
+
+        let smbios_ep: Smbios30Entrypoint = mem.read_obj(GuestAddress(SMBIOS_START)).unwrap();
+        let mut cur = GuestAddress(smbios_ep.physptr);
+
+        let bios: SmbiosBiosInfo = mem.read_obj(cur).unwrap();
+        assert_eq!(bios.r#type, BIOS_INFORMATION);
+        cur = cur.checked_add(bios.length as u64).unwrap();
+        let (bios_strings, next) = read_string_set(&mem, cur);
+        assert_eq!(bios_strings, vec!["cloud-hypervisor", "0"]);
+        cur = next;
+
+        let sys: SmbiosSysInfo = mem.read_obj(cur).unwrap();
+        assert_eq!(sys.r#type, SYSTEM_INFORMATION);
+        assert_eq!(sys.manufacturer, 1);
+        assert_eq!(sys.product_name, 2);
+        assert_eq!(sys.version, 0);
+        assert_eq!(sys.serial_number, 0);
+        assert_eq!(sys.sku, 0);
+        assert_eq!(sys.family, 0);
+        cur = cur.checked_add(sys.length as u64).unwrap();
+        let (sys_strings, next) = read_string_set(&mem, cur);
+        assert_eq!(
+            sys_strings,
+            vec![DEFAULT_SYSTEM_MANUFACTURER, DEFAULT_SYSTEM_PRODUCT_NAME]
+        );
+        cur = next;
+
+        let end: SmbiosEndOfTable = mem.read_obj(cur).unwrap();
+        assert_eq!(end.r#type, END_OF_TABLE);
+    }
+
+    #[test]
+    fn smbios_strings_too_many() {
+        let mut next = 1u8;
+        for _ in 0..255 {
+            alloc_index(&mut next, true).unwrap();
+        }
+        let err = alloc_index(&mut next, true).unwrap_err();
+        assert!(matches!(err, Error::TooManyStrings));
+    }
+
+    #[test]
+    fn smbios_uuid_invalid_rejected() {
+        let mem = GuestMemoryMmap::from_ranges(&[(GuestAddress(SMBIOS_START), 4096)]).unwrap();
+        let smbios = SmbiosConfig {
+            system: Some(SmbiosSystem {
+                uuid: Some("not-a-uuid".to_string()),
+                ..Default::default()
+            }),
+            ..Default::default()
+        };
+
+        let err = setup_smbios(&mem, Some(&smbios)).unwrap_err();
+        assert!(matches!(err, Error::ParseUuid(_, _)));
+    }
+
+    #[test]
+    fn smbios_uuid_written_le() {
+        let mem = GuestMemoryMmap::from_ranges(&[(GuestAddress(SMBIOS_START), 4096)]).unwrap();
+        let uuid_str = "00112233-4455-6677-8899-aabbccddeeff";
+        let smbios = SmbiosConfig {
+            system: Some(SmbiosSystem {
+                uuid: Some(uuid_str.to_string()),
+                ..Default::default()
+            }),
+            ..Default::default()
+        };
+
+        setup_smbios(&mem, Some(&smbios)).unwrap();
+
+        let smbios_ep: Smbios30Entrypoint = mem.read_obj(GuestAddress(SMBIOS_START)).unwrap();
+        let mut cur = GuestAddress(smbios_ep.physptr);
+
+        let bios: SmbiosBiosInfo = mem.read_obj(cur).unwrap();
+        cur = cur.checked_add(bios.length as u64).unwrap();
+        let (_, next) = read_string_set(&mem, cur);
+        cur = next;
+
+        let sys: SmbiosSysInfo = mem.read_obj(cur).unwrap();
+        assert_eq!(sys.uuid, Uuid::parse_str(uuid_str).unwrap().to_bytes_le());
+    }
+
+    #[test]
+    fn smbios_write_fails_with_too_small_memory() {
+        let mem = GuestMemoryMmap::from_ranges(&[(
+            GuestAddress(SMBIOS_START),
+            size_of::<Smbios30Entrypoint>(),
+        )])
+        .unwrap();
+
+        let err = setup_smbios(&mem, None).unwrap_err();
+        assert!(matches!(err, Error::WriteData(_)));
+    }
 }

arch/src/x86_64/tdx/mod.rs

@@ -2,7 +2,8 @@
 //
 // SPDX-License-Identifier: Apache-2.0
 use std::fs::File;
-use std::io::{Read, Seek, SeekFrom};
+use std::io::{self, Read, Seek, SeekFrom};
+use std::slice;
 use std::str::FromStr;
 
 use log::{debug, info};
@@ -15,11 +16,11 @@ use crate::GuestMemoryMmap;
 #[derive(Error, Debug)]
 pub enum TdvfError {
     #[error("Failed read TDVF descriptor")]
-    ReadDescriptor(#[source] std::io::Error),
+    ReadDescriptor(#[source] io::Error),
     #[error("Failed read TDVF descriptor offset")]
-    ReadDescriptorOffset(#[source] std::io::Error),
+    ReadDescriptorOffset(#[source] io::Error),
     #[error("Failed read GUID table")]
-    ReadGuidTable(#[source] std::io::Error),
+    ReadGuidTable(#[source] io::Error),
     #[error("Invalid descriptor signature")]
     InvalidDescriptorSignature,
     #[error("Invalid descriptor size")]
@@ -162,10 +163,7 @@ pub fn parse_tdvf_sections(file: &mut File) -> Result<(Vec<TdvfSection>, bool),
     let mut descriptor: TdvfDescriptor = Default::default();
     // SAFETY: we read exactly the size of the descriptor header
     file.read_exact(unsafe {
-        std::slice::from_raw_parts_mut(
-            (&raw mut descriptor).cast(),
-            std::mem::size_of::<TdvfDescriptor>(),
-        )
+        slice::from_raw_parts_mut((&raw mut descriptor).cast(), size_of::<TdvfDescriptor>())
     })
     .map_err(TdvfError::ReadDescriptor)?;
 
@@ -174,8 +172,7 @@ pub fn parse_tdvf_sections(file: &mut File) -> Result<(Vec<TdvfSection>, bool),
     }
 
     if descriptor.length as usize
-        != std::mem::size_of::<TdvfDescriptor>()
-            + std::mem::size_of::<TdvfSection>() * descriptor.num_sections as usize
+        != size_of::<TdvfDescriptor>() + size_of::<TdvfSection>() * descriptor.num_sections as usize
     {
         return Err(TdvfError::InvalidDescriptorSize);
     }
@@ -189,9 +186,9 @@ pub fn parse_tdvf_sections(file: &mut File) -> Result<(Vec<TdvfSection>, bool),
 
     // SAFETY: we read exactly the advertised sections
     file.read_exact(unsafe {
-        std::slice::from_raw_parts_mut(
+        slice::from_raw_parts_mut(
             sections.as_mut_ptr().cast(),
-            descriptor.num_sections as usize * std::mem::size_of::<TdvfSection>(),
+            descriptor.num_sections as usize * size_of::<TdvfSection>(),
         )
     })
     .map_err(TdvfError::ReadDescriptor)?;
@@ -305,7 +302,7 @@ fn align_hob(v: u64) -> u64 {
 
 impl TdHob {
     fn update_offset<T>(&mut self) {
-        self.current_offset = align_hob(self.current_offset + std::mem::size_of::<T>() as u64);
+        self.current_offset = align_hob(self.current_offset + size_of::<T>() as u64);
     }
 
     pub fn start(offset: u64) -> TdHob {
@@ -322,7 +319,7 @@ impl TdHob {
         // Write end
         let end = HobHeader {
             r#type: HobType::EndOfHobList,
-            length: std::mem::size_of::<HobHeader>() as u16,
+            length: size_of::<HobHeader>() as u16,
             reserved: 0,
         };
         info!("Writing HOB end {:x} {:x?}", self.current_offset, end);
@@ -335,7 +332,7 @@ impl TdHob {
         let handoff = HobHandoffInfoTable {
             header: HobHeader {
                 r#type: HobType::Handoff,
-                length: std::mem::size_of::<HobHandoffInfoTable>() as u16,
+                length: size_of::<HobHandoffInfoTable>() as u16,
                 reserved: 0,
             },
             version: 0x9,
@@ -362,7 +359,7 @@ impl TdHob {
         let resource_descriptor = HobResourceDescriptor {
             header: HobHeader {
                 r#type: HobType::ResourceDescriptor,
-                length: std::mem::size_of::<HobResourceDescriptor>() as u16,
+                length: size_of::<HobResourceDescriptor>() as u16,
                 reserved: 0,
             },
             owner: EfiGuid::default(),
@@ -439,8 +436,7 @@ impl TdHob {
         // We already know the HobGuidType size is 8 bytes multiple, but we
         // need the total size to be 8 bytes multiple. That is why the ACPI
         // table size must be 8 bytes multiple as well.
-        let length = std::mem::size_of::<HobGuidType>() as u16
-            + align_hob(table_content.len() as u64) as u16;
+        let length = size_of::<HobGuidType>() as u16 + align_hob(table_content.len() as u64) as u16;
         let hob_guid_type = HobGuidType {
             header: HobHeader {
                 r#type: HobType::GuidExtension,
@@ -462,7 +458,7 @@ impl TdHob {
         );
         mem.write_obj(hob_guid_type, GuestAddress(self.current_offset))
             .map_err(TdvfError::GuestMemoryWriteHob)?;
-        let current_offset = self.current_offset + std::mem::size_of::<HobGuidType>() as u64;
+        let current_offset = self.current_offset + size_of::<HobGuidType>() as u64;
 
         // In case the table is quite large, let's make sure we can handle
         // retrying until everything has been correctly copied.
@@ -493,7 +489,7 @@ impl TdHob {
             guid_type: HobGuidType {
                 header: HobHeader {
                     r#type: HobType::GuidExtension,
-                    length: std::mem::size_of::<TdPayload>() as u16,
+                    length: size_of::<TdPayload>() as u16,
                     reserved: 0,
                 },
                 // HOB_PAYLOAD_INFO_GUID
@@ -526,7 +522,7 @@ mod unit_tests {
     #[test]
     #[ignore]
     fn test_parse_tdvf_sections() {
-        let mut f = std::fs::File::open("tdvf.fd").unwrap();
+        let mut f = File::open("tdvf.fd").unwrap();
         let (sections, _) = parse_tdvf_sections(&mut f).unwrap();
         for section in sections {
             eprintln!("{section:x?}");

block/Cargo.toml

@@ -8,18 +8,19 @@ version = "0.1.0"
 [features]
 default = []
 io_uring = ["dep:io-uring"]
+test-utils = []
 
 [dependencies]
 bitflags = { workspace = true }
 byteorder = { workspace = true }
-crc-any = "2.5.0"
+crc-any = "3.0.0"
 flate2 = "1.1"
 io-uring = { version = "0.7.12", optional = true }
 libc = { workspace = true }
 log = { workspace = true }
 remain = "0.2.15"
 serde = { workspace = true, features = ["derive"] }
-smallvec = "1.15.1"
+smallvec = { workspace = true }
 thiserror = { workspace = true }
 uuid = { workspace = true, features = ["v4"] }
 virtio-bindings = { workspace = true }
@@ -31,6 +32,7 @@ vm-memory = { workspace = true, features = [
 ] }
 vm-virtio = { path = "../vm-virtio" }
 vmm-sys-util = { workspace = true }
+zerocopy = { workspace = true, features = ["derive"] }
 zstd = "0.13"
 
 [dev-dependencies]

block/src/aligned_buffer.rs

@@ -0,0 +1,269 @@
+// Copyright 2026 The Cloud Hypervisor Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+use std::alloc::{Layout, alloc_zeroed, dealloc};
+use std::os::unix::fs::FileExt;
+use std::{io, slice};
+
+/// RAII aligned heap buffer for O_DIRECT I/O.
+///
+/// Handles the alignment math for offset and length, allocating a buffer
+/// that satisfies O_DIRECT constraints. The caller's logical data lives
+/// at `as_slice()`/`as_mut_slice()` (accounting for head padding when the
+/// requested offset is not alignment-aligned). The full aligned region is
+/// used internally for pread/pwrite via `FileExt`.
+pub(crate) struct AlignedBuffer {
+    ptr: *mut u8,
+    layout: Layout,
+    head_pad: usize,
+    user_len: usize,
+    aligned_len: usize,
+    aligned_offset: u64,
+}
+
+impl AlignedBuffer {
+    /// Create a new aligned buffer for I/O at `offset` of `len` bytes with
+    /// the given `alignment` requirement.
+    ///
+    /// When offset and length are already aligned, `head_pad == 0` and the
+    /// full buffer equals the user's logical portion (no overhead).
+    pub fn new(offset: u64, len: usize, alignment: usize) -> io::Result<Self> {
+        if alignment == 0 || !alignment.is_power_of_two() {
+            return Err(io::Error::new(
+                io::ErrorKind::InvalidInput,
+                "alignment must be a non-zero power of two",
+            ));
+        }
+
+        let mask = alignment as u64 - 1;
+        let aligned_offset = offset & !mask;
+        let head_pad = (offset - aligned_offset) as usize;
+        let min_len = head_pad
+            .checked_add(len)
+            .ok_or_else(|| io::Error::other("aligned buffer length overflow"))?;
+        let aligned_len = if min_len == 0 {
+            0
+        } else {
+            let remainder = min_len % alignment;
+            if remainder == 0 {
+                min_len
+            } else {
+                min_len
+                    .checked_add(alignment - remainder)
+                    .ok_or_else(|| io::Error::other("aligned buffer length overflow"))?
+            }
+        };
+
+        // alloc_zeroed is UB on a zero-sized layout, so round the allocation
+        // up to one alignment unit for the zero-length case. The padding is
+        // never exposed: as_slice/full_slice report aligned_len/user_len (0).
+        let layout = Layout::from_size_align(aligned_len.max(alignment), alignment)
+            .map_err(|e| io::Error::other(format!("invalid aligned layout: {e}")))?;
+        // SAFETY: layout has non-zero size.
+        let ptr = unsafe { alloc_zeroed(layout) };
+        if ptr.is_null() {
+            return Err(io::Error::new(
+                io::ErrorKind::OutOfMemory,
+                "aligned allocation failed",
+            ));
+        }
+
+        Ok(AlignedBuffer {
+            ptr,
+            layout,
+            head_pad,
+            user_len: len,
+            aligned_len,
+            aligned_offset,
+        })
+    }
+
+    /// The caller's logical portion of the buffer (read-only).
+    pub fn as_slice(&self) -> &[u8] {
+        // SAFETY: ptr is valid for layout.size() bytes; head_pad + user_len <= layout.size().
+        unsafe { slice::from_raw_parts(self.ptr.add(self.head_pad), self.user_len) }
+    }
+
+    /// The caller's logical portion of the buffer (mutable).
+    pub fn as_mut_slice(&mut self) -> &mut [u8] {
+        // SAFETY: ptr is valid for layout.size() bytes; head_pad + user_len <= layout.size().
+        unsafe { slice::from_raw_parts_mut(self.ptr.add(self.head_pad), self.user_len) }
+    }
+
+    fn full_slice(&self) -> &[u8] {
+        // SAFETY: ptr is valid for layout.size() bytes; aligned_len <= layout.size().
+        unsafe { slice::from_raw_parts(self.ptr, self.aligned_len) }
+    }
+
+    fn full_mut_slice(&mut self) -> &mut [u8] {
+        // SAFETY: ptr is valid for layout.size() bytes; aligned_len <= layout.size().
+        unsafe { slice::from_raw_parts_mut(self.ptr, self.aligned_len) }
+    }
+
+    /// Read into the buffer from `f`, tolerating a short read at EOF.
+    ///
+    /// Returns the number of caller-logical bytes now valid in `as_slice()`,
+    /// accounting for head padding and any short read.
+    pub fn read_from(&mut self, f: &impl FileExt) -> io::Result<usize> {
+        let mut total = 0usize;
+        while total < self.aligned_len {
+            let offset = self
+                .aligned_offset
+                .checked_add(total as u64)
+                .ok_or_else(|| io::Error::other("aligned buffer offset overflow"))?;
+            match f.read_at(&mut self.full_mut_slice()[total..], offset) {
+                Ok(0) => break,
+                Ok(n) => total += n,
+                Err(e) if e.kind() == io::ErrorKind::Interrupted => {}
+                Err(e) => return Err(e),
+            }
+        }
+        Ok(total.saturating_sub(self.head_pad).min(self.user_len))
+    }
+
+    /// Write the full aligned region from this buffer to `f`.
+    pub fn write_to(&self, f: &impl FileExt) -> io::Result<()> {
+        f.write_all_at(self.full_slice(), self.aligned_offset)
+    }
+}
+
+impl Drop for AlignedBuffer {
+    fn drop(&mut self) {
+        // SAFETY: ptr was allocated by alloc_zeroed with self.layout.
+        unsafe { dealloc(self.ptr, self.layout) };
+    }
+}
+
+// SAFETY: The buffer is a plain heap allocation with no interior references.
+unsafe impl Send for AlignedBuffer {}
+
+#[cfg(test)]
+mod tests {
+    use std::io::Write;
+    use std::os::unix::fs::FileExt;
+
+    use vmm_sys_util::tempfile::TempFile;
+
+    use super::*;
+
+    fn create_pattern_file(size: usize) -> TempFile {
+        let tf = TempFile::new().unwrap();
+        let pattern: Vec<u8> = (0..size).map(|i| (i % 251) as u8).collect();
+        tf.as_file().write_all(&pattern).unwrap();
+        tf.as_file().sync_all().unwrap();
+        tf
+    }
+
+    #[test]
+    fn test_read_aligned() {
+        let size = 4096usize;
+        let tf = create_pattern_file(size);
+        let alignment = 512;
+
+        let mut abuf = AlignedBuffer::new(0, size, alignment).unwrap();
+        abuf.read_from(tf.as_file()).unwrap();
+
+        let expected: Vec<u8> = (0..size).map(|i| (i % 251) as u8).collect();
+        assert_eq!(abuf.as_slice(), &expected[..]);
+    }
+
+    #[test]
+    fn test_zero_len_is_noop() {
+        let tf = create_pattern_file(512);
+        let mut abuf = AlignedBuffer::new(100, 0, 512).unwrap();
+
+        abuf.read_from(tf.as_file()).unwrap();
+        abuf.write_to(tf.as_file()).unwrap();
+
+        assert!(abuf.as_slice().is_empty());
+        assert!(abuf.as_mut_slice().is_empty());
+    }
+
+    #[test]
+    fn test_read_unaligned_offset() {
+        let file_size = 8192usize;
+        let tf = create_pattern_file(file_size);
+        let alignment = 512;
+
+        let offset = 100u64;
+        let len = 200usize;
+        let mut abuf = AlignedBuffer::new(offset, len, alignment).unwrap();
+        abuf.read_from(tf.as_file()).unwrap();
+
+        let expected: Vec<u8> = (offset as usize..offset as usize + len)
+            .map(|i| (i % 251) as u8)
+            .collect();
+        assert_eq!(abuf.as_slice(), &expected[..]);
+    }
+
+    #[test]
+    fn test_write_aligned() {
+        let size = 4096usize;
+        let tf = create_pattern_file(size);
+        let alignment = 512;
+
+        let data: Vec<u8> = (0..size).map(|i| ((i + 1) % 251) as u8).collect();
+        let mut abuf = AlignedBuffer::new(0, size, alignment).unwrap();
+        abuf.as_mut_slice().copy_from_slice(&data);
+        abuf.write_to(tf.as_file()).unwrap();
+
+        let mut readback = vec![0u8; size];
+        tf.as_file().read_exact_at(&mut readback, 0).unwrap();
+        assert_eq!(readback, data);
+    }
+
+    #[test]
+    fn test_write_unaligned_offset_rmw() {
+        let file_size = 8192usize;
+        let tf = create_pattern_file(file_size);
+        let alignment = 512;
+
+        let offset = 100u64;
+        let len = 200usize;
+        let data: Vec<u8> = (0..len).map(|i| ((i + 1) % 239) as u8).collect();
+
+        let mut abuf = AlignedBuffer::new(offset, len, alignment).unwrap();
+        abuf.read_from(tf.as_file()).unwrap();
+        abuf.as_mut_slice().copy_from_slice(&data);
+        abuf.write_to(tf.as_file()).unwrap();
+
+        let mut whole = vec![0u8; file_size];
+        tf.as_file().read_exact_at(&mut whole, 0).unwrap();
+
+        let before: Vec<u8> = (0..offset as usize).map(|i| (i % 251) as u8).collect();
+        assert_eq!(&whole[..offset as usize], &before[..]);
+        assert_eq!(&whole[offset as usize..offset as usize + len], &data[..]);
+        let after_start = offset as usize + len;
+        let after: Vec<u8> = (after_start..file_size).map(|i| (i % 251) as u8).collect();
+        assert_eq!(&whole[after_start..], &after[..]);
+    }
+
+    #[test]
+    fn test_4096_alignment() {
+        let file_size = 16384usize;
+        let tf = create_pattern_file(file_size);
+        let alignment = 4096;
+
+        let offset = 4096u64;
+        let len = 4096usize;
+        let data: Vec<u8> = (0..len).map(|i| ((i + 1) % 239) as u8).collect();
+
+        let mut abuf = AlignedBuffer::new(offset, len, alignment).unwrap();
+        abuf.read_from(tf.as_file()).unwrap();
+        abuf.as_mut_slice().copy_from_slice(&data);
+        abuf.write_to(tf.as_file()).unwrap();
+
+        let mut abuf = AlignedBuffer::new(offset, len, alignment).unwrap();
+        abuf.read_from(tf.as_file()).unwrap();
+        assert_eq!(abuf.as_slice(), &data[..]);
+
+        let mut whole = vec![0u8; file_size];
+        tf.as_file().read_exact_at(&mut whole, 0).unwrap();
+        let before: Vec<u8> = (0..offset as usize).map(|i| (i % 251) as u8).collect();
+        assert_eq!(&whole[..offset as usize], &before[..]);
+        let after_start = offset as usize + len;
+        let after: Vec<u8> = (after_start..file_size).map(|i| (i % 251) as u8).collect();
+        assert_eq!(&whole[after_start..], &after[..]);
+    }
+}

block/src/aligned_file.rs

@@ -0,0 +1,402 @@
+// Copyright 2026 The Cloud Hypervisor Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+use std::fs::{File, Metadata};
+use std::io;
+use std::os::fd::{AsFd, BorrowedFd};
+use std::os::unix::fs::FileExt;
+use std::os::unix::io::{AsRawFd, RawFd};
+
+use vmm_sys_util::file_traits::FileSync;
+use vmm_sys_util::seek_hole::SeekHole;
+use vmm_sys_util::write_zeroes::{PunchHole, WriteZeroesAt};
+
+use crate::aligned_buffer::AlignedBuffer;
+use crate::{SECTOR_SIZE, probe_direct_alignment};
+
+/// True when `buf_ptr`/`len`/`offset` already satisfy `alignment`
+/// (`alignment == 0` means no O_DIRECT, so everything is "aligned").
+fn is_aligned(alignment: usize, buf_ptr: usize, len: usize, offset: u64) -> bool {
+    alignment == 0
+        || (buf_ptr.is_multiple_of(alignment)
+            && len.is_multiple_of(alignment)
+            && offset.is_multiple_of(alignment as u64))
+}
+
+/// A `File` that transparently satisfies O_DIRECT alignment requirements.
+///
+/// `alignment == 0` means no O_DIRECT (all I/O passes straight through).
+/// For unaligned requests under O_DIRECT, I/O is bounced through an
+/// `AlignedBuffer` (read-modify-write for writes).
+#[derive(Debug)]
+pub struct AlignedFile {
+    file: File,
+    alignment: usize,
+}
+
+impl AlignedFile {
+    /// Wrap `file`, querying the O_DIRECT block alignment when `direct_io`.
+    pub fn new(file: File, direct_io: bool) -> Self {
+        let alignment = if direct_io {
+            probe_direct_alignment(file.as_raw_fd()).unwrap_or(SECTOR_SIZE) as usize
+        } else {
+            0
+        };
+        AlignedFile { file, alignment }
+    }
+
+    pub fn alignment(&self) -> usize {
+        self.alignment
+    }
+
+    pub fn file(&self) -> &File {
+        &self.file
+    }
+
+    pub fn file_mut(&mut self) -> &mut File {
+        &mut self.file
+    }
+
+    pub fn try_clone(&self) -> io::Result<Self> {
+        Ok(AlignedFile {
+            file: self.file.try_clone()?,
+            alignment: self.alignment,
+        })
+    }
+
+    pub fn set_len(&self, size: u64) -> io::Result<()> {
+        self.file.set_len(size)
+    }
+
+    pub fn metadata(&self) -> io::Result<Metadata> {
+        self.file.metadata()
+    }
+
+    pub fn sync_all(&self) -> io::Result<()> {
+        self.file.sync_all()
+    }
+
+    pub fn sync_data(&self) -> io::Result<()> {
+        self.file.sync_data()
+    }
+
+    pub fn is_direct(&self) -> bool {
+        self.alignment != 0
+    }
+
+    pub fn is_writable(&self) -> bool {
+        // SAFETY: fcntl with F_GETFL is safe and doesn't modify the file descriptor
+        let flags = unsafe { libc::fcntl(self.file.as_raw_fd(), libc::F_GETFL) };
+        if flags < 0 {
+            return false;
+        }
+        let access_mode = flags & libc::O_ACCMODE;
+        access_mode == libc::O_WRONLY || access_mode == libc::O_RDWR
+    }
+
+    /// Wrap `file` with an explicit alignment, bypassing the probe. Used by
+    /// tests to force the bounce/RMW path without a real O_DIRECT fd.
+    #[cfg(test)]
+    pub fn with_alignment(file: File, alignment: usize) -> Self {
+        AlignedFile { file, alignment }
+    }
+
+    /// Read `len` bytes at `offset` through an aligned bounce buffer.
+    pub(crate) fn read_unaligned(
+        &self,
+        offset: u64,
+        len: usize,
+        scatter: impl FnOnce(&[u8]) -> io::Result<()>,
+    ) -> io::Result<usize> {
+        let mut abuf = AlignedBuffer::new(offset, len, self.alignment)?;
+        let n = abuf.read_from(&self.file)?;
+        scatter(&abuf.as_slice()[..n])?;
+        Ok(n)
+    }
+
+    /// Write `len` bytes at `offset` through an aligned bounce buffer.
+    pub(crate) fn write_unaligned(
+        &self,
+        offset: u64,
+        len: usize,
+        gather: impl FnOnce(&mut [u8]) -> io::Result<()>,
+    ) -> io::Result<usize> {
+        let mut abuf = AlignedBuffer::new(offset, len, self.alignment)?;
+        abuf.read_from(&self.file)?; // RMW: preserve head/tail padding
+        gather(abuf.as_mut_slice())?;
+        abuf.write_to(&self.file)?;
+        Ok(len)
+    }
+}
+
+impl FileExt for AlignedFile {
+    fn read_at(&self, buf: &mut [u8], offset: u64) -> io::Result<usize> {
+        if buf.is_empty() {
+            return Ok(0);
+        }
+        if is_aligned(self.alignment, buf.as_ptr() as usize, buf.len(), offset) {
+            return self.file.read_at(buf, offset);
+        }
+        self.read_unaligned(offset, buf.len(), |data| {
+            buf[..data.len()].copy_from_slice(data);
+            Ok(())
+        })
+    }
+
+    fn write_at(&self, buf: &[u8], offset: u64) -> io::Result<usize> {
+        if buf.is_empty() {
+            return Ok(0);
+        }
+        if is_aligned(self.alignment, buf.as_ptr() as usize, buf.len(), offset) {
+            return self.file.write_at(buf, offset);
+        }
+        self.write_unaligned(offset, buf.len(), |dst| {
+            dst.copy_from_slice(buf);
+            Ok(())
+        })
+    }
+}
+
+impl WriteZeroesAt for AlignedFile {
+    fn write_zeroes_at(&mut self, offset: u64, length: usize) -> io::Result<usize> {
+        self.file.write_zeroes_at(offset, length)
+    }
+}
+
+impl PunchHole for AlignedFile {
+    fn punch_hole(&mut self, offset: u64, length: u64) -> io::Result<()> {
+        self.file.punch_hole(offset, length)
+    }
+}
+
+impl FileSync for AlignedFile {
+    fn fsync(&mut self) -> io::Result<()> {
+        self.file.fsync()
+    }
+}
+
+impl SeekHole for AlignedFile {
+    fn seek_hole(&mut self, offset: u64) -> io::Result<Option<u64>> {
+        self.file.seek_hole(offset)
+    }
+
+    fn seek_data(&mut self, offset: u64) -> io::Result<Option<u64>> {
+        self.file.seek_data(offset)
+    }
+}
+
+impl Clone for AlignedFile {
+    fn clone(&self) -> Self {
+        self.try_clone().expect("AlignedFile cloning failed")
+    }
+}
+
+impl AsRawFd for AlignedFile {
+    fn as_raw_fd(&self) -> RawFd {
+        self.file.as_raw_fd()
+    }
+}
+
+impl AsFd for AlignedFile {
+    fn as_fd(&self) -> BorrowedFd<'_> {
+        self.file.as_fd()
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::io::Write;
+    use std::os::unix::fs::FileExt;
+
+    use vmm_sys_util::tempfile::TempFile;
+
+    use super::*;
+
+    fn pattern_file(size: usize) -> TempFile {
+        let tf = TempFile::new().unwrap();
+        let p: Vec<u8> = (0..size).map(|i| (i % 251) as u8).collect();
+        tf.as_file().write_all(&p).unwrap();
+        tf.as_file().sync_all().unwrap();
+        tf
+    }
+
+    fn forced(file: File, alignment: usize) -> AlignedFile {
+        AlignedFile { file, alignment }
+    }
+
+    #[test]
+    fn new_probes_alignment_and_accessors() {
+        let tf = pattern_file(8192);
+        // Not O_DIRECT, so new() falls back to SECTOR_SIZE (512).
+        let mut af = AlignedFile::new(tf.as_file().try_clone().unwrap(), true);
+        assert_eq!(af.alignment(), 512);
+        let _ = af.file();
+        let _ = af.file_mut();
+        let _ = af.try_clone().unwrap();
+
+        let plain = AlignedFile::new(tf.as_file().try_clone().unwrap(), false);
+        assert_eq!(plain.alignment(), 0);
+    }
+
+    #[test]
+    fn read_unaligned_offset_matches_contents() {
+        let tf = pattern_file(8192);
+        let af = forced(tf.as_file().try_clone().unwrap(), 512);
+        let mut buf = vec![0u8; 200];
+        assert_eq!(af.read_at(&mut buf, 100).unwrap(), 200);
+        let want: Vec<u8> = (100..300).map(|i| (i % 251) as u8).collect();
+        assert_eq!(buf, want);
+    }
+
+    #[test]
+    fn read_unaligned_short_at_eof() {
+        let tf = pattern_file(100);
+        let af = forced(tf.as_file().try_clone().unwrap(), 512);
+        let mut buf = vec![0u8; 200];
+        assert_eq!(af.read_at(&mut buf, 10).unwrap(), 90);
+    }
+
+    #[test]
+    fn write_unaligned_offset_is_rmw() {
+        let tf = pattern_file(8192);
+        let af = forced(tf.as_file().try_clone().unwrap(), 512);
+        let data: Vec<u8> = (0..200).map(|i| ((i + 1) % 239) as u8).collect();
+        assert_eq!(af.write_at(&data, 100).unwrap(), 200);
+
+        let mut whole = vec![0u8; 8192];
+        tf.as_file().read_exact_at(&mut whole, 0).unwrap();
+        let before: Vec<u8> = (0..100).map(|i| (i % 251) as u8).collect();
+        assert_eq!(&whole[..100], &before[..]);
+        assert_eq!(&whole[100..300], &data[..]);
+        let after: Vec<u8> = (300..8192).map(|i| (i % 251) as u8).collect();
+        assert_eq!(&whole[300..], &after[..]);
+    }
+
+    #[test]
+    fn aligned_passthrough_roundtrip() {
+        let tf = pattern_file(4096);
+        let af = forced(tf.as_file().try_clone().unwrap(), 512);
+        let mut buf = vec![0u8; 512];
+        assert_eq!(af.read_at(&mut buf, 512).unwrap(), 512);
+        let want: Vec<u8> = (512..1024).map(|i| (i % 251) as u8).collect();
+        assert_eq!(buf, want);
+    }
+
+    #[test]
+    fn no_alignment_is_plain_passthrough() {
+        let tf = pattern_file(100);
+        let af = forced(tf.as_file().try_clone().unwrap(), 0);
+        let mut buf = vec![0u8; 50];
+        assert_eq!(af.read_at(&mut buf, 10).unwrap(), 50);
+    }
+
+    #[test]
+    fn test_unaligned_read_beyond_eof_returns_zero() {
+        let tf = pattern_file(100);
+        let af = forced(tf.as_file().try_clone().unwrap(), 512);
+        let mut buf = vec![0u8; 16];
+        assert_eq!(af.read_at(&mut buf, 200).unwrap(), 0);
+    }
+
+    #[test]
+    fn test_unaligned_write_extends_at_eof() {
+        let file_size = 100usize;
+        let tf = pattern_file(file_size);
+        let af = forced(tf.as_file().try_clone().unwrap(), 512);
+        let data = b"xyz";
+        assert_eq!(af.write_at(data, file_size as u64).unwrap(), data.len());
+
+        let mut readback = vec![0u8; file_size + data.len()];
+        tf.as_file().read_exact_at(&mut readback, 0).unwrap();
+        let expected_prefix: Vec<u8> = (0..file_size).map(|i| (i % 251) as u8).collect();
+        assert_eq!(&readback[..file_size], &expected_prefix[..]);
+        assert_eq!(&readback[file_size..], data);
+    }
+
+    #[test]
+    fn test_empty_unaligned_io_is_noop() {
+        let tf = pattern_file(100);
+        let af = forced(tf.as_file().try_clone().unwrap(), 512);
+        let mut read_buf = [];
+        assert_eq!(af.read_at(&mut read_buf, 1).unwrap(), 0);
+        assert_eq!(af.write_at(&[], 1).unwrap(), 0);
+    }
+
+    #[test]
+    fn read_unaligned_scatters_in_a_single_copy() {
+        let file = pattern_file(8192);
+        let aligned_file = forced(file.as_file().try_clone().unwrap(), 512);
+        let mut out = vec![0u8; 200];
+        let n = aligned_file
+            .read_unaligned(100, 200, |data| {
+                out.copy_from_slice(data);
+                Ok(())
+            })
+            .unwrap();
+        assert_eq!(n, 200);
+        let want: Vec<u8> = (100..300).map(|i| (i % 251) as u8).collect();
+        assert_eq!(out, want);
+    }
+
+    #[test]
+    fn read_unaligned_closure_short_at_eof() {
+        let file = pattern_file(100);
+        let aligned_file = forced(file.as_file().try_clone().unwrap(), 512);
+        let mut seen = 0usize;
+        let n = aligned_file
+            .read_unaligned(10, 200, |data| {
+                seen = data.len();
+                Ok(())
+            })
+            .unwrap();
+        assert_eq!(n, 90);
+        assert_eq!(seen, 90);
+    }
+
+    #[test]
+    fn write_unaligned_gather_is_rmw() {
+        let file = pattern_file(8192);
+        let aligned_file = forced(file.as_file().try_clone().unwrap(), 512);
+        let data: Vec<u8> = (0..200).map(|i| ((i + 1) % 239) as u8).collect();
+        let n = aligned_file
+            .write_unaligned(100, 200, |buf| {
+                buf.copy_from_slice(&data);
+                Ok(())
+            })
+            .unwrap();
+        assert_eq!(n, 200);
+
+        let mut whole = vec![0u8; 8192];
+        file.as_file().read_exact_at(&mut whole, 0).unwrap();
+        let before: Vec<u8> = (0..100).map(|i| (i % 251) as u8).collect();
+        assert_eq!(&whole[..100], &before[..]);
+        assert_eq!(&whole[100..300], &data[..]);
+        let after: Vec<u8> = (300..8192).map(|i| (i % 251) as u8).collect();
+        assert_eq!(&whole[300..], &after[..]);
+    }
+
+    #[test]
+    fn read_unaligned_propagates_closure_error() {
+        let file = pattern_file(8192);
+        let aligned_file = forced(file.as_file().try_clone().unwrap(), 512);
+        let err = aligned_file
+            .read_unaligned(100, 200, |_| {
+                Err(io::Error::new(io::ErrorKind::InvalidInput, "boom"))
+            })
+            .unwrap_err();
+        assert_eq!(err.kind(), io::ErrorKind::InvalidInput);
+    }
+
+    #[test]
+    fn write_unaligned_propagates_closure_error() {
+        let file = pattern_file(8192);
+        let aligned_file = forced(file.as_file().try_clone().unwrap(), 512);
+        let err = aligned_file
+            .write_unaligned(100, 200, |_| {
+                Err(io::Error::new(io::ErrorKind::InvalidInput, "boom"))
+            })
+            .unwrap_err();
+        assert_eq!(err.kind(), io::ErrorKind::InvalidInput);
+    }
+}

block/src/aligned_operation.rs

@@ -1,90 +0,0 @@
-// Copyright (c) 2026 Meta Platforms, Inc. and affiliates.
-//
-// SPDX-License-Identifier: Apache-2.0 AND BSD-3-Clause
-
-use std::alloc::{Layout, alloc_zeroed, dealloc};
-use std::io;
-
-use vm_memory::GuestAddress;
-
-/// Owns an aligned bounce buffer used when a guest descriptor's host VA
-/// does not meet the disk backend's alignment requirement.
-#[derive(Debug)]
-pub struct AlignedOperation {
-    data_addr: GuestAddress,
-    aligned_ptr: *mut u8,
-    size: usize,
-    layout: Layout,
-}
-
-impl AlignedOperation {
-    /// Allocate a zero-initialized buffer of `size` bytes aligned to
-    /// `alignment`. Returns `InvalidInput` if `size` is zero;
-    /// `alignment` must be a power of two and not exceed `isize::MAX`
-    /// after rounding up.
-    pub fn new(data_addr: GuestAddress, size: usize, alignment: usize) -> io::Result<Self> {
-        if size == 0 {
-            return Err(io::Error::new(
-                io::ErrorKind::InvalidInput,
-                "AlignedOperation requires a non-zero size",
-            ));
-        }
-        let layout = Layout::from_size_align(size, alignment)
-            .map_err(|e| io::Error::new(io::ErrorKind::InvalidInput, e))?;
-        // SAFETY: size is non-zero (checked above) and Layout::from_size_align
-        // rejects alignments that are not a power of two or that overflow.
-        let aligned_ptr = unsafe { alloc_zeroed(layout) };
-        if aligned_ptr.is_null() {
-            return Err(io::Error::last_os_error());
-        }
-        Ok(Self {
-            data_addr,
-            aligned_ptr,
-            size,
-            layout,
-        })
-    }
-
-    /// Gets the raw pointer to the aligned buffer.
-    pub fn as_mut_ptr(&mut self) -> *mut u8 {
-        self.aligned_ptr
-    }
-
-    /// Returns the aligned buffer as a slice.
-    pub fn as_bytes(&self) -> &[u8] {
-        // SAFETY: `new` allocates `size` bytes via alloc_zeroed (so they
-        // are initialized) and AlignedOperation owns the buffer
-        // exclusively.
-        unsafe { std::slice::from_raw_parts(self.aligned_ptr, self.size) }
-    }
-
-    /// Returns the aligned buffer as a mutable slice.
-    pub fn as_bytes_mut(&mut self) -> &mut [u8] {
-        // SAFETY: same invariant as as_bytes; &mut self rules out other
-        // simultaneous borrows.
-        unsafe { std::slice::from_raw_parts_mut(self.aligned_ptr, self.size) }
-    }
-
-    /// Returns the guest address for this op.
-    pub fn data_addr(&self) -> GuestAddress {
-        self.data_addr
-    }
-}
-
-impl Drop for AlignedOperation {
-    fn drop(&mut self) {
-        // SAFETY: `new` is the only constructor, and it stores a pointer
-        // returned by `alloc_zeroed` paired with the exact `layout` used
-        // for that allocation. Ownership has not escaped (the type is
-        // neither `Clone` nor `Copy`).
-        unsafe {
-            dealloc(self.aligned_ptr, self.layout);
-        }
-    }
-}
-
-// SAFETY: AlignedOperation owns its heap allocation exclusively (no Clone/
-// Copy, no shared aliases) and the allocation's lifetime is tied to the
-// value's. Moving an AlignedOperation between threads transfers that
-// ownership — the same rationale Box<T> uses for its Send impl.
-unsafe impl Send for AlignedOperation {}

block/src/async_io.rs

@@ -1,112 +0,0 @@
-// Copyright © 2021 Intel Corporation
-//
-// SPDX-License-Identifier: Apache-2.0 AND BSD-3-Clause
-
-use std::marker::PhantomData;
-use std::os::fd::{AsRawFd, OwnedFd, RawFd};
-
-use thiserror::Error;
-use vmm_sys_util::eventfd::EventFd;
-
-use crate::{BatchRequest, SECTOR_SIZE};
-
-#[derive(Error, Debug)]
-pub enum DiskFileError {
-    /// Failed getting disk file size.
-    #[error("Failed getting disk file size")]
-    Size(#[source] std::io::Error),
-    /// Failed creating a new AsyncIo.
-    #[error("Failed creating a new AsyncIo")]
-    NewAsyncIo(#[source] std::io::Error),
-    /// Unsupported operation.
-    #[error("Unsupported operation")]
-    Unsupported,
-    /// Resize failed
-    #[error("Resize failed")]
-    ResizeError(#[source] std::io::Error),
-    #[error("Failed cloning disk file")]
-    Clone(#[source] std::io::Error),
-}
-
-pub type DiskFileResult<T> = std::result::Result<T, DiskFileError>;
-
-/// A wrapper for [`RawFd`] capturing the lifetime of a corresponding disk file.
-///
-/// This fulfills the same role as [`BorrowedFd`] but is tailored to the limitations
-/// by some disk implementations, which wrap the effective [`File`]
-/// in an `Arc<Mutex<T>>`, making the use of [`BorrowedFd`] impossible.
-///
-/// [`BorrowedFd`]: std::os::fd::BorrowedFd
-#[derive(Copy, Clone, Debug)]
-pub struct BorrowedDiskFd<'fd> {
-    raw_fd: RawFd,
-    _lifetime: PhantomData<&'fd OwnedFd>,
-}
-
-impl BorrowedDiskFd<'_> {
-    pub(super) fn new(raw_fd: RawFd) -> Self {
-        Self {
-            raw_fd,
-            _lifetime: PhantomData,
-        }
-    }
-}
-
-impl AsRawFd for BorrowedDiskFd<'_> {
-    fn as_raw_fd(&self) -> RawFd {
-        self.raw_fd
-    }
-}
-
-#[derive(Error, Debug)]
-pub enum AsyncIoError {
-    /// Failed vectored reading from file.
-    #[error("Failed vectored reading from file")]
-    ReadVectored(#[source] std::io::Error),
-    /// Failed vectored writing to file.
-    #[error("Failed vectored writing to file")]
-    WriteVectored(#[source] std::io::Error),
-    /// Failed synchronizing file.
-    #[error("Failed synchronizing file")]
-    Fsync(#[source] std::io::Error),
-    /// Failed punching hole.
-    #[error("Failed punching hole")]
-    PunchHole(#[source] std::io::Error),
-    /// Failed writing zeroes.
-    #[error("Failed writing zeroes")]
-    WriteZeroes(#[source] std::io::Error),
-    /// Failed submitting batch requests.
-    #[error("Failed submitting batch requests")]
-    SubmitBatchRequests(#[source] std::io::Error),
-}
-
-pub type AsyncIoResult<T> = std::result::Result<T, AsyncIoError>;
-
-pub trait AsyncIo: Send {
-    fn notifier(&self) -> &EventFd;
-    fn read_vectored(
-        &mut self,
-        offset: libc::off_t,
-        iovecs: &[libc::iovec],
-        user_data: u64,
-    ) -> AsyncIoResult<()>;
-    fn write_vectored(
-        &mut self,
-        offset: libc::off_t,
-        iovecs: &[libc::iovec],
-        user_data: u64,
-    ) -> AsyncIoResult<()>;
-    fn fsync(&mut self, user_data: Option<u64>) -> AsyncIoResult<()>;
-    fn punch_hole(&mut self, offset: u64, length: u64, user_data: u64) -> AsyncIoResult<()>;
-    fn write_zeroes(&mut self, offset: u64, length: u64, user_data: u64) -> AsyncIoResult<()>;
-    fn next_completed_request(&mut self) -> Option<(u64, i32)>;
-    fn batch_requests_enabled(&self) -> bool {
-        false
-    }
-    fn submit_batch_requests(&mut self, _batch_request: &[BatchRequest]) -> AsyncIoResult<()> {
-        Ok(())
-    }
-    fn alignment(&self) -> u64 {
-        SECTOR_SIZE
-    }
-}

block/src/factory.rs

@@ -20,10 +20,10 @@ use log::info;
 use crate::block_io_uring_is_supported;
 use crate::disk_file::AsyncFullDiskFile;
 use crate::error::{BlockError, BlockErrorKind, BlockResult};
-use crate::fixed_vhd_disk::FixedVhdDisk;
-use crate::qcow_disk::QcowDisk;
-use crate::raw_disk::{RawBackend, RawDisk};
-use crate::vhdx_sync::VhdxDiskSync;
+use crate::formats::qcow::QcowDisk;
+use crate::formats::raw::{RawBackend, RawDisk};
+use crate::formats::vhd::VhdDisk;
+use crate::formats::vhdx::VhdxDisk;
 use crate::{
     ImageType, block_aio_is_supported, detect_image_type, open_disk_image, preallocate_disk,
 };
@@ -114,7 +114,7 @@ fn open_vhdx(
 ) -> BlockResult<Box<dyn AsyncFullDiskFile>> {
     info!("Opening VHDX disk file with synchronous backend");
     Ok(Box::new(
-        VhdxDiskSync::new(file).map_err(|e| e.with_path(options.path))?,
+        VhdxDisk::new(file, options.direct).map_err(|e| e.with_path(options.path))?,
     ))
 }
 
@@ -127,7 +127,7 @@ fn open_fixed_vhd(
         if io_uring_supported() {
             info!("Opening fixed VHD disk file with io_uring backend");
             return Ok(Box::new(
-                FixedVhdDisk::new(file, true).map_err(|e| e.with_path(options.path))?,
+                VhdDisk::new(file, true, options.direct).map_err(|e| e.with_path(options.path))?,
             ));
         }
         info!("io_uring runtime probe failed for fixed VHD, using synchronous backend");
@@ -135,7 +135,7 @@ fn open_fixed_vhd(
 
     info!("Opening fixed VHD disk file with synchronous backend");
     Ok(Box::new(
-        FixedVhdDisk::new(file, false).map_err(|e| e.with_path(options.path))?,
+        VhdDisk::new(file, false, options.direct).map_err(|e| e.with_path(options.path))?,
     ))
 }
 
@@ -151,7 +151,11 @@ fn open_raw(
     if !options.disable_io_uring {
         if io_uring_supported() {
             info!("Opening RAW disk file with io_uring backend");
-            return Ok(Box::new(RawDisk::new(file, RawBackend::IoUring)));
+            return Ok(Box::new(RawDisk::new(
+                file,
+                RawBackend::IoUring,
+                options.direct,
+            )));
         }
         info!("io_uring runtime probe failed for RAW, trying next backend");
     }
@@ -159,13 +163,21 @@ fn open_raw(
     if !options.disable_aio {
         if aio_supported() {
             info!("Opening RAW disk file with AIO backend");
-            return Ok(Box::new(RawDisk::new(file, RawBackend::Aio)));
+            return Ok(Box::new(RawDisk::new(
+                file,
+                RawBackend::Aio,
+                options.direct,
+            )));
         }
         info!("AIO runtime probe failed for RAW, using synchronous backend");
     }
 
     info!("Opening RAW disk file with synchronous backend");
-    Ok(Box::new(RawDisk::new(file, RawBackend::Sync)))
+    Ok(Box::new(RawDisk::new(
+        file,
+        RawBackend::Sync,
+        options.direct,
+    )))
 }
 
 fn open_qcow2(
@@ -205,13 +217,12 @@ fn open_qcow2(
 
 #[cfg(test)]
 mod unit_tests {
-    use std::io::Write;
     use std::path::Path;
 
     use vmm_sys_util::tempfile::TempFile;
 
     use super::*;
-    use crate::qcow::{QcowFile, RawFile};
+    use crate::formats::qcow;
 
     fn default_options(path: &Path) -> DiskOpenOptions<'_> {
         DiskOpenOptions {
@@ -247,12 +258,9 @@ mod unit_tests {
 
     #[test]
     fn detect_qcow2_image() {
-        let tmp = TempFile::new().unwrap();
-        {
-            let raw = RawFile::new(tmp.as_file().try_clone().unwrap(), false);
-            let mut qcow = QcowFile::new(raw, 3, 100 * 1024 * 1024, true).unwrap();
-            qcow.flush().unwrap();
-        }
+        let tmp = qcow::QcowTempDisk::new(100 * 1024 * 1024, None, false, true, false)
+            .unwrap()
+            .into_tempfile();
         let path = tmp.as_path().to_owned();
         let options = default_options(&path);
         let opened = open_disk(&options).unwrap();

block/src/fixed_vhd.rs

@@ -1,99 +0,0 @@
-// Copyright © 2021 Intel Corporation
-//
-// SPDX-License-Identifier: Apache-2.0
-
-use std::fs::File;
-use std::io::{Read, Seek, SeekFrom, Write};
-use std::os::unix::io::{AsRawFd, RawFd};
-
-use crate::BlockBackend;
-use crate::vhd::VhdFooter;
-
-#[derive(Debug)]
-pub struct FixedVhd {
-    file: File,
-    size: u64,
-    position: u64,
-}
-
-impl FixedVhd {
-    pub fn new(mut file: File) -> std::io::Result<Self> {
-        let footer = VhdFooter::new(&mut file)?;
-
-        Ok(Self {
-            file,
-            size: footer.current_size(),
-            position: 0,
-        })
-    }
-}
-
-impl AsRawFd for FixedVhd {
-    fn as_raw_fd(&self) -> RawFd {
-        self.file.as_raw_fd()
-    }
-}
-
-impl Read for FixedVhd {
-    fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
-        match self.file.read(buf) {
-            Ok(r) => {
-                self.position = self.position.checked_add(r.try_into().unwrap()).unwrap();
-                Ok(r)
-            }
-            Err(e) => Err(e),
-        }
-    }
-}
-
-impl Write for FixedVhd {
-    fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
-        match self.file.write(buf) {
-            Ok(r) => {
-                self.position = self.position.checked_add(r.try_into().unwrap()).unwrap();
-                Ok(r)
-            }
-            Err(e) => Err(e),
-        }
-    }
-
-    fn flush(&mut self) -> std::io::Result<()> {
-        self.file.sync_all()
-    }
-}
-
-impl Seek for FixedVhd {
-    fn seek(&mut self, newpos: SeekFrom) -> std::io::Result<u64> {
-        match self.file.seek(newpos) {
-            Ok(pos) => {
-                self.position = pos;
-                Ok(pos)
-            }
-            Err(e) => Err(e),
-        }
-    }
-}
-
-impl BlockBackend for FixedVhd {
-    fn logical_size(&self) -> Result<u64, crate::Error> {
-        Ok(self.size)
-    }
-
-    /// Returns the physical size of the underlying file.
-    fn physical_size(&self) -> Result<u64, crate::Error> {
-        self.file
-            .metadata()
-            .map(|m| m.len())
-            .map_err(crate::Error::GetFileMetadata)
-    }
-}
-
-impl Clone for FixedVhd {
-    fn clone(&self) -> Self {
-        Self {
-            file: self.file.try_clone().expect("FixedVhd cloning failed"),
-            size: self.size,
-            position: self.position,
-        }
-    }
-}

block/src/fixed_vhd_async.rs

@@ -1,101 +0,0 @@
-// Copyright © 2021 Intel Corporation
-//
-// SPDX-License-Identifier: Apache-2.0
-
-use std::os::unix::io::RawFd;
-
-use vmm_sys_util::eventfd::EventFd;
-
-use crate::BatchRequest;
-use crate::async_io::{AsyncIo, AsyncIoError, AsyncIoResult};
-use crate::error::BlockResult;
-use crate::raw_async::RawFileAsync;
-
-pub struct FixedVhdAsync {
-    raw_file_async: RawFileAsync,
-    size: u64,
-}
-
-impl FixedVhdAsync {
-    pub fn new(fd: RawFd, ring_depth: u32, size: u64) -> BlockResult<Self> {
-        let raw_file_async = RawFileAsync::new(fd, ring_depth)?;
-
-        Ok(FixedVhdAsync {
-            raw_file_async,
-            size,
-        })
-    }
-}
-
-impl AsyncIo for FixedVhdAsync {
-    fn notifier(&self) -> &EventFd {
-        self.raw_file_async.notifier()
-    }
-
-    fn read_vectored(
-        &mut self,
-        offset: libc::off_t,
-        iovecs: &[libc::iovec],
-        user_data: u64,
-    ) -> AsyncIoResult<()> {
-        if offset as u64 >= self.size {
-            return Err(AsyncIoError::ReadVectored(std::io::Error::new(
-                std::io::ErrorKind::InvalidData,
-                format!(
-                    "Invalid offset {}, can't be larger than file size {}",
-                    offset, self.size
-                ),
-            )));
-        }
-
-        self.raw_file_async.read_vectored(offset, iovecs, user_data)
-    }
-
-    fn write_vectored(
-        &mut self,
-        offset: libc::off_t,
-        iovecs: &[libc::iovec],
-        user_data: u64,
-    ) -> AsyncIoResult<()> {
-        if offset as u64 >= self.size {
-            return Err(AsyncIoError::WriteVectored(std::io::Error::new(
-                std::io::ErrorKind::InvalidData,
-                format!(
-                    "Invalid offset {}, can't be larger than file size {}",
-                    offset, self.size
-                ),
-            )));
-        }
-
-        self.raw_file_async
-            .write_vectored(offset, iovecs, user_data)
-    }
-
-    fn fsync(&mut self, user_data: Option<u64>) -> AsyncIoResult<()> {
-        self.raw_file_async.fsync(user_data)
-    }
-
-    fn next_completed_request(&mut self) -> Option<(u64, i32)> {
-        self.raw_file_async.next_completed_request()
-    }
-
-    fn punch_hole(&mut self, _offset: u64, _length: u64, _user_data: u64) -> AsyncIoResult<()> {
-        Err(AsyncIoError::PunchHole(std::io::Error::other(
-            "punch_hole not supported for fixed VHD",
-        )))
-    }
-
-    fn write_zeroes(&mut self, _offset: u64, _length: u64, _user_data: u64) -> AsyncIoResult<()> {
-        Err(AsyncIoError::WriteZeroes(std::io::Error::other(
-            "write_zeroes not supported for fixed VHD",
-        )))
-    }
-
-    fn batch_requests_enabled(&self) -> bool {
-        true
-    }
-
-    fn submit_batch_requests(&mut self, batch_request: &[BatchRequest]) -> AsyncIoResult<()> {
-        self.raw_file_async.submit_batch_requests(batch_request)
-    }
-}

block/src/fixed_vhd_sync.rs

@@ -1,88 +0,0 @@
-// Copyright © 2021 Intel Corporation
-//
-// SPDX-License-Identifier: Apache-2.0
-
-use std::os::unix::io::RawFd;
-
-use vmm_sys_util::eventfd::EventFd;
-
-use crate::async_io::{AsyncIo, AsyncIoError, AsyncIoResult};
-use crate::raw_sync::RawFileSync;
-
-pub struct FixedVhdSync {
-    raw_file_sync: RawFileSync,
-    size: u64,
-}
-
-impl FixedVhdSync {
-    pub fn new(fd: RawFd, size: u64) -> std::io::Result<Self> {
-        Ok(FixedVhdSync {
-            raw_file_sync: RawFileSync::new(fd),
-            size,
-        })
-    }
-}
-
-impl AsyncIo for FixedVhdSync {
-    fn notifier(&self) -> &EventFd {
-        self.raw_file_sync.notifier()
-    }
-
-    fn read_vectored(
-        &mut self,
-        offset: libc::off_t,
-        iovecs: &[libc::iovec],
-        user_data: u64,
-    ) -> AsyncIoResult<()> {
-        if offset as u64 >= self.size {
-            return Err(AsyncIoError::ReadVectored(std::io::Error::new(
-                std::io::ErrorKind::InvalidData,
-                format!(
-                    "Invalid offset {}, can't be larger than file size {}",
-                    offset, self.size
-                ),
-            )));
-        }
-
-        self.raw_file_sync.read_vectored(offset, iovecs, user_data)
-    }
-
-    fn write_vectored(
-        &mut self,
-        offset: libc::off_t,
-        iovecs: &[libc::iovec],
-        user_data: u64,
-    ) -> AsyncIoResult<()> {
-        if offset as u64 >= self.size {
-            return Err(AsyncIoError::WriteVectored(std::io::Error::new(
-                std::io::ErrorKind::InvalidData,
-                format!(
-                    "Invalid offset {}, can't be larger than file size {}",
-                    offset, self.size
-                ),
-            )));
-        }
-
-        self.raw_file_sync.write_vectored(offset, iovecs, user_data)
-    }
-
-    fn fsync(&mut self, user_data: Option<u64>) -> AsyncIoResult<()> {
-        self.raw_file_sync.fsync(user_data)
-    }
-
-    fn next_completed_request(&mut self) -> Option<(u64, i32)> {
-        self.raw_file_sync.next_completed_request()
-    }
-
-    fn punch_hole(&mut self, _offset: u64, _length: u64, _user_data: u64) -> AsyncIoResult<()> {
-        Err(AsyncIoError::PunchHole(std::io::Error::other(
-            "punch_hole not supported for fixed VHD",
-        )))
-    }
-
-    fn write_zeroes(&mut self, _offset: u64, _length: u64, _user_data: u64) -> AsyncIoResult<()> {
-        Err(AsyncIoError::WriteZeroes(std::io::Error::other(
-            "write_zeroes not supported for fixed VHD",
-        )))
-    }
-}

block/src/formats/mod.rs

@@ -0,0 +1,13 @@
+// Copyright 2026 The Cloud Hypervisor Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+//! Disk format implementations.
+//!
+//! Each format lives in its own submodule with a `DiskFile` wrapper,
+//! format specific internals, and sync/async I/O workers.
+
+pub mod qcow;
+pub mod raw;
+pub mod vhd;
+pub mod vhdx;

block/src/formats/qcow/backing.rs

@@ -6,23 +6,25 @@
 
 //! Thread safe backing file readers for QCOW2 images.
 
+use std::fs::File;
 use std::io;
-use std::os::fd::{AsFd, AsRawFd, BorrowedFd, OwnedFd};
+use std::os::fd::{AsFd, BorrowedFd, OwnedFd};
+use std::os::unix::fs::FileExt;
 use std::sync::Arc;
 
+use super::decoder::Decoder;
+use super::metadata::{BackingRead, ClusterReadMapping, QcowMetadata};
+use super::parser::{BackingFile, BackingKind, Error as QcowError};
 use crate::error::{BlockError, BlockErrorKind, BlockResult, ErrorOp};
-use crate::qcow::decoder::Decoder;
-use crate::qcow::metadata::{BackingRead, ClusterReadMapping, QcowMetadata};
-use crate::qcow::{BackingFile, BackingKind, Error as QcowError};
-use crate::qcow_common::{decompress_cluster, pread_alloc, pread_exact};
+use crate::formats::qcow::common::decompress_cluster;
 
-/// Raw backing file using pread64 on a duplicated fd.
+/// Raw backing file using position-independent reads on a duplicated fd.
 pub(crate) struct RawBacking {
-    pub(crate) fd: OwnedFd,
+    pub(crate) file: File,
     pub(crate) virtual_size: u64,
 }
 
-// SAFETY: The only I/O operation is pread64 which is position independent
+// SAFETY: The only I/O operation is read_at which is position independent
 // and safe for concurrent use from multiple threads.
 unsafe impl Sync for RawBacking {}
 
@@ -34,9 +36,9 @@ impl BackingRead for RawBacking {
         }
         let available = (self.virtual_size - address) as usize;
         if available >= buf.len() {
-            pread_exact(self.fd.as_raw_fd(), buf, address)
+            self.file.read_exact_at(buf, address)
         } else {
-            pread_exact(self.fd.as_raw_fd(), &mut buf[..available], address)?;
+            self.file.read_exact_at(&mut buf[..available], address)?;
             buf[available..].fill(0);
             Ok(())
         }
@@ -51,14 +53,14 @@ impl BackingRead for RawBacking {
 /// are handled recursively via the optional `backing_file` field.
 pub(crate) struct Qcow2Backing {
     pub(crate) metadata: Arc<QcowMetadata>,
-    pub(crate) data_fd: OwnedFd,
+    pub(crate) data_file: File,
     pub(crate) backing_file: Option<Arc<dyn BackingRead>>,
     pub(crate) cluster_size: u64,
     pub(crate) decoder: Arc<dyn Decoder>,
 }
 
 // SAFETY: All reads go through QcowMetadata which uses RwLock
-// and pread64 which is position independent and thread safe.
+// and read_exact_at which is position independent and thread safe.
 unsafe impl Sync for Qcow2Backing {}
 
 impl BackingRead for Qcow2Backing {
@@ -79,8 +81,6 @@ impl BackingRead for Qcow2Backing {
 }
 
 impl Qcow2Backing {
-    /// Resolve cluster mappings via metadata then read allocated clusters
-    /// with pread64.
     fn read_clusters(&self, address: u64, buf: &mut [u8]) -> io::Result<()> {
         let total_len = buf.len();
         let has_backing = self.backing_file.is_some();
@@ -100,8 +100,7 @@ impl Qcow2Backing {
                     offset: host_offset,
                     length,
                 } => {
-                    pread_exact(
-                        self.data_fd.as_raw_fd(),
+                    self.data_file.read_exact_at(
                         &mut buf[buf_offset..buf_offset + length as usize],
                         host_offset,
                     )?;
@@ -113,8 +112,8 @@ impl Qcow2Backing {
                     cluster_offset,
                     length,
                 } => {
-                    let compressed =
-                        pread_alloc(self.data_fd.as_raw_fd(), host_offset, compressed_size)?;
+                    let mut compressed = vec![0u8; compressed_size];
+                    self.data_file.read_exact_at(&mut compressed, host_offset)?;
                     let decompressed = decompress_cluster(
                         &compressed,
                         self.cluster_size as usize,
@@ -147,7 +146,7 @@ impl Drop for Qcow2Backing {
 }
 
 /// Construct a thread safe backing file reader.
-pub fn shared_backing_from(bf: BackingFile) -> BlockResult<Arc<dyn BackingRead>> {
+pub(super) fn shared_backing_from(bf: BackingFile) -> BlockResult<Arc<dyn BackingRead>> {
     let (kind, virtual_size) = bf.into_kind();
 
     let dup_fd = |fd: BorrowedFd<'_>| -> BlockResult<OwnedFd> {
@@ -162,23 +161,19 @@ pub fn shared_backing_from(bf: BackingFile) -> BlockResult<Arc<dyn BackingRead>>
 
     match kind {
         BackingKind::Raw(raw_file) => {
-            let fd = dup_fd(raw_file.as_fd())?;
-            Ok(Arc::new(RawBacking { fd, virtual_size }))
+            let file = File::from(dup_fd(raw_file.as_fd())?);
+            Ok(Arc::new(RawBacking { file, virtual_size }))
         }
         BackingKind::Qcow { inner, backing } => {
-            let data_fd = dup_fd(inner.raw_file.as_fd())?;
+            let data_file = File::from(dup_fd(inner.raw_file.as_fd())?);
             let metadata = Arc::new(QcowMetadata::new(*inner));
             Ok(Arc::new(Qcow2Backing {
                 cluster_size: metadata.cluster_size(),
                 decoder: metadata.decoder(),
                 metadata,
-                data_fd,
+                data_file,
                 backing_file: backing.map(|bf| shared_backing_from(*bf)).transpose()?,
             }))
         }
-        #[cfg(test)]
-        BackingKind::QcowFile(_) => {
-            unreachable!("QcowFile variant is only used by set_backing_file() in tests")
-        }
     }
 }

block/src/formats/qcow/common.rs

@@ -0,0 +1,161 @@
+// Copyright © 2021 Intel Corporation
+//
+// Copyright 2026 The Cloud Hypervisor Authors. All rights reserved.
+//
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+//
+// SPDX-License-Identifier: Apache-2.0 AND BSD-3-Clause
+
+//! Shared helpers for QCOW2 sync and async backends.
+
+use std::io;
+
+use super::decoder::Decoder;
+
+/// Decompress a full QCOW2 cluster from compressed data.
+///
+/// Returns a `cluster_size` byte buffer with the decompressed cluster
+/// content. Fails if the decoder does not produce exactly `cluster_size`
+/// bytes.
+pub(super) fn decompress_cluster(
+    compressed: &[u8],
+    cluster_size: usize,
+    decoder: &dyn Decoder,
+) -> io::Result<Vec<u8>> {
+    let mut decompressed = vec![0u8; cluster_size];
+    let n = decoder
+        .decode(compressed, &mut decompressed)
+        .map_err(|_| io::Error::from_raw_os_error(libc::EIO))?;
+    if n != cluster_size {
+        return Err(io::Error::from_raw_os_error(libc::EIO));
+    }
+    Ok(decompressed)
+}
+
+#[cfg(test)]
+pub(crate) mod unit_tests {
+    use std::fs::File;
+    use std::io::Write;
+    use std::os::unix::fs::FileExt;
+
+    use flate2::Compression;
+    use flate2::write::DeflateEncoder;
+
+    use super::super::decoder::ZlibDecoder;
+    use super::decompress_cluster;
+
+    const COMPRESSED_FLAG: u64 = 1 << 62;
+    const CLUSTER_USED_FLAG: u64 = 1 << 63;
+    const COMPRESSED_SECTOR_SIZE: u64 = 512;
+
+    const HEADER_CLUSTER_BITS_OFFSET: u64 = 20;
+    const HEADER_L1_SIZE_OFFSET: u64 = 36;
+    const HEADER_L1_TABLE_OFFSET: u64 = 40;
+
+    const L1_L2_ADDR_MASK: u64 = 0x00ff_ffff_ffff_fe00;
+
+    fn make_compressed_l2_entry(host_offset: u64, compressed_len: usize, cluster_bits: u32) -> u64 {
+        let compressed_size_shift = 62 - (cluster_bits - 8);
+        let intra_sector_offset = host_offset & (COMPRESSED_SECTOR_SIZE - 1);
+        let total_bytes = compressed_len as u64 + intra_sector_offset;
+        let nsectors = total_bytes.div_ceil(COMPRESSED_SECTOR_SIZE);
+        let addr_part = host_offset & ((1 << compressed_size_shift) - 1);
+        let size_part = (nsectors - 1) << compressed_size_shift;
+        COMPRESSED_FLAG | size_part | addr_part
+    }
+
+    /// Compress every allocated cluster in a QCOW2 image file in place.
+    pub fn compress_allocated_clusters(file: &mut File) {
+        let mut buf4 = [0u8; 4];
+        file.read_exact_at(&mut buf4, HEADER_CLUSTER_BITS_OFFSET)
+            .unwrap();
+        let cluster_bits = u32::from_be_bytes(buf4);
+        let cluster_size = 1u64 << cluster_bits;
+
+        file.read_exact_at(&mut buf4, HEADER_L1_SIZE_OFFSET)
+            .unwrap();
+        let l1_size = u32::from_be_bytes(buf4);
+
+        let mut buf8 = [0u8; 8];
+        file.read_exact_at(&mut buf8, HEADER_L1_TABLE_OFFSET)
+            .unwrap();
+        let l1_table_offset = u64::from_be_bytes(buf8);
+
+        let entries_per_l2 = cluster_size / 8;
+
+        let mut append_offset = file.metadata().unwrap().len();
+        append_offset = (append_offset + 511) & !511;
+
+        for l1_idx in 0..l1_size as u64 {
+            let l1_entry_offset = l1_table_offset + l1_idx * 8;
+            file.read_exact_at(&mut buf8, l1_entry_offset).unwrap();
+            let l1_entry = u64::from_be_bytes(buf8);
+
+            let l2_table_addr = l1_entry & L1_L2_ADDR_MASK;
+            if l2_table_addr == 0 {
+                continue;
+            }
+
+            for l2_idx in 0..entries_per_l2 {
+                let l2_entry_offset = l2_table_addr + l2_idx * 8;
+                file.read_exact_at(&mut buf8, l2_entry_offset).unwrap();
+                let l2_entry = u64::from_be_bytes(buf8);
+
+                if l2_entry & CLUSTER_USED_FLAG == 0 || l2_entry & COMPRESSED_FLAG != 0 {
+                    continue;
+                }
+
+                let host_cluster_addr = l2_entry & L1_L2_ADDR_MASK;
+                if host_cluster_addr == 0 {
+                    continue;
+                }
+
+                let mut cluster_data = vec![0u8; cluster_size as usize];
+                file.read_exact_at(&mut cluster_data, host_cluster_addr)
+                    .unwrap();
+
+                let mut encoder = DeflateEncoder::new(Vec::new(), Compression::default());
+                encoder.write_all(&cluster_data).unwrap();
+                let compressed = encoder.finish().unwrap();
+
+                file.write_all_at(&compressed, append_offset).unwrap();
+
+                let padded_len = (compressed.len() + 511) & !511;
+                if padded_len > compressed.len() {
+                    let padding = vec![0u8; padded_len - compressed.len()];
+                    file.write_all_at(&padding, append_offset + compressed.len() as u64)
+                        .unwrap();
+                }
+
+                let new_entry =
+                    make_compressed_l2_entry(append_offset, compressed.len(), cluster_bits);
+                file.write_all_at(&new_entry.to_be_bytes(), l2_entry_offset)
+                    .unwrap();
+
+                append_offset += padded_len as u64;
+            }
+        }
+
+        file.flush().unwrap();
+    }
+
+    #[test]
+    fn test_decompress_cluster() {
+        let cluster_size = 65536;
+        let original: Vec<u8> = (0..=255).cycle().take(cluster_size).collect();
+
+        let mut encoder = DeflateEncoder::new(Vec::new(), Compression::default());
+        encoder.write_all(&original).unwrap();
+        let compressed = encoder.finish().unwrap();
+
+        let result = decompress_cluster(&compressed, cluster_size, &ZlibDecoder {}).unwrap();
+        assert_eq!(result, original);
+    }
+
+    #[test]
+    fn test_decompress_cluster_corrupt_input() {
+        let corrupt = vec![0xffu8; 64];
+        let err = decompress_cluster(&corrupt, 65536, &ZlibDecoder {}).unwrap_err();
+        assert_eq!(err.raw_os_error(), Some(libc::EIO));
+    }
+}

block/src/formats/qcow/decoder.rs

@@ -2,6 +2,8 @@
 //
 // SPDX-License-Identifier: Apache-2.0
 
+use std::{io, result};
+
 use thiserror::Error;
 
 #[derive(Debug, Error)]
@@ -11,12 +13,12 @@ pub enum Error {
     #[error("Zlib unexpected status: {0:?}")]
     ZlibUnexpectedStatus(flate2::Status),
     #[error("Zstd decompress error")]
-    ZstdDecompress(#[source] std::io::Error),
+    ZstdDecompress(#[source] io::Error),
     #[error("Zstd: failed to fill buffer")]
-    ZstdFillBuffer(#[source] std::io::Error),
+    ZstdFillBuffer(#[source] io::Error),
 }
 
-pub type Result<T> = std::result::Result<T, Error>;
+pub(super) type Result<T> = result::Result<T, Error>;
 
 /// Generic trait for decoding zlib/zstd formats
 pub trait Decoder: Send + Sync {
@@ -24,7 +26,7 @@ pub trait Decoder: Send + Sync {
 }
 
 #[derive(Default)]
-pub struct ZlibDecoder {}
+pub(super) struct ZlibDecoder {}
 
 impl Decoder for ZlibDecoder {
     fn decode(&self, input: &[u8], output: &mut [u8]) -> Result<usize> {
@@ -43,7 +45,7 @@ impl Decoder for ZlibDecoder {
 }
 
 #[derive(Default)]
-pub struct ZstdDecoder {}
+pub(super) struct ZstdDecoder {}
 
 impl Decoder for ZstdDecoder {
     fn decode(&self, input: &[u8], output: &mut [u8]) -> Result<usize> {

block/src/formats/qcow/header.rs

@@ -9,17 +9,18 @@
 //! QCOW2 header parsing, validation, and creation.
 
 use std::fmt::{Display, Formatter, Result as FmtResult};
-use std::io::{Read, Seek, SeekFrom, Write};
-use std::mem::size_of;
+use std::os::unix::fs::FileExt;
 use std::str::FromStr;
 
 use bitflags::bitflags;
 use vmm_sys_util::file_traits::FileSync;
+use zerocopy::big_endian::{U32 as BeU32, U64 as BeU64};
+use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout};
 
 use super::decoder::{Decoder, ZlibDecoder, ZstdDecoder};
-use super::qcow_raw_file::BeUint;
-use super::raw_file::RawFile;
-use super::{Error, Result, div_round_up_u32, div_round_up_u64};
+use super::parser::{Error, Result};
+use super::util::{div_round_up_u32, div_round_up_u64};
+use crate::aligned_file::AlignedFile;
 use crate::error::{BlockError, BlockErrorKind, BlockResult};
 
 #[derive(Copy, Clone, Debug, PartialEq, Eq)]
@@ -207,30 +208,115 @@ pub struct QcowHeader {
     pub backing_file: Option<BackingFileConfig>,
 }
 
+/// On-disk layout of the bare qcow2 header shared by v2 and v3 (72 bytes).
+#[repr(C)]
+#[derive(FromBytes, IntoBytes, KnownLayout, Immutable)]
+struct RawHeaderV2 {
+    magic: BeU32,
+    version: BeU32,
+    backing_file_offset: BeU64,
+    backing_file_size: BeU32,
+    cluster_bits: BeU32,
+    size: BeU64,
+    crypt_method: BeU32,
+    l1_size: BeU32,
+    l1_table_offset: BeU64,
+    refcount_table_offset: BeU64,
+    refcount_table_clusters: BeU32,
+    nb_snapshots: BeU32,
+    snapshots_offset: BeU64,
+}
+
+impl RawHeaderV2 {
+    fn from_header(header: &QcowHeader) -> Self {
+        Self {
+            magic: BeU32::new(header.magic),
+            version: BeU32::new(header.version),
+            backing_file_offset: BeU64::new(header.backing_file_offset),
+            backing_file_size: BeU32::new(header.backing_file_size),
+            cluster_bits: BeU32::new(header.cluster_bits),
+            size: BeU64::new(header.size),
+            crypt_method: BeU32::new(header.crypt_method),
+            l1_size: BeU32::new(header.l1_size),
+            l1_table_offset: BeU64::new(header.l1_table_offset),
+            refcount_table_offset: BeU64::new(header.refcount_table_offset),
+            refcount_table_clusters: BeU32::new(header.refcount_table_clusters),
+            nb_snapshots: BeU32::new(header.nb_snapshots),
+            snapshots_offset: BeU64::new(header.snapshots_offset),
+        }
+    }
+}
+
+/// On-disk layout of the fields v3 adds after the bare header (32 bytes).
+#[repr(C)]
+#[derive(FromBytes, IntoBytes, KnownLayout, Immutable)]
+struct RawHeaderV3Tail {
+    incompatible_features: BeU64,
+    compatible_features: BeU64,
+    autoclear_features: BeU64,
+    refcount_order: BeU32,
+    header_size: BeU32,
+}
+
+impl RawHeaderV3Tail {
+    fn from_header(header: &QcowHeader) -> Self {
+        Self {
+            incompatible_features: BeU64::new(header.incompatible_features),
+            compatible_features: BeU64::new(header.compatible_features),
+            autoclear_features: BeU64::new(header.autoclear_features),
+            refcount_order: BeU32::new(header.refcount_order),
+            header_size: BeU32::new(header.header_size),
+        }
+    }
+}
+
+#[repr(C)]
+#[derive(FromBytes, IntoBytes, KnownLayout, Immutable)]
+struct ExtensionHeader {
+    extension_type: BeU32,
+    length: BeU32,
+}
+
+impl ExtensionHeader {
+    fn end() -> Self {
+        Self {
+            extension_type: BeU32::new(HEADER_EXT_END),
+            length: BeU32::ZERO,
+        }
+    }
+}
+
 impl QcowHeader {
     /// Read header extensions, optionally collecting feature names for error reporting.
     pub(super) fn read_header_extensions(
-        f: &mut RawFile,
+        f: &AlignedFile,
         header: &mut QcowHeader,
         mut feature_table: Option<&mut Vec<(u8, String)>>,
     ) -> Result<()> {
-        // Extensions start directly after the header
-        f.seek(SeekFrom::Start(header.header_size as u64))
-            .map_err(Error::ReadingHeader)?;
+        // Extensions start directly after the header.
+        let mut offset = header.header_size as u64;
 
         loop {
-            let ext_type = u32::read_be(f).map_err(Error::ReadingHeader)?;
+            let mut field = [0u8; size_of::<ExtensionHeader>()];
+            f.read_exact_at(&mut field, offset)
+                .map_err(Error::ReadingHeader)?;
+            offset += field.len() as u64;
+
+            let extension =
+                ExtensionHeader::read_from_bytes(&field).expect("buffer covers extension header");
+            let ext_type = extension.extension_type.get();
             if ext_type == HEADER_EXT_END {
                 break;
             }
 
-            let ext_length = u32::read_be(f).map_err(Error::ReadingHeader)?;
+            let ext_length = extension.length.get();
 
             match ext_type {
                 HEADER_EXT_BACKING_FORMAT => {
                     let mut format_bytes = vec![0u8; ext_length as usize];
-                    f.read_exact(&mut format_bytes)
+                    f.read_exact_at(&mut format_bytes, offset)
                         .map_err(Error::ReadingHeader)?;
+                    offset += format_bytes.len() as u64;
                     let format_str = String::from_utf8(format_bytes)
                         .map_err(|err| Error::InvalidBackingFileName(err.utf8_error()))?;
                     if let Some(backing_file) = &mut header.backing_file {
@@ -240,9 +326,11 @@ impl QcowHeader {
                 HEADER_EXT_FEATURE_NAME_TABLE if feature_table.is_some() => {
                     const FEATURE_NAME_ENTRY_SIZE: usize = 1 + 1 + 46; // type + bit + name
                     let mut data = vec![0u8; ext_length as usize];
-                    f.read_exact(&mut data).map_err(Error::ReadingHeader)?;
+                    f.read_exact_at(&mut data, offset)
+                        .map_err(Error::ReadingHeader)?;
+                    offset += data.len() as u64;
                     let table = feature_table.as_mut().unwrap();
-                    for entry in data.chunks_exact(FEATURE_NAME_ENTRY_SIZE) {
+                    for entry in data.as_chunks::<FEATURE_NAME_ENTRY_SIZE>().0 {
                         if entry[0] == FEAT_TYPE_INCOMPATIBLE {
                             let bit_number = entry[1];
                             let name_bytes = &entry[2..];
@@ -254,88 +342,116 @@ impl QcowHeader {
                 }
                 _ => {
                     // Skip unknown extension
-                    f.seek(SeekFrom::Current(ext_length as i64))
-                        .map_err(Error::ReadingHeader)?;
+                    offset += ext_length as u64;
                 }
             }
 
             // Skip to the next 8 byte boundary
             let padding = (8 - (ext_length % 8)) % 8;
-            f.seek(SeekFrom::Current(padding as i64))
-                .map_err(Error::ReadingHeader)?;
+            offset += padding as u64;
         }
 
         Ok(())
     }
 
     /// Creates a QcowHeader from a reference to a file.
-    pub fn new(f: &mut RawFile) -> Result<QcowHeader> {
-        f.rewind().map_err(Error::ReadingHeader)?;
-        let magic = u32::read_be(f).map_err(Error::ReadingHeader)?;
+    pub fn new(f: &AlignedFile) -> Result<QcowHeader> {
+        // The bare header fits in V3_BARE_HEADER_SIZE plus the optional
+        // compression field. Read it once, then decode each region as a typed
+        // view whose layout matches the on-disk header.
+        let mut buf = [0u8; V3_BARE_HEADER_SIZE as usize + size_of::<u64>()];
+        f.read_exact_at(&mut buf, 0).map_err(Error::ReadingHeader)?;
+
+        // `buf` is always larger than the views, and the views are unaligned,
+        // so the casts cannot fail.
+        let (v2, tail) = RawHeaderV2::ref_from_prefix(&buf).expect("buffer covers the v2 header");
+
+        let magic = v2.magic.get();
         if magic != QCOW_MAGIC {
             return Err(Error::InvalidMagic);
         }
-
-        // Reads the next u32 from the file.
-        fn read_u32_be(f: &mut RawFile) -> Result<u32> {
-            u32::read_be(f).map_err(Error::ReadingHeader)
-        }
-
-        // Reads the next u64 from the file.
-        fn read_u64_be(f: &mut RawFile) -> Result<u64> {
-            u64::read_be(f).map_err(Error::ReadingHeader)
-        }
-
-        let version = read_u32_be(f)?;
+        let version = v2.version.get();
 
         let mut header = QcowHeader {
             magic,
             version,
-            backing_file_offset: read_u64_be(f)?,
-            backing_file_size: read_u32_be(f)?,
-            cluster_bits: read_u32_be(f)?,
-            size: read_u64_be(f)?,
-            crypt_method: read_u32_be(f)?,
-            l1_size: read_u32_be(f)?,
-            l1_table_offset: read_u64_be(f)?,
-            refcount_table_offset: read_u64_be(f)?,
-            refcount_table_clusters: read_u32_be(f)?,
-            nb_snapshots: read_u32_be(f)?,
-            snapshots_offset: read_u64_be(f)?,
-            incompatible_features: if version == 2 { 0 } else { read_u64_be(f)? },
-            compatible_features: if version == 2 { 0 } else { read_u64_be(f)? },
-            autoclear_features: if version == 2 { 0 } else { read_u64_be(f)? },
-            refcount_order: if version == 2 {
-                DEFAULT_REFCOUNT_ORDER
-            } else {
-                read_u32_be(f)?
-            },
-            header_size: if version == 2 {
-                V2_BARE_HEADER_SIZE
-            } else {
-                read_u32_be(f)?
-            },
+            backing_file_offset: v2.backing_file_offset.get(),
+            backing_file_size: v2.backing_file_size.get(),
+            cluster_bits: v2.cluster_bits.get(),
+            size: v2.size.get(),
+            crypt_method: v2.crypt_method.get(),
+            l1_size: v2.l1_size.get(),
+            l1_table_offset: v2.l1_table_offset.get(),
+            refcount_table_offset: v2.refcount_table_offset.get(),
+            refcount_table_clusters: v2.refcount_table_clusters.get(),
+            nb_snapshots: v2.nb_snapshots.get(),
+            snapshots_offset: v2.snapshots_offset.get(),
+            incompatible_features: 0,
+            compatible_features: 0,
+            autoclear_features: 0,
+            refcount_order: DEFAULT_REFCOUNT_ORDER,
+            header_size: V2_BARE_HEADER_SIZE,
             compression_type: CompressionType::Zlib,
             backing_file: None,
         };
-        if version == 3 && header.header_size > V3_BARE_HEADER_SIZE {
-            let raw_compression_type = read_u64_be(f)? >> (64 - 8);
-            header.compression_type = if raw_compression_type == COMPRESSION_TYPE_ZLIB {
-                Ok(CompressionType::Zlib)
-            } else if raw_compression_type == COMPRESSION_TYPE_ZSTD {
-                Ok(CompressionType::Zstd)
-            } else {
-                Err(Error::UnsupportedCompressionType)
-            }?;
+
+        if version != 2 {
+            let (v3, rest) =
+                RawHeaderV3Tail::ref_from_prefix(tail).expect("buffer covers the v3 header");
+            header.incompatible_features = v3.incompatible_features.get();
+            header.compatible_features = v3.compatible_features.get();
+            header.autoclear_features = v3.autoclear_features.get();
+            header.refcount_order = v3.refcount_order.get();
+            header.header_size = v3.header_size.get();
+
+            if version == 3 && header.header_size > V3_BARE_HEADER_SIZE {
+                let (compression, _) =
+                    BeU64::ref_from_prefix(rest).expect("buffer covers the compression field");
+                let raw_compression_type = compression.get() >> (64 - 8);
+                header.compression_type = if raw_compression_type == COMPRESSION_TYPE_ZLIB {
+                    Ok(CompressionType::Zlib)
+                } else if raw_compression_type == COMPRESSION_TYPE_ZSTD {
+                    Ok(CompressionType::Zstd)
+                } else {
+                    Err(Error::UnsupportedCompressionType)
+                }?;
+            }
         }
         if header.backing_file_size > MAX_BACKING_FILE_SIZE {
             return Err(Error::BackingFileTooLong(header.backing_file_size as usize));
         }
+        if header.backing_file_offset == 0 && header.backing_file_size != 0 {
+            return Err(Error::BackingFileSizeWithoutOffset(
+                header.backing_file_size,
+            ));
+        }
+        if header.backing_file_offset != 0 && header.backing_file_size == 0 {
+            return Err(Error::BackingFileOffsetWithoutSize(
+                header.backing_file_offset,
+            ));
+        }
         if header.backing_file_offset != 0 {
-            f.seek(SeekFrom::Start(header.backing_file_offset))
-                .map_err(Error::ReadingHeader)?;
+            let cluster_size = 1u64
+                .checked_shl(header.cluster_bits)
+                .ok_or(Error::InvalidClusterSize)?;
+            if header.backing_file_offset < u64::from(header.header_size) {
+                return Err(Error::BackingFileOverlapsHeader(
+                    header.backing_file_offset,
+                    header.backing_file_size,
+                    header.header_size,
+                ));
+            }
+            if header.backing_file_offset >= cluster_size
+                || header.backing_file_offset + u64::from(header.backing_file_size) > cluster_size
+            {
+                return Err(Error::BackingFileOutsideFirstCluster(
+                    header.backing_file_offset,
+                    header.backing_file_size,
+                    cluster_size,
+                ));
+            }
             let mut backing_file_name_bytes = vec![0u8; header.backing_file_size as usize];
-            f.read_exact(&mut backing_file_name_bytes)
+            f.read_exact_at(&mut backing_file_name_bytes, header.backing_file_offset)
                 .map_err(Error::ReadingHeader)?;
             let path = String::from_utf8(backing_file_name_bytes)
                 .map_err(|err| Error::InvalidBackingFileName(err.utf8_error()))?;
@@ -448,78 +564,65 @@ impl QcowHeader {
         })
     }
 
-    /// Write the header to `file`.
-    pub fn write_to<F: Write + Seek>(&self, file: &mut F) -> Result<()> {
-        // Writes the next u32 to the file.
-        fn write_u32_be<F: Write>(f: &mut F, value: u32) -> Result<()> {
-            u32::write_be(f, value).map_err(Error::WritingHeader)
-        }
-
-        // Writes the next u64 to the file.
-        fn write_u64_be<F: Write>(f: &mut F, value: u64) -> Result<()> {
-            u64::write_be(f, value).map_err(Error::WritingHeader)
-        }
-
-        write_u32_be(file, self.magic)?;
-        write_u32_be(file, self.version)?;
-        write_u64_be(file, self.backing_file_offset)?;
-        write_u32_be(file, self.backing_file_size)?;
-        write_u32_be(file, self.cluster_bits)?;
-        write_u64_be(file, self.size)?;
-        write_u32_be(file, self.crypt_method)?;
-        write_u32_be(file, self.l1_size)?;
-        write_u64_be(file, self.l1_table_offset)?;
-        write_u64_be(file, self.refcount_table_offset)?;
-        write_u32_be(file, self.refcount_table_clusters)?;
-        write_u32_be(file, self.nb_snapshots)?;
-        write_u64_be(file, self.snapshots_offset)?;
+    /// Write the header to `f`.
+    pub fn write_to(&self, f: &AlignedFile) -> Result<()> {
+        // Build the header in memory, then write it in one positional write.
+        let mut buf = Vec::new();
+        let v2 = RawHeaderV2::from_header(self);
+        buf.extend_from_slice(v2.as_bytes());
 
         if self.version == 3 {
-            write_u64_be(file, self.incompatible_features)?;
-            write_u64_be(file, self.compatible_features)?;
-            write_u64_be(file, self.autoclear_features)?;
-            write_u32_be(file, self.refcount_order)?;
-            write_u32_be(file, self.header_size)?;
+            let v3 = RawHeaderV3Tail::from_header(self);
+            buf.extend_from_slice(v3.as_bytes());
 
             if self.header_size > V3_BARE_HEADER_SIZE {
-                write_u64_be(file, 0)?; // no compression
+                let compression_type = match &self.compression_type {
+                    CompressionType::Zlib => COMPRESSION_TYPE_ZLIB,
+                    CompressionType::Zstd => COMPRESSION_TYPE_ZSTD,
+                };
+                let compression_type = BeU64::new(compression_type << (64 - 8));
+                buf.extend_from_slice(compression_type.as_bytes());
             }
 
-            write_u32_be(file, 0)?; // header extension type: end of header extension area
-            write_u32_be(file, 0)?; // length of header extension data: 0
+            let end_extension = ExtensionHeader::end();
+            buf.extend_from_slice(end_extension.as_bytes());
         }
 
+        f.write_all_at(&buf, 0).map_err(Error::WritingHeader)?;
+
         if let Some(backing_file_path) = self.backing_file.as_ref().map(|bf| &bf.path) {
-            if self.backing_file_offset > 0 {
-                file.seek(SeekFrom::Start(self.backing_file_offset))
-                    .map_err(Error::WritingHeader)?;
-            }
-            write!(file, "{backing_file_path}").map_err(Error::WritingHeader)?;
+            let offset = if self.backing_file_offset > 0 {
+                self.backing_file_offset
+            } else {
+                buf.len() as u64
+            };
+            f.write_all_at(backing_file_path.as_bytes(), offset)
+                .map_err(Error::WritingHeader)?;
         }
 
-        // Set the file length by seeking and writing a zero to the last byte. This avoids needing
-        // a `File` instead of anything that implements seek as the `file` argument.
-        // Zeros out the l1 and refcount table clusters.
+        // Set the file length by writing a zero to the last byte. This also
+        // zeros the l1 and refcount table clusters.
         let cluster_size = 0x01u64 << self.cluster_bits;
         let refcount_blocks_size = u64::from(self.refcount_table_clusters) * cluster_size;
-        file.seek(SeekFrom::Start(
+        f.write_all_at(
+            &[0u8],
             self.refcount_table_offset + refcount_blocks_size - 2,
-        ))
+        )
         .map_err(Error::WritingHeader)?;
-        file.write(&[0u8]).map_err(Error::WritingHeader)?;
 
         Ok(())
     }
 
     /// Write only the incompatible_features field to the file at its fixed offset.
-    fn write_incompatible_features<F: Seek + Write>(&self, file: &mut F) -> BlockResult<()> {
+    fn write_incompatible_features(&self, file: &AlignedFile) -> BlockResult<()> {
         if self.version != 3 {
             return Ok(());
         }
-        file.seek(SeekFrom::Start(V2_BARE_HEADER_SIZE as u64))
-            .map_err(|e| BlockError::new(BlockErrorKind::Io, Error::WritingHeader(e)))?;
-        u64::write_be(file, self.incompatible_features)
-            .map_err(|e| BlockError::new(BlockErrorKind::Io, Error::WritingHeader(e)))?;
+        file.write_all_at(
+            &self.incompatible_features.to_be_bytes(),
+            V2_BARE_HEADER_SIZE as u64,
+        )
+        .map_err(|e| BlockError::new(BlockErrorKind::Io, Error::WritingHeader(e)))?;
         Ok(())
     }
 
@@ -527,11 +630,7 @@ impl QcowHeader {
     ///
     /// When `dirty` is true, sets the bit to indicate the image is in use.
     /// When `dirty` is false, clears the bit to indicate a clean shutdown.
-    pub fn set_dirty_bit<F: Seek + Write + FileSync>(
-        &mut self,
-        file: &mut F,
-        dirty: bool,
-    ) -> BlockResult<()> {
+    pub fn set_dirty_bit(&mut self, file: &mut AlignedFile, dirty: bool) -> BlockResult<()> {
         if self.version == 3 {
             if dirty {
                 self.incompatible_features |= IncompatFeatures::DIRTY.bits();
@@ -549,7 +648,7 @@ impl QcowHeader {
     ///
     /// This marks the image as corrupted. Once set, the image can only be
     /// opened read-only until repaired.
-    pub fn set_corrupt_bit<F: Seek + Write + FileSync>(&mut self, file: &mut F) -> BlockResult<()> {
+    pub fn set_corrupt_bit(&mut self, file: &mut AlignedFile) -> BlockResult<()> {
         if self.version == 3 {
             self.incompatible_features |= IncompatFeatures::CORRUPT.bits();
             self.write_incompatible_features(file)?;
@@ -568,15 +667,11 @@ impl QcowHeader {
     ///
     /// These bits indicate features that can be safely disabled when modified
     /// by software that doesn't understand them.
-    pub fn clear_autoclear_features<F: Seek + Write + FileSync>(
-        &mut self,
-        file: &mut F,
-    ) -> Result<()> {
+    pub fn clear_autoclear_features(&mut self, file: &mut AlignedFile) -> Result<()> {
         if self.version == 3 && self.autoclear_features != 0 {
             self.autoclear_features = 0;
-            file.seek(SeekFrom::Start(AUTOCLEAR_FEATURES_OFFSET))
+            file.write_all_at(&0u64.to_be_bytes(), AUTOCLEAR_FEATURES_OFFSET)
                 .map_err(Error::WritingHeader)?;
-            u64::write_be(file, 0).map_err(Error::WritingHeader)?;
             file.fsync().map_err(Error::SyncingHeader)?;
         }
         Ok(())

block/src/formats/qcow/metadata.rs

@@ -17,11 +17,12 @@
 //! operations upgrade to a write lock.
 
 use std::cmp::min;
-use std::io::{self, Seek};
-use std::mem;
+use std::os::unix::fs::FileExt;
 use std::sync::{Arc, RwLock};
+use std::{io, mem};
 
 use libc::{EINVAL, EIO};
+use vmm_sys_util::write_zeroes::WriteZeroesAt;
 
 use super::decoder::Decoder;
 use super::qcow_raw_file::QcowRawFile;
@@ -29,7 +30,7 @@ use super::refcount::RefCount;
 use super::util::{
     div_round_up_u64, l1_entry_make, l2_entry_compressed_cluster_layout, l2_entry_is_compressed,
     l2_entry_is_empty, l2_entry_is_zero, l2_entry_make_std, l2_entry_make_zero,
-    l2_entry_std_cluster_addr,
+    l2_entry_make_zero_plain, l2_entry_std_cluster_addr,
 };
 use super::vec_cache::{CacheMap, Cacheable, VecCache};
 use super::{QcowHeader, refcount};
@@ -40,7 +41,7 @@ use super::{QcowHeader, refcount};
 /// the actual data I/O using its own per queue file descriptor without
 /// holding the metadata lock.
 #[derive(Debug)]
-pub enum ClusterReadMapping {
+pub(super) enum ClusterReadMapping {
     /// The cluster is not allocated and the guest should see zeros.
     /// This covers both truly unallocated clusters where the L1 or L2
     /// entry is zero and clusters with the ZERO flag set.
@@ -75,7 +76,7 @@ pub enum ClusterReadMapping {
 /// the actual data I/O using its own per queue file descriptor without
 /// holding the metadata lock.
 #[derive(Debug)]
-pub enum ClusterWriteMapping {
+pub(super) enum ClusterWriteMapping {
     /// The write target is at the given host file offset.
     /// This covers both already allocated clusters and freshly allocated ones.
     /// The offset is the exact byte position combining cluster base and
@@ -93,7 +94,7 @@ pub(crate) trait BackingRead: Send + Sync {
 
 /// Action that the caller must perform after deallocate_bytes.
 #[derive(Debug)]
-pub enum DeallocAction {
+pub(super) enum DeallocAction {
     /// Punch a hole at the given host file offset for a full cluster.
     PunchHole { host_offset: u64, length: u64 },
     /// Write zeros at the given host file offset for a partial cluster.
@@ -114,7 +115,7 @@ pub enum DeallocAction {
 /// L1 to L2 lookup, which completes under a shared read lock. Only
 /// cluster allocation, L2 cache eviction and resize take the exclusive
 /// write lock, so contention stays low and queues scale.
-pub struct QcowMetadata {
+pub(super) struct QcowMetadata {
     inner: RwLock<QcowState>,
     decoder: Arc<dyn Decoder>,
 }
@@ -159,7 +160,7 @@ impl QcowMetadata {
     ///
     /// The has_backing_file flag indicates whether a backing file exists,
     /// needed to distinguish zero versus backing for unallocated clusters.
-    pub fn map_clusters_for_read(
+    pub(super) fn map_clusters_for_read(
         &self,
         address: u64,
         total_length: usize,
@@ -227,7 +228,7 @@ impl QcowMetadata {
     /// unallocated and a backing file exists, the caller should have already
     /// read the backing cluster data and pass it here. If None, the new
     /// cluster is zeroed.
-    pub fn map_cluster_for_write(
+    pub(super) fn map_cluster_for_write(
         &self,
         address: u64,
         backing_data: Option<Vec<u8>>,
@@ -236,7 +237,7 @@ impl QcowMetadata {
         inner.map_write(address, backing_data)
     }
 
-    pub fn flush(&self) -> io::Result<()> {
+    pub(super) fn flush(&self) -> io::Result<()> {
         let mut inner = self.inner.write().unwrap();
         inner.sync_caches()?;
         let mut unref = mem::take(&mut inner.unref_clusters);
@@ -246,7 +247,7 @@ impl QcowMetadata {
 
     /// Flushes dirty metadata caches and clears the dirty bit for
     /// clean shutdown.
-    pub fn shutdown(&self) {
+    pub(super) fn shutdown(&self) {
         let mut inner = self.inner.write().unwrap();
         let _ = inner.sync_caches();
         let QcowState {
@@ -264,21 +265,22 @@ impl QcowMetadata {
     /// clusters beyond the new size and risks data loss.
     ///
     /// Returns an error if the new size is smaller than the current size.
-    pub fn resize(&self, new_size: u64) -> io::Result<()> {
+    pub(super) fn resize(&self, new_size: u64) -> io::Result<()> {
         let mut inner = self.inner.write().unwrap();
         inner.resize(new_size)
     }
 
     /// Deallocates a range of bytes. Full clusters are deallocated via metadata.
-    /// Partial clusters need the caller to write zeros. This method returns a
-    /// list of actions the caller should take.
+    /// If `zero_marker` is true, full-cluster deallocation records a logical
+    /// zero instead of an empty entry where backing data could otherwise be
+    /// exposed. Partial clusters need the caller to write zeros. This method
+    /// returns a list of actions the caller should take.
     pub(crate) fn deallocate_bytes(
         &self,
         address: u64,
         length: usize,
         sparse: bool,
-        virtual_size: u64,
-        cluster_size: u64,
+        zero_marker: bool,
         backing_file: Option<&dyn BackingRead>,
     ) -> io::Result<Vec<DeallocAction>> {
         if address.checked_add(length as u64).is_none() {
@@ -287,7 +289,8 @@ impl QcowMetadata {
         let mut inner = self.inner.write().unwrap();
         let mut actions = Vec::new();
 
-        let file_end = virtual_size;
+        let file_end = inner.header.size;
+        let cluster_size = inner.raw_file.cluster_size();
         let remaining_in_file = file_end.saturating_sub(address);
         let write_count = min(length as u64, remaining_in_file) as usize;
 
@@ -301,7 +304,11 @@ impl QcowMetadata {
             );
 
             if count == cluster_size as usize {
-                let punch_offset = inner.deallocate_cluster(curr_addr, sparse)?;
+                let punch_offset = inner.deallocate_cluster(
+                    curr_addr,
+                    sparse,
+                    zero_marker && backing_file.is_some(),
+                )?;
                 if let Some(host_offset) = punch_offset {
                     actions.push(DeallocAction::PunchHole {
                         host_offset,
@@ -332,18 +339,36 @@ impl QcowMetadata {
         Ok(actions)
     }
 
-    pub fn virtual_size(&self) -> u64 {
+    pub(super) fn virtual_size(&self) -> u64 {
         self.inner.read().unwrap().header.size
     }
 
-    pub fn cluster_size(&self) -> u64 {
+    pub(super) fn cluster_size(&self) -> u64 {
         self.inner.read().unwrap().raw_file.cluster_size()
     }
 
     /// Returns the shared decoder matching the image compression type.
-    pub fn decoder(&self) -> Arc<dyn Decoder> {
+    pub(super) fn decoder(&self) -> Arc<dyn Decoder> {
         Arc::clone(&self.decoder)
     }
+
+    #[cfg(test)]
+    pub fn header(&self) -> QcowHeader {
+        self.inner.read().unwrap().header.clone()
+    }
+
+    #[cfg(test)]
+    pub fn cluster_refcount(&self, address: u64) -> io::Result<u64> {
+        let mut inner = self.inner.write().unwrap();
+        let QcowState {
+            refcounts,
+            raw_file,
+            ..
+        } = &mut *inner;
+        refcounts
+            .get_cluster_refcount(raw_file, address)
+            .map_err(|e| io::Error::other(format!("get_cluster_refcount: {e}")))
+    }
 }
 
 impl QcowState {
@@ -406,18 +431,12 @@ impl QcowState {
                 has_backing_file,
             )))
         } else if l2_entry_is_zero(l2_entry) {
-            // Match original QcowFile::file_read semantics where zero flagged
-            // entries fall through to backing file when one exists or return
-            // zeros otherwise.
-            Ok(Some(self.unallocated_read_mapping(
-                address,
-                count,
-                has_backing_file,
-            )))
+            Ok(Some(ClusterReadMapping::Zero {
+                length: count as u64,
+            }))
         } else {
             let cluster_addr = l2_entry_std_cluster_addr(l2_entry);
-            let cluster_size = self.raw_file.cluster_size();
-            if cluster_addr & (cluster_size - 1) != 0 {
+            if !self.is_refcount_addressable_cluster_offset(cluster_addr) {
                 // Fall through to write lock path which sets the corrupt bit
                 return Ok(None);
             }
@@ -469,17 +488,12 @@ impl QcowState {
                 length: count,
             })
         } else if l2_entry_is_zero(l2_entry) {
-            // Match original QcowFile::file_read semantics where zero flagged
-            // entries fall through to backing file when one exists or return
-            // zeros otherwise.
-            Ok(self.unallocated_read_mapping(address, count, has_backing_file))
+            Ok(ClusterReadMapping::Zero {
+                length: count as u64,
+            })
         } else {
             let cluster_addr = l2_entry_std_cluster_addr(l2_entry);
-            let cluster_size = self.raw_file.cluster_size();
-            if cluster_addr & (cluster_size - 1) != 0 {
-                self.set_corrupt_bit_best_effort();
-                return Err(io::Error::from_raw_os_error(EIO));
-            }
+            self.reject_invalid_cluster_offset(cluster_addr)?;
             let intra_offset = self.raw_file.cluster_offset(address);
             Ok(ClusterReadMapping::Allocated {
                 offset: cluster_addr + intra_offset,
@@ -506,19 +520,6 @@ impl QcowState {
         }
     }
 
-    /// Maps a single cluster region for a sequential read.
-    pub(crate) fn map_cluster_read(
-        &mut self,
-        address: u64,
-        count: usize,
-        has_backing_file: bool,
-    ) -> io::Result<ClusterReadMapping> {
-        match self.try_map_read(address, count, has_backing_file)? {
-            Some(mapping) => Ok(mapping),
-            None => self.map_read_with_populate(address, count, has_backing_file),
-        }
-    }
-
     /// Write path mapping. Always called under write lock.
     fn map_write(
         &mut self,
@@ -547,10 +548,10 @@ impl QcowState {
             let decompressed_cluster = self.decompress_l2_cluster(l2_entry)?;
             let cluster_addr = self.append_data_cluster(None)?;
             self.update_cluster_addr(l1_index, l2_index, cluster_addr, &mut set_refcounts)?;
-            self.raw_file
+            let nwritten = self
+                .raw_file
                 .file_mut()
-                .seek(io::SeekFrom::Start(cluster_addr))?;
-            let nwritten = io::Write::write(self.raw_file.file_mut(), &decompressed_cluster)?;
+                .write_at(&decompressed_cluster, cluster_addr)?;
             if nwritten != decompressed_cluster.len() {
                 self.set_corrupt_bit_best_effort();
                 return Err(io::Error::from_raw_os_error(EIO));
@@ -558,16 +559,16 @@ impl QcowState {
             self.deallocate_compressed_cluster(l2_entry)?;
             cluster_addr
         } else if l2_entry_is_empty(l2_entry) || l2_entry_is_zero(l2_entry) {
-            let cluster_addr = self.append_data_cluster(backing_data)?;
+            let cluster_addr = if l2_entry_is_zero(l2_entry) {
+                self.append_zeroed_data_cluster()?
+            } else {
+                self.append_data_cluster(backing_data)?
+            };
             self.update_cluster_addr(l1_index, l2_index, cluster_addr, &mut set_refcounts)?;
             cluster_addr
         } else {
-            // Already allocated - validate alignment
             let cluster_addr = l2_entry_std_cluster_addr(l2_entry);
-            if cluster_addr & (self.raw_file.cluster_size() - 1) != 0 {
-                self.set_corrupt_bit_best_effort();
-                return Err(io::Error::from_raw_os_error(EIO));
-            }
+            self.reject_invalid_cluster_offset(cluster_addr)?;
             cluster_addr
         };
 
@@ -592,16 +593,26 @@ impl QcowState {
         (address / self.raw_file.cluster_size()) % self.l2_entries
     }
 
+    fn is_refcount_addressable_cluster_offset(&self, cluster_addr: u64) -> bool {
+        cluster_addr & (self.raw_file.cluster_size() - 1) == 0
+            && cluster_addr <= self.refcounts.max_valid_cluster_offset()
+    }
+
+    fn reject_invalid_cluster_offset(&mut self, cluster_addr: u64) -> io::Result<()> {
+        if self.is_refcount_addressable_cluster_offset(cluster_addr) {
+            Ok(())
+        } else {
+            self.set_corrupt_bit_best_effort();
+            Err(io::Error::from_raw_os_error(EIO))
+        }
+    }
+
     // -- Cache and allocation operations requiring exclusive access --
 
     /// Populates the L2 cache for read operations without allocation.
     fn cache_l2_cluster(&mut self, l1_index: usize, l2_addr_disk: u64) -> io::Result<()> {
         if !self.l2_cache.contains_key(l1_index) {
-            let cluster_size = self.raw_file.cluster_size();
-            if l2_addr_disk & (cluster_size - 1) != 0 {
-                self.set_corrupt_bit_best_effort();
-                return Err(io::Error::from_raw_os_error(EIO));
-            }
+            self.reject_invalid_cluster_offset(l2_addr_disk)?;
             let l2_table =
                 VecCache::from_vec(self.raw_file.read_pointer_cluster(l2_addr_disk, None)?);
             let l1_table = &self.l1_table;
@@ -629,11 +640,7 @@ impl QcowState {
                 self.l1_table[l1_index] = new_addr;
                 VecCache::new(self.l2_entries as usize)
             } else {
-                let cluster_size = self.raw_file.cluster_size();
-                if l2_addr_disk & (cluster_size - 1) != 0 {
-                    self.set_corrupt_bit_best_effort();
-                    return Err(io::Error::from_raw_os_error(EIO));
-                }
+                self.reject_invalid_cluster_offset(l2_addr_disk)?;
                 VecCache::from_vec(self.raw_file.read_pointer_cluster(l2_addr_disk, None)?)
             };
             let l1_table = &self.l1_table;
@@ -683,6 +690,17 @@ impl QcowState {
         Ok(new_addr)
     }
 
+    /// Allocates a data cluster and zeroes it without building a cluster-sized buffer.
+    fn append_zeroed_data_cluster(&mut self) -> io::Result<u64> {
+        let new_addr = self.get_new_cluster(None)?;
+        let cluster_size = self.raw_file.cluster_size() as usize;
+        self.raw_file
+            .file_mut()
+            .write_zeroes_at(new_addr, cluster_size)?;
+        self.set_cluster_refcount_track_freed(new_addr, 1)?;
+        Ok(new_addr)
+    }
+
     /// Updates the L1 and L2 tables to point to a new cluster address.
     fn update_cluster_addr(
         &mut self,
@@ -737,7 +755,6 @@ impl QcowState {
 
         self.header.size = new_size;
 
-        self.raw_file.file_mut().rewind()?;
         self.header
             .write_to(self.raw_file.file_mut())
             .map_err(|e| io::Error::other(format!("failed to write header during resize: {e}")))?;
@@ -757,7 +774,7 @@ impl QcowState {
         let new_l1_clusters = div_round_up_u64(new_l1_bytes, cluster_size);
 
         // Allocate contiguous clusters at file end for new L1 table
-        let file_size = self.raw_file.file_mut().seek(io::SeekFrom::End(0))?;
+        let file_size = self.raw_file.physical_size()?;
         let new_l1_offset = self.raw_file.cluster_address(file_size + cluster_size - 1);
 
         let new_file_end = new_l1_offset + new_l1_clusters * cluster_size;
@@ -793,7 +810,6 @@ impl QcowState {
         self.header.l1_size = new_l1_size;
         self.header.l1_table_offset = new_l1_offset;
 
-        self.raw_file.file_mut().rewind()?;
         self.header
             .write_to(self.raw_file.file_mut())
             .map_err(|e| io::Error::other(format!("failed to write header during resize: {e}")))?;
@@ -821,13 +837,15 @@ impl QcowState {
     /// If sparse is true, fully deallocates and returns the host offset if
     /// the underlying storage should be punched after the refcount dropped
     /// to zero. If sparse is false, uses the zero flag optimization when
-    /// possible.
+    /// possible. If `zero_marker` is true, empty entries are replaced with
+    /// logical-zero entries so reads do not fall through to backing data.
     ///
     /// Returns None if no host punch_hole is needed.
     pub(super) fn deallocate_cluster(
         &mut self,
         address: u64,
         sparse: bool,
+        zero_marker: bool,
     ) -> io::Result<Option<u64>> {
         if address >= self.header.size {
             return Err(io::Error::from_raw_os_error(EINVAL));
@@ -839,25 +857,44 @@ impl QcowState {
             None => return Err(io::Error::from_raw_os_error(EINVAL)),
         };
         let l2_index = self.l2_table_index(address) as usize;
+        let dealloc_entry = if zero_marker {
+            l2_entry_make_zero_plain()
+        } else {
+            0
+        };
 
         if l2_addr_disk == 0 {
+            if zero_marker {
+                if let Some(new_addr) = self.cache_l2_cluster_alloc(l1_index, l2_addr_disk)? {
+                    self.set_cluster_refcount_track_freed(new_addr, 1)?;
+                }
+                self.l2_cache.get_mut(l1_index).unwrap()[l2_index] = dealloc_entry;
+            }
             return Ok(None);
         }
 
         self.cache_l2_cluster(l1_index, l2_addr_disk)?;
 
         let l2_entry = self.l2_cache.get(l1_index).unwrap()[l2_index];
-        if l2_entry_is_empty(l2_entry) || l2_entry_is_zero(l2_entry) {
+        if l2_entry_is_empty(l2_entry) {
+            if zero_marker {
+                self.l2_cache.get_mut(l1_index).unwrap()[l2_index] = dealloc_entry;
+            }
             return Ok(None);
         }
-
+        // Compressed entries may use bit 0 as part of their layout, so they
+        // must be classified before zero-flagged standard entries.
         if l2_entry_is_compressed(l2_entry) {
             self.deallocate_compressed_cluster(l2_entry)?;
-            self.l2_cache.get_mut(l1_index).unwrap()[l2_index] = 0;
+            self.l2_cache.get_mut(l1_index).unwrap()[l2_index] = dealloc_entry;
+            return Ok(None);
+        }
+        if l2_entry_is_zero(l2_entry) {
             return Ok(None);
         }
 
         let cluster_addr = l2_entry_std_cluster_addr(l2_entry);
+        self.reject_invalid_cluster_offset(cluster_addr)?;
         let refcount = self
             .refcounts
             .get_cluster_refcount(&mut self.raw_file, cluster_addr)
@@ -877,7 +914,7 @@ impl QcowState {
         if sparse {
             let new_refcount = refcount - 1;
             self.set_cluster_refcount_track_freed(cluster_addr, new_refcount)?;
-            self.l2_cache.get_mut(l1_index).unwrap()[l2_index] = 0;
+            self.l2_cache.get_mut(l1_index).unwrap()[l2_index] = dealloc_entry;
             if new_refcount == 0 {
                 self.unref_clusters.push(cluster_addr);
                 return Ok(Some(cluster_addr));
@@ -886,7 +923,7 @@ impl QcowState {
             self.l2_cache.get_mut(l1_index).unwrap()[l2_index] = l2_entry_make_zero(cluster_addr);
         } else {
             self.set_cluster_refcount_track_freed(cluster_addr, refcount - 1)?;
-            self.l2_cache.get_mut(l1_index).unwrap()[l2_index] = 0;
+            self.l2_cache.get_mut(l1_index).unwrap()[l2_index] = dealloc_entry;
         }
         Ok(None)
     }
@@ -1013,11 +1050,10 @@ impl QcowState {
     fn decompress_l2_cluster(&mut self, l2_entry: u64) -> io::Result<Vec<u8>> {
         let (compressed_addr, compressed_size) =
             l2_entry_compressed_cluster_layout(l2_entry, self.header.cluster_bits);
+        let mut compressed = vec![0u8; compressed_size];
         self.raw_file
             .file_mut()
-            .seek(io::SeekFrom::Start(compressed_addr))?;
-        let mut compressed = vec![0u8; compressed_size];
-        io::Read::read_exact(self.raw_file.file_mut(), &mut compressed)?;
+            .read_exact_at(&mut compressed, compressed_addr)?;
         let decoder = self.header.get_decoder();
         let cluster_size = self.raw_file.cluster_size() as usize;
         let mut decompressed = vec![0u8; cluster_size];

block/src/formats/qcow/mod.rs

@@ -2,21 +2,56 @@
 //
 // SPDX-License-Identifier: Apache-2.0
 
+//! QCOW2 disk image format.
+//!
+//! Provides [`QcowDisk`], the `DiskFile` wrapper for QCOW2 images
+//! with backing file and compression support.
+
+mod backing;
+mod common;
+mod decoder;
+mod engine_sync;
+#[cfg(feature = "io_uring")]
+mod engine_uring;
+mod header;
+mod metadata;
+mod parser;
+mod qcow_raw_file;
+mod refcount;
+mod util;
+mod vec_cache;
+
 use std::fs::File;
 use std::os::unix::io::AsRawFd;
+#[cfg(any(test, feature = "test-utils"))]
+use std::path::Path;
 use std::sync::Arc;
 use std::{fmt, io};
 
+pub use parser::{
+    BackingFileConfig, CompressionType, Error, ImageType, IncompatFeatures, MissingFeatureError,
+    QcowHeader,
+};
+#[cfg(any(test, feature = "test-utils"))]
+use vm_memory::{Bytes, GuestAddress, GuestMemoryMmap};
+#[cfg(any(test, feature = "test-utils"))]
+use vmm_sys_util::tempfile::TempFile;
+
+use self::backing::shared_backing_from;
+use self::engine_sync::QcowSync;
+#[cfg(feature = "io_uring")]
+use self::engine_uring::QcowAsync;
+use self::metadata::{BackingRead, QcowMetadata};
+use self::parser::{MAX_NESTING_DEPTH, parse_qcow};
+use self::qcow_raw_file::QcowRawFile;
+use crate::aligned_file::AlignedFile;
+#[cfg(any(test, feature = "test-utils"))]
+use crate::async_io::GuestMemoryTarget;
 use crate::async_io::{AsyncIo, BorrowedDiskFd, DiskFileError};
 use crate::disk_file;
+#[cfg(any(test, feature = "test-utils"))]
+use crate::disk_file::AsyncDiskFile;
 use crate::error::{BlockError, BlockErrorKind, BlockResult, ErrorOp};
-use crate::qcow::backing::shared_backing_from;
-use crate::qcow::metadata::{BackingRead, QcowMetadata};
-use crate::qcow::qcow_raw_file::QcowRawFile;
-use crate::qcow::{MAX_NESTING_DEPTH, RawFile, parse_qcow};
-#[cfg(feature = "io_uring")]
-use crate::qcow_async::QcowAsync;
-use crate::qcow_sync::QcowSync;
 
 /// Unified DiskFile wrapper for QCOW2 disk images.
 ///
@@ -67,7 +102,7 @@ impl QcowDisk {
         }
 
         let max_nesting_depth = if backing_files { MAX_NESTING_DEPTH } else { 0 };
-        let raw_file = RawFile::new(file, direct_io);
+        let raw_file = AlignedFile::new(file, direct_io);
         let (inner, backing_file, sparse) = parse_qcow(raw_file, max_nesting_depth, sparse)
             .map_err(|e| {
                 let e = if !backing_files && matches!(e.kind(), BlockErrorKind::Overflow) {
@@ -86,6 +121,123 @@ impl QcowDisk {
             use_io_uring,
         })
     }
+
+    /// Synchronous write convenience for tests and benchmarks.
+    #[cfg(any(test, feature = "test-utils"))]
+    pub fn write_all_at(&self, offset: u64, data: &[u8]) {
+        let mut async_io = self.create_async_io(1).unwrap();
+        let mem =
+            Arc::new(GuestMemoryMmap::<()>::from_ranges(&[(GuestAddress(0), data.len())]).unwrap());
+        mem.write_slice(data, GuestAddress(0)).unwrap();
+        let range = [(GuestAddress(0), data.len() as u32)];
+        let target = GuestMemoryTarget::new(Arc::clone(&mem), &range).unwrap();
+        async_io
+            .write_from_memory(offset as libc::off_t, target, 0)
+            .unwrap();
+        while async_io.next_completed_request().is_some() {}
+    }
+
+    /// Synchronous read convenience for tests and benchmarks.
+    #[cfg(test)]
+    pub fn read_all_at(&self, offset: u64, len: usize) -> Vec<u8> {
+        let mut async_io = self.create_async_io(1).unwrap();
+        let mem = Arc::new(GuestMemoryMmap::<()>::from_ranges(&[(GuestAddress(0), len)]).unwrap());
+        let range = [(GuestAddress(0), len as u32)];
+        let target = GuestMemoryTarget::new(Arc::clone(&mem), &range).unwrap();
+        async_io
+            .read_to_memory(offset as libc::off_t, target, 0)
+            .unwrap();
+        while async_io.next_completed_request().is_some() {}
+        let mut buf = vec![0u8; len];
+        mem.read_slice(&mut buf, GuestAddress(0)).unwrap();
+        buf
+    }
+
+    #[cfg(test)]
+    fn metadata(&self) -> &QcowMetadata {
+        &self.metadata
+    }
+}
+
+/// Writes a fresh qcow2 layout into `file`
+#[cfg(any(test, feature = "test-utils"))]
+pub(crate) fn create_image(
+    file: &File,
+    virtual_size: u64,
+    backing_config: Option<&BackingFileConfig>,
+) -> BlockResult<()> {
+    let path = backing_config.map(|cfg| cfg.path.as_str());
+    let mut header = QcowHeader::create_for_size_and_path(3, virtual_size, path)
+        .map_err(|e| BlockError::new(BlockErrorKind::Io, e))?;
+    if let Some(cfg) = backing_config
+        && let Some(backing_file) = &mut header.backing_file
+    {
+        backing_file.format = cfg.format;
+    }
+    let raw = AlignedFile::new(
+        file.try_clone()
+            .map_err(|e| BlockError::new(BlockErrorKind::Io, DiskFileError::Clone(e)))?,
+        false,
+    );
+    header
+        .write_to(&raw)
+        .map_err(|e| BlockError::new(BlockErrorKind::Io, e))?;
+    let (inner, _backing, _sparse) = parse_qcow(raw, MAX_NESTING_DEPTH, true)?;
+    // Flush dirty caches and clear the dirty bit
+    QcowMetadata::new(inner).shutdown();
+    Ok(())
+}
+
+/// Helper struct to create a new qcow2 image in a temporary file.
+#[cfg(any(test, feature = "test-utils"))]
+pub struct QcowTempDisk {
+    tmp: TempFile,
+    disk: QcowDisk,
+}
+
+#[cfg(any(test, feature = "test-utils"))]
+impl QcowTempDisk {
+    /// Creates a new qcow2 image in a temporary file with optional
+    /// backing file. Flags are passed to QcowDisk::new.
+    pub fn new(
+        virtual_size: u64,
+        backing_config: Option<&BackingFileConfig>,
+        direct_io: bool,
+        sparse: bool,
+        use_io_uring: bool,
+    ) -> BlockResult<Self> {
+        let tmp = TempFile::new().map_err(io::Error::from)?;
+        create_image(tmp.as_file(), virtual_size, backing_config)?;
+        let file = tmp
+            .as_file()
+            .try_clone()
+            .map_err(|e| BlockError::new(BlockErrorKind::Io, DiskFileError::Clone(e)))?;
+        let disk = QcowDisk::new(
+            file,
+            direct_io,
+            backing_config.is_some(),
+            sparse,
+            use_io_uring,
+        )?;
+        Ok(Self { tmp, disk })
+    }
+
+    pub fn path(&self) -> &Path {
+        self.tmp.as_path()
+    }
+
+    pub fn as_file(&self) -> &File {
+        self.tmp.as_file()
+    }
+
+    pub fn disk(&self) -> &QcowDisk {
+        &self.disk
+    }
+
+    /// Drops the disk handle and returns the underlying TempFile.
+    pub fn into_tempfile(self) -> TempFile {
+        self.tmp
+    }
 }
 
 impl Drop for QcowDisk {
@@ -189,22 +341,17 @@ impl disk_file::AsyncDiskFile for QcowDisk {
 
 #[cfg(test)]
 mod unit_tests {
-    use vmm_sys_util::tempfile::TempFile;
-
     use super::*;
     use crate::async_io::AsyncIo;
     use crate::disk_file::{AsyncDiskFile, DiskSize, PhysicalSize};
-    use crate::qcow::{QcowFile, RawFile};
 
     const TEST_SIZE: u64 = 0x5566_7788;
 
     fn make_qcow_file() -> File {
-        let temp_file = TempFile::new().unwrap();
-        {
-            let raw = RawFile::new(temp_file.as_file().try_clone().unwrap(), false);
-            QcowFile::new(raw, 3, TEST_SIZE, true).unwrap();
-        }
-        temp_file.into_file()
+        QcowTempDisk::new(TEST_SIZE, None, false, true, false)
+            .unwrap()
+            .into_tempfile()
+            .into_file()
     }
 
     #[test]

block/src/formats/qcow/qcow_raw_file.rs

@@ -0,0 +1,557 @@
+// Copyright 2018 The Chromium OS Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE-BSD-3-Clause file.
+//
+// SPDX-License-Identifier: Apache-2.0 AND BSD-3-Clause
+
+use std::fmt::Debug;
+use std::io::{self, Write};
+use std::os::fd::{AsFd, AsRawFd, BorrowedFd, RawFd};
+use std::os::unix::fs::FileExt;
+
+use byteorder::{BigEndian, WriteBytesExt};
+use vmm_sys_util::write_zeroes::WriteZeroesAt;
+
+use crate::aligned_file::AlignedFile;
+
+// Type aliases for the refcount read/write function pointers
+type RefcountReader = fn(&mut AlignedFile, u64, usize) -> io::Result<Vec<u64>>;
+type RefcountWriter = fn(&mut AlignedFile, u64, &[u64]) -> io::Result<()>;
+
+/// Big-endian file access trait.
+pub(super) trait BeUint: Sized + Copy {
+    fn from_be_slice(bytes: &[u8]) -> u64;
+    fn write_be<W: Write>(w: &mut W, val: Self) -> io::Result<()>;
+}
+
+impl BeUint for u8 {
+    #[inline(always)]
+    fn from_be_slice(bytes: &[u8]) -> u64 {
+        bytes[0] as u64
+    }
+    #[inline(always)]
+    fn write_be<W: Write>(w: &mut W, val: Self) -> io::Result<()> {
+        w.write_u8(val)
+    }
+}
+
+impl BeUint for u16 {
+    #[inline(always)]
+    fn from_be_slice(bytes: &[u8]) -> u64 {
+        u16::from_be_bytes([bytes[0], bytes[1]]) as u64
+    }
+    #[inline(always)]
+    fn write_be<W: Write>(w: &mut W, val: Self) -> io::Result<()> {
+        w.write_u16::<BigEndian>(val)
+    }
+}
+
+impl BeUint for u32 {
+    #[inline(always)]
+    fn from_be_slice(bytes: &[u8]) -> u64 {
+        u32::from_be_bytes([bytes[0], bytes[1], bytes[2], bytes[3]]) as u64
+    }
+    #[inline(always)]
+    fn write_be<W: Write>(w: &mut W, val: Self) -> io::Result<()> {
+        w.write_u32::<BigEndian>(val)
+    }
+}
+
+impl BeUint for u64 {
+    #[inline(always)]
+    fn from_be_slice(bytes: &[u8]) -> u64 {
+        u64::from_be_bytes([
+            bytes[0], bytes[1], bytes[2], bytes[3], bytes[4], bytes[5], bytes[6], bytes[7],
+        ])
+    }
+    #[inline(always)]
+    fn write_be<W: Write>(w: &mut W, val: Self) -> io::Result<()> {
+        w.write_u64::<BigEndian>(val)
+    }
+}
+
+/// Read byte-aligned refcounts.
+fn read_refcount<T: BeUint>(
+    file: &mut AlignedFile,
+    offset: u64,
+    count: usize,
+) -> io::Result<Vec<u64>> {
+    let bytes_per_entry = size_of::<T>();
+    let mut data = vec![0u8; count * bytes_per_entry];
+    file.read_exact_at(&mut data, offset)?;
+    Ok(data
+        .chunks_exact(bytes_per_entry)
+        .map(T::from_be_slice)
+        .collect())
+}
+
+/// Write byte-aligned refcounts.
+fn write_refcount<T: BeUint + TryFrom<u64>>(
+    file: &mut AlignedFile,
+    offset: u64,
+    table: &[u64],
+) -> io::Result<()>
+where
+    <T as TryFrom<u64>>::Error: Debug,
+{
+    let bytes_per_entry = size_of::<T>();
+    let mut buffer = Vec::with_capacity(table.len() * bytes_per_entry);
+    for &val in table {
+        let converted = T::try_from(val).expect("refcount values are validated on increment");
+        T::write_be(&mut buffer, converted)?;
+    }
+    file.write_all_at(&buffer, offset)
+}
+
+/// Read sub-byte refcounts. Bit 0 is the least significant bit.
+fn read_refcount_subbyte<const BITS: usize>(
+    file: &mut AlignedFile,
+    offset: u64,
+    count: usize,
+) -> io::Result<Vec<u64>> {
+    const { assert!(BITS == 1 || BITS == 2 || BITS == 4) };
+    let entries_per_byte = 8 / BITS;
+    let mask = (1u64 << BITS) - 1;
+    let bytes_needed = count.div_ceil(entries_per_byte);
+    let mut bytes = vec![0u8; bytes_needed];
+    file.read_exact_at(&mut bytes, offset)?;
+
+    let mut table = vec![0u64; count];
+    for (i, val) in table.iter_mut().enumerate() {
+        let byte_idx = i / entries_per_byte;
+        let bit_offset = (i % entries_per_byte) * BITS;
+        *val = (bytes[byte_idx] as u64 >> bit_offset) & mask;
+    }
+    Ok(table)
+}
+
+/// Write sub-byte refcounts. Bit 0 is the least significant bit.
+fn write_refcount_subbyte<const BITS: usize>(
+    file: &mut AlignedFile,
+    offset: u64,
+    table: &[u64],
+) -> io::Result<()> {
+    const { assert!(BITS == 1 || BITS == 2 || BITS == 4) };
+    let entries_per_byte = 8 / BITS;
+    let mask = (1u64 << BITS) - 1;
+    let mut buffer = Vec::with_capacity(table.len().div_ceil(entries_per_byte));
+
+    for chunk in table.chunks(entries_per_byte) {
+        let mut byte = 0u8;
+        for (i, &val) in chunk.iter().enumerate() {
+            let bit_offset = i * BITS;
+            byte |= ((val & mask) << bit_offset) as u8;
+        }
+        buffer.push(byte);
+    }
+    file.write_all_at(&buffer, offset)
+}
+
+/// A qcow file. Allows reading/writing clusters and appending clusters.
+#[derive(Debug)]
+pub(super) struct QcowRawFile {
+    file: AlignedFile,
+    cluster_size: u64,
+    cluster_mask: u64,
+    refcount_block_entries: u64,
+    read_refcount_fn: RefcountReader,
+    write_refcount_fn: RefcountWriter,
+}
+
+impl QcowRawFile {
+    /// Creates a `QcowRawFile` from the given `File`, `None` is returned if `cluster_size` is not
+    /// a power of two or refcount_bits is invalid.
+    pub(super) fn from(file: AlignedFile, cluster_size: u64, refcount_bits: u64) -> Option<Self> {
+        if !cluster_size.is_power_of_two() {
+            return None;
+        }
+
+        let (read_refcount_fn, write_refcount_fn): (RefcountReader, RefcountWriter) =
+            match refcount_bits {
+                1 => (read_refcount_subbyte::<1>, write_refcount_subbyte::<1>),
+                2 => (read_refcount_subbyte::<2>, write_refcount_subbyte::<2>),
+                4 => (read_refcount_subbyte::<4>, write_refcount_subbyte::<4>),
+                8 => (read_refcount::<u8>, write_refcount::<u8>),
+                16 => (read_refcount::<u16>, write_refcount::<u16>),
+                32 => (read_refcount::<u32>, write_refcount::<u32>),
+                64 => (read_refcount::<u64>, write_refcount::<u64>),
+                _ => return None,
+            };
+
+        // For sub-byte refcounts (1,2,4 bits), entries pack multiple per byte
+        let refcount_block_entries = cluster_size * 8 / refcount_bits;
+
+        Some(QcowRawFile {
+            file,
+            cluster_size,
+            cluster_mask: cluster_size - 1,
+            refcount_block_entries,
+            read_refcount_fn,
+            write_refcount_fn,
+        })
+    }
+
+    /// Reads `count` 64 bit offsets and returns them as a vector.
+    /// `mask` optionally `&`s out some of the bits on the file.
+    pub(super) fn read_pointer_table(
+        &mut self,
+        offset: u64,
+        count: u64,
+        mask: Option<u64>,
+    ) -> io::Result<Vec<u64>> {
+        let mut bytes = vec![0u8; count as usize * size_of::<u64>()];
+        self.file.read_exact_at(&mut bytes, offset)?;
+        let m = mask.unwrap_or(u64::MAX);
+        let table = bytes
+            .as_chunks::<{ size_of::<u64>() }>()
+            .0
+            .iter()
+            .map(|c| u64::from_be_bytes(*c) & m)
+            .collect();
+        Ok(table)
+    }
+
+    /// Reads a cluster's worth of 64 bit offsets and returns them as a vector.
+    /// `mask` optionally `&`s out some of the bits on the file.
+    pub(super) fn read_pointer_cluster(
+        &mut self,
+        offset: u64,
+        mask: Option<u64>,
+    ) -> io::Result<Vec<u64>> {
+        let count = self.cluster_size / size_of::<u64>() as u64;
+        self.read_pointer_table(offset, count, mask)
+    }
+
+    /// Writes a pointer table to `offset` in the file.
+    /// Entries are computed on-the-fly by the callback.
+    ///
+    /// The callback may perform metadata I/O on this `QcowRawFile`, so all
+    /// entries are materialized before the final positional write.
+    pub(super) fn write_pointer_table<'a, T: Copy + 'a>(
+        &mut self,
+        offset: u64,
+        entries: impl Iterator<Item = &'a T>,
+        mut f: impl FnMut(&mut QcowRawFile, T) -> io::Result<u64>,
+    ) -> io::Result<()> {
+        let mut buffer = Vec::with_capacity(entries.size_hint().0 * size_of::<u64>());
+        for addr in entries {
+            let entry = f(self, *addr)?;
+            buffer.extend_from_slice(&entry.to_be_bytes());
+        }
+        self.file.write_all_at(&buffer, offset)
+    }
+
+    /// Writes a pointer table directly without transforming values.
+    ///
+    /// Uses the same materialize-then-write path as `write_pointer_table`.
+    pub(super) fn write_pointer_table_direct<'a>(
+        &mut self,
+        offset: u64,
+        entries: impl Iterator<Item = &'a u64>,
+    ) -> io::Result<()> {
+        let mut buffer = Vec::with_capacity(entries.size_hint().0 * size_of::<u64>());
+        for &entry in entries {
+            buffer.extend_from_slice(&entry.to_be_bytes());
+        }
+        self.file.write_all_at(&buffer, offset)
+    }
+
+    /// Read a refcount block from the file and returns a Vec containing the block.
+    /// Always returns a cluster's worth of data.
+    #[inline]
+    pub(super) fn read_refcount_block(&mut self, offset: u64) -> io::Result<Vec<u64>> {
+        (self.read_refcount_fn)(&mut self.file, offset, self.refcount_block_entries as usize)
+    }
+
+    /// Writes a refcount block to the file.
+    #[inline]
+    pub(super) fn write_refcount_block(&mut self, offset: u64, table: &[u64]) -> io::Result<()> {
+        (self.write_refcount_fn)(&mut self.file, offset, table)
+    }
+
+    /// Allocates a new cluster at the end of the current file, return the address.
+    pub(super) fn add_cluster_end(
+        &mut self,
+        max_valid_cluster_offset: u64,
+    ) -> io::Result<Option<u64>> {
+        // Determine where the new end of the file should be and set_len, which
+        // translates to truncate(2).
+        let file_end: u64 = self.physical_size()?;
+        let new_cluster_address: u64 = (file_end + self.cluster_size - 1) & !self.cluster_mask;
+
+        if new_cluster_address > max_valid_cluster_offset {
+            return Ok(None);
+        }
+
+        self.file.set_len(new_cluster_address + self.cluster_size)?;
+
+        Ok(Some(new_cluster_address))
+    }
+
+    /// Returns a reference to the underlying file.
+    pub(super) fn file(&self) -> &AlignedFile {
+        &self.file
+    }
+
+    /// Returns a mutable reference to the underlying file.
+    pub(super) fn file_mut(&mut self) -> &mut AlignedFile {
+        &mut self.file
+    }
+
+    /// Returns the size of the file's clusters.
+    pub(super) fn cluster_size(&self) -> u64 {
+        self.cluster_size
+    }
+
+    /// Returns the offset of `address` within a cluster.
+    pub(super) fn cluster_offset(&self, address: u64) -> u64 {
+        address & self.cluster_mask
+    }
+
+    /// Returns the base address of the cluster containing `address`.
+    pub(super) fn cluster_address(&self, address: u64) -> u64 {
+        address & !self.cluster_mask
+    }
+
+    /// Zeros out a cluster in the file.
+    pub(super) fn zero_cluster(&mut self, address: u64) -> io::Result<()> {
+        let cluster_size = self.cluster_size as usize;
+        self.file.write_all_zeroes_at(address, cluster_size)?;
+        Ok(())
+    }
+
+    /// Writes
+    pub(super) fn write_cluster(&mut self, address: u64, data: &[u8]) -> io::Result<()> {
+        let cluster_size = self.cluster_size as usize;
+        self.file.write_all_at(&data[0..cluster_size], address)
+    }
+
+    pub(super) fn physical_size(&self) -> io::Result<u64> {
+        self.file.metadata().map(|m| m.len())
+    }
+}
+
+impl Clone for QcowRawFile {
+    fn clone(&self) -> Self {
+        QcowRawFile {
+            file: self.file.try_clone().expect("QcowRawFile cloning failed"),
+            cluster_size: self.cluster_size,
+            cluster_mask: self.cluster_mask,
+            refcount_block_entries: self.refcount_block_entries,
+            read_refcount_fn: self.read_refcount_fn,
+            write_refcount_fn: self.write_refcount_fn,
+        }
+    }
+}
+
+impl AsRawFd for QcowRawFile {
+    fn as_raw_fd(&self) -> RawFd {
+        self.file.as_raw_fd()
+    }
+}
+
+impl AsFd for QcowRawFile {
+    fn as_fd(&self) -> BorrowedFd<'_> {
+        self.file.as_fd()
+    }
+}
+
+#[cfg(test)]
+mod unit_tests {
+    use std::io::Read;
+    use std::os::unix::fs::FileExt;
+
+    use vmm_sys_util::tempfile::TempFile;
+
+    use super::*;
+
+    fn be_bytes(entries: &[u64]) -> Vec<u8> {
+        let mut v = Vec::with_capacity(size_of_val(entries));
+        for e in entries {
+            v.extend_from_slice(&e.to_be_bytes());
+        }
+        v
+    }
+
+    fn find_all(haystack: &[u8], needle: &[u8]) -> Vec<usize> {
+        haystack
+            .windows(needle.len())
+            .enumerate()
+            .filter(|(_, w)| *w == needle)
+            .map(|(i, _)| i)
+            .collect()
+    }
+
+    const CLUSTER_SIZE: u64 = 0x10000; // 64 KiB
+    const TARGET_OFFSET: u64 = 0x1000; // where the table must be written
+    const FAR_OFFSET: u64 = 0x9000; // where the callback reads (refcount block)
+    const FILE_LEN: u64 = 0x40000; // 256 KiB filler so all offsets are valid
+
+    fn make_qcow_raw() -> (TempFile, QcowRawFile) {
+        make_qcow_raw_bits(16)
+    }
+
+    fn make_qcow_raw_bits(refcount_bits: u64) -> (TempFile, QcowRawFile) {
+        let temp_file = TempFile::new().unwrap();
+        temp_file.as_file().set_len(FILE_LEN).unwrap();
+
+        let file = temp_file.as_file().try_clone().unwrap();
+        let raw = AlignedFile::new(file, false);
+        let qcow_raw =
+            QcowRawFile::from(raw, CLUSTER_SIZE, refcount_bits).expect("QcowRawFile::from");
+        (temp_file, qcow_raw)
+    }
+
+    #[test]
+    fn write_pointer_table_lands_at_offset_despite_callback_seek() {
+        let (temp_file, mut qcow) = make_qcow_raw();
+        let entries: Vec<u64> = vec![0x1111_2222_3333_4444u64; 8]; // 64 bytes
+
+        qcow.write_pointer_table(TARGET_OFFSET, entries.iter(), |q, addr| {
+            let _ = q.read_refcount_block(FAR_OFFSET)?;
+            Ok(addr)
+        })
+        .expect("write_pointer_table");
+
+        let expected = be_bytes(&entries);
+        let mut verify = temp_file.as_file().try_clone().unwrap();
+        let mut whole = Vec::new();
+        verify.read_to_end(&mut whole).unwrap();
+        let found_at = find_all(&whole, &expected);
+
+        let mut at_target = vec![0u8; expected.len()];
+        verify.read_exact_at(&mut at_target, TARGET_OFFSET).unwrap();
+
+        assert_eq!(
+            at_target, expected,
+            "pointer table did NOT land at TARGET_OFFSET {TARGET_OFFSET:#x}; \
+             found matching bytes at {found_at:x?}"
+        );
+    }
+
+    #[test]
+    fn write_pointer_table_direct_lands_at_offset() {
+        let (temp_file, mut qcow) = make_qcow_raw();
+        let entries: Vec<u64> = vec![0xAAAA_BBBB_CCCC_DDDDu64; 8];
+
+        qcow.write_pointer_table_direct(TARGET_OFFSET, entries.iter())
+            .expect("write_pointer_table_direct");
+
+        let expected = be_bytes(&entries);
+        let verify = temp_file.as_file().try_clone().unwrap();
+        let mut at_target = vec![0u8; expected.len()];
+        verify.read_exact_at(&mut at_target, TARGET_OFFSET).unwrap();
+
+        assert_eq!(
+            at_target, expected,
+            "write_pointer_table_direct did not land at {TARGET_OFFSET:#x}"
+        );
+    }
+
+    #[test]
+    fn read_pointer_table_round_trips() {
+        let (_temp_file, mut qcow) = make_qcow_raw();
+        let entries: Vec<u64> = vec![
+            0x0000_0000_0000_0000,
+            0x0011_2233_4455_6677,
+            0x8899_aabb_ccdd_eeff,
+            0xffff_ffff_ffff_ffff,
+        ];
+
+        qcow.write_pointer_table_direct(TARGET_OFFSET, entries.iter())
+            .expect("write_pointer_table_direct");
+
+        let read_back = qcow
+            .read_pointer_table(TARGET_OFFSET, entries.len() as u64, None)
+            .expect("read_pointer_table");
+
+        assert_eq!(read_back, entries);
+    }
+
+    #[test]
+    fn read_pointer_table_applies_mask() {
+        let (_temp_file, mut qcow) = make_qcow_raw();
+        let entries: Vec<u64> = vec![0xffff_ffff_ffff_ffffu64; 4];
+        let mask = 0x00ff_ffff_ffff_fe00u64;
+
+        qcow.write_pointer_table_direct(TARGET_OFFSET, entries.iter())
+            .expect("write_pointer_table_direct");
+
+        let read_back = qcow
+            .read_pointer_table(TARGET_OFFSET, entries.len() as u64, Some(mask))
+            .expect("read_pointer_table");
+
+        assert!(read_back.iter().all(|&e| e == mask));
+    }
+
+    #[test]
+    fn write_cluster_then_zero_cluster_round_trips() {
+        let (temp_file, mut qcow) = make_qcow_raw();
+        let cluster_size = CLUSTER_SIZE as usize;
+        let data: Vec<u8> = (0..cluster_size).map(|i| (i % 251) as u8).collect();
+
+        qcow.write_cluster(CLUSTER_SIZE, &data)
+            .expect("write_cluster");
+
+        let verify = temp_file.as_file().try_clone().unwrap();
+        let mut buf = vec![0u8; cluster_size];
+        verify.read_exact_at(&mut buf, CLUSTER_SIZE).unwrap();
+        assert_eq!(buf, data);
+
+        qcow.zero_cluster(CLUSTER_SIZE).expect("zero_cluster");
+
+        verify.read_exact_at(&mut buf, CLUSTER_SIZE).unwrap();
+        assert!(buf.iter().all(|&b| b == 0));
+    }
+
+    #[test]
+    fn refcount_block_round_trips() {
+        let (_temp_file, mut qcow) = make_qcow_raw_bits(16);
+        let count = qcow.refcount_block_entries as usize;
+        let table: Vec<u64> = (0..count).map(|i| (i % 251) as u64).collect();
+
+        qcow.write_refcount_block(TARGET_OFFSET, &table)
+            .expect("write_refcount_block");
+        let read_back = qcow
+            .read_refcount_block(TARGET_OFFSET)
+            .expect("read_refcount_block");
+
+        assert_eq!(read_back, table);
+    }
+
+    #[test]
+    fn refcount_block_subbyte_round_trips() {
+        let (_temp_file, mut qcow) = make_qcow_raw_bits(4);
+        let count = qcow.refcount_block_entries as usize;
+        let table: Vec<u64> = (0..count).map(|i| (i % 16) as u64).collect();
+
+        qcow.write_refcount_block(TARGET_OFFSET, &table)
+            .expect("write_refcount_block");
+        let read_back = qcow
+            .read_refcount_block(TARGET_OFFSET)
+            .expect("read_refcount_block");
+
+        assert_eq!(read_back, table);
+    }
+
+    #[test]
+    fn add_cluster_end_appends_aligned_cluster() {
+        let (_temp_file, mut qcow) = make_qcow_raw();
+        let before = qcow.physical_size().unwrap();
+
+        let addr = qcow
+            .add_cluster_end(u64::MAX)
+            .expect("add_cluster_end")
+            .expect("a cluster was allocated");
+
+        assert_eq!(addr % CLUSTER_SIZE, 0);
+        assert!(addr >= before);
+        assert_eq!(qcow.physical_size().unwrap(), addr + CLUSTER_SIZE);
+    }
+
+    #[test]
+    fn add_cluster_end_respects_max_offset() {
+        let (_temp_file, mut qcow) = make_qcow_raw();
+        assert!(qcow.add_cluster_end(0).unwrap().is_none());
+    }
+}

block/src/formats/qcow/refcount.rs

@@ -4,13 +4,13 @@
 //
 // SPDX-License-Identifier: Apache-2.0 AND BSD-3-Clause
 
-use std::io;
+use std::{io, result};
 
 use libc::EINVAL;
 use thiserror::Error;
 
-use crate::qcow::qcow_raw_file::QcowRawFile;
-use crate::qcow::vec_cache::{CacheMap, Cacheable, VecCache};
+use super::qcow_raw_file::QcowRawFile;
+use super::vec_cache::{CacheMap, Cacheable, VecCache};
 
 #[derive(Debug, Error)]
 pub enum Error {
@@ -41,11 +41,11 @@ pub enum Error {
     },
 }
 
-pub type Result<T> = std::result::Result<T, Error>;
+pub(super) type Result<T> = result::Result<T, Error>;
 
 /// Represents the refcount entries for an open qcow file.
 #[derive(Clone, Debug)]
-pub struct RefCount {
+pub(super) struct RefCount {
     ref_table: VecCache<u64>,
     refcount_table_offset: u64,
     refblock_cache: CacheMap<VecCache<u64>>,
@@ -62,7 +62,7 @@ impl RefCount {
     /// `refcount_block_entries` indicates the number of refcounts in each refcount block.
     /// `refcount_bits` is the number of bits per refcount (1, 2, 4, 8, 16, 32, or 64).
     /// Each refcount table entry points to a refcount block.
-    pub fn new(
+    pub(super) fn new(
         raw_file: &mut QcowRawFile,
         refcount_table_offset: u64,
         refcount_table_entries: u64,
@@ -95,12 +95,12 @@ impl RefCount {
     }
 
     /// Returns the number of refcounts per block.
-    pub fn refcounts_per_block(&self) -> u64 {
+    pub(super) fn refcounts_per_block(&self) -> u64 {
         self.refcount_block_entries
     }
 
     /// Returns the maximum valid cluster offset in the raw file for this refcount table.
-    pub fn max_valid_cluster_offset(&self) -> u64 {
+    pub(super) fn max_valid_cluster_offset(&self) -> u64 {
         self.max_valid_cluster_offset
     }
 
@@ -109,7 +109,7 @@ impl RefCount {
     /// allocate a cluster or read the required one and call this function again with the cluster.
     /// On success, an optional address of a dropped cluster is returned. The dropped cluster can
     /// be reused for other purposes.
-    pub fn set_cluster_refcount(
+    pub(super) fn set_cluster_refcount(
         &mut self,
         raw_file: &mut QcowRawFile,
         cluster_address: u64,
@@ -167,14 +167,14 @@ impl RefCount {
 
     /// Flush the dirty refcount blocks. This must be done before flushing the table that points to
     /// the blocks.
-    pub fn flush_blocks(&mut self, raw_file: &mut QcowRawFile) -> io::Result<()> {
+    pub(super) fn flush_blocks(&mut self, raw_file: &mut QcowRawFile) -> io::Result<()> {
         // Write out all dirty L2 tables.
         for (table_index, block) in self.refblock_cache.iter_mut().filter(|(_k, v)| v.dirty()) {
             let addr = self.ref_table[*table_index];
             if addr != 0 {
                 raw_file.write_refcount_block(addr, block.get_values())?;
             } else {
-                return Err(std::io::Error::from_raw_os_error(EINVAL));
+                return Err(io::Error::from_raw_os_error(EINVAL));
             }
             block.mark_clean();
         }
@@ -183,7 +183,7 @@ impl RefCount {
 
     /// Flush the refcount table that keeps the address of the refcounts blocks.
     /// Returns true if the table changed since the previous `flush_table()` call.
-    pub fn flush_table(&mut self, raw_file: &mut QcowRawFile) -> io::Result<bool> {
+    pub(super) fn flush_table(&mut self, raw_file: &mut QcowRawFile) -> io::Result<bool> {
         if self.ref_table.dirty() {
             raw_file
                 .write_pointer_table_direct(self.refcount_table_offset, self.ref_table.iter())?;
@@ -195,7 +195,7 @@ impl RefCount {
     }
 
     /// Gets the refcount for a cluster with the given address.
-    pub fn get_cluster_refcount(
+    pub(super) fn get_cluster_refcount(
         &mut self,
         raw_file: &mut QcowRawFile,
         address: u64,
@@ -224,41 +224,6 @@ impl RefCount {
         Ok(self.refblock_cache.get(table_index).unwrap()[block_index])
     }
 
-    /// Returns the refcount table for this file. This is only useful for debugging.
-    pub fn ref_table(&self) -> &[u64] {
-        self.ref_table.get_values()
-    }
-
-    /// Returns the refcounts stored in the given block.
-    pub fn refcount_block(
-        &mut self,
-        raw_file: &mut QcowRawFile,
-        table_index: usize,
-    ) -> Result<Option<&[u64]>> {
-        let block_addr_disk = *self.ref_table.get(table_index).ok_or(Error::InvalidIndex)?;
-        if block_addr_disk == 0 {
-            return Ok(None);
-        }
-        if !self.refblock_cache.contains_key(table_index) {
-            let table = VecCache::from_vec(
-                raw_file
-                    .read_refcount_block(block_addr_disk)
-                    .map_err(Error::ReadingRefCounts)?,
-            );
-            // TODO(dgreid) - closure needs to return an error.
-            let ref_table = &self.ref_table;
-            self.refblock_cache
-                .insert(table_index, table, |index, evicted| {
-                    raw_file.write_refcount_block(ref_table[index], evicted.get_values())
-                })
-                .map_err(Error::EvictingRefCounts)?;
-        }
-        // The index must exist as it was just inserted if it didn't already.
-        Ok(Some(
-            self.refblock_cache.get(table_index).unwrap().get_values(),
-        ))
-    }
-
     // Gets the address of the refcount block and the index into the block for the given address.
     fn get_refcount_index(&self, address: u64) -> (usize, usize) {
         let block_index = (address / self.cluster_size) % self.refcount_block_entries;

block/src/formats/qcow/util.rs

@@ -63,6 +63,11 @@ pub(super) fn l2_entry_make_zero(cluster_addr: u64) -> u64 {
     (cluster_addr & L2_TABLE_OFFSET_MASK) | CLUSTER_USED_FLAG | ZERO_FLAG
 }
 
+/// Make L2 entry for an unallocated cluster that reads as logical zeros.
+pub(super) fn l2_entry_make_zero_plain() -> u64 {
+    ZERO_FLAG
+}
+
 /// Make L1 entry with optional flags.
 pub(super) fn l1_entry_make(cluster_addr: u64, refcount_is_one: bool) -> u64 {
     (cluster_addr & L1_TABLE_OFFSET_MASK) | (refcount_is_one as u64 * CLUSTER_USED_FLAG)

block/src/formats/qcow/vec_cache.rs

@@ -12,21 +12,21 @@ use std::slice::SliceIndex;
 
 /// Trait that allows for checking if an implementor is dirty. Useful for types that are cached so
 /// it can be checked if they need to be committed to disk.
-pub trait Cacheable {
+pub(super) trait Cacheable {
     /// Used to check if the item needs to be written out or if it can be discarded.
     fn dirty(&self) -> bool;
 }
 
 #[derive(Clone, Debug)]
 /// Represents a vector that implements the `Cacheable` trait so it can be held in a cache.
-pub struct VecCache<T: 'static + Copy + Default> {
+pub(super) struct VecCache<T: 'static + Copy + Default> {
     vec: Box<[T]>,
     dirty: bool,
 }
 
 impl<T: 'static + Copy + Default> VecCache<T> {
     /// Creates a `VecCache` that can hold `count` elements.
-    pub fn new(count: usize) -> VecCache<T> {
+    pub(super) fn new(count: usize) -> VecCache<T> {
         VecCache {
             vec: vec![Default::default(); count].into_boxed_slice(),
             dirty: true,
@@ -34,14 +34,14 @@ impl<T: 'static + Copy + Default> VecCache<T> {
     }
 
     /// Creates a `VecCache` from the passed in `vec`.
-    pub fn from_vec(vec: Vec<T>) -> VecCache<T> {
+    pub(super) fn from_vec(vec: Vec<T>) -> VecCache<T> {
         VecCache {
             vec: vec.into_boxed_slice(),
             dirty: false,
         }
     }
 
-    pub fn get<I>(&self, index: I) -> Option<&<I as SliceIndex<[T]>>::Output>
+    pub(super) fn get<I>(&self, index: I) -> Option<&<I as SliceIndex<[T]>>::Output>
     where
         I: SliceIndex<[T]>,
     {
@@ -49,17 +49,17 @@ impl<T: 'static + Copy + Default> VecCache<T> {
     }
 
     /// Gets a reference to the underlying vector.
-    pub fn get_values(&self) -> &[T] {
+    pub(super) fn get_values(&self) -> &[T] {
         &self.vec
     }
 
     /// Mark this cache element as clean.
-    pub fn mark_clean(&mut self) {
+    pub(super) fn mark_clean(&mut self) {
         self.dirty = false;
     }
 
     /// Returns the number of elements in the vector.
-    pub fn len(&self) -> usize {
+    pub(super) fn len(&self) -> usize {
         self.vec.len()
     }
 
@@ -68,7 +68,7 @@ impl<T: 'static + Copy + Default> VecCache<T> {
     /// No-op if `new_len <= self.len()`. Allocates a new buffer, copies
     /// existing data, and fills new elements with default values.
     /// Marks the cache as dirty.
-    pub fn extend(&mut self, new_len: usize) {
+    pub(super) fn extend(&mut self, new_len: usize) {
         if new_len <= self.vec.len() {
             return;
         }
@@ -109,37 +109,37 @@ impl<T: 'static + Copy + Default> Deref for VecCache<T> {
 }
 
 #[derive(Clone, Debug)]
-pub struct CacheMap<T: Cacheable> {
+pub(super) struct CacheMap<T: Cacheable> {
     capacity: usize,
     map: HashMap<usize, T>,
 }
 
 impl<T: Cacheable> CacheMap<T> {
-    pub fn new(capacity: usize) -> Self {
+    pub(super) fn new(capacity: usize) -> Self {
         CacheMap {
             capacity,
             map: HashMap::with_capacity(capacity),
         }
     }
 
-    pub fn contains_key(&self, key: usize) -> bool {
+    pub(super) fn contains_key(&self, key: usize) -> bool {
         self.map.contains_key(&key)
     }
 
-    pub fn get(&self, index: usize) -> Option<&T> {
+    pub(super) fn get(&self, index: usize) -> Option<&T> {
         self.map.get(&index)
     }
 
-    pub fn get_mut(&mut self, index: usize) -> Option<&mut T> {
+    pub(super) fn get_mut(&mut self, index: usize) -> Option<&mut T> {
         self.map.get_mut(&index)
     }
 
-    pub fn iter_mut(&mut self) -> IterMut<'_, usize, T> {
+    pub(super) fn iter_mut(&mut self) -> IterMut<'_, usize, T> {
         self.map.iter_mut()
     }
 
     // Check if the refblock cache is full and we need to evict.
-    pub fn insert<F>(&mut self, index: usize, block: T, write_callback: F) -> io::Result<()>
+    pub(super) fn insert<F>(&mut self, index: usize, block: T, write_callback: F) -> io::Result<()>
     where
         F: FnOnce(usize, T) -> io::Result<()>,
     {

block/src/formats/raw/engine_aio.rs

@@ -0,0 +1,159 @@
+// Copyright © 2023 Intel Corporation
+//
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+//
+// SPDX-License-Identifier: Apache-2.0 AND BSD-3-Clause
+//
+// Copyright © 2023 Crusoe Energy Systems LLC
+//
+
+use std::os::unix::io::AsRawFd;
+
+use vmm_sys_util::eventfd::EventFd;
+
+use super::{operation_is_aligned, run_unaligned_operation};
+use crate::async_io::{
+    AioDataIo, AsyncIo, AsyncIoCompletion, AsyncIoError, AsyncIoOperation, AsyncIoResult,
+};
+use crate::error::{BlockError, BlockErrorKind, BlockResult};
+use crate::sparse::{punch_hole, write_zeroes};
+use crate::{AlignedFile, is_block_device};
+
+pub(super) struct RawAio {
+    raw_file: AlignedFile,
+    data_io: AioDataIo,
+    alignment: u64,
+    is_block_device: bool,
+}
+
+impl RawAio {
+    pub(super) fn new(raw_file: AlignedFile, queue_depth: u32) -> BlockResult<Self> {
+        let data_io =
+            AioDataIo::new(queue_depth).map_err(|e| BlockError::new(BlockErrorKind::Io, e))?;
+        let is_block_device = is_block_device(raw_file.as_raw_fd());
+        let alignment = raw_file.alignment() as u64;
+
+        Ok(RawAio {
+            raw_file,
+            data_io,
+            alignment,
+            is_block_device,
+        })
+    }
+}
+
+impl AsyncIo for RawAio {
+    fn notifier(&self) -> &EventFd {
+        self.data_io.notifier()
+    }
+
+    fn alignment(&self) -> u64 {
+        self.alignment
+    }
+
+    fn submit_data_operation(&mut self, mut op: AsyncIoOperation) -> AsyncIoResult<()> {
+        let is_read = op.is_read();
+
+        if operation_is_aligned(&op, self.alignment) {
+            let fd = self.raw_file.as_raw_fd();
+            return self.data_io.submit_operation(fd, op).map_err(|e| {
+                if is_read {
+                    AsyncIoError::ReadVectored(e)
+                } else {
+                    AsyncIoError::WriteVectored(e)
+                }
+            });
+        }
+
+        let result = run_unaligned_operation(&self.raw_file, &mut op)?;
+        self.data_io
+            .inject_completion(AsyncIoCompletion::from_operation(op, result));
+
+        Ok(())
+    }
+
+    fn fsync(&mut self, user_data: Option<u64>) -> AsyncIoResult<()> {
+        let fd = self.raw_file.as_raw_fd();
+        if let Some(user_data) = user_data {
+            self.data_io
+                .submit_fsync(fd, user_data)
+                .map_err(AsyncIoError::Fsync)?;
+        } else {
+            // SAFETY: FFI call with a valid fd
+            unsafe { libc::fsync(fd) };
+        }
+
+        Ok(())
+    }
+
+    fn next_completed_request(&mut self) -> Option<AsyncIoCompletion> {
+        self.data_io.next_completion()
+    }
+
+    fn punch_hole(&mut self, offset: u64, length: u64, user_data: u64) -> AsyncIoResult<()> {
+        // Linux AIO has no IOCB command for fallocate, so perform the
+        // operation synchronously and signal completion via the completion
+        // list, matching the pattern used by the sync backend (RawSync).
+        punch_hole(
+            self.raw_file.as_raw_fd(),
+            self.is_block_device,
+            offset,
+            length,
+        )
+        .map_err(AsyncIoError::PunchHole)?;
+        self.data_io
+            .inject_completion(AsyncIoCompletion::new(user_data, 0, None));
+
+        Ok(())
+    }
+
+    fn write_zeroes(&mut self, offset: u64, length: u64, user_data: u64) -> AsyncIoResult<()> {
+        // Same as punch_hole().
+        write_zeroes(
+            self.raw_file.as_raw_fd(),
+            self.is_block_device,
+            offset,
+            length,
+        )
+        .map_err(AsyncIoError::WriteZeroes)?;
+        self.data_io
+            .inject_completion(AsyncIoCompletion::new(user_data, 0, None));
+
+        Ok(())
+    }
+}
+
+#[cfg(test)]
+mod unit_tests {
+    use vmm_sys_util::tempfile::TempFile;
+
+    use super::*;
+    use crate::formats::raw::tests;
+
+    #[test]
+    fn test_punch_hole() {
+        let temp_file = TempFile::new().unwrap();
+        let mut file = temp_file.into_file();
+        let mut async_io =
+            RawAio::new(AlignedFile::new(file.try_clone().unwrap(), false), 128).unwrap();
+        tests::test_punch_hole(&mut async_io, &mut file);
+    }
+
+    #[test]
+    fn test_write_zeroes() {
+        let temp_file = TempFile::new().unwrap();
+        let mut file = temp_file.into_file();
+        let mut async_io =
+            RawAio::new(AlignedFile::new(file.try_clone().unwrap(), false), 128).unwrap();
+        tests::test_write_zeroes(&mut async_io, &mut file);
+    }
+
+    #[test]
+    fn test_punch_hole_multiple_operations() {
+        let temp_file = TempFile::new().unwrap();
+        let mut file = temp_file.into_file();
+        let mut async_io =
+            RawAio::new(AlignedFile::new(file.try_clone().unwrap(), false), 128).unwrap();
+        tests::test_punch_hole_multiple_operations(&mut async_io, &mut file);
+    }
+}

block/src/formats/raw/engine_sync.rs

@@ -0,0 +1,181 @@
+// Copyright © 2021 Intel Corporation
+//
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+//
+// SPDX-License-Identifier: Apache-2.0 AND BSD-3-Clause
+
+use std::collections::VecDeque;
+use std::io;
+use std::os::unix::io::AsRawFd;
+
+use vmm_sys_util::eventfd::EventFd;
+
+use super::{operation_is_aligned, run_unaligned_operation};
+use crate::async_io::{AsyncIo, AsyncIoCompletion, AsyncIoError, AsyncIoOperation, AsyncIoResult};
+use crate::sparse::{punch_hole, write_zeroes};
+use crate::{AlignedFile, is_block_device};
+
+pub(crate) struct RawSync {
+    raw_file: AlignedFile,
+    eventfd: EventFd,
+    completion_list: VecDeque<AsyncIoCompletion>,
+    alignment: u64,
+    is_block_device: bool,
+}
+
+impl RawSync {
+    pub(crate) fn new(raw_file: AlignedFile) -> Self {
+        let is_block_device = is_block_device(raw_file.as_raw_fd());
+        let alignment = raw_file.alignment() as u64;
+        RawSync {
+            raw_file,
+            eventfd: EventFd::new(libc::EFD_NONBLOCK).expect("Failed creating EventFd for RawFile"),
+            completion_list: VecDeque::new(),
+            alignment,
+            is_block_device,
+        }
+    }
+}
+
+impl AsyncIo for RawSync {
+    fn notifier(&self) -> &EventFd {
+        &self.eventfd
+    }
+
+    fn alignment(&self) -> u64 {
+        self.alignment
+    }
+
+    fn submit_data_operation(&mut self, mut op: AsyncIoOperation) -> AsyncIoResult<()> {
+        let is_read = op.is_read();
+
+        let result = if operation_is_aligned(&op, self.alignment) {
+            let fd = self.raw_file.as_raw_fd();
+            let offset = op.offset();
+            let iovecs = op.iovecs();
+
+            let result = if is_read {
+                // SAFETY: the memory pointed to by `iovecs` is backed by the op,
+                // and valid for the kernel to write to by construction of
+                // AsyncIoOperation.
+                unsafe {
+                    libc::preadv(
+                        fd as libc::c_int,
+                        iovecs.as_ptr(),
+                        iovecs.len() as libc::c_int,
+                        offset,
+                    )
+                }
+            } else {
+                // SAFETY: the memory pointed to by `iovecs` is backed by the op,
+                // and valid for the kernel to read from by construction of
+                // AsyncIoOperation.
+                unsafe {
+                    libc::pwritev(
+                        fd as libc::c_int,
+                        iovecs.as_ptr(),
+                        iovecs.len() as libc::c_int,
+                        offset,
+                    )
+                }
+            };
+            if result < 0 {
+                let error = io::Error::last_os_error();
+                return Err(if is_read {
+                    AsyncIoError::ReadVectored(error)
+                } else {
+                    AsyncIoError::WriteVectored(error)
+                });
+            }
+            result as i32
+        } else {
+            run_unaligned_operation(&self.raw_file, &mut op)?
+        };
+
+        self.completion_list
+            .push_back(AsyncIoCompletion::from_operation(op, result));
+        self.eventfd.write(1).unwrap();
+
+        Ok(())
+    }
+
+    fn fsync(&mut self, user_data: Option<u64>) -> AsyncIoResult<()> {
+        // SAFETY: FFI call
+        let result = unsafe { libc::fsync(self.raw_file.as_raw_fd() as libc::c_int) };
+        if result < 0 {
+            return Err(AsyncIoError::Fsync(io::Error::last_os_error()));
+        }
+
+        if let Some(user_data) = user_data {
+            self.completion_list
+                .push_back(AsyncIoCompletion::new(user_data, result, None));
+            self.eventfd.write(1).unwrap();
+        }
+
+        Ok(())
+    }
+
+    fn next_completed_request(&mut self) -> Option<AsyncIoCompletion> {
+        self.completion_list.pop_front()
+    }
+
+    fn punch_hole(&mut self, offset: u64, length: u64, user_data: u64) -> AsyncIoResult<()> {
+        punch_hole(
+            self.raw_file.as_raw_fd(),
+            self.is_block_device,
+            offset,
+            length,
+        )
+        .map_err(AsyncIoError::PunchHole)?;
+        self.completion_list
+            .push_back(AsyncIoCompletion::new(user_data, 0, None));
+        self.eventfd.write(1).unwrap();
+        Ok(())
+    }
+
+    fn write_zeroes(&mut self, offset: u64, length: u64, user_data: u64) -> AsyncIoResult<()> {
+        write_zeroes(
+            self.raw_file.as_raw_fd(),
+            self.is_block_device,
+            offset,
+            length,
+        )
+        .map_err(AsyncIoError::WriteZeroes)?;
+        self.completion_list
+            .push_back(AsyncIoCompletion::new(user_data, 0, None));
+        self.eventfd.write(1).unwrap();
+        Ok(())
+    }
+}
+
+#[cfg(test)]
+mod unit_tests {
+    use vmm_sys_util::tempfile::TempFile;
+
+    use super::*;
+    use crate::formats::raw::tests;
+
+    #[test]
+    fn test_punch_hole() {
+        let temp_file = TempFile::new().unwrap();
+        let mut file = temp_file.into_file();
+        let mut async_io = RawSync::new(AlignedFile::new(file.try_clone().unwrap(), false));
+        tests::test_punch_hole(&mut async_io, &mut file);
+    }
+
+    #[test]
+    fn test_write_zeroes() {
+        let temp_file = TempFile::new().unwrap();
+        let mut file = temp_file.into_file();
+        let mut async_io = RawSync::new(AlignedFile::new(file.try_clone().unwrap(), false));
+        tests::test_write_zeroes(&mut async_io, &mut file);
+    }
+
+    #[test]
+    fn test_punch_hole_multiple_operations() {
+        let temp_file = TempFile::new().unwrap();
+        let mut file = temp_file.into_file();
+        let mut async_io = RawSync::new(AlignedFile::new(file.try_clone().unwrap(), false));
+        tests::test_punch_hole_multiple_operations(&mut async_io, &mut file);
+    }
+}

block/src/formats/raw/engine_uring.rs

@@ -0,0 +1,162 @@
+// Copyright © 2021 Intel Corporation
+//
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+//
+// SPDX-License-Identifier: Apache-2.0 AND BSD-3-Clause
+
+use std::os::unix::io::AsRawFd;
+
+use libc::{FALLOC_FL_KEEP_SIZE, FALLOC_FL_PUNCH_HOLE, FALLOC_FL_ZERO_RANGE};
+use vmm_sys_util::eventfd::EventFd;
+
+use super::{operation_is_aligned, run_unaligned_operation};
+use crate::async_io::{
+    AsyncIo, AsyncIoCompletion, AsyncIoError, AsyncIoOperation, AsyncIoResult, UringDataIo,
+};
+use crate::error::{BlockError, BlockErrorKind, BlockResult};
+use crate::sparse::{blkdiscard, blkzeroout};
+use crate::{AlignedFile, is_block_device};
+
+pub(crate) struct RawAsync {
+    raw_file: AlignedFile,
+    data_io: UringDataIo,
+    alignment: u64,
+    is_block_device: bool,
+}
+
+impl RawAsync {
+    pub(crate) fn new(raw_file: AlignedFile, ring_depth: u32) -> BlockResult<Self> {
+        let data_io =
+            UringDataIo::new(ring_depth).map_err(|e| BlockError::new(BlockErrorKind::Io, e))?;
+        let is_block_device = is_block_device(raw_file.as_raw_fd());
+        let alignment = raw_file.alignment() as u64;
+
+        Ok(RawAsync {
+            raw_file,
+            data_io,
+            alignment,
+            is_block_device,
+        })
+    }
+}
+
+impl AsyncIo for RawAsync {
+    fn notifier(&self) -> &EventFd {
+        self.data_io.notifier()
+    }
+
+    fn alignment(&self) -> u64 {
+        self.alignment
+    }
+
+    fn submit_data_operation(&mut self, mut op: AsyncIoOperation) -> AsyncIoResult<()> {
+        let is_read = op.is_read();
+
+        if operation_is_aligned(&op, self.alignment) {
+            let fd = self.raw_file.as_raw_fd();
+            return self.data_io.submit_operation(fd, op).map_err(|e| {
+                if is_read {
+                    AsyncIoError::ReadVectored(e)
+                } else {
+                    AsyncIoError::WriteVectored(e)
+                }
+            });
+        }
+
+        let result = run_unaligned_operation(&self.raw_file, &mut op)?;
+        self.data_io
+            .inject_completion(AsyncIoCompletion::from_operation(op, result));
+
+        Ok(())
+    }
+
+    fn fsync(&mut self, user_data: Option<u64>) -> AsyncIoResult<()> {
+        let fd = self.raw_file.as_raw_fd();
+        if let Some(user_data) = user_data {
+            self.data_io
+                .submit_fsync(fd, user_data)
+                .map_err(AsyncIoError::Fsync)?;
+        } else {
+            // SAFETY: FFI call with a valid fd
+            unsafe { libc::fsync(fd) };
+        }
+
+        Ok(())
+    }
+
+    fn next_completed_request(&mut self) -> Option<AsyncIoCompletion> {
+        self.data_io.next_completion()
+    }
+
+    fn batch_requests_enabled(&self) -> bool {
+        true
+    }
+
+    fn submit_batch_requests(&mut self, batch_request: Vec<AsyncIoOperation>) -> AsyncIoResult<()> {
+        if self.alignment != 0 {
+            let mut aligned_batch = Vec::with_capacity(batch_request.len());
+            for mut op in batch_request {
+                if operation_is_aligned(&op, self.alignment) {
+                    aligned_batch.push(op);
+                } else {
+                    let result = run_unaligned_operation(&self.raw_file, &mut op)?;
+                    self.data_io
+                        .inject_completion(AsyncIoCompletion::from_operation(op, result));
+                }
+            }
+            if aligned_batch.is_empty() {
+                return Ok(());
+            }
+            return self
+                .data_io
+                .submit_batch(self.raw_file.as_raw_fd(), aligned_batch)
+                .map_err(AsyncIoError::SubmitBatchRequests);
+        }
+
+        self.data_io
+            .submit_batch(self.raw_file.as_raw_fd(), batch_request)
+            .map_err(AsyncIoError::SubmitBatchRequests)
+    }
+
+    fn punch_hole(&mut self, offset: u64, length: u64, user_data: u64) -> AsyncIoResult<()> {
+        // Some block devices don't support fallocate(). Use ioctl instead. The assumption is that
+        // this happens rarely and we don't need to introduce unnecessary complexity by submitting
+        // a fallocate request, reaping ENOTSUPP in the completion routine, and reissuing the
+        // request with an ioctl.
+        if self.is_block_device {
+            blkdiscard(self.raw_file.as_raw_fd(), offset, length)
+                .map_err(AsyncIoError::PunchHole)?;
+            // Deliver the completion through the normal io_uring path by
+            // queuing a NOP carrying `user_data`. The registered eventfd will
+            // fire when it completes, just like any other request.
+            return self
+                .data_io
+                .submit_nop(user_data)
+                .map_err(AsyncIoError::PunchHole);
+        }
+
+        let mode = FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE;
+
+        self.data_io
+            .submit_fallocate(self.raw_file.as_raw_fd(), offset, length, mode, user_data)
+            .map_err(AsyncIoError::PunchHole)
+    }
+
+    fn write_zeroes(&mut self, offset: u64, length: u64, user_data: u64) -> AsyncIoResult<()> {
+        // Same rationale as punch_hole().
+        if self.is_block_device {
+            blkzeroout(self.raw_file.as_raw_fd(), offset, length)
+                .map_err(AsyncIoError::WriteZeroes)?;
+            return self
+                .data_io
+                .submit_nop(user_data)
+                .map_err(AsyncIoError::WriteZeroes);
+        }
+
+        let mode = FALLOC_FL_ZERO_RANGE | FALLOC_FL_KEEP_SIZE;
+
+        self.data_io
+            .submit_fallocate(self.raw_file.as_raw_fd(), offset, length, mode, user_data)
+            .map_err(AsyncIoError::WriteZeroes)
+    }
+}

block/src/formats/raw/mod.rs

@@ -2,6 +2,11 @@
 //
 // SPDX-License-Identifier: Apache-2.0 AND BSD-3-Clause
 
+//! Raw disk image format.
+//!
+//! Provides [`RawDisk`], the `DiskFile` wrapper for flat disk images
+//! with no metadata or copy on write layer.
+
 use std::fs::File;
 use std::io;
 use std::os::unix::fs::FileTypeExt;
@@ -9,13 +14,22 @@ use std::os::unix::io::AsRawFd;
 
 use log::warn;
 
-use crate::async_io::{AsyncIo, BorrowedDiskFd, DiskFileError};
-use crate::error::{BlockError, BlockErrorKind, BlockResult};
+use self::engine_aio::RawAio;
+use self::engine_sync::RawSync;
 #[cfg(feature = "io_uring")]
-use crate::raw_async::RawFileAsync;
-use crate::raw_async_aio::RawFileAsyncAio;
-use crate::raw_sync::RawFileSync;
-use crate::{DiskTopology, disk_file, probe_sparse_support, query_device_size};
+use self::engine_uring::RawAsync;
+use crate::async_io::{
+    AsyncIo, AsyncIoError, AsyncIoOperation, AsyncIoResult, BorrowedDiskFd, DiskFileError,
+};
+use crate::error::{BlockError, BlockErrorKind, BlockResult};
+use crate::{AlignedFile, DiskTopology, disk_file, probe_sparse_support, query_device_size};
+
+mod engine_aio;
+pub(crate) mod engine_sync;
+#[cfg(feature = "io_uring")]
+pub(crate) mod engine_uring;
+#[cfg(test)]
+mod tests;
 
 /// Selects which async I/O backend a `RawDisk` uses.
 #[derive(Clone, Copy, Debug, PartialEq)]
@@ -39,11 +53,16 @@ pub enum RawBackend {
 pub struct RawDisk {
     file: File,
     backend: RawBackend,
+    direct: bool,
 }
 
 impl RawDisk {
-    pub fn new(file: File, backend: RawBackend) -> Self {
-        Self { file, backend }
+    pub fn new(file: File, backend: RawBackend, direct: bool) -> Self {
+        Self {
+            file,
+            backend,
+            direct,
+        }
     }
 }
 
@@ -124,25 +143,60 @@ impl disk_file::AsyncDiskFile for RawDisk {
         Ok(Box::new(RawDisk {
             file,
             backend: self.backend,
+            direct: self.direct,
         }))
     }
 
     fn create_async_io(&self, ring_depth: u32) -> BlockResult<Box<dyn AsyncIo>> {
+        let file = self
+            .file
+            .try_clone()
+            .map_err(|e| BlockError::new(BlockErrorKind::Io, DiskFileError::Clone(e)))?;
+        let raw_file = AlignedFile::new(file, self.direct);
         match self.backend {
-            RawBackend::Sync => Ok(Box::new(RawFileSync::new(self.file.as_raw_fd()))),
+            RawBackend::Sync => Ok(Box::new(RawSync::new(raw_file))),
             #[cfg(feature = "io_uring")]
-            RawBackend::IoUring => Ok(Box::new(RawFileAsync::new(
-                self.file.as_raw_fd(),
-                ring_depth,
-            )?)),
-            RawBackend::Aio => Ok(Box::new(RawFileAsyncAio::new(
-                self.file.as_raw_fd(),
-                ring_depth,
-            )?)),
+            RawBackend::IoUring => Ok(Box::new(RawAsync::new(raw_file, ring_depth)?)),
+            RawBackend::Aio => Ok(Box::new(RawAio::new(raw_file, ring_depth)?)),
         }
     }
 }
 
+/// True when `op` satisfies `alignment` and can go straight to the kernel.
+fn operation_is_aligned(op: &AsyncIoOperation, alignment: u64) -> bool {
+    if alignment == 0 {
+        return true;
+    }
+    if !(op.offset() as u64).is_multiple_of(alignment) {
+        return false;
+    }
+    op.iovecs().iter().all(|iov| {
+        (iov.iov_base as u64).is_multiple_of(alignment)
+            && (iov.iov_len as u64).is_multiple_of(alignment)
+    })
+}
+
+/// Runs an unaligned O_DIRECT operation synchronously through `aligned_file`.
+fn run_unaligned_operation(
+    aligned_file: &AlignedFile,
+    op: &mut AsyncIoOperation,
+) -> AsyncIoResult<i32> {
+    let offset = op.offset() as u64;
+    let total_len = op.total_len();
+
+    if op.is_read() {
+        let n = aligned_file
+            .read_unaligned(offset, total_len, |data| op.write_bytes_at(0, data))
+            .map_err(AsyncIoError::ReadVectored)?;
+        Ok(n as i32)
+    } else {
+        let n = aligned_file
+            .write_unaligned(offset, total_len, |data| op.read_bytes_at(0, data))
+            .map_err(AsyncIoError::WriteVectored)?;
+        Ok(n as i32)
+    }
+}
+
 #[cfg(test)]
 mod unit_tests {
     use std::fs::File;
@@ -164,7 +218,7 @@ mod unit_tests {
     #[test]
     fn new_sync_returns_correct_size() {
         let file = make_raw_file();
-        let disk = RawDisk::new(file, RawBackend::Sync);
+        let disk = RawDisk::new(file, RawBackend::Sync, false);
         assert_eq!(disk.logical_size().unwrap(), TEST_SIZE);
     }
 
@@ -200,14 +254,14 @@ mod unit_tests {
     #[test]
     fn sync_backend_disables_batch_requests() {
         let file = make_raw_file();
-        let disk = RawDisk::new(file, RawBackend::Sync);
+        let disk = RawDisk::new(file, RawBackend::Sync, false);
         assert_sync_backend(&disk);
     }
 
     #[test]
     fn aio_backend_disables_batch_requests() {
         let file = make_raw_file();
-        let disk = RawDisk::new(file, RawBackend::Aio);
+        let disk = RawDisk::new(file, RawBackend::Aio, false);
         assert_aio_backend(&disk);
     }
 
@@ -215,7 +269,7 @@ mod unit_tests {
     #[test]
     fn io_uring_backend_enables_batch_requests() {
         let file = make_raw_file();
-        let disk = RawDisk::new(file, RawBackend::IoUring);
+        let disk = RawDisk::new(file, RawBackend::IoUring, false);
         assert_io_uring_backend(&disk);
     }
 
@@ -227,14 +281,14 @@ mod unit_tests {
     #[test]
     fn try_clone_preserves_sync_backend() {
         let file = make_raw_file();
-        let disk = RawDisk::new(file, RawBackend::Sync);
+        let disk = RawDisk::new(file, RawBackend::Sync, false);
         assert_try_clone(&disk, RawBackend::Sync);
     }
 
     #[test]
     fn try_clone_preserves_aio_backend() {
         let file = make_raw_file();
-        let disk = RawDisk::new(file, RawBackend::Aio);
+        let disk = RawDisk::new(file, RawBackend::Aio, false);
         assert_try_clone(&disk, RawBackend::Aio);
     }
 
@@ -242,14 +296,14 @@ mod unit_tests {
     #[test]
     fn try_clone_preserves_io_uring_backend() {
         let file = make_raw_file();
-        let disk = RawDisk::new(file, RawBackend::IoUring);
+        let disk = RawDisk::new(file, RawBackend::IoUring, false);
         assert_try_clone(&disk, RawBackend::IoUring);
     }
 
     #[test]
     fn resize_changes_file_size() {
         let file = make_raw_file();
-        let mut disk = RawDisk::new(file, RawBackend::Aio);
+        let mut disk = RawDisk::new(file, RawBackend::Aio, false);
         let new_size = TEST_SIZE * 2;
         disk.resize(new_size).unwrap();
         assert_eq!(disk.logical_size().unwrap(), new_size);
@@ -258,7 +312,7 @@ mod unit_tests {
     #[test]
     fn physical_size_reports_allocated_blocks() {
         let file = make_raw_file();
-        let disk = RawDisk::new(file, RawBackend::Aio);
+        let disk = RawDisk::new(file, RawBackend::Aio, false);
         // Sparse file: physical size is less than logical size.
         assert!(disk.physical_size().unwrap() < disk.logical_size().unwrap());
     }

block/src/formats/raw/tests.rs

@@ -1,5 +1,7 @@
 // Copyright 2026 The Cloud Hypervisor Authors. All rights reserved.
 //
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+//
 // SPDX-License-Identifier: Apache-2.0 AND BSD-3-Clause
 
 //! Shared test helpers for [`AsyncIo`] backends.
@@ -13,6 +15,11 @@ use std::io::{Read, Seek, SeekFrom, Write};
 
 use crate::async_io::{AsyncIo, AsyncIoError};
 
+fn next_completion(async_io: &mut dyn AsyncIo) -> (u64, i32) {
+    let completion = async_io.next_completed_request().expect("No completion");
+    (completion.user_data, completion.result)
+}
+
 /// Tests punching a hole in the middle of a 4 MB file and verifying data
 /// integrity around the hole.
 pub fn test_punch_hole(async_io: &mut dyn AsyncIo, file: &mut File) {
@@ -27,7 +34,7 @@ pub fn test_punch_hole(async_io: &mut dyn AsyncIo, file: &mut File) {
     async_io.punch_hole(offset, length, 1).unwrap();
 
     // Check completion
-    let (user_data, result) = async_io.next_completed_request().unwrap();
+    let (user_data, result) = next_completion(async_io);
     assert_eq!(user_data, 1);
     assert_eq!(result, 0);
 
@@ -84,7 +91,7 @@ pub fn test_write_zeroes(async_io: &mut dyn AsyncIo, file: &mut File) {
     write_zeroes_result.unwrap();
 
     // Check completion
-    let (user_data, result) = async_io.next_completed_request().unwrap();
+    let (user_data, result) = next_completion(async_io);
     assert_eq!(user_data, 2);
     assert_eq!(result, 0);
 
@@ -134,15 +141,15 @@ pub fn test_punch_hole_multiple_operations(async_io: &mut dyn AsyncIo, file: &mu
         .unwrap();
 
     // Check all completions
-    let (user_data, result) = async_io.next_completed_request().unwrap();
+    let (user_data, result) = next_completion(async_io);
     assert_eq!(user_data, 10);
     assert_eq!(result, 0);
 
-    let (user_data, result) = async_io.next_completed_request().unwrap();
+    let (user_data, result) = next_completion(async_io);
     assert_eq!(user_data, 11);
     assert_eq!(result, 0);
 
-    let (user_data, result) = async_io.next_completed_request().unwrap();
+    let (user_data, result) = next_completion(async_io);
     assert_eq!(user_data, 12);
     assert_eq!(result, 0);
 

block/src/formats/vhd/engine_sync.rs

@@ -0,0 +1,58 @@
+// Copyright © 2021 Intel Corporation
+//
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+use std::io;
+
+use vmm_sys_util::eventfd::EventFd;
+
+use crate::AlignedFile;
+use crate::async_io::{AsyncIo, AsyncIoCompletion, AsyncIoError, AsyncIoOperation, AsyncIoResult};
+use crate::formats::raw::engine_sync::RawSync;
+
+pub(super) struct FixedVhdSync {
+    raw_file_sync: RawSync,
+    size: u64,
+}
+
+impl FixedVhdSync {
+    pub(super) fn new(raw_file: AlignedFile, size: u64) -> Self {
+        FixedVhdSync {
+            raw_file_sync: RawSync::new(raw_file),
+            size,
+        }
+    }
+}
+
+impl AsyncIo for FixedVhdSync {
+    fn notifier(&self) -> &EventFd {
+        self.raw_file_sync.notifier()
+    }
+
+    fn submit_data_operation(&mut self, op: AsyncIoOperation) -> AsyncIoResult<()> {
+        op.validate_bounds(self.size)?;
+        self.raw_file_sync.submit_data_operation(op)
+    }
+
+    fn fsync(&mut self, user_data: Option<u64>) -> AsyncIoResult<()> {
+        self.raw_file_sync.fsync(user_data)
+    }
+
+    fn next_completed_request(&mut self) -> Option<AsyncIoCompletion> {
+        self.raw_file_sync.next_completed_request()
+    }
+
+    fn punch_hole(&mut self, _offset: u64, _length: u64, _user_data: u64) -> AsyncIoResult<()> {
+        Err(AsyncIoError::PunchHole(io::Error::other(
+            "punch_hole not supported for fixed VHD",
+        )))
+    }
+
+    fn write_zeroes(&mut self, _offset: u64, _length: u64, _user_data: u64) -> AsyncIoResult<()> {
+        Err(AsyncIoError::WriteZeroes(io::Error::other(
+            "write_zeroes not supported for fixed VHD",
+        )))
+    }
+}

block/src/formats/vhd/engine_uring.rs

@@ -0,0 +1,73 @@
+// Copyright © 2021 Intel Corporation
+//
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+use std::io;
+
+use vmm_sys_util::eventfd::EventFd;
+
+use crate::AlignedFile;
+use crate::async_io::{AsyncIo, AsyncIoCompletion, AsyncIoError, AsyncIoOperation, AsyncIoResult};
+use crate::error::BlockResult;
+use crate::formats::raw::engine_uring::RawAsync;
+
+pub(super) struct FixedVhdAsync {
+    raw_file_async: RawAsync,
+    size: u64,
+}
+
+impl FixedVhdAsync {
+    pub(super) fn new(raw_file: AlignedFile, ring_depth: u32, size: u64) -> BlockResult<Self> {
+        let raw_file_async = RawAsync::new(raw_file, ring_depth)?;
+
+        Ok(FixedVhdAsync {
+            raw_file_async,
+            size,
+        })
+    }
+}
+
+impl AsyncIo for FixedVhdAsync {
+    fn notifier(&self) -> &EventFd {
+        self.raw_file_async.notifier()
+    }
+
+    fn submit_data_operation(&mut self, op: AsyncIoOperation) -> AsyncIoResult<()> {
+        op.validate_bounds(self.size)?;
+        self.raw_file_async.submit_data_operation(op)
+    }
+
+    fn fsync(&mut self, user_data: Option<u64>) -> AsyncIoResult<()> {
+        self.raw_file_async.fsync(user_data)
+    }
+
+    fn next_completed_request(&mut self) -> Option<AsyncIoCompletion> {
+        self.raw_file_async.next_completed_request()
+    }
+
+    fn punch_hole(&mut self, _offset: u64, _length: u64, _user_data: u64) -> AsyncIoResult<()> {
+        Err(AsyncIoError::PunchHole(io::Error::other(
+            "punch_hole not supported for fixed VHD",
+        )))
+    }
+
+    fn write_zeroes(&mut self, _offset: u64, _length: u64, _user_data: u64) -> AsyncIoResult<()> {
+        Err(AsyncIoError::WriteZeroes(io::Error::other(
+            "write_zeroes not supported for fixed VHD",
+        )))
+    }
+
+    fn batch_requests_enabled(&self) -> bool {
+        true
+    }
+
+    fn submit_batch_requests(&mut self, batch_request: Vec<AsyncIoOperation>) -> AsyncIoResult<()> {
+        for op in &batch_request {
+            op.validate_bounds(self.size)?;
+        }
+
+        self.raw_file_async.submit_batch_requests(batch_request)
+    }
+}

block/src/formats/vhd/fixed.rs

@@ -0,0 +1,59 @@
+// Copyright © 2021 Intel Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+
+use std::fs::File;
+use std::io;
+use std::os::unix::io::{AsRawFd, RawFd};
+
+use super::footer::VhdFooter;
+
+#[derive(Debug)]
+pub(super) struct FixedVhd {
+    file: File,
+    size: u64,
+}
+
+impl FixedVhd {
+    pub(super) fn new(mut file: File) -> io::Result<Self> {
+        let footer = VhdFooter::new(&mut file)?;
+
+        Ok(Self {
+            file,
+            size: footer.current_size(),
+        })
+    }
+
+    pub(crate) fn file(&self) -> &File {
+        &self.file
+    }
+}
+
+impl AsRawFd for FixedVhd {
+    fn as_raw_fd(&self) -> RawFd {
+        self.file.as_raw_fd()
+    }
+}
+
+impl FixedVhd {
+    pub(crate) fn logical_size(&self) -> Result<u64, crate::Error> {
+        Ok(self.size)
+    }
+
+    /// Returns the physical size of the underlying file.
+    pub(crate) fn physical_size(&self) -> Result<u64, crate::Error> {
+        self.file
+            .metadata()
+            .map(|m| m.len())
+            .map_err(crate::Error::GetFileMetadata)
+    }
+}
+
+impl Clone for FixedVhd {
+    fn clone(&self) -> Self {
+        Self {
+            file: self.file.try_clone().expect("FixedVhd cloning failed"),
+            size: self.size,
+        }
+    }
+}

block/src/formats/vhd/footer.rs

@@ -3,12 +3,17 @@
 // SPDX-License-Identifier: Apache-2.0
 
 use std::fs::File;
-use std::io::{Seek, SeekFrom};
+use std::io;
+use std::os::unix::fs::FileExt;
 
-use crate::{DiskTopology, read_aligned_block_size};
+use crate::{AlignedFile, query_device_size};
 
+// Production code uses: cookie, file_format_version, data_offset,
+// current_size, disk_type. The remaining fields are parsed for VHD
+// spec completeness and exercised only by unit tests.
 #[derive(Clone, Copy)]
-pub struct VhdFooter {
+#[cfg_attr(not(test), expect(dead_code))]
+pub(super) struct VhdFooter {
     cookie: u64,
     features: u32,
     file_format_version: u32,
@@ -27,17 +32,14 @@ pub struct VhdFooter {
 }
 
 impl VhdFooter {
-    pub fn new(file: &mut File) -> std::io::Result<VhdFooter> {
-        let blocksize = DiskTopology::probe(file)?.logical_block_size as usize;
-
-        // Place the cursor in the last block of the file
-        file.seek(SeekFrom::End(0 - (blocksize as i64)))?;
-        // Read in the last block
-        let data = read_aligned_block_size(file)?;
-
-        // We only care about the last sector
-        let offset = blocksize - 512;
-        let sector = &data[offset..];
+    pub(super) fn new(file: &mut File) -> io::Result<VhdFooter> {
+        let aligned = AlignedFile::new(file.try_clone()?, true);
+        let size = query_device_size(file)?.0;
+        let footer_offset = size.checked_sub(512).ok_or_else(|| {
+            io::Error::new(io::ErrorKind::InvalidInput, "file too small for VHD footer")
+        })?;
+        let mut sector = [0u8; 512];
+        aligned.read_exact_at(&mut sector, footer_offset)?;
 
         Ok(VhdFooter {
             cookie: u64::from_be_bytes(sector[0..8].try_into().unwrap()),
@@ -58,55 +60,65 @@ impl VhdFooter {
         })
     }
 
-    pub fn cookie(&self) -> u64 {
+    pub(super) fn cookie(&self) -> u64 {
         self.cookie
     }
+    #[cfg(test)]
     pub fn features(&self) -> u32 {
         self.features
     }
-    pub fn file_format_version(&self) -> u32 {
+    pub(super) fn file_format_version(&self) -> u32 {
         self.file_format_version
     }
-    pub fn data_offset(&self) -> u64 {
+    pub(super) fn data_offset(&self) -> u64 {
         self.data_offset
     }
+    #[cfg(test)]
     pub fn time_stamp(&self) -> u32 {
         self.time_stamp
     }
+    #[cfg(test)]
     pub fn creator_application(&self) -> u32 {
         self.creator_application
     }
+    #[cfg(test)]
     pub fn creator_version(&self) -> u32 {
         self.creator_version
     }
+    #[cfg(test)]
     pub fn creator_host_os(&self) -> u32 {
         self.creator_host_os
     }
+    #[cfg(test)]
     pub fn original_size(&self) -> u64 {
         self.original_size
     }
-    pub fn current_size(&self) -> u64 {
+    pub(super) fn current_size(&self) -> u64 {
         self.current_size
     }
+    #[cfg(test)]
     pub fn disk_geometry(&self) -> u32 {
         self.disk_geometry
     }
-    pub fn disk_type(&self) -> u32 {
+    pub(super) fn disk_type(&self) -> u32 {
         self.disk_type
     }
+    #[cfg(test)]
     pub fn checksum(&self) -> u32 {
         self.checksum
     }
+    #[cfg(test)]
     pub fn unique_id(&self) -> u128 {
         self.unique_id
     }
+    #[cfg(test)]
     pub fn saved_state(&self) -> u8 {
         self.saved_state
     }
 }
 
 /// Determine image type through file parsing.
-pub fn is_fixed_vhd(f: &mut File) -> std::io::Result<bool> {
+pub fn is_fixed_vhd(f: &mut File) -> io::Result<bool> {
     let footer = VhdFooter::new(f)?;
 
     // "conectix" => 0x636f6e6563746978

block/src/formats/vhd/mod.rs

@@ -1,28 +1,45 @@
 // Copyright 2026 The Cloud Hypervisor Authors. All rights reserved.
 //
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+//
 // SPDX-License-Identifier: Apache-2.0
 
+//! Fixed VHD disk image format.
+//!
+//! Provides [`VhdDisk`], the `DiskFile` wrapper for fixed size VHD
+//! images.
+
+mod engine_sync;
+#[cfg(feature = "io_uring")]
+mod engine_uring;
+mod fixed;
+mod footer;
+
 use std::fs::File;
 use std::io;
 use std::os::unix::io::AsRawFd;
 
+pub use footer::is_fixed_vhd;
+use log::warn;
+
+use self::engine_sync::FixedVhdSync;
+#[cfg(feature = "io_uring")]
+use self::engine_uring::FixedVhdAsync;
+use self::fixed::FixedVhd;
 use crate::async_io::{AsyncIo, BorrowedDiskFd, DiskFileError};
 use crate::disk_file::DiskSize;
 use crate::error::{BlockError, BlockErrorKind, BlockResult, ErrorOp};
-use crate::fixed_vhd::FixedVhd;
-#[cfg(feature = "io_uring")]
-use crate::fixed_vhd_async::FixedVhdAsync;
-use crate::fixed_vhd_sync::FixedVhdSync;
-use crate::{BlockBackend, Error, disk_file};
+use crate::{AlignedFile, DiskTopology, Error, disk_file};
 
 #[derive(Debug)]
-pub struct FixedVhdDisk {
+pub struct VhdDisk {
     inner: FixedVhd,
     use_io_uring: bool,
+    direct: bool,
 }
 
-impl FixedVhdDisk {
-    pub fn new(file: File, use_io_uring: bool) -> BlockResult<Self> {
+impl VhdDisk {
+    pub fn new(file: File, use_io_uring: bool, direct: bool) -> BlockResult<Self> {
         #[cfg(not(feature = "io_uring"))]
         if use_io_uring {
             return Err(BlockError::new(
@@ -36,11 +53,12 @@ impl FixedVhdDisk {
         Ok(Self {
             inner: FixedVhd::new(file).map_err(|e| BlockError::from(e).with_op(ErrorOp::Open))?,
             use_io_uring,
+            direct,
         })
     }
 }
 
-impl disk_file::DiskSize for FixedVhdDisk {
+impl disk_file::DiskSize for VhdDisk {
     fn logical_size(&self) -> BlockResult<u64> {
         self.inner
             .logical_size()
@@ -48,7 +66,7 @@ impl disk_file::DiskSize for FixedVhdDisk {
     }
 }
 
-impl disk_file::PhysicalSize for FixedVhdDisk {
+impl disk_file::PhysicalSize for VhdDisk {
     fn physical_size(&self) -> BlockResult<u64> {
         self.inner.physical_size().map_err(|e| match e {
             Error::GetFileMetadata(io) => {
@@ -59,17 +77,24 @@ impl disk_file::PhysicalSize for FixedVhdDisk {
     }
 }
 
-impl disk_file::DiskFd for FixedVhdDisk {
+impl disk_file::DiskFd for VhdDisk {
     fn fd(&self) -> BorrowedDiskFd<'_> {
         BorrowedDiskFd::new(self.inner.as_raw_fd())
     }
 }
 
-impl disk_file::Geometry for FixedVhdDisk {}
+impl disk_file::Geometry for VhdDisk {
+    fn topology(&self) -> DiskTopology {
+        DiskTopology::probe(self.inner.file()).unwrap_or_else(|_| {
+            warn!("Unable to get device topology. Using default topology");
+            DiskTopology::default()
+        })
+    }
+}
 
-impl disk_file::SparseCapable for FixedVhdDisk {}
+impl disk_file::SparseCapable for VhdDisk {}
 
-impl disk_file::Resizable for FixedVhdDisk {
+impl disk_file::Resizable for VhdDisk {
     fn resize(&mut self, _size: u64) -> BlockResult<()> {
         Err(BlockError::new(
             BlockErrorKind::UnsupportedFeature,
@@ -79,27 +104,28 @@ impl disk_file::Resizable for FixedVhdDisk {
     }
 }
 
-impl disk_file::DiskFile for FixedVhdDisk {}
+impl disk_file::DiskFile for VhdDisk {}
 
-impl disk_file::AsyncDiskFile for FixedVhdDisk {
+impl disk_file::AsyncDiskFile for VhdDisk {
     fn try_clone(&self) -> BlockResult<Box<dyn disk_file::AsyncDiskFile>> {
-        Ok(Box::new(FixedVhdDisk {
+        Ok(Box::new(VhdDisk {
             inner: self.inner.clone(),
             use_io_uring: self.use_io_uring,
+            direct: self.direct,
         }))
     }
 
     fn create_async_io(&self, ring_depth: u32) -> BlockResult<Box<dyn AsyncIo>> {
         let size = self.logical_size()?;
+        let file = self.inner.file().try_clone().map_err(|e| {
+            BlockError::new(BlockErrorKind::Io, DiskFileError::NewAsyncIo(e)).with_op(ErrorOp::Open)
+        })?;
+        let raw_file = AlignedFile::new(file, self.direct);
 
         if self.use_io_uring {
             #[cfg(feature = "io_uring")]
             {
-                return Ok(Box::new(FixedVhdAsync::new(
-                    self.inner.as_raw_fd(),
-                    ring_depth,
-                    size,
-                )?));
+                return Ok(Box::new(FixedVhdAsync::new(raw_file, ring_depth, size)?));
             }
 
             #[cfg(not(feature = "io_uring"))]
@@ -107,12 +133,7 @@ impl disk_file::AsyncDiskFile for FixedVhdDisk {
         }
 
         let _ = ring_depth;
-        Ok(Box::new(
-            FixedVhdSync::new(self.inner.as_raw_fd(), size).map_err(|e| {
-                BlockError::new(BlockErrorKind::Io, DiskFileError::NewAsyncIo(e))
-                    .with_op(ErrorOp::Open)
-            })?,
-        ))
+        Ok(Box::new(FixedVhdSync::new(raw_file, size)))
     }
 }
 
@@ -124,7 +145,7 @@ mod unit_tests {
     use vmm_sys_util::tempfile::TempFile;
 
     use super::*;
-    use crate::async_io::AsyncIo;
+    use crate::async_io::{AsyncIo, AsyncIoError, AsyncIoOperation, OwnedIoBuffer};
     use crate::disk_file::{AsyncDiskFile, DiskSize, PhysicalSize, Resizable};
 
     /// Minimal fixed VHD footer (disk type = 2, current_size = 0x11223344).
@@ -160,7 +181,7 @@ mod unit_tests {
     #[test]
     fn new_sync_returns_correct_size() {
         let file = make_vhd_file();
-        let disk = FixedVhdDisk::new(file, false).unwrap();
+        let disk = VhdDisk::new(file, false, false).unwrap();
         assert_eq!(disk.logical_size().unwrap(), 0x1122_3344);
     }
 
@@ -169,14 +190,14 @@ mod unit_tests {
         assert_eq!(io.batch_requests_enabled(), expect_batch);
     }
 
-    fn assert_async_io(disk: &FixedVhdDisk, expect_batch: bool) {
+    fn assert_async_io(disk: &VhdDisk, expect_batch: bool) {
         assert_async_io_from_dyn(disk, expect_batch);
     }
 
     #[test]
     fn sync_backend_disables_batch_requests() {
         let file = make_vhd_file();
-        let disk = FixedVhdDisk::new(file, false).unwrap();
+        let disk = VhdDisk::new(file, false, false).unwrap();
         assert_async_io(&disk, false);
     }
 
@@ -184,14 +205,103 @@ mod unit_tests {
     #[test]
     fn io_uring_backend_enables_batch_requests() {
         let file = make_vhd_file();
-        let disk = FixedVhdDisk::new(file, true).unwrap();
+        let disk = VhdDisk::new(file, true, false).unwrap();
         assert_async_io(&disk, true);
     }
 
+    #[test]
+    fn sync_rejects_read_straddling_logical_size() {
+        let file = TempFile::new().unwrap().into_file();
+        file.set_len(0x2000).unwrap();
+        let mut sync_io =
+            FixedVhdSync::new(AlignedFile::new(file.try_clone().unwrap(), false), 0x1000);
+        let op = AsyncIoOperation::read_to_vec(0x800, OwnedIoBuffer::from_vec(vec![0; 0x900]), 1);
+
+        assert!(matches!(
+            sync_io.submit_data_operation(op),
+            Err(AsyncIoError::ReadVectored(_))
+        ));
+    }
+
+    #[test]
+    fn sync_rejects_write_straddling_logical_size() {
+        let file = TempFile::new().unwrap().into_file();
+        file.set_len(0x2000).unwrap();
+        let mut sync_io =
+            FixedVhdSync::new(AlignedFile::new(file.try_clone().unwrap(), false), 0x1000);
+        let op =
+            AsyncIoOperation::write_from_vec(0x800, OwnedIoBuffer::from_vec(vec![0; 0x900]), 1);
+
+        assert!(matches!(
+            sync_io.submit_data_operation(op),
+            Err(AsyncIoError::WriteVectored(_))
+        ));
+    }
+
+    #[test]
+    fn sync_accepts_operation_exactly_filling_logical_size() {
+        let file = TempFile::new().unwrap().into_file();
+        file.set_len(0x2000).unwrap();
+        let mut sync_io =
+            FixedVhdSync::new(AlignedFile::new(file.try_clone().unwrap(), false), 0x1000);
+        // end == size: boundary must be accepted
+        let op = AsyncIoOperation::read_to_vec(0, OwnedIoBuffer::from_vec(vec![0; 0x1000]), 1);
+        sync_io.submit_data_operation(op).unwrap();
+    }
+
+    #[test]
+    fn sync_accepts_operation_at_last_byte() {
+        let file = TempFile::new().unwrap().into_file();
+        file.set_len(0x2000).unwrap();
+        let mut sync_io =
+            FixedVhdSync::new(AlignedFile::new(file.try_clone().unwrap(), false), 0x1000);
+        // end = 0xFFF + 1 = 0x1000 == size: boundary must be accepted
+        let op = AsyncIoOperation::read_to_vec(0xFFF, OwnedIoBuffer::from_vec(vec![0; 1]), 1);
+        sync_io.submit_data_operation(op).unwrap();
+    }
+
+    #[cfg(feature = "io_uring")]
+    #[test]
+    fn io_uring_batch_rejects_request_straddling_logical_size() {
+        let file = TempFile::new().unwrap().into_file();
+        file.set_len(0x2000).unwrap();
+        let mut async_io = FixedVhdAsync::new(
+            AlignedFile::new(file.try_clone().unwrap(), false),
+            8,
+            0x1000,
+        )
+        .unwrap();
+        let op = AsyncIoOperation::read_to_vec(0x800, OwnedIoBuffer::from_vec(vec![0; 0x900]), 1);
+
+        assert!(matches!(
+            async_io.submit_batch_requests(vec![op]),
+            Err(AsyncIoError::ReadVectored(_))
+        ));
+    }
+
+    #[cfg(feature = "io_uring")]
+    #[test]
+    fn io_uring_rejects_single_op_straddling_logical_size() {
+        let file = TempFile::new().unwrap().into_file();
+        file.set_len(0x2000).unwrap();
+        let mut async_io = FixedVhdAsync::new(
+            AlignedFile::new(file.try_clone().unwrap(), false),
+            8,
+            0x1000,
+        )
+        .unwrap();
+        let op = AsyncIoOperation::read_to_vec(0x800, OwnedIoBuffer::from_vec(vec![0; 0x900]), 1);
+
+        assert!(matches!(
+            async_io.submit_data_operation(op),
+            Err(AsyncIoError::ReadVectored(_))
+        ));
+    }
+
     #[test]
     fn try_clone_preserves_sync_dispatch() {
         let file = make_vhd_file();
-        let disk = FixedVhdDisk::new(file, false).unwrap();
+        let disk = VhdDisk::new(file, false, false).unwrap();
         let cloned = disk.try_clone().unwrap();
         assert_async_io_from_dyn(cloned.as_ref(), false);
     }
@@ -200,7 +310,7 @@ mod unit_tests {
     #[test]
     fn try_clone_preserves_io_uring_dispatch() {
         let file = make_vhd_file();
-        let disk = FixedVhdDisk::new(file, true).unwrap();
+        let disk = VhdDisk::new(file, true, false).unwrap();
         let cloned = disk.try_clone().unwrap();
         assert_async_io_from_dyn(cloned.as_ref(), true);
     }
@@ -208,14 +318,14 @@ mod unit_tests {
     #[test]
     fn resize_returns_error() {
         let file = make_vhd_file();
-        let mut disk = FixedVhdDisk::new(file, false).unwrap();
+        let mut disk = VhdDisk::new(file, false, false).unwrap();
         assert!(disk.resize(0x2000_0000).is_err());
     }
 
     #[test]
     fn physical_size_includes_footer() {
         let file = make_vhd_file();
-        let disk = FixedVhdDisk::new(file, false).unwrap();
+        let disk = VhdDisk::new(file, false, false).unwrap();
         // Data region (0x1122_3344) + VHD footer (0x200).
         assert_eq!(disk.physical_size().unwrap(), 0x1122_3344 + 0x200);
     }

block/src/formats/vhdx/bat.rs

@@ -2,29 +2,29 @@
 //
 // SPDX-License-Identifier: Apache-2.0
 
-use std::fs::File;
-use std::io::{self, Seek, SeekFrom};
-use std::mem::size_of;
+use std::os::unix::fs::FileExt;
+use std::{io, result};
 
-use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt};
+use byteorder::{ByteOrder, LittleEndian};
 use remain::sorted;
 use thiserror::Error;
 
-use crate::vhdx::vhdx_header::RegionTableEntry;
-use crate::vhdx::vhdx_metadata::DiskSpec;
+use super::header::RegionTableEntry;
+use super::metadata::DiskSpec;
+use crate::aligned_file::AlignedFile;
 
 // Payload BAT Entry States
-pub const PAYLOAD_BLOCK_NOT_PRESENT: u64 = 0;
-pub const PAYLOAD_BLOCK_UNDEFINED: u64 = 1;
-pub const PAYLOAD_BLOCK_ZERO: u64 = 2;
-pub const PAYLOAD_BLOCK_UNMAPPED: u64 = 3;
-pub const PAYLOAD_BLOCK_FULLY_PRESENT: u64 = 6;
-pub const PAYLOAD_BLOCK_PARTIALLY_PRESENT: u64 = 7;
+pub(super) const PAYLOAD_BLOCK_NOT_PRESENT: u64 = 0;
+pub(super) const PAYLOAD_BLOCK_UNDEFINED: u64 = 1;
+pub(super) const PAYLOAD_BLOCK_ZERO: u64 = 2;
+pub(super) const PAYLOAD_BLOCK_UNMAPPED: u64 = 3;
+pub(super) const PAYLOAD_BLOCK_FULLY_PRESENT: u64 = 6;
+pub(super) const PAYLOAD_BLOCK_PARTIALLY_PRESENT: u64 = 7;
 
 // Mask for the BAT state
-pub const BAT_STATE_BIT_MASK: u64 = 0x07;
+pub(super) const BAT_STATE_BIT_MASK: u64 = 0x07;
 // Mask for the offset within the file in units of 1 MB
-pub const BAT_FILE_OFF_MASK: u64 = 0xFFFFFFFFFFF00000;
+pub(super) const BAT_FILE_OFF_MASK: u64 = 0xFFFFFFFFFFF00000;
 
 #[sorted]
 #[derive(Error, Debug)]
@@ -39,15 +39,15 @@ pub enum VhdxBatError {
     WriteBat(#[source] io::Error),
 }
 
-pub type Result<T> = std::result::Result<T, VhdxBatError>;
+pub(super) type Result<T> = result::Result<T, VhdxBatError>;
 
 #[derive(Default, Clone, Debug)]
-pub struct BatEntry(pub u64);
+pub(super) struct BatEntry(pub u64);
 
 impl BatEntry {
     // Read all BAT entries presented on the disk and insert them to a vector
-    pub fn collect_bat_entries(
-        f: &mut File,
+    pub(super) fn collect_bat_entries(
+        f: &AlignedFile,
         disk_spec: &DiskSpec,
         bat_entry: &RegionTableEntry,
     ) -> Result<Vec<BatEntry>> {
@@ -63,14 +63,10 @@ impl BatEntry {
         let mut bat: Vec<BatEntry> = Vec::with_capacity(bat_entry.length as usize);
         let offset = bat_entry.file_offset;
         for i in 0..entry_count {
-            f.seek(SeekFrom::Start(offset + i * size_of::<u64>() as u64))
+            let mut entry = [0u8; size_of::<u64>()];
+            f.read_exact_at(&mut entry, offset + i * size_of::<u64>() as u64)
                 .map_err(VhdxBatError::ReadBat)?;
-
-            let bat_entry = BatEntry(
-                f.read_u64::<LittleEndian>()
-                    .map_err(VhdxBatError::ReadBat)?,
-            );
-            bat.insert(i as usize, bat_entry);
+            bat.insert(i as usize, BatEntry(LittleEndian::read_u64(&entry)));
         }
 
         Ok(bat)
@@ -83,14 +79,12 @@ impl BatEntry {
     }
 
     // Routine for writing BAT entries to the disk
-    pub fn write_bat_entries(
-        f: &mut File,
+    pub(super) fn write_bat_entries(
+        f: &AlignedFile,
         bat_offset: u64,
         bat_entries: &[BatEntry],
     ) -> Result<()> {
         for i in 0..bat_entries.len() as u64 {
-            f.seek(SeekFrom::Start(bat_offset + i * size_of::<u64>() as u64))
-                .map_err(VhdxBatError::WriteBat)?;
             let bat_entry = match bat_entries.get(i as usize) {
                 Some(entry) => entry.0,
                 None => {
@@ -98,7 +92,9 @@ impl BatEntry {
                 }
             };
 
-            f.write_u64::<LittleEndian>(bat_entry)
+            let mut buf = [0u8; size_of::<u64>()];
+            LittleEndian::write_u64(&mut buf, bat_entry);
+            f.write_all_at(&buf, bat_offset + i * size_of::<u64>() as u64)
                 .map_err(VhdxBatError::WriteBat)?;
         }
         Ok(())

block/src/formats/vhdx/engine_sync.rs

@@ -0,0 +1,208 @@
+// Copyright © 2021 Intel Corporation
+//
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+use std::collections::VecDeque;
+use std::io::{self, Read, Seek, SeekFrom, Write};
+use std::sync::{Arc, Mutex};
+
+use vmm_sys_util::eventfd::EventFd;
+
+use crate::async_io::{AsyncIo, AsyncIoCompletion, AsyncIoError, AsyncIoOperation, AsyncIoResult};
+use crate::formats::vhdx::Vhdx;
+
+pub(super) struct VhdxSync {
+    vhdx_file: Arc<Mutex<Vhdx>>,
+    eventfd: EventFd,
+    completion_list: VecDeque<AsyncIoCompletion>,
+    size: u64,
+}
+
+impl VhdxSync {
+    pub(super) fn new(vhdx_file: Arc<Mutex<Vhdx>>, size: u64) -> Self {
+        VhdxSync {
+            vhdx_file,
+            eventfd: EventFd::new(libc::EFD_NONBLOCK)
+                .expect("Failed creating EventFd for VhdxSync"),
+            completion_list: VecDeque::new(),
+            size,
+        }
+    }
+
+    fn read_operation(&mut self, op: &mut AsyncIoOperation) -> AsyncIoResult<usize> {
+        let offset = op.offset();
+        let mut buf = vec![0u8; op.total_len()];
+        let mut vhdx = self.vhdx_file.lock().unwrap();
+        vhdx.seek(SeekFrom::Start(offset as u64))
+            .map_err(AsyncIoError::ReadVectored)?;
+        let result = vhdx.read(&mut buf).map_err(AsyncIoError::ReadVectored)?;
+        drop(vhdx);
+
+        op.write_bytes_at(0, &buf[..result])
+            .map_err(AsyncIoError::ReadVectored)?;
+        Ok(result)
+    }
+
+    fn write_operation(&mut self, op: &AsyncIoOperation) -> AsyncIoResult<usize> {
+        let offset = op.offset();
+        let mut buf = vec![0u8; op.total_len()];
+        op.read_bytes_at(0, &mut buf)
+            .map_err(AsyncIoError::WriteVectored)?;
+
+        let mut vhdx = self.vhdx_file.lock().unwrap();
+        vhdx.seek(SeekFrom::Start(offset as u64))
+            .map_err(AsyncIoError::WriteVectored)?;
+        let result = vhdx.write(&buf).map_err(AsyncIoError::WriteVectored)?;
+        Ok(result)
+    }
+}
+
+impl AsyncIo for VhdxSync {
+    fn notifier(&self) -> &EventFd {
+        &self.eventfd
+    }
+
+    fn submit_data_operation(&mut self, op: AsyncIoOperation) -> AsyncIoResult<()> {
+        op.validate_bounds(self.size)?;
+        let is_read = op.is_read();
+        let mut op = op;
+        let result = if is_read {
+            self.read_operation(&mut op)?
+        } else {
+            self.write_operation(&op)?
+        };
+
+        self.completion_list
+            .push_back(AsyncIoCompletion::from_operation(op, result as i32));
+        self.eventfd.write(1).unwrap();
+        Ok(())
+    }
+
+    fn fsync(&mut self, user_data: Option<u64>) -> AsyncIoResult<()> {
+        self.vhdx_file
+            .lock()
+            .unwrap()
+            .flush()
+            .map_err(AsyncIoError::Fsync)?;
+        if let Some(user_data) = user_data {
+            self.completion_list
+                .push_back(AsyncIoCompletion::new(user_data, 0, None));
+            self.eventfd.write(1).unwrap();
+        }
+        Ok(())
+    }
+
+    fn next_completed_request(&mut self) -> Option<AsyncIoCompletion> {
+        self.completion_list.pop_front()
+    }
+
+    fn punch_hole(&mut self, _offset: u64, _length: u64, _user_data: u64) -> AsyncIoResult<()> {
+        Err(AsyncIoError::PunchHole(io::Error::other(
+            "punch_hole not supported for VHDX",
+        )))
+    }
+
+    fn write_zeroes(&mut self, _offset: u64, _length: u64, _user_data: u64) -> AsyncIoResult<()> {
+        Err(AsyncIoError::WriteZeroes(io::Error::other(
+            "write_zeroes not supported for VHDX",
+        )))
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::fs;
+    use std::sync::{Arc, Mutex};
+
+    use vmm_sys_util::tempfile::TempFile;
+
+    use super::*;
+    use crate::async_io::{AsyncIo, AsyncIoError, AsyncIoOperation, OwnedIoBuffer};
+    use crate::formats::vhdx::Vhdx;
+    use crate::formats::vhdx::test_util::create_dynamic_vhdx;
+
+    fn make_vhdx_sync(tf: &TempFile) -> (VhdxSync, u64) {
+        let file = fs::OpenOptions::new()
+            .read(true)
+            .write(true)
+            .open(tf.as_path())
+            .unwrap();
+        let vhdx = Vhdx::new(file, false).unwrap();
+        let size = vhdx.virtual_disk_size();
+        let sync = VhdxSync::new(Arc::new(Mutex::new(vhdx)), size);
+        (sync, size)
+    }
+
+    /// Builds a `VhdxSync` from a fresh 1 MiB dynamic VHDX, or `None`
+    /// if `qemu-img` is unavailable to generate one.
+    fn setup() -> Option<(VhdxSync, u64)> {
+        let tf = create_dynamic_vhdx(1)?;
+        Some(make_vhdx_sync(&tf))
+    }
+
+    #[test]
+    fn sync_rejects_read_straddling_logical_size() {
+        let Some((mut sync, size)) = setup() else {
+            eprintln!("skipping: qemu-img unavailable");
+            return;
+        };
+
+        let op = AsyncIoOperation::read_to_vec(
+            (size - 512) as i64,
+            OwnedIoBuffer::from_vec(vec![0u8; 1024]),
+            1,
+        );
+        assert!(matches!(
+            sync.submit_data_operation(op),
+            Err(AsyncIoError::ReadVectored(_))
+        ));
+    }
+
+    #[test]
+    fn sync_rejects_write_straddling_logical_size() {
+        let Some((mut sync, size)) = setup() else {
+            eprintln!("skipping: qemu-img unavailable");
+            return;
+        };
+
+        let op = AsyncIoOperation::write_from_vec(
+            (size - 512) as i64,
+            OwnedIoBuffer::from_vec(vec![0u8; 1024]),
+            1,
+        );
+        assert!(matches!(
+            sync.submit_data_operation(op),
+            Err(AsyncIoError::WriteVectored(_))
+        ));
+    }
+
+    #[test]
+    fn sync_accepts_operation_exactly_filling_logical_size() {
+        let Some((mut sync, size)) = setup() else {
+            eprintln!("skipping: qemu-img unavailable");
+            return;
+        };
+
+        let op =
+            AsyncIoOperation::read_to_vec(0, OwnedIoBuffer::from_vec(vec![0u8; size as usize]), 1);
+        sync.submit_data_operation(op).unwrap();
+    }
+
+    #[test]
+    fn sync_accepts_operation_at_last_sector() {
+        let Some((mut sync, size)) = setup() else {
+            eprintln!("skipping: qemu-img unavailable");
+            return;
+        };
+
+        // VHDX operates in 512-byte sectors; read exactly the last sector.
+        let op = AsyncIoOperation::read_to_vec(
+            (size - 512) as i64,
+            OwnedIoBuffer::from_vec(vec![0u8; 512]),
+            1,
+        );
+        sync.submit_data_operation(op).unwrap();
+    }
+}

block/src/formats/vhdx/header.rs

@@ -3,14 +3,16 @@
 // SPDX-License-Identifier: Apache-2.0
 
 use std::collections::btree_map::BTreeMap;
-use std::fs::File;
-use std::io::{self, Read, Seek, SeekFrom, Write};
-use std::mem::size_of;
+use std::os::unix::fs::FileExt;
+use std::{io, result};
 
-use byteorder::{ByteOrder, LittleEndian, ReadBytesExt};
+use byteorder::{ByteOrder, LittleEndian};
 use remain::sorted;
 use thiserror::Error;
 use uuid::Uuid;
+use zerocopy::{FromBytes, Immutable, IntoBytes};
+
+use crate::aligned_file::AlignedFile;
 
 const VHDX_SIGN: u64 = 0x656C_6966_7864_6876; // "vhdxfile"
 const HEADER_SIGN: u32 = 0x6461_6568; // "head"
@@ -19,7 +21,7 @@ const REGION_SIGN: u32 = 0x6967_6572; // "regi"
 const FILE_START: u64 = 0; // The first element
 const HEADER_1_START: u64 = 64 * 1024; // Header 1 start in Bytes
 const HEADER_2_START: u64 = 128 * 1024; // Header 2 start in Bytes
-pub const REGION_TABLE_1_START: u64 = 192 * 1024; // Region 1 start in Bytes
+pub(super) const REGION_TABLE_1_START: u64 = 192 * 1024; // Region 1 start in Bytes
 const REGION_TABLE_2_START: u64 = 256 * 1024; // Region 2 start in Bytes
 
 const HEADER_SIZE: u64 = 4 * 1024; // Each header is 64 KiB, but only first 4 kiB contains info
@@ -27,8 +29,13 @@ const REGION_SIZE: u64 = 64 * 1024; // Each region size is 64 KiB
 
 const REGION_ENTRY_REQUIRED: u32 = 1;
 
-const BAT_GUID: &str = "2DC27766-F623-4200-9D64-115E9BFD4A08"; // BAT GUID
-const MDR_GUID: &str = "8B7CA206-4790-4B9A-B8FE-575F050F886E"; // Metadata GUID
+// VHDX stores GUIDs using little-endian GUID byte order.
+const BAT_GUID: [u8; 16] = [
+    0x66, 0x77, 0xc2, 0x2d, 0x23, 0xf6, 0x00, 0x42, 0x9d, 0x64, 0x11, 0x5e, 0x9b, 0xfd, 0x4a, 0x08,
+];
+const MDR_GUID: [u8; 16] = [
+    0x06, 0xa2, 0x7c, 0x8b, 0x90, 0x47, 0x9a, 0x4b, 0xb8, 0xfe, 0x57, 0x5f, 0x05, 0x0f, 0x88, 0x6e,
+];
 
 #[sorted]
 #[derive(Error, Debug)]
@@ -39,7 +46,7 @@ pub enum VhdxHeaderError {
     DuplicateBATEntry,
     #[error("Metadata region entry is not unique")]
     DuplicateMDREntry,
-    #[error("Checksum doesn't match for")]
+    #[error("Checksum doesn't match for {0}")]
     InvalidChecksum(String),
     #[error("Invalid entry count")]
     InvalidEntryCount,
@@ -47,8 +54,6 @@ pub enum VhdxHeaderError {
     InvalidHeaderSign,
     #[error("Not a valid VHDx region")]
     InvalidRegionSign,
-    #[error("Couldn't parse Uuid for region entry {0}")]
-    InvalidUuid(#[source] uuid::Error),
     #[error("Not a VHDx file")]
     InvalidVHDXSign,
     #[error("No valid header found")]
@@ -60,46 +65,39 @@ pub enum VhdxHeaderError {
     #[error("Failed to read headers {0}")]
     ReadHeader(#[source] io::Error),
     #[error("Failed to read metadata {0}")]
-    ReadMetadata(#[source] std::io::Error),
+    ReadMetadata(#[source] io::Error),
     #[error("Failed to read region table entries {0}")]
     ReadRegionTableEntries(#[source] io::Error),
     #[error("Failed to read region table header {0}")]
     ReadRegionTableHeader(#[source] io::Error),
     #[error("Failed to read region entries")]
     RegionEntryCollectionFailed,
+    #[error("Region entry file offset ({0}) and length ({1}) overflow u64")]
+    RegionEntryOverflow(u64 /* start */, usize /* length */),
     #[error("Overlapping regions found")]
     RegionOverlap,
     #[error("Reserved region has non-zero value")]
     ReservedIsNonZero,
-    #[error("Failed to seek in File Type Identifier {0}")]
-    SeekFileTypeIdentifier(#[source] io::Error),
-    #[error("Failed to seek in headers {0}")]
-    SeekHeader(#[source] io::Error),
-    #[error("Failed to seek in region table entries {0}")]
-    SeekRegionTableEntries(#[source] io::Error),
-    #[error("Failed to seek in region table header {0}")]
-    SeekRegionTableHeader(#[source] io::Error),
     #[error("We do not recognize this entry")]
     UnrecognizedRegionEntry,
     #[error("Failed to write header {0}")]
     WriteHeader(#[source] io::Error),
 }
 
-pub type Result<T> = std::result::Result<T, VhdxHeaderError>;
+pub(super) type Result<T> = result::Result<T, VhdxHeaderError>;
 
 #[derive(Clone, Debug)]
-pub struct FileTypeIdentifier {
+pub(super) struct FileTypeIdentifier {
     pub _signature: u64,
 }
 
 impl FileTypeIdentifier {
     /// Reads the File Type Identifier structure from a reference VHDx file
-    pub fn new(f: &mut File) -> Result<FileTypeIdentifier> {
-        f.seek(SeekFrom::Start(FILE_START))
-            .map_err(VhdxHeaderError::SeekFileTypeIdentifier)?;
-        let _signature = f
-            .read_u64::<LittleEndian>()
+    pub(super) fn new(f: &AlignedFile) -> Result<FileTypeIdentifier> {
+        let mut buf = [0u8; size_of::<u64>()];
+        f.read_exact_at(&mut buf, FILE_START)
             .map_err(VhdxHeaderError::ReadFileTypeIdentifier)?;
+        let _signature = LittleEndian::read_u64(&buf);
         if _signature != VHDX_SIGN {
             return Err(VhdxHeaderError::InvalidVHDXSign);
         }
@@ -109,8 +107,8 @@ impl FileTypeIdentifier {
 }
 
 #[repr(C, packed)]
-#[derive(Clone, Copy, Debug)]
-pub struct Header {
+#[derive(Clone, Copy, Debug, FromBytes, Immutable, IntoBytes)]
+pub(super) struct Header {
     pub signature: u32,
     pub checksum: u32,
     pub sequence_number: u64,
@@ -125,17 +123,14 @@ pub struct Header {
 
 impl Header {
     /// Reads the Header structure from a reference VHDx file
-    pub fn new(f: &mut File, start: u64) -> Result<Header> {
+    pub(super) fn new(f: &AlignedFile, start: u64) -> Result<Header> {
         // Read the whole header into a buffer. We will need it for
         // calculating checksum.
         let mut buffer = [0; HEADER_SIZE as usize];
-        f.seek(SeekFrom::Start(start))
-            .map_err(VhdxHeaderError::SeekHeader)?;
-        f.read_exact(&mut buffer)
+        f.read_exact_at(&mut buffer, start)
             .map_err(VhdxHeaderError::ReadHeader)?;
 
-        // SAFETY: buffer is of correct size and has been successfully filled.
-        let header: Header = unsafe { *(buffer.as_ptr().cast()) };
+        let header = Header::read_from_prefix(&buffer).unwrap().0;
         if header.signature != HEADER_SIGN {
             return Err(VhdxHeaderError::InvalidHeaderSign);
         }
@@ -148,32 +143,26 @@ impl Header {
         Ok(header)
     }
 
-    /// Converts the header structure into a buffer
-    fn write_to_buffer(&self, buffer: &mut [u8; HEADER_SIZE as usize]) {
-        // SAFETY: self is a valid header.
-        let reference =
-            unsafe { std::slice::from_raw_parts((&raw const *self).cast(), HEADER_SIZE as usize) };
-        *buffer = reference.try_into().unwrap();
-    }
-
     /// Creates and returns new updated header from the provided current header
     fn update_header(
-        f: &mut File,
+        f: &AlignedFile,
         current_header: &Header,
         change_data_guid: bool,
-        mut file_write_guid: u128,
+        file_write_guid: u128,
         start: u64,
     ) -> Result<Header> {
         let mut buffer = [0u8; HEADER_SIZE as usize];
-        let mut data_write_guid = current_header.data_write_guid;
+        let data_write_guid = if change_data_guid {
+            Uuid::new_v4().as_u128()
+        } else {
+            current_header.data_write_guid
+        };
 
-        if change_data_guid {
-            data_write_guid = Uuid::new_v4().as_u128();
-        }
-
-        if file_write_guid == 0 {
-            file_write_guid = current_header.file_write_guid;
-        }
+        let file_write_guid = if file_write_guid == 0 {
+            current_header.file_write_guid
+        } else {
+            file_write_guid
+        };
 
         let mut new_header = Header {
             signature: current_header.signature,
@@ -188,20 +177,19 @@ impl Header {
             log_offset: current_header.log_offset,
         };
 
-        new_header.write_to_buffer(&mut buffer);
+        new_header.write_to_prefix(&mut buffer).unwrap();
         new_header.checksum = calculate_checksum(&mut buffer, size_of::<u32>());
-        new_header.write_to_buffer(&mut buffer);
+        new_header.write_to_prefix(&mut buffer).unwrap();
 
-        f.seek(SeekFrom::Start(start))
-            .map_err(VhdxHeaderError::SeekHeader)?;
-        f.write(&buffer).map_err(VhdxHeaderError::WriteHeader)?;
+        f.write_all_at(&buffer, start)
+            .map_err(VhdxHeaderError::WriteHeader)?;
 
         Ok(new_header)
     }
 }
 
 #[repr(C, packed)]
-#[derive(Clone, Copy, Debug)]
+#[derive(Clone, Copy, Debug, FromBytes)]
 struct RegionTableHeader {
     pub signature: u32,
     pub checksum: u32,
@@ -211,17 +199,14 @@ struct RegionTableHeader {
 
 impl RegionTableHeader {
     /// Reads the Region Table Header structure from a reference VHDx file
-    pub fn new(f: &mut File, start: u64) -> Result<RegionTableHeader> {
+    pub(crate) fn new(f: &AlignedFile, start: u64) -> Result<RegionTableHeader> {
         // Read the whole header into a buffer. We will need it for calculating
         // checksum.
         let mut buffer = [0u8; REGION_SIZE as usize];
-        f.seek(SeekFrom::Start(start))
-            .map_err(VhdxHeaderError::SeekRegionTableHeader)?;
-        f.read_exact(&mut buffer)
+        f.read_exact_at(&mut buffer, start)
             .map_err(VhdxHeaderError::ReadRegionTableHeader)?;
 
-        // SAFETY: buffer is of correct size and has been successfully filled.
-        let region_table_header: RegionTableHeader = unsafe { *(buffer.as_ptr().cast()) };
+        let region_table_header = RegionTableHeader::read_from_prefix(&buffer).unwrap().0;
         if region_table_header.signature != REGION_SIGN {
             return Err(VhdxHeaderError::InvalidRegionSign);
         }
@@ -243,7 +228,13 @@ impl RegionTableHeader {
     }
 }
 
-pub struct RegionInfo {
+/// Returns `true` if the half-open byte ranges `[a_start, a_end)` and
+/// `[b_start, b_end)` overlap.
+fn ranges_overlap(a_start: u64, a_end: u64, b_start: u64, b_end: u64) -> bool {
+    a_start < b_end && b_start < a_end
+}
+
+pub(super) struct RegionInfo {
     pub bat_entry: RegionTableEntry,
     pub mdr_entry: RegionTableEntry,
     pub region_entries: BTreeMap<u64, u64>,
@@ -252,7 +243,7 @@ pub struct RegionInfo {
 impl RegionInfo {
     /// Collect all entries in a BTreeMap from the Region Table and identifies
     /// BAT and metadata regions
-    pub fn new(f: &mut File, region_start: u64, entry_count: u32) -> Result<RegionInfo> {
+    pub(super) fn new(f: &AlignedFile, region_start: u64, entry_count: u32) -> Result<RegionInfo> {
         let mut bat_entry: Option<RegionTableEntry> = None;
         let mut mdr_entry: Option<RegionTableEntry> = None;
 
@@ -260,31 +251,34 @@ impl RegionInfo {
         let mut region_entries = BTreeMap::new();
 
         let mut buffer = [0; REGION_SIZE as usize];
-        // Seek after the Region Table Header
-        f.seek(SeekFrom::Start(
+        // Read after the Region Table Header
+        f.read_exact_at(
+            &mut buffer,
             region_start + size_of::<RegionTableHeader>() as u64,
-        ))
-        .map_err(VhdxHeaderError::SeekRegionTableEntries)?;
-        f.read_exact(&mut buffer)
-            .map_err(VhdxHeaderError::ReadRegionTableEntries)?;
+        )
+        .map_err(VhdxHeaderError::ReadRegionTableEntries)?;
 
         for _ in 0..entry_count {
-            let entry =
-                RegionTableEntry::new(&buffer[offset..offset + size_of::<RegionTableEntry>()])?;
+            let entry = RegionTableEntry::read_from_bytes(
+                &buffer[offset..offset + size_of::<RegionTableEntry>()],
+            )
+            .unwrap();
 
             offset += size_of::<RegionTableEntry>();
             let start = entry.file_offset;
-            let end = start + entry.length as u64;
+            let end = start.checked_add(entry.length as u64).ok_or(
+                VhdxHeaderError::RegionEntryOverflow(start, entry.length as usize),
+            )?;
 
             for (region_ent_start, region_ent_end) in region_entries.iter() {
-                if !((start >= *region_ent_start) || (end <= *region_ent_end)) {
+                if ranges_overlap(start, end, *region_ent_start, *region_ent_end) {
                     return Err(VhdxHeaderError::RegionOverlap);
                 }
             }
 
-            region_entries.insert(entry.file_offset, entry.file_offset + entry.length as u64);
+            region_entries.insert(start, end);
 
-            if entry.guid == Uuid::parse_str(BAT_GUID).map_err(VhdxHeaderError::InvalidUuid)? {
+            if entry.guid == BAT_GUID {
                 if bat_entry.is_none() {
                     bat_entry = Some(entry);
                     continue;
@@ -292,7 +286,7 @@ impl RegionInfo {
                 return Err(VhdxHeaderError::DuplicateBATEntry);
             }
 
-            if entry.guid == Uuid::parse_str(MDR_GUID).map_err(VhdxHeaderError::InvalidUuid)? {
+            if entry.guid == MDR_GUID {
                 if mdr_entry.is_none() {
                     mdr_entry = Some(entry);
                     continue;
@@ -326,28 +320,14 @@ impl RegionInfo {
 }
 
 #[repr(C, packed)]
-#[derive(Clone, Copy, Debug)]
-pub struct RegionTableEntry {
-    pub guid: Uuid,
+#[derive(Clone, Copy, Debug, FromBytes)]
+pub(super) struct RegionTableEntry {
+    guid: [u8; 16],
     pub file_offset: u64,
     pub length: u32,
     pub required: u32,
 }
 
-impl RegionTableEntry {
-    /// Reads one Region Entry from a Region Table index that starts from 0
-    pub fn new(buffer: &[u8]) -> Result<RegionTableEntry> {
-        assert!(buffer.len() == std::mem::size_of::<RegionTableEntry>());
-        // SAFETY: the assertion above makes sure the buffer size is correct.
-        let mut region_table_entry: RegionTableEntry = unsafe { *(buffer.as_ptr().cast()) };
-
-        let uuid = crate::vhdx::uuid_from_guid(buffer);
-        region_table_entry.guid = uuid;
-
-        Ok(region_table_entry)
-    }
-}
-
 enum HeaderNo {
     First,
     Second,
@@ -355,7 +335,7 @@ enum HeaderNo {
 
 /// Contains the information from the header of a VHDx file
 #[derive(Clone, Debug)]
-pub struct VhdxHeader {
+pub(super) struct VhdxHeader {
     _file_type_identifier: FileTypeIdentifier,
     header_1: Header,
     header_2: Header,
@@ -365,7 +345,7 @@ pub struct VhdxHeader {
 
 impl VhdxHeader {
     /// Creates a VhdxHeader from a reference to a file
-    pub fn new(f: &mut File) -> Result<VhdxHeader> {
+    pub(super) fn new(f: &AlignedFile) -> Result<VhdxHeader> {
         Ok(VhdxHeader {
             _file_type_identifier: FileTypeIdentifier::new(f)?,
             header_1: Header::new(f, HEADER_1_START)?,
@@ -402,7 +382,7 @@ impl VhdxHeader {
     /// current one. Returns both headers as a tuple sequenced the way it was
     /// received from the parameter list.
     fn update_header(
-        f: &mut File,
+        f: &AlignedFile,
         header_1: Result<Header>,
         header_2: Result<Header>,
         guid: u128,
@@ -425,7 +405,7 @@ impl VhdxHeader {
 
     // Update the provided headers according to the spec
     fn update_headers(
-        f: &mut File,
+        f: &AlignedFile,
         header_1: Result<Header>,
         header_2: Result<Header>,
         guid: u128,
@@ -435,14 +415,14 @@ impl VhdxHeader {
         VhdxHeader::update_header(f, Ok(header_1), Ok(header_2), guid)
     }
 
-    pub fn update(&mut self, f: &mut File) -> Result<()> {
+    pub(super) fn update(&mut self, f: &AlignedFile) -> Result<()> {
         let headers = VhdxHeader::update_headers(f, Ok(self.header_1), Ok(self.header_2), 0)?;
         self.header_1 = headers.0;
         self.header_2 = headers.1;
         Ok(())
     }
 
-    pub fn region_entry_count(&self) -> u32 {
+    pub(super) fn region_entry_count(&self) -> u32 {
         self.region_table_1.entry_count
     }
 }
@@ -466,3 +446,144 @@ fn calculate_checksum(buffer: &mut [u8], csum_offset: usize) -> u32 {
 
     new_csum
 }
+
+#[cfg(test)]
+mod tests {
+    use std::os::unix::fs::FileExt;
+
+    use vmm_sys_util::tempfile::TempFile;
+    use zerocopy::{FromBytes, IntoBytes};
+
+    use super::{
+        BAT_GUID, HEADER_SIGN, Header, MDR_GUID, REGION_TABLE_1_START, RegionInfo,
+        RegionTableHeader, VhdxHeaderError, ranges_overlap,
+    };
+    use crate::aligned_file::AlignedFile;
+
+    #[test]
+    fn test_header_bytes_round_trip() {
+        let header = Header {
+            signature: HEADER_SIGN,
+            checksum: 0x1122_3344,
+            sequence_number: 0x0102_0304_0506_0708,
+            file_write_guid: 0x0f0e_0d0c_0b0a_0908_0706_0504_0302_0100,
+            data_write_guid: 0x1f1e_1d1c_1b1a_1918_1716_1514_1312_1110,
+            log_guid: 0x2f2e_2d2c_2b2a_2928_2726_2524_2322_2120,
+            log_version: 0xabcd,
+            version: 0x0001,
+            log_length: 0x0010_0000,
+            log_offset: 0x0000_0100_0000_0000,
+        };
+
+        let bytes = header.as_bytes();
+        assert_eq!(&bytes[0..4], &header.signature.to_le_bytes()[..]);
+        assert_eq!(&bytes[8..16], &header.sequence_number.to_le_bytes()[..]);
+        assert_eq!(&bytes[16..32], &header.file_write_guid.to_le_bytes()[..]);
+        assert_eq!(&bytes[64..66], &header.log_version.to_le_bytes()[..]);
+        assert_eq!(&bytes[72..80], &header.log_offset.to_le_bytes()[..]);
+
+        let parsed = Header::read_from_bytes(bytes).unwrap();
+        assert_eq!({ parsed.signature }, { header.signature });
+        assert_eq!({ parsed.checksum }, { header.checksum });
+        assert_eq!({ parsed.sequence_number }, { header.sequence_number });
+        assert_eq!({ parsed.file_write_guid }, { header.file_write_guid });
+        assert_eq!({ parsed.data_write_guid }, { header.data_write_guid });
+        assert_eq!({ parsed.log_guid }, { header.log_guid });
+        assert_eq!({ parsed.log_version }, { header.log_version });
+        assert_eq!({ parsed.version }, { header.version });
+        assert_eq!({ parsed.log_length }, { header.log_length });
+        assert_eq!({ parsed.log_offset }, { header.log_offset });
+    }
+
+    #[test]
+    fn test_ranges_overlap() {
+        // (new [start,end), existing [s,e), expected overlap)
+        let cases: &[(u64, u64, u64, u64, bool)] = &[
+            // Genuine overlaps — all of these must be detected.
+            (0, 10, 0, 10, true), // identical
+            (2, 8, 0, 10, true),  // new fully inside existing
+            (0, 20, 5, 10, true), // new fully contains existing
+            (5, 15, 0, 10, true), // partial, new starts inside existing
+            (0, 8, 5, 15, true),  // partial, new starts before existing
+            // Non-overlapping — must not be flagged.
+            (0, 5, 10, 20, false),   // disjoint, new before existing
+            (30, 40, 10, 20, false), // disjoint, new after existing
+            (0, 10, 10, 20, false),  // touching at the boundary (half-open)
+        ];
+
+        for &(a_start, a_end, b_start, b_end, expected) in cases {
+            assert_eq!(
+                ranges_overlap(a_start, a_end, b_start, b_end),
+                expected,
+                "[{a_start},{a_end}) vs [{b_start},{b_end})"
+            );
+            // Overlap is symmetric.
+            assert_eq!(
+                ranges_overlap(b_start, b_end, a_start, a_end),
+                expected,
+                "symmetry: [{b_start},{b_end}) vs [{a_start},{a_end})"
+            );
+        }
+    }
+
+    /// Builds the 32-byte on-disk region table entry for `guid` describing
+    /// the region `[file_offset, file_offset + length)`.
+    fn region_entry(guid: [u8; 16], file_offset: u64, length: u32) -> [u8; 32] {
+        let mut e = [0u8; 32];
+        e[0..16].copy_from_slice(&guid);
+        e[16..24].copy_from_slice(&file_offset.to_le_bytes());
+        e[24..28].copy_from_slice(&length.to_le_bytes());
+        // `required` (e[28..32]) left zero.
+        e
+    }
+
+    #[test]
+    fn test_region_info_rejects_overlapping_regions() {
+        // BAT region [1 MiB, 3 MiB) and metadata region [2 MiB, 4 MiB) overlap
+        // on [2 MiB, 3 MiB); per [MS-VHDX] all region objects must be
+        // non-overlapping, so this image must be rejected.
+        const MIB: u64 = 1024 * 1024;
+        let region_start = REGION_TABLE_1_START;
+        let entries_at = region_start + size_of::<RegionTableHeader>() as u64;
+
+        let temp = TempFile::new().unwrap();
+        let f = temp.into_file();
+        f.set_len(entries_at + 64 * 1024).unwrap();
+        f.write_all_at(&region_entry(BAT_GUID, MIB, (2 * MIB) as u32), entries_at)
+            .unwrap();
+        f.write_all_at(
+            &region_entry(MDR_GUID, 2 * MIB, (2 * MIB) as u32),
+            entries_at + 32,
+        )
+        .unwrap();
+
+        let af = AlignedFile::new(f, false);
+        let res = RegionInfo::new(&af, region_start, 2);
+        assert!(
+            matches!(res, Err(VhdxHeaderError::RegionOverlap)),
+            "expected RegionOverlap for an overlapping region table"
+        );
+    }
+
+    #[test]
+    fn test_region_info_rejects_overflowing_region() {
+        // A region whose file offset plus length wraps past u64::MAX must be
+        // rejected rather than silently producing a small end offset that
+        // could mask a genuine overlap.
+        let region_start = REGION_TABLE_1_START;
+        let entries_at = region_start + size_of::<RegionTableHeader>() as u64;
+
+        let temp = TempFile::new().unwrap();
+        let f = temp.into_file();
+        f.set_len(entries_at + 64 * 1024).unwrap();
+        f.write_all_at(&region_entry(BAT_GUID, u64::MAX, 0x1000), entries_at)
+            .unwrap();
+
+        let af = AlignedFile::new(f, false);
+        let res = RegionInfo::new(&af, region_start, 1);
+        assert!(
+            matches!(res, Err(VhdxHeaderError::RegionEntryOverflow(..))),
+            "expected RegionEntryOverflow for a wrapping region entry"
+        );
+    }
+}

block/src/formats/vhdx/io.rs

@@ -2,14 +2,15 @@
 //
 // SPDX-License-Identifier: Apache-2.0
 
-use std::fs::File;
-use std::io::{self, Read, Seek, SeekFrom, Write};
+use std::os::unix::fs::FileExt;
+use std::{io, result};
 
 use remain::sorted;
 use thiserror::Error;
 
-use crate::vhdx::vhdx_bat::{self, BatEntry, VhdxBatError};
-use crate::vhdx::vhdx_metadata::{self, DiskSpec};
+use super::bat::{self, BatEntry, VhdxBatError};
+use super::metadata::{self, DiskSpec};
+use crate::aligned_file::AlignedFile;
 
 const SECTOR_SIZE: u64 = 512;
 
@@ -32,7 +33,7 @@ pub enum VhdxIoError {
     WriteBat(#[source] VhdxBatError),
 }
 
-pub type Result<T> = std::result::Result<T, VhdxIoError>;
+pub(super) type Result<T> = result::Result<T, VhdxIoError>;
 
 macro_rules! align {
     ($n:expr, $align:expr) => {{ $n.div_ceil($align) * $align }};
@@ -50,7 +51,7 @@ struct Sector {
 impl Sector {
     /// Translate sector index and count of data in file to actual offsets and
     /// BAT index.
-    pub fn new(
+    pub(crate) fn new(
         disk_spec: &DiskSpec,
         bat: &[BatEntry],
         sector_index: u64,
@@ -74,7 +75,7 @@ impl Sector {
                 return Err(VhdxIoError::InvalidBatIndex);
             }
         };
-        sector.file_offset = bat_entry & vhdx_bat::BAT_FILE_OFF_MASK;
+        sector.file_offset = bat_entry & bat::BAT_FILE_OFF_MASK;
         if sector.file_offset != 0 {
             sector.file_offset += sector.block_offset;
         }
@@ -85,8 +86,8 @@ impl Sector {
 
 /// VHDx IO read routine: requires relative sector index and count for the
 /// requested data.
-pub fn read(
-    f: &mut File,
+pub(super) fn read(
+    f: &AlignedFile,
     buf: &mut [u8],
     disk_spec: &DiskSpec,
     bat: &[BatEntry],
@@ -108,21 +109,20 @@ pub fn read(
             }
         };
 
-        match bat_entry & vhdx_bat::BAT_STATE_BIT_MASK {
-            vhdx_bat::PAYLOAD_BLOCK_NOT_PRESENT
-            | vhdx_bat::PAYLOAD_BLOCK_UNDEFINED
-            | vhdx_bat::PAYLOAD_BLOCK_UNMAPPED
-            | vhdx_bat::PAYLOAD_BLOCK_ZERO => {}
-            vhdx_bat::PAYLOAD_BLOCK_FULLY_PRESENT => {
-                f.seek(SeekFrom::Start(sector.file_offset))
-                    .map_err(VhdxIoError::ReadSectorBlock)?;
-                f.read_exact(
+        match bat_entry & bat::BAT_STATE_BIT_MASK {
+            bat::PAYLOAD_BLOCK_NOT_PRESENT
+            | bat::PAYLOAD_BLOCK_UNDEFINED
+            | bat::PAYLOAD_BLOCK_UNMAPPED
+            | bat::PAYLOAD_BLOCK_ZERO => {}
+            bat::PAYLOAD_BLOCK_FULLY_PRESENT => {
+                f.read_exact_at(
                     &mut buf
                         [read_count..(read_count + (sector.free_sectors * SECTOR_SIZE) as usize)],
+                    sector.file_offset,
                 )
                 .map_err(VhdxIoError::ReadSectorBlock)?;
             }
-            vhdx_bat::PAYLOAD_BLOCK_PARTIALLY_PRESENT => {
+            bat::PAYLOAD_BLOCK_PARTIALLY_PRESENT => {
                 return Err(VhdxIoError::UnsupportedMode);
             }
             _ => {
@@ -138,8 +138,8 @@ pub fn read(
 
 /// VHDx IO write routine: requires relative sector index and count for the
 /// requested data.
-pub fn write(
-    f: &mut File,
+pub(super) fn write(
+    f: &AlignedFile,
     buf: &[u8],
     disk_spec: &mut DiskSpec,
     bat_offset: u64,
@@ -162,49 +162,48 @@ pub fn write(
             }
         };
 
-        match bat_entry & vhdx_bat::BAT_STATE_BIT_MASK {
-            vhdx_bat::PAYLOAD_BLOCK_NOT_PRESENT
-            | vhdx_bat::PAYLOAD_BLOCK_UNDEFINED
-            | vhdx_bat::PAYLOAD_BLOCK_UNMAPPED
-            | vhdx_bat::PAYLOAD_BLOCK_ZERO => {
-                let file_offset =
-                    align!(disk_spec.image_size, vhdx_metadata::BLOCK_SIZE_MIN as u64);
+        match bat_entry & bat::BAT_STATE_BIT_MASK {
+            bat::PAYLOAD_BLOCK_NOT_PRESENT
+            | bat::PAYLOAD_BLOCK_UNDEFINED
+            | bat::PAYLOAD_BLOCK_UNMAPPED
+            | bat::PAYLOAD_BLOCK_ZERO => {
+                let file_offset = align!(disk_spec.image_size, metadata::BLOCK_SIZE_MIN as u64);
                 let new_size = file_offset
                     .checked_add(disk_spec.block_size as u64)
                     .ok_or(VhdxIoError::InvalidDiskSize)?;
 
-                f.set_len(new_size).map_err(VhdxIoError::ResizeFile)?;
+                f.file()
+                    .set_len(new_size)
+                    .map_err(VhdxIoError::ResizeFile)?;
                 disk_spec.image_size = new_size;
 
-                let new_bat_entry = file_offset
-                    | (vhdx_bat::PAYLOAD_BLOCK_FULLY_PRESENT & vhdx_bat::BAT_STATE_BIT_MASK);
+                let new_bat_entry =
+                    file_offset | (bat::PAYLOAD_BLOCK_FULLY_PRESENT & bat::BAT_STATE_BIT_MASK);
                 bat[sector.bat_index as usize] = BatEntry(new_bat_entry);
                 BatEntry::write_bat_entries(f, bat_offset, bat).map_err(VhdxIoError::WriteBat)?;
 
-                if file_offset < vhdx_metadata::BLOCK_SIZE_MIN as u64 {
+                if file_offset < metadata::BLOCK_SIZE_MIN as u64 {
                     break;
                 }
 
-                f.seek(SeekFrom::Start(file_offset))
-                    .map_err(VhdxIoError::ReadSectorBlock)?;
-                f.write_all(
+                f.write_all_at(
                     &buf[write_count..(write_count + (sector.free_sectors * SECTOR_SIZE) as usize)],
+                    file_offset,
                 )
                 .map_err(VhdxIoError::ReadSectorBlock)?;
             }
-            vhdx_bat::PAYLOAD_BLOCK_FULLY_PRESENT => {
-                if sector.file_offset < vhdx_metadata::BLOCK_SIZE_MIN as u64 {
+            bat::PAYLOAD_BLOCK_FULLY_PRESENT => {
+                if sector.file_offset < metadata::BLOCK_SIZE_MIN as u64 {
                     break;
                 }
 
-                f.seek(SeekFrom::Start(sector.file_offset))
-                    .map_err(VhdxIoError::ReadSectorBlock)?;
-                f.write_all(
+                f.write_all_at(
                     &buf[write_count..(write_count + (sector.free_sectors * SECTOR_SIZE) as usize)],
+                    sector.file_offset,
                 )
                 .map_err(VhdxIoError::ReadSectorBlock)?;
             }
-            vhdx_bat::PAYLOAD_BLOCK_PARTIALLY_PRESENT => {
+            bat::PAYLOAD_BLOCK_PARTIALLY_PRESENT => {
                 return Err(VhdxIoError::UnsupportedMode);
             }
             _ => {

block/src/formats/vhdx/metadata.rs

@@ -2,16 +2,16 @@
 //
 // SPDX-License-Identifier: Apache-2.0
 
-use std::fs::File;
-use std::io::{self, Read, Seek, SeekFrom};
-use std::mem::size_of;
+use std::os::unix::fs::FileExt;
+use std::{io, result};
 
-use byteorder::{LittleEndian, ReadBytesExt};
+use byteorder::{ByteOrder, LittleEndian};
 use remain::sorted;
 use thiserror::Error;
-use uuid::Uuid;
+use zerocopy::FromBytes;
 
-use crate::vhdx::vhdx_header::RegionTableEntry;
+use super::header::RegionTableEntry;
+use crate::aligned_file::AlignedFile;
 
 const METADATA_SIGN: u64 = 0x6174_6164_6174_656D;
 const METADATA_ENTRY_SIZE: usize = 32;
@@ -21,19 +21,31 @@ const METADATA_TABLE_MAX_SIZE: usize = METADATA_ENTRY_SIZE * (METADATA_MAX_ENTRI
 
 const METADATA_FLAGS_IS_REQUIRED: u32 = 0x04;
 
-pub const BLOCK_SIZE_MIN: u32 = 1 << 20; // 1 MiB
+pub(super) const BLOCK_SIZE_MIN: u32 = 1 << 20; // 1 MiB
 const BLOCK_SIZE_MAX: u32 = 256 << 20; // 256 MiB
 const MAX_SECTORS_PER_BLOCK: u64 = 1 << 23;
 
 const BLOCK_HAS_PARENT: u32 = 0x02; // Has a parent or a backing file
 
 // GUID for known metadata items
-const METADATA_FILE_PARAMETER: &str = "CAA16737-FA36-4D43-B3B6-33F0AA44E76B";
-const METADATA_VIRTUAL_DISK_SIZE: &str = "2FA54224-CD1B-4876-B211-5DBED83BF4B8";
-const METADATA_VIRTUAL_DISK_ID: &str = "BECA12AB-B2E6-4523-93EF-C309E000C746";
-const METADATA_LOGICAL_SECTOR_SIZE: &str = "8141BF1D-A96F-4709-BA47-F233A8FAAB5F";
-const METADATA_PHYSICAL_SECTOR_SIZE: &str = "CDA348C7-445D-4471-9CC9-E9885251C556";
-const METADATA_PARENT_LOCATOR: &str = "A8D35F2D-B30B-454D-ABF7-D3D84834AB0C";
+const METADATA_FILE_PARAMETER: [u8; 16] = [
+    0x37, 0x67, 0xa1, 0xca, 0x36, 0xfa, 0x43, 0x4d, 0xb3, 0xb6, 0x33, 0xf0, 0xaa, 0x44, 0xe7, 0x6b,
+];
+const METADATA_VIRTUAL_DISK_SIZE: [u8; 16] = [
+    0x24, 0x42, 0xa5, 0x2f, 0x1b, 0xcd, 0x76, 0x48, 0xb2, 0x11, 0x5d, 0xbe, 0xd8, 0x3b, 0xf4, 0xb8,
+];
+const METADATA_VIRTUAL_DISK_ID: [u8; 16] = [
+    0xab, 0x12, 0xca, 0xbe, 0xe6, 0xb2, 0x23, 0x45, 0x93, 0xef, 0xc3, 0x09, 0xe0, 0x00, 0xc7, 0x46,
+];
+const METADATA_LOGICAL_SECTOR_SIZE: [u8; 16] = [
+    0x1d, 0xbf, 0x41, 0x81, 0x6f, 0xa9, 0x09, 0x47, 0xba, 0x47, 0xf2, 0x33, 0xa8, 0xfa, 0xab, 0x5f,
+];
+const METADATA_PHYSICAL_SECTOR_SIZE: [u8; 16] = [
+    0xc7, 0x48, 0xa3, 0xcd, 0x5d, 0x44, 0x71, 0x44, 0x9c, 0xc9, 0xe9, 0x88, 0x52, 0x51, 0xc5, 0x56,
+];
+const METADATA_PARENT_LOCATOR: [u8; 16] = [
+    0x2d, 0x5f, 0xd3, 0xa8, 0x0b, 0xb3, 0x4d, 0x45, 0xab, 0xf7, 0xd3, 0xd8, 0x48, 0x34, 0xab, 0x0c,
+];
 
 const METADATA_FILE_PARAMETER_PRESENT: u16 = 0x01;
 const METADATA_VIRTUAL_DISK_SIZE_PRESENT: u16 = 0x02;
@@ -69,8 +81,6 @@ pub enum VhdxMetadataError {
     InvalidMetadataSign,
     #[error("Invalid physical sector size")]
     InvalidPhysicalSectorSize,
-    #[error("Invalid UUID")]
-    InvalidUuid(#[source] uuid::Error),
     #[error("Invalid value")]
     InvalidValue,
     #[error("Not all required metadata found")]
@@ -83,10 +93,10 @@ pub enum VhdxMetadataError {
     UnsupportedFlag,
 }
 
-pub type Result<T> = std::result::Result<T, VhdxMetadataError>;
+pub(super) type Result<T> = result::Result<T, VhdxMetadataError>;
 
 #[derive(Default, Clone, Debug)]
-pub struct DiskSpec {
+pub(super) struct DiskSpec {
     pub disk_id: u128,
     pub image_size: u64,
     pub block_size: u32,
@@ -102,17 +112,18 @@ pub struct DiskSpec {
 impl DiskSpec {
     /// Parse all metadata from the provided file and store info in DiskSpec
     /// structure.
-    pub fn new(f: &mut File, metadata_region: &RegionTableEntry) -> Result<DiskSpec> {
+    pub(super) fn new(f: &AlignedFile, metadata_region: &RegionTableEntry) -> Result<DiskSpec> {
         let mut disk_spec = DiskSpec::default();
         let mut metadata_presence: u16 = 0;
         let mut offset = 0;
-        let metadata = f.metadata().map_err(VhdxMetadataError::ReadMetadata)?;
+        let metadata = f
+            .file()
+            .metadata()
+            .map_err(VhdxMetadataError::ReadMetadata)?;
         disk_spec.image_size = metadata.len();
 
         let mut buffer = [0u8; METADATA_TABLE_MAX_SIZE];
-        f.seek(SeekFrom::Start(metadata_region.file_offset))
-            .map_err(VhdxMetadataError::ReadMetadata)?;
-        f.read_exact(&mut buffer)
+        f.read_exact_at(&mut buffer, metadata_region.file_offset)
             .map_err(VhdxMetadataError::ReadMetadata)?;
 
         let metadata_header =
@@ -123,18 +134,13 @@ impl DiskSpec {
             let metadata_entry =
                 MetadataTableEntry::new(&buffer[offset..offset + size_of::<MetadataTableEntry>()])?;
 
-            f.seek(SeekFrom::Start(
-                metadata_region.file_offset + metadata_entry.offset as u64,
-            ))
-            .map_err(VhdxMetadataError::ReadMetadata)?;
+            let item_offset = metadata_region.file_offset + metadata_entry.offset as u64;
 
-            if metadata_entry.item_id
-                == Uuid::parse_str(METADATA_FILE_PARAMETER)
-                    .map_err(VhdxMetadataError::InvalidUuid)?
-            {
-                disk_spec.block_size = f
-                    .read_u32::<LittleEndian>()
+            if metadata_entry.item_id == METADATA_FILE_PARAMETER {
+                let mut item = [0u8; 2 * size_of::<u32>()];
+                f.read_exact_at(&mut item, item_offset)
                     .map_err(VhdxMetadataError::ReadMetadata)?;
+                disk_spec.block_size = LittleEndian::read_u32(&item[0..4]);
 
                 // MUST be at least 1 MiB and not greater than 256 MiB
                 if disk_spec.block_size < BLOCK_SIZE_MIN || disk_spec.block_size > BLOCK_SIZE_MAX {
@@ -146,50 +152,40 @@ impl DiskSpec {
                     return Err(VhdxMetadataError::InvalidBlockSize);
                 }
 
-                let bits = f
-                    .read_u32::<LittleEndian>()
-                    .map_err(VhdxMetadataError::ReadMetadata)?;
+                let bits = LittleEndian::read_u32(&item[4..8]);
                 disk_spec.has_parent = bits & BLOCK_HAS_PARENT != 0;
 
                 metadata_presence |= METADATA_FILE_PARAMETER_PRESENT;
-            } else if metadata_entry.item_id
-                == Uuid::parse_str(METADATA_VIRTUAL_DISK_SIZE)
-                    .map_err(VhdxMetadataError::InvalidUuid)?
-            {
-                disk_spec.virtual_disk_size = f
-                    .read_u64::<LittleEndian>()
+            } else if metadata_entry.item_id == METADATA_VIRTUAL_DISK_SIZE {
+                let mut item = [0u8; size_of::<u64>()];
+                f.read_exact_at(&mut item, item_offset)
                     .map_err(VhdxMetadataError::ReadMetadata)?;
+                disk_spec.virtual_disk_size = LittleEndian::read_u64(&item);
 
                 metadata_presence |= METADATA_VIRTUAL_DISK_SIZE_PRESENT;
-            } else if metadata_entry.item_id
-                == Uuid::parse_str(METADATA_VIRTUAL_DISK_ID)
-                    .map_err(VhdxMetadataError::InvalidUuid)?
-            {
-                disk_spec.disk_id = f
-                    .read_u128::<LittleEndian>()
+            } else if metadata_entry.item_id == METADATA_VIRTUAL_DISK_ID {
+                let mut item = [0u8; size_of::<u128>()];
+                f.read_exact_at(&mut item, item_offset)
                     .map_err(VhdxMetadataError::ReadMetadata)?;
+                disk_spec.disk_id = LittleEndian::read_u128(&item);
 
                 metadata_presence |= METADATA_VIRTUAL_DISK_ID_PRESENT;
-            } else if metadata_entry.item_id
-                == Uuid::parse_str(METADATA_LOGICAL_SECTOR_SIZE)
-                    .map_err(VhdxMetadataError::InvalidUuid)?
-            {
-                disk_spec.logical_sector_size = f
-                    .read_u32::<LittleEndian>()
+            } else if metadata_entry.item_id == METADATA_LOGICAL_SECTOR_SIZE {
+                let mut item = [0u8; size_of::<u32>()];
+                f.read_exact_at(&mut item, item_offset)
                     .map_err(VhdxMetadataError::ReadMetadata)?;
+                disk_spec.logical_sector_size = LittleEndian::read_u32(&item);
                 if !(disk_spec.logical_sector_size == 512 || disk_spec.logical_sector_size == 4096)
                 {
                     return Err(VhdxMetadataError::InvalidLogicalSectorSize);
                 }
 
                 metadata_presence |= METADATA_LOGICAL_SECTOR_SIZE_PRESENT;
-            } else if metadata_entry.item_id
-                == Uuid::parse_str(METADATA_PHYSICAL_SECTOR_SIZE)
-                    .map_err(VhdxMetadataError::InvalidUuid)?
-            {
-                disk_spec.physical_sector_size = f
-                    .read_u32::<LittleEndian>()
+            } else if metadata_entry.item_id == METADATA_PHYSICAL_SECTOR_SIZE {
+                let mut item = [0u8; size_of::<u32>()];
+                f.read_exact_at(&mut item, item_offset)
                     .map_err(VhdxMetadataError::ReadMetadata)?;
+                disk_spec.physical_sector_size = LittleEndian::read_u32(&item);
                 if !(disk_spec.physical_sector_size == 512
                     || disk_spec.physical_sector_size == 4096)
                 {
@@ -197,10 +193,7 @@ impl DiskSpec {
                 }
 
                 metadata_presence |= METADATA_PHYSICAL_SECTOR_SIZE_PRESENT;
-            } else if metadata_entry.item_id
-                == Uuid::parse_str(METADATA_PARENT_LOCATOR)
-                    .map_err(VhdxMetadataError::InvalidUuid)?
-            {
+            } else if metadata_entry.item_id == METADATA_PARENT_LOCATOR {
                 metadata_presence |= METADATA_PARENT_LOCATOR_PRESENT;
             } else {
                 return Err(VhdxMetadataError::InvalidMetadataItem);
@@ -268,7 +261,7 @@ impl DiskSpec {
 }
 
 #[repr(C, packed)]
-#[derive(Default, Debug, Clone, Copy)]
+#[derive(Default, Debug, Clone, Copy, FromBytes)]
 struct MetadataTableHeader {
     signature: u64,
     reserved: u16,
@@ -277,10 +270,8 @@ struct MetadataTableHeader {
 }
 
 impl MetadataTableHeader {
-    pub fn new(buffer: &[u8]) -> Result<MetadataTableHeader> {
-        assert!(buffer.len() == std::mem::size_of::<MetadataTableHeader>());
-        // SAFETY: the assertion above makes sure the buffer size is correct.
-        let metadata_table_header: MetadataTableHeader = unsafe { *(buffer.as_ptr().cast()) };
+    pub(crate) fn new(buffer: &[u8]) -> Result<MetadataTableHeader> {
+        let metadata_table_header = MetadataTableHeader::read_from_bytes(buffer).unwrap();
 
         if metadata_table_header.signature != METADATA_SIGN {
             return Err(VhdxMetadataError::InvalidMetadataSign);
@@ -299,9 +290,9 @@ impl MetadataTableHeader {
 }
 
 #[repr(C, packed)]
-#[derive(Default, Debug, Clone, Copy)]
-pub struct MetadataTableEntry {
-    item_id: Uuid,
+#[derive(Default, Debug, Clone, Copy, FromBytes)]
+pub(super) struct MetadataTableEntry {
+    item_id: [u8; 16],
     offset: u32,
     length: u32,
     flag_bits: u32,
@@ -311,12 +302,7 @@ pub struct MetadataTableEntry {
 impl MetadataTableEntry {
     /// Parse one metadata entry from the buffer
     fn new(buffer: &[u8]) -> Result<MetadataTableEntry> {
-        assert!(buffer.len() == std::mem::size_of::<MetadataTableEntry>());
-        // SAFETY: the assertion above makes sure the buffer size is correct.
-        let mut metadata_table_entry: MetadataTableEntry = unsafe { *(buffer.as_ptr().cast()) };
-
-        let uuid = crate::vhdx::uuid_from_guid(buffer);
-        metadata_table_entry.item_id = uuid;
+        let metadata_table_entry = MetadataTableEntry::read_from_bytes(buffer).unwrap();
 
         if metadata_table_entry.length > METADATA_LENGTH_MAX {
             return Err(VhdxMetadataError::InvalidMetadataLength);

block/src/formats/vhdx/mod.rs

@@ -0,0 +1,118 @@
+// Copyright © 2021 Intel Corporation
+//
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+//! VHDX disk format support.
+//!
+//! Provides [`VhdxDisk`], the `DiskFile` wrapper for dynamic VHDX
+//! images.
+
+mod bat;
+mod engine_sync;
+mod header;
+mod io;
+mod metadata;
+mod parser;
+#[cfg(test)]
+mod test_util;
+
+use std::fs::File;
+use std::io::Error as IoError;
+use std::os::fd::AsRawFd;
+use std::sync::{Arc, Mutex};
+
+pub use parser::{Vhdx, VhdxError};
+
+use self::engine_sync::VhdxSync;
+use crate::async_io::{AsyncIo, BorrowedDiskFd, DiskFileError};
+use crate::error::{BlockError, BlockErrorKind, BlockResult, ErrorOp};
+use crate::{Error, disk_file};
+
+#[derive(Debug)]
+pub struct VhdxDisk {
+    // FIXME: The Mutex serializes all VHDX I/O operations across queues, which
+    // is necessary for correctness but eliminates any parallelism benefit from
+    // multiqueue. Vhdx::clone() shares the underlying file description across
+    // threads, so concurrent I/O from multiple queues races on the file offset
+    // causing data corruption.
+    //
+    // A proper fix would require restructuring the VHDX I/O path so that data
+    // operations can proceed in parallel with independent file descriptors.
+    vhdx_file: Arc<Mutex<Vhdx>>,
+}
+
+impl VhdxDisk {
+    pub fn new(f: File, direct_io: bool) -> BlockResult<Self> {
+        Ok(VhdxDisk {
+            vhdx_file: Arc::new(Mutex::new(Vhdx::new(f, direct_io).map_err(|e| {
+                let kind = match &e {
+                    VhdxError::NotVhdx(_)
+                    | VhdxError::ParseVhdxHeader(_)
+                    | VhdxError::ParseVhdxMetadata(_)
+                    | VhdxError::ParseVhdxRegionEntry(_) => BlockErrorKind::InvalidFormat,
+                    VhdxError::ReadBatEntry(_) => BlockErrorKind::CorruptImage,
+                    VhdxError::ReadFailed(_) | VhdxError::WriteFailed(_) => BlockErrorKind::Io,
+                };
+                BlockError::new(kind, e).with_op(ErrorOp::Open)
+            })?)),
+        })
+    }
+}
+
+impl disk_file::DiskSize for VhdxDisk {
+    fn logical_size(&self) -> BlockResult<u64> {
+        Ok(self.vhdx_file.lock().unwrap().virtual_disk_size())
+    }
+}
+
+impl disk_file::PhysicalSize for VhdxDisk {
+    fn physical_size(&self) -> BlockResult<u64> {
+        self.vhdx_file
+            .lock()
+            .unwrap()
+            .physical_size()
+            .map_err(|e| match e {
+                Error::GetFileMetadata(io) => {
+                    BlockError::new(BlockErrorKind::Io, Error::GetFileMetadata(io))
+                }
+                _ => unreachable!("unexpected error from Vhdx::physical_size(): {e}"),
+            })
+    }
+}
+
+impl disk_file::DiskFd for VhdxDisk {
+    fn fd(&self) -> BorrowedDiskFd<'_> {
+        BorrowedDiskFd::new(self.vhdx_file.lock().unwrap().as_raw_fd())
+    }
+}
+
+impl disk_file::Geometry for VhdxDisk {}
+
+impl disk_file::SparseCapable for VhdxDisk {}
+
+impl disk_file::Resizable for VhdxDisk {
+    fn resize(&mut self, _size: u64) -> BlockResult<()> {
+        Err(BlockError::new(
+            BlockErrorKind::UnsupportedFeature,
+            DiskFileError::ResizeError(IoError::other("resize not supported for VHDX")),
+        )
+        .with_op(ErrorOp::Resize))
+    }
+}
+
+impl disk_file::DiskFile for VhdxDisk {}
+
+impl disk_file::AsyncDiskFile for VhdxDisk {
+    fn try_clone(&self) -> BlockResult<Box<dyn disk_file::AsyncDiskFile>> {
+        Ok(Box::new(VhdxDisk {
+            vhdx_file: Arc::clone(&self.vhdx_file),
+        }))
+    }
+
+    fn create_async_io(&self, _ring_depth: u32) -> BlockResult<Box<dyn AsyncIo>> {
+        let size = self.vhdx_file.lock().unwrap().virtual_disk_size();
+        Ok(Box::new(VhdxSync::new(Arc::clone(&self.vhdx_file), size)))
+    }
+}

block/src/formats/vhdx/parser.rs

@@ -4,24 +4,20 @@
 
 use std::collections::btree_map::BTreeMap;
 use std::fs::File;
-use std::io::{Read, Seek, SeekFrom, Write};
+use std::io::{
+    Error as IoError, ErrorKind as IoErrorKind, Read, Result as IoResult, Seek, SeekFrom, Write,
+};
 use std::os::fd::{AsRawFd, RawFd};
+use std::result;
 
-use byteorder::{BigEndian, ByteOrder};
 use remain::sorted;
 use thiserror::Error;
-use uuid::Uuid;
 
-use crate::BlockBackend;
-use crate::vhdx::vhdx_bat::{BatEntry, VhdxBatError};
-use crate::vhdx::vhdx_header::{RegionInfo, RegionTableEntry, VhdxHeader, VhdxHeaderError};
-use crate::vhdx::vhdx_io::VhdxIoError;
-use crate::vhdx::vhdx_metadata::{DiskSpec, VhdxMetadataError};
-
-mod vhdx_bat;
-mod vhdx_header;
-mod vhdx_io;
-mod vhdx_metadata;
+use super::bat::{BatEntry, VhdxBatError};
+use super::header::{self, RegionInfo, RegionTableEntry, VhdxHeader, VhdxHeaderError};
+use super::io::{self, VhdxIoError};
+use super::metadata::{DiskSpec, VhdxMetadataError};
+use crate::aligned_file::AlignedFile;
 
 #[sorted]
 #[derive(Error, Debug)]
@@ -42,11 +38,11 @@ pub enum VhdxError {
     WriteFailed(#[source] VhdxIoError),
 }
 
-pub type Result<T> = std::result::Result<T, VhdxError>;
+pub(super) type Result<T> = result::Result<T, VhdxError>;
 
 #[derive(Debug)]
 pub struct Vhdx {
-    file: File,
+    aligned: AlignedFile,
     vhdx_header: VhdxHeader,
     region_entries: BTreeMap<u64, u64>,
     bat_entry: RegionTableEntry,
@@ -60,12 +56,14 @@ pub struct Vhdx {
 impl Vhdx {
     /// Parse the Vhdx header, BAT, and metadata from a file and store info
     // in Vhdx structure.
-    pub fn new(mut file: File) -> Result<Vhdx> {
-        let vhdx_header = VhdxHeader::new(&mut file).map_err(VhdxError::ParseVhdxHeader)?;
+    pub fn new(file: File, direct_io: bool) -> Result<Vhdx> {
+        let aligned = AlignedFile::new(file, direct_io);
+
+        let vhdx_header = VhdxHeader::new(&aligned).map_err(VhdxError::ParseVhdxHeader)?;
 
         let collected_entries = RegionInfo::new(
-            &mut file,
-            vhdx_header::REGION_TABLE_1_START,
+            &aligned,
+            header::REGION_TABLE_1_START,
             vhdx_header.region_entry_count(),
         )
         .map_err(VhdxError::ParseVhdxRegionEntry)?;
@@ -74,12 +72,12 @@ impl Vhdx {
         let mdr_entry = collected_entries.mdr_entry;
 
         let disk_spec =
-            DiskSpec::new(&mut file, &mdr_entry).map_err(VhdxError::ParseVhdxMetadata)?;
-        let bat_entries = BatEntry::collect_bat_entries(&mut file, &disk_spec, &bat_entry)
+            DiskSpec::new(&aligned, &mdr_entry).map_err(VhdxError::ParseVhdxMetadata)?;
+        let bat_entries = BatEntry::collect_bat_entries(&aligned, &disk_spec, &bat_entry)
             .map_err(VhdxError::ReadBatEntry)?;
 
         Ok(Vhdx {
-            file,
+            aligned,
             vhdx_header,
             region_entries: collected_entries.region_entries,
             bat_entry,
@@ -99,12 +97,12 @@ impl Vhdx {
 impl Read for Vhdx {
     /// Wrapper function to satisfy Read trait implementation for VHDx disk.
     /// Convert the offset to sector index and buffer length to sector count.
-    fn read(&mut self, buf: &mut [u8]) -> std::result::Result<usize, std::io::Error> {
+    fn read(&mut self, buf: &mut [u8]) -> IoResult<usize> {
         let sector_count = (buf.len() as u64).div_ceil(self.disk_spec.logical_sector_size as u64);
         let sector_index = self.current_offset / self.disk_spec.logical_sector_size as u64;
 
-        let result = vhdx_io::read(
-            &mut self.file,
+        let result = io::read(
+            &self.aligned,
             buf,
             &self.disk_spec,
             &self.bat_entries,
@@ -112,7 +110,7 @@ impl Read for Vhdx {
             sector_count,
         )
         .map_err(|e| {
-            std::io::Error::other(format!(
+            IoError::other(format!(
                 "Failed reading {sector_count} sectors from VHDx at index {sector_index}: {e}"
             ))
         })?;
@@ -124,25 +122,25 @@ impl Read for Vhdx {
 }
 
 impl Write for Vhdx {
-    fn flush(&mut self) -> std::result::Result<(), std::io::Error> {
-        self.file.flush()
+    fn flush(&mut self) -> IoResult<()> {
+        self.aligned.file_mut().flush()
     }
 
     /// Wrapper function to satisfy Write trait implementation for VHDx disk.
     /// Convert the offset to sector index and buffer length to sector count.
-    fn write(&mut self, buf: &[u8]) -> std::result::Result<usize, std::io::Error> {
+    fn write(&mut self, buf: &[u8]) -> IoResult<usize> {
         let sector_count = (buf.len() as u64).div_ceil(self.disk_spec.logical_sector_size as u64);
         let sector_index = self.current_offset / self.disk_spec.logical_sector_size as u64;
 
         if self.first_write {
             self.first_write = false;
             self.vhdx_header
-                .update(&mut self.file)
-                .map_err(|e| std::io::Error::other(format!("Failed to update VHDx header: {e}")))?;
+                .update(&self.aligned)
+                .map_err(|e| IoError::other(format!("Failed to update VHDx header: {e}")))?;
         }
 
-        let result = vhdx_io::write(
-            &mut self.file,
+        let result = io::write(
+            &self.aligned,
             buf,
             &mut self.disk_spec,
             self.bat_entry.file_offset,
@@ -151,7 +149,7 @@ impl Write for Vhdx {
             sector_count,
         )
         .map_err(|e| {
-            std::io::Error::other(format!(
+            IoError::other(format!(
                 "Failed writing {sector_count} sectors on VHDx at index {sector_index}: {e}"
             ))
         })?;
@@ -165,7 +163,7 @@ impl Write for Vhdx {
 impl Seek for Vhdx {
     /// Wrapper function to satisfy Seek trait implementation for VHDx disk.
     /// Updates the offset field in the Vhdx struct.
-    fn seek(&mut self, pos: SeekFrom) -> std::io::Result<u64> {
+    fn seek(&mut self, pos: SeekFrom) -> IoResult<u64> {
         let new_offset: Option<u64> = match pos {
             SeekFrom::Start(off) => Some(off),
             SeekFrom::End(off) => {
@@ -194,20 +192,17 @@ impl Seek for Vhdx {
             return Ok(o);
         }
 
-        Err(std::io::Error::new(
-            std::io::ErrorKind::InvalidData,
+        Err(IoError::new(
+            IoErrorKind::InvalidData,
             "Failed seek operation",
         ))
     }
 }
 
-impl BlockBackend for Vhdx {
-    fn logical_size(&self) -> std::result::Result<u64, crate::Error> {
-        Ok(self.virtual_disk_size())
-    }
-
-    fn physical_size(&self) -> std::result::Result<u64, crate::Error> {
-        self.file
+impl Vhdx {
+    pub(crate) fn physical_size(&self) -> result::Result<u64, crate::Error> {
+        self.aligned
+            .file()
             .metadata()
             .map(|m| m.len())
             .map_err(crate::Error::GetFileMetadata)
@@ -217,7 +212,7 @@ impl BlockBackend for Vhdx {
 impl Clone for Vhdx {
     fn clone(&self) -> Self {
         Vhdx {
-            file: self.file.try_clone().unwrap(),
+            aligned: self.aligned.try_clone().unwrap(),
             vhdx_header: self.vhdx_header.clone(),
             region_entries: self.region_entries.clone(),
             bat_entry: self.bat_entry,
@@ -232,18 +227,82 @@ impl Clone for Vhdx {
 
 impl AsRawFd for Vhdx {
     fn as_raw_fd(&self) -> RawFd {
-        self.file.as_raw_fd()
+        self.aligned.file().as_raw_fd()
     }
 }
 
-pub(crate) fn uuid_from_guid(buf: &[u8]) -> Uuid {
-    // The first 3 fields of UUID are stored in Big Endian format, and
-    // the last 8 bytes are stored as byte array. Therefore, we read the
-    // first 3 fields in Big Endian format instead of Little Endian.
-    Uuid::from_fields_le(
-        BigEndian::read_u32(&buf[0..4]),
-        BigEndian::read_u16(&buf[4..6]),
-        BigEndian::read_u16(&buf[6..8]),
-        buf[8..16].try_into().unwrap(),
-    )
+#[cfg(test)]
+mod tests {
+    use std::fs;
+
+    use super::*;
+    use crate::formats::vhdx::test_util::create_dynamic_vhdx;
+
+    /// An unaligned sector write under a forced O_DIRECT alignment must go
+    /// through `AlignedFile`'s read-modify-write bounce (the data block and the
+    /// BAT update both land at unaligned host offsets) and read back intact.
+    #[test]
+    fn unaligned_write_is_rmw() {
+        let Some(tf) = create_dynamic_vhdx(16) else {
+            eprintln!("skipping unaligned_write_is_rmw: qemu-img unavailable");
+            return;
+        };
+
+        let file = fs::OpenOptions::new()
+            .read(true)
+            .write(true)
+            .open(tf.as_path())
+            .unwrap();
+        let mut vhdx = Vhdx::new(file, false).unwrap();
+
+        // Force a non-zero alignment so all of vhdx's positioned I/O exercises
+        // the bounce/RMW path even though the tempfile is not really O_DIRECT.
+        vhdx.aligned = AlignedFile::with_alignment(vhdx.aligned.file().try_clone().unwrap(), 512);
+
+        let sector = vhdx.disk_spec.logical_sector_size as usize;
+        let data: Vec<u8> = (0..sector).map(|i| ((i + 1) % 251) as u8).collect();
+
+        // Write at virtual offset 0 (allocates a new data block + rewrites BAT).
+        vhdx.seek(SeekFrom::Start(0)).unwrap();
+        assert_eq!(vhdx.write(&data).unwrap(), data.len());
+        vhdx.flush().unwrap();
+
+        // Read it back through a fresh, forced-alignment handle.
+        let mut readback = vec![0u8; sector];
+        vhdx.seek(SeekFrom::Start(0)).unwrap();
+        assert_eq!(vhdx.read(&mut readback).unwrap(), readback.len());
+        assert_eq!(readback, data);
+    }
+
+    #[test]
+    fn header_update_survives_reopen() {
+        let Some(tf) = create_dynamic_vhdx(16) else {
+            eprintln!("skipping header_update_survives_reopen: qemu-img unavailable");
+            return;
+        };
+
+        let data = [0xa5u8; 512];
+        {
+            let file = fs::OpenOptions::new()
+                .read(true)
+                .write(true)
+                .open(tf.as_path())
+                .unwrap();
+            let mut vhdx = Vhdx::new(file, false).unwrap();
+            vhdx.seek(SeekFrom::Start(0)).unwrap();
+            assert_eq!(vhdx.write(&data).unwrap(), data.len());
+            vhdx.flush().unwrap();
+        }
+
+        let file = fs::OpenOptions::new()
+            .read(true)
+            .write(true)
+            .open(tf.as_path())
+            .unwrap();
+        let mut vhdx = Vhdx::new(file, false).unwrap();
+        let mut readback = [0u8; 512];
+        vhdx.seek(SeekFrom::Start(0)).unwrap();
+        assert_eq!(vhdx.read(&mut readback).unwrap(), readback.len());
+        assert_eq!(readback, data);
+    }
 }

block/src/formats/vhdx/test_util.rs

@@ -0,0 +1,25 @@
+// Copyright 2026 The Cloud Hypervisor Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+//! Shared test helpers for VHDX image tests.
+
+use std::process::Command;
+
+use vmm_sys_util::tempfile::TempFile;
+
+/// Generate a small dynamic VHDX with `qemu-img`. Returns `None` (and the
+/// test is skipped) when `qemu-img` is unavailable, e.g. in minimal CI.
+pub(crate) fn create_dynamic_vhdx(size_mib: u64) -> Option<TempFile> {
+    let tf = TempFile::new().unwrap();
+    let path = tf.as_path();
+    let status = Command::new("qemu-img")
+        .args(["create", "-f", "vhdx", "-o", "subformat=dynamic"])
+        .arg(path)
+        .arg(format!("{size_mib}M"))
+        .status();
+    match status {
+        Ok(s) if s.success() => Some(tf),
+        _ => None,
+    }
+}

block/src/io/async_io.rs

@@ -0,0 +1,184 @@
+// Copyright © 2021 Intel Corporation
+//
+// SPDX-License-Identifier: Apache-2.0 AND BSD-3-Clause
+
+use std::marker::PhantomData;
+use std::os::fd::{AsRawFd, OwnedFd, RawFd};
+mod aio_data_io;
+mod common;
+mod completion;
+mod guest_memory_target;
+mod operation;
+mod owned_io_buffer;
+#[cfg(feature = "io_uring")]
+mod uring_data_io;
+
+use std::{io, result};
+
+pub use aio_data_io::AioDataIo;
+pub use completion::AsyncIoCompletion;
+pub use guest_memory_target::GuestMemoryTarget;
+pub use operation::AsyncIoOperation;
+pub use owned_io_buffer::OwnedIoBuffer;
+use thiserror::Error;
+#[cfg(feature = "io_uring")]
+pub use uring_data_io::UringDataIo;
+use vmm_sys_util::eventfd::EventFd;
+
+use crate::SECTOR_SIZE;
+
+#[derive(Error, Debug)]
+pub enum DiskFileError {
+    /// Failed getting disk file size.
+    #[error("Failed getting disk file size")]
+    Size(#[source] io::Error),
+    /// Failed creating a new AsyncIo.
+    #[error("Failed creating a new AsyncIo")]
+    NewAsyncIo(#[source] io::Error),
+    /// Unsupported operation.
+    #[error("Unsupported operation")]
+    Unsupported,
+    /// Resize failed
+    #[error("Resize failed")]
+    ResizeError(#[source] io::Error),
+    #[error("Failed cloning disk file")]
+    Clone(#[source] io::Error),
+}
+
+pub type DiskFileResult<T> = result::Result<T, DiskFileError>;
+
+/// A wrapper for [`RawFd`] capturing the lifetime of a corresponding disk file.
+///
+/// This fulfills the same role as [`BorrowedFd`] but is tailored to the limitations
+/// by some disk implementations, which wrap the effective [`File`]
+/// in an `Arc<Mutex<T>>`, making the use of [`BorrowedFd`] impossible.
+///
+/// [`BorrowedFd`]: std::os::fd::BorrowedFd
+#[derive(Copy, Clone, Debug)]
+pub struct BorrowedDiskFd<'fd> {
+    raw_fd: RawFd,
+    _lifetime: PhantomData<&'fd OwnedFd>,
+}
+
+impl BorrowedDiskFd<'_> {
+    pub(crate) fn new(raw_fd: RawFd) -> Self {
+        Self {
+            raw_fd,
+            _lifetime: PhantomData,
+        }
+    }
+}
+
+impl AsRawFd for BorrowedDiskFd<'_> {
+    fn as_raw_fd(&self) -> RawFd {
+        self.raw_fd
+    }
+}
+
+#[derive(Error, Debug)]
+pub enum AsyncIoError {
+    /// Failed vectored reading from file.
+    #[error("Failed vectored reading from file")]
+    ReadVectored(#[source] io::Error),
+    /// Failed vectored writing to file.
+    #[error("Failed vectored writing to file")]
+    WriteVectored(#[source] io::Error),
+    /// Failed synchronizing file.
+    #[error("Failed synchronizing file")]
+    Fsync(#[source] io::Error),
+    /// Failed punching hole.
+    #[error("Failed punching hole")]
+    PunchHole(#[source] io::Error),
+    /// Failed writing zeroes.
+    #[error("Failed writing zeroes")]
+    WriteZeroes(#[source] io::Error),
+    /// Failed submitting batch requests.
+    #[error("Failed submitting batch requests")]
+    SubmitBatchRequests(#[source] io::Error),
+}
+
+pub type AsyncIoResult<T> = result::Result<T, AsyncIoError>;
+
+pub trait AsyncIo: Send {
+    fn notifier(&self) -> &EventFd;
+
+    /// Submits one owned data operation.
+    ///
+    /// Takes ownership of `op`.
+    /// Implementations that complete asynchronously must retain it until its
+    /// completion is returned.
+    fn submit_data_operation(&mut self, op: AsyncIoOperation) -> AsyncIoResult<()>;
+
+    /// Submits a read from `offset` into guest memory.
+    fn read_to_memory(
+        &mut self,
+        offset: libc::off_t,
+        target: GuestMemoryTarget,
+        user_data: u64,
+    ) -> AsyncIoResult<()> {
+        self.submit_data_operation(AsyncIoOperation::read_to_memory(offset, target, user_data))
+    }
+
+    /// Submits a write to `offset` from guest memory.
+    fn write_from_memory(
+        &mut self,
+        offset: libc::off_t,
+        target: GuestMemoryTarget,
+        user_data: u64,
+    ) -> AsyncIoResult<()> {
+        self.submit_data_operation(AsyncIoOperation::write_from_memory(
+            offset, target, user_data,
+        ))
+    }
+
+    /// Submits a read from `offset` into an owned host-memory buffer.
+    fn read_to_vec(
+        &mut self,
+        offset: libc::off_t,
+        buffer: OwnedIoBuffer,
+        user_data: u64,
+    ) -> AsyncIoResult<()> {
+        self.submit_data_operation(AsyncIoOperation::read_to_vec(offset, buffer, user_data))
+    }
+
+    /// Submits a write to `offset` from an owned host-memory buffer.
+    fn write_from_vec(
+        &mut self,
+        offset: libc::off_t,
+        buffer: OwnedIoBuffer,
+        user_data: u64,
+    ) -> AsyncIoResult<()> {
+        self.submit_data_operation(AsyncIoOperation::write_from_vec(offset, buffer, user_data))
+    }
+
+    fn fsync(&mut self, user_data: Option<u64>) -> AsyncIoResult<()>;
+    fn punch_hole(&mut self, offset: u64, length: u64, user_data: u64) -> AsyncIoResult<()>;
+    fn write_zeroes(&mut self, offset: u64, length: u64, user_data: u64) -> AsyncIoResult<()>;
+
+    /// Returns the next owned completion, if one is available.
+    ///
+    /// Read completions from owned host-memory buffers return that buffer here.
+    fn next_completed_request(&mut self) -> Option<AsyncIoCompletion>;
+
+    fn batch_requests_enabled(&self) -> bool {
+        false
+    }
+
+    /// Submits a batch of owned data operations.
+    ///
+    /// Backends either accept the whole batch for eventual completion or return
+    /// an error before taking ownership of any operation.
+    fn submit_batch_requests(&mut self, batch_request: Vec<AsyncIoOperation>) -> AsyncIoResult<()> {
+        if batch_request.is_empty() {
+            Ok(())
+        } else {
+            Err(AsyncIoError::SubmitBatchRequests(io::Error::other(
+                "batch requests are not supported by this backend",
+            )))
+        }
+    }
+
+    fn alignment(&self) -> u64 {
+        SECTOR_SIZE
+    }
+}

block/src/io/async_io/aio_data_io.rs

@@ -0,0 +1,235 @@
+// Copyright © 2023 Intel Corporation
+//
+// Copyright © 2023 Crusoe Energy Systems LLC
+//
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+//
+// SPDX-License-Identifier: Apache-2.0 AND BSD-3-Clause
+
+use std::collections::{HashMap, VecDeque};
+use std::os::fd::{AsRawFd, RawFd};
+use std::{io, slice};
+
+use log::warn;
+use vmm_sys_util::aio;
+use vmm_sys_util::eventfd::EventFd;
+
+use super::common::{duplicate_user_data_error, errno_result, validate_batch};
+use super::{AsyncIoCompletion, AsyncIoOperation};
+
+/// Retained Linux AIO queue for owned async data I/O operations.
+pub struct AioDataIo {
+    // Keep this before `in_flight`: Rust drops fields in declaration order, so
+    // dropping the context destroys kernel AIO state before retained
+    // operations release the buffers referenced by their iovecs.
+    ctx: aio::IoContext,
+    // The `EventFd` for completion signals.
+    eventfd: EventFd,
+    // `in_flight` tracks every user_data value accepted by the kernel. Owned
+    // data operations store `Some(op)` so their iovecs and backing buffers
+    // remain valid until completion; metadata operations store `None`.
+    in_flight: HashMap<u64, Option<AsyncIoOperation>>,
+    // `completions` holds locally produced completions and kernel events that
+    // have been fetched but not yet returned to the caller.
+    completions: VecDeque<AsyncIoCompletion>,
+}
+
+impl AioDataIo {
+    /// Creates a Linux AIO context and its completion eventfd.
+    pub fn new(queue_depth: u32) -> io::Result<Self> {
+        Ok(Self {
+            ctx: aio::IoContext::new(queue_depth)?,
+            eventfd: EventFd::new(libc::EFD_NONBLOCK)?,
+            in_flight: HashMap::new(),
+            completions: VecDeque::new(),
+        })
+    }
+
+    /// Returns the eventfd signaled when completions are available.
+    pub fn notifier(&self) -> &EventFd {
+        &self.eventfd
+    }
+
+    #[allow(unused_unsafe)]
+    fn submit_iocbs(ctx: &aio::IoContext, iocbs: &[&mut aio::IoControlBlock]) -> io::Result<usize> {
+        // SAFETY: vmm_sys_util currently marks IoContext::submit safe, but
+        // io_submit consumes raw pointers asynchronously. Callers must ensure
+        // all iovec and buffer memory referenced by each iocb remains valid
+        // until completion or failed submission.
+        unsafe { ctx.submit(iocbs) }
+    }
+
+    /// Submits one owned read or write operation to the queue.
+    ///
+    /// Submission failures are converted into injected completions so callers
+    /// can observe every accepted request through the normal completion path.
+    pub fn submit_operation(&mut self, fd: RawFd, op: AsyncIoOperation) -> io::Result<()> {
+        validate_batch(
+            |user_data| self.in_flight.contains_key(&user_data),
+            slice::from_ref(&op),
+        )?;
+
+        let user_data = op.user_data();
+        let iovecs = op.iovecs();
+        let opcode = if op.is_read() {
+            aio::IOCB_CMD_PREADV
+        } else {
+            aio::IOCB_CMD_PWRITEV
+        };
+        let mut iocb = aio::IoControlBlock {
+            aio_fildes: fd.as_raw_fd() as u32,
+            aio_lio_opcode: opcode as u16,
+            aio_buf: iovecs.as_ptr() as u64,
+            aio_nbytes: iovecs.len() as u64,
+            aio_offset: op.offset(),
+            aio_data: user_data,
+            aio_flags: aio::IOCB_FLAG_RESFD,
+            aio_resfd: self.eventfd.as_raw_fd() as u32,
+            ..Default::default()
+        };
+        self.in_flight.insert(user_data, Some(op));
+
+        let result = match Self::submit_iocbs(&self.ctx, &[&mut iocb]) {
+            Ok(1) => return Ok(()),
+            Ok(_) => -libc::EAGAIN,
+            Err(e) => errno_result(&e),
+        };
+
+        let buffer = self
+            .in_flight
+            .remove(&user_data)
+            .flatten()
+            .and_then(AsyncIoOperation::into_completion_buffer);
+        self.inject_completion(AsyncIoCompletion::new(user_data, result, buffer));
+        Ok(())
+    }
+
+    /// Submits an fsync operation carrying `user_data`.
+    pub fn submit_fsync(&mut self, fd: RawFd, user_data: u64) -> io::Result<()> {
+        if self.in_flight.contains_key(&user_data) {
+            return Err(duplicate_user_data_error(user_data));
+        }
+
+        let mut iocb = aio::IoControlBlock {
+            aio_fildes: fd.as_raw_fd() as u32,
+            aio_lio_opcode: aio::IOCB_CMD_FSYNC as u16,
+            aio_data: user_data,
+            aio_flags: aio::IOCB_FLAG_RESFD,
+            aio_resfd: self.eventfd.as_raw_fd() as u32,
+            ..Default::default()
+        };
+        self.in_flight.insert(user_data, None);
+        let result = match Self::submit_iocbs(&self.ctx, &[&mut iocb]) {
+            Ok(1) => return Ok(()),
+            Ok(_) => -libc::EAGAIN,
+            Err(e) => errno_result(&e),
+        };
+
+        self.in_flight.remove(&user_data);
+        self.inject_completion(AsyncIoCompletion::new(user_data, result, None));
+        Ok(())
+    }
+
+    /// Injects a completion that did not come from a kernel AIO event.
+    ///
+    /// The notifier is signaled so callers can drain it with
+    /// [`Self::next_completion`].
+    pub fn inject_completion(&mut self, completion: AsyncIoCompletion) {
+        self.completions.push_back(completion);
+        self.eventfd.write(1).unwrap();
+    }
+
+    /// Returns the next kernel or injected completion if one is available.
+    ///
+    /// Consuming a kernel completion returns ownership of any buffer retained
+    /// by the corresponding operation.
+    pub fn next_completion(&mut self) -> Option<AsyncIoCompletion> {
+        if self.completions.is_empty() {
+            let mut events = [aio::IoEvent::default(); 32];
+            let rc = match self.ctx.get_events(0, &mut events, None) {
+                Ok(rc) => rc,
+                Err(e) => {
+                    warn!("Linux AIO get_events failed: {e}");
+                    return None;
+                }
+            };
+            for event in &events[..rc] {
+                self.completions.push_back(AsyncIoCompletion::new(
+                    event.data,
+                    event.res as i32,
+                    self.in_flight
+                        .remove(&event.data)
+                        .flatten()
+                        .and_then(AsyncIoOperation::into_completion_buffer),
+                ));
+            }
+        }
+
+        self.completions.pop_front()
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::io::{self, Write};
+    use std::os::fd::AsRawFd;
+    use std::thread::sleep;
+    use std::time::Duration;
+
+    use vmm_sys_util::tempfile::TempFile;
+
+    use super::AioDataIo;
+    use crate::async_io::{AsyncIoCompletion, AsyncIoOperation, OwnedIoBuffer};
+
+    fn wait_for_completion(data_io: &mut AioDataIo) -> AsyncIoCompletion {
+        for _ in 0..1000 {
+            if let Some(completion) = data_io.next_completion() {
+                return completion;
+            }
+            sleep(Duration::from_millis(1));
+        }
+
+        panic!("timed out waiting for Linux AIO completion");
+    }
+
+    #[test]
+    fn aio_rejects_duplicate_user_data_for_metadata_ops() {
+        let mut file = TempFile::new().unwrap().into_file();
+        file.write_all(&[0xa5; 512]).unwrap();
+        let fd = file.as_raw_fd();
+        let mut data_io = AioDataIo::new(8).unwrap();
+
+        data_io
+            .submit_operation(
+                fd,
+                AsyncIoOperation::read_to_vec(0, OwnedIoBuffer::from_vec(vec![0; 512]), 7),
+            )
+            .unwrap();
+
+        assert_eq!(
+            data_io.submit_fsync(fd, 7).unwrap_err().kind(),
+            io::ErrorKind::AlreadyExists
+        );
+
+        let completion = wait_for_completion(&mut data_io);
+        assert_eq!(completion.user_data, 7);
+        assert_eq!(completion.result, 512);
+        assert_eq!(
+            completion.buffer.unwrap().as_slice(),
+            [0xa5; 512].as_slice()
+        );
+    }
+
+    #[test]
+    fn aio_injected_completion_uses_completion_path() {
+        let mut data_io = AioDataIo::new(8).unwrap();
+
+        data_io.inject_completion(AsyncIoCompletion::new(9, -libc::EIO, None));
+
+        let completion = data_io.next_completion().unwrap();
+        assert_eq!(completion.user_data, 9);
+        assert_eq!(completion.result, -libc::EIO);
+        assert!(completion.buffer.is_none());
+        assert!(data_io.next_completion().is_none());
+    }
+}

block/src/io/async_io/common.rs

@@ -0,0 +1,40 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+//
+// SPDX-License-Identifier: Apache-2.0 AND BSD-3-Clause
+
+//! Helpers used by both aio and uring async io.
+
+use std::collections::HashSet;
+use std::io;
+
+use super::AsyncIoOperation;
+
+/// Converts an I/O error into the negative errno form used in completions.
+pub(super) fn errno_result(error: &io::Error) -> i32 {
+    -error.raw_os_error().unwrap_or(libc::EIO)
+}
+
+/// Builds the error returned when a new request reuses in-flight `user_data`.
+pub(super) fn duplicate_user_data_error(user_data: u64) -> io::Error {
+    io::Error::new(
+        io::ErrorKind::AlreadyExists,
+        format!("duplicate async I/O user_data {user_data}"),
+    )
+}
+
+/// Validates that a batch has unique `user_data` not already in flight.
+pub(super) fn validate_batch<F>(mut is_in_flight: F, batch: &[AsyncIoOperation]) -> io::Result<()>
+where
+    F: FnMut(u64) -> bool,
+{
+    let mut seen = HashSet::with_capacity(batch.len());
+
+    for op in batch {
+        let user_data = op.user_data();
+        if is_in_flight(user_data) || !seen.insert(user_data) {
+            return Err(duplicate_user_data_error(user_data));
+        }
+    }
+
+    Ok(())
+}

block/src/io/async_io/completion.rs

@@ -0,0 +1,42 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+//
+// SPDX-License-Identifier: Apache-2.0 AND BSD-3-Clause
+
+use super::{AsyncIoOperation, OwnedIoBuffer};
+
+/// Completion returned by an owned async I/O backend.
+///
+/// The completion carries the caller provided `user_data`, the result,
+/// and any owned buffer that can now be dropped.
+#[derive(Debug)]
+pub struct AsyncIoCompletion {
+    /// Caller provided identifier associated with the submitted operation.
+    pub user_data: u64,
+    /// I/O result reported by the backend.
+    ///
+    /// Successful operations report a non-negative byte count. Failed
+    /// operations report a negative errno value.
+    pub result: i32,
+    /// The backing buffer that can now be dropped or re-used.
+    pub buffer: Option<OwnedIoBuffer>,
+}
+
+impl AsyncIoCompletion {
+    /// Creates a completion from its parts.
+    pub fn new(user_data: u64, result: i32, buffer: Option<OwnedIoBuffer>) -> Self {
+        Self {
+            user_data,
+            result,
+            buffer,
+        }
+    }
+
+    /// Creates a completion by consuming the operation that just completed.
+    ///
+    /// This returns ownership of any completion buffer carried by the
+    /// operation.
+    pub fn from_operation(op: AsyncIoOperation, result: i32) -> Self {
+        let user_data = op.user_data();
+        Self::new(user_data, result, op.into_completion_buffer())
+    }
+}

block/src/io/async_io/guest_memory_target.rs

@@ -0,0 +1,241 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+//
+// SPDX-License-Identifier: Apache-2.0 AND BSD-3-Clause
+
+use std::cmp::min;
+use std::fmt;
+use std::sync::Arc;
+
+use smallvec::SmallVec;
+use vm_memory::bitmap::Bitmap;
+use vm_memory::{Address, Bytes, GuestAddress, GuestMemory, GuestMemoryError, GuestMemoryMmap};
+
+trait GuestMemoryTargetOwner: Send + Sync {
+    fn iovec_for_range(
+        &self,
+        addr: GuestAddress,
+        len: usize,
+    ) -> Result<libc::iovec, GuestMemoryError>;
+    fn write_guest_slice(&self, buf: &[u8], addr: GuestAddress) -> Result<(), GuestMemoryError>;
+    fn read_guest_slice(&self, buf: &mut [u8], addr: GuestAddress) -> Result<(), GuestMemoryError>;
+}
+
+impl<B> GuestMemoryTargetOwner for GuestMemoryMmap<B>
+where
+    B: Bitmap + Send + Sync + 'static,
+{
+    fn iovec_for_range(
+        &self,
+        addr: GuestAddress,
+        len: usize,
+    ) -> Result<libc::iovec, GuestMemoryError> {
+        let slice = self.get_slice(addr, len)?;
+        let guard = slice.ptr_guard_mut();
+
+        Ok(libc::iovec {
+            iov_base: guard.as_ptr().cast(),
+            iov_len: len,
+        })
+    }
+
+    fn write_guest_slice(&self, buf: &[u8], addr: GuestAddress) -> Result<(), GuestMemoryError> {
+        <Self as Bytes<GuestAddress>>::write_slice(self, buf, addr)
+    }
+
+    fn read_guest_slice(&self, buf: &mut [u8], addr: GuestAddress) -> Result<(), GuestMemoryError> {
+        <Self as Bytes<GuestAddress>>::read_slice(self, buf, addr)
+    }
+}
+
+/// Retains a guest-memory Arc and the validated ranges used for I/O.
+///
+/// Keeping the guest memory arc with the ranges guarantees that the iovecs
+/// remain valid for as long as Self is alive. The iovecs are also shared with
+/// the kernel and must be stable.
+pub struct GuestMemoryTarget {
+    owner: Arc<dyn GuestMemoryTargetOwner>,
+    ranges: SmallVec<[(GuestAddress, usize); 1]>,
+    iovecs: Vec<libc::iovec>,
+}
+
+// SAFETY: GuestMemoryTarget owns an Arc to the guest memory backing and
+// holds its iovecs in a heap allocation, so moving the target leaves the
+// iovec addresses (and the host pointers they reference) stable.
+unsafe impl Send for GuestMemoryTarget {}
+
+impl GuestMemoryTarget {
+    /// Creates a new `GuestMemoryTarget`.
+    ///
+    /// The memory Arc is retained for the life of `Self`, making this
+    /// appropriate for asynchronous I/O operations on the specified ranges.
+    pub fn new<B>(
+        mem: Arc<GuestMemoryMmap<B>>,
+        ranges: &[(GuestAddress, u32)],
+    ) -> Result<Self, GuestMemoryError>
+    where
+        B: Bitmap + Send + Sync + 'static,
+    {
+        let retained_ranges: SmallVec<[(GuestAddress, usize); 1]> = ranges
+            .iter()
+            .copied()
+            .filter(|&(_, len)| len != 0)
+            .map(|(addr, len)| {
+                let len = len as usize;
+                mem.get_slice(addr, len)?;
+                Ok((addr, len))
+            })
+            .collect::<Result<SmallVec<[_; 1]>, GuestMemoryError>>()?;
+
+        // iovec_for_range cannot fail: each range was just validated by
+        // get_slice above and the Arc keeps the mapping alive.
+        let iovecs: Vec<libc::iovec> = retained_ranges
+            .iter()
+            .map(|&(addr, len)| {
+                mem.iovec_for_range(addr, len)
+                    .expect("range validated above and retained by owner Arc")
+            })
+            .collect();
+
+        Ok(Self {
+            owner: mem,
+            ranges: retained_ranges,
+            iovecs,
+        })
+    }
+
+    /// Returns the raw iovecs to be passed to the kernel for asynchronous I/O.
+    pub(super) fn iovecs(&self) -> &[libc::iovec] {
+        &self.iovecs
+    }
+
+    /// Returns the total length of the ranges specified at creation.
+    pub fn total_len(&self) -> usize {
+        self.ranges.iter().map(|(_, len)| len).sum()
+    }
+
+    pub(crate) fn write_bytes_at(&self, start: usize, data: &[u8]) -> Result<(), GuestMemoryError> {
+        self.for_each_range(start, data.len(), |addr, offset, len| {
+            self.owner
+                .write_guest_slice(&data[offset..offset + len], addr)
+        })
+    }
+
+    pub(crate) fn read_bytes_at(
+        &self,
+        start: usize,
+        data: &mut [u8],
+    ) -> Result<(), GuestMemoryError> {
+        self.for_each_range(start, data.len(), |addr, offset, len| {
+            self.owner
+                .read_guest_slice(&mut data[offset..offset + len], addr)
+        })
+    }
+
+    pub(crate) fn fill_zeroes_at(&self, start: usize, len: usize) -> Result<(), GuestMemoryError> {
+        let zeroes = [0u8; 4096];
+        self.for_each_range(start, len, |addr, _, mut len| {
+            let mut offset = 0usize;
+            while len > 0 {
+                let count = min(len, zeroes.len());
+                let addr = addr
+                    .checked_add(offset as u64)
+                    .ok_or(GuestMemoryError::InvalidGuestAddress(addr))?;
+                self.owner.write_guest_slice(&zeroes[..count], addr)?;
+                offset += count;
+                len -= count;
+            }
+            Ok(())
+        })
+    }
+
+    fn for_each_range<F>(&self, start: usize, len: usize, mut f: F) -> Result<(), GuestMemoryError>
+    where
+        F: FnMut(GuestAddress, usize, usize) -> Result<(), GuestMemoryError>,
+    {
+        self.validate_range(start, len)?;
+
+        let mut copied = 0usize;
+        let mut pos = 0usize;
+        for &(addr, range_len) in self.ranges.iter() {
+            let range_end = pos + range_len;
+            if range_end <= start || copied == len {
+                pos = range_end;
+                continue;
+            }
+
+            let range_start = start.saturating_sub(pos);
+            let count = min(range_len - range_start, len - copied);
+            let addr = addr
+                .checked_add(range_start as u64)
+                .ok_or(GuestMemoryError::InvalidGuestAddress(addr))?;
+            f(addr, copied, count)?;
+
+            copied += count;
+            if copied == len {
+                break;
+            }
+            pos = range_end;
+        }
+
+        if copied != len {
+            return Err(GuestMemoryError::PartialBuffer {
+                expected: len,
+                completed: copied,
+            });
+        }
+
+        Ok(())
+    }
+
+    fn validate_range(&self, start: usize, len: usize) -> Result<(), GuestMemoryError> {
+        let total_len = self.total_len();
+        if start <= total_len
+            && let Some(end) = start.checked_add(len)
+            && end <= total_len
+        {
+            return Ok(());
+        }
+
+        Err(GuestMemoryError::PartialBuffer {
+            expected: len,
+            completed: total_len.saturating_sub(start).min(len),
+        })
+    }
+}
+
+impl fmt::Debug for GuestMemoryTarget {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        let mut debug = f.debug_struct("GuestMemoryTarget");
+        debug.field("ranges", &self.ranges.len());
+        debug
+            .field("iovecs", &self.iovecs.len())
+            .finish_non_exhaustive()
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::sync::Arc;
+
+    use vm_memory::{GuestAddress, GuestMemoryMmap};
+
+    use super::GuestMemoryTarget;
+
+    #[test]
+    fn iovecs_survive_move() {
+        // The iovec array must live on the heap so its address stays valid
+        // after the GuestMemoryTarget (and the AsyncIoOperation that owns it)
+        // is moved into an in-flight map. Capture the addresses before the
+        // move and confirm they still match afterwards.
+        let mem = Arc::new(GuestMemoryMmap::<()>::from_ranges(&[(GuestAddress(0), 4096)]).unwrap());
+        let target = GuestMemoryTarget::new(mem, &[(GuestAddress(0), 512)]).unwrap();
+        let iovec_ptr_before = target.iovecs().as_ptr() as usize;
+        let iov_base_before = target.iovecs()[0].iov_base as usize;
+
+        let moved = Box::new(target);
+
+        assert_eq!(moved.iovecs().as_ptr() as usize, iovec_ptr_before);
+        assert_eq!(moved.iovecs()[0].iov_base as usize, iov_base_before);
+        assert_eq!(moved.iovecs().len(), 1);
+    }
+}

block/src/io/async_io/operation.rs

@@ -0,0 +1,308 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+//
+// SPDX-License-Identifier: Apache-2.0 AND BSD-3-Clause
+
+use std::io;
+use std::ops::Range;
+
+use super::{AsyncIoError, AsyncIoResult, GuestMemoryTarget, OwnedIoBuffer};
+
+/// A single async IO operation.
+///
+/// Each operation owns or retains the memory target for the duration of the
+/// operation so backends can submit it to the kernel or copy through safe helper
+/// methods.
+#[derive(Debug)]
+pub enum AsyncIoOperation {
+    /// Read from disk into guest memory.
+    ReadToMemory {
+        /// Disk offset for the operation.
+        offset: libc::off_t,
+        /// Guest-memory destination.
+        target: GuestMemoryTarget,
+        /// Caller-provided completion identifier.
+        user_data: u64,
+    },
+    /// Write from guest memory to disk.
+    WriteFromMemory {
+        /// Disk offset for the operation.
+        offset: libc::off_t,
+        /// Guest-memory source.
+        target: GuestMemoryTarget,
+        /// Caller-provided completion identifier.
+        user_data: u64,
+    },
+    /// Read from disk into an owned host-memory buffer.
+    ReadToVec {
+        /// Disk offset for the operation.
+        offset: libc::off_t,
+        /// Owned destination buffer.
+        buffer: OwnedIoBuffer,
+        /// Caller-provided completion identifier.
+        user_data: u64,
+    },
+    /// Write from an owned host-memory buffer to disk.
+    WriteFromVec {
+        /// Disk offset for the operation.
+        offset: libc::off_t,
+        /// Owned source buffer.
+        buffer: OwnedIoBuffer,
+        /// Caller-provided completion identifier.
+        user_data: u64,
+    },
+}
+
+impl AsyncIoOperation {
+    /// Creates an operation that reads from disk into guest memory.
+    pub fn read_to_memory(offset: libc::off_t, target: GuestMemoryTarget, user_data: u64) -> Self {
+        Self::ReadToMemory {
+            offset,
+            target,
+            user_data,
+        }
+    }
+
+    /// Creates an operation that writes from guest memory to disk.
+    pub fn write_from_memory(
+        offset: libc::off_t,
+        target: GuestMemoryTarget,
+        user_data: u64,
+    ) -> Self {
+        Self::WriteFromMemory {
+            offset,
+            target,
+            user_data,
+        }
+    }
+
+    /// Creates an operation that reads from disk into an owned buffer.
+    pub fn read_to_vec(offset: libc::off_t, buffer: OwnedIoBuffer, user_data: u64) -> Self {
+        Self::ReadToVec {
+            offset,
+            buffer,
+            user_data,
+        }
+    }
+
+    /// Creates an operation that writes from an owned buffer to disk.
+    pub fn write_from_vec(offset: libc::off_t, buffer: OwnedIoBuffer, user_data: u64) -> Self {
+        Self::WriteFromVec {
+            offset,
+            buffer,
+            user_data,
+        }
+    }
+
+    /// Returns the value provided at construction.
+    pub fn user_data(&self) -> u64 {
+        match self {
+            Self::ReadToMemory { user_data, .. }
+            | Self::WriteFromMemory { user_data, .. }
+            | Self::ReadToVec { user_data, .. }
+            | Self::WriteFromVec { user_data, .. } => *user_data,
+        }
+    }
+
+    /// Returns the disk offset for this operation.
+    pub fn offset(&self) -> libc::off_t {
+        match self {
+            Self::ReadToMemory { offset, .. }
+            | Self::WriteFromMemory { offset, .. }
+            | Self::ReadToVec { offset, .. }
+            | Self::WriteFromVec { offset, .. } => *offset,
+        }
+    }
+
+    /// Updates the disk offset for this operation.
+    pub fn set_offset(&mut self, new_offset: libc::off_t) {
+        match self {
+            Self::ReadToMemory { offset, .. }
+            | Self::WriteFromMemory { offset, .. }
+            | Self::ReadToVec { offset, .. }
+            | Self::WriteFromVec { offset, .. } => *offset = new_offset,
+        }
+    }
+
+    /// Returns whether this operation reads from disk.
+    pub fn is_read(&self) -> bool {
+        matches!(self, Self::ReadToMemory { .. } | Self::ReadToVec { .. })
+    }
+
+    /// Rejects an operation whose byte range falls outside a disk of `size` bytes.
+    ///
+    /// Returns the read/write-specific `AsyncIoError` variant, carrying an
+    /// `InvalidData` error, when the offset overflows or `offset + len`
+    /// exceeds `size`.
+    pub(crate) fn validate_bounds(&self, size: u64) -> AsyncIoResult<()> {
+        let bounds_error = || {
+            let error = io::Error::new(
+                io::ErrorKind::InvalidData,
+                format!(
+                    "Invalid request offset {} and length {}, can't exceed file size {}",
+                    self.offset(),
+                    self.total_len(),
+                    size
+                ),
+            );
+            if self.is_read() {
+                AsyncIoError::ReadVectored(error)
+            } else {
+                AsyncIoError::WriteVectored(error)
+            }
+        };
+
+        let offset = u64::try_from(self.offset()).map_err(|_| bounds_error())?;
+        let len = u64::try_from(self.total_len()).map_err(|_| bounds_error())?;
+        let end = offset.checked_add(len).ok_or_else(bounds_error)?;
+
+        if end > size {
+            return Err(bounds_error());
+        }
+
+        Ok(())
+    }
+
+    /// Returns the retained iovec array for kernel submission.
+    ///
+    /// The iovec pointers are valid while this operation is alive.
+    pub fn iovecs(&self) -> &[libc::iovec] {
+        match self {
+            Self::ReadToMemory { target, .. } | Self::WriteFromMemory { target, .. } => {
+                target.iovecs()
+            }
+            Self::ReadToVec { buffer, .. } | Self::WriteFromVec { buffer, .. } => buffer.iovecs(),
+        }
+    }
+
+    /// Returns the total number of bytes described by the operation iovecs.
+    pub fn total_len(&self) -> usize {
+        match self {
+            Self::ReadToMemory { target, .. } | Self::WriteFromMemory { target, .. } => {
+                target.total_len()
+            }
+            Self::ReadToVec { buffer, .. } | Self::WriteFromVec { buffer, .. } => {
+                buffer.total_len()
+            }
+        }
+    }
+
+    fn checked_range(total_len: usize, start: usize, len: usize) -> io::Result<Range<usize>> {
+        if start <= total_len
+            && let Some(end) = start.checked_add(len)
+            && end <= total_len
+        {
+            return Ok(start..end);
+        }
+
+        Err(io::Error::new(
+            io::ErrorKind::InvalidInput,
+            "async I/O buffer range out of bounds",
+        ))
+    }
+
+    /// Copies bytes into a read operation at `start`.
+    pub(crate) fn write_bytes_at(&mut self, start: usize, data: &[u8]) -> io::Result<()> {
+        match self {
+            Self::ReadToMemory { target, .. } => {
+                target.write_bytes_at(start, data).map_err(io::Error::other)
+            }
+            Self::ReadToVec { buffer, .. } => {
+                let range = Self::checked_range(buffer.total_len(), start, data.len())?;
+                buffer.as_mut_slice()[range].copy_from_slice(data);
+                Ok(())
+            }
+            Self::WriteFromMemory { .. } | Self::WriteFromVec { .. } => Err(io::Error::new(
+                io::ErrorKind::InvalidInput,
+                "cannot write into a write operation",
+            )),
+        }
+    }
+
+    /// Fills a read operation with zeroes at `start`.
+    pub(crate) fn fill_zeroes_at(&mut self, start: usize, len: usize) -> io::Result<()> {
+        match self {
+            Self::ReadToMemory { target, .. } => {
+                target.fill_zeroes_at(start, len).map_err(io::Error::other)
+            }
+            Self::ReadToVec { buffer, .. } => {
+                let range = Self::checked_range(buffer.total_len(), start, len)?;
+                buffer.as_mut_slice()[range].fill(0);
+                Ok(())
+            }
+            Self::WriteFromMemory { .. } | Self::WriteFromVec { .. } => Err(io::Error::new(
+                io::ErrorKind::InvalidInput,
+                "cannot write into a write operation",
+            )),
+        }
+    }
+
+    /// Copies bytes out of a write operation at `start`.
+    pub(crate) fn read_bytes_at(&self, start: usize, data: &mut [u8]) -> io::Result<()> {
+        match self {
+            Self::WriteFromMemory { target, .. } => {
+                target.read_bytes_at(start, data).map_err(io::Error::other)
+            }
+            Self::WriteFromVec { buffer, .. } => {
+                let range = Self::checked_range(buffer.total_len(), start, data.len())?;
+                data.copy_from_slice(&buffer.as_slice()[range]);
+                Ok(())
+            }
+            Self::ReadToMemory { .. } | Self::ReadToVec { .. } => Err(io::Error::new(
+                io::ErrorKind::InvalidInput,
+                "cannot read from a read operation",
+            )),
+        }
+    }
+
+    /// Consumes the operation and returns the buffer needed by its completion.
+    ///
+    /// Only `ReadToVec` operations return a buffer because callers need the
+    /// data they read.
+    pub fn into_completion_buffer(self) -> Option<OwnedIoBuffer> {
+        match self {
+            Self::ReadToVec { buffer, .. } => Some(buffer),
+            Self::ReadToMemory { .. }
+            | Self::WriteFromMemory { .. }
+            | Self::WriteFromVec { .. } => None,
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn read_op(offset: libc::off_t, len: usize) -> AsyncIoOperation {
+        AsyncIoOperation::read_to_vec(offset, OwnedIoBuffer::from_vec(vec![0u8; len]), 0)
+    }
+
+    fn write_op(offset: libc::off_t, len: usize) -> AsyncIoOperation {
+        AsyncIoOperation::write_from_vec(offset, OwnedIoBuffer::from_vec(vec![0u8; len]), 0)
+    }
+
+    #[test]
+    fn accepts_operation_exactly_filling_size() {
+        read_op(0, 512).validate_bounds(512).unwrap();
+    }
+
+    #[test]
+    fn rejects_read_straddling_size() {
+        assert!(matches!(
+            read_op(256, 512).validate_bounds(512),
+            Err(AsyncIoError::ReadVectored(_))
+        ));
+    }
+
+    #[test]
+    fn rejects_write_straddling_size() {
+        assert!(matches!(
+            write_op(256, 512).validate_bounds(512),
+            Err(AsyncIoError::WriteVectored(_))
+        ));
+    }
+
+    #[test]
+    fn rejects_offset_at_size() {
+        assert!(read_op(512, 1).validate_bounds(512).is_err());
+    }
+}

block/src/io/async_io/owned_io_buffer.rs

@@ -0,0 +1,174 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+//
+// SPDX-License-Identifier: Apache-2.0 AND BSD-3-Clause
+
+use std::alloc::{Layout, alloc_zeroed, dealloc};
+use std::{fmt, io, slice};
+
+// Storage owned by an async I/O request for host-memory buffers.
+//
+// `Vec` is used when ordinary vector storage is sufficient. `Aligned` is used
+// when the backend requires an alignment that a normal `Vec` cannot
+// guarantee.
+enum OwnedIoBufferStorage {
+    // Buffer backed by a standard `Vec<u8>`.
+    Vec(Vec<u8>),
+    // Buffer backed by an explicitly aligned allocation.
+    Aligned {
+        // Pointer returned by `alloc_zeroed` for `layout`.
+        ptr: *mut u8,
+        // Layout used to allocate and deallocate `ptr`.
+        layout: Layout,
+        // Logical buffer length exposed to I/O.
+        len: usize,
+    },
+}
+
+// SAFETY: OwnedIoBufferStorage owns its allocation exclusively. Moving it to
+// another thread transfers that ownership.
+unsafe impl Send for OwnedIoBufferStorage {}
+
+impl OwnedIoBufferStorage {
+    fn new(len: usize, alignment: usize) -> io::Result<Self> {
+        if alignment <= 1 {
+            return Ok(Self::Vec(vec![0; len]));
+        }
+
+        let alloc_len = len.max(1).next_multiple_of(alignment);
+        let layout = Layout::from_size_align(alloc_len, alignment)
+            .map_err(|e| io::Error::new(io::ErrorKind::InvalidInput, e))?;
+        // SAFETY: layout has non-zero size because alloc_len is at least 1.
+        let ptr = unsafe { alloc_zeroed(layout) };
+        if ptr.is_null() {
+            return Err(io::Error::new(
+                io::ErrorKind::OutOfMemory,
+                "alloc_zeroed returned null",
+            ));
+        }
+
+        Ok(Self::Aligned { ptr, layout, len })
+    }
+
+    fn as_mut_ptr(&mut self) -> *mut u8 {
+        match self {
+            Self::Vec(buf) => buf.as_mut_ptr(),
+            Self::Aligned { ptr, .. } => *ptr,
+        }
+    }
+
+    fn as_slice(&self) -> &[u8] {
+        match self {
+            Self::Vec(buf) => buf.as_slice(),
+            Self::Aligned { ptr, len, .. } => {
+                // SAFETY: alloc_zeroed initialized `len` bytes at `ptr` and the
+                // allocation is owned by Self.
+                unsafe { slice::from_raw_parts(*ptr, *len) }
+            }
+        }
+    }
+
+    fn as_mut_slice(&mut self) -> &mut [u8] {
+        match self {
+            Self::Vec(buf) => buf.as_mut_slice(),
+            Self::Aligned { ptr, len, .. } => {
+                // SAFETY: alloc_zeroed initialized `len` bytes at `ptr`,
+                // &mut self ensures unique access, and the allocation is
+                // owned by Self.
+                unsafe { slice::from_raw_parts_mut(*ptr, *len) }
+            }
+        }
+    }
+}
+
+impl Drop for OwnedIoBufferStorage {
+    fn drop(&mut self) {
+        if let Self::Aligned { ptr, layout, .. } = self {
+            // SAFETY: ptr was allocated by alloc_zeroed with this layout and is
+            // solely owned by Self.
+            unsafe { dealloc(*ptr, *layout) };
+        }
+    }
+}
+
+impl fmt::Debug for OwnedIoBufferStorage {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            Self::Vec(buf) => f.debug_tuple("Vec").field(&buf.len()).finish(),
+            Self::Aligned { len, layout, .. } => f
+                .debug_struct("Aligned")
+                .field("len", len)
+                .field("layout", layout)
+                .finish(),
+        }
+    }
+}
+
+/// Owns host-memory buffer storage and the iovec array that points into it.
+///
+/// The retained iovec is valid for as long as this value is alive.
+/// When used for Async I/O this struct must remain valid for the duration of the op.
+#[derive(Debug)]
+pub struct OwnedIoBuffer {
+    storage: OwnedIoBufferStorage,
+    iovecs: Vec<libc::iovec>,
+}
+
+// SAFETY: OwnedIoBuffer owns the storage referenced by its single iovec, and moving the buffer
+// keeps the allocation address stable.
+unsafe impl Send for OwnedIoBuffer {}
+
+impl OwnedIoBuffer {
+    /// Creates a zeroed buffer with the requested logical length and alignment.
+    ///
+    /// An alignment of 0 or 1 uses ordinary `Vec` storage. Larger alignments use an explicitly
+    /// aligned allocation whose allocated size may be rounded up while the exposed slice length
+    /// remains `len`.
+    pub fn new(len: usize, alignment: usize) -> io::Result<Self> {
+        let mut storage = OwnedIoBufferStorage::new(len, alignment)?;
+        let iovec = libc::iovec {
+            iov_base: storage.as_mut_ptr().cast(),
+            iov_len: len,
+        };
+        Ok(Self {
+            storage,
+            iovecs: vec![iovec],
+        })
+    }
+
+    /// Creates an owned I/O buffer from an existing `Vec<u8>`.
+    ///
+    /// The generated iovec covers the full vector length and remains valid
+    /// until the OwnedIoBuffer is dropped.
+    pub fn from_vec(mut buf: Vec<u8>) -> Self {
+        let iovec = libc::iovec {
+            iov_base: buf.as_mut_ptr().cast(),
+            iov_len: buf.len(),
+        };
+        Self {
+            storage: OwnedIoBufferStorage::Vec(buf),
+            iovecs: vec![iovec],
+        }
+    }
+
+    /// Returns the logical buffer contents.
+    pub fn as_slice(&self) -> &[u8] {
+        self.storage.as_slice()
+    }
+
+    /// Returns the logical buffer contents mutably.
+    pub fn as_mut_slice(&mut self) -> &mut [u8] {
+        self.storage.as_mut_slice()
+    }
+
+    /// Returns the retained iovec array for kernel submission.
+    ///
+    /// The iovec pointers remain valid while this buffer is alive.
+    pub fn iovecs(&self) -> &[libc::iovec] {
+        &self.iovecs
+    }
+
+    /// Returns the total number of bytes described by the retained iovecs.
+    pub fn total_len(&self) -> usize {
+        self.iovecs.iter().map(|iov| iov.iov_len).sum()
+    }
+}

block/src/io/async_io/uring_data_io.rs

@@ -0,0 +1,426 @@
+// Copyright © 2021 Intel Corporation
+//
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+//
+// SPDX-License-Identifier: Apache-2.0 AND BSD-3-Clause
+
+use std::collections::{HashMap, VecDeque};
+use std::os::fd::{AsRawFd, RawFd};
+use std::{io, mem};
+
+use io_uring::{IoUring, opcode, squeue, types};
+use log::{error, warn};
+use vmm_sys_util::eventfd::EventFd;
+
+use super::common::{duplicate_user_data_error, validate_batch};
+use super::{AsyncIoCompletion, AsyncIoOperation};
+
+/// `io_uring` wrapper for async I/O.
+///
+/// Holds the `IoUring` and its `EventFd`. Tracks ops that are pending.
+pub struct UringDataIo {
+    io_uring: IoUring,
+    // The `EventFd` for completion signals.
+    eventfd: EventFd,
+    // `in_flight` tracks every user_data value accepted by the kernel. Owned
+    // data operations store `Some(op)` so their iovecs and backing buffers
+    // remain valid until completion; metadata operations store `None`.
+    in_flight: HashMap<u64, Option<AsyncIoOperation>>,
+    // `injected` holds locally produced completions so synchronous failures
+    // and short-circuited requests use the same drain path as kernel CQEs.
+    injected: VecDeque<AsyncIoCompletion>,
+    // `needs_submit_retry` is set when SQEs have been published to the ring,
+    // but the submit syscall failed before confirming kernel ownership.
+    needs_submit_retry: bool,
+}
+
+impl UringDataIo {
+    /// Creates an io_uring queue and registers its completion eventfd.
+    pub fn new(ring_depth: u32) -> io::Result<Self> {
+        let io_uring = IoUring::new(ring_depth)?;
+        let eventfd = EventFd::new(libc::EFD_NONBLOCK)?;
+        io_uring.submitter().register_eventfd(eventfd.as_raw_fd())?;
+
+        Ok(Self {
+            io_uring,
+            eventfd,
+            in_flight: HashMap::new(),
+            injected: VecDeque::new(),
+            needs_submit_retry: false,
+        })
+    }
+
+    /// Returns the eventfd signaled when completions are available.
+    pub fn notifier(&self) -> &EventFd {
+        &self.eventfd
+    }
+
+    /// Submits one owned read or write operation to the queue.
+    pub fn submit_operation(&mut self, fd: RawFd, op: AsyncIoOperation) -> io::Result<()> {
+        self.submit_batch(fd, vec![op])
+    }
+
+    fn reserve_user_data(&mut self, user_data: u64) -> io::Result<()> {
+        if self.in_flight.contains_key(&user_data) {
+            return Err(duplicate_user_data_error(user_data));
+        }
+        self.in_flight.insert(user_data, None);
+
+        Ok(())
+    }
+
+    fn submit_kernel_entry(&mut self, user_data: u64, entry: &squeue::Entry) -> io::Result<()> {
+        self.reserve_user_data(user_data)?;
+
+        let (submitter, mut sq, _) = self.io_uring.split();
+        // SAFETY: the entry has no caller-owned buffer. `user_data` is retained
+        // in `in_flight` until the CQE is consumed.
+        if let Err(e) = unsafe { sq.push(entry) } {
+            self.in_flight.remove(&user_data);
+            return Err(io::Error::other(format!("Submission queue is full: {e:?}")));
+        }
+        sq.sync();
+
+        match submitter.submit() {
+            Ok(_) => self.needs_submit_retry = false,
+            Err(e) => {
+                self.needs_submit_retry = true;
+                warn!("io_uring submit failed after SQE was published: {e}");
+                self.eventfd.write(1).unwrap();
+            }
+        }
+
+        Ok(())
+    }
+
+    /// Submits a batch of owned read and write operations.
+    ///
+    /// If the io_uring submission queue cannot accept the whole batch, each
+    /// operation is completed locally with `-EAGAIN` so callers can observe
+    /// every request through the normal completion path.
+    pub fn submit_batch(&mut self, fd: RawFd, batch: Vec<AsyncIoOperation>) -> io::Result<()> {
+        if batch.is_empty() {
+            return Ok(());
+        }
+
+        validate_batch(|user_data| self.in_flight.contains_key(&user_data), &batch)?;
+
+        let (submitter, mut sq, _) = self.io_uring.split();
+        let available = sq.capacity() - sq.len();
+        if batch.len() > available {
+            // Not enough space for the batch.
+            // Drop sq, which will re-publish an unmodified tail pointer
+            drop(sq);
+            for op in batch {
+                self.injected
+                    .push_back(AsyncIoCompletion::from_operation(op, -libc::EAGAIN));
+            }
+            self.eventfd.write(1).unwrap();
+            return Ok(());
+        }
+
+        let mut signal_completion = false;
+        let mut batch = batch.into_iter();
+        while let Some(op) = batch.next() {
+            let user_data = op.user_data();
+            let entry = Self::build_entry(fd, &op);
+            self.in_flight.insert(user_data, Some(op));
+
+            // SAFETY: the SQ capacity was just checked. Every iovec's pointer is retained in
+            // self.in_flight before the SQ tail is advanced by sync or drop. in_flight only
+            // drops the memory after a completion.
+            if let Err(e) = unsafe { sq.push(&entry) } {
+                Self::handle_push_failure(
+                    &mut self.in_flight,
+                    &mut self.injected,
+                    user_data,
+                    batch.by_ref(),
+                    &e,
+                );
+                signal_completion = true;
+                break;
+            }
+        }
+
+        sq.sync();
+        match submitter.submit() {
+            Ok(_) => self.needs_submit_retry = false,
+            Err(e) => {
+                self.needs_submit_retry = true;
+                warn!("io_uring submit failed after SQEs were published: {e}");
+                signal_completion = true;
+            }
+        }
+        if signal_completion {
+            self.eventfd.write(1).unwrap();
+        }
+
+        Ok(())
+    }
+
+    #[cold]
+    fn handle_push_failure(
+        in_flight: &mut HashMap<u64, Option<AsyncIoOperation>>,
+        injected: &mut VecDeque<AsyncIoCompletion>,
+        user_data: u64,
+        remaining: impl Iterator<Item = AsyncIoOperation>,
+        error: &squeue::PushError,
+    ) {
+        // Since capacity was just checked, this should only happen if the ring
+        // state changed unexpectedly. Keep all affected operations memory safe
+        // by returning local completions through the normal path.
+        let op = in_flight
+            .remove(&user_data)
+            .flatten()
+            .expect("pending operation missing after failed push");
+        injected.push_back(AsyncIoCompletion::from_operation(op, -libc::EAGAIN));
+        for op in remaining {
+            injected.push_back(AsyncIoCompletion::from_operation(op, -libc::EAGAIN));
+        }
+        warn!("io_uring submission queue became full after capacity check: {error:?}");
+    }
+
+    fn build_entry(fd: RawFd, op: &AsyncIoOperation) -> squeue::Entry {
+        let iovecs = op.iovecs();
+        let fd = types::Fd(fd);
+        if op.is_read() {
+            opcode::Readv::new(fd, iovecs.as_ptr(), iovecs.len() as u32)
+                .offset(op.offset() as u64)
+                .build()
+                .user_data(op.user_data())
+        } else {
+            opcode::Writev::new(fd, iovecs.as_ptr(), iovecs.len() as u32)
+                .offset(op.offset() as u64)
+                .build()
+                .user_data(op.user_data())
+        }
+    }
+
+    /// Submits an io_uring NOP carrying `user_data`.
+    pub fn submit_nop(&mut self, user_data: u64) -> io::Result<()> {
+        self.submit_kernel_entry(user_data, &opcode::Nop::new().build().user_data(user_data))
+    }
+
+    /// Submits an fsync operation carrying `user_data`.
+    pub fn submit_fsync(&mut self, fd: RawFd, user_data: u64) -> io::Result<()> {
+        self.submit_kernel_entry(
+            user_data,
+            &opcode::Fsync::new(types::Fd(fd))
+                .build()
+                .user_data(user_data),
+        )
+    }
+
+    /// Submits a fallocate operation carrying `user_data`.
+    pub fn submit_fallocate(
+        &mut self,
+        fd: RawFd,
+        offset: u64,
+        length: u64,
+        mode: i32,
+        user_data: u64,
+    ) -> io::Result<()> {
+        self.submit_kernel_entry(
+            user_data,
+            &opcode::Fallocate::new(types::Fd(fd), length)
+                .offset(offset)
+                .mode(mode)
+                .build()
+                .user_data(user_data),
+        )
+    }
+
+    /// Injects a completion that did not come from a kernel CQE.
+    ///
+    /// The notifier is signaled so callers can drain it with
+    /// [`Self::next_completion`].
+    pub fn inject_completion(&mut self, completion: AsyncIoCompletion) {
+        self.injected.push_back(completion);
+        self.eventfd.write(1).unwrap();
+    }
+
+    /// Returns the next kernel or injected completion if one is available.
+    ///
+    /// Consuming a kernel completion returns ownership of any buffer retained
+    /// by the corresponding operation.
+    pub fn next_completion(&mut self) -> Option<AsyncIoCompletion> {
+        if self.needs_submit_retry {
+            match self.io_uring.submitter().submit() {
+                Ok(_) => self.needs_submit_retry = false,
+                Err(e) => warn!("io_uring retry submit failed for retained SQEs: {e}"),
+            }
+        }
+
+        if let Some(entry) = self.io_uring.completion().next() {
+            let user_data = entry.user_data();
+            return Some(AsyncIoCompletion::new(
+                user_data,
+                entry.result(),
+                self.in_flight
+                    .remove(&user_data)
+                    .flatten()
+                    .and_then(AsyncIoOperation::into_completion_buffer),
+            ));
+        }
+
+        self.injected.pop_front()
+    }
+}
+
+impl Drop for UringDataIo {
+    fn drop(&mut self) {
+        // Closing the ring fd does not cancel io_uring ops that have started.
+        // Wait for CQEs before releasing retained iovecs.
+        if self.needs_submit_retry {
+            if let Err(e) = self.io_uring.submitter().submit() {
+                warn!("io_uring drain submit failed for retained SQEs: {e}");
+            }
+            self.needs_submit_retry = false;
+        }
+
+        let max_drain_iterations = self.in_flight.len().saturating_mul(2);
+        let mut drain_iterations = 0;
+        while !self.in_flight.is_empty() {
+            if drain_iterations == max_drain_iterations {
+                error!(
+                    "io_uring drain abandoned with {} operations still in flight after {} drain iterations",
+                    self.in_flight.len(),
+                    drain_iterations
+                );
+                // Keep retained buffers mapped if the ring cannot be drained.
+                mem::forget(mem::take(&mut self.in_flight));
+                break;
+            }
+            drain_iterations += 1;
+
+            if let Some(entry) = self.io_uring.completion().next() {
+                self.in_flight.remove(&entry.user_data());
+                continue;
+            }
+
+            // No completion ready: block in the kernel until at least one is.
+            if let Err(e) = self.io_uring.submitter().submit_and_wait(1) {
+                if e.kind() == io::ErrorKind::Interrupted {
+                    continue;
+                }
+                error!(
+                    "io_uring drain abandoned with {} operations still in flight: {e}",
+                    self.in_flight.len()
+                );
+                // Keep retained buffers mapped if the ring cannot be drained.
+                mem::forget(mem::take(&mut self.in_flight));
+                break;
+            }
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::io;
+    use std::os::fd::AsRawFd;
+    use std::thread::sleep;
+    use std::time::Duration;
+
+    use vmm_sys_util::tempfile::TempFile;
+
+    use super::UringDataIo;
+    use crate::async_io::{AsyncIoCompletion, AsyncIoOperation, OwnedIoBuffer};
+
+    fn wait_for_completion(data_io: &mut UringDataIo) -> AsyncIoCompletion {
+        for _ in 0..1000 {
+            if let Some(completion) = data_io.next_completion() {
+                return completion;
+            }
+            sleep(Duration::from_millis(1));
+        }
+
+        panic!("timed out waiting for io_uring completion");
+    }
+
+    #[test]
+    fn uring_rejects_duplicate_user_data_for_metadata_ops() {
+        let file = TempFile::new().unwrap().into_file();
+        file.set_len(512).unwrap();
+        let fd = file.as_raw_fd();
+        let mut data_io = UringDataIo::new(8).unwrap();
+
+        data_io
+            .submit_operation(
+                fd,
+                AsyncIoOperation::read_to_vec(0, OwnedIoBuffer::from_vec(vec![0; 512]), 7),
+            )
+            .unwrap();
+
+        assert_eq!(
+            data_io.submit_fsync(fd, 7).unwrap_err().kind(),
+            io::ErrorKind::AlreadyExists
+        );
+        assert_eq!(
+            data_io.submit_nop(7).unwrap_err().kind(),
+            io::ErrorKind::AlreadyExists
+        );
+        assert_eq!(
+            data_io
+                .submit_fallocate(fd, 0, 512, 0, 7)
+                .unwrap_err()
+                .kind(),
+            io::ErrorKind::AlreadyExists
+        );
+
+        let completion = wait_for_completion(&mut data_io);
+        assert_eq!(completion.user_data, 7);
+        assert_eq!(completion.result, 512);
+    }
+
+    #[test]
+    fn uring_drop_drains_in_flight_operations() {
+        let file = TempFile::new().unwrap().into_file();
+        file.set_len(8192).unwrap();
+        let fd = file.as_raw_fd();
+        let mut data_io = UringDataIo::new(8).unwrap();
+
+        for user_data in 0..4 {
+            data_io
+                .submit_operation(
+                    fd,
+                    AsyncIoOperation::read_to_vec(
+                        0,
+                        OwnedIoBuffer::from_vec(vec![0; 512]),
+                        user_data,
+                    ),
+                )
+                .unwrap();
+        }
+
+        drop(data_io);
+    }
+
+    #[test]
+    fn uring_queue_full_batch_completes_each_operation() {
+        let file = TempFile::new().unwrap().into_file();
+        let fd = file.as_raw_fd();
+        let mut data_io = UringDataIo::new(1).unwrap();
+        let available = {
+            let (_, sq, _) = data_io.io_uring.split();
+            sq.capacity() - sq.len()
+        };
+        let batch_len = available + 1;
+        let batch: Vec<_> = (0..batch_len as u64)
+            .map(|user_data| {
+                AsyncIoOperation::read_to_vec(0, OwnedIoBuffer::from_vec(vec![0; 512]), user_data)
+            })
+            .collect();
+
+        data_io.submit_batch(fd, batch).unwrap();
+
+        let mut completed = Vec::new();
+        while let Some(completion) = data_io.next_completion() {
+            assert_eq!(completion.result, -libc::EAGAIN);
+            assert!(completion.buffer.is_some());
+            completed.push(completion.user_data);
+        }
+        completed.sort_unstable();
+        assert_eq!(completed, (0..batch_len as u64).collect::<Vec<_>>());
+    }
+}

block/src/io/fcntl.rs

@@ -34,7 +34,7 @@ pub enum LockError {
 }
 
 /// Commands for use with [`fcntl`].
-#[allow(non_camel_case_types)]
+#[expect(non_camel_case_types)]
 enum FcntlArg<'a> {
     /// Set an OFD lock from the given lock description.
     F_OFD_SETLK(&'a libc::flock),
@@ -201,20 +201,23 @@ pub fn try_acquire_lock<Fd: AsRawFd>(
 ) -> Result<(), LockError> {
     let flock = get_flock(lock_type, granularity);
 
-    let res = fcntl(file.as_raw_fd(), FcntlArg::F_OFD_SETLK(&flock));
-    match res {
-        0 => Ok(()),
-        -1 => {
-            let io_error = io::Error::last_os_error();
-            let errno = io_error.raw_os_error().unwrap();
-            match errno {
-                // See man page for error code:
-                // <https://man7.org/linux/man-pages/man2/fcntl.2.html>
-                libc::EAGAIN | libc::EACCES => Err(LockError::AlreadyLocked),
-                _ => Err(LockError::Io(io_error)),
+    loop {
+        let res = fcntl(file.as_raw_fd(), FcntlArg::F_OFD_SETLK(&flock));
+        match res {
+            0 => return Ok(()),
+            -1 => {
+                let io_error = io::Error::last_os_error();
+                let errno = io_error.raw_os_error().unwrap();
+                match errno {
+                    // See man page for error code:
+                    // <https://man7.org/linux/man-pages/man2/fcntl.2.html>
+                    libc::EAGAIN | libc::EACCES => return Err(LockError::AlreadyLocked),
+                    libc::EINTR => continue,
+                    _ => return Err(LockError::Io(io_error)),
+                }
             }
+            val => panic!("Unexpected return value from fcntl(): {val}"),
         }
-        val => panic!("Unexpected return value from fcntl(): {val}"),
     }
 }
 

block/src/io/mod.rs

@@ -0,0 +1,12 @@
+// Copyright 2026 The Cloud Hypervisor Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+//! Shared I/O infrastructure for all disk format backends.
+//!
+//! Contains the async I/O trait, request handling, and file locking
+//! helpers.
+
+pub mod async_io;
+pub mod fcntl;
+pub mod request;

block/src/io/request.rs

@@ -6,10 +6,13 @@
 //
 // Copyright © 2020 Intel Corporation
 //
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+//
 // SPDX-License-Identifier: Apache-2.0 AND BSD-3-Clause
 
-use std::io::{Read, Seek, SeekFrom, Write};
 use std::mem;
+use std::os::unix::fs::FileExt;
+use std::sync::Arc;
 use std::time::Instant;
 
 use log::{error, warn};
@@ -24,10 +27,13 @@ use vm_memory::{
     Address as _, Bytes as _, GuestAddress, GuestMemory as _, GuestMemoryError,
     GuestMemoryLoadGuard,
 };
-use vm_virtio::{AccessPlatform, Translatable as _};
+use vm_virtio::AccessPlatform;
+use vm_virtio::checked_descriptor::DescriptorChainExt;
+use vmm_sys_util::file_traits::FileSync;
 
-use crate::aligned_operation::AlignedOperation;
-use crate::async_io::AsyncIo;
+use crate::async_io::{
+    AsyncIo, AsyncIoCompletion, AsyncIoOperation, GuestMemoryTarget, OwnedIoBuffer,
+};
 use crate::{Error, ExecuteError, request_type, sector};
 
 const SECTOR_SHIFT: u8 = 9;
@@ -36,7 +42,7 @@ pub const SECTOR_SIZE: u64 = 0x01 << SECTOR_SHIFT;
 /// Maximum number of segments per DISCARD or WRITE_ZEROES request.
 pub const MAX_DISCARD_WRITE_ZEROES_SEG: u32 = 1;
 /// Size and field offsets within `struct virtio_blk_discard_write_zeroes`.
-const DISCARD_WZ_SEG_SIZE: u32 = mem::size_of::<virtio_blk_discard_write_zeroes>() as u32;
+const DISCARD_WZ_SEG_SIZE: u32 = size_of::<virtio_blk_discard_write_zeroes>() as u32;
 const DISCARD_WZ_MAX_PAYLOAD: u32 = DISCARD_WZ_SEG_SIZE * MAX_DISCARD_WRITE_ZEROES_SEG;
 const DISCARD_WZ_SECTOR_OFFSET: u64 =
     mem::offset_of!(virtio_blk_discard_write_zeroes, sector) as u64;
@@ -56,18 +62,12 @@ pub enum RequestType {
 }
 
 pub const DEFAULT_DESCRIPTOR_VEC_SIZE: usize = 32;
-pub struct BatchRequest {
-    pub offset: libc::off_t,
-    pub iovecs: SmallVec<[libc::iovec; DEFAULT_DESCRIPTOR_VEC_SIZE]>,
-    pub user_data: u64,
-    pub request_type: RequestType,
-}
 
 pub struct ExecuteAsync {
     // `true` if the execution will complete asynchronously
     pub async_complete: bool,
     // request need to be batched for submission if any
-    pub batch_request: Option<BatchRequest>,
+    pub batch_request: Option<AsyncIoOperation>,
 }
 
 #[derive(Debug)]
@@ -77,7 +77,6 @@ pub struct Request {
     data_descriptors: SmallVec<[(GuestAddress, u32); DEFAULT_DESCRIPTOR_VEC_SIZE]>,
     status_addr: GuestAddress,
     pub writeback: bool,
-    aligned_operations: SmallVec<[AlignedOperation; DEFAULT_DESCRIPTOR_VEC_SIZE]>,
     start: Instant,
 }
 
@@ -87,10 +86,11 @@ impl Request {
         access_platform: Option<&dyn AccessPlatform>,
     ) -> Result<Request, Error> {
         let hdr_desc = desc_chain
-            .next()
-            .ok_or(Error::DescriptorChainTooShort)
-            .inspect_err(|_| {
+            .next_checked(access_platform)
+            .map_err(|addr| Error::GuestMemory(GuestMemoryError::InvalidGuestAddress(addr)))?
+            .ok_or_else(|| {
                 error!("Missing head descriptor");
+                Error::DescriptorChainTooShort
             })?;
 
         // The head contains the request type which MUST be readable.
@@ -98,10 +98,7 @@ impl Request {
             return Err(Error::UnexpectedWriteOnlyDescriptor);
         }
 
-        let hdr_desc_addr = hdr_desc
-            .addr()
-            .translate_gva(access_platform, hdr_desc.len() as usize)
-            .map_err(|e| Error::GuestMemory(GuestMemoryError::IOError(e)))?;
+        let hdr_desc_addr = hdr_desc.addr();
 
         let mut req = Request {
             request_type: request_type(desc_chain.memory(), hdr_desc_addr)?,
@@ -109,16 +106,16 @@ impl Request {
             data_descriptors: SmallVec::with_capacity(DEFAULT_DESCRIPTOR_VEC_SIZE),
             status_addr: GuestAddress(0),
             writeback: true,
-            aligned_operations: SmallVec::with_capacity(DEFAULT_DESCRIPTOR_VEC_SIZE),
             start: Instant::now(),
         };
 
         let status_desc;
         let mut desc = desc_chain
-            .next()
-            .ok_or(Error::DescriptorChainTooShort)
-            .inspect_err(|_| {
+            .next_checked(access_platform)
+            .map_err(|addr| Error::GuestMemory(GuestMemoryError::InvalidGuestAddress(addr)))?
+            .ok_or_else(|| {
                 error!("Only head descriptor present: request = {req:?}");
+                Error::DescriptorChainTooShort
             })?;
 
         if desc.has_next() {
@@ -140,17 +137,15 @@ impl Request {
                     return Err(Error::UnexpectedReadOnlyDescriptor);
                 }
 
-                req.data_descriptors.push((
-                    desc.addr()
-                        .translate_gva(access_platform, desc.len() as usize)
-                        .map_err(|e| Error::GuestMemory(GuestMemoryError::IOError(e)))?,
-                    desc.len(),
-                ));
+                req.data_descriptors.push((desc.addr(), desc.len()));
                 desc = desc_chain
-                    .next()
-                    .ok_or(Error::DescriptorChainTooShort)
-                    .inspect_err(|_| {
+                    .next_checked(access_platform)
+                    .map_err(|addr| {
+                        Error::GuestMemory(GuestMemoryError::InvalidGuestAddress(addr))
+                    })?
+                    .ok_or_else(|| {
                         error!("DescriptorChain corrupted: request = {req:?}");
+                        Error::DescriptorChainTooShort
                     })?;
             }
             status_desc = desc;
@@ -168,19 +163,16 @@ impl Request {
             return Err(Error::UnexpectedReadOnlyDescriptor);
         }
 
-        if status_desc.len() < 1 {
+        if status_desc.is_empty() {
             return Err(Error::DescriptorLengthTooSmall);
         }
 
-        req.status_addr = status_desc
-            .addr()
-            .translate_gva(access_platform, status_desc.len() as usize)
-            .map_err(|e| Error::GuestMemory(GuestMemoryError::IOError(e)))?;
+        req.status_addr = status_desc.addr();
 
         Ok(req)
     }
 
-    pub fn execute<T: Seek + Read + Write, B: Bitmap + 'static>(
+    pub fn execute<T: FileExt + FileSync, B: Bitmap + 'static>(
         &self,
         disk: &mut T,
         disk_nsectors: u64,
@@ -189,32 +181,35 @@ impl Request {
     ) -> Result<u32, ExecuteError> {
         self.check_data_bounds(disk_nsectors)?;
 
-        disk.seek(SeekFrom::Start(self.sector << SECTOR_SHIFT))
-            .map_err(ExecuteError::Seek)?;
+        let mut offset = self.sector << SECTOR_SHIFT;
         let mut len = 0;
         for (data_addr, data_len) in &self.data_descriptors {
             match self.request_type {
                 RequestType::In => {
                     let mut buf = vec![0u8; *data_len as usize];
-                    disk.read_exact(&mut buf).map_err(ExecuteError::ReadExact)?;
+                    disk.read_exact_at(&mut buf, offset)
+                        .map_err(ExecuteError::ReadExact)?;
                     mem.read_exact_volatile_from(
                         *data_addr,
                         &mut buf.as_slice(),
                         *data_len as usize,
                     )
                     .map_err(ExecuteError::Read)?;
+                    offset += u64::from(*data_len);
                     len += data_len;
                 }
                 RequestType::Out => {
                     let mut buf: Vec<u8> = Vec::new();
                     mem.write_all_volatile_to(*data_addr, &mut buf, *data_len as usize)
                         .map_err(ExecuteError::Write)?;
-                    disk.write_all(&buf).map_err(ExecuteError::WriteAll)?;
+                    disk.write_all_at(&buf, offset)
+                        .map_err(ExecuteError::WriteAll)?;
                     if !self.writeback {
-                        disk.flush().map_err(ExecuteError::Flush)?;
+                        disk.fsync().map_err(ExecuteError::Flush)?;
                     }
+                    offset += u64::from(*data_len);
                 }
-                RequestType::Flush => disk.flush().map_err(ExecuteError::Flush)?,
+                RequestType::Flush => disk.fsync().map_err(ExecuteError::Flush)?,
                 RequestType::GetDeviceId => {
                     if (*data_len as usize) < serial.len() {
                         return Err(ExecuteError::BadRequest(Error::InvalidOffset));
@@ -234,9 +229,9 @@ impl Request {
         Ok(len)
     }
 
-    pub fn execute_async<B: Bitmap + 'static>(
+    pub fn execute_async<B: Bitmap + Send + Sync + 'static>(
         &mut self,
-        mem: &vm_memory::GuestMemoryMmap<B>,
+        mem: Arc<vm_memory::GuestMemoryMmap<B>>,
         disk_nsectors: u64,
         disk_image: &mut dyn AsyncIo,
         serial: &[u8],
@@ -250,56 +245,6 @@ impl Request {
 
         self.check_data_bounds(disk_nsectors)?;
 
-        let mut iovecs: SmallVec<[libc::iovec; DEFAULT_DESCRIPTOR_VEC_SIZE]> =
-            SmallVec::with_capacity(self.data_descriptors.len());
-        for &(data_addr, data_len) in &self.data_descriptors {
-            let _: u32 = data_len; // compiler-checked documentation
-            const _: () = assert!(
-                core::mem::size_of::<u32>() <= core::mem::size_of::<usize>(),
-                "unsupported platform"
-            );
-            if data_len == 0 {
-                continue;
-            }
-            let data_len = data_len as usize;
-
-            let origin_ptr = mem
-                .get_slice(data_addr, data_len)
-                .map_err(ExecuteError::GetHostAddress)?;
-            assert!(origin_ptr.len() >= data_len);
-            let origin_ptr = origin_ptr.ptr_guard_mut();
-
-            // O_DIRECT requires buffer addresses to be aligned to the
-            // backend device's logical block size. In case it's not properly
-            // aligned, an intermediate buffer is created with the correct
-            // alignment, and a copy from/to the origin buffer is performed,
-            // depending on the type of operation.
-            let iov_base = if (origin_ptr.as_ptr() as u64).is_multiple_of(alignment) {
-                origin_ptr.as_ptr().cast()
-            } else {
-                let mut aligned_op = AlignedOperation::new(data_addr, data_len, alignment as usize)
-                    .map_err(ExecuteError::TemporaryBufferAllocation)?;
-
-                // We need to perform the copy beforehand in case we're writing
-                // data out.
-                if request_type == RequestType::Out {
-                    mem.read_slice(aligned_op.as_bytes_mut(), data_addr)
-                        .map_err(ExecuteError::Read)?;
-                }
-
-                let aligned_ptr = aligned_op.as_mut_ptr();
-                self.aligned_operations.push(aligned_op);
-
-                aligned_ptr.cast()
-            };
-
-            let iovec = libc::iovec {
-                iov_base,
-                iov_len: data_len as libc::size_t,
-            };
-            iovecs.push(iovec);
-        }
-
         let mut ret = ExecuteAsync {
             async_complete: true,
             batch_request: None,
@@ -307,37 +252,52 @@ impl Request {
         // Queue operations expected to be submitted.
         match request_type {
             RequestType::In => {
-                for (data_addr, data_len) in &self.data_descriptors {
-                    mem.get_slice(*data_addr, *data_len as usize)
-                        .map_err(ExecuteError::GetHostAddress)?
-                        .bitmap()
-                        .mark_dirty(0, *data_len as usize);
-                }
+                self.mark_read_dirty(&mem)?;
+                let op = self.build_data_operation(mem, offset, alignment, user_data)?;
                 if disk_image.batch_requests_enabled() {
-                    ret.batch_request = Some(BatchRequest {
-                        offset,
-                        iovecs,
-                        user_data,
-                        request_type,
-                    });
+                    ret.batch_request = Some(op);
                 } else {
-                    disk_image
-                        .read_vectored(offset, &iovecs, user_data)
-                        .map_err(ExecuteError::AsyncRead)?;
+                    match op {
+                        AsyncIoOperation::ReadToMemory {
+                            offset,
+                            target,
+                            user_data,
+                        } => disk_image
+                            .read_to_memory(offset, target, user_data)
+                            .map_err(ExecuteError::AsyncRead)?,
+                        AsyncIoOperation::ReadToVec {
+                            offset,
+                            buffer,
+                            user_data,
+                        } => disk_image
+                            .read_to_vec(offset, buffer, user_data)
+                            .map_err(ExecuteError::AsyncRead)?,
+                        _ => unreachable!("unexpected read operation"),
+                    }
                 }
             }
             RequestType::Out => {
+                let op = self.build_data_operation(mem, offset, alignment, user_data)?;
                 if disk_image.batch_requests_enabled() {
-                    ret.batch_request = Some(BatchRequest {
-                        offset,
-                        iovecs,
-                        user_data,
-                        request_type,
-                    });
+                    ret.batch_request = Some(op);
                 } else {
-                    disk_image
-                        .write_vectored(offset, &iovecs, user_data)
-                        .map_err(ExecuteError::AsyncWrite)?;
+                    match op {
+                        AsyncIoOperation::WriteFromMemory {
+                            offset,
+                            target,
+                            user_data,
+                        } => disk_image
+                            .write_from_memory(offset, target, user_data)
+                            .map_err(ExecuteError::AsyncWrite)?,
+                        AsyncIoOperation::WriteFromVec {
+                            offset,
+                            buffer,
+                            user_data,
+                        } => disk_image
+                            .write_from_vec(offset, buffer, user_data)
+                            .map_err(ExecuteError::AsyncWrite)?,
+                        _ => unreachable!("unexpected write operation"),
+                    }
                 }
             }
             RequestType::Flush => {
@@ -472,11 +432,6 @@ impl Request {
                     });
                 }
 
-                let wz_offset = wz_sector * SECTOR_SIZE;
-                if wz_offset == 0 && disable_sector0_writes {
-                    return Err(ExecuteError::BadRequest(Error::InvalidOffset));
-                }
-
                 let top = wz_sector
                     .checked_add(wz_num_sectors as u64)
                     .ok_or(ExecuteError::BadRequest(Error::InvalidOffset))?;
@@ -484,6 +439,11 @@ impl Request {
                     return Err(ExecuteError::BadRequest(Error::InvalidOffset));
                 }
 
+                let wz_offset = wz_sector * SECTOR_SIZE;
+                if wz_offset == 0 && disable_sector0_writes {
+                    return Err(ExecuteError::BadRequest(Error::InvalidOffset));
+                }
+
                 let wz_length = (wz_num_sectors as u64) * SECTOR_SIZE;
 
                 if wz_flags & VIRTIO_BLK_WRITE_ZEROES_FLAG_UNMAP != 0 {
@@ -502,17 +462,141 @@ impl Request {
         Ok(ret)
     }
 
+    // Builds a read or write operation for IO to or from `mem`.
+    fn build_data_operation<B: Bitmap + Send + Sync + 'static>(
+        &self,
+        mem: Arc<vm_memory::GuestMemoryMmap<B>>,
+        offset: libc::off_t,
+        alignment: u64,
+        user_data: u64,
+    ) -> Result<AsyncIoOperation, ExecuteError> {
+        if self.guest_memory_is_aligned(&mem, alignment)? {
+            let target = GuestMemoryTarget::new(mem, &self.data_descriptors)
+                .map_err(ExecuteError::GetHostAddress)?;
+            return Ok(match self.request_type {
+                RequestType::In => AsyncIoOperation::read_to_memory(offset, target, user_data),
+                RequestType::Out => AsyncIoOperation::write_from_memory(offset, target, user_data),
+                _ => unreachable!("unexpected data operation type"),
+            });
+        }
+
+        // The guest-memory buffers are unaligned, so use an aligned bounce buffer.
+        let mut buffer = OwnedIoBuffer::new(self.data_len(), alignment as usize)
+            .map_err(ExecuteError::TemporaryBufferAllocation)?;
+
+        if self.request_type == RequestType::Out {
+            self.copy_guest_to_buffer(&mem, buffer.as_mut_slice())?;
+        }
+
+        Ok(match self.request_type {
+            RequestType::In => AsyncIoOperation::read_to_vec(offset, buffer, user_data),
+            RequestType::Out => AsyncIoOperation::write_from_vec(offset, buffer, user_data),
+            _ => unreachable!("unexpected data operation type"),
+        })
+    }
+
+    // Checks whether `self.data_descriptors` are aligned to `alignment`.
+    fn guest_memory_is_aligned<B: Bitmap + 'static>(
+        &self,
+        mem: &vm_memory::GuestMemoryMmap<B>,
+        alignment: u64,
+    ) -> Result<bool, ExecuteError> {
+        if alignment <= 1 {
+            return Ok(true);
+        }
+
+        for &(data_addr, data_len) in &self.data_descriptors {
+            let _: u32 = data_len;
+            const _: () = assert!(
+                size_of::<u32>() <= size_of::<usize>(),
+                "unsupported platform"
+            );
+            if data_len == 0 {
+                continue;
+            }
+            let data_len = data_len as usize;
+            let origin_ptr = mem
+                .get_slice(data_addr, data_len)
+                .map_err(ExecuteError::GetHostAddress)?;
+            let origin_ptr = origin_ptr.ptr_guard_mut();
+            if !(origin_ptr.as_ptr() as u64).is_multiple_of(alignment)
+                || !(origin_ptr.len() as u64).is_multiple_of(alignment)
+            {
+                return Ok(false);
+            }
+        }
+
+        Ok(true)
+    }
+
+    // Returns the sum of the lengths of `self.data_descriptors`.
+    fn data_len(&self) -> usize {
+        self.data_descriptors
+            .iter()
+            .map(|(_, len)| *len as usize)
+            .sum()
+    }
+
+    // Marks guest-memory read destinations dirty before submitting async IO.
+    fn mark_read_dirty<B: Bitmap + 'static>(
+        &self,
+        mem: &vm_memory::GuestMemoryMmap<B>,
+    ) -> Result<(), ExecuteError> {
+        for (data_addr, data_len) in &self.data_descriptors {
+            mem.get_slice(*data_addr, *data_len as usize)
+                .map_err(ExecuteError::GetHostAddress)?
+                .bitmap()
+                .mark_dirty(0, *data_len as usize);
+        }
+        Ok(())
+    }
+
+    // Copies guest descriptor contents into a contiguous host buffer.
+    fn copy_guest_to_buffer<B: Bitmap + 'static>(
+        &self,
+        mem: &vm_memory::GuestMemoryMmap<B>,
+        buffer: &mut [u8],
+    ) -> Result<(), ExecuteError> {
+        let mut offset = 0usize;
+        for (data_addr, data_len) in &self.data_descriptors {
+            let data_len = *data_len as usize;
+            mem.read_slice(&mut buffer[offset..offset + data_len], *data_addr)
+                .map_err(ExecuteError::Read)?;
+            offset += data_len;
+        }
+        Ok(())
+    }
+
+    // Copies a host completion buffer back into guest descriptors.
+    fn copy_buffer_to_guest<B: Bitmap + 'static>(
+        &self,
+        mem: &vm_memory::GuestMemoryMmap<B>,
+        buffer: &[u8],
+    ) -> Result<(), Error> {
+        let mut buffer_offset = 0usize;
+        for (data_addr, data_len) in &self.data_descriptors {
+            if buffer_offset >= buffer.len() {
+                break;
+            }
+            let data_len = (*data_len as usize).min(buffer.len() - buffer_offset);
+            mem.write_slice(&buffer[buffer_offset..buffer_offset + data_len], *data_addr)
+                .map_err(Error::GuestMemory)?;
+            buffer_offset += data_len;
+        }
+        Ok(())
+    }
+
     pub fn complete_async<B: Bitmap + 'static>(
         &mut self,
         mem: &vm_memory::GuestMemoryMmap<B>,
+        completion: &mut AsyncIoCompletion,
     ) -> Result<(), Error> {
-        for aligned_op in self.aligned_operations.drain(..) {
-            // We need to perform the copy after the data has been read inside
-            // the aligned buffer in case we're reading data in.
-            if self.request_type == RequestType::In {
-                mem.write_slice(aligned_op.as_bytes(), aligned_op.data_addr())
-                    .map_err(Error::GuestMemory)?;
-            }
+        if self.request_type == RequestType::In
+            && completion.result > 0
+            && let Some(buffer) = completion.buffer.take()
+        {
+            let len = (completion.result as usize).min(buffer.as_slice().len());
+            self.copy_buffer_to_guest(mem, &buffer.as_slice()[..len])?;
         }
 
         Ok(())
@@ -560,7 +644,10 @@ impl Request {
         if total_bytes == 0 {
             return Ok(());
         }
-        let total_sectors = total_bytes.div_ceil(SECTOR_SIZE);
+        if !total_bytes.is_multiple_of(SECTOR_SIZE) {
+            return Err(ExecuteError::BadRequest(Error::InvalidDataLength));
+        }
+        let total_sectors = total_bytes / SECTOR_SIZE;
         let end_sector = self
             .sector
             .checked_add(total_sectors)
@@ -571,3 +658,62 @@ impl Request {
         Ok(())
     }
 }
+
+#[cfg(test)]
+mod unit_tests {
+    use std::sync::Arc;
+
+    use vm_memory::GuestMemoryMmap;
+    use vmm_sys_util::eventfd::EventFd;
+
+    use super::*;
+    use crate::async_io::{AsyncIo, AsyncIoCompletion, AsyncIoOperation, AsyncIoResult};
+
+    struct PanicAsyncIo(EventFd);
+
+    impl AsyncIo for PanicAsyncIo {
+        fn notifier(&self) -> &EventFd {
+            &self.0
+        }
+        fn submit_data_operation(&mut self, _: AsyncIoOperation) -> AsyncIoResult<()> {
+            unreachable!()
+        }
+        fn fsync(&mut self, _: Option<u64>) -> AsyncIoResult<()> {
+            unreachable!()
+        }
+        fn punch_hole(&mut self, _: u64, _: u64, _: u64) -> AsyncIoResult<()> {
+            unreachable!()
+        }
+        fn write_zeroes(&mut self, _: u64, _: u64, _: u64) -> AsyncIoResult<()> {
+            unreachable!()
+        }
+        fn next_completed_request(&mut self) -> Option<AsyncIoCompletion> {
+            None
+        }
+    }
+
+    #[test]
+    fn write_zeroes_rejects_sector_arithmetic_overflow() {
+        let mem = Arc::new(GuestMemoryMmap::<()>::from_ranges(&[(GuestAddress(0), 4096)]).unwrap());
+        mem.write_slice(&(u64::MAX - 100).to_le_bytes(), GuestAddress(0))
+            .unwrap();
+        mem.write_slice(&1000u32.to_le_bytes(), GuestAddress(8))
+            .unwrap();
+
+        let mut request = Request {
+            request_type: RequestType::WriteZeroes,
+            sector: 0,
+            data_descriptors: SmallVec::from_slice(&[(GuestAddress(0), DISCARD_WZ_SEG_SIZE)]),
+            status_addr: GuestAddress(0),
+            writeback: true,
+            start: Instant::now(),
+        };
+        let mut disk = PanicAsyncIo(EventFd::new(0).unwrap());
+
+        let Err(ExecuteError::BadRequest(Error::InvalidOffset)) =
+            request.execute_async(mem, 1024, &mut disk, &[], false, 0)
+        else {
+            panic!("expected BadRequest(InvalidOffset)");
+        };
+    }
+}

block/src/lib.rs

@@ -8,75 +8,52 @@
 //
 // SPDX-License-Identifier: Apache-2.0 AND BSD-3-Clause
 
-mod aligned_operation;
-pub mod async_io;
 pub mod disk_file;
 pub mod error;
 pub mod factory;
-pub mod fcntl;
-pub mod fixed_vhd;
-#[cfg(feature = "io_uring")]
-/// Enabled with the `"io_uring"` feature
-pub mod fixed_vhd_async;
-pub mod fixed_vhd_disk;
-pub mod fixed_vhd_sync;
-pub mod qcow;
-#[cfg(feature = "io_uring")]
-pub(crate) mod qcow_async;
-pub(crate) mod qcow_common;
-pub mod qcow_disk;
-pub(crate) mod qcow_sync;
-#[cfg(feature = "io_uring")]
-pub(crate) mod raw_async;
-pub(crate) mod raw_async_aio;
-#[cfg(test)]
-mod raw_async_io_tests;
-pub mod raw_disk;
-pub(crate) mod raw_sync;
-mod request;
-pub mod vhd;
-pub mod vhdx;
-pub mod vhdx_sync;
-
-use std::alloc::{Layout, alloc_zeroed};
-use std::collections::VecDeque;
+#[path = "io/mod.rs"]
+mod io_impl;
+pub use io_impl::{async_io, fcntl, request};
+pub(crate) mod aligned_buffer;
+pub mod aligned_file;
+pub mod formats;
+mod sparse;
 use std::fmt::{self, Debug};
 use std::fs::{File, OpenOptions};
-use std::io::{self, IoSlice, IoSliceMut, Read, Seek, SeekFrom, Write};
 use std::os::linux::fs::MetadataExt;
-use std::os::unix::fs::FileTypeExt;
-use std::os::unix::io::AsRawFd;
+use std::os::unix::fs::{FileExt, FileTypeExt};
+use std::os::unix::io::{AsRawFd, RawFd};
 use std::path::Path;
 use std::str::FromStr;
-use std::{cmp, mem, result};
+use std::{cmp, io, mem, result};
 
-pub use aligned_operation::AlignedOperation;
+pub use aligned_file::AlignedFile;
+use formats::qcow;
 #[cfg(feature = "io_uring")]
 use io_uring::{IoUring, Probe, opcode};
 use libc::{
     FALLOC_FL_KEEP_SIZE, FALLOC_FL_PUNCH_HOLE, FALLOC_FL_ZERO_RANGE, S_IFBLK, S_IFMT, ioctl,
 };
 use log::{debug, info, warn};
-pub use request::{BatchRequest, ExecuteAsync, MAX_DISCARD_WRITE_ZEROES_SEG, Request, RequestType};
+pub use request::{ExecuteAsync, MAX_DISCARD_WRITE_ZEROES_SEG, Request, RequestType};
 use serde::{Deserialize, Serialize};
-use smallvec::SmallVec;
+pub use sparse::{BLKDISCARD, BLKZEROOUT};
 use thiserror::Error;
 use virtio_bindings::virtio_blk::*;
 use vm_memory::bitmap::Bitmap;
 use vm_memory::{ByteValued, Bytes, GuestAddress, GuestMemory, GuestMemoryError};
-use vmm_sys_util::eventfd::EventFd;
 use vmm_sys_util::{aio, ioctl_io_nr, ioctl_ior_nr};
 
-use crate::async_io::{AsyncIoError, AsyncIoResult};
+use crate::async_io::AsyncIoError;
 use crate::error::{BlockError, BlockErrorKind, BlockResult, ErrorOp};
-use crate::request::{DEFAULT_DESCRIPTOR_VEC_SIZE, SECTOR_SIZE};
-use crate::vhdx::VhdxError;
+use crate::formats::vhdx::VhdxError;
+use crate::request::SECTOR_SIZE;
 
 #[derive(Error, Debug)]
 pub enum Error {
     #[error("Guest gave us bad memory addresses")]
     GuestMemory(#[source] GuestMemoryError),
-    #[error("Guest gave us offsets that would have overflowed a usize")]
+    #[error("Guest address {0:?} with sector offset {1} would overflow a usize")]
     CheckedOffset(GuestAddress, usize /* sector offset */),
     #[error("Guest gave us a write only descriptor that protocol says to read from")]
     UnexpectedWriteOnlyDescriptor,
@@ -87,17 +64,17 @@ pub enum Error {
     #[error("Guest gave us a descriptor that was too short to use")]
     DescriptorLengthTooSmall,
     #[error("Failed to detect image type")]
-    DetectImageType(#[source] std::io::Error),
+    DetectImageType(#[source] io::Error),
     #[error("Failure in fixed vhd")]
-    FixedVhdError(#[source] std::io::Error),
+    FixedVhdError(#[source] io::Error),
     #[error("Getting a block's metadata failed")]
-    GetFileMetadata(#[source] std::io::Error),
+    GetFileMetadata(#[source] io::Error),
     #[error("The requested operation would cause a seek beyond disk end")]
     InvalidOffset,
+    #[error("Request data length is not a multiple of the 512-byte sector size")]
+    InvalidDataLength,
     #[error("Failure in qcow")]
     QcowError(#[source] qcow::Error),
-    #[error("Failure in raw file")]
-    RawFileError(#[source] std::io::Error),
     #[error("The requested operation does not support multiple descriptors")]
     TooManyDescriptors,
     #[error("Request contains too many segments ({0}, max {MAX_DISCARD_WRITE_ZEROES_SEG})")]
@@ -150,8 +127,6 @@ pub enum ExecuteError {
     ReadExact(#[source] io::Error),
     #[error("Can't execute an operation other than `read` or `get_id` on a read-only device")]
     ReadOnly,
-    #[error("Failed to seek")]
-    Seek(#[source] io::Error),
     #[error("Failed to write")]
     Write(#[source] GuestMemoryError),
     #[error("Failed to write_all")]
@@ -186,7 +161,6 @@ impl ExecuteError {
             ExecuteError::Read(_) => VIRTIO_BLK_S_IOERR,
             ExecuteError::ReadExact(_) => VIRTIO_BLK_S_IOERR,
             ExecuteError::ReadOnly => VIRTIO_BLK_S_IOERR,
-            ExecuteError::Seek(_) => VIRTIO_BLK_S_IOERR,
             ExecuteError::Write(_) => VIRTIO_BLK_S_IOERR,
             ExecuteError::WriteAll(_) => VIRTIO_BLK_S_IOERR,
             ExecuteError::Unsupported(_) => VIRTIO_BLK_S_UNSUPP,
@@ -332,26 +306,82 @@ pub fn block_io_uring_is_supported() -> bool {
     }
 }
 
+/// Returns `true` iff `fd` refers to a block device.
+///
+/// Returns `false` if the `fstat()` probe itself fails. Callers that need to
+/// distinguish "not a block device" from "couldn't tell" should fall back to
+/// regular-file behaviour, which is what every current caller already does.
+pub(crate) fn is_block_device(fd: RawFd) -> bool {
+    // SAFETY: `libc::stat` is POD; zero-initialization is a valid bit pattern
+    // and `fstat` overwrites every field it cares about on success.
+    let mut stat: libc::stat = unsafe { mem::zeroed() };
+    // SAFETY: FFI call with a valid fd and a valid out-pointer.
+    let ret = unsafe { libc::fstat(fd, &mut stat) };
+    ret == 0 && stat.st_mode & S_IFMT == S_IFBLK
+}
+
+/// Returns the kernel reported direct I/O alignment for `fd`, or `None`
+/// when `fd` was not opened with O_DIRECT.
+///
+/// When O_DIRECT is set, uses `statx(STATX_DIOALIGN)` (Linux >= 6.1) to obtain
+/// the exact memory and offset alignment the kernel requires for direct I/O on
+/// this specific fd. Unlike `fstatvfs().f_bsize`, which only returns the
+/// filesystem's preferred I/O block size, `STATX_DIOALIGN` reports the true per
+/// fd direct I/O constraint accounting for the filesystem, underlying block
+/// device, and any stacking such as loop or device mapper. Falls back to
+/// [`SECTOR_SIZE`] when the kernel does not report a value.
+pub(crate) fn probe_direct_alignment(fd: RawFd) -> Option<u64> {
+    // SAFETY: fcntl(F_GETFL) is always safe on a valid fd.
+    let flags = unsafe { libc::fcntl(fd, libc::F_GETFL) };
+    if flags < 0 || (flags & libc::O_DIRECT) == 0 {
+        return None;
+    }
+
+    // The libc crate does not expose statx / STATX_DIOALIGN on all targets,
+    // for example musl, so define the constant and a minimal repr(C) struct
+    // locally and invoke the syscall directly.
+    const STATX_DIOALIGN: u32 = 0x2000;
+
+    // Minimal statx layout, only the needed fields, everything else is
+    // padding.
+    #[repr(C)]
+    struct Statx {
+        stx_mask: u32,
+        _pad: [u8; 148],
+        stx_dio_mem_align: u32,
+        stx_dio_offset_align: u32,
+        _pad2: [u8; 96],
+    }
+
+    let mut stx = mem::MaybeUninit::<Statx>::zeroed();
+    // SAFETY: FFI syscall with valid fd and correctly sized buffer.
+    let ret = unsafe {
+        libc::syscall(
+            libc::SYS_statx,
+            fd,
+            c"".as_ptr(),
+            libc::AT_EMPTY_PATH,
+            STATX_DIOALIGN,
+            stx.as_mut_ptr(),
+        )
+    };
+    if ret == 0 {
+        // SAFETY: statx succeeded, the struct is fully initialized.
+        let stx = unsafe { stx.assume_init() };
+        if stx.stx_mask & STATX_DIOALIGN != 0 && stx.stx_dio_mem_align > 0 {
+            return Some(cmp::max(stx.stx_dio_mem_align, stx.stx_dio_offset_align) as u64);
+        }
+    }
+
+    debug!("O_DIRECT alignment query failed, falling back to default {SECTOR_SIZE}");
+    Some(SECTOR_SIZE)
+}
+
 /// Probe whether the file/device supports punch hole and zero range
 pub fn probe_sparse_support(file: &File) -> bool {
     let fd = file.as_raw_fd();
 
-    let is_block_device = {
-        let mut stat = std::mem::MaybeUninit::<libc::stat>::uninit();
-        // SAFETY: FFI call with valid fd and buffer
-        let ret = unsafe { libc::fstat(fd, stat.as_mut_ptr()) };
-        if ret != 0 {
-            warn!(
-                "Failed to stat file descriptor for sparse probe: {}",
-                io::Error::last_os_error()
-            );
-            return false;
-        }
-        // SAFETY: stat result is valid at this point
-        unsafe { (*stat.as_ptr()).st_mode & S_IFMT == S_IFBLK }
-    };
-
-    if is_block_device {
+    if is_block_device(fd) {
         probe_block_device_sparse_support(fd)
     } else {
         probe_file_sparse_support(fd)
@@ -452,110 +482,6 @@ pub fn preallocate_disk<P: AsRef<Path>>(file: &File, path: P) {
     }
 }
 
-pub trait AsyncAdaptor {
-    fn read_vectored_sync(
-        &mut self,
-        offset: libc::off_t,
-        iovecs: &[libc::iovec],
-        user_data: u64,
-        eventfd: &EventFd,
-        completion_list: &mut VecDeque<(u64, i32)>,
-    ) -> AsyncIoResult<()>
-    where
-        Self: Read + Seek,
-    {
-        // Convert libc::iovec into IoSliceMut
-        let mut slices: SmallVec<[IoSliceMut; DEFAULT_DESCRIPTOR_VEC_SIZE]> =
-            SmallVec::with_capacity(iovecs.len());
-        for iovec in iovecs.iter() {
-            // SAFETY: on Linux IoSliceMut wraps around libc::iovec
-            slices.push(IoSliceMut::new(unsafe {
-                std::mem::transmute::<libc::iovec, &mut [u8]>(*iovec)
-            }));
-        }
-
-        let result = {
-            // Move the cursor to the right offset
-            self.seek(SeekFrom::Start(offset as u64))
-                .map_err(AsyncIoError::ReadVectored)?;
-
-            let mut r = 0;
-            for b in slices.iter_mut() {
-                r += self.read(b).map_err(AsyncIoError::ReadVectored)?;
-            }
-            r
-        };
-
-        completion_list.push_back((user_data, result as i32));
-        eventfd.write(1).unwrap();
-
-        Ok(())
-    }
-
-    fn write_vectored_sync(
-        &mut self,
-        offset: libc::off_t,
-        iovecs: &[libc::iovec],
-        user_data: u64,
-        eventfd: &EventFd,
-        completion_list: &mut VecDeque<(u64, i32)>,
-    ) -> AsyncIoResult<()>
-    where
-        Self: Write + Seek,
-    {
-        // Convert libc::iovec into IoSlice
-        let mut slices: SmallVec<[IoSlice; DEFAULT_DESCRIPTOR_VEC_SIZE]> =
-            SmallVec::with_capacity(iovecs.len());
-        for iovec in iovecs.iter() {
-            // SAFETY: on Linux IoSlice wraps around libc::iovec
-            slices.push(IoSlice::new(unsafe {
-                std::mem::transmute::<libc::iovec, &mut [u8]>(*iovec)
-            }));
-        }
-
-        let result = {
-            // Move the cursor to the right offset
-            self.seek(SeekFrom::Start(offset as u64))
-                .map_err(AsyncIoError::WriteVectored)?;
-
-            let mut r = 0;
-            for b in slices.iter() {
-                r += self.write(b).map_err(AsyncIoError::WriteVectored)?;
-            }
-            r
-        };
-
-        completion_list.push_back((user_data, result as i32));
-        eventfd.write(1).unwrap();
-
-        Ok(())
-    }
-
-    fn fsync_sync(
-        &mut self,
-        user_data: Option<u64>,
-        eventfd: &EventFd,
-        completion_list: &mut VecDeque<(u64, i32)>,
-    ) -> AsyncIoResult<()>
-    where
-        Self: Write,
-    {
-        let result: i32 = {
-            // Flush
-            self.flush().map_err(AsyncIoError::Fsync)?;
-
-            0
-        };
-
-        if let Some(user_data) = user_data {
-            completion_list.push_back((user_data, result));
-            eventfd.write(1).unwrap();
-        }
-
-        Ok(())
-    }
-}
-
 #[derive(Serialize, Deserialize, Clone, Copy, Debug, PartialEq, Eq, Default)]
 pub enum ImageType {
     FixedVhd,
@@ -599,23 +525,6 @@ impl FromStr for ImageType {
 const QCOW_MAGIC: u32 = 0x5146_49fb;
 const VHDX_SIGN: u64 = 0x656C_6966_7864_6876;
 
-/// Read a block into memory aligned by the source block size (needed for O_DIRECT)
-pub fn read_aligned_block_size(f: &mut File) -> std::io::Result<Vec<u8>> {
-    let blocksize = DiskTopology::probe(f)?.logical_block_size as usize;
-    // SAFETY: We are allocating memory that is naturally aligned (size = alignment) and we meet
-    // requirements for safety from Vec::from_raw_parts() as we are using the global allocator
-    // and transferring ownership of the memory.
-    let mut data = unsafe {
-        Vec::from_raw_parts(
-            alloc_zeroed(Layout::from_size_align_unchecked(blocksize, blocksize)),
-            blocksize,
-            blocksize,
-        )
-    };
-    f.read_exact(&mut data)?;
-    Ok(data)
-}
-
 /// Open a disk image file, returning a [`BlockError`] with path context
 /// on failure.
 pub fn open_disk_image(path: &Path, options: &OpenOptions) -> BlockResult<File> {
@@ -628,13 +537,16 @@ pub fn open_disk_image(path: &Path, options: &OpenOptions) -> BlockResult<File>
 
 /// Determine image type through file parsing.
 pub fn detect_image_type(f: &mut File) -> BlockResult<ImageType> {
-    let block = read_aligned_block_size(f)
+    let aligned = AlignedFile::new(f.try_clone()?, true);
+    let mut block = vec![0u8; aligned.alignment()];
+    aligned
+        .read_exact_at(&mut block, 0)
         .map_err(|e| BlockError::new(BlockErrorKind::Io, e).with_op(ErrorOp::DetectImageType))?;
 
     // Check 4 first bytes to get the header value and determine the image type
     let image_type = if u32::from_be_bytes(block[0..4].try_into().unwrap()) == QCOW_MAGIC {
         ImageType::Qcow2
-    } else if vhd::is_fixed_vhd(f)
+    } else if formats::vhd::is_fixed_vhd(f)
         .map_err(|e| BlockError::new(BlockErrorKind::Io, e).with_op(ErrorOp::DetectImageType))?
     {
         ImageType::FixedVhd
@@ -647,17 +559,6 @@ pub fn detect_image_type(f: &mut File) -> BlockResult<ImageType> {
     Ok(image_type)
 }
 
-pub trait BlockBackend: Read + Write + Seek + Send + Debug {
-    /// Returns the logical disk size a guest will see.
-    ///
-    /// For raw formats, this is equal to [`Self::physical_size`]. For file formats
-    /// that wrap disk images in a container (e.g. QCOW2), this refers to the
-    /// effective size that the guest will see.
-    fn logical_size(&self) -> Result<u64, Error>;
-    /// Returns the physical size of the underlying file.
-    fn physical_size(&self) -> Result<u64, Error>;
-}
-
 #[derive(Debug)]
 pub struct DiskTopology {
     pub logical_block_size: u64,
@@ -721,21 +622,8 @@ enum BlockSize {
 }
 
 impl DiskTopology {
-    fn is_block_device(f: &File) -> std::io::Result<bool> {
-        let mut stat = std::mem::MaybeUninit::<libc::stat>::uninit();
-        // SAFETY: FFI call with a valid fd and buffer
-        let ret = unsafe { libc::fstat(f.as_raw_fd(), stat.as_mut_ptr()) };
-        if ret != 0 {
-            return Err(std::io::Error::last_os_error());
-        }
-
-        // SAFETY: stat is valid at this point
-        let is_block = unsafe { (*stat.as_ptr()).st_mode & S_IFMT == S_IFBLK };
-        Ok(is_block)
-    }
-
     // libc::ioctl() takes different types on different architectures
-    fn query_block_size(f: &File, block_size_type: BlockSize) -> std::io::Result<u64> {
+    fn query_block_size(f: &File, block_size_type: BlockSize) -> io::Result<u64> {
         let mut block_size = 0;
         // SAFETY: FFI call with correct arguments
         let ret = unsafe {
@@ -751,72 +639,18 @@ impl DiskTopology {
             )
         };
         if ret != 0 {
-            return Err(std::io::Error::last_os_error());
+            return Err(io::Error::last_os_error());
         }
 
         Ok(block_size)
     }
 
-    /// Query the O_DIRECT alignment requirement for a regular file.
-    ///
-    /// Uses `statx(STATX_DIOALIGN)` (Linux >= 6.1) to obtain the exact
-    /// memory and offset alignment the kernel requires for direct I/O on
-    /// this specific file. Unlike `fstatvfs().f_bsize`, which only returns
-    /// the filesystem's preferred I/O block size, `STATX_DIOALIGN` reports
-    /// the true per-file DIO constraints accounting for the filesystem,
-    /// underlying block device, and any stacking (loop, dm, etc.).
-    fn query_file_alignment(f: &File) -> u64 {
-        // The libc crate does not expose statx / STATX_DIOALIGN on all
-        // targets (e.g. musl), so define the constant and a minimal repr(C)
-        // struct locally and invoke the syscall directly.
-        const STATX_DIOALIGN: u32 = 0x2000;
-
-        // Minimal statx layout, only the needed fields,
-        // everything else is padding.
-        #[repr(C)]
-        struct Statx {
-            stx_mask: u32,
-            _pad: [u8; 148],
-            stx_dio_mem_align: u32,
-            stx_dio_offset_align: u32,
-            _pad2: [u8; 96],
-        }
-
-        let mut stx = mem::MaybeUninit::<Statx>::zeroed();
-        // SAFETY: FFI syscall with valid fd and correctly sized buffer.
-        let ret = unsafe {
-            libc::syscall(
-                libc::SYS_statx,
-                f.as_raw_fd(),
-                c"".as_ptr(),
-                libc::AT_EMPTY_PATH,
-                STATX_DIOALIGN,
-                stx.as_mut_ptr(),
-            )
-        };
-        if ret == 0 {
-            // SAFETY: statx succeeded, the struct is fully initialized.
-            let stx = unsafe { stx.assume_init() };
-            if stx.stx_mask & STATX_DIOALIGN != 0 && stx.stx_dio_mem_align > 0 {
-                let align = cmp::max(stx.stx_dio_mem_align, stx.stx_dio_offset_align) as u64;
-                debug!("statx(STATX_DIOALIGN) returned alignment {align}");
-                return align;
-            }
-        }
-
-        debug!("O_DIRECT alignment query failed, falling back to default {SECTOR_SIZE}");
-        SECTOR_SIZE
-    }
-
-    pub fn probe(f: &File) -> std::io::Result<Self> {
-        if !Self::is_block_device(f)? {
+    pub fn probe(f: &File) -> io::Result<Self> {
+        if !is_block_device(f.as_raw_fd()) {
             // For regular files opened with O_DIRECT, the logical block size
             // must reflect the filesystem DIO alignment so the guest issues
             // correctly sized I/O.
-            // SAFETY: fcntl(F_GETFL) is always safe on a valid fd.
-            let flags = unsafe { libc::fcntl(f.as_raw_fd(), libc::F_GETFL) };
-            if flags >= 0 && (flags & libc::O_DIRECT) != 0 {
-                let alignment = Self::query_file_alignment(f);
+            if let Some(alignment) = probe_direct_alignment(f.as_raw_fd()) {
                 return Ok(DiskTopology {
                     logical_block_size: alignment,
                     physical_block_size: alignment,
@@ -842,7 +676,7 @@ mod unit_tests {
     use std::fs::OpenOptions;
     use std::io::Write;
     use std::os::unix::fs::OpenOptionsExt;
-    use std::{ptr, slice};
+    use std::{mem, ptr, slice};
 
     use vmm_sys_util::tempfile::TempFile;
 
@@ -973,6 +807,25 @@ mod unit_tests {
         assert!(physical > 0);
     }
 
+    // A mode-0 fallocate() is not eagerly accounted in st_blocks on every
+    // filesystem: zfs reserves the range but accounts blocks lazily at
+    // transaction-group commit, and FUSE-based filesystems such as virtiofs
+    // report preallocated files as sparse. Identify those by filesystem type
+    // so a skipped physical-size check always names a proven platform
+    // limitation instead of being inferred from the value under test.
+    fn fs_defers_fallocate_block_accounting(f: &File) -> bool {
+        // SAFETY: a zeroed statfs is a valid output buffer for fstatfs and it
+        // is only read after the call succeeds.
+        let mut sfs: libc::statfs = unsafe { mem::zeroed() };
+        // SAFETY: the fd is valid and sfs outlives the call.
+        let ret = unsafe { libc::fstatfs(f.as_raw_fd(), &mut sfs) };
+        assert_eq!(ret, 0, "fstatfs failed: {}", io::Error::last_os_error());
+        // ZFS_SUPER_MAGIC and FUSE_SUPER_MAGIC (statfs(2)), as untyped
+        // literals because the width and signedness of f_type differ
+        // between libc targets.
+        matches!(sfs.f_type, 0x2fc1_2fc1 | 0x6573_5546)
+    }
+
     #[test]
     fn test_query_device_size_sparse_file_punch_hole() {
         let temp_file = TempFile::new().unwrap();
@@ -989,11 +842,25 @@ mod unit_tests {
                 size,
             )
         };
-        assert_eq!(ret, 0, "fallocate failed: {}", io::Error::last_os_error());
+        if ret != 0 {
+            let err = io::Error::last_os_error();
+            if err.raw_os_error() == Some(libc::EOPNOTSUPP) {
+                eprintln!("Skipping test: fallocate() is not supported: {err}");
+                return;
+            }
+            panic!("fallocate failed: {err}");
+        }
         f.sync_all().unwrap();
 
         let (log_before, phys_before) = query_device_size(f).unwrap();
         assert_eq!(log_before, size as u64);
+        if fs_defers_fallocate_block_accounting(f) {
+            eprintln!(
+                "Skipping physical size checks: the filesystem defers \
+                 fallocate() block accounting"
+            );
+            return;
+        }
         assert_eq!(phys_before, size as u64);
 
         // Punch a hole in the middle 512 KiB
@@ -1006,7 +873,16 @@ mod unit_tests {
                 size / 2,
             )
         };
-        assert_eq!(ret, 0, "punch hole failed: {}", io::Error::last_os_error());
+        if ret != 0 {
+            let err = io::Error::last_os_error();
+            if err.raw_os_error() == Some(libc::EOPNOTSUPP) {
+                eprintln!(
+                    "Skipping punch-hole checks: FALLOC_FL_PUNCH_HOLE is not supported: {err}"
+                );
+                return;
+            }
+            panic!("punch hole failed: {err}");
+        }
         f.sync_all().unwrap();
 
         let (logical, physical) = query_device_size(f).unwrap();
@@ -1019,7 +895,7 @@ mod unit_tests {
 
     #[test]
     fn test_query_device_size_rejects_char_device() {
-        let f = std::fs::File::open("/dev/zero").unwrap();
+        let f = File::open("/dev/zero").unwrap();
         let err = query_device_size(&f).unwrap_err();
         assert_eq!(err.kind(), io::ErrorKind::InvalidInput);
     }

block/src/qcow/qcow_raw_file.rs

@@ -1,369 +0,0 @@
-// Copyright 2018 The Chromium OS Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE-BSD-3-Clause file.
-//
-// SPDX-License-Identifier: Apache-2.0 AND BSD-3-Clause
-
-use std::fmt::Debug;
-use std::io::{self, BufWriter, Read, Seek, SeekFrom, Write};
-use std::mem::size_of;
-use std::os::fd::{AsFd, AsRawFd, BorrowedFd, RawFd};
-
-use byteorder::{BigEndian, ReadBytesExt, WriteBytesExt};
-use vmm_sys_util::write_zeroes::WriteZeroes;
-
-use super::RawFile;
-
-// Type aliases for the refcount read/write function pointers
-type RefcountReader = fn(&mut RawFile, usize) -> io::Result<Vec<u64>>;
-type RefcountWriter = fn(&mut RawFile, &[u64]) -> io::Result<()>;
-
-/// Big-endian file access trait.
-pub(super) trait BeUint: Sized + Copy {
-    fn from_be_slice(bytes: &[u8]) -> u64;
-    fn read_be<R: Read>(r: &mut R) -> io::Result<Self>;
-    fn write_be<W: Write>(w: &mut W, val: Self) -> io::Result<()>;
-}
-
-impl BeUint for u8 {
-    #[inline(always)]
-    fn from_be_slice(bytes: &[u8]) -> u64 {
-        bytes[0] as u64
-    }
-    #[inline(always)]
-    fn read_be<R: Read>(r: &mut R) -> io::Result<Self> {
-        r.read_u8()
-    }
-    #[inline(always)]
-    fn write_be<W: Write>(w: &mut W, val: Self) -> io::Result<()> {
-        w.write_u8(val)
-    }
-}
-
-impl BeUint for u16 {
-    #[inline(always)]
-    fn from_be_slice(bytes: &[u8]) -> u64 {
-        u16::from_be_bytes([bytes[0], bytes[1]]) as u64
-    }
-    #[inline(always)]
-    fn read_be<R: Read>(r: &mut R) -> io::Result<Self> {
-        r.read_u16::<BigEndian>()
-    }
-    #[inline(always)]
-    fn write_be<W: Write>(w: &mut W, val: Self) -> io::Result<()> {
-        w.write_u16::<BigEndian>(val)
-    }
-}
-
-impl BeUint for u32 {
-    #[inline(always)]
-    fn from_be_slice(bytes: &[u8]) -> u64 {
-        u32::from_be_bytes([bytes[0], bytes[1], bytes[2], bytes[3]]) as u64
-    }
-    #[inline(always)]
-    fn read_be<R: Read>(r: &mut R) -> io::Result<Self> {
-        r.read_u32::<BigEndian>()
-    }
-    #[inline(always)]
-    fn write_be<W: Write>(w: &mut W, val: Self) -> io::Result<()> {
-        w.write_u32::<BigEndian>(val)
-    }
-}
-
-impl BeUint for u64 {
-    #[inline(always)]
-    fn from_be_slice(bytes: &[u8]) -> u64 {
-        u64::from_be_bytes([
-            bytes[0], bytes[1], bytes[2], bytes[3], bytes[4], bytes[5], bytes[6], bytes[7],
-        ])
-    }
-    #[inline(always)]
-    fn read_be<R: Read>(r: &mut R) -> io::Result<Self> {
-        r.read_u64::<BigEndian>()
-    }
-    #[inline(always)]
-    fn write_be<W: Write>(w: &mut W, val: Self) -> io::Result<()> {
-        w.write_u64::<BigEndian>(val)
-    }
-}
-
-/// Read byte-aligned refcounts.
-fn read_refcount<T: BeUint>(file: &mut RawFile, count: usize) -> io::Result<Vec<u64>> {
-    let bytes_per_entry = size_of::<T>();
-    let mut data = vec![0u8; count * bytes_per_entry];
-    file.read_exact(&mut data)?;
-    Ok(data
-        .chunks_exact(bytes_per_entry)
-        .map(T::from_be_slice)
-        .collect())
-}
-
-/// Write byte-aligned refcounts.
-fn write_refcount<T: BeUint + TryFrom<u64>>(file: &mut RawFile, table: &[u64]) -> io::Result<()>
-where
-    <T as TryFrom<u64>>::Error: Debug,
-{
-    let bytes_per_entry = size_of::<T>();
-    let mut buffer = BufWriter::with_capacity(table.len() * bytes_per_entry, file);
-    for &val in table {
-        let converted = T::try_from(val).expect("refcount values are validated on increment");
-        T::write_be(&mut buffer, converted)?;
-    }
-    buffer.flush()
-}
-
-/// Read sub-byte refcounts. Bit 0 is the least significant bit.
-fn read_refcount_subbyte<const BITS: usize>(
-    file: &mut RawFile,
-    count: usize,
-) -> io::Result<Vec<u64>> {
-    const { assert!(BITS == 1 || BITS == 2 || BITS == 4) };
-    let entries_per_byte = 8 / BITS;
-    let mask = (1u64 << BITS) - 1;
-    let bytes_needed = count.div_ceil(entries_per_byte);
-    let mut bytes = vec![0u8; bytes_needed];
-    file.read_exact(&mut bytes)?;
-
-    let mut table = vec![0u64; count];
-    for (i, val) in table.iter_mut().enumerate() {
-        let byte_idx = i / entries_per_byte;
-        let bit_offset = (i % entries_per_byte) * BITS;
-        *val = (bytes[byte_idx] as u64 >> bit_offset) & mask;
-    }
-    Ok(table)
-}
-
-/// Write sub-byte refcounts. Bit 0 is the least significant bit.
-fn write_refcount_subbyte<const BITS: usize>(file: &mut RawFile, table: &[u64]) -> io::Result<()> {
-    const { assert!(BITS == 1 || BITS == 2 || BITS == 4) };
-    let entries_per_byte = 8 / BITS;
-    let mask = (1u64 << BITS) - 1;
-    let mut buffer = BufWriter::with_capacity(table.len().div_ceil(entries_per_byte), file);
-
-    for chunk in table.chunks(entries_per_byte) {
-        let mut byte = 0u8;
-        for (i, &val) in chunk.iter().enumerate() {
-            let bit_offset = i * BITS;
-            byte |= ((val & mask) << bit_offset) as u8;
-        }
-        buffer.write_u8(byte)?;
-    }
-    buffer.flush()
-}
-
-/// A qcow file. Allows reading/writing clusters and appending clusters.
-#[derive(Debug)]
-pub struct QcowRawFile {
-    file: RawFile,
-    cluster_size: u64,
-    cluster_mask: u64,
-    refcount_block_entries: u64,
-    read_refcount_fn: RefcountReader,
-    write_refcount_fn: RefcountWriter,
-}
-
-impl QcowRawFile {
-    /// Creates a `QcowRawFile` from the given `File`, `None` is returned if `cluster_size` is not
-    /// a power of two or refcount_bits is invalid.
-    pub fn from(file: RawFile, cluster_size: u64, refcount_bits: u64) -> Option<Self> {
-        if !cluster_size.is_power_of_two() {
-            return None;
-        }
-
-        let (read_refcount_fn, write_refcount_fn): (RefcountReader, RefcountWriter) =
-            match refcount_bits {
-                1 => (read_refcount_subbyte::<1>, write_refcount_subbyte::<1>),
-                2 => (read_refcount_subbyte::<2>, write_refcount_subbyte::<2>),
-                4 => (read_refcount_subbyte::<4>, write_refcount_subbyte::<4>),
-                8 => (read_refcount::<u8>, write_refcount::<u8>),
-                16 => (read_refcount::<u16>, write_refcount::<u16>),
-                32 => (read_refcount::<u32>, write_refcount::<u32>),
-                64 => (read_refcount::<u64>, write_refcount::<u64>),
-                _ => return None,
-            };
-
-        // For sub-byte refcounts (1,2,4 bits), entries pack multiple per byte
-        let refcount_block_entries = cluster_size * 8 / refcount_bits;
-
-        Some(QcowRawFile {
-            file,
-            cluster_size,
-            cluster_mask: cluster_size - 1,
-            refcount_block_entries,
-            read_refcount_fn,
-            write_refcount_fn,
-        })
-    }
-
-    /// Reads `count` 64 bit offsets and returns them as a vector.
-    /// `mask` optionally `&`s out some of the bits on the file.
-    pub fn read_pointer_table(
-        &mut self,
-        offset: u64,
-        count: u64,
-        mask: Option<u64>,
-    ) -> io::Result<Vec<u64>> {
-        let mut table = vec![0; count as usize];
-        self.file.seek(SeekFrom::Start(offset))?;
-        self.file.read_u64_into::<BigEndian>(&mut table)?;
-        if let Some(m) = mask {
-            for ptr in &mut table {
-                *ptr &= m;
-            }
-        }
-        Ok(table)
-    }
-
-    /// Reads a cluster's worth of 64 bit offsets and returns them as a vector.
-    /// `mask` optionally `&`s out some of the bits on the file.
-    pub fn read_pointer_cluster(&mut self, offset: u64, mask: Option<u64>) -> io::Result<Vec<u64>> {
-        let count = self.cluster_size / size_of::<u64>() as u64;
-        self.read_pointer_table(offset, count, mask)
-    }
-
-    /// Internal helper for creating a buffered writer for pointer tables.
-    #[inline]
-    fn setup_pointer_table_writer<T>(
-        &mut self,
-        offset: u64,
-        entries: &impl Iterator<Item = T>,
-    ) -> io::Result<BufWriter<RawFile>> {
-        self.file.seek(SeekFrom::Start(offset))?;
-        let my_file = self.file.try_clone()?;
-        let capacity = entries.size_hint().0 * size_of::<u64>();
-        Ok(BufWriter::with_capacity(capacity, my_file))
-    }
-
-    /// Writes a pointer table to `offset` in the file.
-    /// Entries are computed on-the-fly by the callback.
-    pub fn write_pointer_table<'a, T: Copy + 'a>(
-        &mut self,
-        offset: u64,
-        entries: impl Iterator<Item = &'a T>,
-        mut f: impl FnMut(&mut QcowRawFile, T) -> io::Result<u64>,
-    ) -> io::Result<()> {
-        let mut buffer = self.setup_pointer_table_writer(offset, &entries)?;
-
-        for addr in entries {
-            let entry = f(self, *addr)?;
-            u64::write_be(&mut buffer, entry)?;
-        }
-        buffer.flush()?;
-        Ok(())
-    }
-
-    /// Writes a pointer table directly without transforming values.
-    pub fn write_pointer_table_direct<'a>(
-        &mut self,
-        offset: u64,
-        entries: impl Iterator<Item = &'a u64>,
-    ) -> io::Result<()> {
-        let mut buffer = self.setup_pointer_table_writer(offset, &entries)?;
-
-        for &entry in entries {
-            u64::write_be(&mut buffer, entry)?;
-        }
-        buffer.flush()?;
-        Ok(())
-    }
-
-    /// Read a refcount block from the file and returns a Vec containing the block.
-    /// Always returns a cluster's worth of data.
-    #[inline]
-    pub fn read_refcount_block(&mut self, offset: u64) -> io::Result<Vec<u64>> {
-        self.file.seek(SeekFrom::Start(offset))?;
-        (self.read_refcount_fn)(&mut self.file, self.refcount_block_entries as usize)
-    }
-
-    /// Writes a refcount block to the file.
-    #[inline]
-    pub fn write_refcount_block(&mut self, offset: u64, table: &[u64]) -> io::Result<()> {
-        self.file.seek(SeekFrom::Start(offset))?;
-        (self.write_refcount_fn)(&mut self.file, table)
-    }
-
-    /// Allocates a new cluster at the end of the current file, return the address.
-    pub fn add_cluster_end(&mut self, max_valid_cluster_offset: u64) -> io::Result<Option<u64>> {
-        // Determine where the new end of the file should be and set_len, which
-        // translates to truncate(2).
-        let file_end: u64 = self.file.seek(SeekFrom::End(0))?;
-        let new_cluster_address: u64 = (file_end + self.cluster_size - 1) & !self.cluster_mask;
-
-        if new_cluster_address > max_valid_cluster_offset {
-            return Ok(None);
-        }
-
-        self.file.set_len(new_cluster_address + self.cluster_size)?;
-
-        Ok(Some(new_cluster_address))
-    }
-
-    /// Returns a reference to the underlying file.
-    pub fn file(&self) -> &RawFile {
-        &self.file
-    }
-
-    /// Returns a mutable reference to the underlying file.
-    pub fn file_mut(&mut self) -> &mut RawFile {
-        &mut self.file
-    }
-
-    /// Returns the size of the file's clusters.
-    pub fn cluster_size(&self) -> u64 {
-        self.cluster_size
-    }
-
-    /// Returns the offset of `address` within a cluster.
-    pub fn cluster_offset(&self, address: u64) -> u64 {
-        address & self.cluster_mask
-    }
-
-    /// Returns the base address of the cluster containing `address`.
-    pub fn cluster_address(&self, address: u64) -> u64 {
-        address & !self.cluster_mask
-    }
-
-    /// Zeros out a cluster in the file.
-    pub fn zero_cluster(&mut self, address: u64) -> io::Result<()> {
-        let cluster_size = self.cluster_size as usize;
-        self.file.seek(SeekFrom::Start(address))?;
-        self.file.write_zeroes(cluster_size)?;
-        Ok(())
-    }
-
-    /// Writes
-    pub fn write_cluster(&mut self, address: u64, data: &[u8]) -> io::Result<()> {
-        let cluster_size = self.cluster_size as usize;
-        self.file.seek(SeekFrom::Start(address))?;
-        self.file.write_all(&data[0..cluster_size])
-    }
-
-    pub fn physical_size(&self) -> Result<u64, std::io::Error> {
-        self.file.metadata().map(|m| m.len())
-    }
-}
-
-impl Clone for QcowRawFile {
-    fn clone(&self) -> Self {
-        QcowRawFile {
-            file: self.file.try_clone().expect("QcowRawFile cloning failed"),
-            cluster_size: self.cluster_size,
-            cluster_mask: self.cluster_mask,
-            refcount_block_entries: self.refcount_block_entries,
-            read_refcount_fn: self.read_refcount_fn,
-            write_refcount_fn: self.write_refcount_fn,
-        }
-    }
-}
-
-impl AsRawFd for QcowRawFile {
-    fn as_raw_fd(&self) -> RawFd {
-        self.file.as_raw_fd()
-    }
-}
-
-impl AsFd for QcowRawFile {
-    fn as_fd(&self) -> BorrowedFd<'_> {
-        self.file.as_fd()
-    }
-}

block/src/qcow/raw_file.rs

@@ -1,406 +0,0 @@
-// Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
-//
-// Portions Copyright 2017 The Chromium OS Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE-BSD-3-Clause file.
-//
-// Copyright © 2020 Intel Corporation
-//
-// SPDX-License-Identifier: Apache-2.0 AND BSD-3-Clause
-
-use std::alloc::{Layout, alloc_zeroed, dealloc};
-use std::fs::{File, Metadata};
-use std::io::{self, Read, Seek, SeekFrom, Write};
-use std::os::fd::{AsFd, BorrowedFd};
-use std::os::unix::io::{AsRawFd, RawFd};
-use std::slice;
-
-use vmm_sys_util::file_traits::FileSync;
-use vmm_sys_util::seek_hole::SeekHole;
-use vmm_sys_util::write_zeroes::{PunchHole, WriteZeroesAt};
-
-use crate::{BlockBackend, query_device_size};
-
-#[derive(Debug)]
-pub struct RawFile {
-    file: File,
-    alignment: usize,
-    position: u64,
-    direct_io: bool,
-}
-
-const BLK_ALIGNMENTS: [usize; 2] = [512, 4096];
-
-fn is_valid_alignment(fd: RawFd, alignment: usize) -> bool {
-    let layout = Layout::from_size_align(alignment, alignment).unwrap();
-    // SAFETY: layout has non-zero size
-    let ptr = unsafe { alloc_zeroed(layout) };
-    assert!(!ptr.is_null());
-
-    // SAFETY: FFI call
-    let ret = unsafe { ::libc::pread(fd, ptr.cast(), alignment, alignment.try_into().unwrap()) };
-
-    // SAFETY: ptr was allocated by alloc_zeroed with layout
-    unsafe { dealloc(ptr, layout) };
-
-    ret >= 0
-}
-
-impl RawFile {
-    pub fn new(file: File, direct_io: bool) -> Self {
-        // Assume no alignment restrictions if we aren't using O_DIRECT.
-        let mut alignment = 0;
-        if direct_io {
-            for align in &BLK_ALIGNMENTS {
-                if is_valid_alignment(file.as_raw_fd(), *align) {
-                    alignment = *align;
-                    break;
-                }
-            }
-        }
-        RawFile {
-            file,
-            alignment,
-            position: 0,
-            direct_io,
-        }
-    }
-
-    fn round_up(&self, offset: u64) -> u64 {
-        let align: u64 = self.alignment.try_into().unwrap();
-        offset.div_ceil(align) * align
-    }
-
-    fn round_down(&self, offset: u64) -> u64 {
-        let align: u64 = self.alignment.try_into().unwrap();
-        (offset / align) * align
-    }
-
-    fn is_aligned(&self, buf: &[u8]) -> bool {
-        if self.alignment == 0 {
-            return true;
-        }
-
-        let align64: u64 = self.alignment.try_into().unwrap();
-
-        self.position.is_multiple_of(align64)
-            && (buf.as_ptr() as usize).is_multiple_of(self.alignment)
-            && buf.len().is_multiple_of(self.alignment)
-    }
-
-    pub fn set_len(&self, size: u64) -> std::io::Result<()> {
-        self.file.set_len(size)
-    }
-
-    pub fn metadata(&self) -> std::io::Result<Metadata> {
-        self.file.metadata()
-    }
-
-    pub fn try_clone(&self) -> std::io::Result<RawFile> {
-        Ok(RawFile {
-            file: self.file.try_clone().expect("RawFile cloning failed"),
-            alignment: self.alignment,
-            position: self.position,
-            direct_io: self.direct_io,
-        })
-    }
-
-    pub fn sync_all(&self) -> std::io::Result<()> {
-        self.file.sync_all()
-    }
-
-    pub fn sync_data(&self) -> std::io::Result<()> {
-        self.file.sync_data()
-    }
-
-    pub fn is_direct(&self) -> bool {
-        self.direct_io
-    }
-
-    pub fn alignment(&self) -> usize {
-        self.alignment
-    }
-
-    /// Returns true if the file was opened with write access.
-    pub fn is_writable(&self) -> bool {
-        // SAFETY: fcntl with F_GETFL is safe and doesn't modify the file descriptor
-        let flags = unsafe { libc::fcntl(self.file.as_raw_fd(), libc::F_GETFL) };
-        if flags < 0 {
-            return false;
-        }
-        let access_mode = flags & libc::O_ACCMODE;
-        access_mode == libc::O_WRONLY || access_mode == libc::O_RDWR
-    }
-}
-
-impl Read for RawFile {
-    fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
-        if self.is_aligned(buf) {
-            match self.file.read(buf) {
-                Ok(r) => {
-                    self.position = self.position.checked_add(r.try_into().unwrap()).unwrap();
-                    Ok(r)
-                }
-                Err(e) => Err(e),
-            }
-        } else {
-            let rounded_pos: u64 = self.round_down(self.position);
-            let file_offset: usize = self
-                .position
-                .checked_sub(rounded_pos)
-                .unwrap()
-                .try_into()
-                .unwrap();
-            let buf_len: usize = buf.len();
-            let rounded_len: usize = self
-                .round_up(
-                    file_offset
-                        .checked_add(buf_len)
-                        .unwrap()
-                        .try_into()
-                        .unwrap(),
-                )
-                .try_into()
-                .unwrap();
-
-            let layout = Layout::from_size_align(rounded_len, self.alignment).unwrap();
-            // SAFETY: layout has non-zero size
-            let tmp_ptr = unsafe { alloc_zeroed(layout) };
-            if tmp_ptr.is_null() {
-                return Err(io::Error::last_os_error());
-            }
-
-            // SAFETY: tmp_ptr is valid and at least rounded_len long
-            let tmp_buf = unsafe { slice::from_raw_parts_mut(tmp_ptr, rounded_len) };
-
-            // This can eventually replaced with read_at once its interface
-            // has been stabilized.
-            // SAFETY: FFI call. All parameters are valid.
-            let ret = unsafe {
-                ::libc::pread64(
-                    self.file.as_raw_fd(),
-                    tmp_buf.as_mut_ptr().cast(),
-                    tmp_buf.len(),
-                    rounded_pos.try_into().unwrap(),
-                )
-            };
-            if ret < 0 {
-                // SAFETY: tmp_ptr was allocated by alloc_zeroed with layout
-                unsafe { dealloc(tmp_ptr, layout) };
-                return Err(io::Error::last_os_error());
-            }
-
-            let read: usize = ret.try_into().unwrap();
-            if read < file_offset {
-                // SAFETY: tmp_ptr was allocated by alloc_zeroed with layout
-                unsafe { dealloc(tmp_ptr, layout) };
-                return Ok(0);
-            }
-
-            let mut to_copy = read - file_offset;
-            if to_copy > buf_len {
-                to_copy = buf_len;
-            }
-
-            buf.copy_from_slice(&tmp_buf[file_offset..(file_offset + buf_len)]);
-            // SAFETY: tmp_ptr was allocated by alloc_zeroed with layout
-            unsafe { dealloc(tmp_ptr, layout) };
-
-            self.seek(SeekFrom::Current(to_copy.try_into().unwrap()))
-                .unwrap();
-            Ok(to_copy)
-        }
-    }
-}
-
-impl Write for RawFile {
-    fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
-        if self.is_aligned(buf) {
-            match self.file.write(buf) {
-                Ok(r) => {
-                    self.position = self.position.checked_add(r.try_into().unwrap()).unwrap();
-                    Ok(r)
-                }
-                Err(e) => Err(e),
-            }
-        } else {
-            let rounded_pos: u64 = self.round_down(self.position);
-            let file_offset: usize = self
-                .position
-                .checked_sub(rounded_pos)
-                .unwrap()
-                .try_into()
-                .unwrap();
-            let buf_len: usize = buf.len();
-            let rounded_len: usize = self
-                .round_up(
-                    file_offset
-                        .checked_add(buf_len)
-                        .unwrap()
-                        .try_into()
-                        .unwrap(),
-                )
-                .try_into()
-                .unwrap();
-
-            let layout = Layout::from_size_align(rounded_len, self.alignment).unwrap();
-            // SAFETY: layout has non-zero size
-            let tmp_ptr = unsafe { alloc_zeroed(layout) };
-            if tmp_ptr.is_null() {
-                return Err(io::Error::last_os_error());
-            }
-
-            // SAFETY: tmp_ptr is at least rounded_len long
-            let tmp_buf = unsafe { slice::from_raw_parts_mut(tmp_ptr, rounded_len) };
-
-            // This can eventually replaced with read_at once its interface
-            // has been stabilized.
-            // SAFETY: FFI call
-            let ret = unsafe {
-                ::libc::pread64(
-                    self.file.as_raw_fd(),
-                    tmp_buf.as_mut_ptr().cast(),
-                    tmp_buf.len(),
-                    rounded_pos.try_into().unwrap(),
-                )
-            };
-            if ret < 0 {
-                // SAFETY: tmp_ptr was allocated by alloc_zeroed with layout
-                unsafe { dealloc(tmp_ptr, layout) };
-                return Err(io::Error::last_os_error());
-            }
-
-            tmp_buf[file_offset..(file_offset + buf_len)].copy_from_slice(buf);
-
-            // This can eventually replaced with write_at once its interface
-            // has been stabilized.
-            // SAFETY: FFI call
-            let ret = unsafe {
-                ::libc::pwrite64(
-                    self.file.as_raw_fd(),
-                    tmp_buf.as_ptr().cast(),
-                    tmp_buf.len(),
-                    rounded_pos.try_into().unwrap(),
-                )
-            };
-
-            // SAFETY: tmp_ptr was allocated by alloc_zeroed with layout
-            unsafe { dealloc(tmp_ptr, layout) };
-
-            if ret < 0 {
-                return Err(io::Error::last_os_error());
-            }
-
-            let written: usize = ret.try_into().unwrap();
-            if written < file_offset {
-                Ok(0)
-            } else {
-                let mut to_seek = written - file_offset;
-                if to_seek > buf_len {
-                    to_seek = buf_len;
-                }
-
-                self.seek(SeekFrom::Current(to_seek.try_into().unwrap()))
-                    .unwrap();
-                Ok(to_seek)
-            }
-        }
-    }
-
-    fn flush(&mut self) -> std::io::Result<()> {
-        self.file.sync_all()
-    }
-}
-
-impl Seek for RawFile {
-    fn seek(&mut self, newpos: SeekFrom) -> std::io::Result<u64> {
-        match self.file.seek(newpos) {
-            Ok(pos) => {
-                self.position = pos;
-                Ok(pos)
-            }
-            Err(e) => Err(e),
-        }
-    }
-}
-
-impl WriteZeroesAt for RawFile {
-    fn write_zeroes_at(&mut self, offset: u64, length: usize) -> std::io::Result<usize> {
-        self.file.write_zeroes_at(offset, length)
-    }
-}
-
-impl PunchHole for RawFile {
-    fn punch_hole(&mut self, offset: u64, length: u64) -> std::io::Result<()> {
-        self.file.punch_hole(offset, length)
-    }
-}
-
-impl FileSync for RawFile {
-    fn fsync(&mut self) -> std::io::Result<()> {
-        self.file.fsync()
-    }
-}
-
-impl SeekHole for RawFile {
-    fn seek_hole(&mut self, offset: u64) -> std::io::Result<Option<u64>> {
-        match self.file.seek_hole(offset) {
-            Ok(pos) => {
-                if let Some(p) = pos {
-                    self.position = p;
-                }
-                Ok(pos)
-            }
-            Err(e) => Err(e),
-        }
-    }
-
-    fn seek_data(&mut self, offset: u64) -> std::io::Result<Option<u64>> {
-        match self.file.seek_data(offset) {
-            Ok(pos) => {
-                if let Some(p) = pos {
-                    self.position = p;
-                }
-                Ok(pos)
-            }
-            Err(e) => Err(e),
-        }
-    }
-}
-
-impl BlockBackend for RawFile {
-    fn logical_size(&self) -> std::result::Result<u64, crate::Error> {
-        Ok(query_device_size(&self.file)
-            .map_err(crate::Error::RawFileError)?
-            .0)
-    }
-
-    fn physical_size(&self) -> std::result::Result<u64, crate::Error> {
-        Ok(query_device_size(&self.file)
-            .map_err(crate::Error::RawFileError)?
-            .1)
-    }
-}
-
-impl Clone for RawFile {
-    fn clone(&self) -> Self {
-        RawFile {
-            file: self.file.try_clone().expect("RawFile cloning failed"),
-            alignment: self.alignment,
-            position: self.position,
-            direct_io: self.direct_io,
-        }
-    }
-}
-
-impl AsRawFd for RawFile {
-    fn as_raw_fd(&self) -> RawFd {
-        self.file.as_raw_fd()
-    }
-}
-
-impl AsFd for RawFile {
-    fn as_fd(&self) -> BorrowedFd<'_> {
-        self.file.as_fd()
-    }
-}

block/src/qcow_common.rs

@@ -1,446 +0,0 @@
-// Copyright © 2021 Intel Corporation
-//
-// Copyright 2026 The Cloud Hypervisor Authors. All rights reserved.
-//
-// SPDX-License-Identifier: Apache-2.0 AND BSD-3-Clause
-
-//! Shared helpers for QCOW2 sync and async backends.
-//!
-//! Position-independent I/O (`pread_exact`, `pwrite_all`) and iovec
-//! scatter/gather helpers used by both `qcow_sync` and `qcow_async`.
-
-use std::alloc::{Layout, alloc_zeroed, dealloc};
-use std::cmp::min;
-use std::os::fd::RawFd;
-use std::{io, ptr, slice};
-
-use crate::qcow::decoder::Decoder;
-
-// -- Position independent I/O helpers --
-//
-// Duplicated file descriptors share the kernel file description and thus the
-// file position. Using seek then read from multiple queues races on that
-// shared position. pread64 and pwrite64 are atomic and never touch the position.
-
-/// Read exactly the requested bytes at offset, looping on short reads.
-pub fn pread_exact(fd: RawFd, buf: &mut [u8], offset: u64) -> io::Result<()> {
-    let mut total = 0usize;
-    while total < buf.len() {
-        // SAFETY: buf and fd are valid for the lifetime of the call.
-        let ret = unsafe {
-            libc::pread64(
-                fd,
-                buf[total..].as_mut_ptr().cast(),
-                buf.len() - total,
-                (offset + total as u64) as libc::off_t,
-            )
-        };
-        if ret < 0 {
-            return Err(io::Error::last_os_error());
-        }
-        if ret == 0 {
-            return Err(io::Error::from(io::ErrorKind::UnexpectedEof));
-        }
-        total += ret as usize;
-    }
-    Ok(())
-}
-
-/// Allocate a buffer and pread exactly `len` bytes at `offset`.
-pub fn pread_alloc(fd: RawFd, offset: u64, len: usize) -> io::Result<Vec<u8>> {
-    let mut buf = vec![0u8; len];
-    pread_exact(fd, &mut buf, offset)?;
-    Ok(buf)
-}
-
-/// Decompress a full QCOW2 cluster from compressed data.
-///
-/// Returns a `cluster_size` byte buffer with the decompressed cluster
-/// content. Fails if the decoder does not produce exactly `cluster_size`
-/// bytes.
-pub fn decompress_cluster(
-    compressed: &[u8],
-    cluster_size: usize,
-    decoder: &dyn Decoder,
-) -> io::Result<Vec<u8>> {
-    let mut decompressed = vec![0u8; cluster_size];
-    let n = decoder
-        .decode(compressed, &mut decompressed)
-        .map_err(|_| io::Error::from_raw_os_error(libc::EIO))?;
-    if n != cluster_size {
-        return Err(io::Error::from_raw_os_error(libc::EIO));
-    }
-    Ok(decompressed)
-}
-
-/// Write all bytes to fd at offset, looping on short writes.
-pub fn pwrite_all(fd: RawFd, buf: &[u8], offset: u64) -> io::Result<()> {
-    let mut total = 0usize;
-    while total < buf.len() {
-        // SAFETY: buf and fd are valid for the lifetime of the call.
-        let ret = unsafe {
-            libc::pwrite64(
-                fd,
-                buf[total..].as_ptr().cast(),
-                buf.len() - total,
-                (offset + total as u64) as libc::off_t,
-            )
-        };
-        if ret < 0 {
-            return Err(io::Error::last_os_error());
-        }
-        if ret == 0 {
-            return Err(io::Error::other("pwrite64 wrote 0 bytes"));
-        }
-        total += ret as usize;
-    }
-    Ok(())
-}
-
-/// RAII wrapper for an aligned heap buffer required by O_DIRECT.
-pub struct AlignedBuf {
-    ptr: *mut u8,
-    layout: Layout,
-}
-
-impl AlignedBuf {
-    pub fn new(size: usize, alignment: usize) -> io::Result<Self> {
-        let size = size.max(1).next_multiple_of(alignment);
-        let layout = Layout::from_size_align(size, alignment)
-            .map_err(|e| io::Error::other(format!("invalid aligned layout: {e}")))?;
-        // SAFETY: layout has non-zero size.
-        let ptr = unsafe { alloc_zeroed(layout) };
-        if ptr.is_null() {
-            return Err(io::Error::new(
-                io::ErrorKind::OutOfMemory,
-                "aligned allocation failed",
-            ));
-        }
-        Ok(AlignedBuf { ptr, layout })
-    }
-
-    pub fn as_mut_slice(&mut self, len: usize) -> &mut [u8] {
-        let len = len.min(self.layout.size());
-        // SAFETY: ptr is valid for layout.size() bytes; len <= layout.size().
-        unsafe { slice::from_raw_parts_mut(self.ptr, len) }
-    }
-
-    pub fn as_slice(&self, len: usize) -> &[u8] {
-        let len = len.min(self.layout.size());
-        // SAFETY: ptr is valid for layout.size() bytes; len <= layout.size().
-        unsafe { slice::from_raw_parts(self.ptr, len) }
-    }
-
-    #[cfg(test)]
-    pub fn layout(&self) -> &Layout {
-        &self.layout
-    }
-
-    #[cfg(test)]
-    pub fn ptr(&self) -> *const u8 {
-        self.ptr
-    }
-}
-
-impl Drop for AlignedBuf {
-    fn drop(&mut self) {
-        // SAFETY: ptr was allocated by alloc_zeroed with self.layout.
-        unsafe { dealloc(self.ptr, self.layout) };
-    }
-}
-
-/// Read into `buf` via an aligned bounce buffer when O_DIRECT requires it.
-pub fn aligned_pread(fd: RawFd, buf: &mut [u8], offset: u64, alignment: usize) -> io::Result<()> {
-    if alignment == 0
-        || ((buf.as_ptr() as usize).is_multiple_of(alignment)
-            && buf.len().is_multiple_of(alignment)
-            && (offset as usize).is_multiple_of(alignment))
-    {
-        return pread_exact(fd, buf, offset);
-    }
-
-    let aligned_offset = offset & !(alignment as u64 - 1);
-    let head = (offset - aligned_offset) as usize;
-    let aligned_len = (head + buf.len()).next_multiple_of(alignment);
-    let mut bounce = AlignedBuf::new(aligned_len, alignment)?;
-    pread_exact(fd, bounce.as_mut_slice(aligned_len), aligned_offset)?;
-    buf.copy_from_slice(&bounce.as_slice(aligned_len)[head..head + buf.len()]);
-    Ok(())
-}
-
-/// Write `buf` via an aligned bounce buffer when O_DIRECT requires it.
-pub fn aligned_pwrite(fd: RawFd, buf: &[u8], offset: u64, alignment: usize) -> io::Result<()> {
-    if alignment == 0
-        || ((buf.as_ptr() as usize).is_multiple_of(alignment)
-            && buf.len().is_multiple_of(alignment)
-            && (offset as usize).is_multiple_of(alignment))
-    {
-        return pwrite_all(fd, buf, offset);
-    }
-
-    let aligned_offset = offset & !(alignment as u64 - 1);
-    let head = (offset - aligned_offset) as usize;
-    let aligned_len = (head + buf.len()).next_multiple_of(alignment);
-    let mut bounce = AlignedBuf::new(aligned_len, alignment)?;
-
-    // Read-modify-write: read the existing aligned region, overlay our data.
-    pread_exact(fd, bounce.as_mut_slice(aligned_len), aligned_offset)?;
-    bounce.as_mut_slice(aligned_len)[head..head + buf.len()].copy_from_slice(buf);
-    pwrite_all(fd, bounce.as_slice(aligned_len), aligned_offset)
-}
-
-// -- iovec helper functions --
-//
-// Operate on the iovec array as a flat byte stream.
-
-/// Copy data into iovecs starting at the given byte offset.
-///
-/// # Safety
-/// Caller must ensure iovecs point to valid, writable memory of sufficient size.
-pub unsafe fn scatter_to_iovecs(iovecs: &[libc::iovec], start: usize, data: &[u8]) {
-    let mut remaining = data;
-    let mut pos = 0usize;
-    for iov in iovecs {
-        let iov_end = pos + iov.iov_len;
-        if iov_end <= start || remaining.is_empty() {
-            pos = iov_end;
-            continue;
-        }
-        let iov_start = start.saturating_sub(pos);
-        let available = iov.iov_len - iov_start;
-        let count = min(available, remaining.len());
-        // SAFETY: iov_base is valid for iov_len bytes per caller contract.
-        unsafe {
-            let dst = iov.iov_base.cast::<u8>().add(iov_start);
-            ptr::copy_nonoverlapping(remaining.as_ptr(), dst, count);
-        }
-        remaining = &remaining[count..];
-        if remaining.is_empty() {
-            break;
-        }
-        pos = iov_end;
-    }
-}
-
-/// Zero fill iovecs starting at the given byte offset for the given length.
-///
-/// # Safety
-/// Caller must ensure iovecs point to valid, writable memory of sufficient size.
-pub unsafe fn zero_fill_iovecs(iovecs: &[libc::iovec], start: usize, len: usize) {
-    let mut remaining = len;
-    let mut pos = 0usize;
-    for iov in iovecs {
-        let iov_end = pos + iov.iov_len;
-        if iov_end <= start || remaining == 0 {
-            pos = iov_end;
-            continue;
-        }
-        let iov_start = start.saturating_sub(pos);
-        let available = iov.iov_len - iov_start;
-        let count = min(available, remaining);
-        // SAFETY: iov_base is valid for iov_len bytes per caller contract.
-        unsafe {
-            let dst = iov.iov_base.cast::<u8>().add(iov_start);
-            ptr::write_bytes(dst, 0, count);
-        }
-        remaining -= count;
-        if remaining == 0 {
-            break;
-        }
-        pos = iov_end;
-    }
-}
-
-/// Gather bytes from iovecs starting at the given byte offset into `dst`.
-///
-/// # Safety
-/// Caller must ensure iovecs point to valid, readable memory of sufficient size.
-pub unsafe fn gather_from_iovecs_into(iovecs: &[libc::iovec], start: usize, dst: &mut [u8]) {
-    let len = dst.len();
-    let mut written = 0usize;
-    let mut pos = 0usize;
-    for iov in iovecs {
-        let iov_end = pos + iov.iov_len;
-        if iov_end <= start || written == len {
-            pos = iov_end;
-            continue;
-        }
-        let iov_start = start.saturating_sub(pos);
-        let available = iov.iov_len - iov_start;
-        let count = min(available, len - written);
-        // SAFETY: iov_base is valid for iov_len bytes per caller contract.
-        unsafe {
-            let src = iov.iov_base.cast::<u8>().add(iov_start);
-            ptr::copy_nonoverlapping(src, dst.as_mut_ptr().add(written), count);
-        }
-        written += count;
-        if written == len {
-            break;
-        }
-        pos = iov_end;
-    }
-}
-
-/// Gather bytes from iovecs starting at the given byte offset into a Vec.
-///
-/// # Safety
-/// Caller must ensure iovecs point to valid, readable memory of sufficient size.
-pub unsafe fn gather_from_iovecs(iovecs: &[libc::iovec], start: usize, len: usize) -> Vec<u8> {
-    let mut result = vec![0u8; len];
-    // SAFETY: caller guarantees iovecs are valid; result has len bytes.
-    unsafe { gather_from_iovecs_into(iovecs, start, &mut result) };
-    result
-}
-
-#[cfg(test)]
-pub(crate) mod unit_tests {
-    use std::fs::File;
-    use std::io::{Read, Seek, SeekFrom, Write};
-    use std::os::unix::fs::FileExt;
-    use std::os::unix::io::AsRawFd;
-
-    use byteorder::{BigEndian, ReadBytesExt, WriteBytesExt};
-    use flate2::Compression;
-    use flate2::write::DeflateEncoder;
-    use vmm_sys_util::tempfile::TempFile;
-
-    use super::{decompress_cluster, pread_alloc};
-    use crate::qcow::decoder::ZlibDecoder;
-
-    const COMPRESSED_FLAG: u64 = 1 << 62;
-    const CLUSTER_USED_FLAG: u64 = 1 << 63;
-    const COMPRESSED_SECTOR_SIZE: u64 = 512;
-
-    const HEADER_CLUSTER_BITS_OFFSET: u64 = 20;
-    const HEADER_L1_SIZE_OFFSET: u64 = 36;
-    const HEADER_L1_TABLE_OFFSET: u64 = 40;
-
-    const L1_L2_ADDR_MASK: u64 = 0x00ff_ffff_ffff_fe00;
-
-    fn make_compressed_l2_entry(host_offset: u64, compressed_len: usize, cluster_bits: u32) -> u64 {
-        let compressed_size_shift = 62 - (cluster_bits - 8);
-        let intra_sector_offset = host_offset & (COMPRESSED_SECTOR_SIZE - 1);
-        let total_bytes = compressed_len as u64 + intra_sector_offset;
-        let nsectors = total_bytes.div_ceil(COMPRESSED_SECTOR_SIZE);
-        let addr_part = host_offset & ((1 << compressed_size_shift) - 1);
-        let size_part = (nsectors - 1) << compressed_size_shift;
-        COMPRESSED_FLAG | size_part | addr_part
-    }
-
-    /// Compress every allocated cluster in a QCOW2 image file in place.
-    ///
-    /// Walks L1 -> L2 tables, compresses each standard cluster with raw
-    /// deflate, appends the compressed payload at the end of the file,
-    /// and rewrites the L2 entry with the compressed layout.
-    pub fn compress_allocated_clusters(file: &mut File) {
-        file.seek(SeekFrom::Start(HEADER_CLUSTER_BITS_OFFSET))
-            .unwrap();
-        let cluster_bits = file.read_u32::<BigEndian>().unwrap();
-        let cluster_size = 1u64 << cluster_bits;
-
-        file.seek(SeekFrom::Start(HEADER_L1_SIZE_OFFSET)).unwrap();
-        let l1_size = file.read_u32::<BigEndian>().unwrap();
-
-        file.seek(SeekFrom::Start(HEADER_L1_TABLE_OFFSET)).unwrap();
-        let l1_table_offset = file.read_u64::<BigEndian>().unwrap();
-
-        let entries_per_l2 = cluster_size / 8;
-
-        let mut append_offset = file.seek(SeekFrom::End(0)).unwrap();
-        append_offset = (append_offset + 511) & !511;
-
-        for l1_idx in 0..l1_size as u64 {
-            let l1_entry_offset = l1_table_offset + l1_idx * 8;
-            file.seek(SeekFrom::Start(l1_entry_offset)).unwrap();
-            let l1_entry = file.read_u64::<BigEndian>().unwrap();
-
-            let l2_table_addr = l1_entry & L1_L2_ADDR_MASK;
-            if l2_table_addr == 0 {
-                continue;
-            }
-
-            for l2_idx in 0..entries_per_l2 {
-                let l2_entry_offset = l2_table_addr + l2_idx * 8;
-                file.seek(SeekFrom::Start(l2_entry_offset)).unwrap();
-                let l2_entry = file.read_u64::<BigEndian>().unwrap();
-
-                if l2_entry & CLUSTER_USED_FLAG == 0 || l2_entry & COMPRESSED_FLAG != 0 {
-                    continue;
-                }
-
-                let host_cluster_addr = l2_entry & L1_L2_ADDR_MASK;
-                if host_cluster_addr == 0 {
-                    continue;
-                }
-
-                let mut cluster_data = vec![0u8; cluster_size as usize];
-                file.seek(SeekFrom::Start(host_cluster_addr)).unwrap();
-                file.read_exact(&mut cluster_data).unwrap();
-
-                let mut encoder = DeflateEncoder::new(Vec::new(), Compression::default());
-                encoder.write_all(&cluster_data).unwrap();
-                let compressed = encoder.finish().unwrap();
-
-                file.seek(SeekFrom::Start(append_offset)).unwrap();
-                file.write_all(&compressed).unwrap();
-
-                // The L2 entry encodes the compressed size in units of
-                // 512 byte sectors. The reader decodes the sector count
-                // back and computes: nsectors * 512 - (addr & 511).
-                // Because addr is 512 aligned, this yields nsectors * 512
-                // which rounds up to the next sector boundary. The file
-                // must contain enough bytes for that rounded up pread.
-                let padded_len = (compressed.len() + 511) & !511;
-                if padded_len > compressed.len() {
-                    let padding = vec![0u8; padded_len - compressed.len()];
-                    file.write_all(&padding).unwrap();
-                }
-
-                let new_entry =
-                    make_compressed_l2_entry(append_offset, compressed.len(), cluster_bits);
-                file.seek(SeekFrom::Start(l2_entry_offset)).unwrap();
-                file.write_u64::<BigEndian>(new_entry).unwrap();
-
-                append_offset += padded_len as u64;
-            }
-        }
-
-        file.flush().unwrap();
-    }
-
-    #[test]
-    fn test_pread_alloc() {
-        let temp = TempFile::new().unwrap();
-        let file = temp.as_file();
-        let data: Vec<u8> = (0..=255).cycle().take(4096).collect();
-        file.write_all_at(&data, 0).unwrap();
-
-        let buf = pread_alloc(file.as_raw_fd(), 0, 4096).unwrap();
-        assert_eq!(buf, data);
-
-        let buf = pread_alloc(file.as_raw_fd(), 100, 200).unwrap();
-        assert_eq!(buf, &data[100..300]);
-
-        pread_alloc(file.as_raw_fd(), 4000, 200).unwrap_err();
-    }
-
-    #[test]
-    fn test_decompress_cluster() {
-        let cluster_size = 65536;
-        let original: Vec<u8> = (0..=255).cycle().take(cluster_size).collect();
-
-        let mut encoder = DeflateEncoder::new(Vec::new(), Compression::default());
-        encoder.write_all(&original).unwrap();
-        let compressed = encoder.finish().unwrap();
-
-        let result = decompress_cluster(&compressed, cluster_size, &ZlibDecoder {}).unwrap();
-        assert_eq!(result, original);
-    }
-
-    #[test]
-    fn test_decompress_cluster_corrupt_input() {
-        let corrupt = vec![0xffu8; 64];
-        let err = decompress_cluster(&corrupt, 65536, &ZlibDecoder {}).unwrap_err();
-        assert_eq!(err.raw_os_error(), Some(libc::EIO));
-    }
-}

block/src/raw_async.rs

@@ -1,286 +0,0 @@
-// Copyright © 2021 Intel Corporation
-//
-// SPDX-License-Identifier: Apache-2.0 AND BSD-3-Clause
-
-use std::io::Error;
-use std::os::unix::io::{AsRawFd, RawFd};
-
-use io_uring::{IoUring, opcode, types};
-use libc::{FALLOC_FL_KEEP_SIZE, FALLOC_FL_PUNCH_HOLE, FALLOC_FL_ZERO_RANGE};
-use vmm_sys_util::eventfd::EventFd;
-
-use crate::async_io::{AsyncIo, AsyncIoError, AsyncIoResult};
-use crate::error::{BlockError, BlockErrorKind, BlockResult};
-use crate::{BatchRequest, RequestType, SECTOR_SIZE};
-
-pub struct RawFileAsync {
-    fd: RawFd,
-    io_uring: IoUring,
-    eventfd: EventFd,
-    alignment: u64,
-}
-
-impl RawFileAsync {
-    pub fn new(fd: RawFd, ring_depth: u32) -> BlockResult<Self> {
-        let io_uring =
-            IoUring::new(ring_depth).map_err(|e| BlockError::new(BlockErrorKind::Io, e))?;
-        let eventfd =
-            EventFd::new(libc::EFD_NONBLOCK).map_err(|e| BlockError::new(BlockErrorKind::Io, e))?;
-
-        // Register the io_uring eventfd that will notify when something in
-        // the completion queue is ready.
-        io_uring
-            .submitter()
-            .register_eventfd(eventfd.as_raw_fd())
-            .map_err(|e| BlockError::new(BlockErrorKind::Io, e))?;
-
-        Ok(RawFileAsync {
-            fd,
-            io_uring,
-            eventfd,
-            alignment: SECTOR_SIZE,
-        })
-    }
-}
-
-impl AsyncIo for RawFileAsync {
-    fn notifier(&self) -> &EventFd {
-        &self.eventfd
-    }
-
-    fn alignment(&self) -> u64 {
-        self.alignment
-    }
-
-    fn read_vectored(
-        &mut self,
-        offset: libc::off_t,
-        iovecs: &[libc::iovec],
-        user_data: u64,
-    ) -> AsyncIoResult<()> {
-        let (submitter, mut sq, _) = self.io_uring.split();
-
-        // SAFETY: we know the file descriptor is valid and we
-        // relied on vm-memory to provide the buffer address.
-        unsafe {
-            sq.push(
-                &opcode::Readv::new(types::Fd(self.fd), iovecs.as_ptr(), iovecs.len() as u32)
-                    .offset(offset.try_into().unwrap())
-                    .build()
-                    .user_data(user_data),
-            )
-            .map_err(|e| {
-                AsyncIoError::ReadVectored(Error::other(format!("Submission queue is full: {e:?}")))
-            })?;
-        };
-
-        // Update the submission queue and submit new operations to the
-        // io_uring instance.
-        sq.sync();
-        submitter.submit().map_err(AsyncIoError::ReadVectored)?;
-
-        Ok(())
-    }
-
-    fn write_vectored(
-        &mut self,
-        offset: libc::off_t,
-        iovecs: &[libc::iovec],
-        user_data: u64,
-    ) -> AsyncIoResult<()> {
-        let (submitter, mut sq, _) = self.io_uring.split();
-
-        // SAFETY: we know the file descriptor is valid and we
-        // relied on vm-memory to provide the buffer address.
-        unsafe {
-            sq.push(
-                &opcode::Writev::new(types::Fd(self.fd), iovecs.as_ptr(), iovecs.len() as u32)
-                    .offset(offset.try_into().unwrap())
-                    .build()
-                    .user_data(user_data),
-            )
-            .map_err(|e| {
-                AsyncIoError::WriteVectored(Error::other(format!(
-                    "Submission queue is full: {e:?}"
-                )))
-            })?;
-        };
-
-        // Update the submission queue and submit new operations to the
-        // io_uring instance.
-        sq.sync();
-        submitter.submit().map_err(AsyncIoError::WriteVectored)?;
-
-        Ok(())
-    }
-
-    fn fsync(&mut self, user_data: Option<u64>) -> AsyncIoResult<()> {
-        if let Some(user_data) = user_data {
-            let (submitter, mut sq, _) = self.io_uring.split();
-
-            // SAFETY: we know the file descriptor is valid.
-            unsafe {
-                sq.push(
-                    &opcode::Fsync::new(types::Fd(self.fd))
-                        .build()
-                        .user_data(user_data),
-                )
-                .map_err(|e| {
-                    AsyncIoError::Fsync(Error::other(format!("Submission queue is full: {e:?}")))
-                })?;
-            };
-
-            // Update the submission queue and submit new operations to the
-            // io_uring instance.
-            sq.sync();
-            submitter.submit().map_err(AsyncIoError::Fsync)?;
-        } else {
-            // SAFETY: FFI call with a valid fd
-            unsafe { libc::fsync(self.fd) };
-        }
-
-        Ok(())
-    }
-
-    fn next_completed_request(&mut self) -> Option<(u64, i32)> {
-        self.io_uring
-            .completion()
-            .next()
-            .map(|entry| (entry.user_data(), entry.result()))
-    }
-
-    fn batch_requests_enabled(&self) -> bool {
-        true
-    }
-
-    fn submit_batch_requests(&mut self, batch_request: &[BatchRequest]) -> AsyncIoResult<()> {
-        if !self.batch_requests_enabled() {
-            return Ok(());
-        }
-
-        let (submitter, mut sq, _) = self.io_uring.split();
-        let mut submitted = false;
-
-        // Refuse the whole batch if it can't fit in the SQ to avoid having to unroll a partially
-        // successful push.
-        if batch_request.len() > sq.capacity() - sq.len() {
-            return Err(AsyncIoError::SubmitBatchRequests(Error::other(
-                "io_uring submission queue is full",
-            )));
-        }
-
-        for req in batch_request {
-            match req.request_type {
-                RequestType::In => {
-                    // SAFETY: we know the file descriptor is valid and we
-                    // relied on vm-memory to provide the buffer address.
-                    unsafe {
-                        sq.push(
-                            &opcode::Readv::new(
-                                types::Fd(self.fd),
-                                req.iovecs.as_ptr(),
-                                req.iovecs.len() as u32,
-                            )
-                            .offset(req.offset as u64)
-                            .build()
-                            .user_data(req.user_data),
-                        )
-                        .map_err(|e| {
-                            AsyncIoError::ReadVectored(Error::other(format!(
-                                "Submission queue is full: {e:?}"
-                            )))
-                        })?;
-                    };
-                    submitted = true;
-                }
-                RequestType::Out => {
-                    // SAFETY: we know the file descriptor is valid and we
-                    // relied on vm-memory to provide the buffer address.
-                    unsafe {
-                        sq.push(
-                            &opcode::Writev::new(
-                                types::Fd(self.fd),
-                                req.iovecs.as_ptr(),
-                                req.iovecs.len() as u32,
-                            )
-                            .offset(req.offset as u64)
-                            .build()
-                            .user_data(req.user_data),
-                        )
-                        .map_err(|e| {
-                            AsyncIoError::WriteVectored(Error::other(format!(
-                                "Submission queue is full: {e:?}"
-                            )))
-                        })?;
-                    };
-                    submitted = true;
-                }
-                _ => {
-                    unreachable!("Unexpected batch request type: {:?}", req.request_type)
-                }
-            }
-        }
-
-        // Only submit if we actually queued something
-        if submitted {
-            // Update the submission queue and submit new operations to the
-            // io_uring instance.
-            sq.sync();
-            submitter
-                .submit()
-                .map_err(AsyncIoError::SubmitBatchRequests)?;
-        }
-
-        Ok(())
-    }
-
-    fn punch_hole(&mut self, offset: u64, length: u64, user_data: u64) -> AsyncIoResult<()> {
-        let (submitter, mut sq, _) = self.io_uring.split();
-
-        let mode = FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE;
-
-        // SAFETY: The file descriptor is known to be valid.
-        unsafe {
-            sq.push(
-                &opcode::Fallocate::new(types::Fd(self.fd), length)
-                    .offset(offset)
-                    .mode(mode)
-                    .build()
-                    .user_data(user_data),
-            )
-            .map_err(|e| {
-                AsyncIoError::PunchHole(Error::other(format!("Submission queue is full: {e:?}")))
-            })?;
-        };
-
-        sq.sync();
-        submitter.submit().map_err(AsyncIoError::PunchHole)?;
-
-        Ok(())
-    }
-
-    fn write_zeroes(&mut self, offset: u64, length: u64, user_data: u64) -> AsyncIoResult<()> {
-        let (submitter, mut sq, _) = self.io_uring.split();
-
-        let mode = FALLOC_FL_ZERO_RANGE | FALLOC_FL_KEEP_SIZE;
-
-        // SAFETY: The file descriptor is known to be valid.
-        unsafe {
-            sq.push(
-                &opcode::Fallocate::new(types::Fd(self.fd), length)
-                    .offset(offset)
-                    .mode(mode)
-                    .build()
-                    .user_data(user_data),
-            )
-            .map_err(|e| {
-                AsyncIoError::WriteZeroes(Error::other(format!("Submission queue is full: {e:?}")))
-            })?;
-        };
-
-        sq.sync();
-        submitter.submit().map_err(AsyncIoError::WriteZeroes)?;
-
-        Ok(())
-    }
-}

block/src/raw_async_aio.rs

@@ -1,218 +0,0 @@
-// Copyright © 2023 Intel Corporation
-//
-// SPDX-License-Identifier: Apache-2.0 AND BSD-3-Clause
-//
-// Copyright © 2023 Crusoe Energy Systems LLC
-//
-
-use std::collections::VecDeque;
-use std::os::unix::io::{AsRawFd, RawFd};
-
-use libc::{FALLOC_FL_KEEP_SIZE, FALLOC_FL_PUNCH_HOLE, FALLOC_FL_ZERO_RANGE};
-use vmm_sys_util::aio;
-use vmm_sys_util::eventfd::EventFd;
-
-use crate::SECTOR_SIZE;
-use crate::async_io::{AsyncIo, AsyncIoError, AsyncIoResult};
-use crate::error::{BlockError, BlockErrorKind, BlockResult};
-
-pub struct RawFileAsyncAio {
-    fd: RawFd,
-    ctx: aio::IoContext,
-    eventfd: EventFd,
-    alignment: u64,
-    completion_list: VecDeque<(u64, i32)>,
-}
-
-impl RawFileAsyncAio {
-    pub fn new(fd: RawFd, queue_depth: u32) -> BlockResult<Self> {
-        let eventfd =
-            EventFd::new(libc::EFD_NONBLOCK).map_err(|e| BlockError::new(BlockErrorKind::Io, e))?;
-        let ctx =
-            aio::IoContext::new(queue_depth).map_err(|e| BlockError::new(BlockErrorKind::Io, e))?;
-
-        Ok(RawFileAsyncAio {
-            fd,
-            ctx,
-            eventfd,
-            alignment: SECTOR_SIZE,
-            completion_list: VecDeque::new(),
-        })
-    }
-}
-
-impl AsyncIo for RawFileAsyncAio {
-    fn notifier(&self) -> &EventFd {
-        &self.eventfd
-    }
-
-    fn alignment(&self) -> u64 {
-        self.alignment
-    }
-
-    fn read_vectored(
-        &mut self,
-        offset: libc::off_t,
-        iovecs: &[libc::iovec],
-        user_data: u64,
-    ) -> AsyncIoResult<()> {
-        let iocbs = [&mut aio::IoControlBlock {
-            aio_fildes: self.fd.as_raw_fd() as u32,
-            aio_lio_opcode: aio::IOCB_CMD_PREADV as u16,
-            aio_buf: iovecs.as_ptr() as u64,
-            aio_nbytes: iovecs.len() as u64,
-            aio_offset: offset,
-            aio_data: user_data,
-            aio_flags: aio::IOCB_FLAG_RESFD,
-            aio_resfd: self.eventfd.as_raw_fd() as u32,
-            ..Default::default()
-        }];
-        let _ = self
-            .ctx
-            .submit(&iocbs[..])
-            .map_err(AsyncIoError::ReadVectored)?;
-
-        Ok(())
-    }
-
-    fn write_vectored(
-        &mut self,
-        offset: libc::off_t,
-        iovecs: &[libc::iovec],
-        user_data: u64,
-    ) -> AsyncIoResult<()> {
-        let iocbs = [&mut aio::IoControlBlock {
-            aio_fildes: self.fd.as_raw_fd() as u32,
-            aio_lio_opcode: aio::IOCB_CMD_PWRITEV as u16,
-            aio_buf: iovecs.as_ptr() as u64,
-            aio_nbytes: iovecs.len() as u64,
-            aio_offset: offset,
-            aio_data: user_data,
-            aio_flags: aio::IOCB_FLAG_RESFD,
-            aio_resfd: self.eventfd.as_raw_fd() as u32,
-            ..Default::default()
-        }];
-        let _ = self
-            .ctx
-            .submit(&iocbs[..])
-            .map_err(AsyncIoError::WriteVectored)?;
-
-        Ok(())
-    }
-
-    fn fsync(&mut self, user_data: Option<u64>) -> AsyncIoResult<()> {
-        if let Some(user_data) = user_data {
-            let iocbs = [&mut aio::IoControlBlock {
-                aio_fildes: self.fd.as_raw_fd() as u32,
-                aio_lio_opcode: aio::IOCB_CMD_FSYNC as u16,
-                aio_data: user_data,
-                aio_flags: aio::IOCB_FLAG_RESFD,
-                aio_resfd: self.eventfd.as_raw_fd() as u32,
-                ..Default::default()
-            }];
-            let _ = self.ctx.submit(&iocbs[..]).map_err(AsyncIoError::Fsync)?;
-        } else {
-            // SAFETY: FFI call with a valid fd
-            unsafe { libc::fsync(self.fd) };
-        }
-
-        Ok(())
-    }
-
-    fn next_completed_request(&mut self) -> Option<(u64, i32)> {
-        if self.completion_list.is_empty() {
-            // Drain pending AIO completions batched into the same queue.
-            let mut events = [aio::IoEvent::default(); 32];
-            let rc = self.ctx.get_events(0, &mut events, None).unwrap();
-            for event in &events[..rc] {
-                self.completion_list
-                    .push_back((event.data, event.res as i32));
-            }
-        }
-        self.completion_list.pop_front()
-    }
-
-    fn punch_hole(&mut self, offset: u64, length: u64, user_data: u64) -> AsyncIoResult<()> {
-        // Linux AIO has no IOCB command for fallocate, so perform the operation
-        // synchronously and signal completion via the completion list, matching
-        // the pattern used by the sync backend (RawFileSync).
-        let mode = FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE;
-
-        // SAFETY: FFI call with valid arguments
-        let result = unsafe {
-            libc::fallocate(
-                self.fd as libc::c_int,
-                mode,
-                offset as libc::off_t,
-                length as libc::off_t,
-            )
-        };
-        if result < 0 {
-            return Err(AsyncIoError::PunchHole(std::io::Error::last_os_error()));
-        }
-
-        self.completion_list.push_back((user_data, result));
-        self.eventfd.write(1).unwrap();
-
-        Ok(())
-    }
-
-    fn write_zeroes(&mut self, offset: u64, length: u64, user_data: u64) -> AsyncIoResult<()> {
-        // Linux AIO has no IOCB command for fallocate, so perform the operation
-        // synchronously and signal completion via the completion list, matching
-        // the pattern used by the sync backend (RawFileSync).
-        let mode = FALLOC_FL_ZERO_RANGE | FALLOC_FL_KEEP_SIZE;
-
-        // SAFETY: FFI call with valid arguments
-        let result = unsafe {
-            libc::fallocate(
-                self.fd as libc::c_int,
-                mode,
-                offset as libc::off_t,
-                length as libc::off_t,
-            )
-        };
-        if result < 0 {
-            return Err(AsyncIoError::WriteZeroes(std::io::Error::last_os_error()));
-        }
-
-        self.completion_list.push_back((user_data, result));
-        self.eventfd.write(1).unwrap();
-
-        Ok(())
-    }
-}
-
-#[cfg(test)]
-mod unit_tests {
-    use std::os::unix::io::AsRawFd;
-
-    use vmm_sys_util::tempfile::TempFile;
-
-    use super::*;
-    use crate::raw_async_io_tests;
-
-    #[test]
-    fn test_punch_hole() {
-        let temp_file = TempFile::new().unwrap();
-        let mut file = temp_file.into_file();
-        let mut async_io = RawFileAsyncAio::new(file.as_raw_fd(), 128).unwrap();
-        raw_async_io_tests::test_punch_hole(&mut async_io, &mut file);
-    }
-
-    #[test]
-    fn test_write_zeroes() {
-        let temp_file = TempFile::new().unwrap();
-        let mut file = temp_file.into_file();
-        let mut async_io = RawFileAsyncAio::new(file.as_raw_fd(), 128).unwrap();
-        raw_async_io_tests::test_write_zeroes(&mut async_io, &mut file);
-    }
-
-    #[test]
-    fn test_punch_hole_multiple_operations() {
-        let temp_file = TempFile::new().unwrap();
-        let mut file = temp_file.into_file();
-        let mut async_io = RawFileAsyncAio::new(file.as_raw_fd(), 128).unwrap();
-        raw_async_io_tests::test_punch_hole_multiple_operations(&mut async_io, &mut file);
-    }
-}

block/src/raw_sync.rs

@@ -1,187 +0,0 @@
-// Copyright © 2021 Intel Corporation
-//
-// SPDX-License-Identifier: Apache-2.0 AND BSD-3-Clause
-
-use std::collections::VecDeque;
-use std::os::unix::io::RawFd;
-
-use libc::{FALLOC_FL_KEEP_SIZE, FALLOC_FL_PUNCH_HOLE, FALLOC_FL_ZERO_RANGE};
-use vmm_sys_util::eventfd::EventFd;
-
-use crate::SECTOR_SIZE;
-use crate::async_io::{AsyncIo, AsyncIoError, AsyncIoResult};
-
-pub struct RawFileSync {
-    fd: RawFd,
-    eventfd: EventFd,
-    completion_list: VecDeque<(u64, i32)>,
-    alignment: u64,
-}
-
-impl RawFileSync {
-    pub fn new(fd: RawFd) -> Self {
-        RawFileSync {
-            fd,
-            eventfd: EventFd::new(libc::EFD_NONBLOCK).expect("Failed creating EventFd for RawFile"),
-            completion_list: VecDeque::new(),
-            alignment: SECTOR_SIZE,
-        }
-    }
-}
-
-impl AsyncIo for RawFileSync {
-    fn notifier(&self) -> &EventFd {
-        &self.eventfd
-    }
-
-    fn alignment(&self) -> u64 {
-        self.alignment
-    }
-
-    fn read_vectored(
-        &mut self,
-        offset: libc::off_t,
-        iovecs: &[libc::iovec],
-        user_data: u64,
-    ) -> AsyncIoResult<()> {
-        // SAFETY: FFI call with valid arguments
-        let result = unsafe {
-            libc::preadv(
-                self.fd as libc::c_int,
-                iovecs.as_ptr(),
-                iovecs.len() as libc::c_int,
-                offset,
-            )
-        };
-        if result < 0 {
-            return Err(AsyncIoError::ReadVectored(std::io::Error::last_os_error()));
-        }
-
-        self.completion_list.push_back((user_data, result as i32));
-        self.eventfd.write(1).unwrap();
-
-        Ok(())
-    }
-
-    fn write_vectored(
-        &mut self,
-        offset: libc::off_t,
-        iovecs: &[libc::iovec],
-        user_data: u64,
-    ) -> AsyncIoResult<()> {
-        // SAFETY: FFI call with valid arguments
-        let result = unsafe {
-            libc::pwritev(
-                self.fd as libc::c_int,
-                iovecs.as_ptr(),
-                iovecs.len() as libc::c_int,
-                offset,
-            )
-        };
-        if result < 0 {
-            return Err(AsyncIoError::WriteVectored(std::io::Error::last_os_error()));
-        }
-
-        self.completion_list.push_back((user_data, result as i32));
-        self.eventfd.write(1).unwrap();
-
-        Ok(())
-    }
-
-    fn fsync(&mut self, user_data: Option<u64>) -> AsyncIoResult<()> {
-        // SAFETY: FFI call
-        let result = unsafe { libc::fsync(self.fd as libc::c_int) };
-        if result < 0 {
-            return Err(AsyncIoError::Fsync(std::io::Error::last_os_error()));
-        }
-
-        if let Some(user_data) = user_data {
-            self.completion_list.push_back((user_data, result));
-            self.eventfd.write(1).unwrap();
-        }
-
-        Ok(())
-    }
-
-    fn next_completed_request(&mut self) -> Option<(u64, i32)> {
-        self.completion_list.pop_front()
-    }
-
-    fn punch_hole(&mut self, offset: u64, length: u64, user_data: u64) -> AsyncIoResult<()> {
-        let mode = FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE;
-
-        // SAFETY: FFI call with valid arguments
-        let result = unsafe {
-            libc::fallocate(
-                self.fd as libc::c_int,
-                mode,
-                offset as libc::off_t,
-                length as libc::off_t,
-            )
-        };
-        if result < 0 {
-            return Err(AsyncIoError::PunchHole(std::io::Error::last_os_error()));
-        }
-
-        self.completion_list.push_back((user_data, result));
-        self.eventfd.write(1).unwrap();
-
-        Ok(())
-    }
-
-    fn write_zeroes(&mut self, offset: u64, length: u64, user_data: u64) -> AsyncIoResult<()> {
-        let mode = FALLOC_FL_ZERO_RANGE | FALLOC_FL_KEEP_SIZE;
-
-        // SAFETY: FFI call with valid arguments
-        let result = unsafe {
-            libc::fallocate(
-                self.fd as libc::c_int,
-                mode,
-                offset as libc::off_t,
-                length as libc::off_t,
-            )
-        };
-        if result < 0 {
-            return Err(AsyncIoError::WriteZeroes(std::io::Error::last_os_error()));
-        }
-
-        self.completion_list.push_back((user_data, result));
-        self.eventfd.write(1).unwrap();
-
-        Ok(())
-    }
-}
-
-#[cfg(test)]
-mod unit_tests {
-    use std::os::unix::io::AsRawFd;
-
-    use vmm_sys_util::tempfile::TempFile;
-
-    use super::*;
-    use crate::raw_async_io_tests;
-
-    #[test]
-    fn test_punch_hole() {
-        let temp_file = TempFile::new().unwrap();
-        let mut file = temp_file.into_file();
-        let mut async_io = RawFileSync::new(file.as_raw_fd());
-        raw_async_io_tests::test_punch_hole(&mut async_io, &mut file);
-    }
-
-    #[test]
-    fn test_write_zeroes() {
-        let temp_file = TempFile::new().unwrap();
-        let mut file = temp_file.into_file();
-        let mut async_io = RawFileSync::new(file.as_raw_fd());
-        raw_async_io_tests::test_write_zeroes(&mut async_io, &mut file);
-    }
-
-    #[test]
-    fn test_punch_hole_multiple_operations() {
-        let temp_file = TempFile::new().unwrap();
-        let mut file = temp_file.into_file();
-        let mut async_io = RawFileSync::new(file.as_raw_fd());
-        raw_async_io_tests::test_punch_hole_multiple_operations(&mut async_io, &mut file);
-    }
-}

block/src/sparse.rs

@@ -0,0 +1,104 @@
+// Copyright 2026 The Cloud Hypervisor Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0 AND BSD-3-Clause
+
+// Helpers for issuing `BLKDISCARD` / `BLKZEROOUT` ioctls on block devices,
+// and the `punch_hole` / `write_zeroes` dispatchers used by the raw I/O
+// backends.
+//
+// The kernel ioctl numbers and argument layout are stable userspace ABI
+// (see `include/uapi/linux/fs.h`):
+//
+// ```c
+// #define BLKDISCARD  _IO(0x12, 119)   /* arg: const __u64 range[2] = { start, len } */
+// #define BLKZEROOUT  _IO(0x12, 127)   /* arg: const __u64 range[2] = { start, len } */
+// ```
+//
+// The kernel does `copy_from_user(range, arg, sizeof(range))`, i.e. it reads
+// 16 bytes through the single pointer it is given, so we must pass a single
+// `__u64[2]` array rather than two separate `*const u64` pointers.
+
+use std::io;
+use std::os::unix::io::RawFd;
+
+use libc::{FALLOC_FL_KEEP_SIZE, FALLOC_FL_PUNCH_HOLE, FALLOC_FL_ZERO_RANGE};
+
+// `_IO(0x12, 119)` — issue a discard request to a block device.
+pub const BLKDISCARD: libc::c_ulong = 0x1277;
+// `_IO(0x12, 127)` — write zeroes to a range of a block device, with a
+// kernel-side fallback to writing zero pages when the hardware has no native
+// `WRITE_ZEROES`.
+pub const BLKZEROOUT: libc::c_ulong = 0x127f;
+
+// Issue a `BLK*` range ioctl with proper `[start, len]` argument.
+fn blk_range_ioctl(fd: RawFd, request: libc::c_ulong, offset: u64, length: u64) -> io::Result<()> {
+    let range: [u64; 2] = [offset, length];
+    // SAFETY: `fd` is a valid block-device fd owned by the caller; `&range`
+    // is a 16-byte array matching the kernel's expected `__u64[2]` layout
+    // and lives for the duration of the call.
+    let ret = unsafe { libc::ioctl(fd, request as _, &range) };
+    if ret == 0 {
+        Ok(())
+    } else {
+        Err(io::Error::last_os_error())
+    }
+}
+
+// Discard (TRIM/UNMAP) the byte range `[offset, offset + length)` on the
+// block device referenced by `fd`.
+pub(crate) fn blkdiscard(fd: RawFd, offset: u64, length: u64) -> io::Result<()> {
+    blk_range_ioctl(fd, BLKDISCARD, offset, length)
+}
+
+// Zero the byte range `[offset, offset + length)` on the block device
+// referenced by `fd`. The kernel falls back to writing explicit zero pages
+// when the device has no hardware `WRITE_ZEROES`.
+pub(crate) fn blkzeroout(fd: RawFd, offset: u64, length: u64) -> io::Result<()> {
+    blk_range_ioctl(fd, BLKZEROOUT, offset, length)
+}
+
+// Punch a hole in `fd` over the byte range `[offset, offset + length)`.
+//
+// On block devices the kernel rejects `fallocate(PUNCH_HOLE)` (notably ZFS
+// zvols), so route through `BLKDISCARD` instead. On regular files use
+// `fallocate(FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE)`.
+pub(crate) fn punch_hole(fd: RawFd, is_blkdev: bool, offset: u64, length: u64) -> io::Result<()> {
+    if is_blkdev {
+        blkdiscard(fd, offset, length)
+    } else {
+        fallocate(
+            fd,
+            FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE,
+            offset,
+            length,
+        )
+    }
+}
+
+// Zero the byte range `[offset, offset + length)` in `fd`.
+//
+// Uses `BLKZEROOUT` on block devices (see [`punch_hole`] for the rationale)
+// and `fallocate(FALLOC_FL_ZERO_RANGE | FALLOC_FL_KEEP_SIZE)` on regular
+// files.
+pub(crate) fn write_zeroes(fd: RawFd, is_blkdev: bool, offset: u64, length: u64) -> io::Result<()> {
+    if is_blkdev {
+        blkzeroout(fd, offset, length)
+    } else {
+        fallocate(
+            fd,
+            FALLOC_FL_ZERO_RANGE | FALLOC_FL_KEEP_SIZE,
+            offset,
+            length,
+        )
+    }
+}
+
+fn fallocate(fd: RawFd, mode: libc::c_int, offset: u64, length: u64) -> io::Result<()> {
+    // SAFETY: FFI call with a valid fd; fallocate touches no userspace memory.
+    let ret = unsafe { libc::fallocate(fd, mode, offset as libc::off_t, length as libc::off_t) };
+    if ret == 0 {
+        Ok(())
+    } else {
+        Err(io::Error::last_os_error())
+    }
+}

block/src/vhdx_sync.rs

@@ -1,180 +0,0 @@
-// Copyright © 2021 Intel Corporation
-//
-// SPDX-License-Identifier: Apache-2.0
-
-use std::collections::VecDeque;
-use std::fs::File;
-use std::os::fd::AsRawFd;
-use std::sync::{Arc, Mutex};
-
-use vmm_sys_util::eventfd::EventFd;
-
-use crate::async_io::{AsyncIo, AsyncIoError, AsyncIoResult, BorrowedDiskFd, DiskFileError};
-use crate::error::{BlockError, BlockErrorKind, BlockResult, ErrorOp};
-use crate::vhdx::{Vhdx, VhdxError};
-use crate::{AsyncAdaptor, BlockBackend, Error, disk_file};
-
-#[derive(Debug)]
-pub struct VhdxDiskSync {
-    // FIXME: The Mutex serializes all VHDX I/O operations across queues, which
-    // is necessary for correctness but eliminates any parallelism benefit from
-    // multiqueue. Vhdx::clone() shares the underlying file description across
-    // threads, so concurrent I/O from multiple queues races on the file offset
-    // causing data corruption.
-    //
-    // A proper fix would require restructuring the VHDX I/O path so that data
-    // operations can proceed in parallel with independent file descriptors.
-    vhdx_file: Arc<Mutex<Vhdx>>,
-}
-
-impl VhdxDiskSync {
-    pub fn new(f: File) -> BlockResult<Self> {
-        Ok(VhdxDiskSync {
-            vhdx_file: Arc::new(Mutex::new(Vhdx::new(f).map_err(|e| {
-                let kind = match &e {
-                    VhdxError::NotVhdx(_)
-                    | VhdxError::ParseVhdxHeader(_)
-                    | VhdxError::ParseVhdxMetadata(_)
-                    | VhdxError::ParseVhdxRegionEntry(_) => BlockErrorKind::InvalidFormat,
-                    VhdxError::ReadBatEntry(_) => BlockErrorKind::CorruptImage,
-                    VhdxError::ReadFailed(_) | VhdxError::WriteFailed(_) => BlockErrorKind::Io,
-                };
-                BlockError::new(kind, e).with_op(ErrorOp::Open)
-            })?)),
-        })
-    }
-}
-
-impl disk_file::DiskSize for VhdxDiskSync {
-    fn logical_size(&self) -> BlockResult<u64> {
-        Ok(self.vhdx_file.lock().unwrap().virtual_disk_size())
-    }
-}
-
-impl disk_file::PhysicalSize for VhdxDiskSync {
-    fn physical_size(&self) -> BlockResult<u64> {
-        self.vhdx_file
-            .lock()
-            .unwrap()
-            .physical_size()
-            .map_err(|e| match e {
-                Error::GetFileMetadata(io) => {
-                    BlockError::new(BlockErrorKind::Io, Error::GetFileMetadata(io))
-                }
-                _ => unreachable!("unexpected error from Vhdx::physical_size(): {e}"),
-            })
-    }
-}
-
-impl disk_file::DiskFd for VhdxDiskSync {
-    fn fd(&self) -> BorrowedDiskFd<'_> {
-        BorrowedDiskFd::new(self.vhdx_file.lock().unwrap().as_raw_fd())
-    }
-}
-
-impl disk_file::Geometry for VhdxDiskSync {}
-
-impl disk_file::SparseCapable for VhdxDiskSync {}
-
-impl disk_file::Resizable for VhdxDiskSync {
-    fn resize(&mut self, _size: u64) -> BlockResult<()> {
-        Err(BlockError::new(
-            BlockErrorKind::UnsupportedFeature,
-            DiskFileError::ResizeError(std::io::Error::other("resize not supported for VHDX")),
-        )
-        .with_op(ErrorOp::Resize))
-    }
-}
-
-impl disk_file::DiskFile for VhdxDiskSync {}
-
-impl disk_file::AsyncDiskFile for VhdxDiskSync {
-    fn try_clone(&self) -> BlockResult<Box<dyn disk_file::AsyncDiskFile>> {
-        Ok(Box::new(VhdxDiskSync {
-            vhdx_file: Arc::clone(&self.vhdx_file),
-        }))
-    }
-
-    fn create_async_io(&self, _ring_depth: u32) -> BlockResult<Box<dyn AsyncIo>> {
-        Ok(Box::new(VhdxSync::new(Arc::clone(&self.vhdx_file))))
-    }
-}
-
-pub struct VhdxSync {
-    vhdx_file: Arc<Mutex<Vhdx>>,
-    eventfd: EventFd,
-    completion_list: VecDeque<(u64, i32)>,
-}
-
-impl VhdxSync {
-    pub fn new(vhdx_file: Arc<Mutex<Vhdx>>) -> Self {
-        VhdxSync {
-            vhdx_file,
-            eventfd: EventFd::new(libc::EFD_NONBLOCK)
-                .expect("Failed creating EventFd for VhdxSync"),
-            completion_list: VecDeque::new(),
-        }
-    }
-}
-
-impl AsyncAdaptor for Vhdx {}
-
-impl AsyncIo for VhdxSync {
-    fn notifier(&self) -> &EventFd {
-        &self.eventfd
-    }
-
-    fn read_vectored(
-        &mut self,
-        offset: libc::off_t,
-        iovecs: &[libc::iovec],
-        user_data: u64,
-    ) -> AsyncIoResult<()> {
-        self.vhdx_file.lock().unwrap().read_vectored_sync(
-            offset,
-            iovecs,
-            user_data,
-            &self.eventfd,
-            &mut self.completion_list,
-        )
-    }
-
-    fn write_vectored(
-        &mut self,
-        offset: libc::off_t,
-        iovecs: &[libc::iovec],
-        user_data: u64,
-    ) -> AsyncIoResult<()> {
-        self.vhdx_file.lock().unwrap().write_vectored_sync(
-            offset,
-            iovecs,
-            user_data,
-            &self.eventfd,
-            &mut self.completion_list,
-        )
-    }
-
-    fn fsync(&mut self, user_data: Option<u64>) -> AsyncIoResult<()> {
-        self.vhdx_file.lock().unwrap().fsync_sync(
-            user_data,
-            &self.eventfd,
-            &mut self.completion_list,
-        )
-    }
-
-    fn next_completed_request(&mut self) -> Option<(u64, i32)> {
-        self.completion_list.pop_front()
-    }
-
-    fn punch_hole(&mut self, _offset: u64, _length: u64, _user_data: u64) -> AsyncIoResult<()> {
-        Err(AsyncIoError::PunchHole(std::io::Error::other(
-            "punch_hole not supported for VHDX",
-        )))
-    }
-
-    fn write_zeroes(&mut self, _offset: u64, _length: u64, _user_data: u64) -> AsyncIoResult<()> {
-        Err(AsyncIoError::WriteZeroes(std::io::Error::other(
-            "write_zeroes not supported for VHDX",
-        )))
-    }
-}

clippy.toml

@@ -0,0 +1 @@
+absolute-paths-allowed-crates = ["arch"]

cloud-hypervisor/Cargo.toml

@@ -8,15 +8,13 @@ homepage = "https://github.com/cloud-hypervisor/cloud-hypervisor"
 license = "Apache-2.0 AND BSD-3-Clause"
 name = "cloud-hypervisor"
 rust-version.workspace = true
-version = "52.0.0"
+version = "53.0.0"
 
 [dependencies]
-anyhow = { workspace = true }
 api_client = { path = "../api_client" }
 clap = { workspace = true, features = ["string"] }
 dhat = { workspace = true, optional = true }
 env_logger = { workspace = true }
-epoll = { workspace = true }
 event_monitor = { path = "../event_monitor" }
 hypervisor = { path = "../hypervisor" }
 jiff = { workspace = true }
@@ -26,10 +24,10 @@ option_parser = { path = "../option_parser" }
 seccompiler = { workspace = true }
 serde_json = { workspace = true }
 signal-hook = { workspace = true }
+signal-hook-registry = { workspace = true }
 thiserror = { workspace = true }
-tpm = { path = "../tpm" }
 tracer = { path = "../tracer" }
-vm-memory = { workspace = true }
+vm-migration = { path = "../vm-migration" }
 vmm = { path = "../vmm" }
 vmm-sys-util = { workspace = true }
 zbus = { version = "5.15.0", optional = true }
@@ -50,13 +48,13 @@ default = ["io_uring", "kvm"]
 dhat-heap = ["dhat", "vmm/dhat-heap"]       # For heap profiling
 fw_cfg = ["vmm/fw_cfg"]
 guest_debug = ["vmm/guest_debug"]
-igvm = ["mshv", "vmm/igvm"]
+igvm = ["vmm/igvm"]
 io_uring = ["vmm/io_uring"]
 ivshmem = ["vmm/ivshmem"]
 kvm = ["vmm/kvm"]
 mshv = ["vmm/mshv"]
 pvmemcontrol = ["vmm/pvmemcontrol"]
-sev_snp = ["igvm", "mshv", "vmm/sev_snp"]
+sev_snp = ["igvm", "vmm/sev_snp"]
 tdx = ["vmm/tdx"]
 tracing = ["tracer/tracing", "vmm/tracing"]
 

cloud-hypervisor/src/bin/ch-remote.rs

@@ -7,10 +7,10 @@
 #[path = "../test_util.rs"]
 mod test_util;
 
-use std::io::Read;
+use std::io::{self, Read};
 use std::marker::PhantomData;
 use std::os::unix::net::UnixStream;
-use std::process;
+use std::{error, fs, iter, num, process};
 
 use api_client::{
     Error as ApiClientError, simple_api_command, simple_api_command_with_fds,
@@ -22,13 +22,14 @@ use clap::{Arg, ArgMatches, Command};
 use log::error;
 use option_parser::{ByteSized, ByteSizedParseError};
 use thiserror::Error;
-use vmm::config::RestoreConfig;
+use vmm::api;
+use vmm::config::{self, RestoreConfig};
 use vmm::vm_config::{
     DeviceConfig, DiskConfig, FsConfig, GenericVhostUserConfig, NetConfig, PmemConfig,
     UserDeviceConfig, VdpaConfig, VsockConfig,
 };
 #[cfg(feature = "dbus_api")]
-use zbus::{proxy, zvariant::Optional};
+use zbus::{blocking::Connection, proxy, zvariant::Optional};
 
 type ApiResult = Result<(), Error>;
 
@@ -40,39 +41,41 @@ enum Error {
     #[error("dbus api client error")]
     DBusApiClient(#[source] zbus::Error),
     #[error("Error parsing CPU count")]
-    InvalidCpuCount(#[source] std::num::ParseIntError),
+    InvalidCpuCount(#[source] num::ParseIntError),
     #[error("Error parsing memory size")]
     InvalidMemorySize(#[source] ByteSizedParseError),
     #[error("Error parsing balloon size")]
     InvalidBalloonSize(#[source] ByteSizedParseError),
     #[error("Error parsing device syntax")]
-    AddDeviceConfig(#[source] vmm::config::Error),
+    AddDeviceConfig(#[source] config::Error),
     #[error("Error parsing disk syntax")]
-    AddDiskConfig(#[source] vmm::config::Error),
+    AddDiskConfig(#[source] config::Error),
     #[error("Error parsing filesystem syntax")]
-    AddFsConfig(#[source] vmm::config::Error),
+    AddFsConfig(#[source] config::Error),
     #[error("Error parsing generic vhost-user syntax")]
-    AddGenericVhostUserConfig(#[source] vmm::config::Error),
+    AddGenericVhostUserConfig(#[source] config::Error),
     #[error("Error parsing persistent memory syntax")]
-    AddPmemConfig(#[source] vmm::config::Error),
+    AddPmemConfig(#[source] config::Error),
     #[error("Error parsing network syntax")]
-    AddNetConfig(#[source] vmm::config::Error),
+    AddNetConfig(#[source] config::Error),
     #[error("Error parsing user device syntax")]
-    AddUserDeviceConfig(#[source] vmm::config::Error),
+    AddUserDeviceConfig(#[source] config::Error),
     #[error("Error parsing vDPA device syntax")]
-    AddVdpaConfig(#[source] vmm::config::Error),
+    AddVdpaConfig(#[source] config::Error),
     #[error("Error parsing vsock syntax")]
-    AddVsockConfig(#[source] vmm::config::Error),
+    AddVsockConfig(#[source] config::Error),
     #[error("Error parsing restore syntax")]
-    Restore(#[source] vmm::config::Error),
+    Restore(#[source] config::Error),
     #[error("Error reading from stdin")]
-    ReadingStdin(#[source] std::io::Error),
+    ReadingStdin(#[source] io::Error),
     #[error("Error reading from file")]
-    ReadingFile(#[source] std::io::Error),
+    ReadingFile(#[source] io::Error),
     #[error("Invalid disk size")]
     InvalidDiskSize(#[source] ByteSizedParseError),
+    #[error("Error parsing receive migration configuration")]
+    ReceiveMigrationConfig(#[from] api::VmReceiveMigrationConfigError),
     #[error("Error parsing send migration configuration")]
-    SendMigrationConfig(#[from] vmm::api::VmSendMigrationConfigError),
+    SendMigrationConfig(#[from] api::VmSendMigrationConfigError),
 }
 
 enum TargetApi<'a> {
@@ -122,9 +125,9 @@ trait DBusApi1 {
 impl<'a> DBusApi1ProxyBlocking<'a> {
     fn new_connection(name: &'a str, path: &'a str, system_bus: bool) -> Result<Self, zbus::Error> {
         let connection = if system_bus {
-            zbus::blocking::Connection::system()?
+            Connection::system()?
         } else {
-            zbus::blocking::Connection::session()?
+            Connection::session()?
         };
 
         Self::builder(&connection)
@@ -369,14 +372,14 @@ fn rest_api_do_command(matches: &ArgMatches, socket: &mut UnixStream) -> ApiResu
                 .map_err(Error::HttpApiClient)
         }
         Some("add-device") => {
-            let device_config = add_device_config(
+            let (device_config, fds) = add_device_config(
                 matches
                     .subcommand_matches("add-device")
                     .unwrap()
                     .get_one::<String>("device_config")
                     .unwrap(),
             )?;
-            simple_api_command(socket, "PUT", "add-device", Some(&device_config))
+            simple_api_command_with_fds(socket, "PUT", "add-device", Some(&device_config), &fds)
                 .map_err(Error::HttpApiClient)
         }
         Some("remove-device") => {
@@ -534,7 +537,7 @@ fn rest_api_do_command(matches: &ArgMatches, socket: &mut UnixStream) -> ApiResu
                     .unwrap()
                     .get_one::<String>("receive_migration_config")
                     .unwrap(),
-            );
+            )?;
             simple_api_command(
                 socket,
                 "PUT",
@@ -607,7 +610,7 @@ fn dbus_api_do_command(matches: &ArgMatches, proxy: &DBusApi1ProxyBlocking<'_>)
             proxy.api_vm_resize_zone(&resize_zone)
         }
         Some("add-device") => {
-            let device_config = add_device_config(
+            let (device_config, _fds) = add_device_config(
                 matches
                     .subcommand_matches("add-device")
                     .unwrap()
@@ -753,7 +756,7 @@ fn dbus_api_do_command(matches: &ArgMatches, proxy: &DBusApi1ProxyBlocking<'_>)
                     .unwrap()
                     .get_one::<String>("receive_migration_config")
                     .unwrap(),
-            );
+            )?;
             proxy.api_vm_receive_migration(&receive_migration_data)
         }
         Some("create") => {
@@ -803,7 +806,7 @@ fn resize_config(
         None
     };
 
-    let resize = vmm::api::VmResizeData {
+    let resize = api::VmResizeData {
         desired_vcpus,
         desired_ram,
         desired_balloon,
@@ -813,7 +816,7 @@ fn resize_config(
 }
 
 fn resize_disk_config(id: &str, size: &str) -> Result<String, Error> {
-    let resize_disk = vmm::api::VmResizeDiskData {
+    let resize_disk = api::VmResizeDiskData {
         id: id.to_owned(),
         desired_size: size.parse::<ByteSized>().map_err(Error::InvalidDiskSize)?.0,
     };
@@ -822,7 +825,7 @@ fn resize_disk_config(id: &str, size: &str) -> Result<String, Error> {
 }
 
 fn resize_zone_config(id: &str, size: &str) -> Result<String, Error> {
-    let resize_zone = vmm::api::VmResizeZoneData {
+    let resize_zone = api::VmResizeZoneData {
         id: id.to_owned(),
         desired_ram: size
             .parse::<ByteSized>()
@@ -833,11 +836,21 @@ fn resize_zone_config(id: &str, size: &str) -> Result<String, Error> {
     Ok(serde_json::to_string(&resize_zone).unwrap())
 }
 
-fn add_device_config(config: &str) -> Result<String, Error> {
-    let device_config = DeviceConfig::parse(config).map_err(Error::AddDeviceConfig)?;
+fn add_device_config(config: &str) -> Result<(String, Vec<i32>), Error> {
+    let mut device_config = DeviceConfig::parse(config).map_err(Error::AddDeviceConfig)?;
+
+    // DeviceConfig is modified on purpose here by taking the file
+    // descriptor out. Keeping it and sending it over to the server side
+    // process would not make any sense since the file descriptor may be
+    // represented with different values.
+    let fds = device_config
+        .fd
+        .take()
+        .map(|fd| vec![fd])
+        .unwrap_or_default();
     let device_config = serde_json::to_string(&device_config).unwrap();
 
-    Ok(device_config)
+    Ok((device_config, fds))
 }
 
 fn add_user_device_config(config: &str) -> Result<String, Error> {
@@ -848,7 +861,7 @@ fn add_user_device_config(config: &str) -> Result<String, Error> {
 }
 
 fn remove_device_config(id: &str) -> String {
-    let remove_device_data = vmm::api::VmRemoveDeviceData { id: id.to_owned() };
+    let remove_device_data = api::VmRemoveDeviceData { id: id.to_owned() };
 
     serde_json::to_string(&remove_device_data).unwrap()
 }
@@ -910,7 +923,7 @@ fn add_vsock_config(config: &str) -> Result<String, Error> {
 }
 
 fn snapshot_config(url: &str) -> String {
-    let snapshot_config = vmm::api::VmSnapshotConfig {
+    let snapshot_config = api::VmSnapshotConfig {
         destination_url: String::from(url),
     };
 
@@ -934,24 +947,22 @@ fn restore_config(config: &str) -> Result<(String, Vec<i32>), Error> {
 }
 
 fn coredump_config(destination_url: &str) -> String {
-    let coredump_config = vmm::api::VmCoredumpData {
+    let coredump_config = api::VmCoredumpData {
         destination_url: String::from(destination_url),
     };
 
     serde_json::to_string(&coredump_config).unwrap()
 }
 
-fn receive_migration_data(url: &str) -> String {
-    let receive_migration_data = vmm::api::VmReceiveMigrationData {
-        receiver_url: url.to_owned(),
-    };
-
-    serde_json::to_string(&receive_migration_data).unwrap()
+fn receive_migration_data(config: &str) -> Result<String, Error> {
+    let receive_migration_data =
+        api::VmReceiveMigrationData::parse(config).map_err(Error::ReceiveMigrationConfig)?;
+    Ok(serde_json::to_string(&receive_migration_data).unwrap())
 }
 
 fn send_migration_data(config: &str) -> Result<String, Error> {
     let send_migration_data =
-        vmm::api::VmSendMigrationData::parse(config).map_err(Error::SendMigrationConfig)?;
+        api::VmSendMigrationData::parse(config).map_err(Error::SendMigrationConfig)?;
     let send_migration_config = serde_json::to_string(&send_migration_data).unwrap();
     Ok(send_migration_config)
 }
@@ -959,11 +970,11 @@ fn send_migration_data(config: &str) -> Result<String, Error> {
 fn create_data(path: &str) -> Result<String, Error> {
     let mut data = String::default();
     if path == "-" {
-        std::io::stdin()
+        io::stdin()
             .read_to_string(&mut data)
             .map_err(Error::ReadingStdin)?;
     } else {
-        data = std::fs::read_to_string(path).map_err(Error::ReadingFile)?;
+        data = fs::read_to_string(path).map_err(Error::ReadingFile)?;
     }
 
     Ok(data)
@@ -1014,28 +1025,20 @@ fn get_cli_commands_sorted() -> Box<[Command]> {
             .arg(Arg::new("disk_config").index(1).help(DiskConfig::SYNTAX)),
         Command::new("add-fs")
             .about("Add virtio-fs backed fs device")
-            .arg(
-                Arg::new("fs_config")
-                    .index(1)
-                    .help(vmm::vm_config::FsConfig::SYNTAX),
-            ),
+            .arg(Arg::new("fs_config").index(1).help(FsConfig::SYNTAX)),
         Command::new("add-generic-vhost-user")
             .about("Add generic vhost-user device")
             .arg(
                 Arg::new("generic_vhost_user_config")
                     .index(1)
-                    .help(vmm::vm_config::GenericVhostUserConfig::SYNTAX),
+                    .help(GenericVhostUserConfig::SYNTAX),
             ),
         Command::new("add-net")
             .about("Add network device")
             .arg(Arg::new("net_config").index(1).help(NetConfig::SYNTAX)),
         Command::new("add-pmem")
             .about("Add persistent memory device")
-            .arg(
-                Arg::new("pmem_config")
-                    .index(1)
-                    .help(vmm::vm_config::PmemConfig::SYNTAX),
-            ),
+            .arg(Arg::new("pmem_config").index(1).help(PmemConfig::SYNTAX)),
         Command::new("add-user-device")
             .about("Add userspace device")
             .arg(
@@ -1069,7 +1072,7 @@ fn get_cli_commands_sorted() -> Box<[Command]> {
             .arg(
                 Arg::new("receive_migration_config")
                     .index(1)
-                    .help("<receiver_url>"),
+                    .help(api::VmReceiveMigrationData::SYNTAX),
             ),
         Command::new("remove-device")
             .about("Remove VFIO and PCI device")
@@ -1127,6 +1130,7 @@ fn get_cli_commands_sorted() -> Box<[Command]> {
             .arg(
                 Arg::new("restore_config")
                     .index(1)
+                    .required(true)
                     .help(RestoreConfig::SYNTAX),
             ),
         Command::new("resume").about("Resume the VM"),
@@ -1135,7 +1139,7 @@ fn get_cli_commands_sorted() -> Box<[Command]> {
             .arg(
                 Arg::new("send_migration_config")
                     .index(1)
-                    .help(vmm::api::VmSendMigrationData::SYNTAX),
+                    .help(api::VmSendMigrationData::SYNTAX),
             ),
         Command::new("shutdown").about("Shutdown the VM"),
         Command::new("shutdown-vmm").about("Shutdown the VMM"),
@@ -1144,6 +1148,7 @@ fn get_cli_commands_sorted() -> Box<[Command]> {
             .arg(
                 Arg::new("snapshot_config")
                     .index(1)
+                    .required(true)
                     .help("<destination_url>"),
             ),
     ]
@@ -1217,7 +1222,7 @@ fn main() {
 
     if let Err(top_error) = target_api.do_command(&matches) {
         // Helper to join strings with a newline.
-        #[allow(clippy::needless_pass_by_value)]
+        #[expect(clippy::needless_pass_by_value)]
         fn join_strs(mut acc: String, next: String) -> String {
             if !acc.is_empty() {
                 acc.push('\n');
@@ -1237,7 +1242,7 @@ fn main() {
         fn server_api_error_display_modifier(
             level: usize,
             indention: usize,
-            error: &(dyn std::error::Error + 'static),
+            error: &(dyn error::Error + 'static),
         ) -> Option<String> {
             if let Some(api_client::Error::ServerResponse(status_code, body)) =
                 error.downcast_ref::<api_client::Error>()
@@ -1245,7 +1250,7 @@ fn main() {
                 let body = body.as_ref().map_or("", |body| body.as_str());
 
                 // Retrieve the list of error messages back.
-                let lines: Vec<&str> = match serde_json::from_str(body) {
+                let lines: Vec<String> = match serde_json::from_str(body) {
                     Ok(json) => json,
                     Err(e) => {
                         return Some(format!(
@@ -1257,7 +1262,8 @@ fn main() {
 
                 let error_status = format!("Server responded with {status_code:?}");
                 // Prepend the error status line to the lines iter.
-                let lines = std::iter::once(error_status.as_str()).chain(lines);
+                let lines =
+                    iter::once(error_status.as_str()).chain(lines.iter().map(|s| s.as_str()));
                 let error_msg_multiline = lines
                     .enumerate()
                     .map(|(index, error_msg)| (index + level, error_msg))
@@ -1275,7 +1281,7 @@ fn main() {
             None
         }
 
-        let top_error: &dyn std::error::Error = &top_error;
+        let top_error: &dyn error::Error = &top_error;
         cloud_hypervisor::cli_print_error_chain(
             top_error,
             "ch-remote",
@@ -1319,4 +1325,17 @@ mod unit_tests {
             assert_args_sorted(|| command.get_arguments());
         }
     }
+
+    #[test]
+    fn test_error_deserialization() {
+        let body = r#"["Error from API","The VM could not be snapshotted","Cannot send VM snapshot","Failed to send migratable component snapshot","Destination is not a directory: \"/tmp/ch.dump\""]"#;
+        let lines: Result<Vec<String>, _> = serde_json::from_str(body);
+        assert!(lines.is_ok());
+        let lines = lines.unwrap();
+        assert_eq!(lines.len(), 5);
+        assert_eq!(
+            lines[4],
+            r#"Destination is not a directory: "/tmp/ch.dump""#
+        );
+    }
 }

cloud-hypervisor/src/lib.rs

@@ -3,6 +3,7 @@
 // SPDX-License-Identifier: Apache-2.0
 
 use std::error::Error;
+use std::iter;
 
 use log::error;
 
@@ -28,7 +29,7 @@ pub fn cli_print_error_chain<'a>(
         eprintln!("  {top_error}");
     } else {
         eprintln!("chain of errors:");
-        std::iter::successors(Some(top_error), |sub_error| {
+        iter::successors(Some(top_error), |sub_error| {
             // Dereference necessary to mitigate rustc compiler bug.
             // See <https://github.com/rust-lang/rust/issues/141673>
             (*sub_error).source()

cloud-hypervisor/src/logger.rs

@@ -3,11 +3,13 @@
 // SPDX-License-Identifier: Apache-2.0
 //
 
-use std::io::Write;
+use std::io::{self, Write};
 use std::str::FromStr;
 use std::sync::Mutex;
 use std::time::Instant;
+use std::{mem, process, thread};
 
+use jiff::tz::TimeZone;
 use thiserror::Error;
 
 #[derive(Debug, Error)]
@@ -20,30 +22,82 @@ pub enum Error {
     UnknownToken(String),
 }
 
+/// Which time source a date/time field should be read from.
+#[derive(Copy, Clone, Debug, Eq, PartialEq)]
+enum Zone {
+    Utc,
+    Local,
+}
+
+/// An individual broken-down date/time field.
+#[derive(Copy, Clone, Debug, Eq, PartialEq)]
+enum TimeField {
+    Year,
+    Month,
+    Day,
+    Hour,
+    Minute,
+    Second,
+    Micros,
+    /// Timezone offset like `-08:00` (always `+00:00` for `Zone::Utc`).
+    Offset,
+}
+
 enum Token {
     Literal(String),
     BootTime,
     /// Wallclock using RFC 3339 formatting.
     WallClock,
+    /// UTC glog-style timestamp (e.g. `0521 08:02:15.542701`).
+    Glog,
+    /// Local-time glog-style timestamp (e.g. `0521 08:02:15.542701`).
+    LocalGlog,
     Pid,
     Tid,
     Thread,
+    /// Full level word (e.g. `INFO`).
     Level,
+    /// Single-letter level character, glog style (e.g. `I`).
+    LevelChar,
     Location,
     Msg,
+    /// A broken-down date/time field from either UTC or local wallclock.
+    Time(TimeField, Zone),
 }
 
 impl FromStr for Token {
     type Err = Error;
 
     fn from_str(s: &str) -> Result<Self, Self::Err> {
+        // Detect `local`-prefixed variants for the broken-down time fields.
+        let (name, zone) = match s.strip_prefix("local") {
+            Some(rest) => (rest, Zone::Local),
+            None => (s, Zone::Utc),
+        };
+
+        match name {
+            "year" => return Ok(Self::Time(TimeField::Year, zone)),
+            "month" => return Ok(Self::Time(TimeField::Month, zone)),
+            "day" => return Ok(Self::Time(TimeField::Day, zone)),
+            "hour" => return Ok(Self::Time(TimeField::Hour, zone)),
+            "minute" => return Ok(Self::Time(TimeField::Minute, zone)),
+            "second" => return Ok(Self::Time(TimeField::Second, zone)),
+            "micros" => return Ok(Self::Time(TimeField::Micros, zone)),
+            "offset" => return Ok(Self::Time(TimeField::Offset, zone)),
+            _ => {}
+        }
+
+        // Fall back to tokens that don't take a `local` prefix.
         match s {
             "boottime" => Ok(Self::BootTime),
             "wallclock" => Ok(Self::WallClock),
+            "glog" => Ok(Self::Glog),
+            "localglog" => Ok(Self::LocalGlog),
             "pid" => Ok(Self::Pid),
             "tid" => Ok(Self::Tid),
             "thread" => Ok(Self::Thread),
             "level" => Ok(Self::Level),
+            "levelchar" => Ok(Self::LevelChar),
             "location" => Ok(Self::Location),
             "msg" => Ok(Self::Msg),
             _ => Err(Error::UnknownToken(s.to_string())),
@@ -51,6 +105,34 @@ impl FromStr for Token {
     }
 }
 
+/// Convert a `log::Level` to its glog single-letter abbreviation.
+fn level_char(level: log::Level) -> char {
+    match level {
+        log::Level::Error => 'E',
+        log::Level::Warn => 'W',
+        log::Level::Info => 'I',
+        log::Level::Debug => 'D',
+        log::Level::Trace => 'T',
+    }
+}
+
+fn write_time_field<W: Write + ?Sized>(
+    out: &mut W,
+    field: TimeField,
+    zoned: &jiff::Zoned,
+) -> io::Result<()> {
+    match field {
+        TimeField::Year => write!(out, "{:04}", zoned.year()),
+        TimeField::Month => write!(out, "{:02}", zoned.month()),
+        TimeField::Day => write!(out, "{:02}", zoned.day()),
+        TimeField::Hour => write!(out, "{:02}", zoned.hour()),
+        TimeField::Minute => write!(out, "{:02}", zoned.minute()),
+        TimeField::Second => write!(out, "{:02}", zoned.second()),
+        TimeField::Micros => write!(out, "{:06}", zoned.subsec_nanosecond() / 1000),
+        TimeField::Offset => write!(out, "{}", zoned.strftime("%:z")),
+    }
+}
+
 fn parse_format(fmt: &str) -> Result<Vec<Token>, Error> {
     let mut tokens = Vec::new();
     let mut literal = String::new();
@@ -66,7 +148,7 @@ fn parse_format(fmt: &str) -> Result<Vec<Token>, Error> {
                 }
 
                 if !literal.is_empty() {
-                    tokens.push(Token::Literal(std::mem::take(&mut literal)));
+                    tokens.push(Token::Literal(mem::take(&mut literal)));
                 }
 
                 let mut name = String::new();
@@ -105,6 +187,9 @@ pub struct Logger {
     start: Instant,
     pid: u32,
     tokens: Vec<Token>,
+    // Saving the timezone when Logger is constructed avoids potential seccomp violations when the
+    // internal libc timezone cache expires as the affected thread is unpredictable.
+    local_tz: TimeZone,
 }
 
 impl Logger {
@@ -112,8 +197,9 @@ impl Logger {
         Ok(Self {
             output: Mutex::new(output),
             start: Instant::now(),
-            pid: std::process::id(),
+            pid: process::id(),
             tokens: parse_format(format)?,
+            local_tz: TimeZone::try_system().unwrap_or(TimeZone::UTC),
         })
     }
 }
@@ -129,6 +215,10 @@ impl log::Log for Logger {
         }
 
         let duration_s = Instant::now().duration_since(self.start).as_secs_f32();
+        // Compute the wallclock timestamps lazily, but at most once per record so
+        // that multiple `{hour}`/`{minute}`/`{second}`/etc. fields stay coherent.
+        let mut zoned_utc: Option<jiff::Zoned> = None;
+        let mut zoned_local: Option<jiff::Zoned> = None;
         let mut out = self.output.lock().unwrap();
         for token in &self.tokens {
             let _ = match token {
@@ -136,7 +226,20 @@ impl log::Log for Logger {
                 // 10: 6 decimal places + sep => whole seconds in range `0..=999` properly aligned
                 Token::BootTime => write!(&mut *out, "{duration_s:>10.6?}"),
                 Token::WallClock => {
-                    write!(out, "{:.6}", jiff::Timestamp::now())
+                    let zoned = zoned_utc
+                        .get_or_insert_with(|| jiff::Timestamp::now().to_zoned(TimeZone::UTC));
+                    write!(&mut *out, "{:.6}", zoned.timestamp())
+                }
+                Token::Glog => {
+                    let zoned = zoned_utc
+                        .get_or_insert_with(|| jiff::Timestamp::now().to_zoned(TimeZone::UTC));
+                    write!(&mut *out, "{}", zoned.strftime("%m%d %H:%M:%S%.6f"))
+                }
+                Token::LocalGlog => {
+                    let zoned = zoned_local.get_or_insert_with(|| {
+                        jiff::Timestamp::now().to_zoned(self.local_tz.clone())
+                    });
+                    write!(&mut *out, "{}", zoned.strftime("%m%d %H:%M:%S%.6f"))
                 }
                 Token::Pid => write!(&mut *out, "{}", self.pid),
                 // SAFETY: gettid(2) always succeeds
@@ -144,14 +247,25 @@ impl log::Log for Logger {
                 Token::Thread => write!(
                     &mut *out,
                     "{}",
-                    std::thread::current().name().unwrap_or("anonymous")
+                    thread::current().name().unwrap_or("anonymous")
                 ),
                 Token::Level => write!(&mut *out, "{}", record.level()),
+                Token::LevelChar => write!(&mut *out, "{}", level_char(record.level())),
                 Token::Location => match (record.file(), record.line()) {
                     (Some(file), Some(line)) => write!(&mut *out, "{file}:{line}"),
                     _ => write!(&mut *out, "{}", record.target()),
                 },
                 Token::Msg => write!(&mut *out, "{}", record.args()),
+                Token::Time(field, zone) => {
+                    let zoned = match zone {
+                        Zone::Utc => zoned_utc
+                            .get_or_insert_with(|| jiff::Timestamp::now().to_zoned(TimeZone::UTC)),
+                        Zone::Local => zoned_local.get_or_insert_with(|| {
+                            jiff::Timestamp::now().to_zoned(self.local_tz.clone())
+                        }),
+                    };
+                    write_time_field(&mut *out, *field, zoned)
+                }
             };
         }
         let _ = out.write_all(b"\r\n");
@@ -198,12 +312,32 @@ mod tests {
                 Token::Literal(s) => format!("L({s})"),
                 Token::BootTime => "B".to_string(),
                 Token::WallClock => "W".to_string(),
+                Token::Glog => "G".to_string(),
+                Token::LocalGlog => "LG".to_string(),
                 Token::Pid => "P".to_string(),
                 Token::Tid => "I".to_string(),
                 Token::Thread => "T".to_string(),
                 Token::Level => "V".to_string(),
+                Token::LevelChar => "VC".to_string(),
                 Token::Location => "O".to_string(),
                 Token::Msg => "M".to_string(),
+                Token::Time(field, zone) => {
+                    let z = match zone {
+                        Zone::Utc => "u",
+                        Zone::Local => "l",
+                    };
+                    let f = match field {
+                        TimeField::Year => "Y",
+                        TimeField::Month => "Mo",
+                        TimeField::Day => "D",
+                        TimeField::Hour => "H",
+                        TimeField::Minute => "Mi",
+                        TimeField::Second => "S",
+                        TimeField::Micros => "U",
+                        TimeField::Offset => "Z",
+                    };
+                    format!("T({z}:{f})")
+                }
             })
             .collect::<Vec<_>>()
             .join("|")
@@ -224,12 +358,12 @@ mod tests {
     #[test]
     fn parse_all_known_tokens() {
         let tokens = parse_format(
-            "[{boottime}] {wallclock} {pid}/{tid} <{thread}> {level} {location} -- {msg}",
+            "[{boottime}] {wallclock} {glog} {localglog} {pid}/{tid} <{thread}> {level} {levelchar} {location} -- {msg}",
         )
         .unwrap();
         assert_eq!(
             render(&tokens),
-            "L([)|B|L(] )|W|L( )|P|L(/)|I|L( <)|T|L(> )|V|L( )|O|L( -- )|M"
+            "L([)|B|L(] )|W|L( )|G|L( )|LG|L( )|P|L(/)|I|L( <)|T|L(> )|V|L( )|VC|L( )|O|L( -- )|M"
         );
     }
 
@@ -377,6 +511,134 @@ mod tests {
         assert!(out.ends_with('Z'), "got: {out}");
     }
 
+    #[test]
+    fn logger_glog_style_output() {
+        // `{levelchar}{localglog}` => glog-style header like `I0521 08:02:15.542701`.
+        let buf = SharedBuffer::default();
+        let logger = Logger::new(Box::new(buf.clone()), "{levelchar}{localglog}").unwrap();
+
+        logger.log(
+            &log::Record::builder()
+                .args(format_args!(""))
+                .level(log::Level::Info)
+                .target("t")
+                .build(),
+        );
+
+        let out = buf.contents();
+        let out = out.trim();
+        // `IMMDD HH:MM:SS.uuuuuu` => 21 chars.
+        assert_eq!(out.len(), 21, "got: {out}");
+        assert_eq!(&out[0..1], "I", "got: {out}");
+        assert_eq!(&out[5..6], " ", "got: {out}");
+        assert_eq!(&out[8..9], ":", "got: {out}");
+        assert_eq!(&out[11..12], ":", "got: {out}");
+        assert_eq!(&out[14..15], ".", "got: {out}");
+        // Every non-separator character is an ASCII digit.
+        for (i, ch) in out.chars().enumerate() {
+            if [0, 5, 8, 11, 14].contains(&i) {
+                continue;
+            }
+            assert!(ch.is_ascii_digit(), "non-digit at {i}: got {out}");
+        }
+    }
+
+    #[test]
+    fn logger_glog_utc_output_shape() {
+        // `{glog}` alone produces `MMDD HH:MM:SS.uuuuuu` (20 chars).
+        let buf = SharedBuffer::default();
+        let logger = Logger::new(Box::new(buf.clone()), "{glog}").unwrap();
+
+        logger.log(
+            &log::Record::builder()
+                .args(format_args!(""))
+                .level(log::Level::Info)
+                .target("t")
+                .build(),
+        );
+
+        let out = buf.contents();
+        let out = out.trim();
+        assert_eq!(out.len(), 20, "got: {out}");
+        assert_eq!(&out[4..5], " ", "got: {out}");
+        assert_eq!(&out[7..8], ":", "got: {out}");
+        assert_eq!(&out[10..11], ":", "got: {out}");
+        assert_eq!(&out[13..14], ".", "got: {out}");
+    }
+
+    #[test]
+    fn parse_utc_time_fields() {
+        let tokens =
+            parse_format("{year}-{month}-{day}T{hour}:{minute}:{second}.{micros}{offset}").unwrap();
+        assert_eq!(
+            render(&tokens),
+            "T(u:Y)|L(-)|T(u:Mo)|L(-)|T(u:D)|L(T)|T(u:H)|L(:)|T(u:Mi)|L(:)|T(u:S)|L(.)|T(u:U)|T(u:Z)"
+        );
+    }
+
+    #[test]
+    fn parse_local_time_fields() {
+        let tokens = parse_format(
+            "{localyear}-{localmonth}-{localday}T{localhour}:{localminute}:{localsecond}.{localmicros}{localoffset}",
+        )
+        .unwrap();
+        assert_eq!(
+            render(&tokens),
+            "T(l:Y)|L(-)|T(l:Mo)|L(-)|T(l:D)|L(T)|T(l:H)|L(:)|T(l:Mi)|L(:)|T(l:S)|L(.)|T(l:U)|T(l:Z)"
+        );
+    }
+
+    #[test]
+    fn logger_utc_offset_is_zero() {
+        let buf = SharedBuffer::default();
+        let logger = Logger::new(Box::new(buf.clone()), "{offset}").unwrap();
+        logger.log(
+            &log::Record::builder()
+                .args(format_args!(""))
+                .level(log::Level::Info)
+                .target("t")
+                .build(),
+        );
+        assert_eq!(buf.contents().trim(), "+00:00");
+    }
+
+    #[test]
+    fn logger_utc_year_matches_jiff() {
+        let buf = SharedBuffer::default();
+        let logger = Logger::new(Box::new(buf.clone()), "{year}").unwrap();
+        logger.log(
+            &log::Record::builder()
+                .args(format_args!(""))
+                .level(log::Level::Info)
+                .target("t")
+                .build(),
+        );
+        let year: i32 = buf.contents().trim().parse().expect("year is numeric");
+        assert!(year >= 2024, "got: {year}");
+    }
+
+    #[test]
+    fn logger_levelchar_per_level() {
+        for (level, expected) in [
+            (log::Level::Error, "E"),
+            (log::Level::Warn, "W"),
+            (log::Level::Info, "I"),
+            (log::Level::Debug, "D"),
+            (log::Level::Trace, "T"),
+        ] {
+            let buf = SharedBuffer::default();
+            let logger = Logger::new(Box::new(buf.clone()), "{levelchar}").unwrap();
+            logger.log(
+                &log::Record::builder()
+                    .args(format_args!(""))
+                    .level(level)
+                    .target("t")
+                    .build(),
+            );
+            assert_eq!(buf.contents().trim(), expected);
+        }
+    }
+
     #[test]
     fn logger_pid_token() {
         let buf = SharedBuffer::default();
@@ -392,7 +654,7 @@ mod tests {
 
         let out = buf.contents();
         let out = out.trim();
-        assert_eq!(out, std::process::id().to_string(), "got: {out}");
+        assert_eq!(out, process::id().to_string(), "got: {out}");
     }
 
     #[test]

cloud-hypervisor/src/main.rs

@@ -7,10 +7,13 @@ mod logger;
 #[cfg(test)]
 mod test_util;
 
-use std::fs::File;
+use std::fs::{self, File};
 use std::os::unix::io::{AsRawFd, FromRawFd, RawFd};
+use std::path::Path;
+#[cfg(feature = "guest_debug")]
+use std::path::PathBuf;
 use std::sync::mpsc::channel;
-use std::{env, io};
+use std::{any, cmp, env, io, num, process, str, thread};
 
 use clap::{Arg, ArgAction, ArgGroup, ArgMatches, Command};
 use event_monitor::event;
@@ -19,13 +22,16 @@ use log::{LevelFilter, error, info, warn};
 use option_parser::OptionParser;
 use seccompiler::SeccompAction;
 use signal_hook::consts::SIGSYS;
+use signal_hook::low_level;
 use thiserror::Error;
-use vmm::api::ApiAction;
+use vm_migration::protocol;
 #[cfg(feature = "dbus_api")]
 use vmm::api::dbus::{DBusApiOptions, dbus_api_graceful_shutdown};
 use vmm::api::http::http_api_graceful_shutdown;
-use vmm::config::{RestoreConfig, VmParams};
+use vmm::api::{self, ApiAction};
+use vmm::config::{self, RestoreConfig, VmParams};
 use vmm::landlock::{Landlock, LandlockError};
+use vmm::vm::Vm;
 use vmm::vm_config;
 #[cfg(feature = "fw_cfg")]
 use vmm::vm_config::FwCfgConfig;
@@ -34,14 +40,49 @@ use vmm::vm_config::IvshmemConfig;
 use vmm::vm_config::{
     BalloonConfig, ConsoleConfig, DeviceConfig, DiskConfig, FsConfig, GenericVhostUserConfig,
     LandlockConfig, NetConfig, NumaConfig, PciSegmentConfig, PlatformConfig, PmemConfig,
-    RateLimiterGroupConfig, RngConfig, SerialConfig, TpmConfig, UserDeviceConfig, VdpaConfig,
-    VmConfig, VsockConfig,
+    RateLimiterGroupConfig, RngConfig, RtcConfig, SerialConfig, TpmConfig, UserDeviceConfig,
+    VdpaConfig, VmConfig, VsockConfig,
 };
 use vmm_sys_util::eventfd::EventFd;
 use vmm_sys_util::signal::block_signal;
 
 use crate::logger::Logger;
 
+// Linux exposes seccomp's SIGSYS payload via the siginfo_t layout; this struct mirrors the
+// fields we need so the handler can read the syscall and arch.
+#[repr(C)]
+struct SeccompSiginfo {
+    si_signo: libc::c_int,
+    si_errno: libc::c_int,
+    si_code: libc::c_int,
+    _pad0: libc::c_int,
+    si_call_addr: *mut libc::c_void,
+    si_syscall: libc::c_int,
+    si_arch: libc::c_uint,
+}
+
+fn handle_sigsys(info: &libc::siginfo_t) {
+    // SAFETY: The handler only reads the provided siginfo pointer, writes a
+    // diagnostic message, and then delegates to the default SIGSYS handler.
+    unsafe {
+        let current_thread = thread::current();
+        let thread_name = current_thread.name().unwrap_or("<unknown>");
+        let tid = libc::syscall(libc::SYS_gettid) as i64;
+        let info = &*(info as *const libc::siginfo_t as *const SeccompSiginfo);
+        eprintln!(
+            concat!(
+                "\n==== Possible seccomp violation ====\n",
+                "Syscall number: {} (arch: {:#x}, tid: {}, thread: {})\n",
+                "Try running with `strace -ff` to identify the cause and open an issue: ",
+                "https://github.com/cloud-hypervisor/cloud-hypervisor/issues/new",
+            ),
+            info.si_syscall, info.si_arch, tid, thread_name,
+        );
+
+        low_level::emulate_default_handler(SIGSYS).unwrap();
+    }
+}
+
 #[cfg(feature = "dhat-heap")]
 #[global_allocator]
 static ALLOC: dhat::Alloc = dhat::Alloc;
@@ -49,32 +90,32 @@ static ALLOC: dhat::Alloc = dhat::Alloc;
 #[derive(Error, Debug)]
 enum Error {
     #[error("Failed to create API EventFd")]
-    CreateApiEventFd(#[source] std::io::Error),
+    CreateApiEventFd(#[source] io::Error),
     #[cfg(feature = "guest_debug")]
     #[error("Failed to create Debug EventFd")]
-    CreateDebugEventFd(#[source] std::io::Error),
+    CreateDebugEventFd(#[source] io::Error),
     #[error("Failed to create exit EventFd")]
-    CreateExitEventFd(#[source] std::io::Error),
+    CreateExitEventFd(#[source] io::Error),
     #[error("Failed to open hypervisor interface (is hypervisor interface available?)")]
     CreateHypervisor(#[source] hypervisor::HypervisorError),
     #[error("Failed to start the VMM thread")]
     StartVmmThread(#[source] vmm::Error),
     #[error("Error parsing config")]
-    ParsingConfig(#[source] vmm::config::Error),
+    ParsingConfig(#[source] config::Error),
     #[error("Error creating VM")]
-    VmCreate(#[source] vmm::api::ApiError),
+    VmCreate(#[source] api::ApiError),
     #[error("Error booting VM")]
-    VmBoot(#[source] vmm::api::ApiError),
+    VmBoot(#[source] api::ApiError),
     #[error("Error restoring VM")]
-    VmRestore(#[source] vmm::api::ApiError),
+    VmRestore(#[source] api::ApiError),
     #[error("Error parsing restore")]
-    ParsingRestore(#[source] vmm::config::Error),
+    ParsingRestore(#[source] config::Error),
     #[error("Failed to join on VMM thread: {0:?}")]
-    ThreadJoin(std::boxed::Box<dyn std::any::Any + std::marker::Send>),
+    ThreadJoin(Box<dyn any::Any + Send>),
     #[error("VMM thread exited with error")]
     VmmThread(#[source] vmm::Error),
     #[error("Error parsing --api-socket")]
-    ParsingApiSocket(#[source] std::num::ParseIntError),
+    ParsingApiSocket(#[source] num::ParseIntError),
     #[error("Error parsing --event-monitor")]
     ParsingEventMonitor(#[source] option_parser::OptionParserError),
     #[cfg(feature = "dbus_api")]
@@ -86,7 +127,7 @@ enum Error {
     #[error("Error parsing --event-monitor: path or fd required")]
     BareEventMonitor,
     #[error("Error doing event monitor I/O")]
-    EventMonitorIo(#[source] std::io::Error),
+    EventMonitorIo(#[source] io::Error),
     #[error("Event monitor thread failed")]
     EventMonitorThread(#[source] vmm::Error),
     #[cfg(feature = "guest_debug")]
@@ -96,7 +137,7 @@ enum Error {
     #[error("Error parsing --gdb: path required")]
     BareGdb,
     #[error("Error creating log file")]
-    LogFileCreation(#[source] std::io::Error),
+    LogFileCreation(#[source] io::Error),
     #[error("Error parsing logger format")]
     LoggerFormat(#[source] logger::Error),
     #[error("Error setting up logger")]
@@ -112,13 +153,13 @@ enum Error {
 #[derive(Error, Debug)]
 enum FdTableError {
     #[error("Failed to create event fd")]
-    CreateEventFd(#[source] std::io::Error),
+    CreateEventFd(#[source] io::Error),
     #[error("Failed to obtain file limit")]
-    GetRLimit(#[source] std::io::Error),
+    GetRLimit(#[source] io::Error),
     #[error("Error calling fcntl with F_GETFD")]
-    GetFd(#[source] std::io::Error),
+    GetFd(#[source] io::Error),
     #[error("Failed to duplicate file handle")]
-    Dup2(#[source] std::io::Error),
+    Dup2(#[source] io::Error),
 }
 
 fn prepare_default_values() -> (String, String, String) {
@@ -303,7 +344,12 @@ fn get_cli_options_sorted(
             .group("logging"),
         Arg::new("log-format")
             .long("log-format")
-            .help("Log format. Available tokens: {boottime}, {wallclock}, {pid}, {tid}, {thread}, {level}, {location}, {msg}")
+            .help(
+                "Log format. Common tokens: {boottime}, {wallclock}, {glog}, \
+                 {localglog}, {thread}, {level}, {location}, {msg}. See \
+                 docs/logging.md for the full list (per-field date/time tokens, \
+                 local-time variants, glog level letter).",
+            )
             .num_args(1)
             .default_value(logger::DEFAULT_FORMAT)
             .group("logging"),
@@ -316,7 +362,7 @@ fn get_cli_options_sorted(
                      hotplug_method=acpi|virtio-mem,\
                      hotplug_size=<hotpluggable_memory_size>,\
                      hotplugged_size=<hotplugged_memory_size>,\
-                     prefault=on|off,thp=on|off\"",
+                     prefault=on|off,reserve=on|off,thp=on|off\"",
             )
             .default_value(default_memory)
             .group("vm-config"),
@@ -330,7 +376,7 @@ fn get_cli_options_sorted(
                      host_numa_node=<node_id>,\
                      id=<zone_identifier>,hotplug_size=<hotpluggable_memory_size>,\
                      hotplugged_size=<hotplugged_memory_size>,\
-                     prefault=on|off\"",
+                     prefault=on|off,reserve=on|off\"",
             )
             .num_args(1..)
             .action(ArgAction::Append)
@@ -399,10 +445,16 @@ fn get_cli_options_sorted(
             .help(RngConfig::SYNTAX)
             .default_value(default_rng)
             .group("vm-config"),
+        Arg::new("rtc")
+            .long("rtc")
+            .help(RtcConfig::SYNTAX)
+            .num_args(0..=1)
+            .default_missing_value("")
+            .group("vm-config"),
         Arg::new("seccomp")
             .long("seccomp")
             .num_args(1)
-            .value_parser(["true", "false", "log"])
+            .value_parser(["true", "false", "log", "errno"])
             .default_value("true"),
         Arg::new("serial")
             .long("serial")
@@ -514,13 +566,12 @@ fn start_vmm(
         _ => LevelFilter::Trace,
     };
 
-    let log_file: Box<dyn std::io::Write + Send> = if let Some(ref file) =
-        cmd_arguments.get_one::<String>("log-file")
-    {
-        Box::new(std::fs::File::create(std::path::Path::new(file)).map_err(Error::LogFileCreation)?)
-    } else {
-        Box::new(std::io::stderr())
-    };
+    let log_file: Box<dyn io::Write + Send> =
+        if let Some(ref file) = cmd_arguments.get_one::<String>("log-file") {
+            Box::new(File::create(Path::new(file)).map_err(Error::LogFileCreation)?)
+        } else {
+            Box::new(io::stderr())
+        };
 
     let format = cmd_arguments.get_one::<String>("log-format").unwrap();
     let logger = Logger::new(log_file, format).map_err(Error::LoggerFormat)?;
@@ -537,6 +588,7 @@ fn start_vmm(
             "true" => SeccompAction::Trap,
             "false" => SeccompAction::Allow,
             "log" => SeccompAction::Log,
+            "errno" => SeccompAction::Errno(libc::EPERM as u32),
             val => {
                 // The user providing an invalid value will be rejected
                 panic!("Invalid parameter {val} for \"--seccomp\" flag");
@@ -547,20 +599,13 @@ fn start_vmm(
     };
 
     if seccomp_action == SeccompAction::Trap {
-        // SAFETY: We only using signal_hook for managing signals and only execute signal
+        // SAFETY: We only use signal_hook for managing signals and only execute signal
         // handler safe functions (writing to stderr) and manipulating signals.
         unsafe {
-            signal_hook::low_level::register(signal_hook::consts::SIGSYS, || {
-                eprintln!(
-                    "\n==== Possible seccomp violation ====\n\
-                Try running with `strace -ff` to identify the cause and open an issue: \
-                https://github.com/cloud-hypervisor/cloud-hypervisor/issues/new"
-                );
-                signal_hook::low_level::emulate_default_handler(SIGSYS).unwrap();
-            })
+            signal_hook_registry::register_sigaction(SIGSYS, handle_sigsys)
+                .map_err(|e| error!("Error adding SIGSYS signal handler: {e}"))
+                .ok();
         }
-        .map_err(|e| error!("Error adding SIGSYS signal handler: {e}"))
-        .ok();
     }
 
     // SAFETY: Trivially safe.
@@ -571,7 +616,7 @@ fn start_vmm(
     // Before we start any threads, mask the signals we'll be
     // installing handlers for, to make sure they only ever run on the
     // dedicated signal handling thread we'll start in a bit.
-    for sig in &vmm::vm::Vm::HANDLED_SIGNALS {
+    for sig in &Vm::HANDLED_SIGNALS {
         if let Err(e) = block_signal(*sig) {
             error!("Error blocking signals: {e}");
         }
@@ -594,7 +639,7 @@ fn start_vmm(
         parser.parse(gdb_config).map_err(Error::ParsingGdb)?;
 
         if parser.is_set("path") {
-            Some(std::path::PathBuf::from(parser.get("path").unwrap()))
+            Some(PathBuf::from(parser.get("path").unwrap()))
         } else {
             return Err(Error::BareGdb);
         }
@@ -630,7 +675,7 @@ fn start_vmm(
                 Ok(Some(unsafe { File::from_raw_fd(fd) }))
             } else if parser.is_set("path") {
                 Ok(Some(
-                    std::fs::OpenOptions::new()
+                    fs::OpenOptions::new()
                         .write(true)
                         .create(true)
                         .truncate(true)
@@ -679,7 +724,6 @@ fn start_vmm(
             monitor,
             &seccomp_action,
             landlock_enable,
-            hypervisor.hypervisor_type(),
             exit_evt.try_clone().unwrap(),
         )
         .map_err(Error::EventMonitorThread)?;
@@ -725,18 +769,18 @@ fn start_vmm(
 
             // Create and boot the VM based off the VM config we just built.
             let sender = api_request_sender.clone();
-            vmm::api::VmCreate
+            api::VmCreate
                 .send(
                     api_evt.try_clone().unwrap(),
                     api_request_sender,
                     Box::new(vm_config),
                 )
                 .map_err(Error::VmCreate)?;
-            vmm::api::VmBoot
+            api::VmBoot
                 .send(api_evt.try_clone().unwrap(), sender, ())
                 .map_err(Error::VmBoot)?;
         } else if let Some(restore_params) = cmd_arguments.get_one::<String>("restore") {
-            vmm::api::VmRestore
+            api::VmRestore
                 .send(
                     api_evt.try_clone().unwrap(),
                     api_request_sender,
@@ -817,7 +861,7 @@ fn expand_fdtable() -> Result<(), FdTableError> {
     let table_size = if limits.rlim_cur == libc::RLIM_INFINITY {
         4096
     } else {
-        std::cmp::min(limits.rlim_cur, 4096) as libc::c_int
+        cmp::min(limits.rlim_cur, 4096) as libc::c_int
     };
 
     // The first 3 handles are stdin, stdout, stderr. We don't want to touch
@@ -851,12 +895,18 @@ fn expand_fdtable() -> Result<(), FdTableError> {
 }
 
 fn main() {
+    #[cfg(feature = "tdx")]
+    compile_error!("Feature 'tdx' is broken.");
     #[cfg(all(feature = "tdx", feature = "sev_snp"))]
     compile_error!("Feature 'tdx' and 'sev_snp' are mutually exclusive.");
     #[cfg(all(feature = "sev_snp", not(target_arch = "x86_64")))]
     compile_error!("Feature 'sev_snp' needs target 'x86_64'");
     #[cfg(all(feature = "fw_cfg", target_arch = "riscv64"))]
     compile_error!("Feature 'fw_cfg' needs targets 'x86_64' or 'aarch64'");
+    #[cfg(all(feature = "igvm", not(any(feature = "kvm", feature = "mshv"))))]
+    compile_error!("Feature 'igvm' needs feature 'kvm' or 'mshv'");
+    #[cfg(all(feature = "sev_snp", not(any(feature = "kvm", feature = "mshv"))))]
+    compile_error!("Feature 'sev_snp' needs feature 'kvm' or 'mshv'");
 
     #[cfg(feature = "dhat-heap")]
     let _profiler = dhat::Profiler::new_heap();
@@ -870,6 +920,11 @@ fn main() {
 
     if cmd_arguments.get_flag("version") {
         println!("{} {}", env!("CARGO_BIN_NAME"), env!("BUILD_VERSION"));
+        let migration_protocol_versions = protocol::supported_protocol_versions()
+            .map(|version| version.to_string())
+            .collect::<Vec<_>>()
+            .join(", ");
+        println!("Migration Protocol Versions: {migration_protocol_versions}");
 
         if cmd_arguments.get_count("v") != 0 {
             println!("Enabled features: {:?}", vmm::feature_list());
@@ -886,14 +941,20 @@ fn main() {
         Ok(p) => p,
         Err(top_error) => {
             cloud_hypervisor::cli_print_error_chain(&top_error, "Cloud Hypervisor", |_, _, _| None);
-            std::process::exit(1);
+            process::exit(1);
         }
     };
 
     let vmm_result = start_vmm(&cmd_arguments, &api_socket_path, api_socket_fd);
 
-    if let Some(ref p) = api_socket_path {
-        let _ = std::fs::remove_file(p);
+    // Remove the socket only when we actually ran (Ok): a failed start may mean
+    // another instance already holds the path, and removing it would clobber
+    // that live socket. A stale socket left by a crash is cleared under the lock
+    // on the next start.
+    if vmm_result.is_ok()
+        && let Some(ref api_socket_path) = api_socket_path
+    {
+        let _ = fs::remove_file(api_socket_path);
     }
 
     let exit_code = match vmm_result {
@@ -910,7 +971,7 @@ fn main() {
     #[cfg(feature = "dhat-heap")]
     drop(_profiler);
 
-    std::process::exit(exit_code);
+    process::exit(exit_code);
 }
 
 #[cfg(test)]
@@ -975,6 +1036,7 @@ mod unit_tests {
                 features: CpuFeatures::default(),
                 nested: true,
                 core_scheduling: CoreScheduling::Vm,
+                profile: Default::default(),
             },
             memory: MemoryConfig {
                 size: 536_870_912,
@@ -986,6 +1048,7 @@ mod unit_tests {
                 hugepages: false,
                 hugepage_size: None,
                 prefault: false,
+                reserve: false,
                 zones: None,
                 thp: true,
             },
@@ -1039,6 +1102,7 @@ mod unit_tests {
             iommu: false,
             numa: None,
             watchdog: false,
+            rtc: None,
             #[cfg(feature = "guest_debug")]
             gdb: false,
             pci_segments: None,
@@ -1693,12 +1757,12 @@ mod unit_tests {
                     "--serial",
                     "null",
                     "--console",
-                    "tty,pci_segment=1,pci_device_id=7",
+                    "tty,pci_segment=0,pci_device_id=7",
                 ],
                 r#"{
                     "payload": {"kernel": "/path/to/kernel"},
                     "serial": {"mode": "Null"},
-                    "console": {"mode": "Tty", "iommu": false, "pci_segment": 1, "pci_device_id": 7}
+                    "console": {"mode": "Tty", "iommu": false, "pci_segment": 0, "pci_device_id": 7}
                 }"#,
                 true,
             ),