build: Release v38.0

Signed-off-by: Bo Chen <chen.bo@intel.com>

.github/workflows/audit.yaml

@@ -4,7 +4,6 @@ on:
     paths: 
       - '**/Cargo.toml'
       - '**/Cargo.lock'
-
 jobs:
   security_audit:
     name: Audit

.github/workflows/build.yaml

@@ -1,11 +1,9 @@
 name: Cloud Hypervisor Build
-on: [pull_request, merge_group]
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
+on: [pull_request, create]
 
 jobs:
   build:
+    if: github.event_name == 'pull_request'
     name: Build
     runs-on: ubuntu-latest
     strategy:

.github/workflows/dco.yaml

@@ -1,9 +1,8 @@
 name: DCO
-on: [pull_request, merge_group]
-
+on:
+  pull_request:
 jobs:
   check:
-    name: DCO Check ("Signed-Off-By")
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v4
@@ -12,7 +11,6 @@ jobs:
       with:
         python-version: '3.x'
     - name: Check DCO
-      if: ${{ github.event_name == 'pull_request' }}
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       run: |

.github/workflows/docker-image.yaml

@@ -1,13 +1,11 @@
 name: Cloud Hypervisor's Docker image update
+
 on:
   push:
     branches: main
     paths: resources/Dockerfile
   pull_request:
     paths: resources/Dockerfile
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
 
 env:
   REGISTRY: ghcr.io

.github/workflows/fuzz-build.yaml

@@ -1,11 +1,9 @@
 name: Cloud Hypervisor Cargo Fuzz Build
-on: [pull_request, merge_group]
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
+on: [pull_request, create]
 
 jobs:
   build:
+    if: github.event_name == 'pull_request'
     name: Cargo Fuzz Build
     runs-on: ubuntu-latest
     strategy:
@@ -14,6 +12,8 @@ jobs:
           - nightly
         target:
           - x86_64-unknown-linux-gnu
+    env:
+      RUSTFLAGS: -D warnings
     steps:
       - name: Code checkout
         uses: actions/checkout@v4
@@ -24,8 +24,8 @@ jobs:
             target: ${{ matrix.target }}
             override: true
       - name: Install Cargo fuzz
-        # Temporary fix for cargo-fuzz on latest nightly: https://github.com/rust-fuzz/cargo-fuzz/issues/276
-        #run: cargo install cargo-fuzz
-        run: cargo install --git https://github.com/rust-fuzz/cargo-fuzz --rev b4df3e58f767b5cad8d1aa6753961003f56f3609
-      - name: Cargo Fuzz Build
+        run: cargo install cargo-fuzz
+      - name: Fuzz Build
         run: cargo fuzz build
+      - name: Fuzz Check
+        run: cargo fuzz check

.github/workflows/gitlint.yaml

@@ -1,4 +1,5 @@
 name: Commit messages check
+
 on:
   pull_request:
 

.github/workflows/hadolint.yaml

@@ -1,4 +1,5 @@
 name: Lint Dockerfile
+
 on:
   push:
     paths:

.github/workflows/integration-arm64.yaml

@@ -1,54 +0,0 @@
-name: Cloud Hypervisor Tests (ARM64)
-on: [pull_request, merge_group]
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  build:
-    timeout-minutes: 60
-    name: Tests (ARM64)
-    runs-on: focal-arm64
-    steps:
-      - name: Fix workspace permissions
-        run: sudo chown -R github-runner:github-runner ${GITHUB_WORKSPACE}
-      - name: Code checkout
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-      - name: Run unit tests (musl)
-        run: scripts/dev_cli.sh tests --unit --libc musl
-      - name: Load openvswitch module
-        run: sudo modprobe openvswitch
-      - name: Run integration tests (musl)
-        timeout-minutes: 30
-        run: scripts/dev_cli.sh tests --integration --libc musl
-      - name: Install Azure CLI
-        if: ${{ github.event_name != 'pull_request' }}
-        run: |
-          sudo apt install -y ca-certificates curl apt-transport-https lsb-release gnupg
-          curl -sL https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor | sudo tee /etc/apt/trusted.gpg.d/microsoft.gpg > /dev/null
-          echo "deb [arch=arm64] https://packages.microsoft.com/repos/azure-cli/ focal main" | sudo tee /etc/apt/sources.list.d/azure-cli.list
-          sudo apt update
-          sudo apt install -y azure-cli
-      - name: Download Windows image
-        if: ${{ github.event_name != 'pull_request' }}
-        shell: bash
-        run: |
-          IMG_BASENAME=windows-11-iot-enterprise-aarch64.raw
-          IMG_PATH=$HOME/workloads/$IMG_BASENAME
-          IMG_GZ_PATH=$HOME/workloads/$IMG_BASENAME.gz
-          IMG_GZ_BLOB_NAME=windows-11-iot-enterprise-aarch64-9-min.raw.gz
-          cp "scripts/$IMG_BASENAME.sha1" "$HOME/workloads/"
-          pushd "$HOME/workloads"
-          if sha1sum "$IMG_BASENAME.sha1" --check; then
-              exit
-          fi
-          popd
-          mkdir -p "$HOME/workloads"
-          az storage blob download --container-name private-images --file "$IMG_GZ_PATH" --name "$IMG_GZ_BLOB_NAME" --connection-string "${{ secrets.CH_PRIVATE_IMAGES }}"
-          gzip -d $IMG_GZ_PATH
-      - name: Run Windows guest integration tests
-        if: ${{ github.event_name != 'pull_request' }}
-        timeout-minutes: 30
-        run: scripts/dev_cli.sh tests --integration-windows --libc musl

.github/workflows/integration-metrics.yaml

@@ -1,22 +0,0 @@
-name: Cloud Hypervisor Tests (Metrics)
-on:
-  push:
-    branches:
-      - main
-
-jobs:
-  build:
-    name: Tests (Metrics)
-    runs-on: jammy-metrics
-    env:
-      METRICS_PUBLISH_KEY: ${{ secrets.METRICS_PUBLISH_KEY }}
-    steps:
-      - name: Code checkout
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-      - name: Run metrics tests
-        timeout-minutes: 60
-        run: scripts/dev_cli.sh tests --metrics -- -- --report-file /root/workloads/metrics.json
-      - name: Upload metrics report
-        run: 'curl -X PUT https://ch-metrics.azurewebsites.net/api/publishmetrics -H "x-functions-key: $METRICS_PUBLISH_KEY" -T ~/workloads/metrics.json'

.github/workflows/integration-rate-limiter.yaml

@@ -1,28 +0,0 @@
-name: Cloud Hypervisor Tests (Rate-Limiter)
-on: [merge_group, pull_request]
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  build:
-    name: Tests (Rate-Limiter)
-    runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-latest' || 'jammy-rate-limiter' }}
-    env:
-      AUTH_DOWNLOAD_TOKEN: ${{ secrets.AUTH_DOWNLOAD_TOKEN }}
-    steps:
-      - name: Fix workspace permissions
-        if: ${{ github.event_name != 'pull_request' }}
-        run: sudo chown -R github-runner:github-runner ${GITHUB_WORKSPACE}
-      - name: Code checkout
-        if: ${{ github.event_name != 'pull_request' }}
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-      - name: Run rate-limiter integration tests
-        if: ${{ github.event_name != 'pull_request' }}
-        timeout-minutes: 10
-        run: scripts/dev_cli.sh tests --integration-rate-limiter
-      - name: Skipping build for PR
-        if: ${{ github.event_name == 'pull_request' }}
-        run: echo "Skipping build for PR"

.github/workflows/integration-sgx.yaml

@@ -1,32 +0,0 @@
-name: Cloud Hypervisor Tests (SGX)
-on: [merge_group, pull_request]
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  build:
-    name: Tests (SGX)
-    runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-latest' || 'jammy-sgx' }}
-    env:
-      AUTH_DOWNLOAD_TOKEN: ${{ secrets.AUTH_DOWNLOAD_TOKEN }}
-    steps:
-      - name: Fix workspace permissions
-        if: ${{ github.event_name != 'pull_request' }}
-        run: sudo chown -R github-runner:github-runner ${GITHUB_WORKSPACE}
-      - name: Code checkout
-        if: ${{ github.event_name != 'pull_request' }}
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-      - name: Run SGX integration tests
-        if: ${{ github.event_name != 'pull_request' }}
-        timeout-minutes: 10
-        run: scripts/dev_cli.sh tests --integration-sgx
-      - name: Run SGX integration tests for musl
-        if: ${{ github.event_name != 'pull_request' }}
-        timeout-minutes: 10
-        run: scripts/dev_cli.sh tests --integration-sgx --libc musl
-      - name: Skipping build for PR
-        if: ${{ github.event_name == 'pull_request' }}
-        run: echo "Skipping build for PR"

.github/workflows/integration-vfio.yaml

@@ -1,32 +0,0 @@
-name: Cloud Hypervisor Tests (VFIO)
-on: [merge_group, pull_request]
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  build:
-    name: Tests (VFIO)
-    runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-latest' || 'jammy-vfio' }}
-    env:
-      AUTH_DOWNLOAD_TOKEN: ${{ secrets.AUTH_DOWNLOAD_TOKEN }}
-    steps:
-      - name: Fix workspace permissions
-        if: ${{ github.event_name != 'pull_request' }}
-        run: sudo chown -R github-runner:github-runner ${GITHUB_WORKSPACE}
-      - name: Code checkout
-        if: ${{ github.event_name != 'pull_request' }}
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-      - name: Run VFIO integration tests
-        if: ${{ github.event_name != 'pull_request' }}
-        timeout-minutes: 15
-        run: scripts/dev_cli.sh tests --integration-vfio
-      - name: Run VFIO integration tests for musl
-        if: ${{ github.event_name != 'pull_request' }}
-        timeout-minutes: 15
-        run: scripts/dev_cli.sh tests --integration-vfio --libc musl
-      - name: Skipping build for PR
-        if: ${{ github.event_name == 'pull_request' }}
-        run: echo "Skipping build for PR"

.github/workflows/integration-windows.yaml

@@ -1,50 +0,0 @@
-name: Cloud Hypervisor Tests (Windows Guest)
-on: [merge_group, pull_request]
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  build:
-    name: Tests (Windows Guest)
-    runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-latest' || 'garm-jammy-16' }}
-    steps:
-      - name: Code checkout
-        if: ${{ github.event_name != 'pull_request' }}
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-      - name: Install Docker
-        if: ${{ github.event_name != 'pull_request' }}
-        run: |
-          sudo apt-get update
-          sudo apt-get -y install ca-certificates curl gnupg
-          curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg
-          sudo chmod a+r /usr/share/keyrings/docker-archive-keyring.gpg
-          echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
-          sudo apt-get update
-          sudo apt install -y docker-ce docker-ce-cli
-      - name: Install Azure CLI
-        if: ${{ github.event_name != 'pull_request' }}
-        run: |
-          sudo apt install -y ca-certificates curl apt-transport-https lsb-release gnupg
-          curl -sL https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor | sudo tee /etc/apt/trusted.gpg.d/microsoft.gpg > /dev/null
-          echo "deb [arch=amd64] https://packages.microsoft.com/repos/azure-cli/ jammy main" | sudo tee /etc/apt/sources.list.d/azure-cli.list
-          sudo apt update
-          sudo apt install -y azure-cli
-      - name: Download Windows image
-        if: ${{ github.event_name != 'pull_request' }}
-        run: |
-          mkdir $HOME/workloads
-          az storage blob download --container-name private-images --file "$HOME/workloads/windows-server-2022-amd64-2.raw" --name windows-server-2022-amd64-2.raw --connection-string "${{ secrets.CH_PRIVATE_IMAGES }}"
-      - name: Run Windows guest integration tests
-        if: ${{ github.event_name != 'pull_request' }}
-        timeout-minutes: 15
-        run: scripts/dev_cli.sh tests --integration-windows
-      - name: Run Windows guest integration tests for musl
-        if: ${{ github.event_name != 'pull_request' }}
-        timeout-minutes: 15
-        run: scripts/dev_cli.sh tests --integration-windows --libc musl
-      - name: Skipping build for PR
-        if: ${{ github.event_name == 'pull_request' }}
-        run: echo "Skipping build for PR"

.github/workflows/integration-x86-64.yaml

@@ -1,52 +0,0 @@
-name: Cloud Hypervisor Tests (x86-64)
-on: [pull_request, merge_group]
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  build:
-    timeout-minutes: 60
-    strategy:
-      fail-fast: false
-      matrix:
-        runner: ['garm-jammy', "garm-jammy-amd"]
-        libc: ["musl", 'gnu']
-    name: Tests (x86-64)
-    runs-on: ${{ github.event_name == 'pull_request' && !(matrix.runner == 'garm-jammy' && matrix.libc == 'gnu') && 'ubuntu-latest' || format('{0}-16', matrix.runner) }}
-    steps:
-      - name: Code checkout
-        if: ${{ github.event_name != 'pull_request' || (matrix.runner == 'garm-jammy' && matrix.libc == 'gnu') }}
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-      - name: Install Docker
-        if: ${{ github.event_name != 'pull_request' || (matrix.runner == 'garm-jammy' && matrix.libc == 'gnu') }}
-        run: |
-          sudo apt-get update
-          sudo apt-get -y install ca-certificates curl gnupg
-          curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg
-          sudo chmod a+r /usr/share/keyrings/docker-archive-keyring.gpg
-          echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
-          sudo apt-get update
-          sudo apt install -y docker-ce docker-ce-cli
-      - name: Prepare for VDPA
-        if: ${{ github.event_name != 'pull_request' || (matrix.runner == 'garm-jammy' && matrix.libc == 'gnu') }}
-        run: scripts/prepare_vdpa.sh
-      - name: Run unit tests
-        if: ${{ github.event_name != 'pull_request' || (matrix.runner == 'garm-jammy' && matrix.libc == 'gnu') }}
-        run: scripts/dev_cli.sh tests --unit --libc ${{ matrix.libc }}
-      - name: Load openvswitch module
-        if: ${{ github.event_name != 'pull_request' || (matrix.runner == 'garm-jammy' && matrix.libc == 'gnu') }}
-        run: sudo modprobe openvswitch
-      - name: Run integration tests
-        if: ${{ github.event_name != 'pull_request' || (matrix.runner == 'garm-jammy' && matrix.libc == 'gnu') }}
-        timeout-minutes: 40
-        run: scripts/dev_cli.sh tests --integration --libc ${{ matrix.libc }}
-      - name: Run live-migration integration tests
-        if: ${{ github.event_name != 'pull_request' || (matrix.runner == 'garm-jammy' && matrix.libc == 'gnu') }}
-        timeout-minutes: 20
-        run: scripts/dev_cli.sh tests --integration-live-migration --libc ${{ matrix.libc }}
-      - name: Skipping build for PR
-        if: ${{ github.event_name == 'pull_request' && matrix.runner != 'garm-jammy' && matrix.libc != 'gnu' }}
-        run: echo "Skipping build for PR"

.github/workflows/openapi.yaml

@@ -1,5 +1,7 @@
 name: Cloud Hypervisor OpenAPI Validation
-on: [pull_request, merge_group]
+
+on:
+  pull_request:
 
 jobs:
   Validate:

.github/workflows/quality.yaml

@@ -1,11 +1,9 @@
 name: Cloud Hypervisor Quality Checks
-on: [pull_request, merge_group]
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
+on: [pull_request, create]
 
 jobs:
   build:
+    if: github.event_name == 'pull_request'
     name: Quality (clippy, rustfmt)
     runs-on: ubuntu-latest
     continue-on-error: ${{ matrix.experimental }}
@@ -48,8 +46,8 @@ jobs:
           override: true
           components: rustfmt, clippy
 
-      - name: Bisectability Check (default features)
-        if: ${{ github.event_name == 'pull_request' && matrix.target == 'x86_64-unknown-linux-gnu' }}
+      - name: Debug Check (default features)
+        if: ${{ matrix.target == 'x86_64-unknown-linux-gnu' }}
         run: |
           set -e
           commits=$(git rev-list origin/${{ github.base_ref }}..${{ github.sha }})

.github/workflows/release.yaml

@@ -1,12 +1,9 @@
 name: Cloud Hypervisor Release
-on: [create, merge_group]
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }}
-  cancel-in-progress: true
+on: [pull_request, create]
 
 jobs:
   release:
-    if: (github.event_name == 'create' && github.event.ref_type == 'tag') || github.event_name == 'merge_group'
+    if: (github.event_name == 'create' && github.event.ref_type == 'tag') || github.event_name == 'pull_request'
     name: Release
     runs-on: ubuntu-latest
     steps:

.github/workflows/shlint.yaml

@@ -0,0 +1,20 @@
+name: Shell scripts check
+
+on:
+  pull_request:
+  push:
+    branches:
+      - main
+
+jobs:
+  sh-checker:
+    name: Check shell scripts
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v4
+    - name: Run the shell script checkers
+      uses: luizm/action-sh-checker@master
+      env:
+        SHFMT_OPTS: -i 4 -d
+        SHELLCHECK_OPTS: -x --source-path scripts

.gitignore

@@ -6,3 +6,4 @@
 **/rusty-tags.vi
 /rpm/SOURCES
 /.vscode
+/vendor

Cargo.lock

@@ -50,9 +50,9 @@ dependencies = [
 
 [[package]]
 name = "anstyle"
-version = "1.0.1"
+version = "1.0.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3a30da5c5f2d5e72842e00bcb57657162cdabef0931f40e2deb9b4140440cecd"
+checksum = "8901269c6307e8d93993578286ac0edf7f195079ffff5ebdeea6a59ffb7e36bc"
 
 [[package]]
 name = "anstyle-parse"
@@ -237,7 +237,7 @@ checksum = "5fd55a5ba1179988837d24ab4c7cc8ed6efdeff578ede0416b4225a5fca35bd0"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.31",
+ "syn 2.0.48",
 ]
 
 [[package]]
@@ -267,13 +267,13 @@ checksum = "b4eb2cdb97421e01129ccb49169d8279ed21e829929144f4a22a6e54ac549ca1"
 
 [[package]]
 name = "async-trait"
-version = "0.1.74"
+version = "0.1.76"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a66537f1bb974b254c98ed142ff995236e81b9d0fe4db0575f46612cb15eb0f9"
+checksum = "531b97fb4cd3dfdce92c35dedbfdc1f0b9d8091c8ca943d6dae340ef5012d514"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.31",
+ "syn 2.0.48",
 ]
 
 [[package]]
@@ -320,7 +320,7 @@ checksum = "a26b8cea8bb6a81b75a84603b9e096f05fa86db057904ef29be1deee900532bd"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.31",
+ "syn 2.0.48",
 ]
 
 [[package]]
@@ -439,7 +439,7 @@ checksum = "702fc72eb24e5a1e48ce58027a675bc24edd52096d5397d4aea7c6dd9eca0bd1"
 
 [[package]]
 name = "cloud-hypervisor"
-version = "37.1.0"
+version = "38.0.0"
 dependencies = [
  "anyhow",
  "api_client",
@@ -476,9 +476,9 @@ checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7"
 
 [[package]]
 name = "concurrent-queue"
-version = "2.3.0"
+version = "2.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f057a694a54f12365049b0958a1685bb52d567f5593b355fbf685838e873d400"
+checksum = "d16048cd947b08fa32c24458a22f5dc5e835264f689f4f5653210c69fd107363"
 dependencies = [
  "crossbeam-utils",
 ]
@@ -518,12 +518,9 @@ checksum = "2707e3afba5e19b75d582d88bc79237418f2a2a2d673d01cf9b03633b46e98f3"
 
 [[package]]
 name = "crossbeam-utils"
-version = "0.8.16"
+version = "0.8.19"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5a22b2d63d4d1dc0b7f1b6b2747dd0088008a9be28b6ddf0b1e7d335e3037294"
-dependencies = [
- "cfg-if",
-]
+checksum = "248e3bacc7dc6baa3b21e405ee045c3047101a49145e7e9eca583ab4c2ca5345"
 
 [[package]]
 name = "crypto-common"
@@ -537,9 +534,9 @@ dependencies = [
 
 [[package]]
 name = "darling"
-version = "0.20.3"
+version = "0.20.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0209d94da627ab5605dcccf08bb18afa5009cfbef48d8a8b7d7bdbc79be25c5e"
+checksum = "c376d08ea6aa96aafe61237c7200d1241cb177b7d3a542d791f2d118e9cbb955"
 dependencies = [
  "darling_core",
  "darling_macro",
@@ -547,27 +544,27 @@ dependencies = [
 
 [[package]]
 name = "darling_core"
-version = "0.20.3"
+version = "0.20.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "177e3443818124b357d8e76f53be906d60937f0d3a90773a664fa63fa253e621"
+checksum = "33043dcd19068b8192064c704b3f83eb464f91f1ff527b44a4e2b08d9cdb8855"
 dependencies = [
  "fnv",
  "ident_case",
  "proc-macro2",
  "quote",
  "strsim",
- "syn 2.0.31",
+ "syn 2.0.48",
 ]
 
 [[package]]
 name = "darling_macro"
-version = "0.20.3"
+version = "0.20.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "836a9bbc7ad63342d6d6e7b815ccab164bc77a2d95d84bc3117a8c0d5c98e2d5"
+checksum = "c5a91391accf613803c2a9bf9abccdbaa07c54b4244a5b64883f9c3c137c86be"
 dependencies = [
  "darling_core",
  "quote",
- "syn 2.0.31",
+ "syn 2.0.48",
 ]
 
 [[package]]
@@ -614,9 +611,9 @@ dependencies = [
 
 [[package]]
 name = "dhat"
-version = "0.3.2"
+version = "0.3.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4f2aaf837aaf456f6706cb46386ba8dffd4013a757e36f4ea05c20dd46b209a3"
+checksum = "98cd11d84628e233de0ce467de10b8633f4ddaecafadefc86e13b84b8739b827"
 dependencies = [
  "backtrace",
  "lazy_static",
@@ -677,7 +674,7 @@ checksum = "f95e2801cd355d4a1a3e3953ce6ee5ae9603a5c833455343a8bfe3f44d418246"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.31",
+ "syn 2.0.48",
 ]
 
 [[package]]
@@ -810,9 +807,9 @@ checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
 
 [[package]]
 name = "futures"
-version = "0.3.28"
+version = "0.3.30"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "23342abe12aba583913b2e62f22225ff9c950774065e4bfb61a19cd9770fec40"
+checksum = "645c6916888f6cb6350d2550b80fb63e734897a8498abe35cfb732b6487804b0"
 dependencies = [
  "futures-channel",
  "futures-core",
@@ -825,9 +822,9 @@ dependencies = [
 
 [[package]]
 name = "futures-channel"
-version = "0.3.28"
+version = "0.3.30"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "955518d47e09b25bbebc7a18df10b81f0c766eaf4c4f1cccef2fca5f2a4fb5f2"
+checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78"
 dependencies = [
  "futures-core",
  "futures-sink",
@@ -841,9 +838,9 @@ checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d"
 
 [[package]]
 name = "futures-executor"
-version = "0.3.28"
+version = "0.3.30"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ccecee823288125bd88b4d7f565c9e58e41858e47ab72e8ea2d64e93624386e0"
+checksum = "a576fc72ae164fca6b9db127eaa9a9dda0d61316034f33a0a0d4eda41f02b01d"
 dependencies = [
  "futures-core",
  "futures-task",
@@ -852,9 +849,9 @@ dependencies = [
 
 [[package]]
 name = "futures-io"
-version = "0.3.28"
+version = "0.3.30"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4fff74096e71ed47f8e023204cfd0aa1289cd54ae5430a9523be060cdb849964"
+checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1"
 
 [[package]]
 name = "futures-lite"
@@ -886,32 +883,32 @@ dependencies = [
 
 [[package]]
 name = "futures-macro"
-version = "0.3.28"
+version = "0.3.30"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72"
+checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.31",
+ "syn 2.0.48",
 ]
 
 [[package]]
 name = "futures-sink"
-version = "0.3.28"
+version = "0.3.30"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f43be4fe21a13b9781a69afa4985b0f6ee0e1afab2c6f454a8cf30e2b2237b6e"
+checksum = "9fb8e00e87438d937621c1c6269e53f536c14d3fbd6a042bb24879e57d474fb5"
 
 [[package]]
 name = "futures-task"
-version = "0.3.29"
+version = "0.3.30"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "efd193069b0ddadc69c46389b740bbccdd97203899b48d09c5f7969591d6bae2"
+checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004"
 
 [[package]]
 name = "futures-util"
-version = "0.3.28"
+version = "0.3.30"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "26b01e40b772d54cf6c6d721c1d1abd0647a0106a12ecaa1c186273392a69533"
+checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48"
 dependencies = [
  "futures-channel",
  "futures-core",
@@ -927,9 +924,9 @@ dependencies = [
 
 [[package]]
 name = "gdbstub"
-version = "0.7.0"
+version = "0.7.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "09a8b954f9d02b74fe8e89a1c77bd9a6b8206713ebf1b272bfad9573b4a86f88"
+checksum = "6341b3480afbb34eaefc7f92713bc92f2d83e338aaa1c44192f9c2956f4a4903"
 dependencies = [
  "bitflags 2.4.1",
  "cfg-if",
@@ -1152,7 +1149,7 @@ dependencies = [
 [[package]]
 name = "kvm-bindings"
 version = "0.7.0"
-source = "git+https://github.com/cloud-hypervisor/kvm-bindings?branch=ch-live-upgrade-stable-37.x#f03fc575cdf20c3af9ca3d4d203f171943d95be4"
+source = "git+https://github.com/cloud-hypervisor/kvm-bindings?branch=ch-v0.7.0#2dcf85d4f8aa55befcaa996b699ddb18ec9ed059"
 dependencies = [
  "serde",
  "serde_derive",
@@ -1179,9 +1176,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
 
 [[package]]
 name = "libc"
-version = "0.2.151"
+version = "0.2.153"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "302d7ab3130588088d277783b1e2d2e10c9e9e4a16dd9050e6ec93fb3e7048f4"
+checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd"
 
 [[package]]
 name = "libssh2-sys"
@@ -1431,9 +1428,9 @@ dependencies = [
 
 [[package]]
 name = "openssl-sys"
-version = "0.9.93"
+version = "0.9.99"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "db4d56a4c0478783083cfafcc42493dd4a981d41669da64b4572a2a089b51b1d"
+checksum = "22e1bf214306098e4832460f797824c05d25aacdf896f64a985fb0fd992454ae"
 dependencies = [
  "cc",
  "libc",
@@ -1560,22 +1557,22 @@ dependencies = [
 
 [[package]]
 name = "pin-project"
-version = "1.1.3"
+version = "1.1.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fda4ed1c6c173e3fc7a83629421152e01d7b1f9b7f65fb301e490e8cfc656422"
+checksum = "0302c4a0442c456bd56f841aee5c3bfd17967563f6fadc9ceb9f9c23cf3807e0"
 dependencies = [
  "pin-project-internal",
 ]
 
 [[package]]
 name = "pin-project-internal"
-version = "1.1.3"
+version = "1.1.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405"
+checksum = "266c042b60c9c76b8d53061e52b2e0d1116abc57cefc8c5cd671619a56ac3690"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.31",
+ "syn 2.0.48",
 ]
 
 [[package]]
@@ -1652,7 +1649,7 @@ dependencies = [
  "proc-macro2",
  "quote",
  "regex",
- "syn 2.0.31",
+ "syn 2.0.48",
 ]
 
 [[package]]
@@ -1741,9 +1738,9 @@ dependencies = [
 
 [[package]]
 name = "quote"
-version = "1.0.33"
+version = "1.0.35"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae"
+checksum = "291ec9ab5efd934aaf503a6466c5d5251535d108ee747472c3977cc5acc868ef"
 dependencies = [
  "proc-macro2",
 ]
@@ -1788,6 +1785,7 @@ checksum = "8edc89eaa583cf6bc4c6ef16a219f0a60d342ca3bf0eae793560038ac8af1795"
 name = "rate_limiter"
 version = "0.1.0"
 dependencies = [
+ "epoll",
  "libc",
  "log",
  "thiserror",
@@ -1860,7 +1858,7 @@ checksum = "bce3a7139d2ee67d07538ee5dba997364fbc243e7e7143e96eb830c74bfaa082"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.31",
+ "syn 2.0.48",
 ]
 
 [[package]]
@@ -1925,22 +1923,22 @@ dependencies = [
 
 [[package]]
 name = "serde"
-version = "1.0.168"
+version = "1.0.196"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d614f89548720367ded108b3c843be93f3a341e22d5674ca0dd5cd57f34926af"
+checksum = "870026e60fa08c69f064aa766c10f10b1d62db9ccd4d0abb206472bee0ce3b32"
 dependencies = [
  "serde_derive",
 ]
 
 [[package]]
 name = "serde_derive"
-version = "1.0.168"
+version = "1.0.196"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d4fe589678c688e44177da4f27152ee2d190757271dc7f1d5b6b9f68d869d641"
+checksum = "33c85360c95e7d137454dc81d9a4ed2b8efd8fbe19cee57357b32b9771fccb67"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.31",
+ "syn 2.0.48",
 ]
 
 [[package]]
@@ -1962,7 +1960,7 @@ checksum = "3081f5ffbb02284dda55132aa26daecedd7372a42417bbbab6f14ab7d6bb9145"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.31",
+ "syn 2.0.48",
 ]
 
 [[package]]
@@ -1984,7 +1982,7 @@ dependencies = [
  "darling",
  "proc-macro2",
  "quote",
- "syn 2.0.31",
+ "syn 2.0.48",
 ]
 
 [[package]]
@@ -2032,9 +2030,9 @@ dependencies = [
 
 [[package]]
 name = "smallvec"
-version = "1.11.0"
+version = "1.13.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "62bb4feee49fdd9f707ef802e22365a35de4b7b299de4763d44bfea899442ff9"
+checksum = "e6ecd384b10a64542d77071bd64bd7b231f4ed5940fba55e98c3de13824cf3d7"
 
 [[package]]
 name = "socket2"
@@ -2092,9 +2090,9 @@ dependencies = [
 
 [[package]]
 name = "syn"
-version = "2.0.31"
+version = "2.0.48"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "718fa2415bcb8d8bd775917a1bf12a7931b6dfa890753378538118181e0cb398"
+checksum = "0f3531638e407dfc0814761abb7c00a5b54992b849452a0646b7f65c9f770f3f"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -2175,7 +2173,7 @@ checksum = "e7fbe9b594d6568a6a1443250a7e67d80b74e1e96f6d1715e1e21cc1888291d3"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.31",
+ "syn 2.0.48",
 ]
 
 [[package]]
@@ -2244,7 +2242,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.31",
+ "syn 2.0.48",
 ]
 
 [[package]]
@@ -2377,9 +2375,9 @@ dependencies = [
 
 [[package]]
 name = "vhost"
-version = "0.11.0"
+version = "0.10.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6be08d1166d41a78861ad50212ab3f9eca0729c349ac3a7a8f557c62406b87cc"
+checksum = "2b64e816d0d49769fbfaa1494eb77cc2a3ddc526ead05c7f922cb7d64106286f"
 dependencies = [
  "bitflags 2.4.1",
  "libc",
@@ -2389,9 +2387,9 @@ dependencies = [
 
 [[package]]
 name = "vhost-user-backend"
-version = "0.15.0"
+version = "0.13.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1f0ffb1dd8e00a708a0e2c32d5efec5812953819888591fff9ff68236b8a5096"
+checksum = "72c8c447d076ac508d78cb45664d203df7989e891656dce260a7e93d72352c9a"
 dependencies = [
  "libc",
  "log",
@@ -2481,9 +2479,9 @@ dependencies = [
 
 [[package]]
 name = "virtio-queue"
-version = "0.12.0"
+version = "0.11.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "07d8406e7250c934462de585d8f2d2781c31819bca1fbb7c5e964ca6bbaabfe8"
+checksum = "e3f69a13d6610db9312acbb438b0390362af905d37634a2106be70c0f734986d"
 dependencies = [
  "log",
  "virtio-bindings",
@@ -2586,6 +2584,7 @@ dependencies = [
  "option_parser",
  "pci",
  "range_map_vec",
+ "rate_limiter",
  "seccompiler",
  "serde",
  "serde_json",
@@ -2664,7 +2663,7 @@ dependencies = [
  "once_cell",
  "proc-macro2",
  "quote",
- "syn 2.0.31",
+ "syn 2.0.48",
  "wasm-bindgen-shared",
 ]
 
@@ -2686,7 +2685,7 @@ checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.31",
+ "syn 2.0.48",
  "wasm-bindgen-backend",
  "wasm-bindgen-shared",
 ]
@@ -2928,9 +2927,9 @@ checksum = "dff9641d1cd4be8d1a070daf9e3773c5f67e78b4d9d42263020c057706765c04"
 
 [[package]]
 name = "winnow"
-version = "0.5.18"
+version = "0.5.39"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "176b6138793677221d420fd2f0aeeced263f197688b36484660da767bca2fa32"
+checksum = "5389a154b01683d28c77f8f68f49dea75f0a4da32557a58f68ee51ebba472d29"
 dependencies = [
  "memchr",
 ]
@@ -3013,9 +3012,9 @@ dependencies = [
 
 [[package]]
 name = "zerocopy"
-version = "0.7.31"
+version = "0.7.32"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1c4061bedbb353041c12f413700357bec76df2c7e2ca8e4df8bac24c6bf68e3d"
+checksum = "74d4d3961e53fa4c9a25a8637fc2bfaf2595b3d3ae34875568a5cf64787716be"
 dependencies = [
  "byteorder",
  "zerocopy-derive",
@@ -3023,13 +3022,13 @@ dependencies = [
 
 [[package]]
 name = "zerocopy-derive"
-version = "0.7.31"
+version = "0.7.32"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b3c129550b3e6de3fd0ba67ba5c81818f9805e58b8d7fee80a3a59d2c9fc601a"
+checksum = "9ce1b18ccd8e73a9321186f97e46f9f04b778851177567b1975109d26a08d2a6"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.31",
+ "syn 2.0.48",
 ]
 
 [[package]]

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "cloud-hypervisor"
-version = "37.1.0"
+version = "38.0.0"
 authors = ["The Cloud Hypervisor Authors"]
 edition = "2021"
 default-run = "cloud-hypervisor"
@@ -29,20 +29,20 @@ strip = false
 debug = true
 
 [dependencies]
-anyhow = "1.0.75"
+anyhow = "1.0.79"
 api_client = { path = "api_client" }
 clap = { version = "4.4.7", features = ["string"] }
-dhat = { version = "0.3.2", optional = true }
+dhat = { version = "0.3.3", optional = true }
 epoll = "4.3.3"
 event_monitor = { path = "event_monitor" }
 hypervisor = { path = "hypervisor" }
-libc = "0.2.147"
+libc = "0.2.153"
 log = { version = "0.4.20", features = ["std"] }
 option_parser = { path = "option_parser" }
 seccompiler = "0.4.0"
-serde_json = "1.0.107"
+serde_json = "1.0.109"
 signal-hook = "0.3.17"
-thiserror = "1.0.40"
+thiserror = "1.0.52"
 tpm = { path = "tpm"}
 tracer = { path = "tracer" }
 vmm = { path = "vmm" }
@@ -52,14 +52,14 @@ zbus = { version = "3.11.1", optional = true }
 
 # List of patched crates
 [patch.crates-io]
-kvm-bindings = { git = "https://github.com/cloud-hypervisor/kvm-bindings", branch = "ch-live-upgrade-stable-37.x" }
+kvm-bindings = { git = "https://github.com/cloud-hypervisor/kvm-bindings", branch = "ch-v0.7.0" }
 versionize_derive = { git = "https://github.com/cloud-hypervisor/versionize_derive", branch = "ch-0.1.6" }
 
 [dev-dependencies]
 dirs = "5.0.0"
 net_util = { path = "net_util" }
-once_cell = "1.18.0"
-serde_json = "1.0.107"
+once_cell = "1.19.0"
+serde_json = "1.0.109"
 test_infra = { path = "test_infra" }
 wait-timeout = "0.2.0"
 

Jenkinsfile

@@ -0,0 +1,530 @@
+def runWorkers = true
+pipeline {
+    agent none
+    options {
+        timeout(time: 4, unit: 'HOURS')
+    }
+    stages {
+        stage('Early checks') {
+            agent { node { label 'built-in' } }
+            stages {
+                stage('Checkout') {
+                    steps {
+                        checkout scm
+                    }
+                }
+                stage('Check if worker build can be skipped') {
+                    when {
+                        expression {
+                            return skipWorkerBuild()
+                        }
+                    }
+                    steps {
+                        script {
+                            runWorkers = false
+                            echo 'No changes requiring a build'
+                        }
+                    }
+                }
+                stage('Check for RFC/WIP builds') {
+                    when {
+                        changeRequest comparator: 'REGEXP', title: '.*(rfc|RFC|wip|WIP).*'
+                        beforeAgent true
+                    }
+                    steps {
+                        error('Failing as this is marked as a WIP or RFC PR.')
+                    }
+                }
+                stage('Cancel older builds') {
+                    when { not { branch 'main' } }
+                    steps {
+                        cancelPreviousBuilds()
+                    }
+                }
+            }
+        }
+        stage('Build') {
+            parallel {
+                stage('Worker build') {
+                    agent { node { label 'jammy' } }
+                    when {
+                        beforeAgent true
+                        expression {
+                            return runWorkers
+                        }
+                    }
+                    environment {
+                        AUTH_DOWNLOAD_TOKEN = credentials('8a26fd74-d40e-414c-9132-ff3f867806ef')
+                    }
+                    stages {
+                        stage('Checkout') {
+                            steps {
+                                checkout scm
+                            }
+                        }
+                        stage('Prepare environment') {
+                            steps {
+                                sh 'scripts/prepare_vdpa.sh'
+                            }
+                        }
+                        stage('Run unit tests') {
+                            steps {
+                                sh 'scripts/dev_cli.sh tests --unit'
+                            }
+                        }
+                        stage('Run integration tests') {
+                            options {
+                                timeout(time: 1, unit: 'HOURS')
+                            }
+                            steps {
+                                sh 'sudo modprobe openvswitch'
+                                sh 'scripts/dev_cli.sh tests --integration'
+                            }
+                        }
+                        stage('Run live-migration integration tests') {
+                            options {
+                                timeout(time: 1, unit: 'HOURS')
+                            }
+                            steps {
+                                sh 'sudo modprobe openvswitch'
+                                sh 'scripts/dev_cli.sh tests --integration-live-migration'
+                            }
+                        }
+                        stage('Run unit tests for musl') {
+                            steps {
+                                sh 'scripts/dev_cli.sh tests --unit --libc musl'
+                            }
+                        }
+                        stage('Run integration tests for musl') {
+                            options {
+                                timeout(time: 1, unit: 'HOURS')
+                            }
+                            steps {
+                                sh 'sudo modprobe openvswitch'
+                                sh 'scripts/dev_cli.sh tests --integration --libc musl'
+                            }
+                        }
+                        stage('Run live-migration integration tests for musl') {
+                            options {
+                                timeout(time: 1, unit: 'HOURS')
+                            }
+                            steps {
+                                sh 'sudo modprobe openvswitch'
+                                sh 'scripts/dev_cli.sh tests --integration-live-migration --libc musl'
+                            }
+                        }
+                    }
+                }
+                stage('Worker build - AMD') {
+                    agent { node { label 'jammy-amd' } }
+                    when {
+                        beforeAgent true
+                        expression {
+                            return runWorkers
+                        }
+                    }
+                    environment {
+                        AUTH_DOWNLOAD_TOKEN = credentials('8a26fd74-d40e-414c-9132-ff3f867806ef')
+                    }
+                    stages {
+                        stage('Checkout') {
+                            steps {
+                                checkout scm
+                            }
+                        }
+                        stage('Prepare environment') {
+                            steps {
+                                sh 'scripts/prepare_vdpa.sh'
+                            }
+                        }
+                        stage('Run integration tests') {
+                            options {
+                                timeout(time: 1, unit: 'HOURS')
+                            }
+                            steps {
+                                sh 'sudo modprobe openvswitch'
+                                sh 'scripts/dev_cli.sh tests --integration'
+                            }
+                        }
+                        stage('Run live-migration integration tests') {
+                            options {
+                                timeout(time: 1, unit: 'HOURS')
+                            }
+                            steps {
+                                sh 'sudo modprobe openvswitch'
+                                sh 'scripts/dev_cli.sh tests --integration-live-migration'
+                            }
+                        }
+                        stage('Run integration tests for musl') {
+                            options {
+                                timeout(time: 1, unit: 'HOURS')
+                            }
+                            steps {
+                                sh 'sudo modprobe openvswitch'
+                                sh 'scripts/dev_cli.sh tests --integration --libc musl'
+                            }
+                        }
+                        stage('Run live-migration integration tests for musl') {
+                            options {
+                                timeout(time: 1, unit: 'HOURS')
+                            }
+                            steps {
+                                sh 'sudo modprobe openvswitch'
+                                sh 'scripts/dev_cli.sh tests --integration-live-migration --libc musl'
+                            }
+                        }
+                    }
+                }
+                stage('AArch64 worker build') {
+                    agent { node { label 'bionic-arm64' } }
+                    when {
+                        beforeAgent true
+                        expression {
+                            return runWorkers
+                        }
+                    }
+                    environment {
+                            AZURE_CONNECTION_STRING = credentials('46b4e7d6-315f-4cc1-8333-b58780863b9b')
+                    }
+                    stages {
+                        stage('Checkout') {
+                            steps {
+                                checkout scm
+                            }
+                        }
+                        stage('Run unit tests') {
+                            steps {
+                                sh 'scripts/dev_cli.sh tests --unit --libc musl'
+                            }
+                        }
+                        stage('Run integration tests') {
+                            options {
+                                timeout(time: 1, unit: 'HOURS')
+                            }
+                            steps {
+                                sh 'sudo modprobe openvswitch'
+                                sh 'scripts/dev_cli.sh tests --integration --libc musl'
+                            }
+                        }
+                        stage('Install azure-cli') {
+                            steps {
+                                installAzureCli('focal', 'arm64')
+                            }
+                        }
+                        stage('Download Windows image') {
+                            steps {
+                                sh '''#!/bin/bash -x
+                                    IMG_BASENAME=windows-11-iot-enterprise-aarch64.raw
+                                    IMG_PATH=$HOME/workloads/$IMG_BASENAME
+                                    IMG_GZ_PATH=$HOME/workloads/$IMG_BASENAME.gz
+                                    IMG_GZ_BLOB_NAME=windows-11-iot-enterprise-aarch64-9-min.raw.gz
+                                    cp "scripts/$IMG_BASENAME.sha1" "$HOME/workloads/"
+                                    pushd "$HOME/workloads"
+                                    if sha1sum "$IMG_BASENAME.sha1" --check; then
+                                        exit
+                                    fi
+                                    popd
+                                    mkdir -p "$HOME/workloads"
+                                    az storage blob download \
+                                        --container-name private-images \
+                                        --file "$IMG_GZ_PATH" \
+                                        --name "$IMG_GZ_BLOB_NAME" \
+                                        --connection-string "$AZURE_CONNECTION_STRING"
+                                    gzip -d $IMG_GZ_PATH
+                                '''
+                            }
+                        }
+                        stage('Run Windows guest integration tests') {
+                            options {
+                                timeout(time: 1, unit: 'HOURS')
+                            }
+                            steps {
+                                sh 'scripts/dev_cli.sh tests --integration-windows --libc musl'
+                            }
+                        }
+                    }
+                    post {
+                        always {
+                            sh "sudo chown -R jenkins.jenkins ${WORKSPACE}"
+                            deleteDir()
+                        }
+                    }
+                }
+                stage('Worker build - Windows guest') {
+                    agent { node { label 'jammy' } }
+                    when {
+                        beforeAgent true
+                        expression {
+                            return runWorkers
+                        }
+                    }
+                    environment {
+                            AZURE_CONNECTION_STRING = credentials('46b4e7d6-315f-4cc1-8333-b58780863b9b')
+                    }
+                    stages {
+                        stage('Checkout') {
+                            steps {
+                                checkout scm
+                            }
+                        }
+                        stage('Install azure-cli') {
+                            steps {
+                                installAzureCli('jammy', 'amd64')
+                            }
+                        }
+                        stage('Download assets') {
+                            steps {
+                                sh "mkdir ${env.HOME}/workloads"
+                                sh 'az storage blob download --container-name private-images --file "$HOME/workloads/windows-server-2022-amd64-2.raw" --name windows-server-2022-amd64-2.raw --connection-string "$AZURE_CONNECTION_STRING"'
+                            }
+                        }
+                        stage('Run Windows guest integration tests') {
+                            options {
+                                timeout(time: 1, unit: 'HOURS')
+                            }
+                            steps {
+                                sh 'scripts/dev_cli.sh tests --integration-windows'
+                            }
+                        }
+                        stage('Run Windows guest integration tests for musl') {
+                            options {
+                                timeout(time: 1, unit: 'HOURS')
+                            }
+                            steps {
+                                sh 'scripts/dev_cli.sh tests --integration-windows --libc musl'
+                            }
+                        }
+                    }
+                }
+                stage('Worker build - Metrics') {
+                    agent { node { label 'jammy-metrics' } }
+                    when {
+                        branch 'main'
+                        beforeAgent true
+                        expression {
+                            return runWorkers
+                        }
+                    }
+                    environment {
+                        METRICS_PUBLISH_KEY = credentials('52e0945f-ce7a-43d1-87af-67d1d87cc40f')
+                    }
+                    stages {
+                        stage('Checkout') {
+                            steps {
+                                checkout scm
+                            }
+                        }
+                        stage('Run metrics tests') {
+                            options {
+                                timeout(time: 1, unit: 'HOURS')
+                            }
+                            steps {
+                                sh 'scripts/dev_cli.sh tests --metrics -- -- --report-file /root/workloads/metrics.json'
+                            }
+                        }
+                        stage('Upload metrics report') {
+                            steps {
+                                sh 'curl -X PUT https://cloud-hypervisor-metrics.azurewebsites.net/api/publishmetrics -H "x-functions-key: $METRICS_PUBLISH_KEY" -T ~/workloads/metrics.json'
+                            }
+                        }
+                    }
+                }
+                stage('Worker build - Rate Limiter') {
+                    agent { node { label 'focal-metrics' } }
+                    when {
+                        branch 'main'
+                        beforeAgent true
+                        expression {
+                            return runWorkers
+                        }
+                    }
+                    stages {
+                        stage('Checkout') {
+                            steps {
+                                checkout scm
+                            }
+                        }
+                        stage('Run rate-limiter integration tests') {
+                            options {
+                                timeout(time: 10, unit: 'MINUTES')
+                            }
+                            steps {
+                                sh 'scripts/dev_cli.sh tests --integration-rate-limiter'
+                            }
+                        }
+                    }
+                }
+                stage('Worker build - SGX') {
+                    agent { node { label 'jammy-sgx' } }
+                    when {
+                        beforeAgent true
+                        allOf {
+                            branch 'main'
+                            expression {
+                                return runWorkers
+                            }
+                        }
+                    }
+                    environment {
+                        AUTH_DOWNLOAD_TOKEN = credentials('8a26fd74-d40e-414c-9132-ff3f867806ef')
+                    }
+                    stages {
+                        stage('Checkout') {
+                            steps {
+                                checkout scm
+                            }
+                        }
+                        stage('Run SGX integration tests') {
+                            options {
+                                timeout(time: 1, unit: 'HOURS')
+                            }
+                            steps {
+                                sh 'scripts/dev_cli.sh tests --integration-sgx'
+                            }
+                        }
+                        stage('Run SGX integration tests for musl') {
+                            options {
+                                timeout(time: 1, unit: 'HOURS')
+                            }
+                            steps {
+                                sh 'scripts/dev_cli.sh tests --integration-sgx --libc musl'
+                            }
+                        }
+                    }
+                    post {
+                        always {
+                            sh "sudo chown -R jenkins.jenkins ${WORKSPACE}"
+                            deleteDir()
+                        }
+                    }
+                }
+                stage('Worker build - VFIO') {
+                    agent { node { label 'jammy-vfio' } }
+                    when {
+                        beforeAgent true
+                        allOf {
+                            branch 'main'
+                            expression {
+                                return runWorkers
+                            }
+                        }
+                    }
+                    environment {
+                        AUTH_DOWNLOAD_TOKEN = credentials('8a26fd74-d40e-414c-9132-ff3f867806ef')
+                    }
+                    stages {
+                        stage('Checkout') {
+                            steps {
+                                checkout scm
+                            }
+                        }
+                        stage('Run VFIO integration tests') {
+                            options {
+                                timeout(time: 1, unit: 'HOURS')
+                            }
+                            steps {
+                                sh 'scripts/dev_cli.sh tests --integration-vfio'
+                            }
+                        }
+                        stage('Run VFIO integration tests for musl') {
+                            options {
+                                timeout(time: 1, unit: 'HOURS')
+                            }
+                            steps {
+                                sh 'scripts/dev_cli.sh tests --integration-vfio --libc musl'
+                            }
+                        }
+                    }
+                    post {
+                        always {
+                            sh "sudo chown -R jenkins.jenkins ${WORKSPACE}"
+                            deleteDir()
+                        }
+                    }
+                }
+            }
+        }
+    }
+    post {
+        regression {
+            script {
+                if (env.BRANCH_NAME == 'main') {
+                    slackSend(color: '#ff0000', message: '"main" branch build is now failing', channel: '#jenkins-ci')
+                }
+            }
+        }
+        fixed {
+            script {
+                if (env.BRANCH_NAME == 'main') {
+                    slackSend(color: '#00ff00', message: '"main" branch build is now fixed', channel: '#jenkins-ci')
+                }
+            }
+        }
+    }
+}
+
+def cancelPreviousBuilds() {
+    // Check for other instances of this particular build, cancel any that are older than the current one
+    def jobName = env.JOB_NAME
+    def currentBuildNumber = env.BUILD_NUMBER.toInteger()
+    def currentJob = Jenkins.instance.getItemByFullName(jobName)
+
+    // Loop through all instances of this particular job/branch
+    for (def build : currentJob.builds) {
+        if (build.isBuilding() && (build.number.toInteger() < currentBuildNumber)) {
+            echo "Older build still queued. Sending kill signal to build number: ${build.number}"
+            build.doStop()
+        }
+    }
+}
+
+def installAzureCli(distro, arch) {
+    sh 'sudo apt install -y ca-certificates curl apt-transport-https lsb-release gnupg'
+    sh 'curl -sL https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor | sudo tee /etc/apt/trusted.gpg.d/microsoft.gpg > /dev/null'
+    sh "echo \"deb [arch=${arch}] https://packages.microsoft.com/repos/azure-cli/ ${distro} main\" | sudo tee /etc/apt/sources.list.d/azure-cli.list"
+    sh 'sudo apt update'
+    sh 'sudo apt install -y azure-cli'
+}
+
+def boolean skipWorkerBuild() {
+    if (env.CHANGE_TARGET == null) {
+        return false
+    }
+
+    if (sh(
+        returnStatus: true,
+        script: "git diff --name-only origin/${env.CHANGE_TARGET}... | grep -v '\\.md'"
+    ) != 0) {
+        return true
+    }
+
+    if (sh(
+        returnStatus: true,
+        script: "git diff --name-only origin/${env.CHANGE_TARGET}... | grep -v -E 'fuzz/'"
+    ) != 0) {
+        return true
+    }
+
+    if (sh(
+        returnStatus: true,
+        script: "git diff --name-only origin/${env.CHANGE_TARGET}... | grep -v -E '.github/'"
+    ) != 0) {
+        return true
+    }
+
+    if (sh(
+        returnStatus: true,
+        script: "git diff --name-only origin/${env.CHANGE_TARGET}... | grep -v '^\\.'"
+    ) != 0) {
+        return true
+    }
+
+    if (sh(
+        returnStatus: true,
+        script: "git diff --name-only origin/${env.CHANGE_TARGET}... | grep -v 'gitlint'"
+    ) != 0) {
+        return true
+    }
+
+    return false
+}

arch/Cargo.toml

@@ -10,14 +10,14 @@ sev_snp = []
 tdx = []
 
 [dependencies]
-anyhow = "1.0.75"
+anyhow = "1.0.79"
 byteorder = "1.4.3"
 hypervisor = { path = "../hypervisor" }
-libc = "0.2.147"
+libc = "0.2.153"
 linux-loader = { version = "0.11.0", features = ["elf", "bzimage", "pe"] }
 log = "0.4.20"
-serde = { version = "1.0.168", features = ["rc", "derive"] }
-thiserror = "1.0.40"
+serde = { version = "1.0.196", features = ["rc", "derive"] }
+thiserror = "1.0.52"
 uuid = "1.3.4"
 versionize = "0.2.0"
 versionize_derive = "0.1.6"

arch/src/aarch64/layout.rs

@@ -111,8 +111,9 @@ pub const RAM_64BIT_START: GuestAddress = GuestAddress(0x1_0000_0000);
 pub const CMDLINE_MAX_SIZE: usize = 2048;
 
 /// FDT is at the beginning of RAM.
-/// Maximum size of the device tree blob as specified in https://www.kernel.org/doc/Documentation/arm64/booting.txt.
 pub const FDT_START: GuestAddress = RAM_START;
+/// Maximum size of the device tree blob as specified in [the kernel
+/// documentation](https://www.kernel.org/doc/Documentation/arm64/booting.txt).
 pub const FDT_MAX_SIZE: u64 = 0x20_0000;
 
 /// Put ACPI table above dtb

arch/src/x86_64/mod.rs

@@ -934,12 +934,7 @@ type RamRange = (u64, u64);
 
 /// Returns usable physical memory ranges for the guest
 /// These should be used to create e820_RAM memory maps
-///
-/// There are up to two usable physical memory ranges,
-/// divided by the gap at the end of 32bit address space.
-pub fn generate_ram_ranges(
-    guest_mem: &GuestMemoryMmap,
-) -> super::Result<(RamRange, Option<RamRange>)> {
+pub fn generate_ram_ranges(guest_mem: &GuestMemoryMmap) -> super::Result<Vec<RamRange>> {
     // Merge continuous memory regions into one region.
     // Note: memory regions from "GuestMemory" are sorted and non-zero sized.
     let ram_regions = {
@@ -972,15 +967,11 @@ pub fn generate_ram_ranges(
         ram_regions
     };
 
-    if ram_regions.len() > 2 {
-        error!(
-            "There should be up to two usable physical memory ranges, devidided by the
-            gap at the end of 32bit address space (e.g. between 3G and 4G)."
-        );
-        return Err(super::Error::MemmapTableSetup);
-    }
+    // Create the memory map entry for memory region before the gap
+    let mut ram_ranges = vec![];
 
-    // Generate the first usable physical memory range before the gap
+    // Generate the first usable physical memory range before the gap. The e820 map
+    // should only report memory above 1MiB.
     let first_ram_range = {
         let (first_region_start, first_region_end) =
             ram_regions.first().ok_or(super::Error::MemmapTableSetup)?;
@@ -1007,33 +998,19 @@ pub fn generate_ram_ranges(
 
         (high_ram_start, *first_region_end)
     };
+    ram_ranges.push(first_ram_range);
 
-    // Generate the second usable physical memory range after the gap if any
-    let second_ram_range = if let Some((second_region_start, second_region_end)) =
-        ram_regions.get(1)
-    {
-        let ram_64bit_start = layout::RAM_64BIT_START.raw_value();
-
-        if second_region_start != &ram_64bit_start {
-            error!(
-                "Unexpected second memory region layout: start: 0x{:08x}, ram_64bit_start: 0x{:08x}",
-                second_region_start, ram_64bit_start
-            );
-
-            return Err(super::Error::MemmapTableSetup);
-        }
-
+    // Generate additional usable physical memory range after the gap if any.
+    for ram_region in ram_regions.iter().skip(1) {
         info!(
-            "Second usable physical memory range, start: 0x{:08x}, end: 0x{:08x}",
-            ram_64bit_start, second_region_end
+            "found usable physical memory range, start: 0x{:08x}, end: 0x{:08x}",
+            ram_region.0, ram_region.1
         );
 
-        Some((ram_64bit_start, *second_region_end))
-    } else {
-        None
-    };
+        ram_ranges.push(*ram_region);
+    }
 
-    Ok((first_ram_range, second_ram_range))
+    Ok(ram_ranges)
 }
 
 fn configure_pvh(
@@ -1084,30 +1061,18 @@ fn configure_pvh(
     add_memmap_entry(&mut memmap, 0, layout::EBDA_START.raw_value(), E820_RAM);
 
     // Get usable physical memory ranges
-    let (first_ram_range, second_ram_range) = generate_ram_ranges(guest_mem)?;
+    let ram_ranges = generate_ram_ranges(guest_mem)?;
 
-    // Create e820 memory map entry before the gap
-    info!(
-        "create_memmap_entry, start: 0x{:08x}, end: 0x{:08x}",
-        first_ram_range.0, first_ram_range.1
-    );
-    add_memmap_entry(
-        &mut memmap,
-        first_ram_range.0,
-        first_ram_range.1 - first_ram_range.0,
-        E820_RAM,
-    );
-
-    // Create e820 memory map after the gap if any
-    if let Some(second_ram_range) = second_ram_range {
+    // Create e820 memory map entries
+    for ram_range in ram_ranges {
         info!(
             "create_memmap_entry, start: 0x{:08x}, end: 0x{:08x}",
-            second_ram_range.0, second_ram_range.1
+            ram_range.0, ram_range.1
         );
         add_memmap_entry(
             &mut memmap,
-            second_ram_range.0,
-            second_ram_range.1 - second_ram_range.0,
+            ram_range.0,
+            ram_range.1 - ram_range.0,
             E820_RAM,
         );
     }

block/Cargo.toml

@@ -12,16 +12,16 @@ io_uring = ["dep:io-uring"]
 byteorder = "1.4.3"
 crc-any = "2.4.4"
 io-uring = { version = "0.6.2", optional = true }
-libc = "0.2.147"
+libc = "0.2.153"
 log = "0.4.20"
 remain = "0.2.11"
-smallvec = "1.11.0"
-thiserror = "1.0.40"
+smallvec = "1.13.1"
+thiserror = "1.0.52"
 uuid = { version = "1.3.4", features = ["v4"] }
 versionize = "0.2.0"
 versionize_derive = "0.1.6"
 virtio-bindings = { version = "0.2.0", features = ["virtio-v5_0_0"] }
-virtio-queue = "0.12.0"
+virtio-queue = "0.11.0"
 vm-memory = { version = "0.14.0", features = ["backend-mmap", "backend-atomic", "backend-bitmap"] }
 vm-virtio = { path = "../vm-virtio" }
 vmm-sys-util = "0.12.1"

block/src/lib.rs

@@ -58,14 +58,16 @@ use versionize_derive::Versionize;
 use virtio_bindings::virtio_blk::*;
 use virtio_queue::DescriptorChain;
 use vm_memory::{
-    bitmap::Bitmap, ByteValued, Bytes, GuestAddress, GuestMemory, GuestMemoryError,
-    GuestMemoryLoadGuard,
+    bitmap::AtomicBitmap, bitmap::Bitmap, ByteValued, Bytes, GuestAddress, GuestMemory,
+    GuestMemoryError, GuestMemoryLoadGuard,
 };
 use vm_virtio::{AccessPlatform, Translatable};
 use vmm_sys_util::aio;
 use vmm_sys_util::eventfd::EventFd;
 use vmm_sys_util::{ioctl_io_nr, ioctl_ioc_nr};
 
+type GuestMemoryMmap = vm_memory::GuestMemoryMmap<AtomicBitmap>;
+
 const SECTOR_SHIFT: u8 = 9;
 pub const SECTOR_SIZE: u64 = 0x01 << SECTOR_SHIFT;
 
@@ -195,8 +197,8 @@ pub enum RequestType {
     Unsupported(u32),
 }
 
-pub fn request_type<B: Bitmap + 'static>(
-    mem: &vm_memory::GuestMemoryMmap<B>,
+pub fn request_type(
+    mem: &GuestMemoryMmap,
     desc_addr: GuestAddress,
 ) -> result::Result<RequestType, Error> {
     let type_ = mem.read_obj(desc_addr).map_err(Error::GuestMemory)?;
@@ -209,10 +211,7 @@ pub fn request_type<B: Bitmap + 'static>(
     }
 }
 
-fn sector<B: Bitmap + 'static>(
-    mem: &vm_memory::GuestMemoryMmap<B>,
-    desc_addr: GuestAddress,
-) -> result::Result<u64, Error> {
+fn sector(mem: &GuestMemoryMmap, desc_addr: GuestAddress) -> result::Result<u64, Error> {
     const SECTOR_OFFSET: usize = 8;
     let addr = match mem.checked_offset(desc_addr, SECTOR_OFFSET) {
         Some(v) => v,
@@ -242,8 +241,8 @@ pub struct Request {
 }
 
 impl Request {
-    pub fn parse<B: Bitmap + 'static>(
-        desc_chain: &mut DescriptorChain<GuestMemoryLoadGuard<vm_memory::GuestMemoryMmap<B>>>,
+    pub fn parse(
+        desc_chain: &mut DescriptorChain<GuestMemoryLoadGuard<GuestMemoryMmap>>,
         access_platform: Option<&Arc<dyn AccessPlatform>>,
     ) -> result::Result<Request, Error> {
         let hdr_desc = desc_chain
@@ -334,11 +333,11 @@ impl Request {
         Ok(req)
     }
 
-    pub fn execute<T: Seek + Read + Write, B: Bitmap + 'static>(
+    pub fn execute<T: Seek + Read + Write>(
         &self,
         disk: &mut T,
         disk_nsectors: u64,
-        mem: &vm_memory::GuestMemoryMmap<B>,
+        mem: &GuestMemoryMmap,
         serial: &[u8],
     ) -> result::Result<u32, ExecuteError> {
         disk.seek(SeekFrom::Start(self.sector << SECTOR_SHIFT))
@@ -391,9 +390,9 @@ impl Request {
         Ok(len)
     }
 
-    pub fn execute_async<B: Bitmap + 'static>(
+    pub fn execute_async(
         &mut self,
-        mem: &vm_memory::GuestMemoryMmap<B>,
+        mem: &GuestMemoryMmap,
         disk_nsectors: u64,
         disk_image: &mut dyn AsyncIo,
         serial: &[u8],

block/src/vhdx/vhdx_header.rs

@@ -353,12 +353,6 @@ impl RegionTableEntry {
     }
 }
 
-#[derive(Clone, Debug)]
-struct RegionEntry {
-    _start: u64,
-    _end: u64,
-}
-
 enum HeaderNo {
     First,
     Second,

devices/Cargo.toml

@@ -6,16 +6,16 @@ edition = "2021"
 
 [dependencies]
 acpi_tables = { git = "https://github.com/rust-vmm/acpi_tables", branch = "main"  }
-anyhow = "1.0.75"
+anyhow = "1.0.79"
 arch = { path = "../arch" }
 bitflags = "2.4.1"
 byteorder = "1.4.3"
 event_monitor = { path = "../event_monitor" }
 hypervisor = { path = "../hypervisor" }
-libc = "0.2.147"
+libc = "0.2.153"
 log = "0.4.20"
 pci = { path = "../pci" }
-thiserror = "1.0.40"
+thiserror = "1.0.52"
 tpm = { path = "../tpm" }
 versionize = "0.2.0"
 versionize_derive = "0.1.6"

devices/src/debug_console.rs

@@ -0,0 +1,64 @@
+// Copyright © 2023 Cyberus Technology
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+//! Module for [`DebugconState`].
+
+use std::io;
+use std::io::Write;
+use std::sync::{Arc, Barrier};
+use vm_device::BusDevice;
+use vm_migration::{Migratable, MigratableError, Pausable, Snapshot, Snapshottable, Transportable};
+
+/// I/O-port.
+pub const DEFAULT_PORT: u64 = 0xe9;
+
+#[derive(Default)]
+pub struct DebugconState {}
+
+/// Emulates a debug console similar to the QEMU debugcon device. This device
+/// is stateless and only prints the bytes (usually text) that are written to
+/// it.
+///
+/// This device is only available on x86.
+///
+/// Reference:
+/// - https://github.com/qemu/qemu/blob/master/hw/char/debugcon.c
+/// - https://phip1611.de/blog/how-to-use-qemus-debugcon-feature-and-write-to-a-file/
+pub struct DebugConsole {
+    id: String,
+    out: Box<dyn io::Write + Send>,
+}
+
+impl DebugConsole {
+    pub fn new(id: String, out: Box<dyn io::Write + Send>) -> Self {
+        Self { id, out }
+    }
+}
+
+impl BusDevice for DebugConsole {
+    fn read(&mut self, _base: u64, _offset: u64, _data: &mut [u8]) {}
+
+    fn write(&mut self, _base: u64, _offset: u64, data: &[u8]) -> Option<Arc<Barrier>> {
+        if let Err(e) = self.out.write_all(data) {
+            // unlikely
+            error!("debug-console: failed writing data: {e:?}");
+        }
+        None
+    }
+}
+
+impl Snapshottable for DebugConsole {
+    fn id(&self) -> String {
+        self.id.clone()
+    }
+
+    fn snapshot(&mut self) -> Result<Snapshot, MigratableError> {
+        Snapshot::new_from_state(&())
+    }
+}
+
+impl Pausable for DebugConsole {}
+impl Transportable for DebugConsole {}
+impl Migratable for DebugConsole {}

devices/src/lib.rs

@@ -15,6 +15,8 @@ extern crate event_monitor;
 extern crate log;
 
 pub mod acpi;
+#[cfg(target_arch = "x86_64")]
+pub mod debug_console;
 #[cfg(target_arch = "aarch64")]
 pub mod gic;
 pub mod interrupt_controller;

docs/api.md

@@ -147,7 +147,7 @@ We want to create a virtual machine with the following characteristics:
   `/opt/clh/images/focal-server-cloudimg-amd64.raw`
 
 ```shell
-#!/bin/bash
+#!/usr/bin/env bash
 
 curl --unix-socket /tmp/cloud-hypervisor.sock -i \
      -X PUT 'http://localhost/api/v1/vm.create'  \
@@ -167,7 +167,7 @@ curl --unix-socket /tmp/cloud-hypervisor.sock -i \
 Once the VM is created, we can boot it:
 
 ```shell
-#!/bin/bash
+#!/usr/bin/env bash
 
 curl --unix-socket /tmp/cloud-hypervisor.sock -i -X PUT 'http://localhost/api/v1/vm.boot'
 ```
@@ -177,7 +177,7 @@ curl --unix-socket /tmp/cloud-hypervisor.sock -i -X PUT 'http://localhost/api/v1
 We can fetch information about any VM, as soon as it's created:
 
 ```shell
-#!/bin/bash
+#!/usr/bin/env bash
 
 curl --unix-socket /tmp/cloud-hypervisor.sock -i \
      -X GET 'http://localhost/api/v1/vm.info' \
@@ -189,7 +189,7 @@ curl --unix-socket /tmp/cloud-hypervisor.sock -i \
 We can reboot a VM that's already booted:
 
 ```shell
-#!/bin/bash
+#!/usr/bin/env bash
 
 curl --unix-socket /tmp/cloud-hypervisor.sock -i -X PUT 'http://localhost/api/v1/vm.reboot'
 ```
@@ -199,7 +199,7 @@ curl --unix-socket /tmp/cloud-hypervisor.sock -i -X PUT 'http://localhost/api/v1
 Once booted, we can shut a VM down from the REST API:
 
 ```shell
-#!/bin/bash
+#!/usr/bin/env bash
 
 curl --unix-socket /tmp/cloud-hypervisor.sock -i -X PUT 'http://localhost/api/v1/vm.shutdown'
 ```
@@ -385,7 +385,7 @@ APIs work together, let's look at a complete VM creation flow, from the
    [REST API](#rest-api) in order to creates a virtual machine:
    ```
    shell
-   #!/bin/bash
+   #!/usr/bin/env bash
 
 	curl --unix-socket /tmp/cloud-hypervisor.sock -i \
 		-X PUT 'http://localhost/api/v1/vm.create'  \

docs/custom-image.md

@@ -170,7 +170,7 @@ We usually start from one of the custom cloud image we have previously created
 but we can use a stock cloud image as well.
 
 ```bash
-wget https://ch-images.azureedge.net/jammy-server-cloudimg-amd64-custom-20230119-0.raw
+wget https://cloud-hypervisor.azureedge.net/jammy-server-cloudimg-amd64-custom-20230119-0.raw
 mv jammy-server-cloudimg-amd64-custom-20230119-0.raw jammy-server-cloudimg-amd64-nvidia.raw
 ```
 
@@ -326,4 +326,4 @@ VM will be booted with this image.
 
 ```
 sudo cloud-init clean
-```
+```

docs/debug-port.md

@@ -1,7 +1,16 @@
-# `cloud-hypervisor` debug IO port
+# `cloud-hypervisor` debug IO ports
 
-`cloud-hypervisor` uses the [`0x80`](https://www.intel.com/content/www/us/en/support/articles/000005500/boards-and-kits.html)
-I/O port to trace user defined guest events.
+When running x86 guests, `cloud-hypervisor` provides different kinds of debug ports:
+- [`0x80` debug port](https://www.intel.com/content/www/us/en/support/articles/000005500/boards-and-kits.html) 
+- Debug console (by default at `0xe9`). 
+- Firmware debug port at `0x402`.
+
+All of them can be used to trace user-defined guest events and all of them can
+be used simultaneously.
+
+## Debug Ports Overview
+
+### `0x80` I/O port
 
 Whenever the guest write one byte between `0x0` and `0xF` on this particular
 I/O port, `cloud-hypervisor` will log and timestamp that event at the `debug`
@@ -30,7 +39,7 @@ guest will have `cloud-hypervisor` generate timestamped logs of all those steps.
 That provides a basic but convenient way of measuring not only the overall guest
 boot time but all intermediate steps as well.
 
-## Logging
+#### Logging
 
 Assuming parts of the guest software stack have been instrumented to use the
 `cloud-hypervisor` debug I/O port, we may want to gather the related logs.
@@ -59,3 +68,29 @@ $ grep "Debug I/O port" /tmp/ch-fw.log
 cloud-hypervisor: 19.762449ms: DEBUG:vmm/src/vm.rs:510 -- [Debug I/O port: Firmware code 0x0] 0.019004 seconds
 cloud-hypervisor: 403.499628ms: DEBUG:vmm/src/vm.rs:510 -- [Debug I/O port: Firmware code 0x1] 0.402744 seconds
 ```
+
+### Debug console port
+
+The debug console is inspired by QEMU and Bochs, which have a similar feature. 
+By default, the I/O port `0xe9` is used. This port can be configured like a 
+console. Thus, it can print to a tty, a file, or a pty, for example.
+
+### Firmware debug port
+
+The firmware debug port is also a simple port that prints all bytes written to
+it. The firmware debug port only prints to stdout.
+
+## When do I need these ports?
+
+The ports are on the one hand interesting for firmware or kernel developers, as
+they provide an easy way to print debug information from within a guest. 
+Furthermore, you can patch "normal" software to measure certain events, such as
+the boot time of a guest.
+
+## Which port should I choose?
+
+The `0x80` debug port and the port of the firmware debug device are always
+available. The debug console must be activated via the command line, but
+provides more configuration options. 
+
+You can use different ports for different aspect of your logging messages. 

docs/intel_tdx.md

@@ -103,7 +103,7 @@ firmware:
 ### TDShim
 
 > **Note**
-> The latest version of TDShim being tested is [_66bb334_](https://github.com/confidential-containers/td-shim/tree/66bb33451befbf1291abe3cfea7ee9e99d922b0d).
+> The latest version of TDShim being tested is [_v0.8.0_](https://github.com/confidential-containers/td-shim/releases/tag/v0.8.0).
 
 This is a lightweight version of the TDVF, written in Rust and designed for
 direct kernel boot, which is useful for containers use cases.
@@ -113,7 +113,7 @@ and `LLVM` first. The TDshim can be build as follows:
 ```bash
 git clone https://github.com/confidential-containers/td-shim
 cd td-shim
-git checkout 66bb33451befbf1291abe3cfea7ee9e99d922b0d
+git checkout v0.8.0
 cargo install cargo-xbuild
 export CC=clang
 export AR=llvm-ar
@@ -121,15 +121,13 @@ export CC_x86_64_unknown_none=clang
 export AR_x86_64_unknown_none=llvm-ar
 git submodule update --init --recursive
 ./sh_script/preparation.sh
-cargo xbuild -p td-shim --target x86_64-unknown-none --release --features=main,tdx
-cargo run -p td-shim-tools --bin td-shim-ld --features=linker -- target/x86_64-unknown-none/release/ResetVector.bin target/x86_64-unknown-none/release/td-shim -o target/release/final.bin
+cargo image --release
 ```
 
 If debug logs from the TDShim is needed, here are the alternative
 commands:
 ```bash
-cargo xbuild -p td-shim --target x86_64-unknown-none --features=main,tdx
-cargo run -p td-shim-tools --bin td-shim-ld --features=linker -- target/x86_64-unknown-none/debug/ResetVector.bin target/x86_64-unknown-none/debug/td-shim -o target/debug/final.bin
+cargo image
 ```
 
 And run a TDX VM by providing the firmware previously built, along with a guest

docs/io_throttling.md

@@ -42,3 +42,14 @@ actual rate limit users get can be as low as
 generally advisable to keep `bw/ops_refill_time` larger than `100 ms`
 (`cool_down_time`) to make sure the actual rate limit is close to users'
 expectation ("refill-rate").
+
+## Rate Limit Groups
+It is possible to throttle the aggregate bandwidth or operations
+of multiple virtio-blk devices using a `rate_limit_group`. virtio-blk devices may be
+dynamically added and removed from a `rate_limit_group`. The following example
+demonstrates how to throttle the aggregate bandwidth of two disks to 10 MiB/s.
+```
+--disk path=disk0.raw,rate_limit_group=group0 \
+       path=disk1.raw,rate_limit_group=group0 \
+--rate-limit-group bw_size=1048576,bw_refill_time,bw_refill_time=100
+```

docs/iommu.md

@@ -60,10 +60,7 @@ implement a full emulation of a physical IOMMU.
 
 ### Kernel
 
-Since virtio-iommu has landed partially into the version 5.3 of the Linux
-kernel, a special branch is needed to get things working with Cloud Hypervisor.
-By partially, we are talking about x86 specifically, as it is already fully
-functional for ARM architectures.
+As of Kernel 5.14, virtio-iommu is available for both X86-64 and Aarch64.
 
 ## Usage
 

docs/vhost-user-net-testing.md

@@ -2,8 +2,8 @@
 
 The purpose of this document is to illustrate how to test vhost-user-net
 in cloud-hypervisor with OVS/DPDK as the backend. This document was
-tested with Open vSwitch v2.13.1, DPDK v19.11.3, and Cloud Hypervisor
-v15.0 on Ubuntu 20.04.1 (host kernel v5.4.0).
+tested with Open vSwitch v2.17.8, DPDK v21.11.4, and Cloud Hypervisor
+v37.0 on Ubuntu 22.04.3 (host kernel v5.15.0).
 
 ## Framework
 
@@ -74,8 +74,8 @@ Here is an example how to create a bridge and add two DPDK ports to it
 # create a bridge
 ovs-vsctl add-br ovsbr0 -- set bridge ovsbr0 datapath_type=netdev
 # create two DPDK ports and add them to the bridge
-ovs-vsctl add-port ovsbr0 vhost-user1 -- set Interface vhost-user1 type=dpdkvhostuser
-ovs-vsctl add-port ovsbr0 vhost-user2 -- set Interface vhost-user2 type=dpdkvhostuser
+ovs-vsctl add-port ovsbr0 vhost-user1 -- set Interface vhost-user1 type=dpdkvhostuserclient options:vhost-server-path=/tmp/vhost-user1
+ovs-vsctl add-port ovsbr0 vhost-user2 -- set Interface vhost-user2 type=dpdkvhostuserclient options:vhost-server-path=/tmp/vhost-user2
 # set the number of rx queues
 ovs-vsctl set Interface vhost-user1 options:n_rxq=2
 ovs-vsctl set Interface vhost-user2 options:n_rxq=2
@@ -92,7 +92,7 @@ VMs run in client mode. They connect to the socket created by the `dpdkvhostuser
         --kernel vmlinux \
         --cmdline "console=ttyS0 console=hvc0 root=/dev/vda1 rw" \
         --disk path=focal-server-cloudimg-amd64.raw   \
-        --net mac=52:54:00:02:d9:01,vhost_user=true,socket=/var/run/openvswitch/vhost-user1,num_queues=4
+        --net mac=52:54:00:02:d9:01,vhost_user=true,socket=/tmp/vhost-user1,num_queues=4,vhost_mode=server
 
 # From another terminal. We need to give the cloud-hypervisor binary the NET_ADMIN capabilities for it to set TAP interfaces up on the host.
 ./cloud-hypervisor \
@@ -101,21 +101,21 @@ VMs run in client mode. They connect to the socket created by the `dpdkvhostuser
         --kernel vmlinux \
         --cmdline "console=ttyS0 console=hvc0 root=/dev/vda1 rw" \
         --disk path=focal-server-cloudimg-amd64.raw   \
-        --net mac=52:54:20:11:C5:02,vhost_user=true,socket=/var/run/openvswitch/vhost-user2,num_queues=4
+        --net mac=52:54:20:11:C5:02,vhost_user=true,socket=/tmp/vhost-user2,num_queues=4,vhost_mode=server
 ```
 
 _Setup VM1_
 ```bash
 # From inside the guest
-sudo ip addr add 172.100.0.1/24 dev ens2
-sudo ip link set up dev ens2
+sudo ip addr add 172.100.0.1/24 dev ens3
+sudo ip link set up dev ens3
 ```
 
 _Setup VM2_
 ```bash
 # From inside the guest
-sudo ip addr add 172.100.0.2/24 dev ens2
-sudo ip link set up dev ens2
+sudo ip addr add 172.100.0.2/24 dev ens3
+sudo ip link set up dev ens3
 ```
 
 _Ping VM1 from VM2_

event_monitor/Cargo.toml

@@ -6,7 +6,7 @@ edition = "2021"
 
 [dependencies]
 flume = "0.10.14"
-libc = "0.2.147"
-once_cell = "1.18.0"
-serde = { version = "1.0.168", features = ["rc", "derive"] }
-serde_json = "1.0.107"
+libc = "0.2.153"
+once_cell = "1.19.0"
+serde = { version = "1.0.196", features = ["rc", "derive"] }
+serde_json = "1.0.109"

fuzz/Cargo.lock

@@ -26,9 +26,9 @@ dependencies = [
 
 [[package]]
 name = "anstyle"
-version = "1.0.4"
+version = "1.0.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7079075b41f533b8c61d2a4d073c4676e1f8b249ff94a393b0595db304e0dd87"
+checksum = "8901269c6307e8d93993578286ac0edf7f195079ffff5ebdeea6a59ffb7e36bc"
 
 [[package]]
 name = "anstyle-parse"
@@ -184,30 +184,30 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
 
 [[package]]
 name = "clap"
-version = "4.4.18"
+version = "4.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1e578d6ec4194633722ccf9544794b71b1385c3c027efe0c55db226fc880865c"
+checksum = "80c21025abd42669a92efc996ef13cfb2c5c627858421ea58d5c3b331a6c134f"
 dependencies = [
  "clap_builder",
 ]
 
 [[package]]
 name = "clap_builder"
-version = "4.4.18"
+version = "4.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4df4df40ec50c46000231c914968278b1eb05098cf8f1b3a518a95030e71d1c7"
+checksum = "458bf1f341769dfcf849846f65dffdf9146daa56bcd2a47cb4e1de9915567c99"
 dependencies = [
  "anstream",
  "anstyle",
  "clap_lex",
- "strsim",
+ "strsim 0.11.0",
 ]
 
 [[package]]
 name = "clap_lex"
-version = "0.6.0"
+version = "0.7.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "702fc72eb24e5a1e48ce58027a675bc24edd52096d5397d4aea7c6dd9eca0bd1"
+checksum = "98cc8fbded0c607b7ba9dd60cd98df59af97e84d24e49c8557331cfc26d301ce"
 
 [[package]]
 name = "cloud-hypervisor"
@@ -281,9 +281,9 @@ checksum = "2707e3afba5e19b75d582d88bc79237418f2a2a2d673d01cf9b03633b46e98f3"
 
 [[package]]
 name = "darling"
-version = "0.20.3"
+version = "0.20.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0209d94da627ab5605dcccf08bb18afa5009cfbef48d8a8b7d7bdbc79be25c5e"
+checksum = "fc5d6b04b3fd0ba9926f945895de7d806260a2d7431ba82e7edaecb043c4c6b8"
 dependencies = [
  "darling_core",
  "darling_macro",
@@ -291,23 +291,23 @@ dependencies = [
 
 [[package]]
 name = "darling_core"
-version = "0.20.3"
+version = "0.20.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "177e3443818124b357d8e76f53be906d60937f0d3a90773a664fa63fa253e621"
+checksum = "04e48a959bcd5c761246f5d090ebc2fbf7b9cd527a492b07a67510c108f1e7e3"
 dependencies = [
  "fnv",
  "ident_case",
  "proc-macro2",
  "quote",
- "strsim",
+ "strsim 0.10.0",
  "syn 2.0.47",
 ]
 
 [[package]]
 name = "darling_macro"
-version = "0.20.3"
+version = "0.20.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "836a9bbc7ad63342d6d6e7b815ccab164bc77a2d95d84bc3117a8c0d5c98e2d5"
+checksum = "1d1545d67a2149e1d93b7e5c7752dce5a7426eb5d1357ddcfd89336b94444f77"
 dependencies = [
  "darling_core",
  "quote",
@@ -405,9 +405,9 @@ checksum = "9fb8e00e87438d937621c1c6269e53f536c14d3fbd6a042bb24879e57d474fb5"
 
 [[package]]
 name = "getrandom"
-version = "0.2.11"
+version = "0.2.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fe9006bed769170c11f845cf00c7c1e9092aeb3f268e007c3e760ac68008070f"
+checksum = "190092ea657667030ac6a35e305e62fc4dd69fd98ac98631e5d3a2b1575a12b5"
 dependencies = [
  "cfg-if",
  "js-sys",
@@ -442,9 +442,9 @@ checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39"
 
 [[package]]
 name = "io-uring"
-version = "0.6.2"
+version = "0.6.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "460648e47a07a43110fbfa2e0b14afb2be920093c31e5dccc50e49568e099762"
+checksum = "a9febecd4aebbe9c7c23c8e536e966805fdf09944c8a915e7991ee51acb67087"
 dependencies = [
  "bitflags 1.3.2",
  "libc",
@@ -458,18 +458,18 @@ checksum = "b1a46d1a171d865aa5f83f92695765caa047a9b4cbae2cbf37dbd613a793fd4c"
 
 [[package]]
 name = "jobserver"
-version = "0.1.27"
+version = "0.1.28"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8c37f63953c4c63420ed5fd3d6d398c719489b9f872b9fa683262f8edd363c7d"
+checksum = "ab46a6e9526ddef3ae7f787c06f0f2600639ba80ea3eade3d8e670a2230f51d6"
 dependencies = [
  "libc",
 ]
 
 [[package]]
 name = "js-sys"
-version = "0.3.66"
+version = "0.3.68"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cee9c64da59eae3b50095c18d3e74f8b73c0b86d2792824ff01bbce68ba229ca"
+checksum = "406cda4b368d531c842222cf9d2600a9a4acce8d29423695379c6868a143a9ee"
 dependencies = [
  "wasm-bindgen",
 ]
@@ -477,7 +477,7 @@ dependencies = [
 [[package]]
 name = "kvm-bindings"
 version = "0.7.0"
-source = "git+https://github.com/cloud-hypervisor/kvm-bindings?branch=ch-live-upgrade-stable-37.x#f03fc575cdf20c3af9ca3d4d203f171943d95be4"
+source = "git+https://github.com/cloud-hypervisor/kvm-bindings?branch=ch-v0.7.0#2dcf85d4f8aa55befcaa996b699ddb18ec9ed059"
 dependencies = [
  "serde",
  "serde_derive",
@@ -498,9 +498,9 @@ dependencies = [
 
 [[package]]
 name = "libc"
-version = "0.2.152"
+version = "0.2.153"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "13e3bf6590cbc649f4d1a3eefc9d5d6eb746f5200ffb04e5e142700b8faa56e7"
+checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd"
 
 [[package]]
 name = "libfuzzer-sys"
@@ -619,18 +619,18 @@ dependencies = [
 
 [[package]]
 name = "pin-project"
-version = "1.1.3"
+version = "1.1.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fda4ed1c6c173e3fc7a83629421152e01d7b1f9b7f65fb301e490e8cfc656422"
+checksum = "0302c4a0442c456bd56f841aee5c3bfd17967563f6fadc9ceb9f9c23cf3807e0"
 dependencies = [
  "pin-project-internal",
 ]
 
 [[package]]
 name = "pin-project-internal"
-version = "1.1.3"
+version = "1.1.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405"
+checksum = "266c042b60c9c76b8d53061e52b2e0d1116abc57cefc8c5cd671619a56ac3690"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -639,9 +639,9 @@ dependencies = [
 
 [[package]]
 name = "proc-macro2"
-version = "1.0.76"
+version = "1.0.78"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "95fc56cda0b5c3325f5fbbd7ff9fda9e02bb00bb3dac51252d2f1bfa1cb8cc8c"
+checksum = "e2422ad645d89c99f8f3e6b88a9fdeca7fabeac836b1002371c4367c8f984aae"
 dependencies = [
  "unicode-ident",
 ]
@@ -659,6 +659,7 @@ dependencies = [
 name = "rate_limiter"
 version = "0.1.0"
 dependencies = [
+ "epoll",
  "libc",
  "log",
  "thiserror",
@@ -699,18 +700,18 @@ dependencies = [
 
 [[package]]
 name = "serde"
-version = "1.0.195"
+version = "1.0.196"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "63261df402c67811e9ac6def069e4786148c4563f4b50fd4bf30aa370d626b02"
+checksum = "870026e60fa08c69f064aa766c10f10b1d62db9ccd4d0abb206472bee0ce3b32"
 dependencies = [
  "serde_derive",
 ]
 
 [[package]]
 name = "serde_derive"
-version = "1.0.195"
+version = "1.0.196"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "46fe8f8603d81ba86327b23a2e9cdf49e1255fb94a4c5f297f6ee0547178ea2c"
+checksum = "33c85360c95e7d137454dc81d9a4ed2b8efd8fbe19cee57357b32b9771fccb67"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -719,9 +720,9 @@ dependencies = [
 
 [[package]]
 name = "serde_json"
-version = "1.0.111"
+version = "1.0.113"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "176e46fa42316f18edd598015a5166857fc835ec732f5215eac6b7bdbf0a84f4"
+checksum = "69801b70b1c3dac963ecb03a364ba0ceda9cf60c71cfe475e99864759c8b8a79"
 dependencies = [
  "itoa",
  "ryu",
@@ -775,9 +776,9 @@ dependencies = [
 
 [[package]]
 name = "smallvec"
-version = "1.12.0"
+version = "1.13.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2593d31f82ead8df961d8bd23a64c2ccf2eb5dd34b0a34bfb4dd54011c72009e"
+checksum = "e6ecd384b10a64542d77071bd64bd7b231f4ed5940fba55e98c3de13824cf3d7"
 
 [[package]]
 name = "spin"
@@ -794,6 +795,12 @@ version = "0.10.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623"
 
+[[package]]
+name = "strsim"
+version = "0.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5ee073c9e4cd00e28217186dbe12796d692868f432bf2e97ee73bed0c56dfa01"
+
 [[package]]
 name = "syn"
 version = "1.0.109"
@@ -1099,6 +1106,7 @@ dependencies = [
  "once_cell",
  "option_parser",
  "pci",
+ "rate_limiter",
  "seccompiler",
  "serde",
  "serde_json",
@@ -1142,9 +1150,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
 
 [[package]]
 name = "wasm-bindgen"
-version = "0.2.89"
+version = "0.2.91"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0ed0d4f68a3015cc185aff4db9506a015f4b96f95303897bfa23f846db54064e"
+checksum = "c1e124130aee3fb58c5bdd6b639a0509486b0338acaaae0c84a5124b0f588b7f"
 dependencies = [
  "cfg-if",
  "wasm-bindgen-macro",
@@ -1152,9 +1160,9 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen-backend"
-version = "0.2.89"
+version = "0.2.91"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1b56f625e64f3a1084ded111c4d5f477df9f8c92df113852fa5a374dbda78826"
+checksum = "c9e7e1900c352b609c8488ad12639a311045f40a35491fb69ba8c12f758af70b"
 dependencies = [
  "bumpalo",
  "log",
@@ -1167,9 +1175,9 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen-macro"
-version = "0.2.89"
+version = "0.2.91"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0162dbf37223cd2afce98f3d0785506dcb8d266223983e4b5b525859e6e182b2"
+checksum = "b30af9e2d358182b5c7449424f017eba305ed32a7010509ede96cdc4696c46ed"
 dependencies = [
  "quote",
  "wasm-bindgen-macro-support",
@@ -1177,9 +1185,9 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen-macro-support"
-version = "0.2.89"
+version = "0.2.91"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f0eb82fcb7930ae6219a7ecfd55b217f5f0893484b7a13022ebb2b2bf20b5283"
+checksum = "642f325be6301eb8107a83d12a8ac6c1e1c54345a7ef1a9261962dfefda09e66"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -1190,9 +1198,9 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen-shared"
-version = "0.2.89"
+version = "0.2.91"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7ab9b36309365056cd639da3134bf87fa8f3d86008abf99e612384a6eecd459f"
+checksum = "4f186bd2dcf04330886ce82d6f33dd75a7bfcf69ecf5763b89fcde53b6ac9838"
 
 [[package]]
 name = "winapi"

fuzz/Cargo.toml

@@ -23,7 +23,7 @@ net_util = { path = "../net_util" }
 once_cell = "1.19.0"
 seccompiler = "0.4.0"
 virtio-devices = { path = "../virtio-devices" }
-virtio-queue = "0.12.0"
+virtio-queue = "0.11.0"
 vmm = { path = "../vmm" }
 vmm-sys-util = "0.12.1"
 vm-memory = "0.14.0"
@@ -35,7 +35,7 @@ vm-virtio = { path = "../vm-virtio" }
 path = ".."
 
 [patch.crates-io]
-kvm-bindings = { git = "https://github.com/cloud-hypervisor/kvm-bindings", branch = "ch-live-upgrade-stable-37.x" }
+kvm-bindings = { git = "https://github.com/cloud-hypervisor/kvm-bindings", branch = "ch-v0.7.0" }
 versionize_derive = { git = "https://github.com/cloud-hypervisor/versionize_derive", branch = "ch-0.1.6" }
 
 # Prevent this from interfering with workspaces

fuzz/fuzz_targets/block.rs

@@ -21,6 +21,7 @@ use virtio_devices::{Block, VirtioDevice, VirtioInterrupt, VirtioInterruptType};
 use virtio_queue::{Queue, QueueT};
 use vm_memory::{bitmap::AtomicBitmap, Bytes, GuestAddress, GuestMemoryAtomic};
 use vmm_sys_util::eventfd::{EventFd, EFD_NONBLOCK};
+use std::collections::BTreeMap;
 
 type GuestMemoryMmap = vm_memory::GuestMemoryMmap<AtomicBitmap>;
 
@@ -49,6 +50,7 @@ fuzz_target!(|bytes| {
     let shm = memfd_create(&ffi::CString::new("fuzz").unwrap(), 0).unwrap();
     let disk_file: File = unsafe { File::from_raw_fd(shm) };
     let qcow_disk = Box::new(RawFileDiskSync::new(disk_file)) as Box<dyn DiskFile>;
+    let queue_affinity = BTreeMap::new();
     let mut block = Block::new(
         "tmp".to_owned(),
         qcow_disk,
@@ -62,6 +64,7 @@ fuzz_target!(|bytes| {
         None,
         EventFd::new(EFD_NONBLOCK).unwrap(),
         None,
+        queue_affinity,
     )
     .unwrap();
 

fuzz/fuzz_targets/cmos.rs

@@ -6,8 +6,6 @@
 use devices::legacy::Cmos;
 use libc::EFD_NONBLOCK;
 use libfuzzer_sys::fuzz_target;
-use std::sync::atomic::AtomicBool;
-use std::sync::Arc;
 use vm_device::BusDevice;
 use vmm_sys_util::eventfd::EventFd;
 

fuzz/fuzz_targets/http_api.rs

@@ -7,9 +7,18 @@ use libfuzzer_sys::fuzz_target;
 use micro_http::Request;
 use once_cell::sync::Lazy;
 use std::os::unix::io::AsRawFd;
+use std::path::PathBuf;
 use std::sync::mpsc::{channel, Receiver};
+use std::sync::{Arc, Mutex};
 use std::thread;
-use vmm::api::{http::*, ApiRequest, ApiResponsePayload};
+use vm_migration::MigratableError;
+use vmm::api::{
+    http::*, ApiRequest, RequestHandler, VmInfoResponse, VmReceiveMigrationData,
+    VmSendMigrationData, VmmPingResponse,
+};
+use vmm::config::RestoreConfig;
+use vmm::vm::{Error as VmError, VmState};
+use vmm::vm_config::*;
 use vmm::{EpollContext, EpollDispatch};
 use vmm_sys_util::eventfd::EventFd;
 
@@ -69,6 +78,203 @@ fn generate_request(bytes: &[u8]) -> Option<Request> {
     Request::try_from(&request, None).ok()
 }
 
+struct StubApiRequestHandler;
+
+impl RequestHandler for StubApiRequestHandler {
+    fn vm_create(&mut self, _: Arc<Mutex<VmConfig>>) -> Result<(), VmError> {
+        Ok(())
+    }
+
+    fn vm_boot(&mut self) -> Result<(), VmError> {
+        Ok(())
+    }
+
+    fn vm_pause(&mut self) -> Result<(), VmError> {
+        Ok(())
+    }
+
+    fn vm_resume(&mut self) -> Result<(), VmError> {
+        Ok(())
+    }
+
+    fn vm_snapshot(&mut self, _: &str) -> Result<(), VmError> {
+        Ok(())
+    }
+
+    fn vm_restore(&mut self, _: RestoreConfig) -> Result<(), VmError> {
+        Ok(())
+    }
+
+    #[cfg(all(target_arch = "x86_64", feature = "guest_debug"))]
+    fn vm_coredump(&mut self, _: &str) -> Result<(), VmError> {
+        Ok(())
+    }
+
+    fn vm_shutdown(&mut self) -> Result<(), VmError> {
+        Ok(())
+    }
+
+    fn vm_reboot(&mut self) -> Result<(), VmError> {
+        Ok(())
+    }
+
+    fn vm_info(&self) -> Result<VmInfoResponse, VmError> {
+        Ok(VmInfoResponse {
+            config: Arc::new(Mutex::new(VmConfig {
+                cpus: CpusConfig {
+                    boot_vcpus: 1,
+                    max_vcpus: 1,
+                    topology: None,
+                    kvm_hyperv: false,
+                    max_phys_bits: 46,
+                    affinity: None,
+                    features: CpuFeatures::default(),
+                },
+                memory: MemoryConfig {
+                    size: 536_870_912,
+                    mergeable: false,
+                    hotplug_method: HotplugMethod::Acpi,
+                    hotplug_size: None,
+                    hotplugged_size: None,
+                    shared: false,
+                    hugepages: false,
+                    hugepage_size: None,
+                    prefault: false,
+                    zones: None,
+                    thp: true,
+                },
+                payload: Some(PayloadConfig {
+                    kernel: Some(PathBuf::from("/path/to/kernel")),
+                    firmware: None,
+                    cmdline: None,
+                    initramfs: None,
+                    #[cfg(feature = "igvm")]
+                    igvm: None,
+                }),
+                rate_limit_groups: None,
+                disks: None,
+                net: None,
+                rng: RngConfig {
+                    src: PathBuf::from("/dev/urandom"),
+                    iommu: false,
+                },
+                balloon: None,
+                fs: None,
+                pmem: None,
+                serial: ConsoleConfig {
+                    file: None,
+                    mode: ConsoleOutputMode::Null,
+                    iommu: false,
+                    socket: None,
+                },
+                console: ConsoleConfig {
+                    file: None,
+                    mode: ConsoleOutputMode::Tty,
+                    iommu: false,
+                    socket: None,
+                },
+                #[cfg(target_arch = "x86_64")]
+                debug_console: DebugConsoleConfig::default(),
+                devices: None,
+                user_devices: None,
+                vdpa: None,
+                vsock: None,
+                pvpanic: false,
+                iommu: false,
+                #[cfg(target_arch = "x86_64")]
+                sgx_epc: None,
+                numa: None,
+                watchdog: false,
+                #[cfg(feature = "guest_debug")]
+                gdb: false,
+                platform: None,
+                tpm: None,
+                preserved_fds: None,
+            })),
+            state: VmState::Running,
+            memory_actual_size: 0,
+            device_tree: None,
+        })
+    }
+
+    fn vmm_ping(&self) -> VmmPingResponse {
+        VmmPingResponse {
+            build_version: String::new(),
+            version: String::new(),
+            pid: 0,
+            features: Vec::new(),
+        }
+    }
+
+    fn vm_delete(&mut self) -> Result<(), VmError> {
+        Ok(())
+    }
+
+    fn vmm_shutdown(&mut self) -> Result<(), VmError> {
+        Ok(())
+    }
+
+    fn vm_resize(&mut self, _: Option<u8>, _: Option<u64>, _: Option<u64>) -> Result<(), VmError> {
+        Ok(())
+    }
+
+    fn vm_resize_zone(&mut self, _: String, _: u64) -> Result<(), VmError> {
+        Ok(())
+    }
+
+    fn vm_add_device(&mut self, _: DeviceConfig) -> Result<Option<Vec<u8>>, VmError> {
+        Ok(None)
+    }
+
+    fn vm_add_user_device(&mut self, _: UserDeviceConfig) -> Result<Option<Vec<u8>>, VmError> {
+        Ok(None)
+    }
+
+    fn vm_remove_device(&mut self, _: String) -> Result<(), VmError> {
+        Ok(())
+    }
+
+    fn vm_add_disk(&mut self, _: DiskConfig) -> Result<Option<Vec<u8>>, VmError> {
+        Ok(None)
+    }
+
+    fn vm_add_fs(&mut self, _: FsConfig) -> Result<Option<Vec<u8>>, VmError> {
+        Ok(None)
+    }
+
+    fn vm_add_pmem(&mut self, _: PmemConfig) -> Result<Option<Vec<u8>>, VmError> {
+        Ok(None)
+    }
+
+    fn vm_add_net(&mut self, _: NetConfig) -> Result<Option<Vec<u8>>, VmError> {
+        Ok(None)
+    }
+
+    fn vm_add_vdpa(&mut self, _: VdpaConfig) -> Result<Option<Vec<u8>>, VmError> {
+        Ok(None)
+    }
+
+    fn vm_add_vsock(&mut self, _: VsockConfig) -> Result<Option<Vec<u8>>, VmError> {
+        Ok(None)
+    }
+
+    fn vm_counters(&mut self) -> Result<Option<Vec<u8>>, VmError> {
+        Ok(None)
+    }
+
+    fn vm_power_button(&mut self) -> Result<(), VmError> {
+        Ok(())
+    }
+
+    fn vm_receive_migration(&mut self, _: VmReceiveMigrationData) -> Result<(), MigratableError> {
+        Ok(())
+    }
+
+    fn vm_send_migration(&mut self, _: VmSendMigrationData) -> Result<(), MigratableError> {
+        Ok(())
+    }
+}
+
 fn http_receiver_stub(exit_evt: EventFd, api_evt: EventFd, api_receiver: Receiver<ApiRequest>) {
     let mut epoll = EpollContext::new().unwrap();
     epoll.add_event(&exit_evt, EpollDispatch::Exit).unwrap();
@@ -98,89 +304,7 @@ fn http_receiver_stub(exit_evt: EventFd, api_evt: EventFd, api_receiver: Receive
             EpollDispatch::Api => {
                 for _ in 0..api_evt.read().unwrap() {
                     let api_request = api_receiver.recv().unwrap();
-                    match api_request {
-                        ApiRequest::VmCreate(_, sender) => {
-                            sender.send(Ok(ApiResponsePayload::Empty)).unwrap();
-                        }
-                        ApiRequest::VmDelete(sender) => {
-                            sender.send(Ok(ApiResponsePayload::Empty)).unwrap();
-                        }
-                        ApiRequest::VmBoot(sender) => {
-                            sender.send(Ok(ApiResponsePayload::Empty)).unwrap();
-                        }
-                        ApiRequest::VmShutdown(sender) => {
-                            sender.send(Ok(ApiResponsePayload::Empty)).unwrap();
-                        }
-                        ApiRequest::VmReboot(sender) => {
-                            sender.send(Ok(ApiResponsePayload::Empty)).unwrap();
-                        }
-                        ApiRequest::VmInfo(sender) => {
-                            sender.send(Ok(ApiResponsePayload::Empty)).unwrap();
-                        }
-                        ApiRequest::VmmPing(sender) => {
-                            sender.send(Ok(ApiResponsePayload::Empty)).unwrap();
-                        }
-                        ApiRequest::VmPause(sender) => {
-                            sender.send(Ok(ApiResponsePayload::Empty)).unwrap();
-                        }
-                        ApiRequest::VmResume(sender) => {
-                            sender.send(Ok(ApiResponsePayload::Empty)).unwrap();
-                        }
-                        ApiRequest::VmSnapshot(_, sender) => {
-                            sender.send(Ok(ApiResponsePayload::Empty)).unwrap();
-                        }
-                        ApiRequest::VmRestore(_, sender) => {
-                            sender.send(Ok(ApiResponsePayload::Empty)).unwrap();
-                        }
-                        ApiRequest::VmmShutdown(sender) => {
-                            sender.send(Ok(ApiResponsePayload::Empty)).unwrap();
-                        }
-                        ApiRequest::VmResize(_, sender) => {
-                            sender.send(Ok(ApiResponsePayload::Empty)).unwrap();
-                        }
-                        ApiRequest::VmResizeZone(_, sender) => {
-                            sender.send(Ok(ApiResponsePayload::Empty)).unwrap();
-                        }
-                        ApiRequest::VmAddDevice(_, sender) => {
-                            sender.send(Ok(ApiResponsePayload::Empty)).unwrap();
-                        }
-                        ApiRequest::VmAddUserDevice(_, sender) => {
-                            sender.send(Ok(ApiResponsePayload::Empty)).unwrap();
-                        }
-                        ApiRequest::VmRemoveDevice(_, sender) => {
-                            sender.send(Ok(ApiResponsePayload::Empty)).unwrap();
-                        }
-                        ApiRequest::VmAddDisk(_, sender) => {
-                            sender.send(Ok(ApiResponsePayload::Empty)).unwrap();
-                        }
-                        ApiRequest::VmAddFs(_, sender) => {
-                            sender.send(Ok(ApiResponsePayload::Empty)).unwrap();
-                        }
-                        ApiRequest::VmAddPmem(_, sender) => {
-                            sender.send(Ok(ApiResponsePayload::Empty)).unwrap();
-                        }
-                        ApiRequest::VmAddNet(_, sender) => {
-                            sender.send(Ok(ApiResponsePayload::Empty)).unwrap();
-                        }
-                        ApiRequest::VmAddVdpa(_, sender) => {
-                            sender.send(Ok(ApiResponsePayload::Empty)).unwrap();
-                        }
-                        ApiRequest::VmAddVsock(_, sender) => {
-                            sender.send(Ok(ApiResponsePayload::Empty)).unwrap();
-                        }
-                        ApiRequest::VmCounters(sender) => {
-                            sender.send(Ok(ApiResponsePayload::Empty)).unwrap();
-                        }
-                        ApiRequest::VmReceiveMigration(_, sender) => {
-                            sender.send(Ok(ApiResponsePayload::Empty)).unwrap();
-                        }
-                        ApiRequest::VmSendMigration(_, sender) => {
-                            sender.send(Ok(ApiResponsePayload::Empty)).unwrap();
-                        }
-                        ApiRequest::VmPowerButton(sender) => {
-                            sender.send(Ok(ApiResponsePayload::Empty)).unwrap();
-                        }
-                    }
+                    api_request(&mut StubApiRequestHandler).unwrap();
                 }
             }
             _ => {

hypervisor/Cargo.toml

@@ -12,17 +12,17 @@ sev_snp = ["igvm_parser", "igvm_defs"]
 tdx = []
 
 [dependencies]
-anyhow = "1.0.75"
+anyhow = "1.0.79"
 byteorder = "1.4.3"
 igvm_defs = { git = "https://github.com/microsoft/igvm", branch = "main", package = "igvm_defs", optional  = true }
 igvm_parser = { git = "https://github.com/microsoft/igvm", branch = "main", package = "igvm", optional  = true }
-libc = "0.2.147"
+libc = "0.2.153"
 log = "0.4.20"
 kvm-ioctls = { version = "0.16.0", optional = true }
-kvm-bindings = { git = "https://github.com/cloud-hypervisor/kvm-bindings", branch = "ch-live-upgrade-stable-37.x", features = ["with-serde", "fam-wrappers"], optional  = true }
+kvm-bindings = { git = "https://github.com/cloud-hypervisor/kvm-bindings", branch = "ch-v0.7.0", features = ["with-serde", "fam-wrappers"], optional  = true }
 mshv-bindings = { git = "https://github.com/rust-vmm/mshv", branch = "main", features = ["with-serde", "fam-wrappers"], optional  = true }
 mshv-ioctls = { git = "https://github.com/rust-vmm/mshv", branch = "main", optional  = true}
-serde = { version = "1.0.168", features = ["rc", "derive"] }
+serde = { version = "1.0.196", features = ["rc", "derive"] }
 serde_with = { version = "3.4.0", default-features = false, features = ["macros"] }
 vfio-ioctls = { git = "https://github.com/rust-vmm/vfio", branch = "main", default-features = false }
 vm-memory = { version = "0.14.0", features = ["backend-mmap", "backend-atomic"] }
@@ -36,4 +36,4 @@ default-features = false
 features = ["std", "decoder", "op_code_info", "instr_info", "fast_fmt"]
 
 [dev-dependencies]
-env_logger = "0.10.0"
+env_logger = "0.10.1"

hypervisor/src/arch/x86/mod.rs

@@ -312,10 +312,10 @@ pub struct MsrEntry {
     pub data: u64,
 }
 
-#[repr(C)]
 #[serde_with::serde_as]
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
 pub struct XsaveState {
+    #[serde_as(as = "[_; 1024usize]")]
     pub region: [u32; 1024usize],
 }
 
@@ -325,26 +325,3 @@ impl Default for XsaveState {
         unsafe { ::std::mem::zeroed() }
     }
 }
-
-impl<'de> serde::Deserialize<'de> for XsaveState {
-    fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>
-    where
-        D: serde::Deserializer<'de>,
-    {
-        let region: Vec<u32> = Vec::deserialize(deserializer)?;
-        let mut val: XsaveState = XsaveState::default();
-        // This panics if the source and destination have different lengths.
-        val.region.copy_from_slice(&region[..]);
-        Ok(val)
-    }
-}
-
-impl serde::Serialize for XsaveState {
-    fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
-    where
-        S: serde::Serializer,
-    {
-        let region = &self.region[..];
-        region.serialize(serializer)
-    }
-}

hypervisor/src/cpu.rs

@@ -272,6 +272,12 @@ pub enum HypervisorCpuError {
     ///
     #[error("Failed to get CPUID entries: {0}")]
     GetCpuidVales(#[source] anyhow::Error),
+    ///
+    /// Setting SEV control register error
+    ///
+    #[cfg(feature = "sev_snp")]
+    #[error("Failed to set sev control register: {0}")]
+    SetSevControlRegister(#[source] anyhow::Error),
 }
 
 #[derive(Debug)]
@@ -495,4 +501,8 @@ pub trait Vcpu: Send + Sync {
     ) -> Result<[u32; 4]> {
         unimplemented!()
     }
+    #[cfg(feature = "mshv")]
+    fn set_sev_control_register(&self, _reg: u64) -> Result<()> {
+        unimplemented!()
+    }
 }

hypervisor/src/kvm/mod.rs

@@ -315,7 +315,7 @@ impl From<CpuState> for VcpuKvmState {
 #[cfg(target_arch = "x86_64")]
 impl From<kvm_clock_data> for ClockData {
     fn from(d: kvm_clock_data) -> Self {
-        ClockData::Kvm(d.into())
+        ClockData::Kvm(d)
     }
 }
 
@@ -323,7 +323,7 @@ impl From<kvm_clock_data> for ClockData {
 impl From<ClockData> for kvm_clock_data {
     fn from(ms: ClockData) -> Self {
         match ms {
-            ClockData::Kvm(s) => s.into(),
+            ClockData::Kvm(s) => s,
             /* Needed in case other hypervisors are enabled */
             #[allow(unreachable_patterns)]
             _ => panic!("CpuState is not valid"),
@@ -1977,7 +1977,7 @@ impl cpu::Vcpu for KvmVcpu {
             msr_entries
         };
 
-        let vcpu_events = self.get_vcpu_events()?.into();
+        let vcpu_events = self.get_vcpu_events()?;
         let tsc_khz = self.tsc_khz()?;
 
         Ok(VcpuKvmState {
@@ -2130,7 +2130,7 @@ impl cpu::Vcpu for KvmVcpu {
             }
         }
 
-        self.set_vcpu_events(&state.vcpu_events.into())?;
+        self.set_vcpu_events(&state.vcpu_events)?;
 
         Ok(())
     }

hypervisor/src/kvm/x86_64/mod.rs

@@ -22,7 +22,7 @@ pub use {
     kvm_bindings::kvm_cpuid_entry2, kvm_bindings::kvm_dtable, kvm_bindings::kvm_fpu,
     kvm_bindings::kvm_lapic_state, kvm_bindings::kvm_mp_state as MpState,
     kvm_bindings::kvm_msr_entry, kvm_bindings::kvm_regs, kvm_bindings::kvm_segment,
-    kvm_bindings::kvm_sregs, kvm_bindings::kvm_vcpu_events_old as VcpuEvents,
+    kvm_bindings::kvm_sregs, kvm_bindings::kvm_vcpu_events as VcpuEvents,
     kvm_bindings::kvm_xcrs as ExtendedControlRegisters, kvm_bindings::kvm_xsave,
     kvm_bindings::CpuId, kvm_bindings::MsrList, kvm_bindings::Msrs as MsrEntries,
     kvm_bindings::KVM_CPUID_FLAG_SIGNIFCANT_INDEX,

hypervisor/src/lib.rs

@@ -134,6 +134,7 @@ pub const USER_MEMORY_REGION_READ: u32 = 1;
 pub const USER_MEMORY_REGION_WRITE: u32 = 1 << 1;
 pub const USER_MEMORY_REGION_EXECUTE: u32 = 1 << 2;
 pub const USER_MEMORY_REGION_LOG_DIRTY: u32 = 1 << 3;
+pub const USER_MEMORY_REGION_ADJUSTABLE: u32 = 1 << 4;
 
 #[derive(Debug)]
 pub enum MpState {
@@ -162,7 +163,7 @@ pub enum CpuState {
 #[cfg(target_arch = "x86_64")]
 pub enum ClockData {
     #[cfg(feature = "kvm")]
-    Kvm(kvm_bindings::kvm_clock_data_old),
+    Kvm(kvm_bindings::kvm_clock_data),
     #[cfg(feature = "mshv")]
     Mshv, /* MSHV does not support ClockData yet */
 }

hypervisor/src/mshv/mod.rs

@@ -31,7 +31,8 @@ use snp_constants::*;
 
 use crate::{
     ClockData, CpuState, IoEventAddress, IrqRoutingEntry, MpState, UserMemoryRegion,
-    USER_MEMORY_REGION_EXECUTE, USER_MEMORY_REGION_READ, USER_MEMORY_REGION_WRITE,
+    USER_MEMORY_REGION_ADJUSTABLE, USER_MEMORY_REGION_EXECUTE, USER_MEMORY_REGION_READ,
+    USER_MEMORY_REGION_WRITE,
 };
 #[cfg(feature = "sev_snp")]
 use igvm_defs::IGVM_VHS_SNP_ID_BLOCK;
@@ -73,6 +74,9 @@ impl From<mshv_user_mem_region> for UserMemoryRegion {
         if region.flags & HV_MAP_GPA_EXECUTABLE != 0 {
             flags |= USER_MEMORY_REGION_EXECUTE;
         }
+        if region.flags & HV_MAP_GPA_ADJUSTABLE != 0 {
+            flags |= USER_MEMORY_REGION_ADJUSTABLE;
+        }
 
         UserMemoryRegion {
             guest_phys_addr: (region.guest_pfn << PAGE_SHIFT as u64)
@@ -97,6 +101,9 @@ impl From<UserMemoryRegion> for mshv_user_mem_region {
         if region.flags & USER_MEMORY_REGION_EXECUTE != 0 {
             flags |= HV_MAP_GPA_EXECUTABLE;
         }
+        if region.flags & USER_MEMORY_REGION_ADJUSTABLE != 0 {
+            flags |= HV_MAP_GPA_ADJUSTABLE;
+        }
 
         mshv_user_mem_region {
             guest_pfn: region.guest_phys_addr >> PAGE_SHIFT,
@@ -299,6 +306,8 @@ impl hypervisor::Hypervisor for MshvHypervisor {
             fd: vm_fd,
             msrs,
             dirty_log_slots: Arc::new(RwLock::new(HashMap::new())),
+            #[cfg(feature = "sev_snp")]
+            sev_snp_enabled: mshv_vm_type == VmType::Snp,
         }))
     }
 
@@ -597,6 +606,17 @@ impl cpu::Vcpu for MshvVcpu {
 
                     Ok(cpu::VmExit::Ignore)
                 }
+                hv_message_type_HVMSG_UNACCEPTED_GPA => {
+                    let info = x.to_memory_info().unwrap();
+                    let gva = info.guest_virtual_address;
+                    let gpa = info.guest_physical_address;
+
+                    Err(cpu::HypervisorCpuError::RunVcpu(anyhow!(
+                        "Unhandled VCPU exit: Unaccepted GPA({:x}) found at GVA({:x})",
+                        gpa,
+                        gva,
+                    )))
+                }
                 hv_message_type_HVMSG_X64_CPUID_INTERCEPT => {
                     let info = x.to_cpuid_info().unwrap();
                     debug!("cpuid eax: {:x}", { info.rax });
@@ -1241,6 +1261,18 @@ impl cpu::Vcpu for MshvVcpu {
         ]
         .to_vec()
     }
+
+    ///
+    /// Sets the AMD specific vcpu's sev control register.
+    ///
+    #[cfg(feature = "sev_snp")]
+    fn set_sev_control_register(&self, vmsa_pfn: u64) -> cpu::Result<()> {
+        let sev_control_reg = snp::get_sev_control_register(vmsa_pfn);
+
+        self.fd
+            .set_sev_control_register(sev_control_reg)
+            .map_err(|e| cpu::HypervisorCpuError::SetSevControlRegister(e.into()))
+    }
 }
 
 impl MshvVcpu {
@@ -1437,6 +1469,8 @@ pub struct MshvVm {
     fd: Arc<VmFd>,
     msrs: Vec<MsrEntry>,
     dirty_log_slots: Arc<RwLock<HashMap<u64, MshvDirtyLogSlot>>>,
+    #[cfg(feature = "sev_snp")]
+    sev_snp_enabled: bool,
 }
 
 impl MshvVm {
@@ -1556,6 +1590,11 @@ impl vm::Vm for MshvVm {
         addr: &IoEventAddress,
         datamatch: Option<DataMatch>,
     ) -> vm::Result<()> {
+        #[cfg(feature = "sev_snp")]
+        if self.sev_snp_enabled {
+            return Ok(());
+        }
+
         let addr = &mshv_ioctls::IoEventAddress::from(*addr);
         debug!(
             "register_ioevent fd {} addr {:x?} datamatch {:?}",
@@ -1635,7 +1674,7 @@ impl vm::Vm for MshvVm {
         readonly: bool,
         _log_dirty_pages: bool,
     ) -> UserMemoryRegion {
-        let mut flags = HV_MAP_GPA_READABLE | HV_MAP_GPA_EXECUTABLE;
+        let mut flags = HV_MAP_GPA_READABLE | HV_MAP_GPA_EXECUTABLE | HV_MAP_GPA_ADJUSTABLE;
         if !readonly {
             flags |= HV_MAP_GPA_WRITABLE;
         }
@@ -1814,7 +1853,7 @@ impl vm::Vm for MshvVm {
     fn complete_isolated_import(
         &self,
         snp_id_block: IGVM_VHS_SNP_ID_BLOCK,
-        host_data: &[u8],
+        host_data: [u8; 32],
         id_block_enabled: u8,
     ) -> vm::Result<()> {
         let mut auth_info = hv_snp_id_auth_info {
@@ -1847,7 +1886,7 @@ impl vm::Vm for MshvVm {
                         policy: get_default_snp_guest_policy(),
                     },
                     id_auth_info: auth_info,
-                    host_data: host_data[0..32].try_into().unwrap(),
+                    host_data,
                     id_block_enabled,
                     author_key_enabled: 0,
                 },

hypervisor/src/vm.rs

@@ -385,7 +385,7 @@ pub trait Vm: Send + Sync + Any {
     fn complete_isolated_import(
         &self,
         _snp_id_block: IGVM_VHS_SNP_ID_BLOCK,
-        _host_data: &[u8],
+        _host_data: [u8; 32],
         _id_block_enabled: u8,
     ) -> Result<()> {
         unimplemented!()

net_util/Cargo.toml

@@ -6,23 +6,23 @@ edition = "2021"
 
 [dependencies]
 epoll = "4.3.3"
-getrandom = "0.2.10"
-libc = "0.2.147"
+getrandom = "0.2.11"
+libc = "0.2.153"
 log = "0.4.20"
 net_gen = { path = "../net_gen" }
 rate_limiter = { path = "../rate_limiter" }
-serde = "1.0.168"
+serde = "1.0.196"
 thiserror = "1.0.52"
 versionize = "0.2.0"
 versionize_derive = "0.1.6"
 virtio-bindings = "0.2.0"
-virtio-queue = "0.12.0"
+virtio-queue = "0.11.0"
 vm-memory = { version = "0.14.0", features = ["backend-mmap", "backend-atomic", "backend-bitmap"] }
 vm-virtio = { path = "../vm-virtio" }
 vmm-sys-util = "0.12.1"
 
 [dev-dependencies]
-once_cell = "1.18.0"
+once_cell = "1.19.0"
 pnet = "0.34.0"
 pnet_datalink = "0.34.0"
-serde_json = "1.0.107"
+serde_json = "1.0.109"

net_util/src/queue_pair.rs

@@ -3,6 +3,7 @@
 // SPDX-License-Identifier: Apache-2.0 AND BSD-3-Clause
 
 use super::{register_listener, unregister_listener, vnet_hdr_len, Tap};
+use crate::GuestMemoryMmap;
 use rate_limiter::{RateLimiter, TokenType};
 use std::io;
 use std::num::Wrapping;
@@ -11,7 +12,6 @@ use std::sync::atomic::{AtomicU64, Ordering};
 use std::sync::Arc;
 use thiserror::Error;
 use virtio_queue::{Queue, QueueOwnedT, QueueT};
-use vm_memory::bitmap::Bitmap;
 use vm_memory::{Bytes, GuestMemory};
 use vm_virtio::{AccessPlatform, Translatable};
 
@@ -35,9 +35,9 @@ impl TxVirtio {
         }
     }
 
-    pub fn process_desc_chain<B: Bitmap + 'static>(
+    pub fn process_desc_chain(
         &mut self,
-        mem: &vm_memory::GuestMemoryMmap<B>,
+        mem: &GuestMemoryMmap,
         tap: &Tap,
         queue: &mut Queue,
         rate_limiter: &mut Option<RateLimiter>,
@@ -161,9 +161,9 @@ impl RxVirtio {
         }
     }
 
-    pub fn process_desc_chain<B: Bitmap + 'static>(
+    pub fn process_desc_chain(
         &mut self,
-        mem: &vm_memory::GuestMemoryMmap<B>,
+        mem: &GuestMemoryMmap,
         tap: &Tap,
         queue: &mut Queue,
         rate_limiter: &mut Option<RateLimiter>,
@@ -350,9 +350,9 @@ pub struct NetQueuePair {
 }
 
 impl NetQueuePair {
-    pub fn process_tx<B: Bitmap + 'static>(
+    pub fn process_tx(
         &mut self,
-        mem: &vm_memory::GuestMemoryMmap<B>,
+        mem: &GuestMemoryMmap,
         queue: &mut Queue,
     ) -> Result<bool, NetQueuePairError> {
         let tx_tap_retry = self.tx.process_desc_chain(
@@ -400,9 +400,9 @@ impl NetQueuePair {
             .map_err(NetQueuePairError::QueueNeedsNotification)
     }
 
-    pub fn process_rx<B: Bitmap + 'static>(
+    pub fn process_rx(
         &mut self,
-        mem: &vm_memory::GuestMemoryMmap<B>,
+        mem: &GuestMemoryMmap,
         queue: &mut Queue,
     ) -> Result<bool, NetQueuePairError> {
         self.rx_desc_avail = !self.rx.process_desc_chain(

option_parser/src/lib.rs

@@ -290,6 +290,17 @@ impl TupleValue for Vec<u64> {
     }
 }
 
+impl TupleValue for Vec<usize> {
+    fn parse_value(input: &str) -> Result<Self, TupleError> {
+        Ok(IntegerList::from_str(input)
+            .map_err(TupleError::InvalidIntegerList)?
+            .0
+            .iter()
+            .map(|v| *v as usize)
+            .collect())
+    }
+}
+
 pub struct Tuple<S, T>(pub Vec<(S, T)>);
 
 pub enum TupleError {

pci/Cargo.toml

@@ -10,7 +10,7 @@ kvm = ["vfio-ioctls/kvm"]
 mshv = ["vfio-ioctls/mshv"]
 
 [dependencies]
-anyhow = "1.0.75"
+anyhow = "1.0.79"
 byteorder = "1.4.3"
 hypervisor = { path = "../hypervisor" }
 vfio-bindings = { git = "https://github.com/rust-vmm/vfio", branch = "main", features = ["fam-wrappers"] }
@@ -18,9 +18,9 @@ vfio-ioctls = { git = "https://github.com/rust-vmm/vfio", branch = "main", defau
 vfio_user = { git = "https://github.com/rust-vmm/vfio-user", branch = "main" }
 
 vmm-sys-util = "0.12.1"
-libc = "0.2.147"
+libc = "0.2.153"
 log = "0.4.20"
-serde = { version = "1.0.168", features = ["derive"] }
+serde = { version = "1.0.196", features = ["derive"] }
 thiserror = "1.0.52"
 versionize = "0.2.0"
 versionize_derive = "0.1.6"

pci/src/vfio.rs

@@ -1571,6 +1571,16 @@ impl VfioPciDevice {
                     self.vm
                         .create_user_memory_region(mem_region)
                         .map_err(VfioPciError::CreateUserMemoryRegion)?;
+
+                    if !self.iommu_attached {
+                        self.container
+                            .vfio_dma_map(
+                                user_memory_region.start,
+                                user_memory_region.size,
+                                user_memory_region.host_addr,
+                            )
+                            .map_err(VfioPciError::DmaMap)?;
+                    }
                 }
             }
         }
@@ -1581,6 +1591,16 @@ impl VfioPciDevice {
     pub fn unmap_mmio_regions(&mut self) {
         for region in self.common.mmio_regions.iter() {
             for user_memory_region in region.user_memory_regions.iter() {
+                // Unmap from vfio container
+                if !self.iommu_attached {
+                    if let Err(e) = self
+                        .container
+                        .vfio_dma_unmap(user_memory_region.start, user_memory_region.size)
+                    {
+                        error!("Could not unmap mmio region from vfio container: {}", e);
+                    }
+                }
+
                 // Remove region
                 let r = self.vm.make_user_memory_region(
                     user_memory_region.slot,

performance-metrics/Cargo.toml

@@ -8,8 +8,8 @@ build = "../build.rs"
 [dependencies]
 clap = { version = "4.4.7", features = ["wrap_help"] }
 dirs = "5.0.0"
-serde = { version = "1.0.168", features = ["rc", "derive"] }
-serde_json = "1.0.107"
+serde = { version = "1.0.196", features = ["rc", "derive"] }
+serde_json = "1.0.109"
 test_infra = { path = "../test_infra" }
-thiserror = "1.0.40"
+thiserror = "1.0.52"
 wait-timeout = "0.2.0"

rate_limiter/Cargo.toml

@@ -4,7 +4,8 @@ version = "0.1.0"
 edition = "2021"
 
 [dependencies]
-libc = "0.2.147"
+epoll = "4.3.3"
+libc = "0.2.153"
 log = "0.4.20"
 thiserror = "1.0.40"
 vmm-sys-util = "0.12.1"

rate_limiter/src/group.rs

@@ -0,0 +1,526 @@
+// Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+//
+// Copyright 2023 Crusoe Energy Systems LLC
+// SPDX-License-Identifier: Apache-2.0
+
+use crate::{RateLimiter, TokenType};
+use core::panic::AssertUnwindSafe;
+use std::fs::File;
+use std::io;
+use std::os::unix::io::{AsRawFd, FromRawFd, RawFd};
+use std::result;
+use std::sync::{Arc, Mutex};
+use std::thread;
+use thiserror::Error;
+use vmm_sys_util::eventfd::EventFd;
+
+/// Errors associated with rate-limiter group.
+#[derive(Debug, Error)]
+pub enum Error {
+    /// Cannot create thread
+    #[error("Error spawning rate-limiter thread {0}")]
+    ThreadSpawn(#[source] io::Error),
+
+    /// Cannot create epoll context.
+    #[error("Error creating epoll context: {0}")]
+    Epoll(#[source] io::Error),
+
+    /// Cannot create EventFd.
+    #[error("Error creating EventFd: {0}")]
+    EventFd(#[source] io::Error),
+
+    /// Cannot create RateLimiter.
+    #[error("Error creating RateLimiter: {0}")]
+    RateLimiter(#[source] io::Error),
+
+    /// Cannot read from EventFd.
+    #[error("Error reading from EventFd: {0}")]
+    EventFdRead(#[source] io::Error),
+
+    /// Cannot write to EventFd.
+    #[error("Error writing to EventFd: {0}")]
+    EventFdWrite(#[source] io::Error),
+}
+
+/// The RateLimiterGroupHandle is a handle to a RateLimiterGroup that may be
+/// used in exactly the same way as the RateLimiter type. When the RateLimiter
+/// within a RateLimiterGroup is unblocked, each RateLimiterGroupHandle will
+/// be notified.
+pub struct RateLimiterGroupHandle {
+    eventfd: Arc<EventFd>,
+    inner: Arc<RateLimiterGroupInner>,
+}
+
+impl RateLimiterGroupHandle {
+    fn new(inner: Arc<RateLimiterGroupInner>) -> result::Result<Self, Error> {
+        let eventfd = Arc::new(EventFd::new(0).map_err(Error::EventFd)?);
+        inner.handles.lock().unwrap().push(eventfd.clone());
+        Ok(Self { eventfd, inner })
+    }
+
+    /// Attempts to consume tokens and returns whether that is possible.
+    ///
+    /// If rate limiting is disabled on provided `token_type`, this function will always succeed.
+    pub fn consume(&self, tokens: u64, token_type: TokenType) -> bool {
+        self.inner.rate_limiter.consume(tokens, token_type)
+    }
+
+    /// Adds tokens of `token_type` to their respective bucket.
+    ///
+    /// Can be used to *manually* add tokens to a bucket. Useful for reverting a
+    /// `consume()` if needed.
+    pub fn manual_replenish(&self, tokens: u64, token_type: TokenType) {
+        self.inner.rate_limiter.manual_replenish(tokens, token_type)
+    }
+
+    /// This function needs to be called every time there is an event on the
+    /// FD provided by this object's `AsRawFd` trait implementation.
+    ///
+    /// # Errors
+    ///
+    /// If the rate limiter is disabled or is not blocked, an error is returned.
+    pub fn event_handler(&self) -> Result<(), Error> {
+        self.eventfd.read().map_err(Error::EventFdRead).map(|_| ())
+    }
+
+    /// Returns whether this rate limiter is blocked.
+    ///
+    /// The limiter 'blocks' when a `consume()` operation fails because there was not enough
+    /// budget for it.
+    /// An event will be generated on the exported FD when the limiter 'unblocks'.
+    pub fn is_blocked(&self) -> bool {
+        self.inner.rate_limiter.is_blocked()
+    }
+}
+
+impl Clone for RateLimiterGroupHandle {
+    fn clone(&self) -> Self {
+        RateLimiterGroupHandle::new(self.inner.clone()).unwrap()
+    }
+}
+
+impl AsRawFd for RateLimiterGroupHandle {
+    fn as_raw_fd(&self) -> RawFd {
+        self.eventfd.as_raw_fd()
+    }
+}
+
+impl Drop for RateLimiterGroupHandle {
+    fn drop(&mut self) {
+        let mut handles = self.inner.handles.lock().unwrap();
+        let index = handles
+            .iter()
+            .position(|handle| handle.as_raw_fd() == self.eventfd.as_raw_fd())
+            .expect("RateLimiterGroupHandle must be subscribed to RateLimiterGroup");
+        handles.remove(index);
+    }
+}
+
+struct RateLimiterGroupInner {
+    id: String,
+    rate_limiter: RateLimiter,
+    handles: Mutex<Vec<Arc<EventFd>>>,
+}
+
+/// A RateLimiterGroup is an extension of RateLimiter that enables rate-limiting
+/// the aggregate io consumption of multiple consumers.
+pub struct RateLimiterGroup {
+    inner: Arc<RateLimiterGroupInner>,
+    epoll_file: File,
+    kill_evt: EventFd,
+    epoll_thread: Option<thread::JoinHandle<()>>,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+#[repr(u64)]
+enum EpollDispatch {
+    Kill = 1,
+    Unblocked = 2,
+    Unknown,
+}
+
+impl From<u64> for EpollDispatch {
+    fn from(v: u64) -> Self {
+        use EpollDispatch::*;
+        match v {
+            1 => Kill,
+            2 => Unblocked,
+            _ => Unknown,
+        }
+    }
+}
+
+impl RateLimiterGroup {
+    /// Create a new RateLimiterGroup.
+    pub fn new(
+        id: &str,
+        bytes_total_capacity: u64,
+        bytes_one_time_burst: u64,
+        bytes_complete_refill_time_ms: u64,
+        ops_total_capacity: u64,
+        ops_one_time_burst: u64,
+        ops_complete_refill_time_ms: u64,
+    ) -> result::Result<Self, Error> {
+        let rate_limiter = RateLimiter::new(
+            bytes_total_capacity,
+            bytes_one_time_burst,
+            bytes_complete_refill_time_ms,
+            ops_total_capacity,
+            ops_one_time_burst,
+            ops_complete_refill_time_ms,
+        )
+        .map_err(Error::RateLimiter)?;
+
+        let epoll_fd = epoll::create(true).map_err(Error::Epoll)?;
+        let kill_evt = EventFd::new(0).map_err(Error::EventFd)?;
+
+        epoll::ctl(
+            epoll_fd,
+            epoll::ControlOptions::EPOLL_CTL_ADD,
+            kill_evt.as_raw_fd(),
+            epoll::Event::new(epoll::Events::EPOLLIN, EpollDispatch::Kill as u64),
+        )
+        .map_err(Error::Epoll)?;
+
+        epoll::ctl(
+            epoll_fd,
+            epoll::ControlOptions::EPOLL_CTL_ADD,
+            rate_limiter.as_raw_fd(),
+            epoll::Event::new(epoll::Events::EPOLLIN, EpollDispatch::Unblocked as u64),
+        )
+        .map_err(Error::Epoll)?;
+
+        // Use 'File' to enforce closing on 'epoll_fd'
+        // SAFETY: epoll_fd is valid
+        let epoll_file = unsafe { File::from_raw_fd(epoll_fd) };
+
+        Ok(Self {
+            inner: Arc::new(RateLimiterGroupInner {
+                id: id.to_string(),
+                rate_limiter,
+                handles: Mutex::new(Vec::new()),
+            }),
+            epoll_file,
+            kill_evt,
+            epoll_thread: None,
+        })
+    }
+
+    /// Create a new RateLimiterGroupHandle.
+    pub fn new_handle(&self) -> result::Result<RateLimiterGroupHandle, Error> {
+        RateLimiterGroupHandle::new(self.inner.clone())
+    }
+
+    /// Start a worker thread to broadcast an event to each RateLimiterGroupHandle
+    /// when the RateLimiter becomes unblocked.
+    pub fn start_thread(&mut self, exit_evt: EventFd) -> result::Result<(), Error> {
+        let inner = self.inner.clone();
+        let epoll_fd = self.epoll_file.as_raw_fd();
+        thread::Builder::new()
+            .name(format!("rate-limit-group-{}", inner.id))
+            .spawn(move || {
+                let res = std::panic::catch_unwind(AssertUnwindSafe(move || {
+                    const EPOLL_EVENTS_LEN: usize = 2;
+
+                    let mut events =
+                        [epoll::Event::new(epoll::Events::empty(), 0); EPOLL_EVENTS_LEN];
+
+                    loop {
+                        let num_events = match epoll::wait(epoll_fd, -1, &mut events[..]) {
+                            Ok(res) => res,
+                            Err(e) => {
+                                if e.kind() == io::ErrorKind::Interrupted {
+                                    continue;
+                                } else {
+                                    return Err(Error::Epoll(e));
+                                }
+                            }
+                        };
+
+                        for event in events.iter().take(num_events) {
+                            let dispatch_event: EpollDispatch = event.data.into();
+                            match dispatch_event {
+                                EpollDispatch::Unknown => {
+                                    let event = event.data;
+                                    warn!("Unknown rate-limiter loop event: {}", event);
+                                }
+                                EpollDispatch::Unblocked => {
+                                    inner.rate_limiter.event_handler().unwrap();
+                                    let handles = inner.handles.lock().unwrap();
+                                    for handle in handles.iter() {
+                                        handle.write(1).map_err(Error::EventFdWrite)?
+                                    }
+                                }
+                                EpollDispatch::Kill => {
+                                    info!(
+                                        "KILL_EVENT received, stopping rate-limit-group epoll loop"
+                                    );
+                                    return Ok(());
+                                }
+                            }
+                        }
+                    }
+                }));
+
+                match res {
+                    Ok(res) => {
+                        if let Err(e) = res {
+                            error!("Error running rate-limit-group worker: {:?}", e);
+                            exit_evt.write(1).unwrap();
+                        }
+                    }
+                    Err(_) => {
+                        error!("rate-limit-group worker panicked");
+                        exit_evt.write(1).unwrap();
+                    }
+                };
+            })
+            .map(|thread| self.epoll_thread.insert(thread))
+            .map_err(Error::ThreadSpawn)?;
+
+        Ok(())
+    }
+}
+
+impl Drop for RateLimiterGroup {
+    fn drop(&mut self) {
+        self.kill_evt.write(1).unwrap();
+
+        if let Some(t) = self.epoll_thread.take() {
+            if let Err(e) = t.join() {
+                error!("Error joining thread: {:?}", e);
+            }
+        }
+    }
+}
+
+#[cfg(test)]
+pub(crate) mod tests {
+    use super::RateLimiterGroupHandle;
+    use crate::{group::RateLimiterGroup, TokenBucket, TokenType, REFILL_TIMER_INTERVAL_MS};
+    use std::{os::fd::AsRawFd, thread, time::Duration};
+    use vmm_sys_util::eventfd::EventFd;
+
+    impl RateLimiterGroupHandle {
+        pub fn bandwidth(&self) -> Option<TokenBucket> {
+            let guard = self.inner.rate_limiter.inner.lock().unwrap();
+            guard.bandwidth.clone()
+        }
+
+        pub fn ops(&self) -> Option<TokenBucket> {
+            let guard = self.inner.rate_limiter.inner.lock().unwrap();
+            guard.ops.clone()
+        }
+    }
+
+    #[test]
+    fn test_rate_limiter_group_new() {
+        let l = RateLimiterGroup::new("test", 1000, 1001, 1002, 1003, 1004, 1005).unwrap();
+        let h = l.new_handle().unwrap();
+        let bw = h.bandwidth().unwrap();
+        assert_eq!(bw.capacity(), 1000);
+        assert_eq!(bw.one_time_burst(), 1001);
+        assert_eq!(bw.refill_time_ms(), 1002);
+        assert_eq!(bw.budget(), 1000);
+
+        let ops = h.ops().unwrap();
+        assert_eq!(ops.capacity(), 1003);
+        assert_eq!(ops.one_time_burst(), 1004);
+        assert_eq!(ops.refill_time_ms(), 1005);
+        assert_eq!(ops.budget(), 1003);
+    }
+
+    #[test]
+    fn test_rate_limiter_group_manual_replenish() {
+        // rate limiter with limit of 1000 bytes/s and 1000 ops/s
+        let l = RateLimiterGroup::new("test", 1000, 0, 1000, 1000, 0, 1000).unwrap();
+        let h = l.new_handle().unwrap();
+
+        // consume 123 bytes
+        assert!(h.consume(123, TokenType::Bytes));
+        h.manual_replenish(23, TokenType::Bytes);
+        {
+            let bytes_tb = h.bandwidth().unwrap();
+            assert_eq!(bytes_tb.budget(), 900);
+        }
+        // consume 123 ops
+        assert!(h.consume(123, TokenType::Ops));
+        h.manual_replenish(23, TokenType::Ops);
+        {
+            let bytes_tb = h.ops().unwrap();
+            assert_eq!(bytes_tb.budget(), 900);
+        }
+    }
+
+    #[test]
+    fn test_rate_limiter_group_bandwidth() {
+        // rate limiter with limit of 1000 bytes/s
+        let mut l = RateLimiterGroup::new("test", 1000, 0, 1000, 0, 0, 0).unwrap();
+        l.start_thread(EventFd::new(0).unwrap()).unwrap();
+
+        let h = l.new_handle().unwrap();
+
+        // limiter should not be blocked
+        assert!(!h.is_blocked());
+        // raw FD for this disabled should be valid
+        assert!(h.as_raw_fd() > 0);
+
+        // ops/s limiter should be disabled so consume(whatever) should work
+        assert!(h.consume(u64::max_value(), TokenType::Ops));
+
+        // do full 1000 bytes
+        assert!(h.consume(1000, TokenType::Bytes));
+        // try and fail on another 100
+        assert!(!h.consume(100, TokenType::Bytes));
+        // since consume failed, limiter should be blocked now
+        assert!(h.is_blocked());
+        // wait half the timer period
+        thread::sleep(Duration::from_millis(REFILL_TIMER_INTERVAL_MS / 2));
+        // limiter should still be blocked
+        assert!(h.is_blocked());
+        // wait the other half of the timer period
+        thread::sleep(Duration::from_millis(REFILL_TIMER_INTERVAL_MS / 2));
+        // the timer_fd should have an event on it by now
+        assert!(h.event_handler().is_ok());
+        // limiter should now be unblocked
+        assert!(!h.is_blocked());
+        // try and succeed on another 100 bytes this time
+        assert!(h.consume(100, TokenType::Bytes));
+    }
+
+    #[test]
+    fn test_rate_limiter_group_ops() {
+        // rate limiter with limit of 1000 ops/s
+        let mut l = RateLimiterGroup::new("test", 0, 0, 0, 1000, 0, 1000).unwrap();
+        l.start_thread(EventFd::new(0).unwrap()).unwrap();
+
+        let h = l.new_handle().unwrap();
+
+        // limiter should not be blocked
+        assert!(!h.is_blocked());
+        // raw FD for this disabled should be valid
+        assert!(h.as_raw_fd() > 0);
+
+        // bytes/s limiter should be disabled so consume(whatever) should work
+        assert!(h.consume(u64::max_value(), TokenType::Bytes));
+
+        // do full 1000 ops
+        assert!(h.consume(1000, TokenType::Ops));
+        // try and fail on another 100
+        assert!(!h.consume(100, TokenType::Ops));
+        // since consume failed, limiter should be blocked now
+        assert!(h.is_blocked());
+        // wait half the timer period
+        thread::sleep(Duration::from_millis(REFILL_TIMER_INTERVAL_MS / 2));
+        // limiter should still be blocked
+        assert!(h.is_blocked());
+        // wait the other half of the timer period
+        thread::sleep(Duration::from_millis(REFILL_TIMER_INTERVAL_MS / 2));
+        // the timer_fd should have an event on it by now
+        assert!(h.event_handler().is_ok());
+        // limiter should now be unblocked
+        assert!(!h.is_blocked());
+        // try and succeed on another 100 ops this time
+        assert!(h.consume(100, TokenType::Ops));
+    }
+
+    #[test]
+    fn test_rate_limiter_group_full() {
+        // rate limiter with limit of 1000 bytes/s and 1000 ops/s
+        let mut l = RateLimiterGroup::new("test", 1000, 0, 1000, 1000, 0, 1000).unwrap();
+        l.start_thread(EventFd::new(0).unwrap()).unwrap();
+
+        let h = l.new_handle().unwrap();
+
+        // limiter should not be blocked
+        assert!(!h.is_blocked());
+        // raw FD for this disabled should be valid
+        assert!(h.as_raw_fd() > 0);
+
+        // do full 1000 bytes
+        assert!(h.consume(1000, TokenType::Ops));
+        // do full 1000 bytes
+        assert!(h.consume(1000, TokenType::Bytes));
+        // try and fail on another 100 ops
+        assert!(!h.consume(100, TokenType::Ops));
+        // try and fail on another 100 bytes
+        assert!(!h.consume(100, TokenType::Bytes));
+        // since consume failed, limiter should be blocked now
+        assert!(h.is_blocked());
+        // wait half the timer period
+        thread::sleep(Duration::from_millis(REFILL_TIMER_INTERVAL_MS / 2));
+        // limiter should still be blocked
+        assert!(h.is_blocked());
+        // wait the other half of the timer period
+        thread::sleep(Duration::from_millis(REFILL_TIMER_INTERVAL_MS / 2));
+        // the timer_fd should have an event on it by now
+        assert!(h.event_handler().is_ok());
+        // limiter should now be unblocked
+        assert!(!h.is_blocked());
+        // try and succeed on another 100 ops this time
+        assert!(h.consume(100, TokenType::Ops));
+        // try and succeed on another 100 bytes this time
+        assert!(h.consume(100, TokenType::Bytes));
+    }
+
+    #[test]
+    fn test_rate_limiter_group_overconsumption() {
+        // initialize the rate limiter
+        let mut l = RateLimiterGroup::new("test", 1000, 0, 1000, 1000, 0, 1000).unwrap();
+        l.start_thread(EventFd::new(0).unwrap()).unwrap();
+
+        let h = l.new_handle().unwrap();
+
+        // try to consume 2.5x the bucket size
+        // we are "borrowing" 1.5x the bucket size in tokens since
+        // the bucket is full
+        assert!(h.consume(2500, TokenType::Bytes));
+
+        // check that even after a whole second passes, the rate limiter
+        // is still blocked
+        thread::sleep(Duration::from_millis(1000));
+        assert!(h.is_blocked());
+
+        // after 1.5x the replenish time has passed, the rate limiter
+        // is available again
+        thread::sleep(Duration::from_millis(500));
+        assert!(h.event_handler().is_ok());
+        assert!(!h.is_blocked());
+
+        // reset the rate limiter
+        let mut l = RateLimiterGroup::new("test", 1000, 0, 1000, 1000, 0, 1000).unwrap();
+        l.start_thread(EventFd::new(0).unwrap()).unwrap();
+
+        let h = l.new_handle().unwrap();
+        // try to consume 1.5x the bucket size
+        // we are "borrowing" 1.5x the bucket size in tokens since
+        // the bucket is full, should arm the timer to 0.5x replenish
+        // time, which is 500 ms
+        assert!(h.consume(1500, TokenType::Bytes));
+
+        // check that after more than the minimum refill time,
+        // the rate limiter is still blocked
+        thread::sleep(Duration::from_millis(200));
+        assert!(h.is_blocked());
+
+        // try to consume some tokens, which should fail as the timer
+        // is still active
+        assert!(!h.consume(100, TokenType::Bytes));
+        assert!(h.is_blocked());
+
+        // check that after the minimum refill time, the timer was not
+        // overwritten and the rate limiter is still blocked from the
+        // borrowing we performed earlier
+        thread::sleep(Duration::from_millis(100));
+        assert!(h.is_blocked());
+        assert!(!h.consume(100, TokenType::Bytes));
+
+        // after waiting out the full duration, rate limiter should be
+        // available again
+        thread::sleep(Duration::from_millis(200));
+        assert!(h.event_handler().is_ok());
+        assert!(!h.is_blocked());
+        assert!(h.consume(100, TokenType::Bytes));
+    }
+}

rate_limiter/src/lib.rs

@@ -53,6 +53,9 @@ use std::sync::Mutex;
 use std::time::{Duration, Instant};
 use vmm_sys_util::timerfd::TimerFd;
 
+/// Module for group rate limiting.
+pub mod group;
+
 #[derive(Debug)]
 /// Describes the errors that may occur while handling rate limiter events.
 pub enum Error {

release-notes.md

@@ -1,12 +1,20 @@
-- [v37.1](#v371)
+- [v38.0](#v380)
+    - [Group Rate Limiter on Block Devices](#group-rate-limiter-on-block-devices)
+    - [CPU Pinning Support for Block Device Worker Thread](#cpu-pinning-support-for-block-device-worker-thread)
+    - [Optimized Boot Time with Parallel Memory Prefault](#optimized-boot-time-with-parallel-memory-prefault)
+    - [New 'debug-console' Device](#new-debug-console-device)
+    - [Improved VFIO Device Support](#improved-vfio-device-support)
+    - [Extended CPU Affinity Support](#extended-cpu-affinity-support)
+    - [Notable Bug Fixes](#notable-bug-fixes)
+    - [Contributors](#contributors)
 - [v37.0](#v370)
     - [Long Term Support (LTS) Release](#long-term-support-lts-release)
-    - [Improved VFIO Device Passthrough with Multiple PCI Segments](#improved-vfio-device-passthrough-with-multiple-pci-segments)
+    - [Multiple PCI segments Support for 32-bit VFIO devices](#multiple-pci-segments-support-for-32-bit-vfio-devices)
     - [Configurable Named TAP Devices](#configurable-named-tap-devices)
     - [TTY Output from Both Serial Device and Virtio Console](#tty-output-from-both-serial-device-and-virtio-console)
     - [Faster VM Restoration from Snapshots](#faster-vm-restoration-from-snapshots)
-    - [Notable Bug Fixes](#notable-bug-fixes)
-    - [Contributors](#contributors)
+    - [Notable Bug Fixes](#notable-bug-fixes-1)
+    - [Contributors](#contributors-1)
 - [v36.0](#v360)
     - [Command Line Changes](#command-line-changes)
     - [Enabled Features Reported via API Endpoint and CLI](#enabled-features-reported-via-api-endpoint-and-cli)
@@ -15,31 +23,31 @@
     - [Unix Socket Backend for Serial Port](#unix-socket-backend-for-serial-port)
     - [AIO Backend for Block Devices](#aio-backend-for-block-devices)
     - [Documentation Improvements](#documentation-improvements)
-    - [Notable Bug Fixes](#notable-bug-fixes-1)
-    - [Contributors](#contributors-1)
+    - [Notable Bug Fixes](#notable-bug-fixes-2)
+    - [Contributors](#contributors-2)
 - [v35.0](#v350)
     - [`virtio-vsock` Support for Linux Guest Kernel v6.3+](#virtio-vsock-support-for-linux-guest-kernel-v63)
     - [User Specified Serial Number for `virtio-block`](#user-specified-serial-number-for-virtio-block)
     - [vCPU TSC Frequency Included in Migration State](#vcpu-tsc-frequency-included-in-migration-state)
-    - [Notable Bug Fixes](#notable-bug-fixes-2)
-    - [Contributors](#contributors-2)
+    - [Notable Bug Fixes](#notable-bug-fixes-3)
+    - [Contributors](#contributors-3)
 - [v34.0](#v340)
     - [Paravirtualised Panic Device Support](#paravirtualised-panic-device-support)
     - [Improvements to VM Core Dump](#improvements-to-vm-core-dump)
     - [QCOW2 Support for Backing Files](#qcow2-support-for-backing-files)
     - [Minimum Host Kernel Bump](#minimum-host-kernel-bump)
-    - [Notable Bug Fixes](#notable-bug-fixes-3)
-    - [Contributors](#contributors-3)
+    - [Notable Bug Fixes](#notable-bug-fixes-4)
+    - [Contributors](#contributors-4)
 - [v33.0](#v330)
     - [D-Bus based API](#d-bus-based-api)
     - [Expose Host CPU Cache Details for AArch64](#expose-host-cpu-cache-details-for-aarch64)
-    - [Notable Bug Fixes](#notable-bug-fixes-4)
-    - [Contributors](#contributors-4)
+    - [Notable Bug Fixes](#notable-bug-fixes-5)
+    - [Contributors](#contributors-5)
 - [v32.0](#v320)
     - [Increased PCI Segment Limit](#increased-pci-segment-limit)
     - [API Changes](#api-changes)
-    - [Notable Bug Fixes](#notable-bug-fixes-5)
-    - [Contributors](#contributors-5)
+    - [Notable Bug Fixes](#notable-bug-fixes-6)
+    - [Contributors](#contributors-6)
 - [v31.1](#v311)
 - [v31.0](#v310)
     - [Update to Latest `acpi_tables`](#update-to-latest-acpi_tables)
@@ -47,15 +55,15 @@
     - [Improvements on Console `SIGWINCH` Handler](#improvements-on-console-sigwinch-handler)
     - [Remove Directory Support from `MemoryZoneConfig::file`](#remove-directory-support-from-memoryzoneconfigfile)
     - [Documentation Improvements](#documentation-improvements-1)
-    - [Notable Bug Fixes](#notable-bug-fixes-6)
-    - [Contributors](#contributors-6)
+    - [Notable Bug Fixes](#notable-bug-fixes-7)
+    - [Contributors](#contributors-7)
 - [v30.0](#v300)
     - [Command Line Changes for Reduced Binary Size](#command-line-changes-for-reduced-binary-size)
     - [Basic vfio-user Server Support](#basic-vfio-user-server-support)
     - [Heap Profiling Support](#heap-profiling-support)
     - [Documentation Improvements](#documentation-improvements-2)
-    - [Notable Bug Fixes](#notable-bug-fixes-7)
-    - [Contributors](#contributors-7)
+    - [Notable Bug Fixes](#notable-bug-fixes-8)
+    - [Contributors](#contributors-8)
 - [v28.2](#v282)
 - [v29.0](#v290)
     - [Release Binary Supports Both MSHV and KVM](#release-binary-supports-both-mshv-and-kvm)
@@ -65,10 +73,10 @@
     - [`AArch64` Documentation Integration](#aarch64-documentation-integration)
     - [`virtio-block` Counters Enhancement](#virtio-block-counters-enhancement)
     - [TCP Offload Control](#tcp-offload-control)
-    - [Notable Bug Fixes](#notable-bug-fixes-8)
+    - [Notable Bug Fixes](#notable-bug-fixes-9)
     - [Removals](#removals)
     - [Deprecations](#deprecations)
-    - [Contributors](#contributors-8)
+    - [Contributors](#contributors-9)
 - [v28.1](#v281)
 - [v28.0](#v280)
     - [Community Engagement (Reminder)](#community-engagement-reminder)
@@ -76,9 +84,9 @@
     - [Virtualised TPM Support](#virtualised-tpm-support)
     - [Transparent Huge Page Support](#transparent-huge-page-support)
     - [README Quick Start Improved](#readme-quick-start-improved)
-    - [Notable Bug Fixes](#notable-bug-fixes-9)
+    - [Notable Bug Fixes](#notable-bug-fixes-10)
     - [Removals](#removals-1)
-    - [Contributors](#contributors-9)
+    - [Contributors](#contributors-10)
 - [v27.0](#v270)
     - [Community Engagement](#community-engagement)
     - [Prebuilt Packages](#prebuilt-packages)
@@ -87,41 +95,41 @@
     - [Simplified Build Feature Flags](#simplified-build-feature-flags)
     - [Asynchronous Kernel Loading](#asynchronous-kernel-loading)
     - [GDB Support for AArch64](#gdb-support-for-aarch64)
-    - [Notable Bug Fixes](#notable-bug-fixes-10)
+    - [Notable Bug Fixes](#notable-bug-fixes-11)
     - [Deprecations](#deprecations-1)
-    - [Contributors](#contributors-10)
+    - [Contributors](#contributors-11)
 - [v26.0](#v260)
     - [SMBIOS Improvements via `--platform`](#smbios-improvements-via---platform)
     - [Unified Binary MSHV and KVM Support](#unified-binary-mshv-and-kvm-support)
-    - [Notable Bug Fixes](#notable-bug-fixes-11)
+    - [Notable Bug Fixes](#notable-bug-fixes-12)
     - [Deprecations](#deprecations-2)
     - [Removals](#removals-2)
-    - [Contributors](#contributors-11)
+    - [Contributors](#contributors-12)
 - [v25.0](#v250)
     - [`ch-remote` Improvements](#ch-remote-improvements-1)
     - [VM "Coredump" Support](#vm-coredump-support)
-    - [Notable Bug Fixes](#notable-bug-fixes-12)
+    - [Notable Bug Fixes](#notable-bug-fixes-13)
     - [Removals](#removals-3)
-    - [Contributors](#contributors-12)
+    - [Contributors](#contributors-13)
 - [v24.0](#v240)
     - [Bypass Mode for `virtio-iommu`](#bypass-mode-for-virtio-iommu)
     - [Ensure Identifiers Uniqueness](#ensure-identifiers-uniqueness)
     - [Sparse Mmap support](#sparse-mmap-support)
     - [Expose Platform Serial Number](#expose-platform-serial-number)
-    - [Notable Bug Fixes](#notable-bug-fixes-13)
+    - [Notable Bug Fixes](#notable-bug-fixes-14)
     - [Notable Improvements](#notable-improvements)
     - [Deprecations](#deprecations-3)
     - [New on the Website](#new-on-the-website)
-    - [Contributors](#contributors-13)
+    - [Contributors](#contributors-14)
 - [v23.1](#v231)
 - [v23.0](#v230)
     - [vDPA Support](#vdpa-support)
     - [Updated OS Support list](#updated-os-support-list)
     - [`AArch64` Memory Map Improvements](#aarch64-memory-map-improvements)
     - [`AMX` Support](#amx-support)
-    - [Notable Bug Fixes](#notable-bug-fixes-14)
+    - [Notable Bug Fixes](#notable-bug-fixes-15)
     - [Deprecations](#deprecations-4)
-    - [Contributors](#contributors-14)
+    - [Contributors](#contributors-15)
 - [v22.1](#v221)
 - [v22.0](#v220)
     - [GDB Debug Stub Support](#gdb-debug-stub-support)
@@ -132,13 +140,13 @@
     - [PMU Support for AArch64](#pmu-support-for-aarch64)
     - [Documentation Under CC-BY-4.0 License](#documentation-under-cc-by-40-license)
     - [Deprecation of "Classic" `virtiofsd`](#deprecation-of-classic-virtiofsd)
-    - [Notable Bug Fixes](#notable-bug-fixes-15)
-    - [Contributors](#contributors-15)
+    - [Notable Bug Fixes](#notable-bug-fixes-16)
+    - [Contributors](#contributors-16)
 - [v21.0](#v210)
     - [Efficient Local Live Migration (for Live Upgrade)](#efficient-local-live-migration-for-live-upgrade)
     - [Recommended Kernel is Now 5.15](#recommended-kernel-is-now-515)
-    - [Notable Bug fixes](#notable-bug-fixes-16)
-    - [Contributors](#contributors-16)
+    - [Notable Bug fixes](#notable-bug-fixes-17)
+    - [Contributors](#contributors-17)
 - [v20.2](#v202)
 - [v20.1](#v201)
 - [v20.0](#v200)
@@ -147,8 +155,8 @@
     - [Improved VFIO support](#improved-vfio-support)
     - [Safer code](#safer-code)
     - [Extended documentation](#extended-documentation)
-    - [Notable bug fixes](#notable-bug-fixes-17)
-    - [Contributors](#contributors-17)
+    - [Notable bug fixes](#notable-bug-fixes-18)
+    - [Contributors](#contributors-18)
 - [v19.0](#v190)
     - [Improved PTY handling for serial and `virtio-console`](#improved-pty-handling-for-serial-and-virtio-console)
     - [PCI boot time optimisations](#pci-boot-time-optimisations)
@@ -156,8 +164,8 @@
     - [Live migration enhancements](#live-migration-enhancements)
     - [`virtio-mem` support with `vfio-user`](#virtio-mem-support-with-vfio-user)
     - [AArch64 for `virtio-iommu`](#aarch64-for-virtio-iommu)
-    - [Notable bug fixes](#notable-bug-fixes-18)
-    - [Contributors](#contributors-18)
+    - [Notable bug fixes](#notable-bug-fixes-19)
+    - [Contributors](#contributors-19)
 - [v18.0](#v180)
     - [Experimental User Device (`vfio-user`) support](#experimental-user-device-vfio-user-support)
     - [Migration support for `vhost-user` devices](#migration-support-for-vhost-user-devices)
@@ -167,23 +175,23 @@
     - [Live migration on MSHV hypervisor](#live-migration-on-mshv-hypervisor)
     - [AArch64 CPU topology support](#aarch64-cpu-topology-support)
     - [Power button support on AArch64](#power-button-support-on-aarch64)
-    - [Notable bug fixes](#notable-bug-fixes-19)
-    - [Contributors](#contributors-19)
+    - [Notable bug fixes](#notable-bug-fixes-20)
+    - [Contributors](#contributors-20)
 - [v17.0](#v170)
     - [ARM64 NUMA support using ACPI](#arm64-numa-support-using-acpi)
     - [`Seccomp` support for MSHV backend](#seccomp-support-for-mshv-backend)
     - [Hotplug of `macvtap` devices](#hotplug-of-macvtap-devices)
     - [Improved SGX support](#improved-sgx-support)
     - [Inflight tracking for `vhost-user` devices](#inflight-tracking-for-vhost-user-devices)
-    - [Notable bug fixes](#notable-bug-fixes-20)
-    - [Contributors](#contributors-20)
+    - [Notable bug fixes](#notable-bug-fixes-21)
+    - [Contributors](#contributors-21)
 - [v16.0](#v160)
     - [Improved live migration support](#improved-live-migration-support)
     - [Improved `vhost-user` support](#improved-vhost-user-support)
     - [ARM64 ACPI and UEFI support](#arm64-acpi-and-uefi-support)
-    - [Notable bug fixes](#notable-bug-fixes-21)
+    - [Notable bug fixes](#notable-bug-fixes-22)
     - [Removed functionality](#removed-functionality)
-    - [Contributors](#contributors-21)
+    - [Contributors](#contributors-22)
 - [v15.0](#v150)
     - [Version numbering and stability guarantees](#version-numbering-and-stability-guarantees)
     - [Network device rate limiting](#network-device-rate-limiting)
@@ -191,7 +199,7 @@
     - [`--api-socket` supports file descriptor parameter](#--api-socket-supports-file-descriptor-parameter)
     - [Bug fixes](#bug-fixes)
     - [Deprecations](#deprecations-5)
-    - [Contributors](#contributors-22)
+    - [Contributors](#contributors-23)
 - [v0.14.1](#v0141)
 - [v0.14.0](#v0140)
     - [Structured event monitoring](#structured-event-monitoring)
@@ -201,7 +209,7 @@
     - [PTY control for serial and `virtio-console`](#pty-control-for-serial-and-virtio-console)
     - [Block device rate limiting](#block-device-rate-limiting)
     - [Deprecations](#deprecations-6)
-    - [Contributors](#contributors-23)
+    - [Contributors](#contributors-24)
 - [v0.13.0](#v0130)
     - [Wider VFIO device support](#wider-vfio-device-support)
     - [Improved huge page support](#improved-huge-page-support)
@@ -209,13 +217,13 @@
     - [VHD disk image support](#vhd-disk-image-support)
     - [Improved Virtio device threading](#improved-virtio-device-threading)
     - [Clean shutdown support via synthetic power button](#clean-shutdown-support-via-synthetic-power-button)
-    - [Contributors](#contributors-24)
+    - [Contributors](#contributors-25)
 - [v0.12.0](#v0120)
     - [ARM64 enhancements](#arm64-enhancements)
     - [Removal of `vhost-user-net` and `vhost-user-block` self spawning](#removal-of-vhost-user-net-and-vhost-user-block-self-spawning)
     - [Migration of `vhost-user-fs` backend](#migration-of-vhost-user-fs-backend)
     - [Enhanced "info" API](#enhanced-info-api)
-    - [Contributors](#contributors-25)
+    - [Contributors](#contributors-26)
 - [v0.11.0](#v0110)
     - [`io_uring` support by default for `virtio-block`](#io_uring-support-by-default-for-virtio-block)
     - [Windows Guest Support](#windows-guest-support)
@@ -227,15 +235,15 @@
     - [Default Log Level Changed](#default-log-level-changed)
     - [New `--balloon` Parameter Added](#new---balloon-parameter-added)
     - [Experimental `virtio-watchdog` Support](#experimental-virtio-watchdog-support)
-    - [Notable Bug Fixes](#notable-bug-fixes-22)
-    - [Contributors](#contributors-26)
+    - [Notable Bug Fixes](#notable-bug-fixes-23)
+    - [Contributors](#contributors-27)
 - [v0.10.0](#v0100)
     - [`virtio-block` Support for Multiple Descriptors](#virtio-block-support-for-multiple-descriptors)
     - [Memory Zones](#memory-zones)
     - [`Seccomp` Sandbox Improvements](#seccomp-sandbox-improvements)
     - [Preliminary KVM HyperV Emulation Control](#preliminary-kvm-hyperv-emulation-control)
-    - [Notable Bug Fixes](#notable-bug-fixes-23)
-    - [Contributors](#contributors-27)
+    - [Notable Bug Fixes](#notable-bug-fixes-24)
+    - [Contributors](#contributors-28)
 - [v0.9.0](#v090)
     - [`io_uring` Based Block Device Support](#io_uring-based-block-device-support)
     - [Block and Network Device Statistics](#block-and-network-device-statistics)
@@ -248,17 +256,17 @@
     - [Enhancements to ARM64 Support](#enhancements-to-arm64-support)
     - [Intel SGX Support](#intel-sgx-support)
     - [`Seccomp` Sandbox Improvements](#seccomp-sandbox-improvements-1)
-    - [Notable Bug Fixes](#notable-bug-fixes-24)
-    - [Contributors](#contributors-28)
+    - [Notable Bug Fixes](#notable-bug-fixes-25)
+    - [Contributors](#contributors-29)
 - [v0.8.0](#v080)
     - [Experimental Snapshot and Restore Support](#experimental-snapshot-and-restore-support)
     - [Experimental ARM64 Support](#experimental-arm64-support)
     - [Support for Using 5-level Paging in Guests](#support-for-using-5-level-paging-in-guests)
     - [Virtio Device Interrupt Suppression for Network Devices](#virtio-device-interrupt-suppression-for-network-devices)
     - [`vhost_user_fs` Improvements](#vhost_user_fs-improvements)
-    - [Notable Bug Fixes](#notable-bug-fixes-25)
+    - [Notable Bug Fixes](#notable-bug-fixes-26)
     - [Command Line and API Changes](#command-line-and-api-changes)
-    - [Contributors](#contributors-29)
+    - [Contributors](#contributors-30)
 - [v0.7.0](#v070)
     - [Block, Network, Persistent Memory (PMEM), VirtioFS and Vsock hotplug](#block-network-persistent-memory-pmem-virtiofs-and-vsock-hotplug)
     - [Alternative `libc` Support](#alternative-libc-support)
@@ -268,14 +276,14 @@
     - [`Seccomp` Sandboxing](#seccomp-sandboxing)
     - [Updated Distribution Support](#updated-distribution-support)
     - [Command Line and API Changes](#command-line-and-api-changes-1)
-    - [Contributors](#contributors-30)
+    - [Contributors](#contributors-31)
 - [v0.6.0](#v060)
     - [Directly Assigned Devices Hotplug](#directly-assigned-devices-hotplug)
     - [Shared Filesystem Improvements](#shared-filesystem-improvements)
     - [Block and Networking IO Self Offloading](#block-and-networking-io-self-offloading)
     - [Command Line Interface](#command-line-interface)
     - [PVH Boot](#pvh-boot)
-    - [Contributors](#contributors-31)
+    - [Contributors](#contributors-32)
 - [v0.5.1](#v051)
 - [v0.5.0](#v050)
     - [Virtual Machine Dynamic Resizing](#virtual-machine-dynamic-resizing)
@@ -283,7 +291,7 @@
     - [New Interrupt Management Framework](#new-interrupt-management-framework)
     - [Development Tools](#development-tools)
     - [Kata Containers Integration](#kata-containers-integration)
-    - [Contributors](#contributors-32)
+    - [Contributors](#contributors-33)
 - [v0.4.0](#v040)
     - [Dynamic virtual CPUs addition](#dynamic-virtual-cpus-addition)
     - [Programmatic firmware tables generation](#programmatic-firmware-tables-generation)
@@ -292,7 +300,7 @@
     - [Userspace IOAPIC by default](#userspace-ioapic-by-default)
     - [PCI BAR reprogramming](#pci-bar-reprogramming)
     - [New `cloud-hypervisor` organization](#new-cloud-hypervisor-organization)
-    - [Contributors](#contributors-33)
+    - [Contributors](#contributors-34)
 - [v0.3.0](#v030)
     - [Block device offloading](#block-device-offloading)
     - [Network device backend](#network-device-backend)
@@ -319,18 +327,76 @@
     - [Unit testing](#unit-testing)
     - [Integration tests parallelization](#integration-tests-parallelization)
 
-# v37.1
+# v38.0
 
-This is a bug fix release. The following issues have been addressed:
+This release has been tracked in our [roadmap
+project](https://github.com/orgs/cloud-hypervisor/projects/6) as iteration
+v38.0. The following user visible changes have been made:
+
+### Group Rate Limiter on Block Devices
+
+Users now can throttle a group of block devices with the new
+`--rate-limiter-group` option. Details can be found from the [I/O
+Throttling documentation](docs/io_throttling.md)
+
+### CPU Pinning Support for Block Device Worker Thread
+
+Users now have the option to pin virt-queue threads for block devices
+to specific host cpus.
+
+### Optimized Boot Time with Parallel Memory Prefault
+
+The boot time with `prefault` option enabled is optimized via parallel
+memory prefault.
+
+### New 'debug-console' Device
+
+A 'debug-console' device is added to provide a user-configurable debug
+port for logging guest information. Details can be found from the [Debug
+IO Ports documentation](docs/debug-port.md).
+
+### Improved VFIO Device Support
+
+All non-emulated MMIO regions of VFIO devices are now mapped to the VFIO
+container, allowing PCIe P2P between all VFIO devices on the same
+VM. This is required for a wide variety of multi-GPU workloads involving
+GPUDirect P2P (DMA between two GPUs), GPUDirect RDMA (DMA between a GPU
+and an IB device).
+
+### Extended CPU Affinity Support
+
+Users now can set the vcpu affinity to a host CPU with index larger
+than 255.
+
+### Notable Bug Fixes
 
-* Fix several security advisories from dependencies (#6134, #6141)
 * Enable HTT flag to avoid crashing cpu topology enumeration software
 such as hwloc in the guest (#6146)
-* Enable nested virtualization on AMD if supported (#6106)
+* Fix several security advisories from dependencies (#6134, #6141)
 * Handle non-power-of-two CPU topology properly (#6062)
-* Various bug fixes around virtio-vsock(#6080, #6091, #6095)
+* Various bug fixes around `virtio-vsock`(#6080, #6091, #6095)
+* Enable nested virtualization on AMD if supported (#6106)
 * Align VFIO devices PCI BARs naturally (#6196)
 
+### Contributors
+
+Many thanks to everyone who has contributed to our release:
+* Alyssa Ross <hi@alyssa.is>
+* Bo Chen <chen.bo@intel.com>
+* Daniel Farina <daniel@ubicloud.com>
+* Jinank Jain <jinankjain@microsoft.com>
+* Muminul Islam <muislam@microsoft.com>
+* Peteris Rudzusiks <rye@stripe.com>
+* Philipp Schuster <philipp.schuster@cyberus-technology.de>
+* Ravi kumar Veeramally <ravikumar.veeramally@intel.com>
+* Rob Bradford <rbradford@rivosinc.com>
+* Ruslan Mstoi <ruslan.mstoi@intel.com>
+* Sean Banko <sbanko@crusoeenergy.com>
+* Thomas Barrett <tbarrett@crusoeenergy.com>
+* Wei Liu <liuwe@microsoft.com>
+* Yi Wang <foxywang@tencent.com>
+* acarp <acarp@crusoeenergy.com>
+
 # v37.0
 
 This release has been tracked in our [roadmap

scripts/check-image-compatibility.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 : '
     This script checks if an image is compatible with Cloud Hypervisor.
     At first, it detects the image type(raw or qcow2),
@@ -8,8 +8,6 @@
     a message about the compatibility of the image.
 '
 
-
-
 usage="$(basename "$0") [-h] -f -w -- program to check Cloud Hypervisor compatible image
 
 where:
@@ -18,12 +16,11 @@ where:
     -w  directory to be used for temporary files"
 
 function check_command {
-    if ! command -v  $1 &> /dev/null
-        then
-            echo "Command $1 could not be found"
+    if ! command -v "$1" &>/dev/null; then
+        echo "Command $1 could not be found"
         exit 1
     fi
-};
+}
 
 function check_if_root {
     if [ "$EUID" -ne 0 ]; then
@@ -31,27 +28,32 @@ function check_if_root {
         exit 1
     fi
 
-};
+}
 
 check_if_root
 working_dir=""
 while getopts ':hf:w:' option; do
     case "$option" in
-        h) echo "$usage"
-            exit
-            ;;
-        f) file_name=$OPTARG
-            ;;
-        w) working_dir=$OPTARG
-            ;;
-        :) printf "missing argument for -%s\n" "$OPTARG" >&2
-            echo "$usage" >&2
-            exit 1
-            ;;
-        \?) printf "illegal option: -%s\n" "$OPTARG" >&2
-            echo "$usage" >&2
-            exit 1
-            ;;
+    h)
+        echo "$usage"
+        exit
+        ;;
+    f)
+        file_name=$OPTARG
+        ;;
+    w)
+        working_dir=$OPTARG
+        ;;
+    :)
+        printf "missing argument for -%s\n" "$OPTARG" >&2
+        echo "$usage" >&2
+        exit 1
+        ;;
+    \?)
+        printf "illegal option: -%s\n" "$OPTARG" >&2
+        echo "$usage" >&2
+        exit 1
+        ;;
     esac
 done
 
@@ -59,67 +61,64 @@ shift $((OPTIND - 1))
 
 if [ -z "${file_name}" ]; then
     echo "You must provide the image file name"
-	exit 1
+    exit 1
 fi
 if [[ ! -f ${file_name} ]]; then
     echo "File ${file_name} does not exist"
     exit 1
 fi
 
-file_abs_path=`readlink -m ${file_name}`
+file_abs_path=$(readlink -m "${file_name}")
 if [[ "${working_dir}" != "" && ! -d "${working_dir}" ]]; then
     echo "Directory ${working_dir} does not exist"
     exit 1
 elif [[ "${working_dir}" == "" ]]; then
-    working_dir=`mktemp -d`
+    working_dir=$(mktemp -d)
     tmp_created=1
 else
-    working_dir=`readlink -m ${working_dir}`
+    working_dir=$(readlink -m "${working_dir}")
 fi
 
-#get file extension and image type
-extension="${file_name##*.}"
 filename="${file_name%.*}"
 dest_file=${working_dir}/${filename}.raw
-image_type=$(qemu-img info ${file_abs_path} | grep 'file format:' | awk '{ print $3 }')
+image_type=$(qemu-img info "${file_abs_path}" | grep 'file format:' | awk '{ print $3 }')
 echo "Image type detected as ${image_type}"
 
-
 if [[ "${image_type}" == "raw" ]]; then
-	dest_file=${file_abs_path}
+    dest_file=${file_abs_path}
 elif [[ "$image_type" == "qcow2" ]]; then
-    if lsmod | grep "nbd" &> /dev/null ; then
+    if lsmod | grep "nbd" &>/dev/null; then
         echo "Module nbd is loaded!"
     else
         echo "Module nbd is not loaded. Trying to load the module"
-        modprobe nbd max_part=8
-        if [ $? != 0 ]; then
+
+        if ! modprobe nbd max_part=8; then
             echo "failed to load nbd module. Exiting"
             exit 1
         fi
     fi
     check_command qemu-img
     dest_file=/dev/nbd0
-    qemu-nbd --connect=${dest_file} ${file_abs_path} --read-only
+    qemu-nbd --connect=${dest_file} "${file_abs_path}" --read-only
 fi
 
 check_command blkid
 #get part info
-part_type=$(blkid -o value -s PTTYPE ${dest_file})
+part_type=$(blkid -o value -s PTTYPE "${dest_file}")
 
 check_command partx
-nr_partitions=`partx -g ${dest_file}  | wc -l`
+nr_partitions=$(partx -g "${dest_file}" | wc -l)
 
 check_command fdisk
-out=`fdisk -l  ${dest_file} --bytes | grep -i -A ${nr_partitions} 'Device' | tail -n +2`
+out=$(fdisk -l "${dest_file}" --bytes | grep -i -A "${nr_partitions}" 'Device' | tail -n +2)
 
 IFS='
 '
 i=0
-declare -A liness
-for x in $out ; do
-	lines[$i]=$x
-	i=$((i+1))
+declare -A lines
+for x in $out; do
+    lines[$i]=$x
+    i=$((i + 1))
 done
 
 declare -A partitions
@@ -127,89 +126,86 @@ IFS=' '
 i=0
 ROWS=${#lines[@]}
 
-for line in "${lines[@]}";
-do
-	j=0
-	read -a str_arr <<< "$line"
-	for val in "${str_arr[@]}";
-	do
-		if [[ "$val" != "*" ]]; then
-			partitions[$i,$j]=$val
-			j=$((j+1))
-		fi
-	done
-	i=$((i+1))
+for line in "${lines[@]}"; do
+    j=0
+    read -a -r str_arr <<<"$line"
+    for val in "${str_arr[@]}"; do
+        if [[ "$val" != "*" ]]; then
+            partitions[$i, $j]=$val
+            j=$((j + 1))
+        fi
+    done
+    i=$((i + 1))
 done
 
 COLUMNS=$j
-COUNT=${#partitions[@]}
 START_ADDRESS_INDEX=1
-FILE_SYS_INDEX2=$((COLUMNS-1))
-FILE_SYS_INDEX1=$((COLUMNS-2))
+FILE_SYS_INDEX2=$((COLUMNS - 1))
+FILE_SYS_INDEX1=$((COLUMNS - 2))
 DEVICE_INDEX=0
 # Here we have all the partition info now lets mount and analyze the contents
-for ((i=0;i<ROWS;i++)) do
-	if [[ "$part_type" == "gpt" && "${partitions[$i,${FILE_SYS_INDEX1}]}" == "Linux" && "${partitions[$i,${FILE_SYS_INDEX2}]}" == "filesystem" ]]; then
+for ((i = 0; i < ROWS; i++)); do
+    if [[ "$part_type" == "gpt" && "${partitions[$i, ${FILE_SYS_INDEX1}]}" == "Linux" && "${partitions[$i, ${FILE_SYS_INDEX2}]}" == "filesystem" ]]; then
         echo "The image has GPT partitions"
         MOUNT_ROW=$i
-		break
-	elif [[ "$part_type" == "dos" && "${partitions[$i,${FILE_SYS_INDEX1}]}" == "Linux" && "${partitions[$i,${FILE_SYS_INDEX2}]}" == "" ]]; then
+        break
+    elif [[ "$part_type" == "dos" && "${partitions[$i, ${FILE_SYS_INDEX1}]}" == "Linux" && "${partitions[$i, ${FILE_SYS_INDEX2}]}" == "" ]]; then
         echo "The image has DOS partitions"
         MOUNT_ROW=$i
         break
-	fi
+    fi
 done
 
-start_address=${partitions[${MOUNT_ROW},${START_ADDRESS_INDEX}]}
-offset=$((start_address*512))
+start_address=${partitions[${MOUNT_ROW}, ${START_ADDRESS_INDEX}]}
+offset=$((start_address * 512))
 
 MOUNT_DIR=/mnt/clh-img-check/
 rm -rf ${MOUNT_DIR}
 mkdir ${MOUNT_DIR}
 if [[ "${image_type}" == "raw" ]]; then
-    mount -o ro,loop,offset=$offset ${dest_file} ${MOUNT_DIR}
+    mount -o ro,loop,offset=$offset "${dest_file}" ${MOUNT_DIR}
 elif [[ "${image_type}" == "qcow2" ]]; then
-    mount -o ro ${partitions[${MOUNT_ROW},${DEVICE_INDEX}]} ${MOUNT_DIR}
+    mount -o ro "${partitions[${MOUNT_ROW}, ${DEVICE_INDEX}]}" ${MOUNT_DIR}
 fi
 
 CONFIG_DIR=${MOUNT_DIR}boot/
 if [[ "$part_type" == "dos" ]]; then
-	CONFIG_DIR=${MOUNT_DIR}
+    CONFIG_DIR=${MOUNT_DIR}
 fi
 
 #check VIRTIO
 HAS_VIRTIO=1
-for conf_file in ${CONFIG_DIR}config*; do
-    out=`grep -E "CONFIG_VIRTIO=y|CONFIG_VIRTIO_BLK=y|CONFIG_VIRTIO_BLK=m" ${conf_file} | wc -l`
+for conf_file in "${CONFIG_DIR}"config*; do
+    out=$(grep -cE "CONFIG_VIRTIO=y|CONFIG_VIRTIO_BLK=y|CONFIG_VIRTIO_BLK=m" "${conf_file}")
     if [[ "$out" != "2" ]]; then
-	    echo "VIRTIO not found"
-		HAS_VIRTIO=0
-	fi
+        echo "VIRTIO not found"
+        HAS_VIRTIO=0
+    fi
 done
 
 #clean up
 umount ${MOUNT_DIR}
 
 if [[ "${tmp_created}" == "1" ]]; then
-    rm -rf ${working_dir}
+    rm -rf "${working_dir}"
 fi
 
-if [[ "${image_type}" == "qcow2" ]];then
-    qemu-nbd --disconnect ${dest_file} > /dev/null
+if [[ "${image_type}" == "qcow2" ]]; then
+    qemu-nbd --disconnect "${dest_file}" >/dev/null
 fi
 
 result=""
 if [[ "${part_type}" == "dos" ]]; then
-	result="dos mode not supported"
+    result="dos mode not supported"
 fi
 if [[ "${HAS_VIRTIO}" == "0" ]]; then
-	if [[ "$result" != "" ]]; then
-		result="${result},"
-	fi
-	result="$result VirtIO module not found in the image"
+    if [[ "$result" != "" ]]; then
+        result="${result},"
+    fi
+    result="$result VirtIO module not found in the image"
 fi
-if [[ "$result" == "" ]];then
-	echo "No incompatibilities found"
+if [[ "$result" == "" ]]; then
+    echo "No incompatibilities found"
 else
-	echo "$result"
+    echo "$result"
 fi

scripts/common-aarch64.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 WORKLOADS_DIR="$HOME/workloads"
 
@@ -22,26 +22,26 @@ build_edk2() {
 
     # Prepare source code
     checkout_repo "$EDK2_DIR" "$EDK2_REPO" master "46b4606ba23498d3d0e66b53e498eb3d5d592586"
-    pushd "$EDK2_DIR"
+    pushd "$EDK2_DIR" || exit
     git submodule update --init
-    popd
+    popd || exit
     checkout_repo "$EDK2_PLAT_DIR" "$EDK2_PLAT_REPO" master "8227e9e9f6a8aefbd772b40138f835121ccb2307"
     checkout_repo "$ACPICA_DIR" "$ACPICA_REPO" master "b9c69f81a05c45611c91ea9cbce8756078d76233"
 
-    if [[ ! -f "$EDK2_DIR/.built" || \
-          ! -f "$EDK2_PLAT_DIR/.built" || \
-          ! -f "$ACPICA_DIR/.built" ]]; then
-        pushd "$EDK2_BUILD_DIR"
+    if [[ ! -f "$EDK2_DIR/.built" ||
+        ! -f "$EDK2_PLAT_DIR/.built" ||
+        ! -f "$ACPICA_DIR/.built" ]]; then
+        pushd "$EDK2_BUILD_DIR" || exit
         # Build
-        make -C acpica -j `nproc`
+        make -C acpica -j "$(nproc)"
+        # shellcheck disable=SC1091
         source edk2/edksetup.sh
-        make -C edk2/BaseTools -j `nproc`
+        make -C edk2/BaseTools -j "$(nproc)"
         build -a AARCH64 -t GCC5 -p ArmVirtPkg/ArmVirtCloudHv.dsc -b RELEASE -n 0
         cp Build/ArmVirtCloudHv-AARCH64/RELEASE_GCC5/FV/CLOUDHV_EFI.fd "$WORKLOADS_DIR"
         touch "$EDK2_DIR"/.built
         touch "$EDK2_PLAT_DIR"/.built
         touch "$ACPICA_DIR"/.built
-        popd
+        popd || exit
     fi
 }
-

scripts/create-cloud-init.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 set -x
 
 rm -f /tmp/ubuntu-cloudinit.img
@@ -6,4 +6,3 @@ mkdosfs -n CIDATA -C /tmp/ubuntu-cloudinit.img 8192
 mcopy -oi /tmp/ubuntu-cloudinit.img -s test_data/cloud-init/ubuntu/local/user-data ::
 mcopy -oi /tmp/ubuntu-cloudinit.img -s test_data/cloud-init/ubuntu/local/meta-data ::
 mcopy -oi /tmp/ubuntu-cloudinit.img -s test_data/cloud-init/ubuntu/local/network-config ::
-

scripts/dev_cli.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 # Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
 # Copyright © 2020 Intel Corporation
@@ -121,9 +121,7 @@ ensure_latest_ctr() {
     if [ "$CTR_IMAGE_VERSION" = "local" ]; then
         build_container
     else
-        $DOCKER_RUNTIME pull "$CTR_IMAGE"
-
-        if [ $? -ne 0 ]; then
+        if ! $DOCKER_RUNTIME pull "$CTR_IMAGE"; then
             build_container
         fi
 
@@ -143,7 +141,8 @@ fix_dir_perms() {
         --workdir "$CTR_CLH_ROOT_DIR" \
         --rm \
         --volume /dev:/dev \
-        --volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR" $exported_volumes \
+        --volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR" \
+        ${exported_volumes:+"$exported_volumes"} \
         "$CTR_IMAGE" \
         chown -R "$(id -u):$(id -g)" "$CTR_CLH_ROOT_DIR"
 
@@ -158,9 +157,9 @@ process_volumes_args() {
         return
     fi
     exported_volumes=""
-    arr_vols=(${arg_vols//#/ })
+    arr_vols=("${arg_vols//#/ }")
     for var in "${arr_vols[@]}"; do
-        parts=(${var//:/ })
+        parts=("${var//:/ }")
         if [[ ! -e "${parts[0]}" ]]; then
             echo "The volume ${parts[0]} does not exist."
             exit 1
@@ -233,10 +232,10 @@ cmd_build() {
         "--debug") { build="debug"; } ;;
         "--release") { build="release"; } ;;
         "--runtime")
-	    shift
-	    DOCKER_RUNTIME="$1"
-	    export DOCKER_RUNTIME
-	    ;;
+            shift
+            DOCKER_RUNTIME="$1"
+            export DOCKER_RUNTIME
+            ;;
         "--libc")
             shift
             [[ "$1" =~ ^(musl|gnu)$ ]] ||
@@ -282,6 +281,7 @@ cmd_build() {
     [ $build = "release" ] && cargo_args+=("--release")
     cargo_args+=(--target "$target")
 
+    # shellcheck disable=SC2153
     rustflags="$RUSTFLAGS"
     target_cc=""
     if [ "$(uname -m)" = "aarch64" ] && [ "$libc" = "musl" ]; then
@@ -293,11 +293,12 @@ cmd_build() {
         --workdir "$CTR_CLH_ROOT_DIR" \
         --rm \
         --volume $exported_device \
-        --volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR" $exported_volumes \
+        --volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR" \
+        ${exported_volumes:+"$exported_volumes"} \
         --env RUSTFLAGS="$rustflags" \
         --env TARGET_CC="$target_cc" \
         "$CTR_IMAGE" \
-        cargo build --all $features_build \
+        cargo build --all "$features_build" \
         --target-dir "$CTR_CLH_CARGO_TARGET" \
         "${cargo_args[@]}" && say "Binaries placed under $CLH_CARGO_TARGET/$target/$build"
 }
@@ -312,7 +313,8 @@ cmd_clean() {
         --user "$(id -u):$(id -g)" \
         --workdir "$CTR_CLH_ROOT_DIR" \
         --rm \
-        --volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR" $exported_volumes \
+        --volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR" \
+        ${exported_volumes:+"$exported_volumes"} \
         "$CTR_IMAGE" \
         cargo clean \
         --target-dir "$CTR_CLH_CARGO_TARGET" \
@@ -361,7 +363,6 @@ cmd_tests() {
             hypervisor="$1"
             ;;
         "--all") {
-            cargo=true
             unit=true
             integration=true
         } ;;
@@ -383,7 +384,7 @@ cmd_tests() {
         exported_device="/dev/mshv"
     fi
 
-    if [ ! -e "${exported_device}" ] ; then
+    if [ ! -e "${exported_device}" ]; then
         die "${exported_device} does not exist on the system"
     fi
 
@@ -409,7 +410,8 @@ cmd_tests() {
             --device $exported_device \
             --device /dev/net/tun \
             --cap-add net_admin \
-            --volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR" $exported_volumes \
+            --volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR" \
+            ${exported_volumes:+"$exported_volumes"} \
             --env BUILD_TARGET="$target" \
             --env RUSTFLAGS="$rustflags" \
             --env TARGET_CC="$target_cc" \
@@ -428,7 +430,8 @@ cmd_tests() {
             --net="$CTR_CLH_NET" \
             --mount type=tmpfs,destination=/tmp \
             --volume /dev:/dev \
-            --volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR" $exported_volumes \
+            --volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR" \
+            ${exported_volumes:+"$exported_volumes"} \
             --volume "$CLH_INTEGRATION_WORKLOADS:$CTR_CLH_INTEGRATION_WORKLOADS" \
             --env USER="root" \
             --env BUILD_TARGET="$target" \
@@ -450,7 +453,8 @@ cmd_tests() {
             --net="$CTR_CLH_NET" \
             --mount type=tmpfs,destination=/tmp \
             --volume /dev:/dev \
-            --volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR" $exported_volumes \
+            --volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR" \
+            ${exported_volumes:+"$exported_volumes"} \
             --volume "$CLH_INTEGRATION_WORKLOADS:$CTR_CLH_INTEGRATION_WORKLOADS" \
             --env USER="root" \
             --env BUILD_TARGET="$target" \
@@ -472,7 +476,8 @@ cmd_tests() {
             --net="$CTR_CLH_NET" \
             --mount type=tmpfs,destination=/tmp \
             --volume /dev:/dev \
-            --volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR" $exported_volumes \
+            --volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR" \
+            ${exported_volumes:+"$exported_volumes"} \
             --volume "$CLH_INTEGRATION_WORKLOADS:$CTR_CLH_INTEGRATION_WORKLOADS" \
             --env USER="root" \
             --env BUILD_TARGET="$target" \
@@ -494,7 +499,8 @@ cmd_tests() {
             --net="$CTR_CLH_NET" \
             --mount type=tmpfs,destination=/tmp \
             --volume /dev:/dev \
-            --volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR" $exported_volumes \
+            --volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR" \
+            ${exported_volumes:+"$exported_volumes"} \
             --volume "$CLH_INTEGRATION_WORKLOADS:$CTR_CLH_INTEGRATION_WORKLOADS" \
             --env USER="root" \
             --env BUILD_TARGET="$target" \
@@ -516,7 +522,8 @@ cmd_tests() {
             --net="$CTR_CLH_NET" \
             --mount type=tmpfs,destination=/tmp \
             --volume /dev:/dev \
-            --volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR" $exported_volumes \
+            --volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR" \
+            ${exported_volumes:+"$exported_volumes"} \
             --volume "$CLH_INTEGRATION_WORKLOADS:$CTR_CLH_INTEGRATION_WORKLOADS" \
             --env USER="root" \
             --env BUILD_TARGET="$target" \
@@ -538,7 +545,8 @@ cmd_tests() {
             --net="$CTR_CLH_NET" \
             --mount type=tmpfs,destination=/tmp \
             --volume /dev:/dev \
-            --volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR" $exported_volumes \
+            --volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR" \
+            ${exported_volumes:+"$exported_volumes"} \
             --volume "$CLH_INTEGRATION_WORKLOADS:$CTR_CLH_INTEGRATION_WORKLOADS" \
             --env USER="root" \
             --env BUILD_TARGET="$target" \
@@ -560,7 +568,8 @@ cmd_tests() {
             --net="$CTR_CLH_NET" \
             --mount type=tmpfs,destination=/tmp \
             --volume /dev:/dev \
-            --volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR" $exported_volumes \
+            --volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR" \
+            ${exported_volumes:+"$exported_volumes"} \
             --volume "$CLH_INTEGRATION_WORKLOADS:$CTR_CLH_INTEGRATION_WORKLOADS" \
             --env USER="root" \
             --env BUILD_TARGET="$target" \
@@ -588,9 +597,9 @@ build_container() {
 
     $DOCKER_RUNTIME build \
         --target dev \
-        -t $CTR_IMAGE \
+        -t "$CTR_IMAGE" \
         -f $BUILD_DIR/Dockerfile \
-        --build-arg TARGETARCH=$TARGETARCH \
+        --build-arg TARGETARCH="$TARGETARCH" \
         $BUILD_DIR
 }
 
@@ -650,7 +659,8 @@ cmd_shell() {
         --net="$CTR_CLH_NET" \
         --tmpfs /tmp:exec \
         --volume /dev:/dev \
-        --volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR" $exported_volumes \
+        --volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR" \
+        ${exported_volumes:+"$exported_volumes"} \
         --volume "$CLH_INTEGRATION_WORKLOADS:$CTR_CLH_INTEGRATION_WORKLOADS" \
         --env USER="root" \
         --entrypoint bash \

scripts/gitlint/rules.py

@@ -42,6 +42,7 @@ class TitleStartsWithComponent(LineRule):
             'gitignore',
             'gitlint',
             'hypervisor',
+            'Jenkinsfile',
             'main',
             'misc',
             'net_gen',

scripts/prepare_vdpa.sh

@@ -1,20 +1,20 @@
-#!/bin/bash
+#!/usr/bin/env bash
 set -x
 
 sudo apt install -y libncurses-dev gawk flex bison openssl libssl-dev dkms libelf-dev libudev-dev libpci-dev libiberty-dev autoconf git make dpkg-dev libmnl-dev pkg-config iproute2
 sudo sed -i -- 's/# deb-src/deb-src/g' /etc/apt/sources.list
 sudo apt update
-apt-get source linux-image-unsigned-`uname -r`
-pushd linux-azure*/drivers/vdpa/vdpa_sim/
-cat <<'EOF' > Makefile
+apt-get source linux-image-unsigned-"$(uname -r)"
+pushd linux-azure*/drivers/vdpa/vdpa_sim/ || exit
+cat <<'EOF' >Makefile
 # SPDX-License-Identifier: GPL-2.0
 obj-m += vdpa_sim.o
 obj-m += vdpa_sim_net.o
 obj-m += vdpa_sim_blk.o
 EOF
-make -C /lib/modules/`uname -r`/build M=$PWD
-sudo make -C /lib/modules/`uname -r`/build M=$PWD modules_install
-popd
+make -C /lib/modules/"$(uname -r)"/build M="$PWD"
+sudo make -C /lib/modules/"$(uname -r)"/build M="$PWD" modules_install
+popd || exit
 sudo depmod -a
 sudo modprobe vdpa
 sudo modprobe vhost_vdpa

scripts/run_integration_tests_aarch64.sh

@@ -1,9 +1,11 @@
-#!/bin/bash
+#!/usr/bin/env bash
+# shellcheck disable=SC2048,SC2086
 set -x
 
-source $HOME/.cargo/env
-source $(dirname "$0")/test-util.sh
-source $(dirname "$0")/common-aarch64.sh
+# shellcheck source=/dev/null
+source "$HOME"/.cargo/env
+source "$(dirname "$0")"/test-util.sh
+source "$(dirname "$0")"/common-aarch64.sh
 
 WORKLOADS_LOCK="$WORKLOADS_DIR/integration_test.lock"
 
@@ -14,16 +16,16 @@ build_spdk_nvme() {
     checkout_repo "$SPDK_DIR" "$SPDK_REPO" master "ef8bcce58f3f02b79c0619a297e4f17e81e62b24"
 
     if [ ! -f "$SPDK_DIR/.built" ]; then
-        pushd $SPDK_DIR
+        pushd "$SPDK_DIR" || exit
         git submodule update --init
         apt-get update
         sed -i "/grpcio/d" scripts/pkgdep/debian.sh
         ./scripts/pkgdep.sh
         ./configure --with-vfio-user
         chmod +x /usr/local/lib/python3.10/dist-packages/ninja/data/bin/ninja
-        make -j `nproc` || exit 1
+        make -j "$(nproc)" || exit 1
         touch .built
-        popd
+        popd || exit
     fi
     if [ ! -d "/usr/local/bin/spdk-nvme" ]; then
         mkdir -p $SPDK_DEPLOY_DIR
@@ -41,109 +43,135 @@ build_virtiofsd() {
     checkout_repo "$VIRTIOFSD_DIR" "$VIRTIOFSD_REPO" v1.8.0 "97ea7908fe7f9bc59916671a771bdcfaf4044b45"
 
     if [ ! -f "$VIRTIOFSD_DIR/.built" ]; then
-        pushd $VIRTIOFSD_DIR
+        pushd "$VIRTIOFSD_DIR" || exit
         rm -rf target/
         time RUSTFLAGS="" TARGET_CC="" cargo build --release
         cp target/release/virtiofsd "$WORKLOADS_DIR/" || exit 1
         touch .built
-        popd
+        popd || exit
     fi
 }
 
 update_workloads() {
-    cp scripts/sha1sums-aarch64 $WORKLOADS_DIR
+    cp scripts/sha1sums-aarch64 "$WORKLOADS_DIR"
+
+    BIONIC_OS_IMAGE_DOWNLOAD_NAME="bionic-server-cloudimg-arm64.img"
+    BIONIC_OS_IMAGE_DOWNLOAD_URL="https://cloud-hypervisor.azureedge.net/$BIONIC_OS_IMAGE_DOWNLOAD_NAME"
+    BIONIC_OS_DOWNLOAD_IMAGE="$WORKLOADS_DIR/$BIONIC_OS_IMAGE_DOWNLOAD_NAME"
+    if [ ! -f "$BIONIC_OS_DOWNLOAD_IMAGE" ]; then
+        pushd "$WORKLOADS_DIR" || exit
+        time wget --quiet $BIONIC_OS_IMAGE_DOWNLOAD_URL || exit 1
+        popd || exit
+    fi
+
+    BIONIC_OS_RAW_IMAGE_NAME="bionic-server-cloudimg-arm64.raw"
+    BIONIC_OS_RAW_IMAGE="$WORKLOADS_DIR/$BIONIC_OS_RAW_IMAGE_NAME"
+    if [ ! -f "$BIONIC_OS_RAW_IMAGE" ]; then
+        pushd "$WORKLOADS_DIR" || exit
+        time qemu-img convert -p -f qcow2 -O raw $BIONIC_OS_IMAGE_DOWNLOAD_NAME $BIONIC_OS_RAW_IMAGE_NAME || exit 1
+        popd || exit
+    fi
+
+    # Convert the raw image to qcow2 image to remove compressed blocks from the disk. Therefore letting the
+    # qcow2 format image can be directly used in the integration test.
+    BIONIC_OS_QCOW2_IMAGE_UNCOMPRESSED_NAME="bionic-server-cloudimg-arm64.qcow2"
+    BIONIC_OS_QCOW2_UNCOMPRESSED_IMAGE="$WORKLOADS_DIR/$BIONIC_OS_QCOW2_IMAGE_UNCOMPRESSED_NAME"
+    if [ ! -f "$BIONIC_OS_QCOW2_UNCOMPRESSED_IMAGE" ]; then
+        pushd "$WORKLOADS_DIR" || exit
+        time qemu-img convert -p -f raw -O qcow2 $BIONIC_OS_RAW_IMAGE_NAME "$BIONIC_OS_QCOW2_UNCOMPRESSED_IMAGE" || exit 1
+        popd || exit
+    fi
 
     FOCAL_OS_RAW_IMAGE_NAME="focal-server-cloudimg-arm64-custom-20210929-0.raw"
-    FOCAL_OS_RAW_IMAGE_DOWNLOAD_URL="https://ch-images.azureedge.net/$FOCAL_OS_RAW_IMAGE_NAME"
+    FOCAL_OS_RAW_IMAGE_DOWNLOAD_URL="https://cloud-hypervisor.azureedge.net/$FOCAL_OS_RAW_IMAGE_NAME"
     FOCAL_OS_RAW_IMAGE="$WORKLOADS_DIR/$FOCAL_OS_RAW_IMAGE_NAME"
     if [ ! -f "$FOCAL_OS_RAW_IMAGE" ]; then
-        pushd $WORKLOADS_DIR
+        pushd "$WORKLOADS_DIR" || exit
         time wget --quiet $FOCAL_OS_RAW_IMAGE_DOWNLOAD_URL || exit 1
-        popd
+        popd || exit
     fi
 
     FOCAL_OS_QCOW2_IMAGE_UNCOMPRESSED_NAME="focal-server-cloudimg-arm64-custom-20210929-0.qcow2"
-    FOCAL_OS_QCOW2_IMAGE_UNCOMPRESSED_DOWNLOAD_URL="https://ch-images.azureedge.net/$FOCAL_OS_QCOW2_IMAGE_UNCOMPRESSED_NAME"
+    FOCAL_OS_QCOW2_IMAGE_UNCOMPRESSED_DOWNLOAD_URL="https://cloud-hypervisor.azureedge.net/$FOCAL_OS_QCOW2_IMAGE_UNCOMPRESSED_NAME"
     FOCAL_OS_QCOW2_UNCOMPRESSED_IMAGE="$WORKLOADS_DIR/$FOCAL_OS_QCOW2_IMAGE_UNCOMPRESSED_NAME"
     if [ ! -f "$FOCAL_OS_QCOW2_UNCOMPRESSED_IMAGE" ]; then
-        pushd $WORKLOADS_DIR
+        pushd "$WORKLOADS_DIR" || exit
         time wget --quiet $FOCAL_OS_QCOW2_IMAGE_UNCOMPRESSED_DOWNLOAD_URL || exit 1
-        popd
+        popd || exit
     fi
 
     FOCAL_OS_QCOW2_IMAGE_BACKING_FILE_NAME="focal-server-cloudimg-arm64-custom-20210929-0-backing.qcow2"
     FOCAL_OS_QCOW2_BACKING_FILE_IMAGE="$WORKLOADS_DIR/$FOCAL_OS_QCOW2_IMAGE_BACKING_FILE_NAME"
     if [ ! -f "$FOCAL_OS_QCOW2_BACKING_FILE_IMAGE" ]; then
-        pushd $WORKLOADS_DIR
-        time qemu-img create -f qcow2 -b $FOCAL_OS_QCOW2_UNCOMPRESSED_IMAGE -F qcow2 $FOCAL_OS_QCOW2_IMAGE_BACKING_FILE_NAME
-        popd
+        pushd "$WORKLOADS_DIR" || exit
+        time qemu-img create -f qcow2 -b "$FOCAL_OS_QCOW2_UNCOMPRESSED_IMAGE" -F qcow2 $FOCAL_OS_QCOW2_IMAGE_BACKING_FILE_NAME
+        popd || exit
     fi
 
     JAMMY_OS_RAW_IMAGE_NAME="jammy-server-cloudimg-arm64-custom-20220329-0.raw"
-    JAMMY_OS_RAW_IMAGE_DOWNLOAD_URL="https://ch-images.azureedge.net/$JAMMY_OS_RAW_IMAGE_NAME"
+    JAMMY_OS_RAW_IMAGE_DOWNLOAD_URL="https://cloud-hypervisor.azureedge.net/$JAMMY_OS_RAW_IMAGE_NAME"
     JAMMY_OS_RAW_IMAGE="$WORKLOADS_DIR/$JAMMY_OS_RAW_IMAGE_NAME"
     if [ ! -f "$JAMMY_OS_RAW_IMAGE" ]; then
-        pushd $WORKLOADS_DIR
+        pushd "$WORKLOADS_DIR" || exit
         time wget --quiet $JAMMY_OS_RAW_IMAGE_DOWNLOAD_URL || exit 1
-        popd
+        popd || exit
     fi
 
     JAMMY_OS_QCOW2_IMAGE_UNCOMPRESSED_NAME="jammy-server-cloudimg-arm64-custom-20220329-0.qcow2"
-    JAMMY_OS_QCOW2_IMAGE_UNCOMPRESSED_DOWNLOAD_URL="https://ch-images.azureedge.net/$JAMMY_OS_QCOW2_IMAGE_UNCOMPRESSED_NAME"
+    JAMMY_OS_QCOW2_IMAGE_UNCOMPRESSED_DOWNLOAD_URL="https://cloud-hypervisor.azureedge.net/$JAMMY_OS_QCOW2_IMAGE_UNCOMPRESSED_NAME"
     JAMMY_OS_QCOW2_UNCOMPRESSED_IMAGE="$WORKLOADS_DIR/$JAMMY_OS_QCOW2_IMAGE_UNCOMPRESSED_NAME"
     if [ ! -f "$JAMMY_OS_QCOW2_UNCOMPRESSED_IMAGE" ]; then
-        pushd $WORKLOADS_DIR
+        pushd "$WORKLOADS_DIR" || exit
         time wget --quiet $JAMMY_OS_QCOW2_IMAGE_UNCOMPRESSED_DOWNLOAD_URL || exit 1
-        popd
+        popd || exit
     fi
 
     ALPINE_MINIROOTFS_URL="http://dl-cdn.alpinelinux.org/alpine/v3.11/releases/aarch64/alpine-minirootfs-3.11.3-aarch64.tar.gz"
     ALPINE_MINIROOTFS_TARBALL="$WORKLOADS_DIR/alpine-minirootfs-aarch64.tar.gz"
     if [ ! -f "$ALPINE_MINIROOTFS_TARBALL" ]; then
-        pushd $WORKLOADS_DIR
-        time wget --quiet $ALPINE_MINIROOTFS_URL -O $ALPINE_MINIROOTFS_TARBALL || exit 1
-        popd
+        pushd "$WORKLOADS_DIR" || exit
+        time wget --quiet $ALPINE_MINIROOTFS_URL -O "$ALPINE_MINIROOTFS_TARBALL" || exit 1
+        popd || exit
     fi
 
     ALPINE_INITRAMFS_IMAGE="$WORKLOADS_DIR/alpine_initramfs.img"
     if [ ! -f "$ALPINE_INITRAMFS_IMAGE" ]; then
-        pushd $WORKLOADS_DIR
+        pushd "$WORKLOADS_DIR" || exit
         mkdir alpine-minirootfs
         tar xf "$ALPINE_MINIROOTFS_TARBALL" -C alpine-minirootfs
-        cat > alpine-minirootfs/init <<-EOF
+        cat >alpine-minirootfs/init <<-EOF
 			#! /bin/sh
 			mount -t devtmpfs dev /dev
 			echo \$TEST_STRING > /dev/console
 			poweroff -f
 		EOF
         chmod +x alpine-minirootfs/init
-        cd alpine-minirootfs
+        cd alpine-minirootfs || exit
         find . -print0 |
-            cpio --null --create --verbose --owner root:root --format=newc > "$ALPINE_INITRAMFS_IMAGE"
-        popd
+            cpio --null --create --verbose --owner root:root --format=newc >"$ALPINE_INITRAMFS_IMAGE"
+        popd || exit
     fi
 
-    pushd $WORKLOADS_DIR
-    sha1sum sha1sums-aarch64 --check
-    if [ $? -ne 0 ]; then
+    pushd "$WORKLOADS_DIR" || exit
+
+    if ! sha1sum sha1sums-aarch64 --check; then
         echo "sha1sum validation of images failed, remove invalid images to fix the issue."
         exit 1
     fi
-    popd
+    popd || exit
 
     # Download Cloud Hypervisor binary from its last stable release
     LAST_RELEASE_VERSION="v36.0"
     CH_RELEASE_URL="https://github.com/cloud-hypervisor/cloud-hypervisor/releases/download/$LAST_RELEASE_VERSION/cloud-hypervisor-static-aarch64"
     CH_RELEASE_NAME="cloud-hypervisor-static-aarch64"
-    pushd $WORKLOADS_DIR
+    pushd "$WORKLOADS_DIR" || exit
     # Repeat a few times to workaround a random wget failure
     WGET_RETRY_MAX=10
     wget_retry=0
 
-    until [ "$wget_retry" -ge "$WGET_RETRY_MAX" ]
-    do
+    until [ "$wget_retry" -ge "$WGET_RETRY_MAX" ]; do
         time wget $CH_RELEASE_URL -O "$CH_RELEASE_NAME" && break
-        wget_retry=$((wget_retry+1))
+        wget_retry=$((wget_retry + 1))
     done
 
     if [ $wget_retry -ge "$WGET_RETRY_MAX" ]; then
@@ -151,7 +179,7 @@ update_workloads() {
     else
         chmod +x $CH_RELEASE_NAME
     fi
-    popd
+    popd || exit
 
     # Build custom kernel for guest VMs
     build_custom_linux
@@ -172,22 +200,22 @@ update_workloads() {
     BLK_IMAGE="$WORKLOADS_DIR/blk.img"
     MNT_DIR="mount_image"
     if [ ! -f "$BLK_IMAGE" ]; then
-        pushd $WORKLOADS_DIR
-        fallocate -l 16M $BLK_IMAGE
-        mkfs.ext4 -j $BLK_IMAGE
+        pushd "$WORKLOADS_DIR" || exit
+        fallocate -l 16M "$BLK_IMAGE"
+        mkfs.ext4 -j "$BLK_IMAGE"
         mkdir $MNT_DIR
-        sudo mount -t ext4 $BLK_IMAGE $MNT_DIR
+        sudo mount -t ext4 "$BLK_IMAGE" $MNT_DIR
         sudo bash -c "echo bar > $MNT_DIR/foo" || exit 1
-        sudo umount $BLK_IMAGE
+        sudo umount "$BLK_IMAGE"
         rm -r $MNT_DIR
-        popd
+        popd || exit
     fi
 
     SHARED_DIR="$WORKLOADS_DIR/shared_dir"
     if [ ! -d "$SHARED_DIR" ]; then
-        mkdir -p $SHARED_DIR
-        echo "foo" > "$SHARED_DIR/file1"
-        echo "bar" > "$SHARED_DIR/file3" || exit 1
+        mkdir -p "$SHARED_DIR"
+        echo "foo" >"$SHARED_DIR/file1"
+        echo "bar" >"$SHARED_DIR/file3" || exit 1
     fi
 
     # Checkout and build SPDK NVMe
@@ -205,12 +233,11 @@ if [[ "$hypervisor" = "mshv" ]]; then
     exit 1
 fi
 
-
 # lock the workloads folder to avoid parallel updating by different containers
 (
     echo "try to lock $WORKLOADS_DIR folder and update"
     flock -x 12 && update_workloads
-) 12>$WORKLOADS_LOCK
+) 12>"$WORKLOADS_LOCK"
 
 # Check if there is any error in the execution of `update_workloads`.
 # If there is any error, then kill the shell. Otherwise the script will continue
@@ -222,7 +249,7 @@ fi
 
 export RUST_BACKTRACE=1
 
-cargo build --all --release --target $BUILD_TARGET
+cargo build --all --release --target "$BUILD_TARGET"
 
 # Enable KSM with some reasonable parameters so that it won't take too long
 # for the memory to be merged between two processes.
@@ -231,19 +258,19 @@ sudo bash -c "echo 10 > /sys/kernel/mm/ksm/sleep_millisecs"
 sudo bash -c "echo 1 > /sys/kernel/mm/ksm/run"
 
 # Both test_vfio and ovs-dpdk rely on hugepages
-HUGEPAGESIZE=`grep Hugepagesize /proc/meminfo | awk '{print $2}'`
-PAGE_NUM=`echo $((12288 * 1024 / $HUGEPAGESIZE))`
-echo $PAGE_NUM | sudo tee /proc/sys/vm/nr_hugepages
+HUGEPAGESIZE=$(grep Hugepagesize /proc/meminfo | awk '{print $2}')
+PAGE_NUM=$((12288 * 1024 / HUGEPAGESIZE))
+echo "$PAGE_NUM" | sudo tee /proc/sys/vm/nr_hugepages
 sudo chmod a+rwX /dev/hugepages
 
 # Run all direct kernel boot (Device Tree) test cases in mod `parallel`
-time cargo test "common_parallel::$test_filter" --target $BUILD_TARGET -- ${test_binary_args[*]}
+time cargo test "common_parallel::$test_filter" --target "$BUILD_TARGET" -- ${test_binary_args[*]}
 RES=$?
 
 # Run some tests in sequence since the result could be affected by other tests
 # running in parallel.
 if [ $RES -eq 0 ]; then
-    time cargo test "common_sequential::$test_filter" --target $BUILD_TARGET -- --test-threads=1 ${test_binary_args[*]}
+    time cargo test "common_sequential::$test_filter" --target "$BUILD_TARGET" -- --test-threads=1 ${test_binary_args[*]}
     RES=$?
 else
     exit $RES
@@ -251,7 +278,7 @@ fi
 
 # Run all ACPI test cases
 if [ $RES -eq 0 ]; then
-    time cargo test "aarch64_acpi::$test_filter" --target $BUILD_TARGET -- ${test_binary_args[*]}
+    time cargo test "aarch64_acpi::$test_filter" --target "$BUILD_TARGET" -- ${test_binary_args[*]}
     RES=$?
 else
     exit $RES
@@ -259,14 +286,14 @@ fi
 
 # Run all test cases related to live migration
 if [ $RES -eq 0 ]; then
-    time cargo test "live_migration_parallel::$test_filter" --target $BUILD_TARGET -- ${test_binary_args[*]}
+    time cargo test "live_migration_parallel::$test_filter" --target "$BUILD_TARGET" -- ${test_binary_args[*]}
     RES=$?
 else
     exit $RES
 fi
 
 if [ $RES -eq 0 ]; then
-    time cargo test "live_migration_sequential::$test_filter" --target $BUILD_TARGET -- --test-threads=1 ${test_binary_args[*]}
+    time cargo test "live_migration_sequential::$test_filter" --target "$BUILD_TARGET" -- --test-threads=1 ${test_binary_args[*]}
     RES=$?
 else
     exit $RES
@@ -274,9 +301,9 @@ fi
 
 # Run tests on dbus_api
 if [ $RES -eq 0 ]; then
-    cargo build --features "dbus_api" --all --release --target $BUILD_TARGET
+    cargo build --features "dbus_api" --all --release --target "$BUILD_TARGET"
     export RUST_BACKTRACE=1
-    time cargo test "dbus_api::$test_filter" --target $BUILD_TARGET -- ${test_binary_args[*]}
+    time cargo test "dbus_api::$test_filter" --target "$BUILD_TARGET" -- ${test_binary_args[*]}
     RES=$?
 fi
 

scripts/run_integration_tests_live_migration.sh

@@ -1,8 +1,10 @@
-#!/bin/bash
+#!/usr/bin/env bash
+# shellcheck disable=SC2048,SC2086
 set -x
 
-source $HOME/.cargo/env
-source $(dirname "$0")/test-util.sh
+# shellcheck source=/dev/null
+source "$HOME"/.cargo/env
+source "$(dirname "$0")"/test-util.sh
 
 WORKLOADS_DIR="$HOME/workloads"
 mkdir -p "$WORKLOADS_DIR"
@@ -12,45 +14,44 @@ process_common_args "$@"
 # For now these values are default for kvm
 test_features=""
 
-if [ "$hypervisor" = "mshv" ] ;  then
+if [ "$hypervisor" = "mshv" ]; then
     test_features="--features mshv"
 fi
 
-cp scripts/sha1sums-x86_64 $WORKLOADS_DIR
+cp scripts/sha1sums-x86_64 "$WORKLOADS_DIR"
 
 FOCAL_OS_IMAGE_NAME="focal-server-cloudimg-amd64-custom-20210609-0.qcow2"
-FOCAL_OS_IMAGE_URL="https://ch-images.azureedge.net/$FOCAL_OS_IMAGE_NAME"
+FOCAL_OS_IMAGE_URL="https://cloud-hypervisor.azureedge.net/$FOCAL_OS_IMAGE_NAME"
 FOCAL_OS_IMAGE="$WORKLOADS_DIR/$FOCAL_OS_IMAGE_NAME"
 if [ ! -f "$FOCAL_OS_IMAGE" ]; then
-    pushd $WORKLOADS_DIR
+    pushd "$WORKLOADS_DIR" || exit
     time wget --quiet $FOCAL_OS_IMAGE_URL || exit 1
-    popd
+    popd || exit
 fi
 
 FOCAL_OS_RAW_IMAGE_NAME="focal-server-cloudimg-amd64-custom-20210609-0.raw"
 FOCAL_OS_RAW_IMAGE="$WORKLOADS_DIR/$FOCAL_OS_RAW_IMAGE_NAME"
 if [ ! -f "$FOCAL_OS_RAW_IMAGE" ]; then
-    pushd $WORKLOADS_DIR
+    pushd "$WORKLOADS_DIR" || exit
     time qemu-img convert -p -f qcow2 -O raw $FOCAL_OS_IMAGE_NAME $FOCAL_OS_RAW_IMAGE_NAME || exit 1
-    popd
+    popd || exit
 fi
 
-pushd $WORKLOADS_DIR
-grep focal sha1sums-x86_64 | sha1sum --check
-if [ $? -ne 0 ]; then
+pushd "$WORKLOADS_DIR" || exit
+if ! grep focal sha1sums-x86_64 | sha1sum --check; then
     echo "sha1sum validation of images failed, remove invalid images to fix the issue."
     exit 1
 fi
-popd
+popd || exit
 
 # Download Cloud Hypervisor binary from its last stable release
 LAST_RELEASE_VERSION="v36.0"
 CH_RELEASE_URL="https://github.com/cloud-hypervisor/cloud-hypervisor/releases/download/$LAST_RELEASE_VERSION/cloud-hypervisor-static"
 CH_RELEASE_NAME="cloud-hypervisor-static"
-pushd $WORKLOADS_DIR
+pushd "$WORKLOADS_DIR" || exit
 time wget --quiet $CH_RELEASE_URL -O "$CH_RELEASE_NAME" || exit 1
 chmod +x $CH_RELEASE_NAME
-popd
+popd || exit
 
 # Build custom kernel based on virtio-pmem and virtio-fs upstream patches
 VMLINUX_IMAGE="$WORKLOADS_DIR/vmlinux"
@@ -60,15 +61,16 @@ fi
 
 CFLAGS=""
 if [[ "${BUILD_TARGET}" == "x86_64-unknown-linux-musl" ]]; then
+    # shellcheck disable=SC2034
     CFLAGS="-I /usr/include/x86_64-linux-musl/ -idirafter /usr/include/"
 fi
 
-cargo build --features mshv --all --release --target $BUILD_TARGET
+cargo build --features mshv --all --release --target "$BUILD_TARGET"
 
 # Test ovs-dpdk relies on hugepages
-HUGEPAGESIZE=`grep Hugepagesize /proc/meminfo | awk '{print $2}'`
-PAGE_NUM=`echo $((12288 * 1024 / $HUGEPAGESIZE))`
-echo $PAGE_NUM | sudo tee /proc/sys/vm/nr_hugepages
+HUGEPAGESIZE=$(grep Hugepagesize /proc/meminfo | awk '{print $2}')
+PAGE_NUM=$((12288 * 1024 / HUGEPAGESIZE))
+echo "$PAGE_NUM" | sudo tee /proc/sys/vm/nr_hugepages
 sudo chmod a+rwX /dev/hugepages
 
 export RUST_BACKTRACE=1

scripts/run_integration_tests_rate_limiter.sh

@@ -1,8 +1,10 @@
-#!/bin/bash
+#!/usr/bin/env bash
+# shellcheck disable=SC2048,SC2086
 set -x
 
-source $HOME/.cargo/env
-source $(dirname "$0")/test-util.sh
+# shellcheck source=/dev/null
+source "$HOME"/.cargo/env
+source "$(dirname "$0")"/test-util.sh
 
 WORKLOADS_DIR="$HOME/workloads"
 mkdir -p "$WORKLOADS_DIR"
@@ -12,45 +14,45 @@ process_common_args "$@"
 # For now these values are default for kvm
 test_features=""
 
-if [ "$hypervisor" = "mshv" ] ;  then
+if [ "$hypervisor" = "mshv" ]; then
     test_features="--features mshv"
 fi
 
-cp scripts/sha1sums-x86_64 $WORKLOADS_DIR
+cp scripts/sha1sums-x86_64 "$WORKLOADS_DIR"
 
 FOCAL_OS_IMAGE_NAME="focal-server-cloudimg-amd64-custom-20210609-0.qcow2"
-FOCAL_OS_IMAGE_URL="https://ch-images.azureedge.net/$FOCAL_OS_IMAGE_NAME"
+FOCAL_OS_IMAGE_URL="https://cloud-hypervisor.azureedge.net/$FOCAL_OS_IMAGE_NAME"
 FOCAL_OS_IMAGE="$WORKLOADS_DIR/$FOCAL_OS_IMAGE_NAME"
 if [ ! -f "$FOCAL_OS_IMAGE" ]; then
-    pushd $WORKLOADS_DIR
+    pushd "$WORKLOADS_DIR" || exit
     time wget --quiet $FOCAL_OS_IMAGE_URL || exit 1
-    popd
+    popd || exit
 fi
 
 FOCAL_OS_RAW_IMAGE_NAME="focal-server-cloudimg-amd64-custom-20210609-0.raw"
 FOCAL_OS_RAW_IMAGE="$WORKLOADS_DIR/$FOCAL_OS_RAW_IMAGE_NAME"
 if [ ! -f "$FOCAL_OS_RAW_IMAGE" ]; then
-    pushd $WORKLOADS_DIR
+    pushd "$WORKLOADS_DIR" || exit
     time qemu-img convert -p -f qcow2 -O raw $FOCAL_OS_IMAGE_NAME $FOCAL_OS_RAW_IMAGE_NAME || exit 1
-    popd
+    popd || exit
 fi
 
-pushd $WORKLOADS_DIR
-grep focal sha1sums-x86_64 | sha1sum --check
-if [ $? -ne 0 ]; then
+pushd "$WORKLOADS_DIR" || exit
+if ! grep focal sha1sums-x86_64 | sha1sum --check; then
     echo "sha1sum validation of images failed, remove invalid images to fix the issue."
     exit 1
 fi
-popd
+popd || exit
 
 build_custom_linux
 
 CFLAGS=""
 if [[ "${BUILD_TARGET}" == "x86_64-unknown-linux-musl" ]]; then
+    # shellcheck disable=SC2034
     CFLAGS="-I /usr/include/x86_64-linux-musl/ -idirafter /usr/include/"
 fi
 
-cargo build --features mshv --all --release --target $BUILD_TARGET
+cargo build --features mshv --all --release --target "$BUILD_TARGET"
 
 export RUST_BACKTRACE=1
 time cargo test $test_features "rate_limiter::$test_filter" -- --test-threads=1 ${test_binary_args[*]}

scripts/run_integration_tests_sgx.sh

@@ -1,8 +1,10 @@
-#!/bin/bash
+#!/usr/bin/env bash
+# shellcheck disable=SC2048,SC2086
 set -x
 
-source $HOME/.cargo/env
-source $(dirname "$0")/test-util.sh
+# shellcheck source=/dev/null
+source "$HOME"/.cargo/env
+source "$(dirname "$0")"/test-util.sh
 
 process_common_args "$@"
 
@@ -17,28 +19,29 @@ mkdir -p "$WORKLOADS_DIR"
 download_hypervisor_fw
 
 JAMMY_OS_IMAGE_NAME="jammy-server-cloudimg-amd64-custom-20230119-0.qcow2"
-JAMMY_OS_IMAGE_URL="https://ch-images.azureedge.net/$JAMMY_OS_IMAGE_NAME"
+JAMMY_OS_IMAGE_URL="https://cloud-hypervisor.azureedge.net/$JAMMY_OS_IMAGE_NAME"
 JAMMY_OS_IMAGE="$WORKLOADS_DIR/$JAMMY_OS_IMAGE_NAME"
 if [ ! -f "$JAMMY_OS_IMAGE" ]; then
-    pushd $WORKLOADS_DIR
+    pushd "$WORKLOADS_DIR" || exit
     time wget --quiet $JAMMY_OS_IMAGE_URL || exit 1
-    popd
+    popd || exit
 fi
 
 JAMMY_OS_RAW_IMAGE_NAME="jammy-server-cloudimg-amd64-custom-20230119-0.raw"
 JAMMY_OS_RAW_IMAGE="$WORKLOADS_DIR/$JAMMY_OS_RAW_IMAGE_NAME"
 if [ ! -f "$JAMMY_OS_RAW_IMAGE" ]; then
-    pushd $WORKLOADS_DIR
+    pushd "$WORKLOADS_DIR" || exit
     time qemu-img convert -p -f qcow2 -O raw $JAMMY_OS_IMAGE_NAME $JAMMY_OS_RAW_IMAGE_NAME || exit 1
-    popd
+    popd || exit
 fi
 
 CFLAGS=""
 if [[ "${BUILD_TARGET}" == "x86_64-unknown-linux-musl" ]]; then
+    # shellcheck disable=SC2034
     CFLAGS="-I /usr/include/x86_64-linux-musl/ -idirafter /usr/include/"
 fi
 
-cargo build --features mshv --all --release --target $BUILD_TARGET
+cargo build --features mshv --all --release --target "$BUILD_TARGET"
 
 export RUST_BACKTRACE=1
 

scripts/run_integration_tests_vfio.sh

@@ -1,4 +1,5 @@
-#!/bin/bash
+#!/usr/bin/env bash
+# shellcheck disable=SC2048,SC2086
 set -x
 
 # This set of vfio tests require to be ran on a specific machine with
@@ -7,21 +8,23 @@ set -x
 # out of the scope of this script, including the custom guest image with
 # Nvidia drivers installed, and properly configured Nvidia Tesla T4 card.
 
-source $HOME/.cargo/env
-source $(dirname "$0")/test-util.sh
+# shellcheck source=/dev/null
+source "$HOME"/.cargo/env
+source "$(dirname "$0")"/test-util.sh
 
 process_common_args "$@"
 
 WORKLOADS_DIR="$HOME/workloads"
 
-download_hypervisor_fw 
+download_hypervisor_fw
 
 CFLAGS=""
 if [[ "${BUILD_TARGET}" == "x86_64-unknown-linux-musl" ]]; then
+    # shellcheck disable=SC2034
     CFLAGS="-I /usr/include/x86_64-linux-musl/ -idirafter /usr/include/"
 fi
 
-cargo build --features mshv --all --release --target $BUILD_TARGET
+cargo build --features mshv --all --release --target "$BUILD_TARGET"
 
 export RUST_BACKTRACE=1
 time cargo test "vfio::test_nvidia" -- --test-threads=1 ${test_binary_args[*]}

scripts/run_integration_tests_windows_aarch64.sh

@@ -1,9 +1,11 @@
-#!/bin/bash
+#!/usr/bin/env bash
+# shellcheck disable=SC2048,SC2086
 set -x
 
-source $HOME/.cargo/env
-source $(dirname "$0")/test-util.sh
-source $(dirname "$0")/common-aarch64.sh
+# shellcheck source=/dev/null
+source "$HOME"/.cargo/env
+source "$(dirname "$0")"/test-util.sh
+source "$(dirname "$0")"/common-aarch64.sh
 
 process_common_args "$@"
 
@@ -27,8 +29,8 @@ if [[ ! -f ${WIN_IMAGE_FILE} || ! -f ${OVMF_FW} ]]; then
 fi
 
 # Use device mapper to create a snapshot of the Windows image
-img_blk_size=$(du -b -B 512 ${WIN_IMAGE_FILE} | awk '{print $1;}')
-loop_device=$(losetup --find --show --read-only ${WIN_IMAGE_FILE})
+img_blk_size=$(du -b -B 512 "${WIN_IMAGE_FILE}" | awk '{print $1;}')
+loop_device=$(losetup --find --show --read-only "${WIN_IMAGE_FILE}")
 dmsetup create windows-base --table "0 $img_blk_size linear $loop_device 0"
 dmsetup mknodes
 dmsetup create windows-snapshot-base --table "0 $img_blk_size snapshot-origin /dev/mapper/windows-base"
@@ -36,11 +38,11 @@ dmsetup mknodes
 
 export RUST_BACKTRACE=1
 
-cargo build --all --release --target $BUILD_TARGET
+cargo build --all --release --target "$BUILD_TARGET"
 
 # Only run with 1 thread to avoid tests interfering with one another because
 # Windows has a static IP configured
-time cargo test "windows::$test_filter" --target $BUILD_TARGET -- ${test_binary_args[*]}
+time cargo test "windows::$test_filter" --target "$BUILD_TARGET" -- ${test_binary_args[*]}
 RES=$?
 
 dmsetup remove_all -f

scripts/run_integration_tests_windows_x86_64.sh

@@ -1,29 +1,27 @@
-#!/bin/bash
+#!/usr/bin/env bash
+# shellcheck disable=SC2048,SC2086
 set -x
 
-source $HOME/.cargo/env
-source $(dirname "$0")/test-util.sh
+# shellcheck source=/dev/null
+source "$HOME"/.cargo/env
+source "$(dirname "$0")"/test-util.sh
 
 process_common_args "$@"
 # For now these values are default for kvm
 test_features=""
 
-if [ "$hypervisor" = "mshv" ] ;  then
+if [ "$hypervisor" = "mshv" ]; then
     test_features="--features mshv"
 fi
 WIN_IMAGE_FILE="/root/workloads/windows-server-2022-amd64-2.raw"
 
 WORKLOADS_DIR="/root/workloads"
-OVMF_FW_URL=$(curl --silent https://api.github.com/repos/cloud-hypervisor/edk2/releases/latest | grep "browser_download_url" | grep -o 'https://.*[^ "]')
-OVMF_FW="$WORKLOADS_DIR/CLOUDHV.fd"
-if [ ! -f "$OVMF_FW" ]; then
-    pushd $WORKLOADS_DIR
-    time wget --quiet $OVMF_FW_URL || exit 1
-    popd
-fi
+
+download_ovmf
 
 CFLAGS=""
 if [[ "${BUILD_TARGET}" == "x86_64-unknown-linux-musl" ]]; then
+    # shellcheck disable=SC2034
     CFLAGS="-I /usr/include/x86_64-linux-musl/ -idirafter /usr/include/"
 fi
 
@@ -41,7 +39,7 @@ dmsetup mknodes
 dmsetup create windows-snapshot-base --table "0 $img_blk_size snapshot-origin /dev/mapper/windows-base"
 dmsetup mknodes
 
-cargo build --features mshv --all --release --target $BUILD_TARGET
+cargo build --features mshv --all --release --target "$BUILD_TARGET"
 
 export RUST_BACKTRACE=1
 

scripts/run_integration_tests_x86_64.sh

@@ -1,8 +1,10 @@
-#!/bin/bash
+#!/usr/bin/env bash
+# shellcheck disable=SC2048,SC2086
 set -x
 
-source $HOME/.cargo/env
-source $(dirname "$0")/test-util.sh
+# shellcheck source=/dev/null
+source "$HOME"/.cargo/env
+source "$(dirname "$0")"/test-util.sh
 
 WORKLOADS_DIR="$HOME/workloads"
 mkdir -p "$WORKLOADS_DIR"
@@ -12,97 +14,90 @@ process_common_args "$@"
 # For now these values are default for kvm
 test_features=""
 
-if [ "$hypervisor" = "mshv" ] ;  then
+if [ "$hypervisor" = "mshv" ]; then
     test_features="--features mshv"
 fi
 
-cp scripts/sha1sums-x86_64 $WORKLOADS_DIR
+cp scripts/sha1sums-x86_64 "$WORKLOADS_DIR"
 
 download_hypervisor_fw
 
-OVMF_FW_URL=$(curl --silent https://api.github.com/repos/cloud-hypervisor/edk2/releases/latest | grep "browser_download_url" | grep -o 'https://.*[^ "]')
-OVMF_FW="$WORKLOADS_DIR/CLOUDHV.fd"
-if [ ! -f "$OVMF_FW" ]; then
-    pushd $WORKLOADS_DIR
-    time wget --quiet $OVMF_FW_URL || exit 1
-    popd
-fi
+download_ovmf
 
 FOCAL_OS_IMAGE_NAME="focal-server-cloudimg-amd64-custom-20210609-0.qcow2"
-FOCAL_OS_IMAGE_URL="https://ch-images.azureedge.net/$FOCAL_OS_IMAGE_NAME"
+FOCAL_OS_IMAGE_URL="https://cloud-hypervisor.azureedge.net/$FOCAL_OS_IMAGE_NAME"
 FOCAL_OS_IMAGE="$WORKLOADS_DIR/$FOCAL_OS_IMAGE_NAME"
 if [ ! -f "$FOCAL_OS_IMAGE" ]; then
-    pushd $WORKLOADS_DIR
+    pushd "$WORKLOADS_DIR" || exit
     time wget --quiet $FOCAL_OS_IMAGE_URL || exit 1
-    popd
+    popd || exit
 fi
 
 FOCAL_OS_RAW_IMAGE_NAME="focal-server-cloudimg-amd64-custom-20210609-0.raw"
 FOCAL_OS_RAW_IMAGE="$WORKLOADS_DIR/$FOCAL_OS_RAW_IMAGE_NAME"
 if [ ! -f "$FOCAL_OS_RAW_IMAGE" ]; then
-    pushd $WORKLOADS_DIR
+    pushd "$WORKLOADS_DIR" || exit
     time qemu-img convert -p -f qcow2 -O raw $FOCAL_OS_IMAGE_NAME $FOCAL_OS_RAW_IMAGE_NAME || exit 1
-    popd
+    popd || exit
 fi
 
 FOCAL_OS_QCOW_BACKING_FILE_IMAGE_NAME="focal-server-cloudimg-amd64-custom-20210609-0-backing.qcow2"
 FOCAL_OS_QCOW_BACKING_FILE_IMAGE="$WORKLOADS_DIR/$FOCAL_OS_QCOW_BACKING_FILE_IMAGE_NAME"
 if [ ! -f "$FOCAL_OS_QCOW_BACKING_FILE_IMAGE" ]; then
-    pushd $WORKLOADS_DIR
-    time qemu-img create -f qcow2 -b $FOCAL_OS_IMAGE -F qcow2 $FOCAL_OS_QCOW_BACKING_FILE_IMAGE_NAME
-    popd
+    pushd "$WORKLOADS_DIR" || exit
+    time qemu-img create -f qcow2 -b "$FOCAL_OS_IMAGE" -F qcow2 $FOCAL_OS_QCOW_BACKING_FILE_IMAGE_NAME
+    popd || exit
 fi
 
 JAMMY_OS_IMAGE_NAME="jammy-server-cloudimg-amd64-custom-20230119-0.qcow2"
-JAMMY_OS_IMAGE_URL="https://ch-images.azureedge.net/$JAMMY_OS_IMAGE_NAME"
+JAMMY_OS_IMAGE_URL="https://cloud-hypervisor.azureedge.net/$JAMMY_OS_IMAGE_NAME"
 JAMMY_OS_IMAGE="$WORKLOADS_DIR/$JAMMY_OS_IMAGE_NAME"
 if [ ! -f "$JAMMY_OS_IMAGE" ]; then
-    pushd $WORKLOADS_DIR
+    pushd "$WORKLOADS_DIR" || exit
     time wget --quiet $JAMMY_OS_IMAGE_URL || exit 1
-    popd
+    popd || exit
 fi
 
 JAMMY_OS_RAW_IMAGE_NAME="jammy-server-cloudimg-amd64-custom-20230119-0.raw"
 JAMMY_OS_RAW_IMAGE="$WORKLOADS_DIR/$JAMMY_OS_RAW_IMAGE_NAME"
 if [ ! -f "$JAMMY_OS_RAW_IMAGE" ]; then
-    pushd $WORKLOADS_DIR
+    pushd "$WORKLOADS_DIR" || exit
     time qemu-img convert -p -f qcow2 -O raw $JAMMY_OS_IMAGE_NAME $JAMMY_OS_RAW_IMAGE_NAME || exit 1
-    popd
+    popd || exit
 fi
 
 ALPINE_MINIROOTFS_URL="http://dl-cdn.alpinelinux.org/alpine/v3.11/releases/x86_64/alpine-minirootfs-3.11.3-x86_64.tar.gz"
 ALPINE_MINIROOTFS_TARBALL="$WORKLOADS_DIR/alpine-minirootfs-x86_64.tar.gz"
 if [ ! -f "$ALPINE_MINIROOTFS_TARBALL" ]; then
-    pushd $WORKLOADS_DIR
-    time wget --quiet $ALPINE_MINIROOTFS_URL -O $ALPINE_MINIROOTFS_TARBALL || exit 1
-    popd
+    pushd "$WORKLOADS_DIR" || exit
+    time wget --quiet $ALPINE_MINIROOTFS_URL -O "$ALPINE_MINIROOTFS_TARBALL" || exit 1
+    popd || exit
 fi
 
 ALPINE_INITRAMFS_IMAGE="$WORKLOADS_DIR/alpine_initramfs.img"
 if [ ! -f "$ALPINE_INITRAMFS_IMAGE" ]; then
-    pushd $WORKLOADS_DIR
+    pushd "$WORKLOADS_DIR" || exit
     mkdir alpine-minirootfs
     tar xf "$ALPINE_MINIROOTFS_TARBALL" -C alpine-minirootfs
-    cat > alpine-minirootfs/init <<-EOF
+    cat >alpine-minirootfs/init <<-EOF
 		#! /bin/sh
 		mount -t devtmpfs dev /dev
 		echo \$TEST_STRING > /dev/console
 		poweroff -f
 	EOF
     chmod +x alpine-minirootfs/init
-    cd alpine-minirootfs
+    cd alpine-minirootfs || exit
     find . -print0 |
-        cpio --null --create --verbose --owner root:root --format=newc > "$ALPINE_INITRAMFS_IMAGE"
-    popd
+        cpio --null --create --verbose --owner root:root --format=newc >"$ALPINE_INITRAMFS_IMAGE"
+    popd || exit
 fi
 
-pushd $WORKLOADS_DIR
-sha1sum sha1sums-x86_64 --check
-if [ $? -ne 0 ]; then
+pushd "$WORKLOADS_DIR" || exit
+if ! sha1sum sha1sums-x86_64 --check; then
     echo "sha1sum validation of images failed, remove invalid images to fix the issue."
     exit 1
 fi
-popd
+popd || exit
 
 # Build custom kernel based on virtio-pmem and virtio-fs upstream patches
 VMLINUX_IMAGE="$WORKLOADS_DIR/vmlinux"
@@ -113,52 +108,51 @@ fi
 VIRTIOFSD="$WORKLOADS_DIR/virtiofsd"
 VIRTIOFSD_DIR="virtiofsd_build"
 if [ ! -f "$VIRTIOFSD" ]; then
-    pushd $WORKLOADS_DIR
+    pushd "$WORKLOADS_DIR" || exit
     git clone "https://gitlab.com/virtio-fs/virtiofsd.git" $VIRTIOFSD_DIR
-    pushd $VIRTIOFSD_DIR
+    pushd $VIRTIOFSD_DIR || exit
     git checkout v1.8.0
     time cargo build --release
-    cp target/release/virtiofsd $VIRTIOFSD || exit 1
-    popd
+    cp target/release/virtiofsd "$VIRTIOFSD" || exit 1
+    popd || exit
     rm -rf $VIRTIOFSD_DIR
-    popd
+    popd || exit
 fi
 
-
 BLK_IMAGE="$WORKLOADS_DIR/blk.img"
 MNT_DIR="mount_image"
 if [ ! -f "$BLK_IMAGE" ]; then
-   pushd $WORKLOADS_DIR
-   fallocate -l 16M $BLK_IMAGE
-   mkfs.ext4 -j $BLK_IMAGE
-   mkdir $MNT_DIR
-   sudo mount -t ext4 $BLK_IMAGE $MNT_DIR
-   sudo bash -c "echo bar > $MNT_DIR/foo" || exit 1
-   sudo umount $BLK_IMAGE
-   rm -r $MNT_DIR
-   popd
+    pushd "$WORKLOADS_DIR" || exit
+    fallocate -l 16M "$BLK_IMAGE"
+    mkfs.ext4 -j "$BLK_IMAGE"
+    mkdir $MNT_DIR
+    sudo mount -t ext4 "$BLK_IMAGE" $MNT_DIR
+    sudo bash -c "echo bar > $MNT_DIR/foo" || exit 1
+    sudo umount "$BLK_IMAGE"
+    rm -r $MNT_DIR
+    popd || exit
 fi
 
 SHARED_DIR="$WORKLOADS_DIR/shared_dir"
 if [ ! -d "$SHARED_DIR" ]; then
-    mkdir -p $SHARED_DIR
-    echo "foo" > "$SHARED_DIR/file1"
-    echo "bar" > "$SHARED_DIR/file3" || exit 1
+    mkdir -p "$SHARED_DIR"
+    echo "foo" >"$SHARED_DIR/file1"
+    echo "bar" >"$SHARED_DIR/file3" || exit 1
 fi
 
 VFIO_DIR="$WORKLOADS_DIR/vfio"
 VFIO_DISK_IMAGE="$WORKLOADS_DIR/vfio.img"
-rm -rf $VFIO_DIR $VFIO_DISK_IMAGE
-mkdir -p $VFIO_DIR
-cp $FOCAL_OS_RAW_IMAGE $VFIO_DIR
-cp $FW $VFIO_DIR
-cp $VMLINUX_IMAGE $VFIO_DIR || exit 1
+rm -rf "$VFIO_DIR" "$VFIO_DISK_IMAGE"
+mkdir -p "$VFIO_DIR"
+cp "$FOCAL_OS_RAW_IMAGE" "$VFIO_DIR"
+cp "$FW" "$VFIO_DIR"
+cp "$VMLINUX_IMAGE" "$VFIO_DIR" || exit 1
 
-cargo build --features mshv --all  --release --target $BUILD_TARGET
+cargo build --features mshv --all --release --target "$BUILD_TARGET"
 
 # We always copy a fresh version of our binary for our L2 guest.
-cp target/$BUILD_TARGET/release/cloud-hypervisor $VFIO_DIR
-cp target/$BUILD_TARGET/release/ch-remote $VFIO_DIR
+cp target/"$BUILD_TARGET"/release/cloud-hypervisor "$VFIO_DIR"
+cp target/"$BUILD_TARGET"/release/ch-remote "$VFIO_DIR"
 
 # Enable KSM with some reasonable parameters so that it won't take too long
 # for the memory to be merged between two processes.
@@ -167,9 +161,9 @@ sudo bash -c "echo 10 > /sys/kernel/mm/ksm/sleep_millisecs"
 sudo bash -c "echo 1 > /sys/kernel/mm/ksm/run"
 
 # Both test_vfio, ovs-dpdk and vDPA tests rely on hugepages
-HUGEPAGESIZE=`grep Hugepagesize /proc/meminfo | awk '{print $2}'`
-PAGE_NUM=`echo $((12288 * 1024 / $HUGEPAGESIZE))`
-echo $PAGE_NUM | sudo tee /proc/sys/vm/nr_hugepages
+HUGEPAGESIZE=$(grep Hugepagesize /proc/meminfo | awk '{print $2}')
+PAGE_NUM=$((12288 * 1024 / HUGEPAGESIZE))
+echo "$PAGE_NUM" | sudo tee /proc/sys/vm/nr_hugepages
 sudo chmod a+rwX /dev/hugepages
 
 # Update max locked memory to 'unlimited' to avoid issues with vDPA
@@ -192,7 +186,7 @@ fi
 
 # Run tests on dbus_api
 if [ $RES -eq 0 ]; then
-    cargo build --features "mshv,dbus_api" --all  --release --target $BUILD_TARGET
+    cargo build --features "mshv,dbus_api" --all --release --target "$BUILD_TARGET"
     export RUST_BACKTRACE=1
     # integration tests now do not reply on build feature "dbus_api"
     time cargo test $test_features "dbus_api::$test_filter" -- ${test_binary_args[*]}

scripts/run_metrics.sh

@@ -1,10 +1,12 @@
-#!/bin/bash
+#!/usr/bin/env bash
 set -x
 
-source $HOME/.cargo/env
-source $(dirname "$0")/test-util.sh
+# shellcheck source=/dev/null
+source "$HOME"/.cargo/env
+source "$(dirname "$0")"/test-util.sh
 
-export TEST_ARCH=$(uname -m)
+TEST_ARCH=$(uname -m)
+export TEST_ARCH
 
 WORKLOADS_DIR="$HOME/workloads"
 mkdir -p "$WORKLOADS_DIR"
@@ -15,34 +17,34 @@ build_fio() {
 
     checkout_repo "$FIO_DIR" "$FIO_REPO" master "1953e1adb5a28ed21370e85991d7f5c3cdc699f3"
     if [ ! -f "$FIO_DIR/.built" ]; then
-        pushd $FIO_DIR
+        pushd "$FIO_DIR" || exit
         ./configure
-        make -j `nproc`
+        make -j "$(nproc)"
         cp fio "$WORKLOADS_DIR/fio"
         touch .built
-        popd
+        popd || exit
     fi
 }
 
 process_common_args "$@"
 
-cp scripts/sha1sums-${TEST_ARCH} $WORKLOADS_DIR
+cp scripts/sha1sums-"${TEST_ARCH}" "$WORKLOADS_DIR"
 
-if [ ${TEST_ARCH} == "aarch64" ]; then
-     FOCAL_OS_IMAGE_NAME="focal-server-cloudimg-arm64-custom-20210929-0.qcow2"
+if [ "${TEST_ARCH}" == "aarch64" ]; then
+    FOCAL_OS_IMAGE_NAME="focal-server-cloudimg-arm64-custom-20210929-0.qcow2"
 else
-     FOCAL_OS_IMAGE_NAME="focal-server-cloudimg-amd64-custom-20210609-0.qcow2"
+    FOCAL_OS_IMAGE_NAME="focal-server-cloudimg-amd64-custom-20210609-0.qcow2"
 fi
 
-FOCAL_OS_IMAGE_URL="https://ch-images.azureedge.net/$FOCAL_OS_IMAGE_NAME"
+FOCAL_OS_IMAGE_URL="https://cloud-hypervisor.azureedge.net/$FOCAL_OS_IMAGE_NAME"
 FOCAL_OS_IMAGE="$WORKLOADS_DIR/$FOCAL_OS_IMAGE_NAME"
 if [ ! -f "$FOCAL_OS_IMAGE" ]; then
-    pushd $WORKLOADS_DIR
+    pushd "$WORKLOADS_DIR" || exit
     time wget --quiet $FOCAL_OS_IMAGE_URL || exit 1
-    popd
+    popd || exit
 fi
 
-if [ ${TEST_ARCH} == "aarch64" ]; then
+if [ "${TEST_ARCH}" == "aarch64" ]; then
     FOCAL_OS_RAW_IMAGE_NAME="focal-server-cloudimg-arm64-custom-20210929-0.raw"
 else
     FOCAL_OS_RAW_IMAGE_NAME="focal-server-cloudimg-amd64-custom-20210609-0.raw"
@@ -50,20 +52,19 @@ fi
 
 FOCAL_OS_RAW_IMAGE="$WORKLOADS_DIR/$FOCAL_OS_RAW_IMAGE_NAME"
 if [ ! -f "$FOCAL_OS_RAW_IMAGE" ]; then
-    pushd $WORKLOADS_DIR
+    pushd "$WORKLOADS_DIR" || exit
     time qemu-img convert -p -f qcow2 -O raw $FOCAL_OS_IMAGE_NAME $FOCAL_OS_RAW_IMAGE_NAME || exit 1
-    popd
+    popd || exit
 fi
 
-pushd $WORKLOADS_DIR
-grep focal sha1sums-${TEST_ARCH} | sha1sum --check
-if [ $? -ne 0 ]; then
+pushd "$WORKLOADS_DIR" || exit
+if ! grep focal sha1sums-"${TEST_ARCH}" | sha1sum --check; then
     echo "sha1sum validation of images failed, remove invalid images to fix the issue."
     exit 1
 fi
-popd
+popd || exit
 
-if [ ${TEST_ARCH} == "aarch64" ]; then
+if [ "${TEST_ARCH}" == "aarch64" ]; then
     build_fio
 
     # Update the fio in the cloud image to use io_uring on AArch64
@@ -84,15 +85,16 @@ build_custom_linux
 
 CFLAGS=""
 if [[ "${BUILD_TARGET}" == "${TEST_ARCH}-unknown-linux-musl" ]]; then
+    # shellcheck disable=SC2034
     CFLAGS="-I /usr/include/${TEST_ARCH}-linux-musl/ -idirafter /usr/include/"
 fi
 
-cargo build --features mshv --all --release --target $BUILD_TARGET
+cargo build --features mshv --all --release --target "$BUILD_TARGET"
 
 # setup hugepages
-HUGEPAGESIZE=`grep Hugepagesize /proc/meminfo | awk '{print $2}'`
-PAGE_NUM=`echo $((12288 * 1024 / $HUGEPAGESIZE))`
-echo $PAGE_NUM | sudo tee /proc/sys/vm/nr_hugepages
+HUGEPAGESIZE=$(grep Hugepagesize /proc/meminfo | awk '{print $2}')
+PAGE_NUM=$((12288 * 1024 / HUGEPAGESIZE))
+echo "$PAGE_NUM" | sudo tee /proc/sys/vm/nr_hugepages
 sudo chmod a+rwX /dev/hugepages
 
 if [ -n "$test_filter" ]; then
@@ -100,15 +102,16 @@ if [ -n "$test_filter" ]; then
 fi
 
 # Ensure that git commands can be run in this directory (for metrics report)
-git config --global --add safe.directory $PWD
+git config --global --add safe.directory "$PWD"
 
-RUST_BACKTRACE_VALUE=`echo $RUST_BACKTRACE`
-if [ -z $RUST_BACKTRACE_VALUE ];then
-	export RUST_BACKTRACE=1
+RUST_BACKTRACE_VALUE=$RUST_BACKTRACE
+if [ -z "$RUST_BACKTRACE_VALUE" ]; then
+    export RUST_BACKTRACE=1
 else
-	echo "RUST_BACKTRACE is set to: $RUST_BACKTRACE_VALUE"
+    echo "RUST_BACKTRACE is set to: $RUST_BACKTRACE_VALUE"
 fi
-time target/$BUILD_TARGET/release/performance-metrics ${test_binary_args[*]}
+# shellcheck disable=SC2048,SC2086
+time target/"$BUILD_TARGET"/release/performance-metrics ${test_binary_args[*]}
 RES=$?
 
 exit $RES

scripts/run_openapi_tests.sh

@@ -1,5 +1,5 @@
-#!/bin/bash
+#!/usr/bin/env bash
 set -e
 set -x
 
-sudo docker run --rm -v ${PWD}:/local openapitools/openapi-generator-cli validate -i /local/vmm/src/api/openapi/cloud-hypervisor.yaml
+sudo docker run --rm -v "${PWD}":/local openapitools/openapi-generator-cli validate -i /local/vmm/src/api/openapi/cloud-hypervisor.yaml

scripts/run_unit_tests.sh

@@ -1,12 +1,15 @@
-#!/bin/bash
+#!/usr/bin/env bash
+# shellcheck disable=SC2068
 
-source $HOME/.cargo/env
-source $(dirname "$0")/test-util.sh
+# shellcheck source=/dev/null
+source "$HOME"/.cargo/env
+source "$(dirname "$0")"/test-util.sh
 
 process_common_args "$@"
 
 cargo_args=("")
 
+# shellcheck disable=SC2154
 if [[ $hypervisor = "mshv" ]]; then
     cargo_args+=("--features $hypervisor")
 elif [[ $(uname -m) = "x86_64" ]]; then
@@ -14,5 +17,5 @@ elif [[ $(uname -m) = "x86_64" ]]; then
 fi
 
 export RUST_BACKTRACE=1
-cargo test --lib --bins --target $BUILD_TARGET --workspace ${cargo_args[@]} || exit 1
-cargo test --doc --target $BUILD_TARGET --workspace ${cargo_args[@]} || exit 1
+cargo test --lib --bins --target "$BUILD_TARGET" --workspace ${cargo_args[@]} || exit 1
+cargo test --doc --target "$BUILD_TARGET" --workspace ${cargo_args[@]} || exit 1

scripts/sha1sums-aarch64

@@ -1,3 +1,6 @@
+6fee67adbfed8db7a225be23ee9d90b5bd7f19e6  bionic-server-cloudimg-arm64.img
+786fe1c33588334e92b35c65e414da068df180bc  bionic-server-cloudimg-arm64.raw
+6e66f9f4b01adc72c884c1c1111e60afadc9c871  bionic-server-cloudimg-arm64.qcow2
 e4addb6e212a298144f9eb0eb6e36019d013f0e7  alpine-minirootfs-aarch64.tar.gz
 25b4f9ac308898d63b73d7db0e0e2d4768853723  focal-server-cloudimg-arm64-custom-20210929-0.qcow2
 9953b31bb1923cdd8d91b1b7cc9ad3a9be1e0a59  focal-server-cloudimg-arm64-custom-20210929-0.raw

scripts/test-util.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 hypervisor="kvm"
 test_filter=""
 
@@ -18,13 +18,13 @@ checkout_repo() {
     # If commit is not specified, compare local HEAD and remote HEAD.
     # Remove the folder if there is difference.
     if [ -d "$SRC_DIR" ]; then
-        pushd $SRC_DIR
+        pushd "$SRC_DIR" || exit
         git fetch
         SRC_LOCAL_COMMIT=$(git rev-parse HEAD)
         if [ -z "$GIT_COMMIT" ]; then
             GIT_COMMIT=$(git rev-parse remotes/origin/"$GIT_BRANCH")
         fi
-        popd
+        popd || exit
         if [ "$SRC_LOCAL_COMMIT" != "$GIT_COMMIT" ]; then
             rm -rf "$SRC_DIR"
         fi
@@ -34,10 +34,10 @@ checkout_repo() {
     if [ ! -d "$SRC_DIR" ]; then
         git clone --depth 1 "$GIT_URL" -b "$GIT_BRANCH" "$SRC_DIR"
         if [ "$GIT_COMMIT" ]; then
-            pushd "$SRC_DIR"
+            pushd "$SRC_DIR" || exit
             git fetch --depth 1 origin "$GIT_COMMIT"
             git reset --hard FETCH_HEAD
-            popd
+            popd || exit
         fi
     fi
 }
@@ -51,23 +51,23 @@ build_custom_linux() {
 
     checkout_repo "$LINUX_CUSTOM_DIR" "$LINUX_CUSTOM_URL" "$LINUX_CUSTOM_BRANCH"
 
-    cp $SRCDIR/resources/linux-config-${ARCH} $LINUX_CUSTOM_DIR/.config
+    cp "$SRCDIR"/resources/linux-config-"${ARCH}" "$LINUX_CUSTOM_DIR"/.config
 
-    pushd $LINUX_CUSTOM_DIR
-    make -j `nproc`
-    if [ ${ARCH} == "x86_64" ]; then
-       cp vmlinux "$WORKLOADS_DIR/" || exit 1
-    elif [ ${ARCH} == "aarch64" ]; then
-       cp arch/arm64/boot/Image "$WORKLOADS_DIR/" || exit 1
-       cp arch/arm64/boot/Image.gz "$WORKLOADS_DIR/" || exit 1
+    pushd "$LINUX_CUSTOM_DIR" || exit
+    make -j "$(nproc)"
+    if [ "${ARCH}" == "x86_64" ]; then
+        cp vmlinux "$WORKLOADS_DIR/" || exit 1
+    elif [ "${ARCH}" == "aarch64" ]; then
+        cp arch/arm64/boot/Image "$WORKLOADS_DIR/" || exit 1
+        cp arch/arm64/boot/Image.gz "$WORKLOADS_DIR/" || exit 1
     fi
-    popd
+    popd || exit
 }
 
 cmd_help() {
     echo ""
-    echo "Cloud Hypervisor $(basename $0)"
-    echo "Usage: $(basename $0) [<args>]"
+    echo "Cloud Hypervisor $(basename "$0")"
+    echo "Usage: $(basename "$0") [<args>]"
     echo ""
     echo "Available arguments:"
     echo ""
@@ -80,47 +80,61 @@ cmd_help() {
 
 process_common_args() {
     while [ $# -gt 0 ]; do
-	case "$1" in
-            "-h"|"--help")  { cmd_help; exit 1; } ;;
-            "--hypervisor")
-                shift
-                hypervisor="$1"
-                ;;
-            "--test-filter")
-                shift
-                test_filter="$1"
-                ;;
-            "--") {
-                shift
-                break
-            } ;;
-            *)
-                echo "Unknown test scripts argument: $1. Please use '-- --help' for help."
-                exit
-                ;;
-	esac
-	shift
+        case "$1" in
+        "-h" | "--help") {
+            cmd_help
+            exit 1
+        } ;;
+        "--hypervisor")
+            shift
+            hypervisor="$1"
+            ;;
+        "--test-filter")
+            shift
+            # shellcheck disable=SC2034
+            test_filter="$1"
+            ;;
+        "--") {
+            shift
+            break
+        } ;;
+        *)
+            echo "Unknown test scripts argument: $1. Please use '-- --help' for help."
+            exit
+            ;;
+        esac
+        shift
     done
-    if [[ ! ("$hypervisor" = "kvm" ||  "$hypervisor" = "mshv") ]]; then
+    if [[ ! ("$hypervisor" = "kvm" || "$hypervisor" = "mshv") ]]; then
         die "Hypervisor value must be kvm or mshv"
     fi
-
-    test_binary_args=($@)
+    # shellcheck disable=SC2034
+    test_binary_args=("$@")
 }
 
 download_hypervisor_fw() {
     if [ -n "$AUTH_DOWNLOAD_TOKEN" ]; then
         echo "Using authenticated download from GitHub"
         FW_URL=$(curl --silent https://api.github.com/repos/cloud-hypervisor/rust-hypervisor-firmware/releases/latest \
-                 --header "Authorization: Token $AUTH_DOWNLOAD_TOKEN" \
-                 --header "X-GitHub-Api-Version: 2022-11-28" | grep "browser_download_url" | grep -o 'https://.*[^ "]')
+            --header "Authorization: Token $AUTH_DOWNLOAD_TOKEN" \
+            --header "X-GitHub-Api-Version: 2022-11-28" | grep "browser_download_url" | grep -o 'https://.*[^ "]')
     else
         echo "Using anonymous download from GitHub"
         FW_URL=$(curl --silent https://api.github.com/repos/cloud-hypervisor/rust-hypervisor-firmware/releases/latest | grep "browser_download_url" | grep -o 'https://.*[^ "]')
     fi
     FW="$WORKLOADS_DIR/hypervisor-fw"
-    pushd $WORKLOADS_DIR
-    rm -f $FW
-    time wget --quiet $FW_URL || exit 1
-    popd
+    pushd "$WORKLOADS_DIR" || exit
+    rm -f "$FW"
+    time wget --quiet "$FW_URL" || exit 1
+    popd || exit
+}
+
+download_ovmf() {
+    OVMF_FW_TAG="ch-6624aa331f"
+    OVMF_FW_URL="https://github.com/cloud-hypervisor/edk2/releases/download/$OVMF_FW_TAG/CLOUDHV.fd"
+    OVMF_FW="$WORKLOADS_DIR/CLOUDHV.fd"
+    pushd "$WORKLOADS_DIR" || exit
+    rm -f "$OVMF_FW"
+    time wget --quiet $OVMF_FW_URL || exit 1
+    popd || exit
 }

src/main.rs

@@ -20,6 +20,7 @@ use std::sync::{Arc, Mutex};
 use thiserror::Error;
 #[cfg(feature = "dbus_api")]
 use vmm::api::dbus::{dbus_api_graceful_shutdown, DBusApiOptions};
+use vmm::api::ApiAction;
 use vmm::config;
 use vmm_sys_util::eventfd::EventFd;
 use vmm_sys_util::signal::block_signal;
@@ -240,6 +241,13 @@ fn create_app(default_vcpus: String, default_memory: String, default_rng: String
                 .num_args(1)
                 .group("vm-config"),
         )
+        .arg(
+            Arg::new("rate-limit-group")
+                .long("rate-limit-group")
+                .help(config::RateLimiterGroupConfig::SYNTAX)
+                .num_args(1..)
+                .group("vm-config"),
+        )
         .arg(
             Arg::new("disk")
                 .long("disk")
@@ -410,6 +418,15 @@ fn create_app(default_vcpus: String, default_memory: String, default_rng: String
             .group("vm-config"),
     );
 
+    #[cfg(target_arch = "x86_64")]
+    let app = app.arg(
+        Arg::new("debug-console")
+            .long("debug-console")
+            .help("Debug console: off|pty|tty|file=</path/to/a/file>,iobase=<port in hex>")
+            .default_value("off,iobase=0xe9")
+            .group("vm-config"),
+    );
+
     #[cfg(feature = "guest_debug")]
     let app = app.arg(
         Arg::new("gdb")
@@ -680,6 +697,11 @@ fn start_vmm(cmd_arguments: ArgMatches) -> Result<Option<String>, Error> {
     .map_err(Error::StartVmmThread)?;
 
     let r: Result<(), Error> = (|| {
+        #[cfg(feature = "igvm")]
+        let payload_present = cmd_arguments.contains_id("kernel")
+            || cmd_arguments.contains_id("firmware")
+            || cmd_arguments.contains_id("igvm");
+        #[cfg(not(feature = "igvm"))]
         let payload_present =
             cmd_arguments.contains_id("kernel") || cmd_arguments.contains_id("firmware");
 
@@ -689,22 +711,24 @@ fn start_vmm(cmd_arguments: ArgMatches) -> Result<Option<String>, Error> {
 
             // Create and boot the VM based off the VM config we just built.
             let sender = api_request_sender.clone();
-            vmm::api::vm_create(
-                api_evt.try_clone().unwrap(),
-                api_request_sender,
-                Arc::new(Mutex::new(vm_config)),
-            )
-            .map_err(Error::VmCreate)?;
-            vmm::api::vm_boot(api_evt.try_clone().unwrap(), sender).map_err(Error::VmBoot)?;
+            vmm::api::VmCreate
+                .send(
+                    api_evt.try_clone().unwrap(),
+                    api_request_sender,
+                    Arc::new(Mutex::new(vm_config)),
+                )
+                .map_err(Error::VmCreate)?;
+            vmm::api::VmBoot
+                .send(api_evt.try_clone().unwrap(), sender, ())
+                .map_err(Error::VmBoot)?;
         } else if let Some(restore_params) = cmd_arguments.get_one::<String>("restore") {
-            vmm::api::vm_restore(
-                api_evt.try_clone().unwrap(),
-                api_request_sender,
-                Arc::new(
+            vmm::api::VmRestore
+                .send(
+                    api_evt.try_clone().unwrap(),
+                    api_request_sender,
                     config::RestoreConfig::parse(restore_params).map_err(Error::ParsingRestore)?,
-                ),
-            )
-            .map_err(Error::VmRestore)?;
+                )
+                .map_err(Error::VmRestore)?;
         }
 
         Ok(())
@@ -734,6 +758,9 @@ fn main() {
     #[cfg(all(feature = "tdx", feature = "sev_snp"))]
     compile_error!("Feature 'tdx' and 'sev_snp' are mutually exclusive.");
 
+    #[cfg(all(feature = "sev_snp", not(target_arch = "x86_64")))]
+    compile_error!("Feature 'sev_snp' needs target 'x86_64'");
+
     #[cfg(feature = "dhat-heap")]
     let _profiler = dhat::Profiler::new_heap();
 
@@ -780,6 +807,8 @@ mod unit_tests {
         ConsoleConfig, ConsoleOutputMode, CpuFeatures, CpusConfig, MemoryConfig, PayloadConfig,
         RngConfig, VmConfig, VmParams,
     };
+    #[cfg(target_arch = "x86_64")]
+    use vmm::vm_config::DebugConsoleConfig;
 
     fn get_vm_config_from_vec(args: &[&str]) -> VmConfig {
         let (default_vcpus, default_memory, default_rng) = prepare_default_values();
@@ -841,8 +870,13 @@ mod unit_tests {
             },
             payload: Some(PayloadConfig {
                 kernel: Some(PathBuf::from("/path/to/kernel")),
-                ..Default::default()
+                firmware: None,
+                cmdline: None,
+                initramfs: None,
+                #[cfg(feature = "igvm")]
+                igvm: None,
             }),
+            rate_limit_groups: None,
             disks: None,
             net: None,
             rng: RngConfig {
@@ -864,6 +898,8 @@ mod unit_tests {
                 iommu: false,
                 socket: None,
             },
+            #[cfg(target_arch = "x86_64")]
+            debug_console: DebugConsoleConfig::default(),
             devices: None,
             user_devices: None,
             vdpa: None,
@@ -1114,6 +1150,29 @@ mod unit_tests {
                 }"#,
                 true,
             ),
+            (
+                vec![
+                    "cloud-hypervisor",
+                    "--kernel",
+                    "/path/to/kernel",
+                    "--disk",
+                    "path=/path/to/disk/1,rate_limit_group=group0",
+                    "path=/path/to/disk/2,rate_limit_group=group0",
+                    "--rate-limit-group",
+                    "id=group0,bw_size=1000,bw_refill_time=100",
+                ],
+                r#"{
+                    "payload": {"kernel": "/path/to/kernel"},
+                    "disks": [
+                        {"path": "/path/to/disk/1", "rate_limit_group": "group0"},
+                        {"path": "/path/to/disk/2", "rate_limit_group": "group0"}
+                    ],
+                    "rate_limit_groups": [
+                        {"id": "group0", "rate_limiter_config": {"bandwidth": {"size": 1000, "one_time_burst": 0, "refill_time": 100}}}
+                    ]
+                }"#,
+                true,
+            ),
         ]
         .iter()
         .for_each(|(cli, openapi, equal)| {
@@ -1472,6 +1531,30 @@ mod unit_tests {
         });
     }
 
+    #[cfg(target_arch = "x86_64")]
+    #[test]
+    fn test_valid_vm_config_debug_console() {
+        [(
+            vec![
+                "cloud-hypervisor",
+                "--kernel",
+                "/path/to/kernel",
+                "--debug-console",
+                "tty,iobase=0xe9",
+            ],
+            // 233 == 0xe9
+            r#"{
+                "payload": {"kernel": "/path/to/kernel" },
+                "debug_console": {"mode": "Tty", "iobase": 233 }
+            }"#,
+            true,
+        )]
+        .iter()
+        .for_each(|(cli, openapi, equal)| {
+            compare_vm_config_cli_vs_json(cli, openapi, *equal);
+        });
+    }
+
     #[test]
     fn test_valid_vm_config_serial_console() {
         [

test_infra/Cargo.toml

@@ -7,10 +7,10 @@ edition = "2021"
 [dependencies]
 dirs = "5.0.0"
 epoll = "4.3.3"
-libc = "0.2.147"
-once_cell = "1.18.0"
-serde = { version = "1.0.168", features = ["rc", "derive"] }
-serde_json = "1.0.107"
+libc = "0.2.153"
+once_cell = "1.19.0"
+serde = { version = "1.0.196", features = ["rc", "derive"] }
+serde_json = "1.0.109"
 ssh2 = { version = "0.9.4", features = ["vendored-openssl"] }
 vmm-sys-util = "0.12.1"
 wait-timeout = "0.2.0"

tests/integration.rs

@@ -53,6 +53,7 @@ use x86_64::*;
 
 #[cfg(target_arch = "aarch64")]
 mod aarch64 {
+    pub const BIONIC_IMAGE_NAME: &str = "bionic-server-cloudimg-arm64.raw";
     pub const FOCAL_IMAGE_NAME: &str = "focal-server-cloudimg-arm64-custom-20210929-0.raw";
     pub const FOCAL_IMAGE_UPDATE_KERNEL_NAME: &str =
         "focal-server-cloudimg-arm64-custom-20210929-0-update-kernel.raw";
@@ -2574,7 +2575,63 @@ mod common_parallel {
 
         let _ = child.kill();
         let output = child.wait_with_output().unwrap();
+        handle_child_output(r, &output);
+    }
 
+    #[test]
+    fn test_virtio_queue_affinity() {
+        let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string());
+        let guest = Guest::new(Box::new(focal));
+
+        // We need the host to have at least 4 CPUs if we want to be able
+        // to run this test.
+        let host_cpus_count = exec_host_command_output("nproc");
+        assert!(
+            String::from_utf8_lossy(&host_cpus_count.stdout)
+                .trim()
+                .parse::<u16>()
+                .unwrap_or(0)
+                >= 4
+        );
+
+        let mut child = GuestCommand::new(&guest)
+            .args(["--cpus", "boot=4"])
+            .args(["--memory", "size=512M"])
+            .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()])
+            .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE])
+            .args([
+                "--disk",
+                format!(
+                    "path={}",
+                    guest.disk_config.disk(DiskType::OperatingSystem).unwrap()
+                )
+                .as_str(),
+                format!(
+                    "path={},num_queues=4,queue_affinity=[0@[0,2],1@[1,3],2@[1],3@[3]]",
+                    guest.disk_config.disk(DiskType::CloudInit).unwrap()
+                )
+                .as_str(),
+            ])
+            .default_net()
+            .capture_output()
+            .spawn()
+            .unwrap();
+
+        let r = std::panic::catch_unwind(|| {
+            guest.wait_vm_boot(None).unwrap();
+            let pid = child.id();
+            let taskset_q0 = exec_host_command_output(format!("taskset -pc $(ps -T -p {pid} | grep disk1_q0 | xargs | cut -f 2 -d \" \") | cut -f 6 -d \" \"").as_str());
+            assert_eq!(String::from_utf8_lossy(&taskset_q0.stdout).trim(), "0,2");
+            let taskset_q1 = exec_host_command_output(format!("taskset -pc $(ps -T -p {pid} | grep disk1_q1 | xargs | cut -f 2 -d \" \") | cut -f 6 -d \" \"").as_str());
+            assert_eq!(String::from_utf8_lossy(&taskset_q1.stdout).trim(), "1,3");
+            let taskset_q2 = exec_host_command_output(format!("taskset -pc $(ps -T -p {pid} | grep disk1_q2 | xargs | cut -f 2 -d \" \") | cut -f 6 -d \" \"").as_str());
+            assert_eq!(String::from_utf8_lossy(&taskset_q2.stdout).trim(), "1");
+            let taskset_q3 = exec_host_command_output(format!("taskset -pc $(ps -T -p {pid} | grep disk1_q3 | xargs | cut -f 2 -d \" \") | cut -f 6 -d \" \"").as_str());
+            assert_eq!(String::from_utf8_lossy(&taskset_q3.stdout).trim(), "3");
+        });
+
+        let _ = child.kill();
+        let output = child.wait_with_output().unwrap();
         handle_child_output(r, &output);
     }
 
@@ -6091,7 +6148,7 @@ mod common_parallel {
             .unwrap();
 
         // Wait for the VM to be restored
-        thread::sleep(std::time::Duration::new(20, 0));
+        thread::sleep(std::time::Duration::new(10, 0));
         let expected_events = [
             &MetaEvent {
                 event: "starting".to_string(),
@@ -9596,44 +9653,51 @@ mod live_migration {
         }
 
         #[test]
+        #[ignore = "See #6134"]
         fn test_live_upgrade_basic() {
             _test_live_migration(true, false)
         }
 
         #[test]
+        #[ignore = "See #6134"]
         fn test_live_upgrade_local() {
             _test_live_migration(true, true)
         }
 
         #[test]
-        #[cfg(target_arch = "aarch64")] // see: #6272
+        #[ignore = "See #6134"]
         #[cfg(not(feature = "mshv"))]
         fn test_live_upgrade_numa() {
             _test_live_migration_numa(true, false)
         }
 
         #[test]
+        #[ignore = "See #6134"]
         #[cfg(not(feature = "mshv"))]
         fn test_live_upgrade_numa_local() {
             _test_live_migration_numa(true, true)
         }
 
         #[test]
+        #[ignore = "See #6134"]
         fn test_live_upgrade_watchdog() {
             _test_live_migration_watchdog(true, false)
         }
 
         #[test]
+        #[ignore = "See #6134"]
         fn test_live_upgrade_watchdog_local() {
             _test_live_migration_watchdog(true, true)
         }
 
         #[test]
+        #[ignore = "See #6134"]
         fn test_live_upgrade_balloon() {
             _test_live_migration_balloon(true, false)
         }
 
         #[test]
+        #[ignore = "See #6134"]
         fn test_live_upgrade_balloon_local() {
             _test_live_migration_balloon(true, true)
         }
@@ -9823,9 +9887,8 @@ mod rate_limiter {
         _test_rate_limiter_net(false);
     }
 
-    fn _test_rate_limiter_block(bandwidth: bool) {
+    fn _test_rate_limiter_block(bandwidth: bool, num_queues: u32) {
         let test_timeout = 10;
-        let num_queues = 1;
         let fio_ops = FioOps::RandRW;
 
         let bw_size = if bandwidth {
@@ -9852,11 +9915,11 @@ mod rate_limiter {
 
         let test_blk_params = if bandwidth {
             format!(
-                "path={blk_rate_limiter_test_img},bw_size={bw_size},bw_refill_time={bw_refill_time}"
+                "path={blk_rate_limiter_test_img},num_queues={num_queues},bw_size={bw_size},bw_refill_time={bw_refill_time}"
             )
         } else {
             format!(
-                "path={blk_rate_limiter_test_img},ops_size={bw_size},ops_refill_time={bw_refill_time}"
+                "path={blk_rate_limiter_test_img},num_queues={num_queues},ops_size={bw_size},ops_refill_time={bw_refill_time}"
             )
         };
 
@@ -9909,13 +9972,137 @@ mod rate_limiter {
         handle_child_output(r, &output);
     }
 
+    fn _test_rate_limiter_group_block(bandwidth: bool, num_queues: u32, num_disks: u32) {
+        let test_timeout = 10;
+        let fio_ops = FioOps::RandRW;
+
+        let bw_size = if bandwidth {
+            10485760_u64 // bytes
+        } else {
+            100_u64 // I/O
+        };
+        let bw_refill_time = 100; // ms
+        let limit_rate = (bw_size * 1000) as f64 / bw_refill_time as f64;
+
+        let focal = UbuntuDiskConfig::new(FOCAL_IMAGE_NAME.to_string());
+        let guest = Guest::new(Box::new(focal));
+        let api_socket = temp_api_path(&guest.tmp_dir);
+        let test_img_dir = TempDir::new_with_prefix("/var/tmp/ch").unwrap();
+
+        let rate_limit_group_arg = if bandwidth {
+            format!("id=group0,bw_size={bw_size},bw_refill_time={bw_refill_time}")
+        } else {
+            format!("id=group0,ops_size={bw_size},ops_refill_time={bw_refill_time}")
+        };
+
+        let mut disk_args = vec![
+            "--disk".to_string(),
+            format!(
+                "path={}",
+                guest.disk_config.disk(DiskType::OperatingSystem).unwrap()
+            ),
+            format!(
+                "path={}",
+                guest.disk_config.disk(DiskType::CloudInit).unwrap()
+            ),
+        ];
+
+        for i in 0..num_disks {
+            let test_img_path = String::from(
+                test_img_dir
+                    .as_path()
+                    .join(format!("blk{}.img", i))
+                    .to_str()
+                    .unwrap(),
+            );
+
+            assert!(exec_host_command_output(&format!(
+                "dd if=/dev/zero of={test_img_path} bs=1M count=1024"
+            ))
+            .status
+            .success());
+
+            disk_args.push(format!(
+                "path={test_img_path},num_queues={num_queues},rate_limit_group=group0"
+            ));
+        }
+
+        let mut child = GuestCommand::new(&guest)
+            .args(["--cpus", &format!("boot={}", num_queues * num_disks)])
+            .args(["--memory", "size=4G"])
+            .args(["--kernel", direct_kernel_boot_path().to_str().unwrap()])
+            .args(["--cmdline", DIRECT_KERNEL_BOOT_CMDLINE])
+            .args(["--rate-limit-group", &rate_limit_group_arg])
+            .args(disk_args)
+            .default_net()
+            .args(["--api-socket", &api_socket])
+            .capture_output()
+            .spawn()
+            .unwrap();
+
+        let r = std::panic::catch_unwind(|| {
+            guest.wait_vm_boot(None).unwrap();
+
+            let mut fio_command = format!(
+                "sudo fio --name=global --output-format=json \
+                --direct=1 --bs=4k --ioengine=io_uring --iodepth=64 \
+                --rw={fio_ops} --runtime={test_timeout} --numjobs={num_queues}"
+            );
+
+            // Generate additional argument for each disk:
+            // --name=job0 --filename=/dev/vdc \
+            // --name=job1 --filename=/dev/vdd \
+            // --name=job2 --filename=/dev/vde \
+            // ...
+            for i in 0..num_disks {
+                let c: char = 'c';
+                let arg = format!(
+                    " --name=job{i} --filename=/dev/vd{}",
+                    char::from_u32((c as u32) + i).unwrap()
+                );
+                fio_command += &arg;
+            }
+            let output = guest.ssh_command(&fio_command).unwrap();
+
+            // Parse fio output
+            let measured_rate = if bandwidth {
+                parse_fio_output(&output, &fio_ops, num_queues * num_disks).unwrap()
+            } else {
+                parse_fio_output_iops(&output, &fio_ops, num_queues * num_disks).unwrap()
+            };
+            assert!(check_rate_limit(measured_rate, limit_rate, 0.1));
+        });
+
+        let _ = child.kill();
+        let output = child.wait_with_output().unwrap();
+        handle_child_output(r, &output);
+    }
+
     #[test]
     fn test_rate_limiter_block_bandwidth() {
-        _test_rate_limiter_block(true)
+        _test_rate_limiter_block(true, 1);
+        _test_rate_limiter_block(true, 2)
+    }
+
+    #[test]
+    fn test_rate_limiter_group_block_bandwidth() {
+        _test_rate_limiter_group_block(true, 1, 1);
+        _test_rate_limiter_group_block(true, 2, 1);
+        _test_rate_limiter_group_block(true, 1, 2);
+        _test_rate_limiter_group_block(true, 2, 2);
     }
 
     #[test]
     fn test_rate_limiter_block_iops() {
-        _test_rate_limiter_block(false)
+        _test_rate_limiter_block(false, 1);
+        _test_rate_limiter_block(false, 2);
+    }
+
+    #[test]
+    fn test_rate_limiter_group_block_iops() {
+        _test_rate_limiter_group_block(false, 1, 1);
+        _test_rate_limiter_group_block(false, 2, 1);
+        _test_rate_limiter_group_block(false, 1, 2);
+        _test_rate_limiter_group_block(false, 2, 2);
     }
 }

tracer/Cargo.toml

@@ -5,11 +5,11 @@ authors = ["The Cloud Hypervisor Authors"]
 edition = "2021"
 
 [dependencies]
-libc = "0.2.147"
+libc = "0.2.153"
 log = "0.4.20"
-once_cell = "1.18.0"
-serde = { version = "1.0.168", features = ["rc", "derive"] }
-serde_json = "1.0.107"
+once_cell = "1.19.0"
+serde = { version = "1.0.196", features = ["rc", "derive"] }
+serde_json = "1.0.109"
 
 [features]
 tracing = []

vhost_user_block/Cargo.toml

@@ -8,14 +8,14 @@ build = "../build.rs"
 [dependencies]
 clap = { version = "4.4.7", features = ["wrap_help","cargo"] }
 block = { path = "../block" }
-env_logger = "0.10.0"
+env_logger = "0.10.1"
 epoll = "4.3.3"
-libc = "0.2.147"
+libc = "0.2.153"
 log = "0.4.20"
 option_parser = { path = "../option_parser" }
-vhost = { version = "0.11.0", features = ["vhost-user-backend"] }
-vhost-user-backend = "0.15.0"
+vhost = { version = "0.10.0", features = ["vhost-user-backend"] }
+vhost-user-backend = "0.13.1"
 virtio-bindings = "0.2.0"
-virtio-queue = "0.12.0"
+virtio-queue = "0.11.0"
 vm-memory = "0.14.0"
 vmm-sys-util = "0.12.1"

vhost_user_block/src/lib.rs

@@ -33,18 +33,16 @@ use std::vec::Vec;
 use std::{convert, error, fmt, io};
 use vhost::vhost_user::message::*;
 use vhost::vhost_user::Listener;
-use vhost_user_backend::{
-    bitmap::BitmapMmapRegion, VhostUserBackendMut, VhostUserDaemon, VringRwLock, VringState, VringT,
-};
+use vhost_user_backend::{VhostUserBackendMut, VhostUserDaemon, VringRwLock, VringState, VringT};
 use virtio_bindings::virtio_blk::*;
 use virtio_bindings::virtio_config::VIRTIO_F_VERSION_1;
 use virtio_bindings::virtio_ring::VIRTIO_RING_F_EVENT_IDX;
 use virtio_queue::QueueT;
 use vm_memory::GuestAddressSpace;
-use vm_memory::{ByteValued, Bytes, GuestMemoryAtomic};
+use vm_memory::{bitmap::AtomicBitmap, ByteValued, Bytes, GuestMemoryAtomic};
 use vmm_sys_util::{epoll::EventSet, eventfd::EventFd};
 
-type GuestMemoryMmap = vm_memory::GuestMemoryMmap<BitmapMmapRegion>;
+type GuestMemoryMmap = vm_memory::GuestMemoryMmap<AtomicBitmap>;
 
 const SECTOR_SHIFT: u8 = 9;
 const SECTOR_SIZE: u64 = 0x01 << SECTOR_SHIFT;
@@ -303,7 +301,7 @@ impl VhostUserBlkBackend {
 }
 
 impl VhostUserBackendMut for VhostUserBlkBackend {
-    type Bitmap = BitmapMmapRegion;
+    type Bitmap = AtomicBitmap;
     type Vring = VringRwLock<GuestMemoryAtomic<GuestMemoryMmap>>;
 
     fn num_queues(&self) -> usize {

vhost_user_net/Cargo.toml

@@ -7,14 +7,14 @@ build = "../build.rs"
 
 [dependencies]
 clap = { version = "4.4.7", features = ["wrap_help","cargo"] }
-env_logger = "0.10.0"
+env_logger = "0.10.1"
 epoll = "4.3.3"
-libc = "0.2.147"
+libc = "0.2.153"
 log = "0.4.20"
 net_util = { path = "../net_util" }
 option_parser = { path = "../option_parser" }
-vhost = { version = "0.11.0", features = ["vhost-user-backend"] }
-vhost-user-backend = "0.15.0"
+vhost = { version = "0.10.0", features = ["vhost-user-backend"] }
+vhost-user-backend = "0.13.1"
 virtio-bindings = "0.2.0"
 vm-memory = "0.14.0"
 vmm-sys-util = "0.12.1"

vhost_user_net/src/lib.rs

@@ -23,15 +23,14 @@ use std::sync::{Arc, Mutex, RwLock};
 use std::vec::Vec;
 use vhost::vhost_user::message::*;
 use vhost::vhost_user::Listener;
-use vhost_user_backend::bitmap::BitmapMmapRegion;
 use vhost_user_backend::{VhostUserBackendMut, VhostUserDaemon, VringRwLock, VringT};
 use virtio_bindings::virtio_config::{VIRTIO_F_NOTIFY_ON_EMPTY, VIRTIO_F_VERSION_1};
 use virtio_bindings::virtio_net::*;
 use vm_memory::GuestAddressSpace;
-use vm_memory::GuestMemoryAtomic;
+use vm_memory::{bitmap::AtomicBitmap, GuestMemoryAtomic};
 use vmm_sys_util::{epoll::EventSet, eventfd::EventFd};
 
-type GuestMemoryMmap = vm_memory::GuestMemoryMmap<BitmapMmapRegion>;
+type GuestMemoryMmap = vm_memory::GuestMemoryMmap<AtomicBitmap>;
 
 pub type Result<T> = std::result::Result<T, Error>;
 type VhostUserBackendResult<T> = std::result::Result<T, std::io::Error>;
@@ -160,7 +159,7 @@ impl VhostUserNetBackend {
 }
 
 impl VhostUserBackendMut for VhostUserNetBackend {
-    type Bitmap = BitmapMmapRegion;
+    type Bitmap = AtomicBitmap;
     type Vring = VringRwLock<GuestMemoryAtomic<GuestMemoryMmap>>;
 
     fn num_queues(&self) -> usize {

virtio-devices/Cargo.toml

@@ -8,28 +8,28 @@ edition = "2021"
 default = []
 
 [dependencies]
-anyhow = "1.0.75"
+anyhow = "1.0.79"
 arc-swap = "1.5.1"
 block = { path = "../block" }
 byteorder = "1.4.3"
 epoll = "4.3.3"
 event_monitor = { path = "../event_monitor" }
-libc = "0.2.147"
+libc = "0.2.153"
 log = "0.4.20"
 net_gen = { path = "../net_gen" }
 net_util = { path = "../net_util" }
 pci = { path = "../pci" }
 rate_limiter = { path = "../rate_limiter" }
 seccompiler = "0.4.0"
-serde = { version = "1.0.168", features = ["derive"] }
-serde_json = "1.0.107"
+serde = { version = "1.0.196", features = ["derive"] }
+serde_json = "1.0.109"
 serial_buffer = { path = "../serial_buffer" }
 thiserror = "1.0.52"
 versionize = "0.2.0"
 versionize_derive = "0.1.6"
-vhost = { version = "0.11.0", features = ["vhost-user-frontend", "vhost-user-backend", "vhost-kern", "vhost-vdpa"] }
+vhost = { version = "0.10.0", features = ["vhost-user-frontend", "vhost-user-backend", "vhost-kern", "vhost-vdpa"] }
 virtio-bindings = { version = "0.2.0", features = ["virtio-v5_0_0"] }
-virtio-queue = "0.12.0"
+virtio-queue = "0.11.0"
 vm-allocator = { path = "../vm-allocator" }
 vm-device = { path = "../vm-device" }
 vm-memory = { version = "0.14.0", features = ["backend-mmap", "backend-atomic", "backend-bitmap"] }

virtio-devices/src/block.rs

@@ -10,9 +10,8 @@
 
 use super::Error as DeviceError;
 use super::{
-    ActivateError, ActivateResult, EpollHelper, EpollHelperError, EpollHelperHandler,
-    RateLimiterConfig, VirtioCommon, VirtioDevice, VirtioDeviceType, VirtioInterruptType,
-    EPOLL_HELPER_EVENT_LAST,
+    ActivateError, ActivateResult, EpollHelper, EpollHelperError, EpollHelperHandler, VirtioCommon,
+    VirtioDevice, VirtioDeviceType, VirtioInterruptType, EPOLL_HELPER_EVENT_LAST,
 };
 use crate::seccomp_filters::Thread;
 use crate::thread_helper::spawn_virtio_thread;
@@ -23,8 +22,10 @@ use block::{
     async_io::AsyncIo, async_io::AsyncIoError, async_io::DiskFile, build_serial, Request,
     RequestType, VirtioBlockConfig,
 };
-use rate_limiter::{RateLimiter, TokenType};
+use rate_limiter::group::{RateLimiterGroup, RateLimiterGroupHandle};
+use rate_limiter::TokenType;
 use seccompiler::SeccompAction;
+use std::collections::BTreeMap;
 use std::collections::VecDeque;
 use std::io;
 use std::num::Wrapping;
@@ -131,9 +132,10 @@ struct BlockEpollHandler {
     counters: BlockCounters,
     queue_evt: EventFd,
     inflight_requests: VecDeque<(u16, Request)>,
-    rate_limiter: Option<RateLimiter>,
+    rate_limiter: Option<RateLimiterGroupHandle>,
     access_platform: Option<Arc<dyn AccessPlatform>>,
     read_only: bool,
+    host_cpus: Option<Vec<usize>>,
 }
 
 impl BlockEpollHandler {
@@ -408,6 +410,41 @@ impl BlockEpollHandler {
             })
     }
 
+    fn set_queue_thread_affinity(&self) {
+        // Prepare the CPU set the current queue thread is expected to run onto.
+        let cpuset = self.host_cpus.as_ref().map(|host_cpus| {
+            // SAFETY: all zeros is a valid pattern
+            let mut cpuset: libc::cpu_set_t = unsafe { std::mem::zeroed() };
+            // SAFETY: FFI call, trivially safe
+            unsafe { libc::CPU_ZERO(&mut cpuset) };
+            for host_cpu in host_cpus {
+                // SAFETY: FFI call, trivially safe
+                unsafe { libc::CPU_SET(*host_cpu, &mut cpuset) };
+            }
+            cpuset
+        });
+
+        // Schedule the thread to run on the expected CPU set
+        if let Some(cpuset) = cpuset.as_ref() {
+            // SAFETY: FFI call with correct arguments
+            let ret = unsafe {
+                libc::sched_setaffinity(
+                    0,
+                    std::mem::size_of::<libc::cpu_set_t>(),
+                    cpuset as *const libc::cpu_set_t,
+                )
+            };
+
+            if ret != 0 {
+                error!(
+                    "Failed scheduling the virtqueue thread {} on the expected CPU set: {}",
+                    self.queue_index,
+                    io::Error::last_os_error()
+                )
+            }
+        }
+    }
+
     fn run(
         &mut self,
         paused: Arc<AtomicBool>,
@@ -419,6 +456,7 @@ impl BlockEpollHandler {
         if let Some(rate_limiter) = &self.rate_limiter {
             helper.add_event(rate_limiter.as_raw_fd(), RATE_LIMITER_EVENT)?;
         }
+        self.set_queue_thread_affinity();
         helper.run(paused, paused_sync, self)?;
 
         Ok(())
@@ -507,10 +545,11 @@ pub struct Block {
     writeback: Arc<AtomicBool>,
     counters: BlockCounters,
     seccomp_action: SeccompAction,
-    rate_limiter_config: Option<RateLimiterConfig>,
+    rate_limiter: Option<Arc<RateLimiterGroup>>,
     exit_evt: EventFd,
     read_only: bool,
     serial: Vec<u8>,
+    queue_affinity: BTreeMap<u16, Vec<usize>>,
 }
 
 #[derive(Versionize)]
@@ -537,9 +576,10 @@ impl Block {
         queue_size: u16,
         serial: Option<String>,
         seccomp_action: SeccompAction,
-        rate_limiter_config: Option<RateLimiterConfig>,
+        rate_limiter: Option<Arc<RateLimiterGroup>>,
         exit_evt: EventFd,
         state: Option<BlockState>,
+        queue_affinity: BTreeMap<u16, Vec<usize>>,
     ) -> io::Result<Self> {
         let (disk_nsectors, avail_features, acked_features, config, paused) =
             if let Some(state) = state {
@@ -639,10 +679,11 @@ impl Block {
             writeback: Arc::new(AtomicBool::new(true)),
             counters: BlockCounters::default(),
             seccomp_action,
-            rate_limiter_config,
+            rate_limiter,
             exit_evt,
             read_only,
             serial,
+            queue_affinity,
         })
     }
 
@@ -746,15 +787,10 @@ impl VirtioDevice for Block {
             let (_, queue, queue_evt) = queues.remove(0);
             let queue_size = queue.size();
             let (kill_evt, pause_evt) = self.common.dup_eventfds();
-
-            let rate_limiter: Option<RateLimiter> = self
-                .rate_limiter_config
-                .map(RateLimiterConfig::try_into)
-                .transpose()
-                .map_err(ActivateError::CreateRateLimiter)?;
+            let queue_idx = i as u16;
 
             let mut handler = BlockEpollHandler {
-                queue_index: i as u16,
+                queue_index: queue_idx,
                 queue,
                 mem: mem.clone(),
                 disk_image: self
@@ -776,9 +812,15 @@ impl VirtioDevice for Block {
                 // This gives head room for systems with slower I/O without
                 // compromising the cost of the reallocation or memory overhead
                 inflight_requests: VecDeque::with_capacity(64),
-                rate_limiter,
+                rate_limiter: self
+                    .rate_limiter
+                    .as_ref()
+                    .map(|r| r.new_handle())
+                    .transpose()
+                    .unwrap(),
                 access_platform: self.common.access_platform.clone(),
                 read_only: self.read_only,
+                host_cpus: self.queue_affinity.get(&queue_idx).cloned(),
             };
 
             let paused = self.common.paused.clone();

virtio-devices/src/net.rs

@@ -16,11 +16,12 @@ use crate::thread_helper::spawn_virtio_thread;
 use crate::GuestMemoryMmap;
 use crate::VirtioInterrupt;
 use anyhow::anyhow;
+#[cfg(not(fuzzing))]
+use net_util::virtio_features_to_tap_offload;
 use net_util::CtrlQueue;
 use net_util::{
-    build_net_config_space, build_net_config_space_with_mq, open_tap,
-    virtio_features_to_tap_offload, MacAddr, NetCounters, NetQueuePair, OpenTapError, RxVirtio,
-    Tap, TapError, TxVirtio, VirtioNetConfig,
+    build_net_config_space, build_net_config_space_with_mq, open_tap, MacAddr, NetCounters,
+    NetQueuePair, OpenTapError, RxVirtio, Tap, TapError, TxVirtio, VirtioNetConfig,
 };
 use seccompiler::SeccompAction;
 use std::net::Ipv4Addr;

virtio-devices/src/seccomp_filters.rs

@@ -96,6 +96,7 @@ fn virtio_block_thread_rules() -> Vec<(i64, Vec<SeccompRule>)> {
         (libc::SYS_pwritev, vec![]),
         (libc::SYS_pwrite64, vec![]),
         (libc::SYS_sched_getaffinity, vec![]),
+        (libc::SYS_sched_setaffinity, vec![]),
         (libc::SYS_set_robust_list, vec![]),
         (libc::SYS_timerfd_settime, vec![]),
     ]

vm-allocator/Cargo.toml

@@ -5,7 +5,7 @@ authors = ["The Chromium OS Authors"]
 edition = "2021"
 
 [dependencies]
-libc = "0.2.147"
+libc = "0.2.153"
 vm-memory = "0.14.0"
 
 [target.'cfg(target_arch = "aarch64")'.dependencies]

vm-device/Cargo.toml

@@ -10,10 +10,10 @@ kvm = ["vfio-ioctls/kvm"]
 mshv = ["vfio-ioctls/mshv"]
 
 [dependencies]
-anyhow = "1.0.75"
+anyhow = "1.0.79"
 hypervisor = { path = "../hypervisor" }
-thiserror = "1.0.40"
-serde = { version = "1.0.168", features = ["rc", "derive"] }
+thiserror = "1.0.52"
+serde = { version = "1.0.196", features = ["rc", "derive"] }
 vfio-ioctls = { git = "https://github.com/rust-vmm/vfio", branch = "main", default-features = false }
 vm-memory = { version = "0.14.0", features = ["backend-mmap"] }
 vmm-sys-util = "0.12.1"

vm-migration/Cargo.toml

@@ -5,9 +5,9 @@ authors = ["The Cloud Hypervisor Authors"]
 edition = "2021"
 
 [dependencies]
-anyhow = "1.0.75"
-thiserror = "1.0.40"
-serde = { version = "1.0.168", features = ["rc", "derive"] }
+anyhow = "1.0.79"
+thiserror = "1.0.52"
+serde = { version = "1.0.196", features = ["rc", "derive"] }
 serde_json = "1.0.109"
 versionize = "0.2.0"
 versionize_derive = "0.1.6"

vm-migration/src/lib.rs

@@ -12,8 +12,8 @@ use versionize::{VersionMap, Versionize};
 pub mod protocol;
 
 /// Global VMM version for versioning
-const MAJOR_VERSION: u16 = 37;
-const MINOR_VERSION: u16 = 1;
+const MAJOR_VERSION: u16 = 38;
+const MINOR_VERSION: u16 = 0;
 const VMM_VERSION: u16 = MAJOR_VERSION << 12 | MINOR_VERSION & 0b1111;
 
 pub trait VersionMapped {

vm-virtio/Cargo.toml

@@ -9,5 +9,5 @@ default = []
 
 [dependencies]
 log = "0.4.20"
-virtio-queue = "0.12.0"
+virtio-queue = "0.11.0"
 vm-memory = { version = "0.14.0", features = ["backend-mmap", "backend-atomic", "backend-bitmap"] }

vmm/Cargo.toml

@@ -18,41 +18,42 @@ tracing = ["tracer/tracing"]
 
 [dependencies]
 acpi_tables = { git = "https://github.com/rust-vmm/acpi_tables", branch = "main"  }
-anyhow = "1.0.75"
+anyhow = "1.0.79"
 arc-swap = "1.5.1"
 arch = { path = "../arch" }
 bitflags = "2.4.1"
 block = { path = "../block" }
-blocking = { version = "1.3.0", optional = true }
+blocking = { version = "1.5.1", optional = true }
 cfg-if = "1.0.0"
 clap = "4.4.7"
 devices = { path = "../devices" }
 epoll = "4.3.3"
 event_monitor = { path = "../event_monitor" }
 flume = "0.10.14"
-futures = { version = "0.3.27", optional = true }
-gdbstub = { version = "0.7.0", optional = true }
+futures = { version = "0.3.30", optional = true }
+gdbstub = { version = "0.7.1", optional = true }
 gdbstub_arch = { version = "0.3.0", optional = true }
 hex = { version = "0.4.3", optional = true }
 hypervisor = { path = "../hypervisor" }
 igvm_defs = { git = "https://github.com/microsoft/igvm", branch = "main", package = "igvm_defs", optional  = true }
 igvm_parser = { git = "https://github.com/microsoft/igvm", branch = "main", package = "igvm", optional  = true }
-libc = "0.2.147"
+libc = "0.2.153"
 linux-loader = { version = "0.11.0", features = ["elf", "bzimage", "pe"] }
 log = "0.4.20"
 micro_http = { git = "https://github.com/firecracker-microvm/micro-http", branch = "main" }
 mshv-bindings = { git = "https://github.com/rust-vmm/mshv", branch = "main", features = ["with-serde", "fam-wrappers"], optional  = true }
 net_util = { path = "../net_util" }
-once_cell = "1.18.0"
+once_cell = "1.19.0"
 option_parser = { path = "../option_parser" }
 pci = { path = "../pci" }
 range_map_vec = { version = "0.1.0", optional = true }
+rate_limiter = { path = "../rate_limiter" }
 seccompiler = "0.4.0"
-serde = { version = "1.0.168", features = ["rc", "derive"] }
-serde_json = "1.0.107"
+serde = { version = "1.0.196", features = ["rc", "derive"] }
+serde_json = "1.0.109"
 serial_buffer = { path = "../serial_buffer" }
 signal-hook = "0.3.17"
-thiserror = "1.0.40"
+thiserror = "1.0.52"
 tracer = { path = "../tracer" }
 uuid = "1.3.4"
 versionize = "0.2.0"
@@ -60,7 +61,7 @@ versionize_derive = "0.1.6"
 vfio-ioctls = { git = "https://github.com/rust-vmm/vfio", branch = "main", default-features = false }
 vfio_user = { git = "https://github.com/rust-vmm/vfio-user", branch = "main" }
 virtio-devices = { path = "../virtio-devices" }
-virtio-queue = "0.12.0"
+virtio-queue = "0.11.0"
 vm-allocator = { path = "../vm-allocator" }
 vm-device = { path = "../vm-device" }
 vm-memory = { version = "0.14.0", features = ["backend-mmap", "backend-atomic", "backend-bitmap"] }
@@ -68,4 +69,4 @@ vm-migration = { path = "../vm-migration" }
 vm-virtio = { path = "../vm-virtio" }
 vmm-sys-util = { version = "0.12.1", features = ["with-serde"] }
 zbus = { version = "3.11.1", optional = true }
-zerocopy = { version = "0.7.21", features = ["alloc","derive"] }
+zerocopy = { version = "0.7.32", features = ["alloc","derive"] }

vmm/src/api/dbus/mod.rs

@@ -2,7 +2,15 @@
 //
 // SPDX-License-Identifier: Apache-2.0
 //
-use super::{ApiRequest, VmAction};
+use super::{ApiAction, ApiRequest};
+#[cfg(all(target_arch = "x86_64", feature = "guest_debug"))]
+use crate::api::VmCoredump;
+use crate::api::{
+    AddDisk, Body, VmAddDevice, VmAddFs, VmAddNet, VmAddPmem, VmAddUserDevice, VmAddVdpa,
+    VmAddVsock, VmBoot, VmCounters, VmCreate, VmDelete, VmInfo, VmPause, VmPowerButton, VmReboot,
+    VmReceiveMigration, VmRemoveDevice, VmResize, VmResizeZone, VmRestore, VmResume,
+    VmSendMigration, VmShutdown, VmSnapshot, VmmPing, VmmShutdown,
+};
 use crate::seccomp_filters::{get_seccomp_filter, Thread};
 use crate::{Error as VmmError, Result as VmmResult};
 use crate::{NetConfig, VmConfig};
@@ -78,11 +86,15 @@ impl DBusApi {
             .map_err(|err| fdo::Error::IOError(format!("{err:?}")))
     }
 
-    async fn vm_action(&self, action: VmAction) -> Result<Optional<String>> {
+    async fn vm_action<Action: ApiAction<ResponseBody = Option<Body>>>(
+        &self,
+        action: &'static Action,
+        body: Action::RequestBody,
+    ) -> Result<Optional<String>> {
         let api_sender = self.clone_api_sender().await;
         let api_notifier = self.clone_api_notifier()?;
 
-        let result = blocking::unblock(move || super::vm_action(api_notifier, api_sender, action))
+        let result = blocking::unblock(move || action.send(api_notifier, api_sender, body))
             .await
             .map_err(api_error)?
             // We're using `from_utf8_lossy` here to not deal with the
@@ -99,7 +111,7 @@ impl DBusApi {
         let api_sender = self.clone_api_sender().await;
         let api_notifier = self.clone_api_notifier()?;
 
-        let result = blocking::unblock(move || super::vmm_ping(api_notifier, api_sender))
+        let result = blocking::unblock(move || VmmPing.send(api_notifier, api_sender, ()))
             .await
             .map_err(api_error)?;
         serde_json::to_string(&result).map_err(api_error)
@@ -109,26 +121,24 @@ impl DBusApi {
         let api_sender = self.clone_api_sender().await;
         let api_notifier = self.clone_api_notifier()?;
 
-        blocking::unblock(move || super::vmm_shutdown(api_notifier, api_sender))
+        blocking::unblock(move || VmmShutdown.send(api_notifier, api_sender, ()))
             .await
             .map_err(api_error)
     }
 
     async fn vm_add_device(&self, device_config: String) -> Result<Optional<String>> {
         let device_config = serde_json::from_str(&device_config).map_err(api_error)?;
-        self.vm_action(VmAction::AddDevice(Arc::new(device_config)))
-            .await
+        self.vm_action(&VmAddDevice, device_config).await
     }
 
     async fn vm_add_disk(&self, disk_config: String) -> Result<Optional<String>> {
         let disk_config = serde_json::from_str(&disk_config).map_err(api_error)?;
-        self.vm_action(VmAction::AddDisk(Arc::new(disk_config)))
-            .await
+        self.vm_action(&AddDisk, disk_config).await
     }
 
     async fn vm_add_fs(&self, fs_config: String) -> Result<Optional<String>> {
         let fs_config = serde_json::from_str(&fs_config).map_err(api_error)?;
-        self.vm_action(VmAction::AddFs(Arc::new(fs_config))).await
+        self.vm_action(&VmAddFs, fs_config).await
     }
 
     async fn vm_add_net(&self, net_config: String) -> Result<Optional<String>> {
@@ -137,35 +147,31 @@ impl DBusApi {
             warn!("Ignoring FDs sent via the D-Bus request body");
             net_config.fds = None;
         }
-        self.vm_action(VmAction::AddNet(Arc::new(net_config))).await
+        self.vm_action(&VmAddNet, net_config).await
     }
 
     async fn vm_add_pmem(&self, pmem_config: String) -> Result<Optional<String>> {
         let pmem_config = serde_json::from_str(&pmem_config).map_err(api_error)?;
-        self.vm_action(VmAction::AddPmem(Arc::new(pmem_config)))
-            .await
+        self.vm_action(&VmAddPmem, pmem_config).await
     }
 
     async fn vm_add_user_device(&self, vm_add_user_device: String) -> Result<Optional<String>> {
         let vm_add_user_device = serde_json::from_str(&vm_add_user_device).map_err(api_error)?;
-        self.vm_action(VmAction::AddUserDevice(Arc::new(vm_add_user_device)))
-            .await
+        self.vm_action(&VmAddUserDevice, vm_add_user_device).await
     }
 
     async fn vm_add_vdpa(&self, vdpa_config: String) -> Result<Optional<String>> {
         let vdpa_config = serde_json::from_str(&vdpa_config).map_err(api_error)?;
-        self.vm_action(VmAction::AddVdpa(Arc::new(vdpa_config)))
-            .await
+        self.vm_action(&VmAddVdpa, vdpa_config).await
     }
 
     async fn vm_add_vsock(&self, vsock_config: String) -> Result<Optional<String>> {
         let vsock_config = serde_json::from_str(&vsock_config).map_err(api_error)?;
-        self.vm_action(VmAction::AddVsock(Arc::new(vsock_config)))
-            .await
+        self.vm_action(&VmAddVsock, vsock_config).await
     }
 
     async fn vm_boot(&self) -> Result<()> {
-        self.vm_action(VmAction::Boot).await.map(|_| ())
+        self.vm_action(&VmBoot, ()).await.map(|_| ())
     }
 
     #[allow(unused_variables)]
@@ -176,7 +182,7 @@ impl DBusApi {
         #[cfg(all(target_arch = "x86_64", feature = "guest_debug"))]
         {
             let vm_coredump_data = serde_json::from_str(&vm_coredump_data).map_err(api_error)?;
-            self.vm_action(VmAction::Coredump(Arc::new(vm_coredump_data)))
+            self.vm_action(&VmCoredump, vm_coredump_data)
                 .await
                 .map(|_| ())
         }
@@ -188,7 +194,7 @@ impl DBusApi {
     }
 
     async fn vm_counters(&self) -> Result<Optional<String>> {
-        self.vm_action(VmAction::Counters).await
+        self.vm_action(&VmCounters, ()).await
     }
 
     async fn vm_create(&self, vm_config: String) -> Result<()> {
@@ -207,7 +213,7 @@ impl DBusApi {
         }
 
         blocking::unblock(move || {
-            super::vm_create(api_notifier, api_sender, Arc::new(Mutex::new(vm_config)))
+            VmCreate.send(api_notifier, api_sender, Arc::new(Mutex::new(vm_config)))
         })
         .await
         .map_err(api_error)?;
@@ -216,85 +222,81 @@ impl DBusApi {
     }
 
     async fn vm_delete(&self) -> Result<()> {
-        self.vm_action(VmAction::Delete).await.map(|_| ())
+        self.vm_action(&VmDelete, ()).await.map(|_| ())
     }
 
     async fn vm_info(&self) -> Result<String> {
         let api_sender = self.clone_api_sender().await;
         let api_notifier = self.clone_api_notifier()?;
 
-        let result = blocking::unblock(move || super::vm_info(api_notifier, api_sender))
+        let result = blocking::unblock(move || VmInfo.send(api_notifier, api_sender, ()))
             .await
             .map_err(api_error)?;
         serde_json::to_string(&result).map_err(api_error)
     }
 
     async fn vm_pause(&self) -> Result<()> {
-        self.vm_action(VmAction::Pause).await.map(|_| ())
+        self.vm_action(&VmPause, ()).await.map(|_| ())
     }
 
     async fn vm_power_button(&self) -> Result<()> {
-        self.vm_action(VmAction::PowerButton).await.map(|_| ())
+        self.vm_action(&VmPowerButton, ()).await.map(|_| ())
     }
 
     async fn vm_reboot(&self) -> Result<()> {
-        self.vm_action(VmAction::Reboot).await.map(|_| ())
+        self.vm_action(&VmReboot, ()).await.map(|_| ())
     }
 
     async fn vm_remove_device(&self, vm_remove_device: String) -> Result<()> {
         let vm_remove_device = serde_json::from_str(&vm_remove_device).map_err(api_error)?;
-        self.vm_action(VmAction::RemoveDevice(Arc::new(vm_remove_device)))
+        self.vm_action(&VmRemoveDevice, vm_remove_device)
             .await
             .map(|_| ())
     }
 
     async fn vm_resize(&self, vm_resize: String) -> Result<()> {
         let vm_resize = serde_json::from_str(&vm_resize).map_err(api_error)?;
-        self.vm_action(VmAction::Resize(Arc::new(vm_resize)))
-            .await
-            .map(|_| ())
+        self.vm_action(&VmResize, vm_resize).await.map(|_| ())
     }
 
     async fn vm_resize_zone(&self, vm_resize_zone: String) -> Result<()> {
         let vm_resize_zone = serde_json::from_str(&vm_resize_zone).map_err(api_error)?;
-        self.vm_action(VmAction::ResizeZone(Arc::new(vm_resize_zone)))
+        self.vm_action(&VmResizeZone, vm_resize_zone)
             .await
             .map(|_| ())
     }
 
     async fn vm_restore(&self, restore_config: String) -> Result<()> {
         let restore_config = serde_json::from_str(&restore_config).map_err(api_error)?;
-        self.vm_action(VmAction::Restore(Arc::new(restore_config)))
-            .await
-            .map(|_| ())
+        self.vm_action(&VmRestore, restore_config).await.map(|_| ())
     }
 
     async fn vm_receive_migration(&self, receive_migration_data: String) -> Result<()> {
         let receive_migration_data =
             serde_json::from_str(&receive_migration_data).map_err(api_error)?;
-        self.vm_action(VmAction::ReceiveMigration(Arc::new(receive_migration_data)))
+        self.vm_action(&VmReceiveMigration, receive_migration_data)
             .await
             .map(|_| ())
     }
 
     async fn vm_send_migration(&self, send_migration_data: String) -> Result<()> {
         let send_migration_data = serde_json::from_str(&send_migration_data).map_err(api_error)?;
-        self.vm_action(VmAction::SendMigration(Arc::new(send_migration_data)))
+        self.vm_action(&VmSendMigration, send_migration_data)
             .await
             .map(|_| ())
     }
 
     async fn vm_resume(&self) -> Result<()> {
-        self.vm_action(VmAction::Resume).await.map(|_| ())
+        self.vm_action(&VmResume, ()).await.map(|_| ())
     }
 
     async fn vm_shutdown(&self) -> Result<()> {
-        self.vm_action(VmAction::Shutdown).await.map(|_| ())
+        self.vm_action(&VmShutdown, ()).await.map(|_| ())
     }
 
     async fn vm_snapshot(&self, vm_snapshot_config: String) -> Result<()> {
         let vm_snapshot_config = serde_json::from_str(&vm_snapshot_config).map_err(api_error)?;
-        self.vm_action(VmAction::Snapshot(Arc::new(vm_snapshot_config)))
+        self.vm_action(&VmSnapshot, vm_snapshot_config)
             .await
             .map(|_| ())
     }