scripts: Remove downloads from integration test scripts

Assisted-by: Claude:Opus-4.6
Signed-off-by: Rob Bradford <rbradford@meta.com>

scripts/dev_cli.sh

@@ -583,7 +583,6 @@ cmd_tests() {
             "${common_env_args[@]}" \
             --env PARALLEL_INTEGRATION_TESTS_NUM="${PARALLEL_INTEGRATION_TESTS_NUM:-}" \
             --env LLVM_PROFILE_FILE="$LLVM_PROFILE_FILE" \
-            --env MIGRATABLE_VERSION="$MIGRATABLE_VERSION" \
             "$CTR_IMAGE" \
             dbus-run-session ./scripts/run_integration_tests_"$(uname -m)".sh "$@" || fix_dir_perms $? || exit $?
     fi

scripts/run_integration_tests_aarch64.sh

@@ -10,24 +10,25 @@ source "$(dirname "$0")"/common-aarch64.sh
 WORKLOADS_LOCK="$WORKLOADS_DIR/integration_test.lock"
 
 update_workloads() {
-    cp scripts/sha1sums-aarch64-common "$WORKLOADS_DIR"
-
     JAMMY_OS_RAW_IMAGE_NAME="jammy-server-cloudimg-arm64-custom-20220329-0.raw"
-    JAMMY_OS_RAW_IMAGE_DOWNLOAD_URL="https://ch-images.azureedge.net/$JAMMY_OS_RAW_IMAGE_NAME"
     JAMMY_OS_RAW_IMAGE="$WORKLOADS_DIR/$JAMMY_OS_RAW_IMAGE_NAME"
-    if [ ! -f "$JAMMY_OS_RAW_IMAGE" ]; then
-        pushd "$WORKLOADS_DIR" || exit
-        time wget --quiet $JAMMY_OS_RAW_IMAGE_DOWNLOAD_URL || exit 1
-        popd || exit
-    fi
 
     JAMMY_OS_QCOW2_IMAGE_UNCOMPRESSED_NAME="jammy-server-cloudimg-arm64-custom-20220329-0.qcow2"
-    JAMMY_OS_QCOW2_IMAGE_UNCOMPRESSED_DOWNLOAD_URL="https://ch-images.azureedge.net/$JAMMY_OS_QCOW2_IMAGE_UNCOMPRESSED_NAME"
     JAMMY_OS_QCOW2_UNCOMPRESSED_IMAGE="$WORKLOADS_DIR/$JAMMY_OS_QCOW2_IMAGE_UNCOMPRESSED_NAME"
-    if [ ! -f "$JAMMY_OS_QCOW2_UNCOMPRESSED_IMAGE" ]; then
+
-        pushd "$WORKLOADS_DIR" || exit
+    for required in "$JAMMY_OS_RAW_IMAGE" "$JAMMY_OS_QCOW2_UNCOMPRESSED_IMAGE" \
-        time wget --quiet $JAMMY_OS_QCOW2_IMAGE_UNCOMPRESSED_DOWNLOAD_URL || exit 1
+        "$WORKLOADS_DIR/CLOUDHV_EFI.fd" \
-        popd || exit
+        "$WORKLOADS_DIR/cloud-hypervisor-static-aarch64" \
+        "$WORKLOADS_DIR/alpine-minirootfs-aarch64.tar.gz" \
+        "$WORKLOADS_DIR/Image-arm64"; do
+        if [ ! -f "$required" ]; then
+            echo "Missing: $required — run: python3 scripts/fetch_workloads.py --test integration"
+            exit 1
+        fi
+    done
+
+    if [ ! -f "$WORKLOADS_DIR/virtiofsd" ]; then
+        cp /usr/local/bin/virtiofsd "$WORKLOADS_DIR/virtiofsd"
     fi
 
     JAMMY_OS_QCOW2_ZLIB_FILE_IMAGE_NAME="jammy-server-cloudimg-arm64-custom-20220329-0-zlib.qcow2"
@@ -76,13 +77,7 @@ update_workloads() {
         popd || exit
     fi
 
-    ALPINE_MINIROOTFS_URL="http://dl-cdn.alpinelinux.org/alpine/v3.11/releases/aarch64/alpine-minirootfs-3.11.3-aarch64.tar.gz"
     ALPINE_MINIROOTFS_TARBALL="$WORKLOADS_DIR/alpine-minirootfs-aarch64.tar.gz"
-    if [ ! -f "$ALPINE_MINIROOTFS_TARBALL" ]; then
-        pushd "$WORKLOADS_DIR" || exit
-        time wget --quiet $ALPINE_MINIROOTFS_URL -O "$ALPINE_MINIROOTFS_TARBALL" || exit 1
-        popd || exit
-    fi
 
     ALPINE_INITRAMFS_IMAGE="$WORKLOADS_DIR/alpine_initramfs.img"
     if [ ! -f "$ALPINE_INITRAMFS_IMAGE" ]; then
@@ -102,54 +97,6 @@ update_workloads() {
         popd || exit
     fi
 
-    # Download aarch64 ovmf
-    if [ ! -f "$WORKLOADS_DIR/CLOUDHV_EFI.fd" ]; then
-        download_aarch64_ovmf
-    fi
-
-    pushd "$WORKLOADS_DIR" || exit
-
-    # Skip checksum verification for custom-provided workloads
-    if [ -n "$CH_CUSTOM_OVMF" ]; then
-        if ! grep -v "CLOUDHV_EFI.fd" sha1sums-aarch64-common | sha1sum --check; then
-            echo "sha1sum validation of images failed, remove invalid images to fix the issue."
-            exit 1
-        fi
-    else
-        if ! sha1sum sha1sums-aarch64-common --check; then
-            echo "sha1sum validation of images failed, remove invalid images to fix the issue."
-            exit 1
-        fi
-    fi
-    popd || exit
-
-    # Download Cloud Hypervisor binary from its last stable release
-    CH_RELEASE_URL="https://github.com/cloud-hypervisor/cloud-hypervisor/releases/download/${migratable_version}/cloud-hypervisor-static-aarch64"
-    CH_RELEASE_NAME="cloud-hypervisor-static-aarch64"
-    pushd "$WORKLOADS_DIR" || exit
-    # Repeat a few times to workaround a random wget failure
-    WGET_RETRY_MAX=10
-    wget_retry=0
-
-    until [ "$wget_retry" -ge "$WGET_RETRY_MAX" ]; do
-        time wget $CH_RELEASE_URL -O "$CH_RELEASE_NAME" && break
-        wget_retry=$((wget_retry + 1))
-    done
-
-    if [ $wget_retry -ge "$WGET_RETRY_MAX" ]; then
-        exit 1
-    else
-        chmod +x $CH_RELEASE_NAME
-    fi
-    popd || exit
-
-    # Prepare linux image (build from source or download pre-built)
-    prepare_linux
-
-    if [ ! -f "$WORKLOADS_DIR/virtiofsd" ]; then
-        cp /usr/local/bin/virtiofsd "$WORKLOADS_DIR/virtiofsd"
-    fi
-
     BLK_IMAGE="$WORKLOADS_DIR/blk.img"
     MNT_DIR="mount_image"
     if [ ! -f "$BLK_IMAGE" ]; then
@@ -174,23 +121,12 @@ update_workloads() {
 
 process_common_args "$@"
 
-migratable_version=v39.0
 test_features=""
 
 if [ "$hypervisor" = "mshv" ]; then
     test_features="--features mshv"
 fi
 
-# if migratable version is set to override the default
-if [ -n "${MIGRATABLE_VERSION}" ]; then
-    # validate the version if matched with vxx.0
-    if ! [[ "${MIGRATABLE_VERSION}" =~ ^v[0-9]{2,}\.[0-9]$ ]]; then
-        echo "MIGRATABLE_VERSION should be in format vxx.0, e.g. v47.0"
-        exit 1
-    fi
-    migratable_version=${MIGRATABLE_VERSION}
-fi
-
 # lock the workloads folder to avoid parallel updating by different containers
 (
     echo "try to lock $WORKLOADS_DIR folder and update"

scripts/run_integration_tests_cvm.sh

@@ -15,15 +15,20 @@ process_common_args "$@"
 test_features="--features mshv,igvm,sev_snp"
 build_features="mshv,igvm,sev_snp"
 
-download_x86_guest_images
+JAMMY_OS_IMAGE_NAME="jammy-server-cloudimg-amd64-custom-20241017-0.qcow2"
-cp scripts/sha1sums-x86_64-common "$WORKLOADS_DIR"
+JAMMY_OS_IMAGE="$WORKLOADS_DIR/$JAMMY_OS_IMAGE_NAME"
-
+if [ ! -f "$JAMMY_OS_IMAGE" ]; then
-pushd "$WORKLOADS_DIR" || exit
+    echo "Missing: $JAMMY_OS_IMAGE — run: python3 scripts/fetch_workloads.py --test cvm"
-if ! sha1sum sha1sums-x86_64-common --check; then
-    echo "sha1sum validation of images failed, remove invalid images to fix the issue."
     exit 1
 fi
-popd || exit
+
+JAMMY_OS_RAW_IMAGE_NAME="jammy-server-cloudimg-amd64-custom-20241017-0.raw"
+JAMMY_OS_RAW_IMAGE="$WORKLOADS_DIR/$JAMMY_OS_RAW_IMAGE_NAME"
+if [ ! -f "$JAMMY_OS_RAW_IMAGE" ]; then
+    pushd "$WORKLOADS_DIR" || exit
+    time qemu-img convert -p -f qcow2 -O raw $JAMMY_OS_IMAGE_NAME $JAMMY_OS_RAW_IMAGE_NAME || exit 1
+    popd || exit
+fi
 
 cargo build --features $build_features --all --release --target "$BUILD_TARGET"
 

scripts/run_integration_tests_rate_limiter.sh

@@ -18,15 +18,11 @@ if [ "$hypervisor" = "mshv" ]; then
     test_features="--features mshv"
 fi
 
-cp scripts/sha1sums-x86_64* "$WORKLOADS_DIR"
-
 JAMMY_OS_IMAGE_NAME="jammy-server-cloudimg-amd64-custom-20241017-0.qcow2"
-JAMMY_OS_IMAGE_URL="https://ch-images.azureedge.net/$JAMMY_OS_IMAGE_NAME"
 JAMMY_OS_IMAGE="$WORKLOADS_DIR/$JAMMY_OS_IMAGE_NAME"
 if [ ! -f "$JAMMY_OS_IMAGE" ]; then
-    pushd "$WORKLOADS_DIR" || exit
+    echo "Missing: $JAMMY_OS_IMAGE — run: python3 scripts/fetch_workloads.py --test rate-limiter"
-    time wget --quiet $JAMMY_OS_IMAGE_URL || exit 1
+    exit 1
-    popd || exit
 fi
 
 JAMMY_OS_RAW_IMAGE_NAME="jammy-server-cloudimg-amd64-custom-20241017-0.raw"
@@ -37,15 +33,11 @@ if [ ! -f "$JAMMY_OS_RAW_IMAGE" ]; then
     popd || exit
 fi
 
-pushd "$WORKLOADS_DIR" || exit
+VMLINUX_IMAGE="$WORKLOADS_DIR/vmlinux-x86_64"
-if ! grep jammy sha1sums-x86_64-common | sha1sum --check; then
+if [ ! -f "$VMLINUX_IMAGE" ]; then
-    echo "sha1sum validation of images failed, remove invalid images to fix the issue."
+    echo "Missing: $VMLINUX_IMAGE — run: python3 scripts/fetch_workloads.py --test rate-limiter"
     exit 1
 fi
-popd || exit
-
-# Prepare linux image (build from source or download pre-built)
-prepare_linux
 
 CFLAGS=""
 if [[ "${BUILD_TARGET}" == "x86_64-unknown-linux-musl" ]]; then

scripts/run_integration_tests_vfio.sh

@@ -17,7 +17,9 @@ process_common_args "$@"
 WORKLOADS_DIR="$HOME/workloads"
 
 if [ ! -f "$WORKLOADS_DIR/hypervisor-fw" ]; then
-    download_hypervisor_fw
+    echo "Missing workload asset: $WORKLOADS_DIR/hypervisor-fw"
+    echo "Run: python3 scripts/fetch_workloads.py --test vfio"
+    exit 1
 fi
 
 CFLAGS=""

scripts/run_integration_tests_windows_aarch64.sh

@@ -18,10 +18,11 @@ fi
 WIN_IMAGE_BASENAME="windows-11-iot-enterprise-aarch64.raw"
 WIN_IMAGE_FILE="$WORKLOADS_DIR/$WIN_IMAGE_BASENAME"
 
-# Download aarch64 OVMF
 OVMF_FW="$WORKLOADS_DIR/CLOUDHV_EFI.fd"
 if [ ! -f "$OVMF_FW" ]; then
-    download_aarch64_ovmf
+    echo "Missing workload asset: $OVMF_FW"
+    echo "Run: python3 scripts/fetch_workloads.py --test windows"
+    exit 1
 fi
 
 # Check if the images are present

scripts/run_integration_tests_windows_x86_64.sh

@@ -18,9 +18,10 @@ WIN_IMAGE_FILE="/root/workloads/windows-server-2025-amd64-1.raw"
 WORKLOADS_DIR="/root/workloads"
 OVMF_FW="$WORKLOADS_DIR/CLOUDHV.fd"
 
-# Download amd64 ovmf
 if [ ! -f "$OVMF_FW" ]; then
-    download_amd64_ovmf
+    echo "Missing workload asset: $OVMF_FW"
+    echo "Run: python3 scripts/fetch_workloads.py --test windows"
+    exit 1
 fi
 
 CFLAGS=""

scripts/run_integration_tests_x86_64.sh

@@ -12,7 +12,6 @@ mkdir -p "$WORKLOADS_DIR/junit"
 
 process_common_args "$@"
 
-migratable_version=v39.0
 # For now these values are default for kvm
 test_features=""
 
@@ -21,27 +20,28 @@ if [ "$hypervisor" = "mshv" ]; then
 fi
 
 # if migratable version is set to override the default
-if [ -n "${MIGRATABLE_VERSION}" ]; then
+FW="$WORKLOADS_DIR/hypervisor-fw"
-    # validate the version if matched with vxx.0
+JAMMY_OS_IMAGE_NAME="jammy-server-cloudimg-amd64-custom-20241017-0.qcow2"
-    if ! [[ "${MIGRATABLE_VERSION}" =~ ^v[0-9]{2,}\.[0-9]$ ]]; then
+JAMMY_OS_IMAGE="$WORKLOADS_DIR/$JAMMY_OS_IMAGE_NAME"
-        echo "MIGRATABLE_VERSION should be in format vxx.0, e.g. v47.0"
+JAMMY_OS_RAW_IMAGE_NAME="jammy-server-cloudimg-amd64-custom-20241017-0.raw"
+JAMMY_OS_RAW_IMAGE="$WORKLOADS_DIR/$JAMMY_OS_RAW_IMAGE_NAME"
+
+for required in "$FW" "$WORKLOADS_DIR/CLOUDHV.fd" "$JAMMY_OS_IMAGE" \
+    "$WORKLOADS_DIR/vmlinux-x86_64" "$WORKLOADS_DIR/bzImage-x86_64" \
+    "$WORKLOADS_DIR/alpine-minirootfs-x86_64.tar.gz" \
+    "$WORKLOADS_DIR/cloud-hypervisor-static"; do
+    if [ ! -f "$required" ]; then
+        echo "Missing: $required — run: python3 scripts/fetch_workloads.py --test integration"
         exit 1
     fi
-    migratable_version=${MIGRATABLE_VERSION}
+done
+
+if [ ! -f "$JAMMY_OS_RAW_IMAGE" ]; then
+    pushd "$WORKLOADS_DIR" || exit
+    time qemu-img convert -p -f qcow2 -O raw $JAMMY_OS_IMAGE_NAME $JAMMY_OS_RAW_IMAGE_NAME || exit 1
+    popd || exit
 fi
 
-cp scripts/sha1sums-x86_64* "$WORKLOADS_DIR"
-
-if [ ! -f "$WORKLOADS_DIR/hypervisor-fw" ]; then
-    download_hypervisor_fw
-fi
-
-if [ ! -f "$WORKLOADS_DIR/CLOUDHV.fd" ]; then
-    download_amd64_ovmf
-fi
-
-download_x86_guest_images
-
 JAMMY_OS_QCOW_ZLIB_FILE_IMAGE_NAME="jammy-server-cloudimg-amd64-custom-20241017-0-zlib.qcow2"
 JAMMY_OS_QCOW_ZLIB_FILE_IMAGE="$WORKLOADS_DIR/$JAMMY_OS_QCOW_ZLIB_FILE_IMAGE_NAME"
 if [ ! -f "$JAMMY_OS_QCOW_ZLIB_FILE_IMAGE" ]; then
@@ -90,12 +90,10 @@ if [ ! -f "$JAMMY_OS_QCOW_BACKING_RAW_FILE_IMAGE" ]; then
     popd || exit
 fi
 
-ALPINE_MINIROOTFS_URL="http://dl-cdn.alpinelinux.org/alpine/v3.11/releases/x86_64/alpine-minirootfs-3.11.3-x86_64.tar.gz"
 ALPINE_MINIROOTFS_TARBALL="$WORKLOADS_DIR/alpine-minirootfs-x86_64.tar.gz"
 if [ ! -f "$ALPINE_MINIROOTFS_TARBALL" ]; then
-    pushd "$WORKLOADS_DIR" || exit
+    echo "Missing: $ALPINE_MINIROOTFS_TARBALL — run: python3 scripts/fetch_workloads.py --test integration"
-    time wget --quiet $ALPINE_MINIROOTFS_URL -O "$ALPINE_MINIROOTFS_TARBALL" || exit 1
+    exit 1
-    popd || exit
 fi
 
 ALPINE_INITRAMFS_IMAGE="$WORKLOADS_DIR/alpine_initramfs.img"
@@ -116,39 +114,9 @@ if [ ! -f "$ALPINE_INITRAMFS_IMAGE" ]; then
     popd || exit
 fi
 
-pushd "$WORKLOADS_DIR" || exit
+python3 scripts/fetch_workloads.py --test integration --verify-only || exit 1
-# Skip checksum verification for custom-provided workloads
-sha1_exclude=""
-[ -n "$CH_CUSTOM_FIRMWARE" ] && sha1_exclude="${sha1_exclude}|hypervisor-fw"
-[ -n "$CH_CUSTOM_OVMF" ] && sha1_exclude="${sha1_exclude}|CLOUDHV\\.fd"
-sha1_exclude="${sha1_exclude#|}"
-if [ -n "$sha1_exclude" ]; then
-    if ! cat sha1sums-x86_64 sha1sums-x86_64-common | grep -Ev "$sha1_exclude" | sha1sum --check; then
-        echo "sha1sum validation of images failed, remove invalid images to fix the issue."
-        exit 1
-    fi
-else
-    if ! sha1sum sha1sums-x86_64 sha1sums-x86_64-common --check; then
-        echo "sha1sum validation of images failed, remove invalid images to fix the issue."
-        exit 1
-    fi
-fi
-popd || exit
 
-# Download Cloud Hypervisor binary from its last stable release for live-upgrade tests
-CH_RELEASE_URL="https://github.com/cloud-hypervisor/cloud-hypervisor/releases/download/${migratable_version}/cloud-hypervisor-static"
-CH_RELEASE_NAME="cloud-hypervisor-static"
-pushd "$WORKLOADS_DIR" || exit
-time wget --quiet $CH_RELEASE_URL -O "$CH_RELEASE_NAME" || exit 1
-chmod +x $CH_RELEASE_NAME
-popd || exit
-
-# Build custom kernel based on virtio-pmem and virtio-fs upstream patches
 VMLINUX_IMAGE="$WORKLOADS_DIR/vmlinux-x86_64"
-if [ ! -f "$VMLINUX_IMAGE" ]; then
-    # Prepare linux image (build from source or download pre-built)
-    prepare_linux
-fi
 
 VIRTIOFSD="$WORKLOADS_DIR/virtiofsd"
 if [ ! -f "$VIRTIOFSD" ]; then

scripts/run_metrics.sh

@@ -20,28 +20,20 @@ if [ "$VM_TYPE" = "confidential" ]; then
     vm_type_arg="--vm-type confidential"
 fi
 
-cp scripts/sha1sums-"${TEST_ARCH}"-common "$WORKLOADS_DIR"
-
 if [ "${TEST_ARCH}" == "aarch64" ]; then
     JAMMY_OS_IMAGE_NAME="jammy-server-cloudimg-arm64-custom-20220329-0.qcow2"
-else
-    JAMMY_OS_IMAGE_NAME="jammy-server-cloudimg-amd64-custom-20241017-0.qcow2"
-fi
-
-JAMMY_OS_IMAGE_URL="https://ch-images.azureedge.net/$JAMMY_OS_IMAGE_NAME"
-JAMMY_OS_IMAGE="$WORKLOADS_DIR/$JAMMY_OS_IMAGE_NAME"
-if [ ! -f "$JAMMY_OS_IMAGE" ]; then
-    pushd "$WORKLOADS_DIR" || exit
-    time wget --quiet $JAMMY_OS_IMAGE_URL || exit 1
-    popd || exit
-fi
-
-if [ "${TEST_ARCH}" == "aarch64" ]; then
     JAMMY_OS_RAW_IMAGE_NAME="jammy-server-cloudimg-arm64-custom-20220329-0.raw"
 else
+    JAMMY_OS_IMAGE_NAME="jammy-server-cloudimg-amd64-custom-20241017-0.qcow2"
     JAMMY_OS_RAW_IMAGE_NAME="jammy-server-cloudimg-amd64-custom-20241017-0.raw"
 fi
 
+JAMMY_OS_IMAGE="$WORKLOADS_DIR/$JAMMY_OS_IMAGE_NAME"
+if [ ! -f "$JAMMY_OS_IMAGE" ]; then
+    echo "Missing: $JAMMY_OS_IMAGE — run: python3 scripts/fetch_workloads.py --test metrics"
+    exit 1
+fi
+
 JAMMY_OS_RAW_IMAGE="$WORKLOADS_DIR/$JAMMY_OS_RAW_IMAGE_NAME"
 if [ ! -f "$JAMMY_OS_RAW_IMAGE" ]; then
     pushd "$WORKLOADS_DIR" || exit
@@ -49,17 +41,16 @@ if [ ! -f "$JAMMY_OS_RAW_IMAGE" ]; then
     popd || exit
 fi
 
-pushd "$WORKLOADS_DIR" || exit
+if [ "${TEST_ARCH}" == "aarch64" ]; then
-if ! grep jammy sha1sums-"${TEST_ARCH}"-common | sha1sum --check; then
+    KERNEL_IMAGE="$WORKLOADS_DIR/Image-arm64"
-    echo "sha1sum validation of images failed, remove invalid images to fix the issue."
+else
+    KERNEL_IMAGE="$WORKLOADS_DIR/vmlinux-x86_64"
+fi
+if [ ! -f "$KERNEL_IMAGE" ]; then
+    echo "Missing: $KERNEL_IMAGE — run: python3 scripts/fetch_workloads.py --test metrics"
     exit 1
 fi
 
-popd || exit
-
-# Prepare linux image (build from source or download pre-built)
-prepare_linux
-
 CFLAGS=""
 if [[ "${BUILD_TARGET}" == "${TEST_ARCH}-unknown-linux-musl" ]]; then
     # shellcheck disable=SC2034