From 531666bc67dfb4fc5f3e087f3c025f53cf1e6034 Mon Sep 17 00:00:00 2001 From: Rob Bradford Date: Fri, 22 May 2026 15:44:24 +0100 Subject: [PATCH] scripts: Remove downloads from integration test scripts Assisted-by: Claude:Opus-4.6 Signed-off-by: Rob Bradford --- scripts/dev_cli.sh | 1 - scripts/run_integration_tests_aarch64.sh | 92 +++---------------- scripts/run_integration_tests_cvm.sh | 19 ++-- scripts/run_integration_tests_rate_limiter.sh | 18 +--- scripts/run_integration_tests_vfio.sh | 4 +- .../run_integration_tests_windows_aarch64.sh | 5 +- .../run_integration_tests_windows_x86_64.sh | 5 +- scripts/run_integration_tests_x86_64.sh | 74 +++++---------- scripts/run_metrics.sh | 37 +++----- 9 files changed, 75 insertions(+), 180 deletions(-) diff --git a/scripts/dev_cli.sh b/scripts/dev_cli.sh index dcd6ba4c6..3fa86bdd3 100755 --- a/scripts/dev_cli.sh +++ b/scripts/dev_cli.sh @@ -583,7 +583,6 @@ cmd_tests() { "${common_env_args[@]}" \ --env PARALLEL_INTEGRATION_TESTS_NUM="${PARALLEL_INTEGRATION_TESTS_NUM:-}" \ --env LLVM_PROFILE_FILE="$LLVM_PROFILE_FILE" \ - --env MIGRATABLE_VERSION="$MIGRATABLE_VERSION" \ "$CTR_IMAGE" \ dbus-run-session ./scripts/run_integration_tests_"$(uname -m)".sh "$@" || fix_dir_perms $? || exit $? fi diff --git a/scripts/run_integration_tests_aarch64.sh b/scripts/run_integration_tests_aarch64.sh index e31b02423..d3c92efe4 100755 --- a/scripts/run_integration_tests_aarch64.sh +++ b/scripts/run_integration_tests_aarch64.sh @@ -10,24 +10,25 @@ source "$(dirname "$0")"/common-aarch64.sh WORKLOADS_LOCK="$WORKLOADS_DIR/integration_test.lock" update_workloads() { - cp scripts/sha1sums-aarch64-common "$WORKLOADS_DIR" - JAMMY_OS_RAW_IMAGE_NAME="jammy-server-cloudimg-arm64-custom-20220329-0.raw" - JAMMY_OS_RAW_IMAGE_DOWNLOAD_URL="https://ch-images.azureedge.net/$JAMMY_OS_RAW_IMAGE_NAME" JAMMY_OS_RAW_IMAGE="$WORKLOADS_DIR/$JAMMY_OS_RAW_IMAGE_NAME" - if [ ! -f "$JAMMY_OS_RAW_IMAGE" ]; then - pushd "$WORKLOADS_DIR" || exit - time wget --quiet $JAMMY_OS_RAW_IMAGE_DOWNLOAD_URL || exit 1 - popd || exit - fi JAMMY_OS_QCOW2_IMAGE_UNCOMPRESSED_NAME="jammy-server-cloudimg-arm64-custom-20220329-0.qcow2" - JAMMY_OS_QCOW2_IMAGE_UNCOMPRESSED_DOWNLOAD_URL="https://ch-images.azureedge.net/$JAMMY_OS_QCOW2_IMAGE_UNCOMPRESSED_NAME" JAMMY_OS_QCOW2_UNCOMPRESSED_IMAGE="$WORKLOADS_DIR/$JAMMY_OS_QCOW2_IMAGE_UNCOMPRESSED_NAME" - if [ ! -f "$JAMMY_OS_QCOW2_UNCOMPRESSED_IMAGE" ]; then - pushd "$WORKLOADS_DIR" || exit - time wget --quiet $JAMMY_OS_QCOW2_IMAGE_UNCOMPRESSED_DOWNLOAD_URL || exit 1 - popd || exit + + for required in "$JAMMY_OS_RAW_IMAGE" "$JAMMY_OS_QCOW2_UNCOMPRESSED_IMAGE" \ + "$WORKLOADS_DIR/CLOUDHV_EFI.fd" \ + "$WORKLOADS_DIR/cloud-hypervisor-static-aarch64" \ + "$WORKLOADS_DIR/alpine-minirootfs-aarch64.tar.gz" \ + "$WORKLOADS_DIR/Image-arm64"; do + if [ ! -f "$required" ]; then + echo "Missing: $required — run: python3 scripts/fetch_workloads.py --test integration" + exit 1 + fi + done + + if [ ! -f "$WORKLOADS_DIR/virtiofsd" ]; then + cp /usr/local/bin/virtiofsd "$WORKLOADS_DIR/virtiofsd" fi JAMMY_OS_QCOW2_ZLIB_FILE_IMAGE_NAME="jammy-server-cloudimg-arm64-custom-20220329-0-zlib.qcow2" @@ -76,13 +77,7 @@ update_workloads() { popd || exit fi - ALPINE_MINIROOTFS_URL="http://dl-cdn.alpinelinux.org/alpine/v3.11/releases/aarch64/alpine-minirootfs-3.11.3-aarch64.tar.gz" ALPINE_MINIROOTFS_TARBALL="$WORKLOADS_DIR/alpine-minirootfs-aarch64.tar.gz" - if [ ! -f "$ALPINE_MINIROOTFS_TARBALL" ]; then - pushd "$WORKLOADS_DIR" || exit - time wget --quiet $ALPINE_MINIROOTFS_URL -O "$ALPINE_MINIROOTFS_TARBALL" || exit 1 - popd || exit - fi ALPINE_INITRAMFS_IMAGE="$WORKLOADS_DIR/alpine_initramfs.img" if [ ! -f "$ALPINE_INITRAMFS_IMAGE" ]; then @@ -102,54 +97,6 @@ update_workloads() { popd || exit fi - # Download aarch64 ovmf - if [ ! -f "$WORKLOADS_DIR/CLOUDHV_EFI.fd" ]; then - download_aarch64_ovmf - fi - - pushd "$WORKLOADS_DIR" || exit - - # Skip checksum verification for custom-provided workloads - if [ -n "$CH_CUSTOM_OVMF" ]; then - if ! grep -v "CLOUDHV_EFI.fd" sha1sums-aarch64-common | sha1sum --check; then - echo "sha1sum validation of images failed, remove invalid images to fix the issue." - exit 1 - fi - else - if ! sha1sum sha1sums-aarch64-common --check; then - echo "sha1sum validation of images failed, remove invalid images to fix the issue." - exit 1 - fi - fi - popd || exit - - # Download Cloud Hypervisor binary from its last stable release - CH_RELEASE_URL="https://github.com/cloud-hypervisor/cloud-hypervisor/releases/download/${migratable_version}/cloud-hypervisor-static-aarch64" - CH_RELEASE_NAME="cloud-hypervisor-static-aarch64" - pushd "$WORKLOADS_DIR" || exit - # Repeat a few times to workaround a random wget failure - WGET_RETRY_MAX=10 - wget_retry=0 - - until [ "$wget_retry" -ge "$WGET_RETRY_MAX" ]; do - time wget $CH_RELEASE_URL -O "$CH_RELEASE_NAME" && break - wget_retry=$((wget_retry + 1)) - done - - if [ $wget_retry -ge "$WGET_RETRY_MAX" ]; then - exit 1 - else - chmod +x $CH_RELEASE_NAME - fi - popd || exit - - # Prepare linux image (build from source or download pre-built) - prepare_linux - - if [ ! -f "$WORKLOADS_DIR/virtiofsd" ]; then - cp /usr/local/bin/virtiofsd "$WORKLOADS_DIR/virtiofsd" - fi - BLK_IMAGE="$WORKLOADS_DIR/blk.img" MNT_DIR="mount_image" if [ ! -f "$BLK_IMAGE" ]; then @@ -174,23 +121,12 @@ update_workloads() { process_common_args "$@" -migratable_version=v39.0 test_features="" if [ "$hypervisor" = "mshv" ]; then test_features="--features mshv" fi -# if migratable version is set to override the default -if [ -n "${MIGRATABLE_VERSION}" ]; then - # validate the version if matched with vxx.0 - if ! [[ "${MIGRATABLE_VERSION}" =~ ^v[0-9]{2,}\.[0-9]$ ]]; then - echo "MIGRATABLE_VERSION should be in format vxx.0, e.g. v47.0" - exit 1 - fi - migratable_version=${MIGRATABLE_VERSION} -fi - # lock the workloads folder to avoid parallel updating by different containers ( echo "try to lock $WORKLOADS_DIR folder and update" diff --git a/scripts/run_integration_tests_cvm.sh b/scripts/run_integration_tests_cvm.sh index d0022c7fa..1bd300f48 100755 --- a/scripts/run_integration_tests_cvm.sh +++ b/scripts/run_integration_tests_cvm.sh @@ -15,15 +15,20 @@ process_common_args "$@" test_features="--features mshv,igvm,sev_snp" build_features="mshv,igvm,sev_snp" -download_x86_guest_images -cp scripts/sha1sums-x86_64-common "$WORKLOADS_DIR" - -pushd "$WORKLOADS_DIR" || exit -if ! sha1sum sha1sums-x86_64-common --check; then - echo "sha1sum validation of images failed, remove invalid images to fix the issue." +JAMMY_OS_IMAGE_NAME="jammy-server-cloudimg-amd64-custom-20241017-0.qcow2" +JAMMY_OS_IMAGE="$WORKLOADS_DIR/$JAMMY_OS_IMAGE_NAME" +if [ ! -f "$JAMMY_OS_IMAGE" ]; then + echo "Missing: $JAMMY_OS_IMAGE — run: python3 scripts/fetch_workloads.py --test cvm" exit 1 fi -popd || exit + +JAMMY_OS_RAW_IMAGE_NAME="jammy-server-cloudimg-amd64-custom-20241017-0.raw" +JAMMY_OS_RAW_IMAGE="$WORKLOADS_DIR/$JAMMY_OS_RAW_IMAGE_NAME" +if [ ! -f "$JAMMY_OS_RAW_IMAGE" ]; then + pushd "$WORKLOADS_DIR" || exit + time qemu-img convert -p -f qcow2 -O raw $JAMMY_OS_IMAGE_NAME $JAMMY_OS_RAW_IMAGE_NAME || exit 1 + popd || exit +fi cargo build --features $build_features --all --release --target "$BUILD_TARGET" diff --git a/scripts/run_integration_tests_rate_limiter.sh b/scripts/run_integration_tests_rate_limiter.sh index 17bb3373a..e09861ebc 100755 --- a/scripts/run_integration_tests_rate_limiter.sh +++ b/scripts/run_integration_tests_rate_limiter.sh @@ -18,15 +18,11 @@ if [ "$hypervisor" = "mshv" ]; then test_features="--features mshv" fi -cp scripts/sha1sums-x86_64* "$WORKLOADS_DIR" - JAMMY_OS_IMAGE_NAME="jammy-server-cloudimg-amd64-custom-20241017-0.qcow2" -JAMMY_OS_IMAGE_URL="https://ch-images.azureedge.net/$JAMMY_OS_IMAGE_NAME" JAMMY_OS_IMAGE="$WORKLOADS_DIR/$JAMMY_OS_IMAGE_NAME" if [ ! -f "$JAMMY_OS_IMAGE" ]; then - pushd "$WORKLOADS_DIR" || exit - time wget --quiet $JAMMY_OS_IMAGE_URL || exit 1 - popd || exit + echo "Missing: $JAMMY_OS_IMAGE — run: python3 scripts/fetch_workloads.py --test rate-limiter" + exit 1 fi JAMMY_OS_RAW_IMAGE_NAME="jammy-server-cloudimg-amd64-custom-20241017-0.raw" @@ -37,15 +33,11 @@ if [ ! -f "$JAMMY_OS_RAW_IMAGE" ]; then popd || exit fi -pushd "$WORKLOADS_DIR" || exit -if ! grep jammy sha1sums-x86_64-common | sha1sum --check; then - echo "sha1sum validation of images failed, remove invalid images to fix the issue." +VMLINUX_IMAGE="$WORKLOADS_DIR/vmlinux-x86_64" +if [ ! -f "$VMLINUX_IMAGE" ]; then + echo "Missing: $VMLINUX_IMAGE — run: python3 scripts/fetch_workloads.py --test rate-limiter" exit 1 fi -popd || exit - -# Prepare linux image (build from source or download pre-built) -prepare_linux CFLAGS="" if [[ "${BUILD_TARGET}" == "x86_64-unknown-linux-musl" ]]; then diff --git a/scripts/run_integration_tests_vfio.sh b/scripts/run_integration_tests_vfio.sh index dcdc9d1c0..26f0c31a3 100755 --- a/scripts/run_integration_tests_vfio.sh +++ b/scripts/run_integration_tests_vfio.sh @@ -17,7 +17,9 @@ process_common_args "$@" WORKLOADS_DIR="$HOME/workloads" if [ ! -f "$WORKLOADS_DIR/hypervisor-fw" ]; then - download_hypervisor_fw + echo "Missing workload asset: $WORKLOADS_DIR/hypervisor-fw" + echo "Run: python3 scripts/fetch_workloads.py --test vfio" + exit 1 fi CFLAGS="" diff --git a/scripts/run_integration_tests_windows_aarch64.sh b/scripts/run_integration_tests_windows_aarch64.sh index 17fb28cc9..b6c0e352f 100755 --- a/scripts/run_integration_tests_windows_aarch64.sh +++ b/scripts/run_integration_tests_windows_aarch64.sh @@ -18,10 +18,11 @@ fi WIN_IMAGE_BASENAME="windows-11-iot-enterprise-aarch64.raw" WIN_IMAGE_FILE="$WORKLOADS_DIR/$WIN_IMAGE_BASENAME" -# Download aarch64 OVMF OVMF_FW="$WORKLOADS_DIR/CLOUDHV_EFI.fd" if [ ! -f "$OVMF_FW" ]; then - download_aarch64_ovmf + echo "Missing workload asset: $OVMF_FW" + echo "Run: python3 scripts/fetch_workloads.py --test windows" + exit 1 fi # Check if the images are present diff --git a/scripts/run_integration_tests_windows_x86_64.sh b/scripts/run_integration_tests_windows_x86_64.sh index 7d4800296..a312cd09e 100755 --- a/scripts/run_integration_tests_windows_x86_64.sh +++ b/scripts/run_integration_tests_windows_x86_64.sh @@ -18,9 +18,10 @@ WIN_IMAGE_FILE="/root/workloads/windows-server-2025-amd64-1.raw" WORKLOADS_DIR="/root/workloads" OVMF_FW="$WORKLOADS_DIR/CLOUDHV.fd" -# Download amd64 ovmf if [ ! -f "$OVMF_FW" ]; then - download_amd64_ovmf + echo "Missing workload asset: $OVMF_FW" + echo "Run: python3 scripts/fetch_workloads.py --test windows" + exit 1 fi CFLAGS="" diff --git a/scripts/run_integration_tests_x86_64.sh b/scripts/run_integration_tests_x86_64.sh index a018ae59c..d64094f07 100755 --- a/scripts/run_integration_tests_x86_64.sh +++ b/scripts/run_integration_tests_x86_64.sh @@ -12,7 +12,6 @@ mkdir -p "$WORKLOADS_DIR/junit" process_common_args "$@" -migratable_version=v39.0 # For now these values are default for kvm test_features="" @@ -21,27 +20,28 @@ if [ "$hypervisor" = "mshv" ]; then fi # if migratable version is set to override the default -if [ -n "${MIGRATABLE_VERSION}" ]; then - # validate the version if matched with vxx.0 - if ! [[ "${MIGRATABLE_VERSION}" =~ ^v[0-9]{2,}\.[0-9]$ ]]; then - echo "MIGRATABLE_VERSION should be in format vxx.0, e.g. v47.0" +FW="$WORKLOADS_DIR/hypervisor-fw" +JAMMY_OS_IMAGE_NAME="jammy-server-cloudimg-amd64-custom-20241017-0.qcow2" +JAMMY_OS_IMAGE="$WORKLOADS_DIR/$JAMMY_OS_IMAGE_NAME" +JAMMY_OS_RAW_IMAGE_NAME="jammy-server-cloudimg-amd64-custom-20241017-0.raw" +JAMMY_OS_RAW_IMAGE="$WORKLOADS_DIR/$JAMMY_OS_RAW_IMAGE_NAME" + +for required in "$FW" "$WORKLOADS_DIR/CLOUDHV.fd" "$JAMMY_OS_IMAGE" \ + "$WORKLOADS_DIR/vmlinux-x86_64" "$WORKLOADS_DIR/bzImage-x86_64" \ + "$WORKLOADS_DIR/alpine-minirootfs-x86_64.tar.gz" \ + "$WORKLOADS_DIR/cloud-hypervisor-static"; do + if [ ! -f "$required" ]; then + echo "Missing: $required — run: python3 scripts/fetch_workloads.py --test integration" exit 1 fi - migratable_version=${MIGRATABLE_VERSION} +done + +if [ ! -f "$JAMMY_OS_RAW_IMAGE" ]; then + pushd "$WORKLOADS_DIR" || exit + time qemu-img convert -p -f qcow2 -O raw $JAMMY_OS_IMAGE_NAME $JAMMY_OS_RAW_IMAGE_NAME || exit 1 + popd || exit fi -cp scripts/sha1sums-x86_64* "$WORKLOADS_DIR" - -if [ ! -f "$WORKLOADS_DIR/hypervisor-fw" ]; then - download_hypervisor_fw -fi - -if [ ! -f "$WORKLOADS_DIR/CLOUDHV.fd" ]; then - download_amd64_ovmf -fi - -download_x86_guest_images - JAMMY_OS_QCOW_ZLIB_FILE_IMAGE_NAME="jammy-server-cloudimg-amd64-custom-20241017-0-zlib.qcow2" JAMMY_OS_QCOW_ZLIB_FILE_IMAGE="$WORKLOADS_DIR/$JAMMY_OS_QCOW_ZLIB_FILE_IMAGE_NAME" if [ ! -f "$JAMMY_OS_QCOW_ZLIB_FILE_IMAGE" ]; then @@ -90,12 +90,10 @@ if [ ! -f "$JAMMY_OS_QCOW_BACKING_RAW_FILE_IMAGE" ]; then popd || exit fi -ALPINE_MINIROOTFS_URL="http://dl-cdn.alpinelinux.org/alpine/v3.11/releases/x86_64/alpine-minirootfs-3.11.3-x86_64.tar.gz" ALPINE_MINIROOTFS_TARBALL="$WORKLOADS_DIR/alpine-minirootfs-x86_64.tar.gz" if [ ! -f "$ALPINE_MINIROOTFS_TARBALL" ]; then - pushd "$WORKLOADS_DIR" || exit - time wget --quiet $ALPINE_MINIROOTFS_URL -O "$ALPINE_MINIROOTFS_TARBALL" || exit 1 - popd || exit + echo "Missing: $ALPINE_MINIROOTFS_TARBALL — run: python3 scripts/fetch_workloads.py --test integration" + exit 1 fi ALPINE_INITRAMFS_IMAGE="$WORKLOADS_DIR/alpine_initramfs.img" @@ -116,39 +114,9 @@ if [ ! -f "$ALPINE_INITRAMFS_IMAGE" ]; then popd || exit fi -pushd "$WORKLOADS_DIR" || exit -# Skip checksum verification for custom-provided workloads -sha1_exclude="" -[ -n "$CH_CUSTOM_FIRMWARE" ] && sha1_exclude="${sha1_exclude}|hypervisor-fw" -[ -n "$CH_CUSTOM_OVMF" ] && sha1_exclude="${sha1_exclude}|CLOUDHV\\.fd" -sha1_exclude="${sha1_exclude#|}" -if [ -n "$sha1_exclude" ]; then - if ! cat sha1sums-x86_64 sha1sums-x86_64-common | grep -Ev "$sha1_exclude" | sha1sum --check; then - echo "sha1sum validation of images failed, remove invalid images to fix the issue." - exit 1 - fi -else - if ! sha1sum sha1sums-x86_64 sha1sums-x86_64-common --check; then - echo "sha1sum validation of images failed, remove invalid images to fix the issue." - exit 1 - fi -fi -popd || exit +python3 scripts/fetch_workloads.py --test integration --verify-only || exit 1 -# Download Cloud Hypervisor binary from its last stable release for live-upgrade tests -CH_RELEASE_URL="https://github.com/cloud-hypervisor/cloud-hypervisor/releases/download/${migratable_version}/cloud-hypervisor-static" -CH_RELEASE_NAME="cloud-hypervisor-static" -pushd "$WORKLOADS_DIR" || exit -time wget --quiet $CH_RELEASE_URL -O "$CH_RELEASE_NAME" || exit 1 -chmod +x $CH_RELEASE_NAME -popd || exit - -# Build custom kernel based on virtio-pmem and virtio-fs upstream patches VMLINUX_IMAGE="$WORKLOADS_DIR/vmlinux-x86_64" -if [ ! -f "$VMLINUX_IMAGE" ]; then - # Prepare linux image (build from source or download pre-built) - prepare_linux -fi VIRTIOFSD="$WORKLOADS_DIR/virtiofsd" if [ ! -f "$VIRTIOFSD" ]; then diff --git a/scripts/run_metrics.sh b/scripts/run_metrics.sh index fcc8597e8..3e2f91ff7 100755 --- a/scripts/run_metrics.sh +++ b/scripts/run_metrics.sh @@ -20,28 +20,20 @@ if [ "$VM_TYPE" = "confidential" ]; then vm_type_arg="--vm-type confidential" fi -cp scripts/sha1sums-"${TEST_ARCH}"-common "$WORKLOADS_DIR" - if [ "${TEST_ARCH}" == "aarch64" ]; then JAMMY_OS_IMAGE_NAME="jammy-server-cloudimg-arm64-custom-20220329-0.qcow2" -else - JAMMY_OS_IMAGE_NAME="jammy-server-cloudimg-amd64-custom-20241017-0.qcow2" -fi - -JAMMY_OS_IMAGE_URL="https://ch-images.azureedge.net/$JAMMY_OS_IMAGE_NAME" -JAMMY_OS_IMAGE="$WORKLOADS_DIR/$JAMMY_OS_IMAGE_NAME" -if [ ! -f "$JAMMY_OS_IMAGE" ]; then - pushd "$WORKLOADS_DIR" || exit - time wget --quiet $JAMMY_OS_IMAGE_URL || exit 1 - popd || exit -fi - -if [ "${TEST_ARCH}" == "aarch64" ]; then JAMMY_OS_RAW_IMAGE_NAME="jammy-server-cloudimg-arm64-custom-20220329-0.raw" else + JAMMY_OS_IMAGE_NAME="jammy-server-cloudimg-amd64-custom-20241017-0.qcow2" JAMMY_OS_RAW_IMAGE_NAME="jammy-server-cloudimg-amd64-custom-20241017-0.raw" fi +JAMMY_OS_IMAGE="$WORKLOADS_DIR/$JAMMY_OS_IMAGE_NAME" +if [ ! -f "$JAMMY_OS_IMAGE" ]; then + echo "Missing: $JAMMY_OS_IMAGE — run: python3 scripts/fetch_workloads.py --test metrics" + exit 1 +fi + JAMMY_OS_RAW_IMAGE="$WORKLOADS_DIR/$JAMMY_OS_RAW_IMAGE_NAME" if [ ! -f "$JAMMY_OS_RAW_IMAGE" ]; then pushd "$WORKLOADS_DIR" || exit @@ -49,17 +41,16 @@ if [ ! -f "$JAMMY_OS_RAW_IMAGE" ]; then popd || exit fi -pushd "$WORKLOADS_DIR" || exit -if ! grep jammy sha1sums-"${TEST_ARCH}"-common | sha1sum --check; then - echo "sha1sum validation of images failed, remove invalid images to fix the issue." +if [ "${TEST_ARCH}" == "aarch64" ]; then + KERNEL_IMAGE="$WORKLOADS_DIR/Image-arm64" +else + KERNEL_IMAGE="$WORKLOADS_DIR/vmlinux-x86_64" +fi +if [ ! -f "$KERNEL_IMAGE" ]; then + echo "Missing: $KERNEL_IMAGE — run: python3 scripts/fetch_workloads.py --test metrics" exit 1 fi -popd || exit - -# Prepare linux image (build from source or download pre-built) -prepare_linux - CFLAGS="" if [[ "${BUILD_TARGET}" == "${TEST_ARCH}-unknown-linux-musl" ]]; then # shellcheck disable=SC2034