From c118606d645f210d3eded192c6eb73d88c8696d6 Mon Sep 17 00:00:00 2001 From: Rob Bradford Date: Sat, 25 Apr 2026 18:41:24 +0100 Subject: [PATCH] tests: Fold live migration tests into x86-64 script Move the live migration test running from their own script into the x86-64 script (on aarch64 they were already in the same script.) They were historically separate as they were new. Now they are established it makes sense for them to be combined. The timeout in the GitHub workflow has been extended to accommodate the extra work in the same step. The Rust test scopes are unchanged - the running of the tests has been moved. Assisted-by: Claude:Opus-4.7 Signed-off-by: Rob Bradford --- .github/workflows/integration-x86-64.yaml | 6 +- scripts/dev_cli.sh | 31 +---- .../run_integration_tests_live_migration.sh | 108 ------------------ scripts/run_integration_tests_x86_64.sh | 30 +++++ 4 files changed, 32 insertions(+), 143 deletions(-) delete mode 100755 scripts/run_integration_tests_live_migration.sh diff --git a/.github/workflows/integration-x86-64.yaml b/.github/workflows/integration-x86-64.yaml index 213442d98..38dbcb06f 100644 --- a/.github/workflows/integration-x86-64.yaml +++ b/.github/workflows/integration-x86-64.yaml @@ -47,12 +47,8 @@ jobs: run: sudo modprobe openvswitch - name: Run integration tests if: ${{ github.event_name != 'pull_request' || (matrix.runner == 'garm-jammy' && matrix.libc == 'gnu') }} - timeout-minutes: 60 + timeout-minutes: 80 run: scripts/dev_cli.sh tests --integration --libc ${{ matrix.libc }} - - name: Run live-migration integration tests - if: ${{ github.event_name != 'pull_request' || (matrix.runner == 'garm-jammy' && matrix.libc == 'gnu') }} - timeout-minutes: 20 - run: scripts/dev_cli.sh tests --integration-live-migration --libc ${{ matrix.libc }} - name: Skipping build for PR if: ${{ github.event_name == 'pull_request' && matrix.runner != 'garm-jammy' && matrix.libc != 'gnu' }} run: echo "Skipping build for PR" diff --git a/scripts/dev_cli.sh b/scripts/dev_cli.sh index a5d3947ce..21d636280 100755 --- a/scripts/dev_cli.sh +++ b/scripts/dev_cli.sh @@ -259,7 +259,6 @@ cmd_help() { echo " --integration Run the integration tests." echo " --integration-vfio Run the VFIO integration tests." echo " --integration-windows Run the Windows guest integration tests." - echo " --integration-live-migration Run the live-migration integration tests." echo " --integration-rate-limiter Run the rate-limiter integration tests." echo " --integration-cvm Run the Confidential VM integration tests." echo " --libc Select the C library Cloud Hypervisor will be built against. Default is gnu" @@ -395,7 +394,6 @@ cmd_tests() { integration=false integration_vfio=false integration_windows=false - integration_live_migration=false integration_rate_limiter=false integration_cvm=false metrics=false @@ -414,7 +412,6 @@ cmd_tests() { "--integration") { integration=true; } ;; "--integration-vfio") { integration_vfio=true; } ;; "--integration-windows") { integration_windows=true; } ;; - "--integration-live-migration") { integration_live_migration=true; } ;; "--integration-rate-limiter") { integration_rate_limiter=true; } ;; "--integration-cvm") { integration_cvm=true; } ;; "--metrics") { metrics=true; } ;; @@ -515,6 +512,7 @@ cmd_tests() { --env PARALLEL_INTEGRATION_TESTS_NUM="${PARALLEL_INTEGRATION_TESTS_NUM:-}" \ --env AUTH_DOWNLOAD_TOKEN="$AUTH_DOWNLOAD_TOKEN" \ --env LLVM_PROFILE_FILE="$LLVM_PROFILE_FILE" \ + --env MIGRATABLE_VERSION="$MIGRATABLE_VERSION" \ "$CTR_IMAGE" \ dbus-run-session ./scripts/run_integration_tests_"$(uname -m)".sh "$@" || fix_dir_perms $? || exit $? fi @@ -595,33 +593,6 @@ cmd_tests() { ./scripts/run_integration_tests_windows_"$(uname -m)".sh "$@" || fix_dir_perms $? || exit $? fi - if [ "$integration_live_migration" = true ]; then - say "Running 'live migration' integration tests for $target..." - run_container "$DOCKER_RUNTIME" run \ - --name "$CLH_CTR_NAME" \ - --workdir "$CTR_CLH_ROOT_DIR" \ - --rm \ - --privileged \ - --security-opt seccomp=unconfined \ - --ipc=host \ - --net="$CTR_CLH_NET" \ - --mount type=tmpfs,destination=/tmp \ - --volume /dev:/dev \ - --volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR" \ - ${exported_volumes:+$exported_volumes} \ - --volume "$CLH_INTEGRATION_WORKLOADS:$CTR_CLH_INTEGRATION_WORKLOADS" \ - --env USER="root" \ - --env BUILD_TARGET="$target" \ - --env RUSTFLAGS="$rustflags" \ - --env TARGET_CC="$target_cc" \ - --env PARALLEL_INTEGRATION_TESTS_NUM="${PARALLEL_INTEGRATION_TESTS_NUM:-}" \ - --env AUTH_DOWNLOAD_TOKEN="$AUTH_DOWNLOAD_TOKEN" \ - --env LLVM_PROFILE_FILE="$LLVM_PROFILE_FILE" \ - --env MIGRATABLE_VERSION="$MIGRATABLE_VERSION" \ - "$CTR_IMAGE" \ - ./scripts/run_integration_tests_live_migration.sh "$@" || fix_dir_perms $? || exit $? - fi - if [ "$integration_rate_limiter" = true ]; then say "Running 'rate limiter' integration tests for $target..." run_container "$DOCKER_RUNTIME" run \ diff --git a/scripts/run_integration_tests_live_migration.sh b/scripts/run_integration_tests_live_migration.sh deleted file mode 100755 index 5198e2c89..000000000 --- a/scripts/run_integration_tests_live_migration.sh +++ /dev/null @@ -1,108 +0,0 @@ -#!/usr/bin/env bash -# shellcheck disable=SC2048,SC2086 -set -x - -# shellcheck source=/dev/null -source "$HOME"/.cargo/env -source "$(dirname "$0")"/test-util.sh - -WORKLOADS_DIR="$HOME/workloads" -mkdir -p "$WORKLOADS_DIR" - -process_common_args "$@" - -migratable_version=v39.0 -# For now these values are default for kvm -test_features="" - -if [ "$hypervisor" = "mshv" ]; then - test_features="--features mshv" -fi - -# if migratable version is set to override the default -if [ -n "${MIGRATABLE_VERSION}" ]; then - # validate the version if matched with vxx.0 - if ! [[ "${MIGRATABLE_VERSION}" =~ ^v[0-9]{2,}\.[0-9]$ ]]; then - echo "MIGRATABLE_VERSION should be in format vxx.0, e.g. v47.0" - exit 1 - fi - migratable_version=${MIGRATABLE_VERSION} -fi -cp scripts/sha1sums-x86_64* "$WORKLOADS_DIR" - -FOCAL_OS_IMAGE_NAME="focal-server-cloudimg-amd64-custom-20210609-0.qcow2" -FOCAL_OS_IMAGE_URL="https://ch-images.azureedge.net/$FOCAL_OS_IMAGE_NAME" -FOCAL_OS_IMAGE="$WORKLOADS_DIR/$FOCAL_OS_IMAGE_NAME" -if [ ! -f "$FOCAL_OS_IMAGE" ]; then - pushd "$WORKLOADS_DIR" || exit - time wget --quiet $FOCAL_OS_IMAGE_URL || exit 1 - popd || exit -fi - -FOCAL_OS_RAW_IMAGE_NAME="focal-server-cloudimg-amd64-custom-20210609-0.raw" -FOCAL_OS_RAW_IMAGE="$WORKLOADS_DIR/$FOCAL_OS_RAW_IMAGE_NAME" -if [ ! -f "$FOCAL_OS_RAW_IMAGE" ]; then - pushd "$WORKLOADS_DIR" || exit - time qemu-img convert -p -f qcow2 -O raw $FOCAL_OS_IMAGE_NAME $FOCAL_OS_RAW_IMAGE_NAME || exit 1 - popd || exit -fi - -pushd "$WORKLOADS_DIR" || exit -if ! grep focal sha1sums-x86_64-common | sha1sum --check; then - echo "sha1sum validation of images failed, remove invalid images to fix the issue." - exit 1 -fi -popd || exit - -# Download Cloud Hypervisor binary from its last stable release -CH_RELEASE_URL="https://github.com/cloud-hypervisor/cloud-hypervisor/releases/download/${migratable_version}/cloud-hypervisor-static" -CH_RELEASE_NAME="cloud-hypervisor-static" -pushd "$WORKLOADS_DIR" || exit -time wget --quiet $CH_RELEASE_URL -O "$CH_RELEASE_NAME" || exit 1 -chmod +x $CH_RELEASE_NAME -popd || exit - -# Build custom kernel based on virtio-pmem and virtio-fs upstream patches -VMLINUX_IMAGE="$WORKLOADS_DIR/vmlinux-x86_64" -if [ ! -f "$VMLINUX_IMAGE" ]; then - # Prepare linux image (build from source or download pre-built) - prepare_linux -fi - -CFLAGS="" -if [[ "${BUILD_TARGET}" == "x86_64-unknown-linux-musl" ]]; then - # shellcheck disable=SC2034 - CFLAGS="-I /usr/include/x86_64-linux-musl/ -idirafter /usr/include/" -fi - -cargo build --features mshv --all --release --target "$BUILD_TARGET" - -# Test ovs-dpdk relies on hugepages -HUGEPAGESIZE=$(grep Hugepagesize /proc/meminfo | awk '{print $2}') -PAGE_NUM=$((12288 * 1024 / HUGEPAGESIZE)) -echo "$PAGE_NUM" | sudo tee /proc/sys/vm/nr_hugepages -sudo chmod a+rwX /dev/hugepages - -# Common configuration for every test run -export RUST_BACKTRACE=1 -export RUSTFLAGS="$RUSTFLAGS" - -TEST_THREADS_DEFAULT="$(($(nproc) / 4))" -if ! [[ "${PARALLEL_INTEGRATION_TESTS_NUM:-}" =~ ^[1-9][0-9]*$ ]]; then - PARALLEL_INTEGRATION_TESTS_NUM="${TEST_THREADS_DEFAULT}" -fi -echo "nproc:$(nproc), parallel_integration_tests:${PARALLEL_INTEGRATION_TESTS_NUM}" - -time cargo nextest run $test_features --retries 3 --no-fail-fast --no-tests=pass --test-threads="${PARALLEL_INTEGRATION_TESTS_NUM}" "live_migration_parallel::$test_filter" -- ${test_binary_args[*]} - -RES=$? - -# Run some tests in sequence since the result could be affected by other tests -# running in parallel. -if [ $RES -eq 0 ]; then - export RUST_BACKTRACE=1 - time cargo nextest run $test_features --retries 3 --no-fail-fast --no-tests=pass --test-threads=1 "live_migration_sequential::$test_filter" -- ${test_binary_args[*]} - RES=$? -fi - -exit $RES diff --git a/scripts/run_integration_tests_x86_64.sh b/scripts/run_integration_tests_x86_64.sh index d4bf92847..eab250809 100755 --- a/scripts/run_integration_tests_x86_64.sh +++ b/scripts/run_integration_tests_x86_64.sh @@ -11,6 +11,7 @@ mkdir -p "$WORKLOADS_DIR" process_common_args "$@" +migratable_version=v39.0 # For now these values are default for kvm test_features="" @@ -18,6 +19,16 @@ if [ "$hypervisor" = "mshv" ]; then test_features="--features mshv" fi +# if migratable version is set to override the default +if [ -n "${MIGRATABLE_VERSION}" ]; then + # validate the version if matched with vxx.0 + if ! [[ "${MIGRATABLE_VERSION}" =~ ^v[0-9]{2,}\.[0-9]$ ]]; then + echo "MIGRATABLE_VERSION should be in format vxx.0, e.g. v47.0" + exit 1 + fi + migratable_version=${MIGRATABLE_VERSION} +fi + cp scripts/sha1sums-x86_64* "$WORKLOADS_DIR" if [ ! -f "$WORKLOADS_DIR/hypervisor-fw" ]; then @@ -111,6 +122,14 @@ if ! sha1sum sha1sums-x86_64 sha1sums-x86_64-common --check; then fi popd || exit +# Download Cloud Hypervisor binary from its last stable release for live-upgrade tests +CH_RELEASE_URL="https://github.com/cloud-hypervisor/cloud-hypervisor/releases/download/${migratable_version}/cloud-hypervisor-static" +CH_RELEASE_NAME="cloud-hypervisor-static" +pushd "$WORKLOADS_DIR" || exit +time wget --quiet $CH_RELEASE_URL -O "$CH_RELEASE_NAME" || exit 1 +chmod +x $CH_RELEASE_NAME +popd || exit + # Build custom kernel based on virtio-pmem and virtio-fs upstream patches VMLINUX_IMAGE="$WORKLOADS_DIR/vmlinux-x86_64" if [ ! -f "$VMLINUX_IMAGE" ]; then @@ -204,6 +223,17 @@ if [ $RES -eq 0 ]; then RES=$? fi +# Run all live-migration test cases +if [ $RES -eq 0 ]; then + time cargo nextest run $test_features --retries 3 --no-fail-fast --no-tests=pass --test-threads="${PARALLEL_INTEGRATION_TESTS_NUM}" "live_migration_parallel::$test_filter" -- ${test_binary_args[*]} + RES=$? +fi + +if [ $RES -eq 0 ]; then + time cargo nextest run $test_features --retries 3 --no-fail-fast --no-tests=pass --test-threads=1 "live_migration_sequential::$test_filter" -- ${test_binary_args[*]} + RES=$? +fi + # Run tests on dbus_api if [ $RES -eq 0 ]; then cargo build --features "mshv,dbus_api" --all --release --target "$BUILD_TARGET"