diff --git a/.github/workflows/integration-x86-64.yaml b/.github/workflows/integration-x86-64.yaml index 213442d98..38dbcb06f 100644 --- a/.github/workflows/integration-x86-64.yaml +++ b/.github/workflows/integration-x86-64.yaml @@ -47,12 +47,8 @@ jobs: run: sudo modprobe openvswitch - name: Run integration tests if: ${{ github.event_name != 'pull_request' || (matrix.runner == 'garm-jammy' && matrix.libc == 'gnu') }} - timeout-minutes: 60 + timeout-minutes: 80 run: scripts/dev_cli.sh tests --integration --libc ${{ matrix.libc }} - - name: Run live-migration integration tests - if: ${{ github.event_name != 'pull_request' || (matrix.runner == 'garm-jammy' && matrix.libc == 'gnu') }} - timeout-minutes: 20 - run: scripts/dev_cli.sh tests --integration-live-migration --libc ${{ matrix.libc }} - name: Skipping build for PR if: ${{ github.event_name == 'pull_request' && matrix.runner != 'garm-jammy' && matrix.libc != 'gnu' }} run: echo "Skipping build for PR" diff --git a/scripts/dev_cli.sh b/scripts/dev_cli.sh index a5d3947ce..21d636280 100755 --- a/scripts/dev_cli.sh +++ b/scripts/dev_cli.sh @@ -259,7 +259,6 @@ cmd_help() { echo " --integration Run the integration tests." echo " --integration-vfio Run the VFIO integration tests." echo " --integration-windows Run the Windows guest integration tests." - echo " --integration-live-migration Run the live-migration integration tests." echo " --integration-rate-limiter Run the rate-limiter integration tests." echo " --integration-cvm Run the Confidential VM integration tests." echo " --libc Select the C library Cloud Hypervisor will be built against. Default is gnu" @@ -395,7 +394,6 @@ cmd_tests() { integration=false integration_vfio=false integration_windows=false - integration_live_migration=false integration_rate_limiter=false integration_cvm=false metrics=false @@ -414,7 +412,6 @@ cmd_tests() { "--integration") { integration=true; } ;; "--integration-vfio") { integration_vfio=true; } ;; "--integration-windows") { integration_windows=true; } ;; - "--integration-live-migration") { integration_live_migration=true; } ;; "--integration-rate-limiter") { integration_rate_limiter=true; } ;; "--integration-cvm") { integration_cvm=true; } ;; "--metrics") { metrics=true; } ;; @@ -515,6 +512,7 @@ cmd_tests() { --env PARALLEL_INTEGRATION_TESTS_NUM="${PARALLEL_INTEGRATION_TESTS_NUM:-}" \ --env AUTH_DOWNLOAD_TOKEN="$AUTH_DOWNLOAD_TOKEN" \ --env LLVM_PROFILE_FILE="$LLVM_PROFILE_FILE" \ + --env MIGRATABLE_VERSION="$MIGRATABLE_VERSION" \ "$CTR_IMAGE" \ dbus-run-session ./scripts/run_integration_tests_"$(uname -m)".sh "$@" || fix_dir_perms $? || exit $? fi @@ -595,33 +593,6 @@ cmd_tests() { ./scripts/run_integration_tests_windows_"$(uname -m)".sh "$@" || fix_dir_perms $? || exit $? fi - if [ "$integration_live_migration" = true ]; then - say "Running 'live migration' integration tests for $target..." - run_container "$DOCKER_RUNTIME" run \ - --name "$CLH_CTR_NAME" \ - --workdir "$CTR_CLH_ROOT_DIR" \ - --rm \ - --privileged \ - --security-opt seccomp=unconfined \ - --ipc=host \ - --net="$CTR_CLH_NET" \ - --mount type=tmpfs,destination=/tmp \ - --volume /dev:/dev \ - --volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR" \ - ${exported_volumes:+$exported_volumes} \ - --volume "$CLH_INTEGRATION_WORKLOADS:$CTR_CLH_INTEGRATION_WORKLOADS" \ - --env USER="root" \ - --env BUILD_TARGET="$target" \ - --env RUSTFLAGS="$rustflags" \ - --env TARGET_CC="$target_cc" \ - --env PARALLEL_INTEGRATION_TESTS_NUM="${PARALLEL_INTEGRATION_TESTS_NUM:-}" \ - --env AUTH_DOWNLOAD_TOKEN="$AUTH_DOWNLOAD_TOKEN" \ - --env LLVM_PROFILE_FILE="$LLVM_PROFILE_FILE" \ - --env MIGRATABLE_VERSION="$MIGRATABLE_VERSION" \ - "$CTR_IMAGE" \ - ./scripts/run_integration_tests_live_migration.sh "$@" || fix_dir_perms $? || exit $? - fi - if [ "$integration_rate_limiter" = true ]; then say "Running 'rate limiter' integration tests for $target..." run_container "$DOCKER_RUNTIME" run \ diff --git a/scripts/run_integration_tests_live_migration.sh b/scripts/run_integration_tests_live_migration.sh deleted file mode 100755 index 5198e2c89..000000000 --- a/scripts/run_integration_tests_live_migration.sh +++ /dev/null @@ -1,108 +0,0 @@ -#!/usr/bin/env bash -# shellcheck disable=SC2048,SC2086 -set -x - -# shellcheck source=/dev/null -source "$HOME"/.cargo/env -source "$(dirname "$0")"/test-util.sh - -WORKLOADS_DIR="$HOME/workloads" -mkdir -p "$WORKLOADS_DIR" - -process_common_args "$@" - -migratable_version=v39.0 -# For now these values are default for kvm -test_features="" - -if [ "$hypervisor" = "mshv" ]; then - test_features="--features mshv" -fi - -# if migratable version is set to override the default -if [ -n "${MIGRATABLE_VERSION}" ]; then - # validate the version if matched with vxx.0 - if ! [[ "${MIGRATABLE_VERSION}" =~ ^v[0-9]{2,}\.[0-9]$ ]]; then - echo "MIGRATABLE_VERSION should be in format vxx.0, e.g. v47.0" - exit 1 - fi - migratable_version=${MIGRATABLE_VERSION} -fi -cp scripts/sha1sums-x86_64* "$WORKLOADS_DIR" - -FOCAL_OS_IMAGE_NAME="focal-server-cloudimg-amd64-custom-20210609-0.qcow2" -FOCAL_OS_IMAGE_URL="https://ch-images.azureedge.net/$FOCAL_OS_IMAGE_NAME" -FOCAL_OS_IMAGE="$WORKLOADS_DIR/$FOCAL_OS_IMAGE_NAME" -if [ ! -f "$FOCAL_OS_IMAGE" ]; then - pushd "$WORKLOADS_DIR" || exit - time wget --quiet $FOCAL_OS_IMAGE_URL || exit 1 - popd || exit -fi - -FOCAL_OS_RAW_IMAGE_NAME="focal-server-cloudimg-amd64-custom-20210609-0.raw" -FOCAL_OS_RAW_IMAGE="$WORKLOADS_DIR/$FOCAL_OS_RAW_IMAGE_NAME" -if [ ! -f "$FOCAL_OS_RAW_IMAGE" ]; then - pushd "$WORKLOADS_DIR" || exit - time qemu-img convert -p -f qcow2 -O raw $FOCAL_OS_IMAGE_NAME $FOCAL_OS_RAW_IMAGE_NAME || exit 1 - popd || exit -fi - -pushd "$WORKLOADS_DIR" || exit -if ! grep focal sha1sums-x86_64-common | sha1sum --check; then - echo "sha1sum validation of images failed, remove invalid images to fix the issue." - exit 1 -fi -popd || exit - -# Download Cloud Hypervisor binary from its last stable release -CH_RELEASE_URL="https://github.com/cloud-hypervisor/cloud-hypervisor/releases/download/${migratable_version}/cloud-hypervisor-static" -CH_RELEASE_NAME="cloud-hypervisor-static" -pushd "$WORKLOADS_DIR" || exit -time wget --quiet $CH_RELEASE_URL -O "$CH_RELEASE_NAME" || exit 1 -chmod +x $CH_RELEASE_NAME -popd || exit - -# Build custom kernel based on virtio-pmem and virtio-fs upstream patches -VMLINUX_IMAGE="$WORKLOADS_DIR/vmlinux-x86_64" -if [ ! -f "$VMLINUX_IMAGE" ]; then - # Prepare linux image (build from source or download pre-built) - prepare_linux -fi - -CFLAGS="" -if [[ "${BUILD_TARGET}" == "x86_64-unknown-linux-musl" ]]; then - # shellcheck disable=SC2034 - CFLAGS="-I /usr/include/x86_64-linux-musl/ -idirafter /usr/include/" -fi - -cargo build --features mshv --all --release --target "$BUILD_TARGET" - -# Test ovs-dpdk relies on hugepages -HUGEPAGESIZE=$(grep Hugepagesize /proc/meminfo | awk '{print $2}') -PAGE_NUM=$((12288 * 1024 / HUGEPAGESIZE)) -echo "$PAGE_NUM" | sudo tee /proc/sys/vm/nr_hugepages -sudo chmod a+rwX /dev/hugepages - -# Common configuration for every test run -export RUST_BACKTRACE=1 -export RUSTFLAGS="$RUSTFLAGS" - -TEST_THREADS_DEFAULT="$(($(nproc) / 4))" -if ! [[ "${PARALLEL_INTEGRATION_TESTS_NUM:-}" =~ ^[1-9][0-9]*$ ]]; then - PARALLEL_INTEGRATION_TESTS_NUM="${TEST_THREADS_DEFAULT}" -fi -echo "nproc:$(nproc), parallel_integration_tests:${PARALLEL_INTEGRATION_TESTS_NUM}" - -time cargo nextest run $test_features --retries 3 --no-fail-fast --no-tests=pass --test-threads="${PARALLEL_INTEGRATION_TESTS_NUM}" "live_migration_parallel::$test_filter" -- ${test_binary_args[*]} - -RES=$? - -# Run some tests in sequence since the result could be affected by other tests -# running in parallel. -if [ $RES -eq 0 ]; then - export RUST_BACKTRACE=1 - time cargo nextest run $test_features --retries 3 --no-fail-fast --no-tests=pass --test-threads=1 "live_migration_sequential::$test_filter" -- ${test_binary_args[*]} - RES=$? -fi - -exit $RES diff --git a/scripts/run_integration_tests_x86_64.sh b/scripts/run_integration_tests_x86_64.sh index d4bf92847..eab250809 100755 --- a/scripts/run_integration_tests_x86_64.sh +++ b/scripts/run_integration_tests_x86_64.sh @@ -11,6 +11,7 @@ mkdir -p "$WORKLOADS_DIR" process_common_args "$@" +migratable_version=v39.0 # For now these values are default for kvm test_features="" @@ -18,6 +19,16 @@ if [ "$hypervisor" = "mshv" ]; then test_features="--features mshv" fi +# if migratable version is set to override the default +if [ -n "${MIGRATABLE_VERSION}" ]; then + # validate the version if matched with vxx.0 + if ! [[ "${MIGRATABLE_VERSION}" =~ ^v[0-9]{2,}\.[0-9]$ ]]; then + echo "MIGRATABLE_VERSION should be in format vxx.0, e.g. v47.0" + exit 1 + fi + migratable_version=${MIGRATABLE_VERSION} +fi + cp scripts/sha1sums-x86_64* "$WORKLOADS_DIR" if [ ! -f "$WORKLOADS_DIR/hypervisor-fw" ]; then @@ -111,6 +122,14 @@ if ! sha1sum sha1sums-x86_64 sha1sums-x86_64-common --check; then fi popd || exit +# Download Cloud Hypervisor binary from its last stable release for live-upgrade tests +CH_RELEASE_URL="https://github.com/cloud-hypervisor/cloud-hypervisor/releases/download/${migratable_version}/cloud-hypervisor-static" +CH_RELEASE_NAME="cloud-hypervisor-static" +pushd "$WORKLOADS_DIR" || exit +time wget --quiet $CH_RELEASE_URL -O "$CH_RELEASE_NAME" || exit 1 +chmod +x $CH_RELEASE_NAME +popd || exit + # Build custom kernel based on virtio-pmem and virtio-fs upstream patches VMLINUX_IMAGE="$WORKLOADS_DIR/vmlinux-x86_64" if [ ! -f "$VMLINUX_IMAGE" ]; then @@ -204,6 +223,17 @@ if [ $RES -eq 0 ]; then RES=$? fi +# Run all live-migration test cases +if [ $RES -eq 0 ]; then + time cargo nextest run $test_features --retries 3 --no-fail-fast --no-tests=pass --test-threads="${PARALLEL_INTEGRATION_TESTS_NUM}" "live_migration_parallel::$test_filter" -- ${test_binary_args[*]} + RES=$? +fi + +if [ $RES -eq 0 ]; then + time cargo nextest run $test_features --retries 3 --no-fail-fast --no-tests=pass --test-threads=1 "live_migration_sequential::$test_filter" -- ${test_binary_args[*]} + RES=$? +fi + # Run tests on dbus_api if [ $RES -eq 0 ]; then cargo build --features "mshv,dbus_api" --all --release --target "$BUILD_TARGET"