diff --git a/.github/workflows/integration-arm64.yaml b/.github/workflows/integration-arm64.yaml index 873daaa74..d34d28c3c 100644 --- a/.github/workflows/integration-arm64.yaml +++ b/.github/workflows/integration-arm64.yaml @@ -7,6 +7,11 @@ concurrency: jobs: build: timeout-minutes: 120 + env: + # Our runner has 80 cores (nproc). + # We limit parallelism only to avoid exhausting disk space and memory + # resources, not to save CPU resources. + PARALLEL_INTEGRATION_TESTS_NUM: 25 name: Tests (ARM64) runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-latest' || 'bookworm-arm64' }} steps: diff --git a/.github/workflows/integration-x86-64.yaml b/.github/workflows/integration-x86-64.yaml index 98e2a8dcd..213442d98 100644 --- a/.github/workflows/integration-x86-64.yaml +++ b/.github/workflows/integration-x86-64.yaml @@ -7,6 +7,11 @@ concurrency: jobs: build: timeout-minutes: 80 + env: + # Our runner has 16 cores (nproc). + # We limit parallelism only to avoid exhausting disk space and memory + # resources, not to save CPU resources. + PARALLEL_INTEGRATION_TESTS_NUM: 12 strategy: fail-fast: false matrix: diff --git a/scripts/dev_cli.sh b/scripts/dev_cli.sh index 3b9451321..a5d3947ce 100755 --- a/scripts/dev_cli.sh +++ b/scripts/dev_cli.sh @@ -512,6 +512,7 @@ cmd_tests() { --env BUILD_TARGET="$target" \ --env RUSTFLAGS="$rustflags" \ --env TARGET_CC="$target_cc" \ + --env PARALLEL_INTEGRATION_TESTS_NUM="${PARALLEL_INTEGRATION_TESTS_NUM:-}" \ --env AUTH_DOWNLOAD_TOKEN="$AUTH_DOWNLOAD_TOKEN" \ --env LLVM_PROFILE_FILE="$LLVM_PROFILE_FILE" \ "$CTR_IMAGE" \ @@ -613,6 +614,7 @@ cmd_tests() { --env BUILD_TARGET="$target" \ --env RUSTFLAGS="$rustflags" \ --env TARGET_CC="$target_cc" \ + --env PARALLEL_INTEGRATION_TESTS_NUM="${PARALLEL_INTEGRATION_TESTS_NUM:-}" \ --env AUTH_DOWNLOAD_TOKEN="$AUTH_DOWNLOAD_TOKEN" \ --env LLVM_PROFILE_FILE="$LLVM_PROFILE_FILE" \ --env MIGRATABLE_VERSION="$MIGRATABLE_VERSION" \ diff --git a/scripts/run_integration_tests_aarch64.sh b/scripts/run_integration_tests_aarch64.sh index 5a1dce52b..fb9d5daa5 100755 --- a/scripts/run_integration_tests_aarch64.sh +++ b/scripts/run_integration_tests_aarch64.sh @@ -247,8 +247,13 @@ PAGE_NUM=$((12288 * 1024 / HUGEPAGESIZE)) echo "$PAGE_NUM" | sudo tee /proc/sys/vm/nr_hugepages sudo chmod a+rwX /dev/hugepages +TEST_THREADS_DEFAULT="$(($(nproc) / 4))" +if ! [[ "${PARALLEL_INTEGRATION_TESTS_NUM:-}" =~ ^[1-9][0-9]*$ ]]; then + PARALLEL_INTEGRATION_TESTS_NUM="${TEST_THREADS_DEFAULT}" +fi +echo "nproc:$(nproc), parallel_integration_tests:${PARALLEL_INTEGRATION_TESTS_NUM}" # Run all direct kernel boot (Device Tree) test cases in mod `parallel` -time cargo nextest run $test_features --retries 3 --no-fail-fast --no-tests=pass --test-threads=$(($(nproc) / 4)) "common_parallel::$test_filter" -- ${test_binary_args[*]} +time cargo nextest run $test_features --retries 3 --no-fail-fast --no-tests=pass --test-threads="${PARALLEL_INTEGRATION_TESTS_NUM}" "common_parallel::$test_filter" -- ${test_binary_args[*]} RES=$? # Run some tests in sequence since the result could be affected by other tests @@ -262,7 +267,7 @@ fi # Run all ACPI test cases if [ $RES -eq 0 ]; then - time cargo nextest run $test_features --retries 3 --no-fail-fast --no-tests=pass --test-threads=$(($(nproc) / 4)) "aarch64_acpi::$test_filter" -- ${test_binary_args[*]} + time cargo nextest run $test_features --retries 3 --no-fail-fast --no-tests=pass --test-threads="$TEST_THREADS_DEFAULT" "aarch64_acpi::$test_filter" -- ${test_binary_args[*]} RES=$? else exit $RES @@ -270,7 +275,7 @@ fi # Run all test cases related to live migration if [ $RES -eq 0 ]; then - time cargo nextest run $test_features --retries 3 --no-fail-fast --no-tests=pass --test-threads=$(($(nproc) / 4)) "live_migration_parallel::$test_filter" -- ${test_binary_args[*]} + time cargo nextest run $test_features --retries 3 --no-fail-fast --no-tests=pass --test-threads="$TEST_THREADS_DEFAULT" "live_migration_parallel::$test_filter" -- ${test_binary_args[*]} RES=$? else exit $RES @@ -288,7 +293,7 @@ if [ $RES -eq 0 ]; then cargo build --features "mshv,dbus_api" --all --release --target "$BUILD_TARGET" export RUST_BACKTRACE=1 # integration tests now do not reply on build feature "dbus_api" - time cargo nextest run $test_features --retries 3 --no-fail-fast --no-tests=pass --test-threads=$(($(nproc) / 4)) "dbus_api::$test_filter" -- ${test_binary_args[*]} + time cargo nextest run $test_features --retries 3 --no-fail-fast --no-tests=pass --test-threads="$TEST_THREADS_DEFAULT" "dbus_api::$test_filter" -- ${test_binary_args[*]} RES=$? fi @@ -296,14 +301,14 @@ fi if [ $RES -eq 0 ]; then cargo build --features "mshv,fw_cfg" --all --release --target "$BUILD_TARGET" export RUST_BACKTRACE=1 - time cargo nextest run $test_features --retries 3 --no-fail-fast --no-tests=pass --test-threads=$(($(nproc) / 4)) "fw_cfg::$test_filter" -- ${test_binary_args[*]} + time cargo nextest run $test_features --retries 3 --no-fail-fast --no-tests=pass --test-threads="$TEST_THREADS_DEFAULT" "fw_cfg::$test_filter" -- ${test_binary_args[*]} RES=$? fi if [ $RES -eq 0 ]; then cargo build --features "mshv,ivshmem" --all --release --target "$BUILD_TARGET" export RUST_BACKTRACE=1 - time cargo nextest run $test_features --retries 3 --no-fail-fast --no-tests=pass --test-threads=$(($(nproc) / 4)) "ivshmem::$test_filter" -- ${test_binary_args[*]} + time cargo nextest run $test_features --retries 3 --no-fail-fast --no-tests=pass --test-threads="$TEST_THREADS_DEFAULT" "ivshmem::$test_filter" -- ${test_binary_args[*]} RES=$? fi diff --git a/scripts/run_integration_tests_live_migration.sh b/scripts/run_integration_tests_live_migration.sh index 11cf59b41..5198e2c89 100755 --- a/scripts/run_integration_tests_live_migration.sh +++ b/scripts/run_integration_tests_live_migration.sh @@ -87,7 +87,13 @@ sudo chmod a+rwX /dev/hugepages export RUST_BACKTRACE=1 export RUSTFLAGS="$RUSTFLAGS" -time cargo nextest run $test_features --retries 3 --no-fail-fast --no-tests=pass --test-threads=$(($(nproc) / 4)) "live_migration_parallel::$test_filter" -- ${test_binary_args[*]} +TEST_THREADS_DEFAULT="$(($(nproc) / 4))" +if ! [[ "${PARALLEL_INTEGRATION_TESTS_NUM:-}" =~ ^[1-9][0-9]*$ ]]; then + PARALLEL_INTEGRATION_TESTS_NUM="${TEST_THREADS_DEFAULT}" +fi +echo "nproc:$(nproc), parallel_integration_tests:${PARALLEL_INTEGRATION_TESTS_NUM}" + +time cargo nextest run $test_features --retries 3 --no-fail-fast --no-tests=pass --test-threads="${PARALLEL_INTEGRATION_TESTS_NUM}" "live_migration_parallel::$test_filter" -- ${test_binary_args[*]} RES=$? diff --git a/scripts/run_integration_tests_x86_64.sh b/scripts/run_integration_tests_x86_64.sh index f12fa9aaa..d4bf92847 100755 --- a/scripts/run_integration_tests_x86_64.sh +++ b/scripts/run_integration_tests_x86_64.sh @@ -189,7 +189,12 @@ ulimit -n 4096 export RUST_BACKTRACE=1 export RUSTFLAGS="$RUSTFLAGS" -time cargo nextest run $test_features --retries 3 --no-fail-fast --no-tests=pass --test-threads=$(($(nproc) / 4)) "common_parallel::$test_filter" -- ${test_binary_args[*]} +TEST_THREADS_DEFAULT="$(($(nproc) / 4))" +if ! [[ "${PARALLEL_INTEGRATION_TESTS_NUM:-}" =~ ^[1-9][0-9]*$ ]]; then + PARALLEL_INTEGRATION_TESTS_NUM="${TEST_THREADS_DEFAULT}" +fi +echo "nproc:$(nproc), parallel_integration_tests:${PARALLEL_INTEGRATION_TESTS_NUM}" +time cargo nextest run $test_features --retries 3 --no-fail-fast --no-tests=pass --test-threads="${PARALLEL_INTEGRATION_TESTS_NUM}" "common_parallel::$test_filter" -- ${test_binary_args[*]} RES=$? # Run some tests in sequence since the result could be affected by other tests @@ -203,20 +208,20 @@ fi if [ $RES -eq 0 ]; then cargo build --features "mshv,dbus_api" --all --release --target "$BUILD_TARGET" # integration tests now do not reply on build feature "dbus_api" - time cargo nextest run $test_features --retries 3 --no-fail-fast --no-tests=pass --test-threads=$(($(nproc) / 4)) "dbus_api::$test_filter" -- ${test_binary_args[*]} + time cargo nextest run $test_features --retries 3 --no-fail-fast --no-tests=pass --test-threads="$TEST_THREADS_DEFAULT" "dbus_api::$test_filter" -- ${test_binary_args[*]} RES=$? fi # Run tests on fw_cfg if [ $RES -eq 0 ]; then cargo build --features "mshv,fw_cfg" --all --release --target "$BUILD_TARGET" - time cargo nextest run $test_features --retries 3 --no-fail-fast --no-tests=pass --test-threads=$(($(nproc) / 4)) "fw_cfg::$test_filter" -- ${test_binary_args[*]} + time cargo nextest run $test_features --retries 3 --no-fail-fast --no-tests=pass --test-threads="$TEST_THREADS_DEFAULT" "fw_cfg::$test_filter" -- ${test_binary_args[*]} RES=$? fi if [ $RES -eq 0 ]; then cargo build --features "mshv,ivshmem" --all --release --target "$BUILD_TARGET" - time cargo nextest run $test_features --retries 3 --no-fail-fast --no-tests=pass --test-threads=$(($(nproc) / 4)) "ivshmem::$test_filter" -- ${test_binary_args[*]} + time cargo nextest run $test_features --retries 3 --no-fail-fast --no-tests=pass --test-threads="$TEST_THREADS_DEFAULT" "ivshmem::$test_filter" -- ${test_binary_args[*]} RES=$? fi