From 441da416738ff83d3c2c74e461588d9dc1e96de4 Mon Sep 17 00:00:00 2001 From: cx Date: Thu, 29 Oct 2026 10:02:09 +0000 Subject: [PATCH 1/2] fix: retry failed model tests and summarize final results --- scripts/run_models_and_profile.bash | 154 ++++++++++++++++++++++------ 1 file changed, 122 insertions(+), 32 deletions(-) diff --git a/scripts/run_models_and_profile.bash b/scripts/run_models_and_profile.bash index 84c6351f6..b3629d3bc 100755 --- a/scripts/run_models_and_profile.bash +++ b/scripts/run_models_and_profile.bash @@ -1,5 +1,7 @@ #!/bin/bash +# Parse the entire run up front so edits during execution cannot shift Bash's read position. +{ set -e set -o pipefail @@ -90,6 +92,9 @@ done < <(jq -r '.variables | to_entries[] | "\(.key)=\(.value)"' "$CONFIG_FILE") # Global variable to save the last cmake command LAST_CMAKE_CMD="" declare -A SELECTED_TAGS=() +TOTAL_TESTS=0 +FAILED_TEST_CASES=() +FAILED_TEST_LOGS=() RUN_STARTED_AT="$(date '+%Y-%m-%d %H:%M:%S')" RUN_ID="$(date '+%Y%m%d_%H%M%S')" @@ -188,15 +193,43 @@ clean_build_dir() { # Clean checkpoint directories (called once at start of script) clean_checkpoints() { - echo -e "\033[1;31m[CLEAN] Removing checkpoint directories from previous run\033[0m" if [[ -d "$CKPT_ROOT_DIR" ]]; then echo -e "\033[1;31m[CLEAN] Removing: ${CKPT_ROOT_DIR}\033[0m" rm -rf "${CKPT_ROOT_DIR:?}" fi } +record_test_result() { + TOTAL_TESTS=$((TOTAL_TESTS + 1)) + if [[ "$1" -ne 0 ]]; then + FAILED_TEST_CASES+=("$2") + FAILED_TEST_LOGS+=("$3") + fi +} + +print_test_summary() { + local failed="${#FAILED_TEST_CASES[@]}" + local color='\033[1;32m' + if ((failed > 0)); then + color='\033[1;31m' + fi + + printf '\n%b' "$color" + { + echo "============================================================" + printf '[SUMMARY] Failed / Total test cases: %d / %d\n' "$failed" "$TOTAL_TESTS" + local idx + for ((idx=0; idx /dev/null - - # Write the last cmake command into the log file if available - if [[ -n "$LAST_CMAKE_CMD" ]]; then - echo "[LAST_CMAKE] $LAST_CMAKE_CMD" > "$log_path" - else - # If no cmake command has been run yet, clear the log - > "$log_path" + local expanded_cmd="${cmd//\$LORA_WEIGHTS_DIR/$LORA_WEIGHTS_DIR}" + local max_attempts=2 + # Build failures remain fatal; CTest retries individual tests internally. + if [[ "$tag" == "build" || "$tag" == "ctest" ]]; then + max_attempts=1 fi - # Write the current run command to the log - echo "[RUN_METADATA] $RUN_METADATA_FILE" >> "$log_path" - echo "[RUN_STARTED_AT] $RUN_STARTED_AT" >> "$log_path" - echo "[GIT_BRANCH] $GIT_BRANCH" >> "$log_path" - echo "[GIT_COMMIT] $GIT_COMMIT_FULL" >> "$log_path" - echo "[GIT_COMMIT_SHORT] $GIT_COMMIT_SHORT" >> "$log_path" - local expanded_cmd="${cmd//\$LORA_WEIGHTS_DIR/$LORA_WEIGHTS_DIR}" - echo "[COMMAND] $expanded_cmd" >> "$log_path" + local attempt exit_code=0 + for ((attempt=1; attempt<=max_attempts; attempt++)); do + # Overwrite the previous attempt while preserving metadata for the final log. + { + if [[ -n "$LAST_CMAKE_CMD" ]]; then + echo "[LAST_CMAKE] $LAST_CMAKE_CMD" + fi + echo "[RUN_METADATA] $RUN_METADATA_FILE" + echo "[RUN_STARTED_AT] $RUN_STARTED_AT" + echo "[GIT_BRANCH] $GIT_BRANCH" + echo "[GIT_COMMIT] $GIT_COMMIT_FULL" + echo "[GIT_COMMIT_SHORT] $GIT_COMMIT_SHORT" + echo "[COMMAND] $expanded_cmd" + } > "$log_path" + pushd "$BUILD_DIR" > /dev/null + + # Append this attempt's stdout and stderr after the metadata. + if eval "$cmd" >> "$log_path" 2>&1; then + exit_code=0 + else + exit_code=$? + fi + + popd > /dev/null + echo "[EXIT_CODE] $exit_code" >> "$log_path" + + if [[ "$is_profile" == "yes" ]]; then + move_profile_logs "$log_name" "$tag" + fi + + if [[ "$exit_code" -eq 0 ]]; then + break + fi - # Run the command and append both stdout and stderr to the log file - if ! eval "$cmd" >> "$log_path" 2>&1; then echo -e "\033[1;31m============================================================\033[0m" - echo -e "\033[1;31m[ERROR] Command failed: ${cmd}\033[0m" + echo -e "\033[1;31m[ERROR] Attempt ${attempt}/${max_attempts} failed (exit code ${exit_code}): ${cmd}\033[0m" echo -e "\033[1;31m[ERROR] See log file for details: ${log_path}\033[0m" echo -e "\033[1;31m============================================================\033[0m" echo "" echo "[ERROR] Last 20 lines of log:" tail -20 "$log_path" - exit 1 - fi + if ((attempt < max_attempts)); then + echo "[RETRY] Retrying ${log_name} once." | tee -a "$log_path" + fi + done - popd > /dev/null + if [[ "$tag" == "ctest" ]]; then + # Collect parallel worker results from the existing CTest log into parent-shell memory. + local result_marker result_code result_name + while IFS=$'\t' read -r result_marker result_code result_name; do + if [[ "$result_marker" == "[CTEST_RESULT]" ]]; then + record_test_result "$result_code" "$result_name" "$log_path" + fi + done < "$log_path" + elif [[ "$tag" != "build" ]]; then + record_test_result "$exit_code" "${build_id}/${tag}/${log_name}" "$log_path" + fi - # If profiling is enabled, move profiling files to the target directory - if [[ "$is_profile" == "yes" ]]; then - move_profile_logs "$log_name" "$tag" + if [[ "$exit_code" -ne 0 ]]; then + if [[ "$tag" == "build" ]]; then + exit "$exit_code" + fi + echo "[FAILED] ${log_name}: continuing to the next test." | tee -a "$log_path" fi } @@ -521,7 +608,6 @@ for ((id=0; id Date: Fri, 20 Nov 2026 07:49:13 +0000 Subject: [PATCH 2/2] fix(tests): retry unstable tests and report retry results - Add configurable MAX_TEST_ATTEMPTS for CTest and model test retries - Classify results as pass, pass-on-retry, and fail in the summary - List retry-passed cases and preserve nonzero exit status on failures - Clean checkpoint save directories before retrying save cases --- scripts/run_models_and_profile.bash | 177 +++++++++++++++++++++++----- 1 file changed, 146 insertions(+), 31 deletions(-) diff --git a/scripts/run_models_and_profile.bash b/scripts/run_models_and_profile.bash index b3629d3bc..9699567c3 100755 --- a/scripts/run_models_and_profile.bash +++ b/scripts/run_models_and_profile.bash @@ -13,6 +13,9 @@ Options: --test-config PATH Path to test config JSON. Default: test_config.json. --only-run TAGS Only run the specified tag groups, separated by commas. -h, --help Show this help message. + +Configuration: + MAX_TEST_ATTEMPTS Maximum attempts for model tests and CTest retries. Set it in the environment or test config. Default: 2. EOF } @@ -82,6 +85,7 @@ LLAMA3_TEST_GROUPS="$(read_var LLAMA3_TEST_GROUPS)"; : "${LLAMA3_TEST_GROUP QWEN3_TEST_GROUPS="$(read_var QWEN3_TEST_GROUPS)"; : "${QWEN3_TEST_GROUPS:=qwen3}" MIXTRAL_TEST_GROUPS="$(read_var MIXTRAL_TEST_GROUPS)"; : "${MIXTRAL_TEST_GROUPS:=moe}" DEVICE_BACKEND="$(read_var DEVICE_BACKEND)"; : "${DEVICE_BACKEND:=cuda}" +MAX_TEST_ATTEMPTS="$(read_var MAX_TEST_ATTEMPTS)"; : "${MAX_TEST_ATTEMPTS:=2}" # export custom variables from config first. LOG_DIR/PROFILE_LOG_DIR are normalized below. while IFS="=" read -r k v; do @@ -89,12 +93,19 @@ while IFS="=" read -r k v; do export "$k"="$v" done < <(jq -r '.variables | to_entries[] | "\(.key)=\(.value)"' "$CONFIG_FILE") +if [[ ! "$MAX_TEST_ATTEMPTS" =~ ^[1-9][0-9]*$ ]]; then + echo "Error: MAX_TEST_ATTEMPTS must be a positive integer, got '${MAX_TEST_ATTEMPTS}'." >&2 + exit 1 +fi + # Global variable to save the last cmake command LAST_CMAKE_CMD="" declare -A SELECTED_TAGS=() TOTAL_TESTS=0 FAILED_TEST_CASES=() FAILED_TEST_LOGS=() +PASSED_ON_RETRY_TEST_CASES=() +PASSED_ON_RETRY_TEST_LOGS=() RUN_STARTED_AT="$(date '+%Y-%m-%d %H:%M:%S')" RUN_ID="$(date '+%Y%m%d_%H%M%S')" @@ -200,15 +211,103 @@ clean_checkpoints() { } record_test_result() { + local result="$1" + local test_case="$2" + local log_path="$3" + TOTAL_TESTS=$((TOTAL_TESTS + 1)) - if [[ "$1" -ne 0 ]]; then - FAILED_TEST_CASES+=("$2") - FAILED_TEST_LOGS+=("$3") + case "$result" in + pass) + ;; + pass-on-retry) + PASSED_ON_RETRY_TEST_CASES+=("$test_case") + PASSED_ON_RETRY_TEST_LOGS+=("$log_path") + ;; + fail) + FAILED_TEST_CASES+=("$test_case") + FAILED_TEST_LOGS+=("$log_path") + ;; + *) + echo "Error: Unknown test result '${result}' for ${test_case}." >&2 + return 1 + ;; + esac +} + +emit_ctest_results() { + local output_file="$1" + + awk ' + function is_test_status_line() { + return $0 ~ /^[[:space:]]*([0-9]+\/[0-9]+[[:space:]]+)?Test[[:space:]]+#[0-9]+:/ + } + + is_test_status_line() && (/Passed/ || /\*\*\*/) { + for (i = 1; i <= NF; i++) { + if ($i == "Test") { + test_number = $(i + 1) + break + } + } + sub(/^#/, "", test_number) + sub(/:$/, "", test_number) + + test_name = $0 + sub(/^[[:space:]]*([0-9]+\/[0-9]+[[:space:]]+)?Test[[:space:]]+#[0-9]+:[[:space:]]*/, "", test_name) + sub(/[[:space:]]+[.]+[[:space:]]*(Passed|\*\*\*[^[:space:]]*).*/, "", test_name) + failed = $0 ~ /\*\*\*/ && $0 !~ /\*\*\*Skipped/ + + if (!(test_number in seen_tests)) { + seen_tests[test_number] = 1 + test_order[++test_count] = test_number + } + test_names[test_number] = test_name + if (failed) { + saw_failure[test_number] = 1 + } + final_results[test_number] = failed ? "fail" : "pass" + } + + END { + for (i = 1; i <= test_count; i++) { + test_number = test_order[i] + if (final_results[test_number] == "fail") { + result = "fail" + } else if (saw_failure[test_number]) { + result = "pass-on-retry" + } else { + result = "pass" + } + printf "[CTEST_RESULT]\t%s\t%s/%s\n", result, ENVIRON["LOG_NAME"], test_names[test_number] + } + } + ' "$output_file" +} + +clean_retry_checkpoint_dir() { + local cmd="$1" + local save_dir="" + + if [[ "$cmd" =~ [[:space:]]--save=([^[:space:]]+) ]]; then + save_dir="${BASH_REMATCH[1]}" + fi + if [[ -z "$save_dir" ]]; then + return + fi + + local ckpt_root_dir save_realpath + ckpt_root_dir="$(realpath -m "$CKPT_ROOT_DIR")" + save_realpath="$(realpath -m "$save_dir")" + if [[ "$save_realpath" == "$ckpt_root_dir"/* ]]; then + rm -rf -- "${save_realpath:?}" + echo "[RETRY CLEAN] Removed checkpoint save directory: ${save_realpath}" fi } print_test_summary() { local failed="${#FAILED_TEST_CASES[@]}" + local passed_on_retry="${#PASSED_ON_RETRY_TEST_CASES[@]}" + local passed=$((TOTAL_TESTS - failed - passed_on_retry)) local color='\033[1;32m' if ((failed > 0)); then color='\033[1;31m' @@ -218,10 +317,15 @@ print_test_summary() { { echo "============================================================" printf '[SUMMARY] Failed / Total test cases: %d / %d\n' "$failed" "$TOTAL_TESTS" + printf '[SUMMARY] Pass / Pass on retry / Fail: %d / %d / %d\n' "$passed" "$passed_on_retry" "$failed" + printf '[SUMMARY] Passed on retry: %d\n' "$passed_on_retry" local idx for ((idx=0; idx "$ctest_output_file" 2>&1; then failed=1 fi - - # Read CPU failures before concurrent CUDA runs overwrite CTest's failure list. - local -A failed_cpu_tests=() - local test_number test_name - if [[ -f Testing/Temporary/LastTestsFailed.log ]]; then - while IFS=: read -r test_number test_name; do - [[ -z "$test_name" ]] && continue - failed_cpu_tests["$test_name"]=1 - done < Testing/Temporary/LastTestsFailed.log - fi - for test_name in "${cpu_tests[@]}"; do - printf '[CTEST_RESULT]\t%s\t%s\n' "${failed_cpu_tests[$test_name]:-0}" "${log_name}/${test_name}" - done + cat "$ctest_output_file" + LOG_NAME="$log_name" emit_ctest_results "$ctest_output_file" + rm -f "$ctest_output_file" mapfile -t cuda_tests < <(ctest -N -L cuda | sed -n 's/^ *Test *#[0-9][0-9]*: //p') if [[ ${#cuda_tests[@]} -eq 0 ]]; then @@ -285,19 +379,22 @@ run_ctest() { ( local worker_failed=0 local test_idx="$worker_idx" - local test_name test_failed + local test_name + local ctest_output_file local assigned_gpu="${filtered_gpu_list[$worker_idx]}" while ((test_idx < ${#cuda_tests[@]})); do test_name="${cuda_tests[$test_idx]}" echo "[CUDA GPU ${assigned_gpu}] ${test_name}" - if CUDA_VISIBLE_DEVICES="$assigned_gpu" ctest --output-on-failure --repeat until-pass:2 -R "^${test_name}$" -j1; then - test_failed=0 + ctest_output_file="$(mktemp "${TMPDIR:-/tmp}/run_models_ctest.XXXXXX")" + if CUDA_VISIBLE_DEVICES="$assigned_gpu" ctest --output-on-failure --repeat until-pass:"$MAX_TEST_ATTEMPTS" -R "^${test_name}$" -j1 > "$ctest_output_file" 2>&1; then + : else - test_failed=1 worker_failed=1 fi - printf '[CTEST_RESULT]\t%s\t%s\n' "$test_failed" "${log_name}/${test_name}" + cat "$ctest_output_file" + LOG_NAME="$log_name" emit_ctest_results "$ctest_output_file" + rm -f "$ctest_output_file" test_idx=$((test_idx + worker_count)) done @@ -349,14 +446,18 @@ run_and_log() { echo -e "\033[1;32m============================================================\033[0m" local expanded_cmd="${cmd//\$LORA_WEIGHTS_DIR/$LORA_WEIGHTS_DIR}" - local max_attempts=2 + local max_attempts="$MAX_TEST_ATTEMPTS" # Build failures remain fatal; CTest retries individual tests internally. if [[ "$tag" == "build" || "$tag" == "ctest" ]]; then max_attempts=1 fi - local attempt exit_code=0 + local attempt exit_code=0 passed_on_retry=0 for ((attempt=1; attempt<=max_attempts; attempt++)); do + if ((attempt > 1)); then + clean_retry_checkpoint_dir "$expanded_cmd" + fi + # Overwrite the previous attempt while preserving metadata for the final log. { if [[ -n "$LAST_CMAKE_CMD" ]]; then @@ -386,6 +487,9 @@ run_and_log() { fi if [[ "$exit_code" -eq 0 ]]; then + if ((attempt > 1)); then + passed_on_retry=1 + fi break fi @@ -403,14 +507,20 @@ run_and_log() { if [[ "$tag" == "ctest" ]]; then # Collect parallel worker results from the existing CTest log into parent-shell memory. - local result_marker result_code result_name - while IFS=$'\t' read -r result_marker result_code result_name; do + local result_marker result_status result_name + while IFS=$'\t' read -r result_marker result_status result_name; do if [[ "$result_marker" == "[CTEST_RESULT]" ]]; then - record_test_result "$result_code" "$result_name" "$log_path" + record_test_result "$result_status" "$result_name" "$log_path" fi done < "$log_path" elif [[ "$tag" != "build" ]]; then - record_test_result "$exit_code" "${build_id}/${tag}/${log_name}" "$log_path" + if [[ "$exit_code" -ne 0 ]]; then + record_test_result "fail" "${build_id}/${tag}/${log_name}" "$log_path" + elif ((passed_on_retry)); then + record_test_result "pass-on-retry" "${build_id}/${tag}/${log_name}" "$log_path" + else + record_test_result "pass" "${build_id}/${tag}/${log_name}" "$log_path" + fi fi if [[ "$exit_code" -ne 0 ]]; then @@ -699,5 +809,10 @@ echo -e "\n\033[1;33mNext step:\033[0m" echo "python3 write_to_feishu_sheet.py token.json --log-dir \"$(realpath "$RUN_OUTPUT_DIR")\"" print_test_summary -exit +if (( ${#FAILED_TEST_CASES[@]} > 0 )); then + exit 1 +fi +exit 0 + +# Close the block opened near the top of the file to preload this script before execution. }