Fix redundant NPU validation reruns

Author: Zhendong404

test/npu_validation/scripts/run_remote_npu_validation.sh

@@ -12,7 +12,7 @@ set -euo pipefail
 STAGE="${STAGE:-run}"         # build|run
 RUN_MODE="${RUN_MODE:-npu}"   # npu|sim
 SOC_VERSION="${SOC_VERSION:-Ascend910}"
-GOLDEN_MODE="${GOLDEN_MODE:-npu}"  # sim|npu|skip
+GOLDEN_MODE="${GOLDEN_MODE:-npu}"  # sim|npu|npu_consistency|skip
 PTO_ISA_REPO="${PTO_ISA_REPO:-https://gitcode.com/cann/pto-isa.git}"
 PTO_ISA_COMMIT="${PTO_ISA_COMMIT:-}"
 DEVICE_ID="${DEVICE_ID:-0}"
@@ -307,6 +307,33 @@ while IFS= read -r -d '' cpp; do
       done
     }
 
+    ensure_golden_outputs_present() {
+      local missing=0
+      if [[ -f "./outputs.txt" ]]; then
+        while IFS= read -r name; do
+          [[ -n "${name}" ]] || continue
+          if [[ ! -f "./golden_${name}.bin" ]]; then
+            log "ERROR: missing golden output ./golden_${name}.bin"
+            missing=1
+          fi
+        done < "./outputs.txt"
+        if [[ ${missing} -ne 0 ]]; then
+          log "ERROR: GOLDEN_MODE=npu expects golden.py/custom assets to prepare golden_*.bin"
+          log "ERROR: use GOLDEN_MODE=npu_consistency for run-to-run NPU consistency checks"
+          return 1
+        fi
+        return 0
+      fi
+
+      if compgen -G "./golden_*.bin" > /dev/null; then
+        return 0
+      fi
+
+      log "ERROR: no golden_*.bin found for GOLDEN_MODE=npu"
+      log "ERROR: use GOLDEN_MODE=npu_consistency for run-to-run NPU consistency checks"
+      return 1
+    }
+
     case "${GOLDEN_MODE}" in
       sim)
         python3 ./golden.py
@@ -325,12 +352,26 @@ while IFS= read -r -d '' cpp; do
           exit 2
         fi
         python3 ./golden.py
+        ensure_golden_outputs_present
+        if [[ "${CUSTOM_GOLDEN}" == "1" ]]; then
+          log "Using custom golden for ${testcase}"
+        fi
         LD_LIBRARY_PATH="${LD_LIBRARY_PATH_NPU}" ./build/${testcase}
-        if [[ "${CUSTOM_GOLDEN}" != "1" ]]; then
-          copy_outputs_as_golden
-          python3 ./golden.py
-          LD_LIBRARY_PATH="${LD_LIBRARY_PATH_NPU}" ./build/${testcase}
+        COMPARE_STRICT=1 python3 ./compare.py
+        ;;
+      npu_consistency)
+        if [[ "${RUN_MODE}" != "npu" ]]; then
+          log "ERROR: GOLDEN_MODE=npu_consistency requires RUN_MODE=npu"
+          exit 2
+        fi
+        python3 ./golden.py
+        LD_LIBRARY_PATH="${LD_LIBRARY_PATH_NPU}" ./build/${testcase}
+        if [[ "${CUSTOM_GOLDEN}" == "1" ]]; then
+          log "Ignoring custom golden for ${testcase}; using first NPU run as determinism baseline"
         fi
+        copy_outputs_as_golden
+        python3 ./golden.py
+        LD_LIBRARY_PATH="${LD_LIBRARY_PATH_NPU}" ./build/${testcase}
         COMPARE_STRICT=1 python3 ./compare.py
         ;;
       skip)
@@ -341,7 +382,7 @@ while IFS= read -r -d '' cpp; do
         log "WARN: compare skipped (GOLDEN_MODE=skip)"
         ;;
       *)
-        log "ERROR: unknown GOLDEN_MODE=${GOLDEN_MODE} (expected: sim|npu|skip)"
+        log "ERROR: unknown GOLDEN_MODE=${GOLDEN_MODE} (expected: sim|npu|npu_consistency|skip)"
         exit 2
         ;;
     esac

test/npu_validation/templates/run_sh_template.sh

@@ -11,13 +11,19 @@ set -euo pipefail
 
 RUN_MODE="@RUN_MODE@"
 SOC_VERSION="@SOC_VERSION@"
-GOLDEN_MODE="${GOLDEN_MODE:-npu}"  # sim|npu|skip
+GOLDEN_MODE="${GOLDEN_MODE:-npu}"  # sim|npu|npu_consistency|skip
 BUILD_DIR="${BUILD_DIR:-build}"
 
 ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 
 cd "${ROOT_DIR}"
-python3 "${ROOT_DIR}/golden.py"
+
+CUSTOM_GOLDEN=0
+CUSTOM_COMPARE=0
+if [[ -f "${ROOT_DIR}/validation_meta.env" ]]; then
+  # shellcheck disable=SC1091
+  source "${ROOT_DIR}/validation_meta.env"
+fi
 
 # Best-effort resolve PTO_ISA_ROOT for generated CMakeLists.txt.
 if [[ -z "${PTO_ISA_ROOT:-}" ]]; then
@@ -110,10 +116,40 @@ copy_outputs_as_golden() {
   done
 }
 
+ensure_golden_outputs_present() {
+  local missing=0
+  if [[ -f "${ROOT_DIR}/outputs.txt" ]]; then
+    while IFS= read -r name; do
+      [[ -n "${name}" ]] || continue
+      if [[ ! -f "${ROOT_DIR}/golden_${name}.bin" ]]; then
+        echo "[ERROR] Missing golden output: ${ROOT_DIR}/golden_${name}.bin" >&2
+        missing=1
+      fi
+    done < "${ROOT_DIR}/outputs.txt"
+    if [[ ${missing} -ne 0 ]]; then
+      echo "[ERROR] GOLDEN_MODE=npu expects golden.py/custom assets to prepare golden_*.bin" >&2
+      echo "[ERROR] Use GOLDEN_MODE=npu_consistency for run-to-run NPU consistency checks" >&2
+      return 1
+    fi
+    return 0
+  fi
+
+  if compgen -G "${ROOT_DIR}/golden_*.bin" > /dev/null; then
+    return 0
+  fi
+
+  echo "[ERROR] No golden_*.bin found for GOLDEN_MODE=npu" >&2
+  echo "[ERROR] Use GOLDEN_MODE=npu_consistency for run-to-run NPU consistency checks" >&2
+  return 1
+}
+
 case "${GOLDEN_MODE}" in
   sim)
+    python3 "${ROOT_DIR}/golden.py"
     LD_LIBRARY_PATH="${LD_LIBRARY_PATH_SIM}" "${ROOT_DIR}/${BUILD_DIR}/@EXECUTABLE@_sim"
-    copy_outputs_as_golden
+    if [[ "${CUSTOM_GOLDEN}" != "1" ]]; then
+      copy_outputs_as_golden
+    fi
     if [[ "${RUN_MODE}" == "npu" ]]; then
       LD_LIBRARY_PATH="${LD_LIBRARY_PATH_NPU}" "${ROOT_DIR}/${BUILD_DIR}/@EXECUTABLE@"
     fi
@@ -125,7 +161,23 @@ case "${GOLDEN_MODE}" in
       exit 2
     fi
     python3 "${ROOT_DIR}/golden.py"
+    ensure_golden_outputs_present
+    if [[ "${CUSTOM_GOLDEN}" == "1" ]]; then
+      echo "[INFO] Using custom golden for @EXECUTABLE@"
+    fi
     LD_LIBRARY_PATH="${LD_LIBRARY_PATH_NPU}" "${ROOT_DIR}/${BUILD_DIR}/@EXECUTABLE@"
+    COMPARE_STRICT=1 python3 "${ROOT_DIR}/compare.py"
+    ;;
+  npu_consistency)
+    if [[ "${RUN_MODE}" != "npu" ]]; then
+      echo "[ERROR] GOLDEN_MODE=npu_consistency requires RUN_MODE=npu" >&2
+      exit 2
+    fi
+    python3 "${ROOT_DIR}/golden.py"
+    LD_LIBRARY_PATH="${LD_LIBRARY_PATH_NPU}" "${ROOT_DIR}/${BUILD_DIR}/@EXECUTABLE@"
+    if [[ "${CUSTOM_GOLDEN}" == "1" ]]; then
+      echo "[INFO] Ignoring custom golden for @EXECUTABLE@; using first NPU run as determinism baseline"
+    fi
     copy_outputs_as_golden
     python3 "${ROOT_DIR}/golden.py"
     LD_LIBRARY_PATH="${LD_LIBRARY_PATH_NPU}" "${ROOT_DIR}/${BUILD_DIR}/@EXECUTABLE@"
@@ -139,7 +191,7 @@ case "${GOLDEN_MODE}" in
     echo "[WARN] compare skipped (GOLDEN_MODE=skip)"
     ;;
   *)
-    echo "[ERROR] Unknown GOLDEN_MODE=${GOLDEN_MODE} (expected: sim|npu|skip)" >&2
+    echo "[ERROR] Unknown GOLDEN_MODE=${GOLDEN_MODE} (expected: sim|npu|npu_consistency|skip)" >&2
     exit 2
     ;;
 esac