Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 1 addition & 9 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -57,15 +57,7 @@ jobs:
pip install torch --index-url https://download.pytorch.org/whl/cpu

- name: Run simulation examples
id: sim
run: ./ci.sh -p a2a3sim
continue-on-error: true

- name: Retry sim with pinned PTO-ISA on failure
if: steps.sim.outcome == 'failure'
run: |
rm -rf examples/scripts/_deps/pto-isa
./ci.sh -p a2a3sim --pto-isa-commit 1b22fea
run: ./ci.sh -p a2a3sim -c 1b22fea

run-example-on-device:
runs-on: self-hosted
Expand Down
33 changes: 28 additions & 5 deletions ci.sh
Original file line number Diff line number Diff line change
Expand Up @@ -92,11 +92,25 @@ cleanup() {
trap cleanup INT TERM
trap 'rm -rf "$LOG_DIR"' EXIT

# Build commit flag for run_example.py
# commit_flag starts empty (try latest PTO-ISA first).
# If -c is given AND a test fails, pin_pto_isa_on_failure sets commit_flag.
commit_flag=()
if [[ -n "$PTO_ISA_COMMIT" ]]; then

# Pin PTO-ISA to the specified commit on first failure.
# On first failure: cleans cached clone, sets commit_flag, returns 0 (caller retries).
# On subsequent failures (already pinned): returns 1 (real failure).
pin_pto_isa_on_failure() {
if [[ -z "$PTO_ISA_COMMIT" ]]; then
return 1 # No fallback commit configured
fi
if [[ ${#commit_flag[@]} -gt 0 ]]; then
return 1 # Already pinned, real failure
fi
echo "[CI] First failure detected, pinning PTO-ISA to commit $PTO_ISA_COMMIT"
rm -rf examples/scripts/_deps/pto-isa
commit_flag=(-c "$PTO_ISA_COMMIT")
fi
return 0 # Pinned, caller should retry
}

# ---- Discover all tasks ----
EXAMPLES_DIR="examples"
Expand Down Expand Up @@ -215,7 +229,7 @@ if [[ "$PARALLEL" == "false" ]]; then
fi
done
done
# SIM tasks
# SIM tasks (with pin-on-first-failure for PTO-ISA)
for i in "${!SIM_TASK_NAMES[@]}"; do
name="${SIM_TASK_NAMES[$i]}"
dir="${SIM_TASK_DIRS[$i]}"
Expand All @@ -226,6 +240,15 @@ if [[ "$PARALLEL" == "false" ]]; then
-k "${dir}/kernels" -g "${dir}/golden.py" \
-p a2a3sim "${commit_flag[@]}"; then
echo "${name}:a2a3sim|PASS" >> "$RESULTS_FILE"
elif pin_pto_isa_on_failure; then
echo "[CI] Retrying: $name with pinned PTO-ISA"
if python examples/scripts/run_example.py \
-k "${dir}/kernels" -g "${dir}/golden.py" \
-p a2a3sim "${commit_flag[@]}"; then
echo "${name}:a2a3sim|PASS" >> "$RESULTS_FILE"
else
echo "${name}:a2a3sim|FAIL" >> "$RESULTS_FILE"
fi
else
echo "${name}:a2a3sim|FAIL" >> "$RESULTS_FILE"
fi
Expand All @@ -245,7 +268,7 @@ else
echo "Running: $name (a2a3sim)"
echo "========================================"
if python examples/scripts/run_example.py \
-k "${dir}/kernels" -g "${dir}/golden.py" -p a2a3sim $COMMIT_FLAG; then
-k "${dir}/kernels" -g "${dir}/golden.py" -p a2a3sim "${commit_flag[@]}"; then
echo "${name}:a2a3sim|PASS" >> "$RESULTS_FILE"
else
echo "${name}:a2a3sim|FAIL" >> "$RESULTS_FILE"
Expand Down
2 changes: 1 addition & 1 deletion src/platform/include/common/perf_profiling.h
Original file line number Diff line number Diff line change
Expand Up @@ -205,7 +205,7 @@ enum class AicpuPhaseId : uint32_t {
SCHED_COMPLETE = 0, // Process completed tasks (fanout traversal)
SCHED_DISPATCH = 1, // Dispatch ready tasks to idle cores
SCHED_SCAN = 2, // Incremental scan for root tasks
SCHED_EARLY_READY = 3, // Drain orchestrator's early-ready queue
SCHED_IDLE_WAIT = 3, // Idle time (no progress made)
// Orchestrator phases (16-24)
ORCH_SYNC = 16, // tensormap sync
ORCH_ALLOC = 17, // task_ring_alloc
Expand Down
2 changes: 1 addition & 1 deletion src/platform/src/host/performance_collector.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -511,7 +511,7 @@ int PerformanceCollector::export_swimlane_json(const std::string& output_path) {
case AicpuPhaseId::SCHED_COMPLETE: phase_name = "complete"; break;
case AicpuPhaseId::SCHED_DISPATCH: phase_name = "dispatch"; break;
case AicpuPhaseId::SCHED_SCAN: phase_name = "scan"; break;
case AicpuPhaseId::SCHED_EARLY_READY: phase_name = "early_ready"; break;
case AicpuPhaseId::SCHED_IDLE_WAIT: phase_name = "idle"; break;
default: break;
}

Expand Down
Loading
Loading