Merge pull request #11 from yoinspiration/feat/runner-lock-cancel-safety

ZCShou · web-flow · commit 86118775228f · 2026-02-26T14:16:36.000+08:00
fix: 加固多组织 Runner 锁机制并修复 cancel 场景并行竞态
diff --git a/docs/多组织共享Runner使用说明.md b/docs/多组织共享Runner使用说明.md
@@ -0,0 +1,180 @@
+# 多组织共享 Runner 使用说明
+
+本文档用于指导在同一台主机上部署两套（或多套）GitHub Actions Runner，并通过 `runner-wrapper` 实现：
+
+- 同板卡任务串行（避免硬件冲突）
+- 异板卡任务并行（提升吞吐）
+- 支持网页手动 `Cancel` 后的安全恢复
+
+---
+
+## 1. 适用场景
+
+- 多个组织（或多个账号）共享同一块测试板卡。
+- 需要保证同一资源标签（如 `roc-rk3568-pc`）不会并发操作硬件。
+- 允许不同硬件标签（如 `roc-rk3568-pc` 与 `phytiumpi`）并行执行。
+
+---
+
+## 2. 前置条件
+
+- 同一台 Linux 主机（锁基于本机文件锁，跨主机不生效）。
+- 已安装并可用：
+  - `docker` / `docker compose`
+  - `bash`
+- 两套有效的 GitHub 凭据（组织或账号均可）。
+
+> 注意：不要提交 `.env*`（含 PAT）到仓库。
+
+---
+
+## 3. 环境变量准备
+
+为每个组织（或账号）准备独立 env 文件，例如：
+
+- `.env.orgA`
+- `.env.orgB`
+
+示例（两份都类似）：
+
+```env
+ORG=your-org-or-user
+REPO=test-runner
+GH_PAT=ghp_xxx
+
+RUNNER_RESOURCE_ID_PHYTIUMPI=board-phytiumpi
+RUNNER_RESOURCE_ID_ROC_RK3568_PC=board-roc-rk3568-pc
+RUNNER_LOCK_HOST_PATH=/tmp/github-runner-locks
+RUNNER_LOCK_DIR=/tmp/github-runner-locks
+```
+
+关键要求：
+
+- 两套配置的 `RUNNER_RESOURCE_ID_*` 必须一致（同板卡共享同一锁）。
+- 两套配置的 `RUNNER_LOCK_HOST_PATH` 必须一致（指向同一宿主机目录）。
+
+---
+
+## 4. 初次部署
+
+在仓库根目录执行：
+
+```bash
+ENV_FILE=.env.orgA ./runner.sh init -n 2
+ENV_FILE=.env.orgB ./runner.sh init -n 2
+```
+
+检查状态：
+
+```bash
+ENV_FILE=.env.orgA ./runner.sh ps
+ENV_FILE=.env.orgB ./runner.sh ps
+```
+
+预期：两套都出现 `runner-roc-rk3568-pc`、`runner-phytiumpi` 且 `online`。
+
+---
+
+## 5. 日常更新（脚本/配置改动后）
+
+当修改了 `runner.sh` 或 `.env` 后：
+
+```bash
+ENV_FILE=.env.orgA ./runner.sh compose
+ENV_FILE=.env.orgB ./runner.sh compose
+docker compose -f docker-compose.<orgA>.<repo>.yml up -d --force-recreate
+docker compose -f docker-compose.<orgB>.<repo>.yml up -d --force-recreate
+```
+
+如果修改了镜像内依赖（例如 Dockerfile），再执行：
+
+```bash
+ENV_FILE=.env.orgA ./runner.sh image
+```
+
+> 当前实现已把 `./runner-wrapper` 目录只读挂载进板卡容器，`pre/post` 脚本改动通常不需要重建镜像，只需 `compose + force-recreate`。
+
+---
+
+## 6. 验证方法
+
+### 6.1 同板卡串行验证（应串行）
+
+两边同时触发：
+
+```yaml
+runs-on: [self-hosted, linux, roc-rk3568-pc]
+```
+
+步骤里包含：
+
+```yaml
+- run: echo "START $(date -Iseconds)"
+- run: sleep 120
+- run: echo "END $(date -Iseconds)"
+```
+
+预期：
+
+- 一个先 Running，另一个先 Waiting；
+- 前者结束后后者开始；
+- 两个 `sleep 120` 时间段不重叠。
+
+### 6.2 异板卡并行验证（应并行）
+
+- 任务 A：`roc-rk3568-pc`
+- 任务 B：`phytiumpi`
+
+预期：两者可同时 Running，执行时间有重叠。
+
+---
+
+## 7. Cancel 场景说明
+
+允许在网页点 `Cancel`，但建议遵循：
+
+- 重要验证尽量让任务自然结束；
+- 若中途取消后出现异常（如等待异常、状态不同步），执行一次清场：
+
+```bash
+sudo rm -f /tmp/github-runner-locks/*.holder /tmp/github-runner-locks/*.release
+sudo chmod 1777 /tmp/github-runner-locks
+sudo find /tmp/github-runner-locks -maxdepth 1 -type f -name 'board-*' -exec chmod 666 {} \;
+docker restart <orgA-roc-container>
+docker restart <orgB-roc-container>
+```
+
+当前锁实现已包含：
+
+- 按 `RUNNER_NAME + GITHUB_RUN_ID + GITHUB_RUN_ATTEMPT` 生成唯一 release 文件；
+- 防止旧任务 post-hook 误释放新任务锁（cancel 竞态保护）。
+
+---
+
+## 8. 常见问题
+
+### 8.1 一直 `Waiting for a runner to pick up this job...`
+
+优先检查：
+
+- 该组织/仓库下 runner 是否 `online`
+- 标签是否精确匹配（`self-hosted, linux, roc-rk3568-pc`）
+- Runner group 是否授权目标仓库
+
+### 8.2 Runner 全部 `offline`
+
+常见原因是代理配置错误（例如容器内 `127.0.0.1:7890` 不可达）。
+
+当前脚本已改为：仅当显式设置 `HTTP_PROXY/HTTPS_PROXY/NO_PROXY` 时才注入代理变量。
+
+### 8.3 明明改了脚本，但容器没生效
+
+执行：
+
+```bash
+ENV_FILE=.env.<org> ./runner.sh compose
+docker compose -f docker-compose.<org>.<repo>.yml up -d --force-recreate
+```
+
+并在容器内检查脚本关键字是否存在。
+
diff --git a/runner-wrapper/post-job-lock.sh b/runner-wrapper/post-job-lock.sh
@@ -6,12 +6,49 @@
 #
 # 依赖：RUNNER_RESOURCE_ID、RUNNER_LOCK_DIR 环境变量
 
+set -e
+
 LOCK_DIR="${RUNNER_LOCK_DIR:-/tmp/github-runner-locks}"
 RESOURCE_ID="${RUNNER_RESOURCE_ID:-default-hardware}"
-RELEASE_FILE="${LOCK_DIR}/${RESOURCE_ID}.release"
+RUNNER_NAME_SAFE="${RUNNER_NAME:-unknown-runner}"
+RUN_ID_SAFE="${GITHUB_RUN_ID:-unknown}"
+RUN_ATTEMPT_SAFE="${GITHUB_RUN_ATTEMPT:-unknown}"
+RUN_KEY="${RUNNER_NAME_SAFE}.${RUN_ID_SAFE}.${RUN_ATTEMPT_SAFE}"
+RELEASE_FILE="${LOCK_DIR}/${RESOURCE_ID}.${RUN_KEY}.release"
+HOLDER_PID_FILE="${LOCK_DIR}/${RESOURCE_ID}.holder"
 
 echo "[$(date -Iseconds)] 🔓 Releasing lock for ${RESOURCE_ID}" >&2
-touch "${RELEASE_FILE}"
+mkdir -p "${LOCK_DIR}"
+chmod 1777 "${LOCK_DIR}" || true
+
+# 仅允许当前持锁 runner 释放，防止 cancel 后旧 post-hook 误释放新任务锁
+if [ ! -f "${HOLDER_PID_FILE}" ]; then
+  echo "[$(date -Iseconds)] ⚠️ Holder file not found, skip releasing: ${RESOURCE_ID}" >&2
+  exit 0
+fi
+
+holder_pid=""
+holder_runner=""
+holder_run_id=""
+holder_run_attempt=""
+read -r holder_pid holder_runner holder_run_id holder_run_attempt < "${HOLDER_PID_FILE}" || true
+
+if [ -z "${holder_runner}" ] || [ "${holder_runner}" != "${RUNNER_NAME_SAFE}" ]; then
+  echo "[$(date -Iseconds)] ⚠️ Holder runner mismatch (holder=${holder_runner:-unknown}, current=${RUNNER_NAME_SAFE}), skip releasing ${RESOURCE_ID}" >&2
+  exit 0
+fi
+
+if [ -z "${holder_run_id}" ] || [ "${holder_run_id}" != "${RUN_ID_SAFE}" ] || \
+   [ -z "${holder_run_attempt}" ] || [ "${holder_run_attempt}" != "${RUN_ATTEMPT_SAFE}" ]; then
+  echo "[$(date -Iseconds)] ⚠️ Holder run mismatch (holder=${holder_run_id:-unknown}/${holder_run_attempt:-unknown}, current=${RUN_ID_SAFE}/${RUN_ATTEMPT_SAFE}), skip releasing ${RESOURCE_ID}" >&2
+  exit 0
+fi
+
+touch "${RELEASE_FILE}" || {
+  echo "[$(date -Iseconds)] ⚠️ Failed to create release mark: ${RELEASE_FILE}" >&2
+  # 避免因释放标记写入失败让 job 直接失败，后续由运维处理锁目录权限
+  exit 0
+}
 
 # Holder 会在 1 秒内检测到并退出，锁随之释放
 exit 0
diff --git a/runner-wrapper/pre-job-lock.sh b/runner-wrapper/pre-job-lock.sh
@@ -10,26 +10,50 @@ set -e
 
 LOCK_DIR="${RUNNER_LOCK_DIR:-/tmp/github-runner-locks}"
 RESOURCE_ID="${RUNNER_RESOURCE_ID:-default-hardware}"
+RUNNER_NAME_SAFE="${RUNNER_NAME:-unknown-runner}"
+RUN_ID_SAFE="${GITHUB_RUN_ID:-unknown}"
+RUN_ATTEMPT_SAFE="${GITHUB_RUN_ATTEMPT:-unknown}"
+RUN_KEY="${RUNNER_NAME_SAFE}.${RUN_ID_SAFE}.${RUN_ATTEMPT_SAFE}"
 LOCK_FILE="${LOCK_DIR}/${RESOURCE_ID}.lock"
-RELEASE_FILE="${LOCK_DIR}/${RESOURCE_ID}.release"
+RELEASE_FILE="${LOCK_DIR}/${RESOURCE_ID}.${RUN_KEY}.release"
 HOLDER_PID_FILE="${LOCK_DIR}/${RESOURCE_ID}.holder"
 
 mkdir -p "${LOCK_DIR}"
+chmod 1777 "${LOCK_DIR}" || true
+# 清理当前 run 的残留释放标记，避免误判为可释放
+rm -f "${RELEASE_FILE}" || true
 
 # 打开锁文件并获取排他锁（阻塞等待）
 exec 200>"${LOCK_FILE}"
+chmod 666 "${LOCK_FILE}" || true
 echo "[$(date -Iseconds)] ⏳ Waiting for lock: ${RESOURCE_ID}" >&2
 flock -x 200
 echo "[$(date -Iseconds)] ✅ Acquired lock for ${RESOURCE_ID}" >&2
 
 # 后台子进程继承 fd 200 并持有锁，等待 post-job 创建释放文件
 (
+  holder_pid="${BASHPID:-$$}"
+  printf '%s %s %s %s\n' \
+    "${holder_pid}" \
+    "${RUNNER_NAME_SAFE}" \
+    "${RUN_ID_SAFE}" \
+    "${RUN_ATTEMPT_SAFE}" > "${HOLDER_PID_FILE}"
+  chmod 666 "${HOLDER_PID_FILE}" || true
+
   while [ ! -f "${RELEASE_FILE}" ]; do
     sleep 1
   done
-  rm -f "${RELEASE_FILE}" "${HOLDER_PID_FILE}"
+
+  # 仅清理由自己写入的 holder 记录，避免并发切换时误删新 holder 信息
+  current_holder_pid=""
+  if [ -f "${HOLDER_PID_FILE}" ]; then
+    read -r current_holder_pid _ < "${HOLDER_PID_FILE}" || true
+  fi
+  rm -f "${RELEASE_FILE}" || true
+  if [ "${current_holder_pid}" = "${holder_pid}" ]; then
+    rm -f "${HOLDER_PID_FILE}" || true
+  fi
 ) &
-echo $! > "${HOLDER_PID_FILE}"
 
 # 主脚本退出，子进程继续持有 fd 200，锁保持到 post-job 执行
 exit 0
diff --git a/runner.sh b/runner.sh
@@ -520,9 +520,13 @@ shell_generate_compose_file() {
     # 原因：两种板子 runner 都可能需要文件锁机制
     local extra_env_phytiumpi=()
     local extra_env_roc=()
+    local extra_proxy_env=()
     # 只有设置了相应的资源 ID，才为该类型 runner 添加锁相关环境变量
     [[ -n "$res_phytiumpi" ]] && extra_env_phytiumpi=("      RUNNER_RESOURCE_ID: \"$res_phytiumpi\"" "      RUNNER_SCRIPT: \"/home/runner/run.sh\"" "      RUNNER_LOCK_DIR: \"${RUNNER_LOCK_DIR:-/tmp/github-runner-locks}\"")
     [[ -n "$res_roc" ]] && extra_env_roc=("      RUNNER_RESOURCE_ID: \"$res_roc\"" "      RUNNER_SCRIPT: \"/home/runner/run.sh\"" "      RUNNER_LOCK_DIR: \"${RUNNER_LOCK_DIR:-/tmp/github-runner-locks}\"")
+    [[ -n "${HTTP_PROXY:-}" ]] && extra_proxy_env+=("    HTTP_PROXY: \"${HTTP_PROXY}\"")
+    [[ -n "${HTTPS_PROXY:-}" ]] && extra_proxy_env+=("    HTTPS_PROXY: \"${HTTPS_PROXY}\"")
+    [[ -n "${NO_PROXY:-}" ]] && extra_proxy_env+=("    NO_PROXY: \"${NO_PROXY}\"")
 
     # ════════════════════════════════════════════════════════════════
     # 第四步：为两种板子 runner 类型准备卷挂载配置
@@ -554,9 +558,7 @@ shell_generate_compose_file() {
         "    RUNNER_REMOVE_ON_STOP: \"false\"" \
         "    DISABLE_AUTO_UPDATE: \"${DISABLE_AUTO_UPDATE}\"" \
         "    RUNNER_WORKDIR: \"${RUNNER_WORKDIR}\"" \
-        "    HTTP_PROXY: \"http://127.0.0.1:7890\"" \
-        "    HTTPS_PROXY: \"http://127.0.0.1:7890\"" \
-        "    NO_PROXY: localhost,127.0.0.1,.internal" \
+        "${extra_proxy_env[@]}" \
         "  network_mode: host" \
         "  privileged: true" \
         "" \
@@ -645,6 +647,7 @@ shell_generate_compose_file() {
             "    volumes:" \
             "      - /home/$(whoami)/test/phytiumpi:/home/runner/tftp" \
             "$extra_vol_phytiumpi" \
+            "      - ./runner-wrapper:/home/runner/runner-wrapper:ro" \
             "      - ${RUNNER_NAME_PREFIX}runner-phytiumpi-data:/home/runner" \
             "      - ${RUNNER_NAME_PREFIX}runner-phytiumpi-udev-rules:/etc/udev/rules.d" \
             "" >> "${COMPOSE_FILE}"
@@ -695,6 +698,7 @@ shell_generate_compose_file() {
             "      BOARD_COMM_UART_BAUD: \"1500000\"" \
             "${extra_env_roc[@]}" \
             "    volumes:" \
+            "      - ./runner-wrapper:/home/runner/runner-wrapper:ro" \
             "$extra_vol_roc" \
             "      - ${RUNNER_NAME_PREFIX}runner-roc-rk3568-pc-data:/home/runner" \
             "      - ${RUNNER_NAME_PREFIX}runner-roc-rk3568-pc-udev-rules:/etc/udev/rules.d" \
@@ -1220,7 +1224,7 @@ if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then
                 shell_info "Successfully built ${RUNNER_CUSTOM_IMAGE} image"
                 
                 # Update hash file
-                local new_hash=""
+                new_hash=""
                 if command -v sha256sum >/dev/null 2>&1; then
                     new_hash=$(sha256sum Dockerfile | awk '{print $1}')
                 elif command -v shasum >/dev/null 2>&1; then
