Skip to content

Commit d78994d

Browse files
ZCShouZCShou
committed
feat: 支持多组织并行隔离与通用 runner 资源锁
1 parent e529ea3 commit d78994d

3 files changed

Lines changed: 117 additions & 52 deletions

File tree

‎.env.example‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ GH_PAT=
2323
# ---------- 可选：多组织共享硬件锁 ----------
2424
# 板子 runner 使用 runner-wrapper；锁 ID 为「有定义用定义的，否则用该板默认值」，不回退全局 RUNNER_RESOURCE_ID，避免所有板子共一把锁
2525
# 默认：phytiumpi=board-phytiumpi，roc-rk3568-pc=board-roc-rk3568-pc（不同板子可并行）。多组织共享同一块板时显式设相同 ID：
26+
# RUNNER_RESOURCE_ID_GENERAL=runner-gereral
2627
# RUNNER_RESOURCE_ID_PHYTIUMPI=board-phytiumpi
2728
# RUNNER_RESOURCE_ID_ROC_RK3568_PC=board-roc-rk3568-pc
2829
# RUNNER_LOCK_DIR=/tmp/github-runner-locks

‎.gitignore‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
.env
2-
.reg_token.cache
3-
.dockerfile.sha256
4-
docker-compose.yml
1+
.env*
2+
.reg_token*
3+
.dockerfile*
4+
docker-compose*

‎runner.sh‎

Lines changed: 112 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,9 @@
11
#!/usr/bin/env bash
22
set -euo pipefail
33

4+
# 忽略多组织部署时的 orphan 容器警告（同一主机运行多个组织的 runner 是正常场景）
5+
export COMPOSE_IGNORE_ORPHANS=1
6+
47
ENV_FILE="${ENV_FILE:-.env}"
58

69
# ------------------------------- load .env file -------------------------------
@@ -13,8 +16,6 @@ fi
1316
ORG="${ORG:-}"
1417
GH_PAT="${GH_PAT:-}"
1518
REPO="${REPO:-}"
16-
REG_TOKEN_CACHE_FILE="${REG_TOKEN_CACHE_FILE:-.reg_token.cache}"
17-
REG_TOKEN_CACHE_TTL="${REG_TOKEN_CACHE_TTL:-300}" # seconds, default 5 minutes
1819

1920
# Runner container related parameters
2021
RUNNER_IMAGE="${RUNNER_IMAGE:-ghcr.io/actions/actions-runner:latest}"
@@ -38,15 +39,46 @@ RUNNER_WORKDIR="${RUNNER_WORKDIR:-}"
3839
RUNNER_LABELS="${RUNNER_LABELS:-intel}"
3940
RUNNER_BOARD="2"
4041
DISABLE_AUTO_UPDATE="${DISABLE_AUTO_UPDATE:-false}"
41-
# 多组织共享硬件锁：板子 runner 使用 runner-wrapper 时按锁 ID 串行。每块板子「有定义用定义的，否则用该板子默认值」，不再回退到全局 RUNNER_RESOURCE_ID，避免所有板子共一把锁导致无法并行
42-
RUNNER_RESOURCE_ID="${RUNNER_RESOURCE_ID:-}"
42+
# 普通 runners 的全局资源 ID：可选，设置后普通 runners 会通过 flock 串行执行以避免硬件竞争。多组织部署时推荐设置为相同值（如 "general-runners"）
43+
RUNNER_RESOURCE_ID_GENERAL="${RUNNER_RESOURCE_ID_GENERAL:-}"
4344
# 板子级：未设置时用本板默认值（同类型板串行、不同类型板并行）；多组织共享同一块板时显式设为相同 ID 即可
44-
RUNNER_RESOURCE_ID_PHYTIUMPI="${RUNNER_RESOURCE_ID_PHYTIUMPI:-board-phytiumpi}"
45-
RUNNER_RESOURCE_ID_ROC_RK3568_PC="${RUNNER_RESOURCE_ID_ROC_RK3568_PC:-board-roc-rk3568-pc}"
45+
RUNNER_RESOURCE_ID_PHYTIUMPI="${RUNNER_RESOURCE_ID_PHYTIUMPI:-}"
46+
RUNNER_RESOURCE_ID_ROC_RK3568_PC="${RUNNER_RESOURCE_ID_ROC_RK3568_PC:-}"
4647
RUNNER_LOCK_DIR="${RUNNER_LOCK_DIR:-/tmp/github-runner-locks}"
4748
RUNNER_LOCK_HOST_PATH="${RUNNER_LOCK_HOST_PATH:-/tmp/github-runner-locks}"
48-
COMPOSE_FILE="docker-compose.yml"
49-
DOCKERFILE_HASH_FILE="${DOCKERFILE_HASH_FILE:-.dockerfile.sha256}"
49+
# Compose 文件名：未显式设置时自动拼入 ORG/REPO，避免同一主机多组织时文件冲突
50+
# 组织级默认：docker-compose.<org>.yml 仓库级默认：docker-compose.<org>.<repo>.yml
51+
if [[ -z "${COMPOSE_FILE:-}" ]]; then
52+
if [[ -n "${ORG:-}" && -n "${REPO:-}" ]]; then
53+
COMPOSE_FILE="docker-compose.${ORG}.${REPO}.yml"
54+
elif [[ -n "${ORG:-}" ]]; then
55+
COMPOSE_FILE="docker-compose.${ORG}.yml"
56+
else
57+
COMPOSE_FILE="docker-compose.yml"
58+
fi
59+
fi
60+
# Dockerfile hash 文件名：同样根据 ORG/REPO 区分，避免多组织时 hash 冲突
61+
if [[ -z "${DOCKERFILE_HASH_FILE:-}" ]]; then
62+
if [[ -n "${ORG:-}" && -n "${REPO:-}" ]]; then
63+
DOCKERFILE_HASH_FILE=".dockerfile.${ORG}.${REPO}.sha256"
64+
elif [[ -n "${ORG:-}" ]]; then
65+
DOCKERFILE_HASH_FILE=".dockerfile.${ORG}.sha256"
66+
else
67+
DOCKERFILE_HASH_FILE=".dockerfile.sha256"
68+
fi
69+
fi
70+
# REG_TOKEN_CACHE_FILE 文件名：未显式设置时自动拼入 ORG/REPO，避免同一主机多组织时文件冲突
71+
# 组织级默认：.reg_token.cache.<org> 仓库级默认：.reg_token.cache.<org>.<repo>
72+
if [[ -z "${REG_TOKEN_CACHE_FILE:-}" ]]; then
73+
if [[ -n "${ORG:-}" && -n "${REPO:-}" ]]; then
74+
REG_TOKEN_CACHE_FILE=".reg_token.cache.${ORG}.${REPO}"
75+
elif [[ -n "${ORG:-}" ]]; then
76+
REG_TOKEN_CACHE_FILE=".reg_token.cache.${ORG}"
77+
else
78+
REG_TOKEN_CACHE_FILE=".reg_token.cache"
79+
fi
80+
fi
81+
REG_TOKEN_CACHE_TTL="${REG_TOKEN_CACHE_TTL:-300}" # seconds, default 5 minutes
5082

5183
# ------------------------------- Helpers -------------------------------
5284
shell_usage() {
@@ -92,7 +124,7 @@ shell_usage() {
92124
printf " %-${KEYW}s %s\n" "RUNNER_NAME_PREFIX" "Container name prefix (default: <hostname>-<org>[-<repo>]-); auto includes ORG/REPO to avoid name conflicts"
93125
printf " %-${KEYW}s %s\n" "RUNNER_IMAGE" "Image used for compose generation (default ghcr.io/actions/actions-runner:latest)"
94126
printf " %-${KEYW}s %s\n" "RUNNER_CUSTOM_IMAGE" "Image tag used for auto-build (can override)"
95-
printf " %-${KEYW}s %s\n" "RUNNER_RESOURCE_ID" "Optional global lock ID (e.g. for manual wrapper config); board vars take per-board default if unset"
127+
printf " %-${KEYW}s %s\n" "RUNNER_RESOURCE_ID_GENERAL" "Lock ID for generic runners (optional); prevents hardware contention in multi-org deployments"
96128
printf " %-${KEYW}s %s\n" "RUNNER_RESOURCE_ID_PHYTIUMPI" "Lock ID for phytiumpi board (default: board-phytiumpi); same ID = serial across runners"
97129
printf " %-${KEYW}s %s\n" "RUNNER_RESOURCE_ID_ROC_RK3568_PC" "Lock ID for roc-rk3568-pc board (default: board-roc-rk3568-pc); same ID = serial"
98130
printf " %-${KEYW}s %s\n" "RUNNER_LOCK_DIR" "Lock dir in container (default /tmp/github-runner-locks)"
@@ -102,7 +134,7 @@ shell_usage() {
102134

103135
echo
104136
echo "Tips:"
105-
echo "- docker-compose.yml must exist. The script will not generate or modify it."
137+
echo "- Compose file is auto-generated per ORG/REPO (e.g., docker-compose.<org>.yml)"
106138
echo "- Re-start/up will reuse existing volumes; Runner configuration and tool caches will not be lost."
107139
}
108140

@@ -288,21 +320,9 @@ shell_get_reg_token() {
288320
local now ts cached_token
289321
now=$(date +%s)
290322

291-
# 先确保 ORG 已设置，以便构建正确的缓存文件名
292-
shell_get_org_and_pat
293-
294-
# 根据组织/仓库构建缓存文件名，支持多组织场景
295-
local cache_suffix=""
296-
if [[ -n "${REPO:-}" ]]; then
297-
cache_suffix=".${ORG}.${REPO}"
298-
elif [[ -n "${ORG:-}" ]]; then
299-
cache_suffix=".${ORG}"
300-
fi
301-
local token_cache_file="${REG_TOKEN_CACHE_FILE}${cache_suffix}"
302-
303-
if [[ -f "$token_cache_file" ]]; then
304-
ts=$(head -n1 "$token_cache_file" 2>/dev/null || true)
305-
cached_token=$(sed -n '2p' "$token_cache_file" 2>/dev/null || true)
323+
if [[ -f "$REG_TOKEN_CACHE_FILE" ]]; then
324+
ts=$(head -n1 "$REG_TOKEN_CACHE_FILE" 2>/dev/null || true)
325+
cached_token=$(sed -n '2p' "$REG_TOKEN_CACHE_FILE" 2>/dev/null || true)
306326
if [[ -n "$ts" && -n "$cached_token" && "$ts" =~ ^[0-9]+$ ]]; then
307327
if (( now - ts < REG_TOKEN_CACHE_TTL )); then
308328
REG_TOKEN="$cached_token"
@@ -313,18 +333,19 @@ shell_get_reg_token() {
313333
fi
314334

315335
if [[ -n "${REG_TOKEN:-}" && "${REG_TOKEN:-}" != "null" ]]; then
316-
printf '%s\n%s\n' "$now" "$REG_TOKEN" > "$token_cache_file"
336+
printf '%s\n%s\n' "$now" "$REG_TOKEN" > "$REG_TOKEN_CACHE_FILE"
317337
printf '%s\n' "$REG_TOKEN"
318338
return 0
319339
fi
320340

321-
shell_info "Requesting <${ORG}${REPO:+/}${REPO}> registration token..." >&2
341+
shell_get_org_and_pat
342+
shell_info "Requesting <${ORG:-${REPO}}> registration token..." >&2
322343
local new_token
323344
new_token="$(github_fetch_reg_token || true)"
324345
[[ -n "$new_token" && "$new_token" != "null" ]] || shell_die "Failed to fetch registration token!"
325346
REG_TOKEN="$new_token"
326347
export REG_TOKEN
327-
printf '%s\n%s\n' "$now" "$REG_TOKEN" > "$token_cache_file"
348+
printf '%s\n%s\n' "$now" "$REG_TOKEN" > "$REG_TOKEN_CACHE_FILE"
328349
# Keep compose file in sync when fetching a fresh token
329350
printf '%s\n' "$REG_TOKEN"
330351
}
@@ -423,21 +444,61 @@ shell_get_compose_file() {
423444

424445
shell_generate_compose_file() {
425446
local general_count=$1
426-
# 板子级：有定义用定义的，否则用该板默认值；不同板默认不同 ID 可并行，多组织共享同一块板时显式设相同 ID
427-
local res_phytiumpi="${RUNNER_RESOURCE_ID_PHYTIUMPI:-board-phytiumpi}"
428-
local res_roc="${RUNNER_RESOURCE_ID_ROC_RK3568_PC:-board-roc-rk3568-pc}"
429-
local runner_entrypoint_phytiumpi="exec /home/runner/run.sh"
430-
local runner_entrypoint_roc="exec /home/runner/run.sh"
431-
[[ -n "$res_phytiumpi" ]] && runner_entrypoint_phytiumpi="exec /home/runner/runner-wrapper/runner-wrapper.sh"
432-
[[ -n "$res_roc" ]] && runner_entrypoint_roc="exec /home/runner/runner-wrapper/runner-wrapper.sh"
447+
# ════════════════════════════════════════════════════════════════
448+
# 第一步：为三种 runner 类型定义资源 ID
449+
# ════════════════════════════════════════════════════════════════
450+
# 硬件板 phytiumpi - 总是启用文件锁
451+
local res_phytiumpi="${RUNNER_RESOURCE_ID_PHYTIUMPI:-}"
452+
# 硬件板 roc - 总是启用文件锁
453+
local res_roc="${RUNNER_RESOURCE_ID_ROC_RK3568_PC:-}"
454+
# 普通 runners（可选）- 用户可选择启用文件锁，默认不启用（向后兼容）
455+
local res_general="${RUNNER_RESOURCE_ID_GENERAL:-}"
456+
457+
# ════════════════════════════════════════════════════════════════
458+
# 第二步：三种 runner 类型的 entrypoint 配置
459+
# ════════════════════════════════════════════════════════════════
460+
# 设计说明：若设置了资源 ID（RUNNER_RESOURCE_ID_*），所有 runner 类型都改用
461+
# runner-wrapper.sh 来管理文件锁——包括普通 runners
462+
# 默认：所有 runner 类型都使用 /home/runner/run.sh（不经过 runner-wrapper）
463+
local runner_entrypoint_phytiumpi="/home/runner/run.sh"
464+
local runner_entrypoint_roc="/home/runner/run.sh"
465+
local runner_entrypoint_general="/home/runner/run.sh"
466+
# 若设置了资源 ID，则改用 runner-wrapper 来处理文件锁（适用于所有 runner 类型）
467+
[[ -n "$res_phytiumpi" ]] && runner_entrypoint_phytiumpi="/home/runner/runner-wrapper/runner-wrapper.sh"
468+
[[ -n "$res_roc" ]] && runner_entrypoint_roc="/home/runner/runner-wrapper/runner-wrapper.sh"
469+
[[ -n "$res_general" ]] && runner_entrypoint_general="/home/runner/runner-wrapper/runner-wrapper.sh"
470+
471+
# ════════════════════════════════════════════════════════════════
472+
# 第三步：为三种 runner 类型准备额外的环境变量数组
473+
# ════════════════════════════════════════════════════════════════
474+
# 重复模式说明：以下三部分几乎完全相同，都是：
475+
# 1. 定义空数组：extra_env_X=()
476+
# 2. 如果有资源 ID，则添加三个环境变量：
477+
# - RUNNER_RESOURCE_ID: 用于锁机制
478+
# - RUNNER_SCRIPT: 给 runner-wrapper 使用的脚本路径
479+
# - RUNNER_LOCK_DIR: 容器内锁文件目录
480+
# 原因：三种 runner 都可能需要文件锁机制，但都是可选的
433481
local extra_env_phytiumpi=()
434482
local extra_env_roc=()
483+
local extra_env_general=()
484+
# 只有设置了相应的资源 ID，才为该类型 runner 添加锁相关环境变量
435485
[[ -n "$res_phytiumpi" ]] && extra_env_phytiumpi=(" RUNNER_RESOURCE_ID: \"$res_phytiumpi\"" " RUNNER_SCRIPT: \"/home/runner/run.sh\"" " RUNNER_LOCK_DIR: \"${RUNNER_LOCK_DIR:-/tmp/github-runner-locks}\"")
436486
[[ -n "$res_roc" ]] && extra_env_roc=(" RUNNER_RESOURCE_ID: \"$res_roc\"" " RUNNER_SCRIPT: \"/home/runner/run.sh\"" " RUNNER_LOCK_DIR: \"${RUNNER_LOCK_DIR:-/tmp/github-runner-locks}\"")
487+
[[ -n "$res_general" ]] && extra_env_general=(" RUNNER_RESOURCE_ID: \"$res_general\"" " RUNNER_SCRIPT: \"/home/runner/run.sh\"" " RUNNER_LOCK_DIR: \"${RUNNER_LOCK_DIR:-/tmp/github-runner-locks}\"")
488+
489+
# ════════════════════════════════════════════════════════════════
490+
# 第四步：为三种 runner 类型准备卷挂载配置
491+
# ════════════════════════════════════════════════════════════════
492+
# 重复模式说明：以下三部分完全相同（除变量名），都实现：
493+
# 如果设置了资源 ID，则挂载主机的锁文件目录到容器内
494+
# 原因：文件锁机制需要在主机和容器间共享锁文件
437495
local extra_vol_phytiumpi=""
438496
local extra_vol_roc=""
497+
local extra_vol_general=""
498+
# 只有设置了相应的资源 ID，才为该类型 runner 挂载锁文件目录
439499
[[ -n "$res_phytiumpi" ]] && extra_vol_phytiumpi=" - ${RUNNER_LOCK_HOST_PATH:-/tmp/github-runner-locks}:${RUNNER_LOCK_DIR:-/tmp/github-runner-locks}"
440500
[[ -n "$res_roc" ]] && extra_vol_roc=" - ${RUNNER_LOCK_HOST_PATH:-/tmp/github-runner-locks}:${RUNNER_LOCK_DIR:-/tmp/github-runner-locks}"
501+
[[ -n "$res_general" ]] && extra_vol_general=" - ${RUNNER_LOCK_HOST_PATH:-/tmp/github-runner-locks}:${RUNNER_LOCK_DIR:-/tmp/github-runner-locks}"
441502

442503
# 使用 printf 输出文件头
443504
printf '%s\n' \
@@ -463,16 +524,16 @@ shell_generate_compose_file() {
463524
" network_mode: host" \
464525
" privileged: true" \
465526
"" \
466-
"services:" > docker-compose.yml
527+
"services:" > "${COMPOSE_FILE}"
467528

468529
# 生成普通 runners
469-
echo " # 普通 runners" >> docker-compose.yml
530+
echo " # 普通 runners" >> ${COMPOSE_FILE}
470531
for i in $(seq 1 $general_count); do
471532
printf '%s\n' \
472533
" ${RUNNER_NAME_PREFIX}runner-${i}:" \
473534
" <<: *runner_base" \
474535
" container_name: \"${RUNNER_NAME_PREFIX}runner-${i}\"" \
475-
" command: [\"/home/runner/run.sh\"]" \
536+
" command: [\"${runner_entrypoint_general}\"]" \
476537
" devices:" \
477538
" - /dev/loop-control:/dev/loop-control" \
478539
" - /dev/loop0:/dev/loop0" \
@@ -486,16 +547,18 @@ shell_generate_compose_file() {
486547
" <<: *runner_env" \
487548
" RUNNER_NAME: \"${RUNNER_NAME_PREFIX}runner-${i}\"" \
488549
" RUNNER_LABELS: \"${RUNNER_LABELS}\"" \
550+
"${extra_env_general[@]}" \
489551
" volumes:" \
490552
" - ${RUNNER_NAME_PREFIX}runner-${i}-data:/home/runner" \
491553
" - ${RUNNER_NAME_PREFIX}runner-${i}-udev-rules:/etc/udev/rules.d" \
492-
"" >> docker-compose.yml
554+
"$extra_vol_general" \
555+
"" >> "${COMPOSE_FILE}"
493556
done
494557

495558
# 只有当 RUNNER_BOARD 大于 0 时才生成板子 runners
496559
if [[ "${RUNNER_BOARD}" -gt 0 ]]; then
497560
# 生成板子 runners
498-
echo " # 板子专用 runners" >> docker-compose.yml
561+
echo " # 板子专用 runners" >> "${COMPOSE_FILE}"
499562

500563
# phytiumpi 板子配置
501564
printf '%s\n' \
@@ -550,7 +613,7 @@ shell_generate_compose_file() {
550613
"$extra_vol_phytiumpi" \
551614
" - ${RUNNER_NAME_PREFIX}runner-phytiumpi-data:/home/runner" \
552615
" - ${RUNNER_NAME_PREFIX}runner-phytiumpi-udev-rules:/etc/udev/rules.d" \
553-
"" >> docker-compose.yml
616+
"" >> "${COMPOSE_FILE}"
554617

555618
# roc-rk3568-pc 板子配置
556619
printf '%s\n' \
@@ -601,18 +664,18 @@ shell_generate_compose_file() {
601664
"$extra_vol_roc" \
602665
" - ${RUNNER_NAME_PREFIX}runner-roc-rk3568-pc-data:/home/runner" \
603666
" - ${RUNNER_NAME_PREFIX}runner-roc-rk3568-pc-udev-rules:/etc/udev/rules.d" \
604-
"" >> docker-compose.yml
667+
"" >> "${COMPOSE_FILE}"
605668
fi
606669

607670
# 生成 volumes
608-
echo "volumes:" >> docker-compose.yml
671+
echo "volumes:" >> ${COMPOSE_FILE}
609672

610673
for i in $(seq 1 $general_count); do
611674
printf '%s\n' \
612675
" ${RUNNER_NAME_PREFIX}runner-${i}-data:" \
613676
" name: ${RUNNER_NAME_PREFIX}runner-${i}-data" \
614677
" ${RUNNER_NAME_PREFIX}runner-${i}-udev-rules:" \
615-
" name: ${RUNNER_NAME_PREFIX}runner-${i}-udev-rules" >> docker-compose.yml
678+
" name: ${RUNNER_NAME_PREFIX}runner-${i}-udev-rules" >> "${COMPOSE_FILE}"
616679
done
617680

618681
# 只有当 RUNNER_BOARD 大于 0 时才生成板子相关的 volumes
@@ -622,14 +685,14 @@ shell_generate_compose_file() {
622685
" ${RUNNER_NAME_PREFIX}runner-phytiumpi-data:" \
623686
" name: ${RUNNER_NAME_PREFIX}runner-phytiumpi-data" \
624687
" ${RUNNER_NAME_PREFIX}runner-phytiumpi-udev-rules:" \
625-
" name: ${RUNNER_NAME_PREFIX}runner-phytiumpi-udev-rules" >> docker-compose.yml
688+
" name: ${RUNNER_NAME_PREFIX}runner-phytiumpi-udev-rules" >> "${COMPOSE_FILE}"
626689

627690
# 为 roc-rk3568-pc 板子生成 volumes
628691
printf '%s\n' \
629692
" ${RUNNER_NAME_PREFIX}runner-roc-rk3568-pc-data:" \
630693
" name: ${RUNNER_NAME_PREFIX}runner-roc-rk3568-pc-data" \
631694
" ${RUNNER_NAME_PREFIX}runner-roc-rk3568-pc-udev-rules:" \
632-
" name: ${RUNNER_NAME_PREFIX}runner-roc-rk3568-pc-udev-rules" >> docker-compose.yml
695+
" name: ${RUNNER_NAME_PREFIX}runner-roc-rk3568-pc-udev-rules" >> "${COMPOSE_FILE}"
633696
fi
634697
}
635698

@@ -1092,9 +1155,10 @@ if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then
10921155
shell_delete_all_execute "Confirm unregister of all Runners, delete all containers and volumes, and remove all generated files? [y / N] " || { echo "Operation cancelled!"; exit 0; }
10931156
fi
10941157
for f in \
1095-
"${REG_TOKEN_CACHE_FILE}" \
1158+
"${REG_TOKEN_CACHE_FILE}"* \
10961159
"${DOCKERFILE_HASH_FILE}" \
1097-
"$ENV_FILE"; do
1160+
"$ENV_FILE" \
1161+
"$COMPOSE_FILE"; do
10981162
if [[ -f "$f" ]]; then
10991163
shell_info "Removing file $f"
11001164
rm -f "$f" || true

0 commit comments

Comments
 (0)