Skip to content

Commit 74a7449

Browse files
author
Vincent Li
committed
updates
1 parent da5fe95 commit 74a7449

3 files changed

Lines changed: 193 additions & 7 deletions

File tree

‎.ci‎

Submodule .ci updated 1 file

‎.github/workflows/ci_child.yml‎

Lines changed: 192 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -69,10 +69,196 @@ jobs:
6969
--platform "${{ matrix.platform }}" \
7070
--force
7171
72-
- name: Run stages via .ci/run.py (sequential inside job)
72+
- name: Build docker args (same helpers as .ci/run.py) and write GITHUB_ENV
73+
env:
74+
CONFIG: .ci/config.yaml
75+
BRANCH: None
76+
STAGE: None
77+
IMAGE_TAG: None
78+
GPU_ID: all
79+
RESULT_DIR: None
80+
TEST: None
81+
LOCAL: None
82+
DRY_RUN: None
7383
run: |
74-
python3 .ci/run.py \
75-
--config .ci/config.yaml \
76-
--job "${{ matrix.id }}" \
77-
--local
78-
84+
PYTHONPATH=.ci python3 - <<'PY'
85+
import os
86+
import shlex
87+
import sys
88+
from pathlib import Path
89+
90+
from run import (
91+
apply_test_override,
92+
build_docker_args,
93+
build_results_dir,
94+
resolve_image,
95+
)
96+
from utils import get_git_commit, load_config
97+
98+
99+
def env_opt(key):
100+
v = os.environ.get(key, "")
101+
return None if not v or v == "None" else v
102+
103+
104+
config = load_config(Path(os.environ["CONFIG"]))
105+
repo = config.get("repo", {})
106+
repo_url = repo.get("url", "https://github.com/InfiniTensor/InfiniOps.git")
107+
branch = env_opt("BRANCH") or repo.get("branch", "master")
108+
109+
platform = "${{ matrix.platform }}"
110+
job_key = "${{ matrix.id }}"
111+
112+
if not platform:
113+
print("error: matrix.platform is empty", file=sys.stderr)
114+
sys.exit(1)
115+
116+
jobs = config.get("jobs", {})
117+
if not jobs:
118+
print("error: no jobs in config", file=sys.stderr)
119+
sys.exit(1)
120+
121+
if job_key not in jobs:
122+
print(f"error: job {job_key!r} not in config", file=sys.stderr)
123+
sys.exit(1)
124+
125+
job = jobs[job_key]
126+
all_stages = job.get("stages", [])
127+
stage_filter = env_opt("STAGE")
128+
129+
if stage_filter:
130+
stages = [s for s in all_stages if s["name"] == stage_filter]
131+
if not stages:
132+
print(
133+
f"error: stage {stage_filter!r} not found in {job_key}",
134+
file=sys.stderr,
135+
)
136+
sys.exit(1)
137+
else:
138+
stages = all_stages
139+
140+
test_override = env_opt("TEST")
141+
if test_override:
142+
stages = [
143+
{
144+
**s,
145+
"run": apply_test_override(s.get("run", ""), test_override),
146+
}
147+
for s in stages
148+
]
149+
150+
commit = get_git_commit()
151+
results_base = env_opt("RESULT_DIR")
152+
if results_base is None:
153+
results_base = Path("ci-results")
154+
else:
155+
results_base = Path(results_base)
156+
157+
results_dir = build_results_dir(results_base, platform, stages, commit)
158+
159+
local_path = Path.cwd().resolve() if env_opt("LOCAL") else None
160+
161+
docker_args = build_docker_args(
162+
config,
163+
job_key,
164+
repo_url,
165+
branch,
166+
stages,
167+
"/workspace",
168+
env_opt("IMAGE_TAG"),
169+
gpu_id_override=os.environ.get("GPU_ID") or None,
170+
results_dir=results_dir,
171+
local_path=local_path,
172+
)
173+
174+
dry = os.environ.get("DRY_RUN", "")
175+
if dry and dry != "None":
176+
print(shlex.join(docker_args))
177+
raise SystemExit(0)
178+
179+
github_env = os.environ.get("GITHUB_ENV")
180+
if not github_env:
181+
print("error: GITHUB_ENV is not set", file=sys.stderr)
182+
sys.exit(1)
183+
184+
image_ref = resolve_image(
185+
config,
186+
job.get("platform", platform),
187+
env_opt("IMAGE_TAG") or job.get("image", "latest"),
188+
)
189+
190+
with open(github_env, "a", encoding="utf-8") as f:
191+
f.write(f"DOCKER_ARGS={shlex.join(docker_args)}\n")
192+
f.write(f"RESULT_DIR={results_dir}\n")
193+
f.write(f"IMAGE_TAG={image_ref}\n")
194+
PY
195+
196+
- name: Trigger ${{ matrix.platform }} XPU Smoke Test job
197+
env:
198+
${{ matrix.job_env }}
199+
run: |
200+
set -e
201+
echo "==> running job: ${{ matrix.job_name }}"
202+
203+
if [ -n "${RESULT_DIR:-}" ] && [ "${RESULT_DIR}" != "None" ]; then
204+
mkdir -p "${RESULT_DIR}"
205+
fi
206+
207+
if [ -n "${ENGINE:-}" ] && [ "${ENGINE}" != "None" ]; then
208+
ENGINE_ARGS="${ENGINE}"
209+
else
210+
ENGINE_ARGS="InfiniTensor"
211+
fi
212+
213+
if [ -n "${MODEL_LIST:-}" ] && [ "${MODEL_LIST}" != "None" ]; then
214+
MODEL_LIST_ARGS="${MODEL_LIST}"
215+
else
216+
MODEL_LIST_ARGS="None"
217+
fi
218+
219+
if [ -n "${IMAGE:-}" ] && [ "${IMAGE}" != "None" ]; then
220+
IMAGE_TAG_ARGS="${IMAGE}"
221+
else
222+
IMAGE_TAG_ARGS="${IMAGE_TAG}"
223+
fi
224+
225+
set -m
226+
cd /home/zkjh
227+
if [ ! -d "${{ matrix.platform }}_test" ]; then
228+
mkdir -p "${{ matrix.platform }}_test"
229+
fi
230+
cd ${{ matrix.platform }}_test
231+
mkdir -p ${{ matrix.platform }}_${{ github.run_id }}_${{ matrix.id }}
232+
cd ${{ matrix.platform }}_${{ github.run_id }}_${{ matrix.id }}
233+
234+
git init
235+
git remote add origin git@github.com:Vincent777/ci_autotest.git
236+
git fetch --depth=1 origin main
237+
git show origin/main:ascend_test_suite/daemon.sh > daemon.sh
238+
chmod a+x daemon.sh
239+
240+
DOCKER_ARGS_MATRIX="${{ join(matrix.docker_args, ' ') }}"
241+
if [ -z "${DOCKER_ARGS:-}" ]; then
242+
DOCKER_ARGS="${DOCKER_ARGS_MATRIX}"
243+
fi
244+
245+
VOLUME_ARGS=""
246+
if [ -n "${{ join(matrix.volumes, ' ') }}" ]; then
247+
VOLUME_ARGS=$(printf ' -v %s' ${{ join(matrix.volumes, ' ') }})
248+
fi
249+
250+
SHM_SIZE="--shm-size=${{ matrix.shm_size }}"
251+
252+
echo "DOCKER_ARGS=$DOCKER_ARGS"
253+
echo "VOLUME_ARGS=$VOLUME_ARGS"
254+
echo "SHM_SIZE=$SHM_SIZE"
255+
256+
PLATFORM=${{ matrix.platform }}
257+
258+
echo "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"
259+
echo "${PLATFORM}"
260+
echo "${ENGINE_ARGS}"
261+
echo "${MODEL_LIST_ARGS}"
262+
echo "${DOCKER_ARGS}"
263+
echo "${{ github.run_id }}"
264+
echo "${IMAGE_TAG_ARGS}"

.github/workflows/ci_child.yml.1 renamed to .github/workflows/ci_child_1.yml

File renamed without changes.

0 commit comments

Comments
 (0)