From db890589f3b56f862c65208ad1d72a5b149ea371 Mon Sep 17 00:00:00 2001 From: ZhangYang Date: Thu, 21 May 2026 11:48:32 +0800 Subject: [PATCH] ci: introduce uv to accelerate pip install on Ascend workflows Migrate pip to uv in citest.yaml, citest_npu.yaml, and related scripts. Add full UV environment variables with cache-service and ascend repos. Add requirements/npu.txt for NPU-specific dependencies. Co-Authored-By: Claude Sonnet 4.6 --- .dev_scripts/ci_container_test.sh | 23 +++++++++++++---------- .dev_scripts/dockerci.sh | 14 ++++++++++++++ .github/workflows/citest.yaml | 8 ++++++++ .github/workflows/citest_npu.yaml | 9 +++++++++ requirements/npu.txt | 5 +++++ 5 files changed, 49 insertions(+), 10 deletions(-) create mode 100644 requirements/npu.txt diff --git a/.dev_scripts/ci_container_test.sh b/.dev_scripts/ci_container_test.sh index 4323be1899..8282d8ee76 100644 --- a/.dev_scripts/ci_container_test.sh +++ b/.dev_scripts/ci_container_test.sh @@ -1,6 +1,6 @@ if [ "$MODELSCOPE_SDK_DEBUG" == "True" ]; then - # pip config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple - pip install -r requirements/tests.txt -i https://mirrors.aliyun.com/pypi/simple/ + pip install uv -i https://mirrors.aliyun.com/pypi/simple/ + uv pip install -r requirements/tests.txt git config --global --add safe.directory /ms-swift git config --global user.email tmp git config --global user.name tmp.com @@ -20,17 +20,20 @@ if [ "$MODELSCOPE_SDK_DEBUG" == "True" ]; then fi fi - pip install -r requirements/framework.txt -U -i https://mirrors.aliyun.com/pypi/simple/ - pip install decord einops -U -i https://mirrors.aliyun.com/pypi/simple/ - pip uninstall autoawq -y - pip install optimum - pip install diffusers - pip install "transformers<5.0" "peft<0.19" + uv pip install -r requirements/framework.txt -U + uv pip install decord einops -U + uv pip uninstall autoawq + uv pip install optimum + uv pip install diffusers + uv pip install "transformers<5.0" "peft<0.19" # pip install autoawq -U --no-deps # test with install - pip install . - pip install auto_gptq bitsandbytes deepspeed -U -i https://mirrors.aliyun.com/pypi/simple/ + uv pip install . + uv pip install auto_gptq bitsandbytes deepspeed -U + if [ -f requirements/npu.txt ]; then + uv pip install -r requirements/npu.txt + fi else echo "Running case in release image, run case directly!" fi diff --git a/.dev_scripts/dockerci.sh b/.dev_scripts/dockerci.sh index f42e819fb4..999fa0fd15 100644 --- a/.dev_scripts/dockerci.sh +++ b/.dev_scripts/dockerci.sh @@ -49,6 +49,13 @@ do -e MODEL_TAG_URL=$MODEL_TAG_URL \ -e MODELSCOPE_API_TOKEN=$MODELSCOPE_API_TOKEN \ -e PR_CHANGED_FILES=$PR_CHANGED_FILES \ + -e UV_INDEX_URL=${UV_INDEX_URL:-https://mirrors.aliyun.com/pypi/simple/} \ + -e UV_EXTRA_INDEX_URL=$UV_EXTRA_INDEX_URL \ + -e UV_INDEX_STRATEGY=$UV_INDEX_STRATEGY \ + -e UV_NO_CACHE=$UV_NO_CACHE \ + -e UV_INSECURE_HOST=$UV_INSECURE_HOST \ + -e UV_HTTP_TIMEOUT=${UV_HTTP_TIMEOUT:-120} \ + -e UV_SYSTEM_PYTHON=${UV_SYSTEM_PYTHON:-1} \ --workdir=$CODE_DIR_IN_CONTAINER \ ${IMAGE_NAME}:${IMAGE_VERSION} \ $CI_COMMAND @@ -73,6 +80,13 @@ do -e MODEL_TAG_URL=$MODEL_TAG_URL \ -e MODELSCOPE_API_TOKEN=$MODELSCOPE_API_TOKEN \ -e PR_CHANGED_FILES=$PR_CHANGED_FILES \ + -e UV_INDEX_URL=${UV_INDEX_URL:-https://mirrors.aliyun.com/pypi/simple/} \ + -e UV_EXTRA_INDEX_URL=$UV_EXTRA_INDEX_URL \ + -e UV_INDEX_STRATEGY=$UV_INDEX_STRATEGY \ + -e UV_NO_CACHE=$UV_NO_CACHE \ + -e UV_INSECURE_HOST=$UV_INSECURE_HOST \ + -e UV_HTTP_TIMEOUT=${UV_HTTP_TIMEOUT:-120} \ + -e UV_SYSTEM_PYTHON=${UV_SYSTEM_PYTHON:-1} \ --workdir=$CODE_DIR_IN_CONTAINER \ ${IMAGE_NAME}:${IMAGE_VERSION} \ $CI_COMMAND diff --git a/.github/workflows/citest.yaml b/.github/workflows/citest.yaml index 101e9f3b1a..a7530c68bf 100644 --- a/.github/workflows/citest.yaml +++ b/.github/workflows/citest.yaml @@ -71,6 +71,14 @@ jobs: run: git lfs checkout - name: Run unittest shell: bash + env: + UV_INDEX_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple" + UV_EXTRA_INDEX_URL: "https://repo.huaweicloud.com/ascend/repos/pypi" + UV_INDEX_STRATEGY: "unsafe-best-match" + UV_INSECURE_HOST: "cache-service.nginx-pypi-cache.svc.cluster.local" + UV_HTTP_TIMEOUT: 120 + UV_NO_CACHE: 1 + UV_SYSTEM_PYTHON: 1 run: | set -e source /mnt/modelscope/ci_env.sh diff --git a/.github/workflows/citest_npu.yaml b/.github/workflows/citest_npu.yaml index eb30bcc2c2..a7947eb065 100644 --- a/.github/workflows/citest_npu.yaml +++ b/.github/workflows/citest_npu.yaml @@ -43,12 +43,21 @@ jobs: timeout-minutes: 240 container: image: 'ascendai/cann:8.3.rc2-910b-ubuntu22.04-py3.11' + env: + UV_INDEX_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple" + UV_EXTRA_INDEX_URL: "https://repo.huaweicloud.com/ascend/repos/pypi" + UV_INDEX_STRATEGY: "unsafe-best-match" + UV_INSECURE_HOST: "cache-service.nginx-pypi-cache.svc.cluster.local" + UV_HTTP_TIMEOUT: 120 + UV_NO_CACHE: 1 + UV_SYSTEM_PYTHON: 1 steps: - name: Config mirrors run: | sed -Ei 's@(ports|archive).ubuntu.com@cache-service.nginx-pypi-cache.svc.cluster.local:8081@g' /etc/apt/sources.list pip config set global.index-url http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple pip config set global.trusted-host cache-service.nginx-pypi-cache.svc.cluster.local + pip install uv - name: Checkout uses: actions/checkout@v3 diff --git a/requirements/npu.txt b/requirements/npu.txt new file mode 100644 index 0000000000..20287b4ed9 --- /dev/null +++ b/requirements/npu.txt @@ -0,0 +1,5 @@ +decorator +torch==2.7.1 +torch-npu==2.7.1.post4 +torchaudio==2.7.1 +torchvision==0.22.1