From 4f54db66d9f1b6116d950f85e485314be28e4e6e Mon Sep 17 00:00:00 2001 From: Ivan Basov Date: Thu, 9 Jul 2026 21:45:17 +0000 Subject: [PATCH] Remove model weights from repo; CI fetches them from a release asset The pre-trained model weights are distributed via Hugging Face: - https://huggingface.co/nvidia/Ising-Decoder-SurfaceCode-1-Fast - https://huggingface.co/nvidia/Ising-Decoder-SurfaceCode-1-Accurate Changes: - Delete models/*.pt and the .gitattributes Git LFS rule; ignore /models/. - Add .github/actions/fetch-models, a composite action that downloads the model assets of the ci-models-v1 release into models/. - Wire the action into the five jobs whose tests load the weights (unit-tests, unit-tests-coverage, gpu-tests, gpu-coverage, ler-regression); those jobs grant the token the push access needed to resolve the release. All workflow triggers are same-repo (push/merge_group/schedule/dispatch), never fork pull_request. - Drop lfs: true, git lfs install and git-lfs apt packages everywhere. - Point README and test error messages at Hugging Face; remove the unimplemented PREDECODER_MODEL_URL mention. Co-Authored-By: Claude Fable 5 --- .gitattributes | 2 - .github/actions/fetch-models/action.yml | 86 +++++++++++++++++++ .github/workflows/ci-gpu.yml | 33 +++---- .github/workflows/ci.yml | 17 ++-- .github/workflows/long-running-tests.yml | 27 +++--- .gitignore | 5 ++ README.md | 28 ++++-- code/tests/test_inference_public_model.py | 10 ++- .../Ising-Decoder-SurfaceCode-1-Accurate.pt | 3 - models/Ising-Decoder-SurfaceCode-1-Fast.pt | 3 - 10 files changed, 158 insertions(+), 56 deletions(-) delete mode 100644 .gitattributes create mode 100644 .github/actions/fetch-models/action.yml delete mode 100644 models/Ising-Decoder-SurfaceCode-1-Accurate.pt delete mode 100644 models/Ising-Decoder-SurfaceCode-1-Fast.pt diff --git a/.gitattributes b/.gitattributes deleted file mode 100644 index e0f0a56..0000000 --- a/.gitattributes +++ /dev/null @@ -1,2 +0,0 @@ -# Track large model binaries with Git LFS -models/*.pt filter=lfs diff=lfs merge=lfs -text diff --git a/.github/actions/fetch-models/action.yml b/.github/actions/fetch-models/action.yml new file mode 100644 index 0000000..c560468 --- /dev/null +++ b/.github/actions/fetch-models/action.yml @@ -0,0 +1,86 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Fetch pre-trained models +description: > + Downloads the pre-trained model weights from this repository's + permanent-draft release into models/. The weights are not committed to the + repo because their license differs from the code license; users get them + from Hugging Face (see README), while CI uses the draft-release assets. + Draft releases are invisible without push access, so the calling job must + grant the token write permission (permissions: contents: write). Draft + tags do not exist as git tags, so the release is looked up by tag_name + over the release list rather than by the tags API. +inputs: + release-tag: + description: tag_name of the permanent-draft release that holds the model assets + required: false + default: ci-models-v1 + dest: + description: Directory (relative to the workspace) to place the model files in + required: false + default: models + token: + description: GitHub token with push access (drafts are invisible otherwise) + required: false + default: ${{ github.token }} +runs: + using: composite + steps: + - name: Download model assets from draft release + uses: actions/github-script@v7 + env: + RELEASE_TAG: ${{ inputs.release-tag }} + DEST_DIR: ${{ inputs.dest }} + with: + github-token: ${{ inputs.token }} + script: | + const fs = require('fs'); + const path = require('path'); + const tag = process.env.RELEASE_TAG; + const dest = process.env.DEST_DIR; + const { owner, repo } = context.repo; + + const releases = await github.paginate(github.rest.repos.listReleases, + { owner, repo, per_page: 100 }); + const release = releases.find(r => r.tag_name === tag); + if (!release) { + core.setFailed( + `Release '${tag}' not found. It is a draft release, which is only ` + + `visible to tokens with push access — make sure the job sets ` + + `'permissions: contents: write' and that the draft still exists.`); + return; + } + + const assets = release.assets.filter(a => a.name.endsWith('.pt')); + if (assets.length === 0) { + core.setFailed(`Release '${tag}' has no *.pt assets.`); + return; + } + + fs.mkdirSync(dest, { recursive: true }); + for (const asset of assets) { + const res = await github.request( + 'GET /repos/{owner}/{repo}/releases/assets/{asset_id}', + { owner, repo, asset_id: asset.id, + headers: { accept: 'application/octet-stream' } }); + const buf = Buffer.from(res.data); + if (buf.length !== asset.size) { + throw new Error( + `Size mismatch for ${asset.name}: got ${buf.length}, expected ${asset.size}`); + } + fs.writeFileSync(path.join(dest, asset.name), buf); + core.info(`Downloaded ${asset.name} (${asset.size} bytes) -> ${dest}/`); + } diff --git a/.github/workflows/ci-gpu.yml b/.github/workflows/ci-gpu.yml index 27bcbee..ec98fd3 100644 --- a/.github/workflows/ci-gpu.yml +++ b/.github/workflows/ci-gpu.yml @@ -41,6 +41,10 @@ env: jobs: gpu-tests: runs-on: linux-amd64-gpu-rtxpro6000-latest-1 + # contents: write gives the token push access, which is required to see the + # permanent-draft release that fetch-models downloads the weights from. + permissions: + contents: write container: image: ubuntu:24.04 options: -u root --security-opt seccomp=unconfined --shm-size 16g @@ -59,16 +63,16 @@ jobs: run: | export DEBIAN_FRONTEND=noninteractive apt-get update - apt-get install -y git git-lfs gcc - git lfs install + apt-get install -y git gcc - uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - uses: actions/checkout@v4 - with: - lfs: true + + - name: Fetch pre-trained models + uses: ./.github/actions/fetch-models - name: Verify GPU run: nvidia-smi @@ -134,16 +138,13 @@ jobs: run: | export DEBIAN_FRONTEND=noninteractive apt-get update - apt-get install -y git git-lfs gcc - git lfs install + apt-get install -y git gcc - uses: actions/setup-python@v5 with: python-version: "3.13" - uses: actions/checkout@v4 - with: - lfs: true - name: Verify GPU run: nvidia-smi @@ -205,12 +206,9 @@ jobs: run: | export DEBIAN_FRONTEND=noninteractive apt-get update - apt-get install -y git git-lfs python3 python3-pip python3-venv - git lfs install + apt-get install -y git python3 python3-pip python3-venv - uses: actions/checkout@v4 - with: - lfs: true - name: Verify 2 GPUs are visible run: | @@ -300,6 +298,9 @@ jobs: # --------------------------------------------------------------------------- gpu-coverage: runs-on: linux-amd64-gpu-rtxpro6000-latest-1 + # contents: write — see gpu-tests: needed to read the draft release. + permissions: + contents: write container: image: ubuntu:24.04 options: -u root --security-opt seccomp=unconfined --shm-size 16g @@ -311,12 +312,12 @@ jobs: run: | export DEBIAN_FRONTEND=noninteractive apt-get update - apt-get install -y git git-lfs python3 python3-pip python3-venv - git lfs install + apt-get install -y git python3 python3-pip python3-venv - uses: actions/checkout@v4 - with: - lfs: true + + - name: Fetch pre-trained models + uses: ./.github/actions/fetch-models - name: Verify GPU run: nvidia-smi diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8a4efad..dc54a55 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -56,6 +56,10 @@ jobs: unit-tests: runs-on: linux-amd64-cpu4 + # contents: write gives the token push access, which is required to see the + # permanent-draft release that fetch-models downloads the weights from. + permissions: + contents: write strategy: fail-fast: false matrix: @@ -63,8 +67,8 @@ jobs: name: "unit-tests / py${{ matrix.python-version }}" steps: - uses: actions/checkout@v4 - with: - lfs: true + - name: Fetch pre-trained models + uses: ./.github/actions/fetch-models - uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} @@ -85,8 +89,6 @@ jobs: runs-on: linux-amd64-cpu4 steps: - uses: actions/checkout@v4 - with: - lfs: true - uses: actions/setup-python@v5 with: python-version: "3.12" @@ -102,10 +104,13 @@ jobs: unit-tests-coverage: runs-on: linux-amd64-cpu4 + # contents: write — see unit-tests: needed to read the draft release. + permissions: + contents: write steps: - uses: actions/checkout@v4 - with: - lfs: true + - name: Fetch pre-trained models + uses: ./.github/actions/fetch-models - uses: actions/setup-python@v5 with: python-version: "3.12" diff --git a/.github/workflows/long-running-tests.yml b/.github/workflows/long-running-tests.yml index a7a357c..92d3ae5 100644 --- a/.github/workflows/long-running-tests.yml +++ b/.github/workflows/long-running-tests.yml @@ -99,16 +99,13 @@ jobs: run: | export DEBIAN_FRONTEND=noninteractive apt-get update - apt-get install -y git git-lfs gcc - git lfs install + apt-get install -y git gcc - uses: actions/setup-python@v5 with: python-version: ${{ env.PYTHON_VERSION }} - uses: actions/checkout@v4 - with: - lfs: true - name: Install Python dependencies run: | @@ -152,16 +149,13 @@ jobs: run: | export DEBIAN_FRONTEND=noninteractive apt-get update - apt-get install -y git git-lfs gcc - git lfs install + apt-get install -y git gcc - uses: actions/setup-python@v5 with: python-version: ${{ env.PYTHON_VERSION }} - uses: actions/checkout@v4 - with: - lfs: true - name: Install Python dependencies run: | @@ -213,6 +207,10 @@ jobs: # --------------------------------------------------------------------------- ler-regression: needs: check-for-changes + # contents: write gives the token push access, which is required to see the + # permanent-draft release that fetch-models downloads the weights from. + permissions: + contents: write if: >- needs.check-for-changes.outputs.has_changes == 'true' && (github.event_name == 'schedule' || @@ -230,16 +228,16 @@ jobs: run: | export DEBIAN_FRONTEND=noninteractive apt-get update - apt-get install -y git git-lfs gcc - git lfs install + apt-get install -y git gcc - uses: actions/setup-python@v5 with: python-version: ${{ env.PYTHON_VERSION }} - uses: actions/checkout@v4 - with: - lfs: true + + - name: Fetch pre-trained models + uses: ./.github/actions/fetch-models - name: Install Python dependencies run: | @@ -282,16 +280,13 @@ jobs: run: | export DEBIAN_FRONTEND=noninteractive apt-get update - apt-get install -y git git-lfs gcc - git lfs install + apt-get install -y git gcc - uses: actions/setup-python@v5 with: python-version: ${{ env.PYTHON_VERSION }} - uses: actions/checkout@v4 - with: - lfs: true - name: Install Python dependencies run: | diff --git a/.gitignore b/.gitignore index 035f62e..284f86b 100644 --- a/.gitignore +++ b/.gitignore @@ -53,6 +53,11 @@ env_stim/ .venv_*/ .conda +# Pre-trained model weights: distributed via Hugging Face (users) and the +# permanent-draft GitHub release consumed by .github/actions/fetch-models (CI). +# Never commit weights — their license differs from the code license. +/models/ + # Outputs & logs outputs/ logs/ diff --git a/README.md b/README.md index 76222a2..7878ced 100644 --- a/README.md +++ b/README.md @@ -202,16 +202,30 @@ If you are not training locally, you can run inference using pre-trained models. ``` 2. **Get the pre-trained models** - This repo ships two pre-trained model files (tracked with Git LFS): - - `models/Ising-Decoder-SurfaceCode-1-Fast.pt` (receptive field R=9) - - `models/Ising-Decoder-SurfaceCode-1-Accurate.pt` (receptive field R=13) + Two pre-trained models are published on Hugging Face (they are licensed + separately from the code in this repo and are not part of it): + - [nvidia/Ising-Decoder-SurfaceCode-1-Fast](https://huggingface.co/nvidia/Ising-Decoder-SurfaceCode-1-Fast) (receptive field R=9) + - [nvidia/Ising-Decoder-SurfaceCode-1-Accurate](https://huggingface.co/nvidia/Ising-Decoder-SurfaceCode-1-Accurate) (receptive field R=13) + + The models are access-controlled: sign in with a Hugging Face token + (create one at ), then download + the files into `models/`: + + ```bash + pip install -U "huggingface_hub[cli]" + hf auth login + hf download nvidia/Ising-Decoder-SurfaceCode-1-Fast --local-dir models/ + hf download nvidia/Ising-Decoder-SurfaceCode-1-Accurate --local-dir models/ + ``` + + See each model card for the available files and formats. The scripts below + expect `models/Ising-Decoder-SurfaceCode-1-Fast.pt` and + `models/Ising-Decoder-SurfaceCode-1-Accurate.pt`. These checkpoints target the uniform circuit-level depolarizing setting encoded by the public configs. Custom, non-uniform 25-parameter noise models are supported for training by the pipeline below; they are a training-time - customization rather than a property of the shipped checkpoints. - - Clones get the files via `git lfs pull`. Optionally, set `PREDECODER_MODEL_URL` to the LFS/raw URL to fetch files when not in the working tree (e.g. in a minimal checkout or CI). + customization rather than a property of the published checkpoints. 3. Set: @@ -255,7 +269,7 @@ The pre-trained public models use `--model-id 1` (R=9) and `--model-id 4` (R=13) ### ONNX export and quantization (optional, post-training) -After training (or starting from the shipped `.safetensors` files), you can export the model to +After training (or starting from the `.safetensors` files downloaded from Hugging Face), you can export the model to ONNX and optionally apply INT8 or FP8 post-training quantization for deployment. You may also change the surface code distance and number of rounds at inference diff --git a/code/tests/test_inference_public_model.py b/code/tests/test_inference_public_model.py index bfd3aba..9f17dc5 100644 --- a/code/tests/test_inference_public_model.py +++ b/code/tests/test_inference_public_model.py @@ -69,7 +69,10 @@ def _run_inference_rtest(distance: int, n_rounds: int, model_info: dict): model_file = (MODELS_DIR / model_info["filename"]).resolve() if not model_file.exists(): raise FileNotFoundError( - f"Missing model file: {model_file}. It must be in the repo (Git LFS). Run 'git lfs pull' or restore the file." + f"Missing model file: {model_file}. Download it from Hugging Face " + "(https://huggingface.co/nvidia/Ising-Decoder-SurfaceCode-1-Fast or " + "https://huggingface.co/nvidia/Ising-Decoder-SurfaceCode-1-Accurate); " + "in CI it is fetched by .github/actions/fetch-models." ) merged.model_checkpoint_file = str(model_file) @@ -121,12 +124,13 @@ class TestPublicInferenceModels(unittest.TestCase): """Tests for pre-trained model files (r9 and r13).""" def test_required_model_files_present(self): - """Fail if any tracked model file is missing. Must not skip.""" + """Fail if any required model file is missing. Must not skip.""" for model_file in REQUIRED_MODEL_FILES: self.assertTrue( model_file.exists(), msg=f"Required model file missing: {model_file}. " - "It must be in the repo (Git LFS). Run 'git lfs pull' or restore the file.", + "Download it from Hugging Face (see README, 'Inference (pre-trained " + "models)'); in CI it is fetched by .github/actions/fetch-models.", ) # -- R=9 model tests -- diff --git a/models/Ising-Decoder-SurfaceCode-1-Accurate.pt b/models/Ising-Decoder-SurfaceCode-1-Accurate.pt deleted file mode 100644 index 0d91feb..0000000 --- a/models/Ising-Decoder-SurfaceCode-1-Accurate.pt +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:f49df7fc70d2ca890d5155982dae79e531f012d584203308617e9f6316b19fa9 -size 7196619 diff --git a/models/Ising-Decoder-SurfaceCode-1-Fast.pt b/models/Ising-Decoder-SurfaceCode-1-Fast.pt deleted file mode 100644 index cc13c8b..0000000 --- a/models/Ising-Decoder-SurfaceCode-1-Fast.pt +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:08ce4b1e09e3f396401d606b7c476225101012a6206bc7877efe153daae7b8c4 -size 3655297