diff --git a/.envrc b/.envrc new file mode 100644 index 00000000..3550a30f --- /dev/null +++ b/.envrc @@ -0,0 +1 @@ +use flake diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml new file mode 100644 index 00000000..d0a19dc0 --- /dev/null +++ b/.github/actionlint.yaml @@ -0,0 +1,9 @@ +# Custom self-hosted runner labels used across this repo's workflows +# (Blacksmith runners). Declaring them here stops actionlint from +# reporting "unknown runner label" for runs-on: blacksmith-*. +self-hosted-runner: + labels: + - blacksmith-16vcpu-ubuntu-2204 + - blacksmith-4vcpu-ubuntu-2204 + - blacksmith-16vcpu-ubuntu-2404 + - blacksmith-4vcpu-ubuntu-2404 diff --git a/.github/actions/docker-setup/action.yml b/.github/actions/docker-setup/action.yml index 8b917dcf..dca5b391 100644 --- a/.github/actions/docker-setup/action.yml +++ b/.github/actions/docker-setup/action.yml @@ -104,4 +104,4 @@ runs: BRANCH_NAME=$(echo "${RAW_BRANCH}" | sed 's/\//-/g') SUFFIX="-dev-${BRANCH_NAME}" echo "RELEASE=${SUFFIX}" >> $GITHUB_ENV - echo "release_suffix=${SUFFIX}" >> $GITHUB_OUTPUT \ No newline at end of file + echo "release_suffix=${SUFFIX}" >> $GITHUB_OUTPUT diff --git a/.github/actions/hadolint/action.yml b/.github/actions/hadolint/action.yml index 18eaca43..e1833632 100644 --- a/.github/actions/hadolint/action.yml +++ b/.github/actions/hadolint/action.yml @@ -32,4 +32,4 @@ runs: failure-threshold: ${{ inputs.failure-threshold }} ignore: ${{ inputs.ignore }} format: ${{ inputs.format }} - output-file: ${{ inputs.output-file }} \ No newline at end of file + output-file: ${{ inputs.output-file }} diff --git a/.github/actions/update-readme/action.yml b/.github/actions/update-readme/action.yml index e673c314..d3726279 100644 --- a/.github/actions/update-readme/action.yml +++ b/.github/actions/update-readme/action.yml @@ -1,40 +1,39 @@ -name: "Update Template README" -description: "Updates a Runpod template README via the API" -inputs: - template_id: - description: "Runpod Template ID" - required: true - template_path: - description: "Path to the template directory" - required: true - -runs: - using: "composite" - steps: - - name: Check if README exists - id: check_readme - shell: bash - env: - TEMPLATE_PATH: ${{ inputs.template_path }} - run: | - if [[ -f "${TEMPLATE_PATH}/README.md" ]]; then - echo "skip=false" >> "$GITHUB_OUTPUT" - else - echo "skip=true" >> "$GITHUB_OUTPUT" - echo "README.md not found at ${TEMPLATE_PATH}/README.md, skipping update" - fi - - - name: Make script executable - if: steps.check_readme.outputs.skip != 'true' - shell: bash - run: chmod +x "${GITHUB_WORKSPACE}/scripts/update-template-readme.sh" - - - name: Update template README - if: steps.check_readme.outputs.skip != 'true' - shell: bash - env: - TEMPLATE_ID: ${{ inputs.template_id }} - TEMPLATE_PATH: ${{ inputs.template_path }} - run: | - "${GITHUB_WORKSPACE}/scripts/update-template-readme.sh" "${TEMPLATE_ID}" "${TEMPLATE_PATH}" - \ No newline at end of file +name: "Update Template README" +description: "Updates a Runpod template README via the API" +inputs: + template_id: + description: "Runpod Template ID" + required: true + template_path: + description: "Path to the template directory" + required: true + +runs: + using: "composite" + steps: + - name: Check if README exists + id: check_readme + shell: bash + env: + TEMPLATE_PATH: ${{ inputs.template_path }} + run: | + if [[ -f "${TEMPLATE_PATH}/README.md" ]]; then + echo "skip=false" >> "$GITHUB_OUTPUT" + else + echo "skip=true" >> "$GITHUB_OUTPUT" + echo "README.md not found at ${TEMPLATE_PATH}/README.md, skipping update" + fi + + - name: Make script executable + if: steps.check_readme.outputs.skip != 'true' + shell: bash + run: chmod +x "${GITHUB_WORKSPACE}/scripts/update-template-readme.sh" + + - name: Update template README + if: steps.check_readme.outputs.skip != 'true' + shell: bash + env: + TEMPLATE_ID: ${{ inputs.template_id }} + TEMPLATE_PATH: ${{ inputs.template_path }} + run: | + "${GITHUB_WORKSPACE}/scripts/update-template-readme.sh" "${TEMPLATE_ID}" "${TEMPLATE_PATH}" diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 050a42c0..60f45ba0 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -7,13 +7,13 @@ version: 2 updates: # Enable version updates for github-actions - - package-ecosystem: "github-actions" - directory: "/" + - package-ecosystem: "github-actions" + directory: "/" schedule: interval: "weekly" - # Enable version updates for Docker + # Enable version updates for Docker - package-ecosystem: "docker" - directory: "/" + directory: "/" schedule: interval: "daily" diff --git a/.github/workflows/base.yml b/.github/workflows/base.yml index 6d060c34..96771373 100644 --- a/.github/workflows/base.yml +++ b/.github/workflows/base.yml @@ -363,7 +363,7 @@ jobs: with: image-refs: ${{ steps.refs.outputs.refs }} ref-digests: ${{ steps.refs.outputs.ref-digests }} - + - name: Install Cosign if: github.event_name == 'workflow_dispatch' || steps.changes.outputs.pytorch_any_changed == 'true' uses: sigstore/cosign-installer@6f9f17788090df1f26f669e9d70d6ae9567deba6 diff --git a/.github/workflows/hadolint-pr.yml b/.github/workflows/hadolint-pr.yml index 2b8aa4c3..444c9e7d 100644 --- a/.github/workflows/hadolint-pr.yml +++ b/.github/workflows/hadolint-pr.yml @@ -36,4 +36,3 @@ jobs: format: tty output-file: /dev/stdout ignore: ${{ matrix.ignore }} - \ No newline at end of file diff --git a/.github/workflows/hadolint-push.yml b/.github/workflows/hadolint-push.yml index dd603954..fdfcc2ff 100644 --- a/.github/workflows/hadolint-push.yml +++ b/.github/workflows/hadolint-push.yml @@ -37,10 +37,9 @@ jobs: format: sarif output-file: hadolint.sarif ignore: ${{ matrix.ignore }} - + - name: Upload Hadolint SARIF to Code Scanning uses: github/codeql-action/upload-sarif@8aad20d150bbac5944a9f9d289da16a4b0d87c1e with: sarif_file: hadolint.sarif category: hadolint-${{ strategy.job-index }} - \ No newline at end of file diff --git a/.github/workflows/nix.yml b/.github/workflows/nix.yml new file mode 100644 index 00000000..383e9bcf --- /dev/null +++ b/.github/workflows/nix.yml @@ -0,0 +1,120 @@ +name: Nix static analysis + +on: + push: + branches: [main] + pull_request: + workflow_dispatch: + inputs: + heavy: + description: "Which heavy build-only job(s) to run" + type: choice + default: all + options: [none, cuda, fleet, all] + +permissions: + contents: read + +jobs: + flake-check: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 + + - name: Install Nix + uses: DeterminateSystems/nix-installer-action@ef8a148080ab6020fd15196c2084a2eea5ff2d25 + with: + extra-conf: | + experimental-features = nix-command flakes + + # Runs the sandbox-safe gated checks (nixfmt/statix/deadnix/nil, + # shellcheck, yamllint, actionlint, hadolint, markdown, taplo, + # editorconfig, gitleaks). --keep-going so every check's findings + # surface in one run instead of stopping at the first failure. + - name: nix flake check + run: nix flake check --print-build-logs --keep-going + + # Experimental: build the base-cpu image with Nix and compare its size to the + # published runpod/base CPU image. Build-only — nothing is pushed. Kept in a + # separate job so it never slows the gated flake-check above. + image-size: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 + + - name: Install Nix + uses: DeterminateSystems/nix-installer-action@ef8a148080ab6020fd15196c2084a2eea5ff2d25 + with: + extra-conf: | + experimental-features = nix-command flakes + + - name: Build Nix base-cpu image + size comparison (build-only) + run: bash nix/images/compare-size.sh + + # Cross-image layer-sharing footprint for the Nix image family: bytes a host + # caches once shared layers dedupe, vs the naïve per-image sum. Build-only. + family-footprint: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 + + - name: Install Nix + uses: DeterminateSystems/nix-installer-action@ef8a148080ab6020fd15196c2084a2eea5ff2d25 + with: + extra-conf: | + experimental-features = nix-command flakes + + - name: Image-family layer-sharing footprint + run: bash nix/images/footprint.sh + + # Experimental hybrid CUDA image (nvidia base + Nix userland). Heavy: pulls a + # ~5.6 GB CUDA base and streams ~11 GB, so it is manual-dispatch only — it + # does NOT run on every push. Trigger with: + # gh workflow run "Nix static analysis" --ref + cuda-image-size: + if: github.event_name == 'workflow_dispatch' && (inputs.heavy == 'cuda' || inputs.heavy == 'all') + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 + + - name: Free disk space + run: | + sudo rm -rf /opt/hostedtoolcache /usr/local/lib/android /usr/share/dotnet /usr/local/.ghcup + df -h / + + - name: Install Nix + uses: DeterminateSystems/nix-installer-action@ef8a148080ab6020fd15196c2084a2eea5ff2d25 + with: + extra-conf: | + experimental-features = nix-command flakes + + - name: Build hybrid CUDA image + size comparison (build-only) + run: bash nix/images/compare-cuda-size.sh + + # Fleet-wide layer-sharing footprint of the published images across all + # container types (skopeo manifests only, no pulls). Manual-dispatch: ~38 + # images = many Docker Hub manifest requests, so authenticate + don't run on + # every push. Trigger with: + # gh workflow run "Nix static analysis" --ref -f heavy=fleet + fleet-footprint: + if: github.event_name == 'workflow_dispatch' && (inputs.heavy == 'fleet' || inputs.heavy == 'all') + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 + + - name: Install Nix + uses: DeterminateSystems/nix-installer-action@ef8a148080ab6020fd15196c2084a2eea5ff2d25 + with: + extra-conf: | + experimental-features = nix-command flakes + + - name: Published-fleet layer-sharing footprint + env: + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + run: bash nix/images/fleet-footprint.sh diff --git a/.github/workflows/rocm.yml b/.github/workflows/rocm.yml index b4029edf..1f071694 100644 --- a/.github/workflows/rocm.yml +++ b/.github/workflows/rocm.yml @@ -33,7 +33,7 @@ jobs: uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 with: fetch-depth: 0 - + - name: Setup Docker uses: ./.github/actions/docker-setup id: setup @@ -99,7 +99,7 @@ jobs: path: /tmp/rocm-refs/refs.json retention-days: 1 if-no-files-found: error - + test-rocm: runs-on: blacksmith-4vcpu-ubuntu-2404 needs: build-rocm diff --git a/.github/workflows/sonarqube.yml b/.github/workflows/sonarqube.yml index a4d5a3e2..3a61d109 100644 --- a/.github/workflows/sonarqube.yml +++ b/.github/workflows/sonarqube.yml @@ -32,4 +32,4 @@ jobs: pollingTimeoutSec: 300 env: SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} - SONAR_HOST_URL: https://sonarcloud.io + SONAR_HOST_URL: https://sonarcloud.io diff --git a/.gitignore b/.gitignore index 9c3fcfd0..f0977532 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,9 @@ *.safetensors *.pth -__pycache__/ \ No newline at end of file +__pycache__/ + +# Nix +result +result-* +.direnv/ \ No newline at end of file diff --git a/.gitleaks.toml b/.gitleaks.toml new file mode 100644 index 00000000..34e0461c --- /dev/null +++ b/.gitleaks.toml @@ -0,0 +1,21 @@ +title = "runpod/containers gitleaks config" + +# Start from gitleaks' built-in rule set, then add repo-specific allowlists. +[extend] +useDefault = true + +# The smoke-test composite action greps a user-supplied SSH key for PEM header +# lines to validate its shape. The "-----BEGIN ... PRIVATE KEY-----" strings are +# literals in a case statement, not committed secrets. Scope the allowlist to +# that file AND the PEM-header pattern (matchCondition = AND) so a genuine key +# committed elsewhere still trips the scanner. +[[allowlists]] +description = "PEM header literals in the smoke-test SSH key validator" +matchCondition = "AND" +paths = [ + '''\.github/actions/smoke-test/action\.yml''', +] +regexTarget = "line" +regexes = [ + '''BEGIN (OPENSSH |RSA |EC )?PRIVATE KEY''', +] diff --git a/.hadolint.yaml b/.hadolint.yaml new file mode 100644 index 00000000..714da7d6 --- /dev/null +++ b/.hadolint.yaml @@ -0,0 +1,14 @@ +# Align the Nix gated hadolint check with the ignore rules the repo's own +# hadolint workflow (.github/workflows/hadolint-*.yml) already applies +# per-Dockerfile, so the two agree on policy: +# DL3006 — always tag image version +# DL3008 — pin apt package versions +# DL3013 — pin pip package versions +# DL3022 — COPY --from references a prior FROM alias +# DL3042 — pip --no-cache-dir +ignored: + - DL3006 + - DL3008 + - DL3013 + - DL3022 + - DL3042 diff --git a/.markdownlint.yaml b/.markdownlint.yaml new file mode 100644 index 00000000..4eb0df63 --- /dev/null +++ b/.markdownlint.yaml @@ -0,0 +1,9 @@ +# Pragmatic markdownlint config. Keep structural rules; disable the +# stylistic ones that commonly fire on READMEs. +default: true +# Line length — prose and tables routinely exceed a fixed width. +MD013: false +# Allow inline HTML (badges,
, etc.). +MD033: false +# Do not require the first line to be a top-level heading. +MD041: false diff --git a/.yamllint.yaml b/.yamllint.yaml new file mode 100644 index 00000000..1e27248d --- /dev/null +++ b/.yamllint.yaml @@ -0,0 +1,15 @@ +# Pragmatic yamllint config. Extends the default rules but relaxes the +# ones that fight normal GitHub Actions / Docker-compose style YAML. +extends: default + +rules: + # GitHub Actions workflows use `on:` which yamllint's default flags as + # a truthy value; and long lines are common in run: blocks. + line-length: disable + truthy: + allowed-values: ["true", "false"] + check-keys: false + comments: + min-spaces-from-content: 1 + comments-indentation: disable + document-start: disable diff --git a/README.md b/README.md index 9b933375..008b7eca 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ This repository uses Docker Buildx with [bake files](https://docs.docker.com/bui ### Using the Bake Script -`./bake.sh` automatically combines shared version definitions with template specific bake files. +`./bake.sh` automatically combines shared version definitions with template specific bake files. Use it like this: diff --git a/docs/COSIGN.md b/docs/COSIGN.md index 29bae2eb..649b624a 100644 --- a/docs/COSIGN.md +++ b/docs/COSIGN.md @@ -22,13 +22,13 @@ Contents: ### TL;DR - **No long-lived private signing keys exist** anywhere in this repository, in -GitHub Secrets, on Runpod machines, or in any vault. There is nothing to -rotate, back up, or leak. + GitHub Secrets, on Runpod machines, or in any vault. There is nothing to + rotate, back up, or leak. - Every signing run mints a **fresh ephemeral key pair on the GitHub Actions -runner**, uses it once, and discards it when the runner shuts down. + runner**, uses it once, and discards it when the runner shuts down. - Trust is anchored in (a) the **public Sigstore PKI** (Fulcio + Rekor) and -(b) the **GitHub OIDC identity** of the workflow that signed the image, -enforced via the regex policy consumers pass to `cosign verify`. + (b) the **GitHub OIDC identity** of the workflow that signed the image, + enforced via the regex policy consumers pass to `cosign verify`. ### What "keyless" actually means @@ -53,11 +53,8 @@ sequenceDiagram R->>R: discard private key (process exits) ``` - - Per signing event the runner produces: - | Artifact | Lifetime | Stored where | | ---------------------- | ----------------------- | ----------------------------------------------------- | | Ephemeral private key | seconds (runner RAM) | nowhere — discarded with the runner | @@ -66,44 +63,41 @@ Per signing event the runner produces: | Signature blob | permanent | OCI artifact next to the image: `sha256-.sig` | | Rekor log entry | permanent (append-only) | `https://rekor.sigstore.dev` (publicly auditable) | - ### Where the "CA" lives We do **not** operate our own CA. The CA stack is Sigstore public-good infrastructure: - | Component | Operator | Endpoint | Role | | ------------------- | ----------------------------------- | ----------------------------------------------------------- | ----------------------------------------------------------------- | | Fulcio | Sigstore project (Linux Foundation) | `https://fulcio.sigstore.dev` | Issues short-lived code-signing certs from OIDC identities. | | Rekor | Sigstore project (Linux Foundation) | `https://rekor.sigstore.dev` | Append-only public log of all signatures. | | Root of trust (TUF) | Sigstore project | bundled inside cosign + `https://tuf-repo-cdn.sigstore.dev` | Distributes Fulcio root cert + Rekor public key, rotated via TUF. | - Cosign ships the Sigstore TUF root **inside the binary**. Upgrading cosign automatically picks up any upstream rotation — see [§5.1](#51-cosign-version-upgrade-every-612-months-or-on-cve). ### What is secret? **Nothing on our side.** No shared secret, signing key, or HSM. What we -*do* depend on: +_do_ depend on: - The **integrity of GitHub's OIDC issuer** (`token.actions.githubusercontent.com`). -If compromised, an attacker could mint tokens that impersonate this repo's -workflows. That is an industry-wide incident, not something we can -unilaterally mitigate. -- The `**id-token: write` permission** declared on signing workflows. Any -workflow in this repo with that permission can request an OIDC token whose -`sub` points at its own workflow URI. Branch protection on `main` and PR -review gate what code can introduce or modify such a workflow. + If compromised, an attacker could mint tokens that impersonate this repo's + workflows. That is an industry-wide incident, not something we can + unilaterally mitigate. +- The `**id-token: write` permission\*\* declared on signing workflows. Any + workflow in this repo with that permission can request an OIDC token whose + `sub` points at its own workflow URI. Branch protection on `main` and PR + review gate what code can introduce or modify such a workflow. - The **verification policy** consumers pin (regex + issuer). A loose regex -defeats the whole scheme; see [§3](#3-verification-process-consumers). + defeats the whole scheme; see [§3](#3-verification-process-consumers). ### Identity baked into each signature After signing, the certificate's SAN encodes the GitHub workflow URI, e.g.: -``` +```text https://github.com/runpod/containers/.github/workflows/base.yml@refs/heads/main ``` @@ -117,28 +111,26 @@ workflow name, ref, commit SHA, and trigger. This is exactly what ### Where it is implemented - -| File | Purpose | -| --------------------------------------- | ---------------------------------------------------------------------------- | -| `.github/actions/cosign/action.yml` | Reusable sign + verify composite action. | -| `.github/actions/image-name/action.yml` | Extracts image refs, signable digests, and ref-to-digest mappings from `docker buildx bake` metadata. | -| `.github/actions/docker-push/action.yml` | Pushes tags and verifies Docker Hub resolved each exact ref to the expected digest. | -| `.github/workflows/base.yml` | Builds and signs `runpod/base`, `runpod/pytorch`, `runpod/autoresearch`. | -| `.github/workflows/nvidia.yml` | Builds and signs `runpod/nvidia-*` images. | -| `.github/workflows/rocm.yml` | Builds and signs `runpod/rocm` images. | - +| File | Purpose | +| ---------------------------------------- | ----------------------------------------------------------------------------------------------------- | +| `.github/actions/cosign/action.yml` | Reusable sign + verify composite action. | +| `.github/actions/image-name/action.yml` | Extracts image refs, signable digests, and ref-to-digest mappings from `docker buildx bake` metadata. | +| `.github/actions/docker-push/action.yml` | Pushes tags and verifies Docker Hub resolved each exact ref to the expected digest. | +| `.github/workflows/base.yml` | Builds and signs `runpod/base`, `runpod/pytorch`, `runpod/autoresearch`. | +| `.github/workflows/nvidia.yml` | Builds and signs `runpod/nvidia-*` images. | +| `.github/workflows/rocm.yml` | Builds and signs `runpod/rocm` images. | Each signing workflow declares the permissions cosign keyless requires: ```yaml permissions: contents: read - id-token: write # required to request the OIDC token consumed by Fulcio + id-token: write # required to request the OIDC token consumed by Fulcio ``` ### Pipeline order -``` +```text docker/bake-action → image-name (extract digests) → grype → docker-push (push + registry digest check) → cosign-installer (pinned) → cosign action (sign + verify) ``` @@ -163,14 +155,14 @@ cosign verify \ Notes: - `--yes` is non-interactive consent to the Sigstore Terms of Service — that -is the only reason `cosign sign` would otherwise prompt. It does **not** -affect what gets signed. + is the only reason `cosign sign` would otherwise prompt. It does **not** + affect what gets signed. - No `--key` is passed → cosign 2.x defaults to keyless / OIDC. - The `verify` step in the same job is a smoke test: a malformed signature or -a regex mismatch fails the build immediately rather than landing a broken -signature in production. + a regex mismatch fails the build immediately rather than landing a broken + signature in production. - Signatures are written to Docker Hub next to the image as -`docker.io/runpod/:sha256-.sig`. + `docker.io/runpod/:sha256-.sig`. ### Why we sign the digest, not the tag @@ -187,11 +179,11 @@ consumers should verify too. ### Prerequisites - `cosign` ≥ 2.0 — we run 2.6.x in CI. Install via `brew install cosign`, -your distro's package manager, or -[the upstream releases page](https://github.com/sigstore/cosign/releases). + your distro's package manager, or + [the upstream releases page](https://github.com/sigstore/cosign/releases). - Anything that can resolve `image:tag → sha256:digest`. Docker / buildx is -the path of least resistance; `crane digest` is a lighter alternative if -the Docker daemon is not available. + the path of least resistance; `crane digest` is a lighter alternative if + the Docker daemon is not available. ### Step-by-step @@ -212,7 +204,7 @@ cosign verify \ A successful run prints: -``` +```text Verification for index.docker.io/runpod/base@sha256:... -- The following checks were performed on each of these signatures: - The cosign claims were validated @@ -270,17 +262,15 @@ For each image push, CI publishes exactly one cosign signature OCI artifact plus one Rekor log entry. Concretely, for an image with digest `sha256:` you get: - | Artifact | Where | Contents | | ---------------------------- | ------------------------------------------------------------------------------ | -------------------------------------------------------------------------------------------------------------- | | Signature OCI artifact | `docker.io/runpod/:sha256-.sig` | Payload (image digest + reference), signature, signing certificate chain, Rekor bundle (SET + log index). | | Rekor transparency-log entry | `https://rekor.sigstore.dev` (`hashedrekord`, queryable by hash or `logIndex`) | Tamper-evident public record of the above; what `cosign verify` checks against in the "transparency log" line. | - What is currently **not** published: - `cosign attest` attestations for SBOM, SLSA provenance, vuln-scan results, -in-toto, etc. + in-toto, etc. - `cosign attach sbom` SBOM blobs (the upstream-deprecated form). - OCI Referrers API attestations (Grype, Syft outputs). @@ -301,26 +291,26 @@ images, and no key escrow**. The maintenance surface is small but not zero. ### 5.1 Cosign version upgrade (every 6–12 months, or on CVE) - Bump `cosign-release: vX.Y.Z` in every workflow that installs cosign: -`base.yml`, `nvidia.yml`, `rocm.yml`. -- Watch `[sigstore/cosign` releases](https://github.com/sigstore/cosign/releases) -for security advisories; treat any cosign GHSA as a P1 bump. + `base.yml`, `nvidia.yml`, `rocm.yml`. +- Watch `[sigstore/cosign` releases]() + for security advisories; treat any cosign GHSA as a P1 bump. - A PR build covers the validation: a regressed sign or verify step fails -the job loudly before merge. + the job loudly before merge. ### 5.2 OIDC issuer / Fulcio endpoint change - `--certificate-oidc-issuer` is currently `https://token.actions.githubusercontent.com` -(GitHub's, not Sigstore's). It only changes if **GitHub** changes their -OIDC issuer — historically rare and pre-announced. + (GitHub's, not Sigstore's). It only changes if **GitHub** changes their + OIDC issuer — historically rare and pre-announced. - Fulcio / Rekor endpoints are taken from cosign defaults, not pinned in our -workflows. They have changed in the past (Sigstore GA transition); cosign -upgrades cover those moves. + workflows. They have changed in the past (Sigstore GA transition); cosign + upgrades cover those moves. ### 5.3 Workflow / identity renames If a workflow file is renamed, moved, or split, the cert SAN changes: -``` +```text .../.github/workflows/.yml@refs/... → .../.github/workflows/.yml@refs/... ``` @@ -334,7 +324,6 @@ to move off keyless. ### 5.4 Compromise / incident response - | Scenario | Action | | ------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | | Signing workflow is compromised (malicious push merged to `main`). | Narrow consumer regex to exclude the bad commit via `--certificate-github-workflow-sha` deny-listing; rebuild + resign affected tags from a clean commit; publish a security advisory. | @@ -342,8 +331,7 @@ to move off keyless. | A cosign release is yanked / found vulnerable. | Apply 5.1 (bump pinned release); rebuild affected images; notify consumers if the vuln affects signature integrity. | | GitHub OIDC issuer is compromised. | Industry-wide incident: halt releases, follow GitHub's guidance, plan to re-sign every published image with the post-incident issuer (signatures from the compromised period must be treated as untrusted by consumers). | - -### 5.5 What does *not* need maintenance +### 5.5 What does _not_ need maintenance For clarity, none of the following belongs in a playbook: @@ -351,18 +339,17 @@ For clarity, none of the following belongs in a playbook: - Managing a private CA or shipping a CA-bundle update. - Backing up signing keys. - Distributing a public key to consumers (consumers verify via -identity + issuer, not `--key`). + identity + issuer, not `--key`). --- ## 6. Troubleshooting - -| Symptom | Likely cause | Fix | -| ---------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------- | -| `Error: no matching signatures: none of the expected identities matched what was in the certificate` | Regex too strict, or you are verifying an image from a fork / different repo. | Loosen the regex during debugging, confirm the image came from `runpod/containers`, then re-pin. | -| `Error: parsing reference: could not parse reference: docker.io/runpod/base@` | Digest variable captured multi-line output of `docker manifest inspect`. | Use the `docker buildx imagetools inspect ... | awk '/^Digest:/ {print $2; exit}'` recipe from [§3](#3-verification-process-consumers). | -| `Error: signing ... unauthorized` in CI | Workflow missing `id-token: write`. | Add the `permissions` block ([§2](#2-signing-process-ci)). | -| `cosign verify` succeeds locally but admission controller rejects | Different regex / issuer pinned in the controller policy. | Align the controller policy with the actual signing workflow URI ([§3](#tightening-the-policy-for-production)). | -| Verification slow / hangs | Rekor lookup against `rekor.sigstore.dev` is slow or blocked. | Use `cosign verify --offline ...` if the bundle is attached, or whitelist `*.sigstore.dev`. | -| `Error: fetching signatures: GET ... manifest unknown` | Image was not signed (older tag from before this workflow landed). | Re-trigger the build pipeline to sign in place, or verify a newer tag. | +| Symptom | Likely cause | Fix | +| ---------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------- | +| `Error: no matching signatures: none of the expected identities matched what was in the certificate` | Regex too strict, or you are verifying an image from a fork / different repo. | Loosen the regex during debugging, confirm the image came from `runpod/containers`, then re-pin. | +| `Error: parsing reference: could not parse reference: docker.io/runpod/base@` | Digest variable captured multi-line output of `docker manifest inspect`. | Use the `docker buildx imagetools inspect ... | awk '/^Digest:/ {print $2; exit}'` recipe from [§3](#3-verification-process-consumers). | +| `Error: signing ... unauthorized` in CI | Workflow missing `id-token: write`. | Add the `permissions` block ([§2](#2-signing-process-ci)). | +| `cosign verify` succeeds locally but admission controller rejects | Different regex / issuer pinned in the controller policy. | Align the controller policy with the actual signing workflow URI ([§3](#tightening-the-policy-for-production)). | +| Verification slow / hangs | Rekor lookup against `rekor.sigstore.dev` is slow or blocked. | Use `cosign verify --offline ...` if the bundle is attached, or whitelist `*.sigstore.dev`. | +| `Error: fetching signatures: GET ... manifest unknown` | Image was not signed (older tag from before this workflow landed). | Re-trigger the build pipeline to sign in place, or verify a newer tag. | diff --git a/flake.lock b/flake.lock new file mode 100644 index 00000000..13b06e1d --- /dev/null +++ b/flake.lock @@ -0,0 +1,82 @@ +{ + "nodes": { + "flake-utils": { + "inputs": { + "systems": "systems" + }, + "locked": { + "lastModified": 1731533236, + "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "nix2container": { + "inputs": { + "nixpkgs": [ + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1775487831, + "narHash": "sha256-2lguQpLPQaxpQCJjXhmEEAfabwsAhkP29Z7fgLzHARA=", + "owner": "nlewo", + "repo": "nix2container", + "rev": "76be9608a7f4d6c985d28b0e7be903ae2547df3e", + "type": "github" + }, + "original": { + "owner": "nlewo", + "repo": "nix2container", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1783776592, + "narHash": "sha256-UgCQzxeWI75XM8G+hPrPh+MKzEPjG3SpAj7dtqSbksA=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "e7a3ca8092b61ff85b6a45bf863ea2b2d6a661b3", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "flake-utils": "flake-utils", + "nix2container": "nix2container", + "nixpkgs": "nixpkgs" + } + }, + "systems": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 00000000..87a88870 --- /dev/null +++ b/flake.nix @@ -0,0 +1,104 @@ +{ + description = "runpod/containers — static-analysis surface (Python, shell, Dockerfiles, YAML/GitHub Actions, JSON/TOML, Markdown, Nix, secrets, SAST, supply chain)."; + + inputs = { + nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; + flake-utils.url = "github:numtide/flake-utils"; + # For the experimental nix2container image targets (compared against + # dockerTools). Follows our nixpkgs so there's only one pin. + nix2container.url = "github:nlewo/nix2container"; + nix2container.inputs.nixpkgs.follows = "nixpkgs"; + }; + + outputs = + { + nixpkgs, + flake-utils, + nix2container, + ... + }: + flake-utils.lib.eachSystem + [ + "x86_64-linux" + "aarch64-linux" + # x86_64-darwin dropped: nixpkgs 26.11+ (nixos-unstable) no longer supports it. + "aarch64-darwin" + ] + ( + system: + let + pkgs = import nixpkgs { + inherit system; + config.allowUnfree = true; + }; + inherit (pkgs) lib; + + src = lib.cleanSourceWith { + src = ./.; + filter = + path: _type: + let + baseName = baseNameOf (toString path); + in + baseName != "result" + && baseName != ".direnv" + && baseName != ".git" + && baseName != "node_modules" + && baseName != "dist" + && baseName != "build"; + }; + + repoLib = import ./nix/lib.nix { inherit pkgs lib; }; + + checks = import ./nix/checks.nix { inherit pkgs src; }; + + lints = import ./nix/lints { inherit pkgs lib; }; + + devShell = import ./nix/devshell { inherit pkgs repoLib; }; + + # Experimental Nix-built OCI images (build-only, not published). + # Linux-only — these are Linux container images. Passing the + # nix2container builder in adds the *-n2c variants for comparison. + n2c = nix2container.packages.${system}; + images = lib.optionalAttrs pkgs.stdenv.isLinux ( + import ./nix/images { + inherit pkgs; + n2c = n2c.nix2container; + } + ); + + # Native-parity ("prod") images — a separate tree from the PoC above, + # rebuilding base + pytorch entirely from nixpkgs (source builds, + # full-Nix CUDA). Build-only, not published. + prodImages = lib.optionalAttrs pkgs.stdenv.isLinux (import ./nix/prod-images { inherit pkgs; }); + + formatter = pkgs.nixfmt-rfc-style; + in + { + inherit checks; + + packages = + images + // prodImages + // lib.optionalAttrs pkgs.stdenv.isLinux { + # Patched skopeo that understands nix2container's `nix:` transport, + # used by footprint.sh to inspect the n2c images' real OCI layers. + skopeo-n2c = n2c.skopeo-nix2container; + }; + + devShells.default = devShell; + + apps = { + default = + lints.apps.lint-extreme or { + type = "app"; + program = "${pkgs.writeShellScript "default" "echo 'See nix flake show for the static-analysis surface.'"}"; + meta.description = "Default app — points at lint-extreme."; + }; + } + // lints.apps; + + inherit formatter; + } + ); +} diff --git a/helper-templates/verify-nccl/README.md b/helper-templates/verify-nccl/README.md index 595cb08b..2db3bf05 100644 --- a/helper-templates/verify-nccl/README.md +++ b/helper-templates/verify-nccl/README.md @@ -1,6 +1,5 @@ # Verify NCCL - ## Running Test SSH into the pod and then run the following command: diff --git a/nix/README.md b/nix/README.md new file mode 100644 index 00000000..ceeaef2f --- /dev/null +++ b/nix/README.md @@ -0,0 +1,66 @@ +# Static-analysis flake + +A small `flake.nix` at the repo root pulls in the modular `*.nix` files in +this directory to expose a reproducible static-analysis surface for the +`containers` repo: Python, shell, Dockerfiles, YAML / GitHub Actions, +JSON / TOML, Markdown, and the flake's own Nix. + +No global installs are needed — every tool is pinned by `flake.lock`. + +## Quick start + +```sh +# Enter a shell with every tool on PATH (or use direnv + .envrc): +nix develop + +# Run the gated, sandbox-safe checks (what CI would gate on): +nix flake check --print-build-logs + +# Run a whole domain: +nix run .#lint-py +nix run .#lint-shell +nix run .#lint-docker + +# Run absolutely everything: +nix run .#everything +``` + +## Layout + +| Path | Purpose | +| --- | --- | +| `../flake.nix` | Thin entry point: inputs, `src` filter, wires the four modules below, `formatter`. | +| `lib.nix`, `lib/*.nix` | Shared builders: `mk-check` (gated derivations), `mk-lint` (per-tool apps), `mk-all-app` (composites), `maybe-tool` (graceful skip for tools absent from the pin). | +| `checks.nix` | Sandbox-safe derivations for `nix flake check`. | +| `lints/*.nix` | One file per tool + `default.nix` registry + domain composites + `everything.nix`. | +| `devshell/*.nix` | `nix develop` shell carrying every tool. | + +## Two execution surfaces + +- **`nix flake check`** — gated derivations that run in the Nix sandbox + (no network, no language runtime with third-party imports): nixfmt, + statix, deadnix, nil, shellcheck, yamllint, actionlint, hadolint, + markdownlint, taplo, editorconfig-checker, gitleaks. +- **`nix run .#lint-*`** — manual apps, including tools that need the + network (trivy, osv-scanner, lychee, trufflehog, semgrep) or the + project's own dependencies to resolve imports (pyright, pylint). + +## Posture + +Pragmatic tool defaults, honoring the repo-root config files +(`.yamllint.yaml`, `.markdownlint.yaml`, `pyproject.toml`). Tighten by +editing those configs or the individual `lints/*.nix` commands. + +## Known coverage gaps + +Some file types in this repo have no first-class linter here and are +left to manual review: + +- **HCL** (`docker-bake.hcl`, `versions.hcl`) — no HCL linter is wired + in; review by hand (or `docker buildx bake --print` to validate). +- **INI** (`grafana.ini`) — only `editorconfig-checker` hygiene applies. +- **CSV** (`extended-counters.csv`) — only `editorconfig-checker` + hygiene applies. + +Large generated JSON (e.g. Grafana dashboards) is validated for +well-formedness by `lint-jsonlint`; deep semantic review is manual. diff --git a/nix/checks.nix b/nix/checks.nix new file mode 100644 index 00000000..c69bd7c3 --- /dev/null +++ b/nix/checks.nix @@ -0,0 +1,189 @@ +# Gated derivations consumed by `nix flake check`. These are the +# sandbox-safe checks (no network, no populated language toolchain). +# +# Tools that need network access (osv-scanner, trivy, lychee, +# trufflehog) or a language runtime with resolvable imports +# (pyright/mypy against third-party deps) are exposed as +# `nix run .#lint-*` manual apps only — see ./lints. +# +# Posture is pragmatic: tool defaults, honoring any repo-root config +# file (.yamllint.yaml, .markdownlint.yaml, pyproject.toml, ...). + +{ pkgs, src }: + +let + mkCheck = import ./lib/mk-check.nix { inherit pkgs src; }; +in +{ + # --- Nix (lint our own flake) --- + nixfmt-check = mkCheck { + name = "nixfmt-check"; + buildInputs = [ + pkgs.nixfmt-rfc-style + pkgs.findutils + ]; + script = '' + find . -name '*.nix' \ + -not -path './.git/*' \ + -not -path './result*' \ + -exec nixfmt --check {} + + ''; + }; + + deadnix = mkCheck { + name = "deadnix"; + buildInputs = [ + pkgs.deadnix + pkgs.findutils + ]; + script = '' + find . -name '*.nix' \ + -not -path './.git/*' \ + -not -path './result*' \ + -exec deadnix --fail {} + + ''; + }; + + statix = mkCheck { + name = "statix"; + buildInputs = [ pkgs.statix ]; + script = "statix check ."; + }; + + nil = mkCheck { + name = "nil"; + buildInputs = [ + pkgs.nil + pkgs.findutils + ]; + script = '' + failed=0 + while IFS= read -r f; do + diag=$(nil diagnostics "$f" 2>&1) || failed=1 + if [ -n "$diag" ]; then + printf '=== %s ===\n%s\n' "$f" "$diag" + failed=1 + fi + done < <(find . -name '*.nix' \ + -not -path './.git/*' \ + -not -path './result*') + if [ "$failed" -ne 0 ]; then + exit 1 + fi + ''; + }; + + # --- Shell --- + shellcheck = mkCheck { + name = "shellcheck"; + buildInputs = [ + pkgs.shellcheck + pkgs.findutils + ]; + script = '' + shopt -s nullglob globstar + mapfile -t shfiles < <(find . \ + -path './.git' -prune -o \ + -path './result*' -prune -o \ + -type f \( -name '*.sh' -o -name '*.bash' \) -print) + if [ ''${#shfiles[@]} -eq 0 ]; then + echo "no shell scripts found, skipping" + exit 0 + fi + shellcheck -x "''${shfiles[@]}" + ''; + }; + + # --- YAML / GitHub Actions --- + yamllint = mkCheck { + name = "yamllint"; + buildInputs = [ + pkgs.yamllint + pkgs.findutils + ]; + script = '' + mapfile -t yfiles < <(find . \ + -path './.git' -prune -o \ + -path './result*' -prune -o \ + -type f \( -name '*.yml' -o -name '*.yaml' \) -print) + if [ ''${#yfiles[@]} -eq 0 ]; then exit 0; fi + yamllint "''${yfiles[@]}" + ''; + }; + + actionlint = mkCheck { + name = "actionlint"; + buildInputs = [ + pkgs.actionlint + pkgs.git + ]; + script = '' + if [ -d .github/workflows ]; then + # actionlint finds workflows + reads .github/actionlint.yaml via the + # git project root; the check sandbox has no .git, so init a throwaway + # repo to give it one. + git init -q . + actionlint + fi + ''; + }; + + # --- Docker --- + hadolint = mkCheck { + name = "hadolint"; + buildInputs = [ + pkgs.hadolint + pkgs.findutils + ]; + script = '' + mapfile -t df < <(find . \ + -path './.git' -prune -o \ + -type f \( -name 'Dockerfile' -o -name 'Dockerfile.*' -o -name '*.Dockerfile' \) -print) + if [ ''${#df[@]} -eq 0 ]; then exit 0; fi + hadolint "''${df[@]}" + ''; + }; + + # --- Markdown --- + markdown = mkCheck { + name = "markdown"; + buildInputs = [ pkgs.markdownlint-cli2 ]; + script = '' + markdownlint-cli2 \ + '**/*.md' \ + '!node_modules/**' \ + '!dist/**' \ + '!build/**' + ''; + }; + + # --- TOML --- + taplo = mkCheck { + name = "taplo"; + buildInputs = [ + pkgs.taplo + pkgs.findutils + ]; + script = '' + mapfile -t tf < <(find . \ + -path './.git' -prune -o \ + -type f -name '*.toml' -print) + if [ ''${#tf[@]} -eq 0 ]; then exit 0; fi + taplo check "''${tf[@]}" + ''; + }; + + # --- Hygiene --- + editorconfig-checker = mkCheck { + name = "editorconfig-checker"; + buildInputs = [ pkgs.editorconfig-checker ]; + script = "editorconfig-checker"; + }; + + # --- Secrets (no-git working-tree scan; sandbox has no .git) --- + gitleaks = mkCheck { + name = "gitleaks"; + buildInputs = [ pkgs.gitleaks ]; + script = "gitleaks detect --no-git --source=. --no-banner --redact"; + }; +} diff --git a/nix/devshell/default.nix b/nix/devshell/default.nix new file mode 100644 index 00000000..7a050185 --- /dev/null +++ b/nix/devshell/default.nix @@ -0,0 +1,34 @@ +# Development shell. Contains every tool the lint apps need so a +# contributor with `nix develop` can run any `nix run .#lint-*` +# without further setup. + +{ pkgs, repoLib }: + +let + packages = import ./packages.nix { inherit pkgs; }; +in +pkgs.mkShell { + inherit packages; + meta.description = "runpod/containers static-analysis toolchain (Python, shell, Docker, YAML/Actions, JSON/TOML, Markdown, Nix, secrets, SAST, supply chain)."; + + shellHook = '' + echo " ${repoLib.repoName} — static-analysis shell" + echo + echo " Composites:" + echo " nix run .#lint-shell # shellcheck, shfmt, bashate, checkbashisms" + echo " nix run .#lint-yaml # yamllint, actionlint" + echo " nix run .#lint-json # jsonlint, biome, taplo" + echo " nix run .#lint-py # ruff, pyright, mypy, bandit, pylint, vulture" + echo " nix run .#lint-nix-files # nixfmt, statix, deadnix, nil" + echo " nix run .#lint-docker # hadolint" + echo " nix run .#lint-md # markdownlint, mdformat, vale, lychee" + echo " nix run .#lint-secrets # gitleaks, trufflehog, detect-secrets" + echo " nix run .#lint-sast # semgrep, bandit" + echo " nix run .#lint-supply # osv-scanner, trivy, syft" + echo " nix run .#lint-spelling # cspell, typos, codespell" + echo " nix run .#lint-extreme # every lint sequentially" + echo " nix run .#everything # nix flake check + lint-extreme" + echo " nix flake check # gated subset (sandbox-safe only)" + echo + ''; +} diff --git a/nix/devshell/packages.nix b/nix/devshell/packages.nix new file mode 100644 index 00000000..86a88911 --- /dev/null +++ b/nix/devshell/packages.nix @@ -0,0 +1,81 @@ +# Union of all tool binaries used by `nix run .#lint-*` apps and the +# gated `nix flake check` derivations. A contributor with `nix develop` +# can run any tool directly. +# +# A few tools are not packaged in every nixpkgs pin; `attr or null` +# keeps the shell evaluating and we filter the nulls out below. Their +# lint apps degrade to a visible skip via ../lib/maybe-tool.nix. + +{ pkgs }: + +let + optional = builtins.filter (p: p != null) [ + (pkgs.bashate or null) + (pkgs.checkbashisms or null) + (pkgs.vulture or pkgs.python3Packages.vulture or null) + (pkgs.python3Packages.mdformat or null) + (pkgs.vale or null) + (pkgs.dotenv-linter or null) + (pkgs.detect-secrets or pkgs.python3Packages.detect-secrets or null) + (pkgs.cspell or null) + ]; +in +optional +++ (with pkgs; [ + # Core + git + bash + gnugrep + coreutils + findutils + python3 + + # Shell + shellcheck + shfmt + + # YAML / Actions + yamllint + actionlint + + # JSON / TOML + biome + taplo + + # Python + ruff + pyright + mypy + bandit + pylint + + # Nix + nixfmt-rfc-style + statix + deadnix + nil + + # Docker + hadolint + + # Markdown / prose + markdownlint-cli2 + lychee + + # Hygiene + editorconfig-checker + + # Secrets + gitleaks + trufflehog + + # SAST / vuln / supply chain + semgrep + osv-scanner + trivy + syft + + # Spelling + typos + codespell +]) diff --git a/nix/images/README.md b/nix/images/README.md new file mode 100644 index 00000000..31999341 --- /dev/null +++ b/nix/images/README.md @@ -0,0 +1,245 @@ +# Nix-built container images (experimental, build-only) + +A proof-of-concept for building RunPod's container images with Nix +(`pkgs.dockerTools`) instead of Dockerfiles, to measure how much smaller and +more deterministic the result is. **Nothing here is published** — this is a +build + measure experiment only. + +## TL;DR — measured on the CPU base + +| image | compressed (download) | layers | +| --- | --- | --- | +| `runpod/base:1.0.7-ubuntu2404` (Dockerfile) | **714 MB** | 20 | +| `nix base-cpu` (this PoC, Python 3.14) | **295 MB** | ~98 | + +≈ **59% smaller** compressed download. Numbers are reproduced on every CI run +by [`compare-size.sh`](./compare-size.sh) (job `image-size` in +`.github/workflows/nix.yml`). + +**Fair-comparison caveat:** the published base bundles Python 3.9–3.13 plus a +large apt userland; this PoC ships a single Python (3.14) and a curated tool +set. Some of the win is "fewer Pythons," but the determinism, layer dedup, and +no-package-manager-in-the-image properties are structural and hold regardless. + +## Hybrid CUDA image (`base-cuda.nix`) + +The images people actually run are CUDA/PyTorch. Rather than rebuild CUDA in +Nix (unfree, huge, CI-fragile), the **hybrid** pins NVIDIA's tested CUDA base as +`fromImage` and layers the same Nix userland on top. Directly comparable to +`runpod/base:1.0.7-cuda1281-ubuntu2404`, which is the *same* nvidia base + an +apt userland layer. Torch is pip-installed identically on top of both, so it's +excluded — this compares the **userland layer**. + +| image | compressed | note | +| --- | --- | --- | +| `nvidia/cuda:12.8.1-cudnn-devel-ubuntu24.04` | 5593 MB | shared base | +| **hybrid base-cuda** (nvidia + Nix userland) | **5846 MB** | Nix layer ≈ **253 MB** | +| `runpod/base:1.0.7-cuda1281-ubuntu2404` (nvidia + apt) | 6359 MB | apt layer ≈ 766 MB | +| `runpod/pytorch:1.0.7-cu1281-torch280` (base + torch) | 11211 MB | torch ≈ +4.9 GB | + +The Nix userland layer is **~67% smaller** than the apt layer (253 vs 766 MB). +But the shared CUDA base and torch wheels dominate, so the **whole-image** win +is modest (~8% on the base). The big further levers: a **runtime** (non-devel) +base, and a **Nix-built torch**. Reproduced by +[`compare-cuda-size.sh`](./compare-cuda-size.sh) (manual-dispatch CI job +`cuda-image-size` — heavy, so not run on every push). Base pinned by digest + +Nix hash via `nix-prefetch-docker`. + +## Maximizing shared layers across an image family + +These images co-deploy on the same hosts, so cross-image Docker layer reuse is +where the real fleet-level win is. Nix helps here in a way Dockerfiles cannot: + +- **Automatic:** `dockerTools` layers are keyed by **store path**; an identical + store path produces a byte-identical layer with an identical digest. So any + two Nix images that share dependencies already share those layers in a host's + Docker cache — pulled once, reused everywhere. `base-cpu` and `base-cuda` both + `import ./userland.nix`, so their userland layers are literally the same + digests today. +- **The subtlety:** `streamLayeredImage` assigns layers *per image* via a + popularity heuristic capped at `maxLayers` (default 100); everything past the + cap collapses into one "customisation layer." Two images with different + closures can put a shared path in a dedicated (shared) layer in one and fold + it into the customisation bundle of the other — so sharing is good but not + guaranteed-maximal with naive independent builds. + +### Prototyped here: a shared-userland family + footprint metric + +`family.nix` defines a small family that all reuse the exact same userland store +paths and only add their own Python stack: + +- `family-base` — plain base userland +- `family-data` — + numpy/pandas/scikit-learn/matplotlib/scipy +- `family-serve` — + fastapi/uvicorn/pydantic/pillow/requests + +`footprint.sh` (CI job `family-footprint`) measures the **real OCI layers** each +image emits (streamed docker-archive → `skopeo inspect` → dedupe by blob +digest) — what a host's Docker cache actually shares: + +| | real OCI layers | +| --- | --- | +| family-base | 99 layers, 921 MB | +| family-data | 99 layers, 1623 MB | +| family-serve | 99 layers, 1010 MB | +| **naïve sum** (no sharing) | **3555 MB** | +| **unique cached on a host** (deduped by digest) | **2312 MB** | +| **saved by sharing** | **1243 MB — 34%** | + +**Important:** this is *real OCI* sharing, not Nix store-path sharing. +`streamLayeredImage` gives only the top `maxLayers` (default 100) store paths a +dedicated layer and bundles the rest into a single per-image *customisation +layer* whose digest differs between images. So identical store paths do **not** +all become shared layers — the store-path closure (which would show ~53%) is +only a ceiling. Raising `maxLayers` recovers it, at the cost of layer count: + +| builder | `maxLayers` | real OCI shared | layers / image | +| --- | --- | --- | --- | +| dockerTools | 100 (default) | **34%** | 99 | +| dockerTools | 200 | 38% | 199 | +| dockerTools | 400 | 53% | 275 | +| nix2container | 100 | **16%** | 100 | +| nix2container | 300 | 48% | 283 | +| nix2container | 600 | 53% | 288 | + +Two findings, both measured (`family-*` = dockerTools, `family-*-n2c` = +nix2container; run by `footprint.sh` / the `family-footprint` CI job): + +1. **The sharing ceiling is set by the closure, not the tool.** Both builders + converge at ~53% — the store-path ceiling — only when `maxLayers` ≥ the + closure size (~275) so every path is its own layer. Neither exceeds it. +2. **nix2container does *not* raise the ceiling, and at its default (100) it + shares *less* (16%)** because it bundles the non-dedicated closure into + fewer, larger per-image layers. Its real advantage is that many layers are + *cheap to build* — it references store paths lazily instead of writing a tar + per layer — so `maxLayers=600` is practical to build, whereas dockerTools at + 400 must materialise 275 layer tarballs. + +**The catch either way:** hitting 53% needs ~275+ OCI layers/image, which +overlay2 and some registries discourage. So per-path auto-layering is not the +real answer — deliberate tiering is. + +### Tiering wins: near-ceiling sharing at a low layer count + +`family-tiered.nix` (`family-*-tiered`) puts the **whole shared userland into +one explicit nix2container `buildLayer`** that all three images reuse +byte-for-byte, with only a small per-flavor delta on top: + +| approach | real OCI shared | layers / image | +| --- | --- | --- | +| dockerTools (auto, default) | 34% | 99 | +| nix2container (auto, maxLayers 600) | 53% | 288 | +| **nix2container (tiered `buildLayer`)** | **51%** | **~15–21** | + +Tiering reaches ~the store-path ceiling (51% vs 53%) at **~19 layers/image +instead of ~288** — the shared userland (~900 MB) is one blob cached once, and +`family-data` / `family-serve` add only their extra Python stack. This is the +practical way to get fleet-wide sharing, and it extends naturally to the GPU +tiers (`common userland` → `+CUDA` → `+torch`). + +### The published fleet today (all container types) + +`fleet-footprint.sh` (manual-dispatch CI job `fleet-footprint`) measures the +*current* published images across every type — base (cpu/cuda/rocm), the full +pytorch matrix, autoresearch, nvidia-pytorch — via `skopeo` manifests (no +pulls). It answers: how much does the existing Dockerfile fleet already share? + +| container type | images | naïve sum | +| --- | --- | --- | +| runpod/base | 10 | 111.0 GB | +| runpod/pytorch | 25 | 306.5 GB | +| runpod/autoresearch | 2 | 22.1 GB | +| runpod/nvidia-pytorch | 1 | 9.1 GB | + +- **Fleet:** 38 images +- **Naïve sum:** 448.7 GB +- **Unique cached on a host** (deduped by blob digest): **258.6 GB** +- **Already shared:** 190.1 GB — **42%** of the naïve total + +So the Dockerfile fleet already dedupes ~42% via its shared `FROM` chain +(`nvidia/cuda` → `runpod/base` → `runpod/pytorch`). That reframes the Nix +opportunity honestly: the big base/CUDA layers are *already* shared, so Nix's +incremental win is the **userland tier** (smaller + deterministic, ~67% smaller +than the apt layer) plus **finer, guaranteed store-path sharing** and +reproducibility — not the CUDA base, which is shared either way. The largest +absolute lever remains the torch wheels (~4.9 GB/image, 25 pytorch images), a +Nix-built-torch question for later. + +### Pushing overlap further + +The winning approach is **deliberate tiering**, not cranking `maxLayers`: put +the whole shared userland into *one* explicit layer every image reuses, so a +host caches it once — high sharing at a *low* layer count. + +1. **Explicit shared layers (`nix2container` `buildLayer`).** Define one shared + userland layer (the common store paths) and give each image only a small + per-flavor delta layer on top. This beats per-path auto-layering: near-ceiling + sharing with a handful of layers instead of ~275. nix2container's `buildLayer` + is built for exactly this; it's the natural next step for this family. +2. **Tiered shared bases via `fromImage`.** Chain `common userland` → `+CUDA + (per cuda version)` → `+torch (per combo)` so every variant reuses the exact + lower-tier layer digests. (The CUDA hybrid already does the first hop off the + nvidia base.) +3. **Footprint as a CI guardrail:** track unique-vs-naïve over time so a change + that accidentally breaks sharing (e.g. bumping one variant's nixpkgs pin) + shows up as a regression. + +## How it works + +- **`base-cpu.nix`** — `pkgs.dockerTools.streamLayeredImage` assembling a + Python 3.14 env (JupyterLab, notebook, ipywidgets, hf_transfer), `uv`, + filebrowser, nginx, openssh and the common CLI tooling, plus the repo's + `container-template/start.sh` and banner. `config.Cmd = ["/start.sh"]`. +- **`default.nix`** — registers the images; wired into the flake as + `packages..base-cpu` (Linux only). `nix flake check` does **not** + build it, so the fast gated-lint job is unaffected. +- **`compare-size.sh`** — builds the stream script, measures uncompressed + + `gzip -9` size, verifies the stream is byte-identical across two runs + (determinism), and fetches the published baseline's compressed size via + `skopeo inspect --raw`. + +Build and measure locally: + +```sh +nix build .#packages.x86_64-linux.base-cpu -o result-image +./result-image | gzip -9 | wc -c # compressed size +bash nix/images/compare-size.sh # full comparison table +``` + +## Why Nix images (the structural wins) + +- **Deterministic:** `streamLayeredImage` zeroes timestamps and derives content + from the pinned `flake.lock`. Two builds → byte-identical tar (CI asserts + this). No `apt-get update` drift, no "works on the build host" surprises. +- **Smaller:** only the exact runtime closure ships — no apt caches, no + `build-essential`/`-dev` headers, no compilers, no package manager. Nothing + is "installed then deleted in another layer." +- **Better layering / caching:** one store path per layer, content-addressed, + so unrelated images that share (say) glibc or Python reuse the same layers on + a host — big win across a family of images. +- **Auditable supply chain:** the full dependency graph is the Nix closure; + `nix path-info -rS` gives an exact SBOM, and everything is pinned. + +## Tradeoffs / open questions + +- **Layer count:** ~98 fine-grained layers vs 20. Great for dedup, but some + registries/pull paths prefer fewer layers — `maxLayers` is tunable. +- **`start.sh` assumes Debian:** it calls `service nginx/ssh start`. A Nix image + has no sysvinit; a small entrypoint shim (or s6/supervisord) is needed for a + runnable image. Out of scope for this size PoC. +- **Multi-Python parity:** adding 3.9–3.13 would grow the image (est. + +150–250 MB) but should still beat the Dockerfile build. + +## Path to GPU parity (not done here) + +The images people actually run are CUDA/ROCm PyTorch. Options, in rough order of +effort: + +1. **Hybrid** — keep the vendor CUDA/ROCm base image as `fromImage` and layer + the Nix userland on top. Lowest risk, keeps NVIDIA's tested CUDA stack, + still gets deterministic userland layers. +2. **Full Nix** — `python3Packages.torch` with `cudaSupport = true` from + `cudaPackages`. Fully reproducible but unfree, very large closures, and a + heavier CI build; needs validation against the current images. + +Recommended next step: prototype option 1 for one CUDA PyTorch tag and re-run +`compare-size.sh` against it. diff --git a/nix/images/base-cpu.nix b/nix/images/base-cpu.nix new file mode 100644 index 00000000..4c6d1253 --- /dev/null +++ b/nix/images/base-cpu.nix @@ -0,0 +1,27 @@ +# Proof-of-concept: build the runpod/base CPU image with Nix instead of a +# Dockerfile, for a build-only size/determinism comparison. NOT published. +# +# Faithful subset of official-templates/base (CPU variant); see ./userland.nix +# for the package set. GPU/CUDA parity lives in ./base-cuda.nix. +# +# Built with dockerTools.streamLayeredImage: deterministic (no timestamps / +# build-host entropy), daemon-less, and automatically split into content- +# addressed layers so shared store paths dedupe across images. + +{ pkgs }: + +let + userland = import ./userland.nix { inherit pkgs; }; +in +pkgs.dockerTools.streamLayeredImage { + name = "runpod-base-cpu-nix"; + tag = "poc"; + + inherit (userland) contents; + + config = { + Cmd = [ "/start.sh" ]; + WorkingDir = "/"; + Env = userland.baseEnv ++ [ "PATH=/bin:/usr/bin" ]; + }; +} diff --git a/nix/images/base-cuda.nix b/nix/images/base-cuda.nix new file mode 100644 index 00000000..214a5515 --- /dev/null +++ b/nix/images/base-cuda.nix @@ -0,0 +1,48 @@ +# Proof-of-concept: HYBRID CUDA image — pin NVIDIA's tested CUDA base as +# `fromImage` and layer the deterministic Nix userland on top. Build-only, +# NOT published. +# +# Directly comparable to runpod/base:-cuda1281-ubuntu2404, which is the +# same nvidia/cuda devel base + an apt/5-python/jupyter userland layer. Torch +# is pip-installed on top of BOTH identically (see the pytorch template), so +# excluding it here keeps the comparison of the *userland layer* fair. +# +# The base is pinned by digest + Nix hash (from `nix-prefetch-docker`); bump +# both together when the CUDA version in shared/versions.hcl changes: +# nix run nixpkgs#nix-prefetch-docker -- \ +# --image-name nvidia/cuda --image-tag --arch amd64 --os linux + +{ pkgs }: + +let + userland = import ./userland.nix { inherit pkgs; }; + + cudaBase = pkgs.dockerTools.pullImage { + imageName = "nvidia/cuda"; + imageDigest = "sha256:24c8e3581ea6330038b0d374920721983312627f8adbfcf390bdb4b399d280ed"; + hash = "sha256-SY6Xgmrn9K2Owl0Hel7uM+6KO12Uua5hKqVCLt9gjKw="; + finalImageName = "nvidia/cuda"; + finalImageTag = "12.8.1-cudnn-devel-ubuntu24.04"; + }; +in +pkgs.dockerTools.streamLayeredImage { + name = "runpod-base-cuda-nix"; + tag = "poc"; + + fromImage = cudaBase; + inherit (userland) contents; + + config = { + Cmd = [ "/start.sh" ]; + WorkingDir = "/"; + # NOTE: streamLayeredImage does not inherit the base image's env, so we + # re-declare the CUDA paths a runnable image needs. Env does not affect the + # size measurement; a production hybrid would merge the base's full env. + Env = userland.baseEnv ++ [ + "PATH=/bin:/usr/bin:/usr/local/nvidia/bin:/usr/local/cuda/bin" + "LD_LIBRARY_PATH=/usr/local/nvidia/lib:/usr/local/nvidia/lib64" + "NVIDIA_VISIBLE_DEVICES=all" + "NVIDIA_DRIVER_CAPABILITIES=compute,utility" + ]; + }; +} diff --git a/nix/images/compare-cuda-size.sh b/nix/images/compare-cuda-size.sh new file mode 100644 index 00000000..7f167628 --- /dev/null +++ b/nix/images/compare-cuda-size.sh @@ -0,0 +1,72 @@ +#!/usr/bin/env bash +# Build-only comparison for the HYBRID CUDA image (nvidia/cuda base + Nix +# userland) vs the published runpod CUDA chain. No image is pushed. +# +# Usage: nix/images/compare-cuda-size.sh +# Needs on PATH: nix, gzip, wc; skopeo + jq reachable via `nix run nixpkgs#...`. +set -euo pipefail + +SYSTEM="x86_64-linux" +NVIDIA_REF="docker://docker.io/nvidia/cuda:12.8.1-cudnn-devel-ubuntu24.04" +BASE_REF="docker://docker.io/runpod/base:1.0.7-cuda1281-ubuntu2404" +PYTORCH_REF="docker://docker.io/runpod/pytorch:1.0.7-cu1281-torch280-ubuntu2404" + +nixf() { nix --extra-experimental-features 'nix-command flakes' "$@"; } +mb() { echo $(($1 / 1048576)); } + +# Compressed download size (amd64) of a published image, via skopeo (no pull). +compressed_mb() { + local ref="$1" raw + export CONTAINERS_REGISTRIES_CONF="${CONTAINERS_REGISTRIES_CONF:-/dev/null}" + if raw=$(nixf run nixpkgs#skopeo -- inspect --override-os linux --override-arch amd64 "$ref" 2>/dev/null); then + printf '%s' "$raw" | nixf run nixpkgs#jq -- -r '[.LayersData[].Size] | add' 2>/dev/null || echo "" + else + echo "" + fi +} + +echo "==> building hybrid base-cuda image (pulls pinned nvidia base; build-only)" +nixf build ".#packages.${SYSTEM}.base-cuda" -o result-base-cuda-image + +echo "==> measuring hybrid image (streaming ~6 GB, please wait)" +# gzip -6 (registry-default-ish) — -9 on an ~11 GB stream is far too slow. +raw=$(./result-base-cuda-image | wc -c) +gz=$(./result-base-cuda-image | gzip -6 | wc -c) + +echo "==> fetching published baseline sizes (skopeo, no pull)" +nvidia_c=$(compressed_mb "$NVIDIA_REF") +base_c=$(compressed_mb "$BASE_REF") +pt_c=$(compressed_mb "$PYTORCH_REF") + +emit() { + echo "$1" + if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then + echo "$1" >>"$GITHUB_STEP_SUMMARY" + fi +} + +emit "" +emit "## Hybrid CUDA image — build-only size comparison" +emit "" +emit "Hybrid = pinned \`nvidia/cuda:12.8.1-cudnn-devel-ubuntu24.04\` (fromImage) + the Nix userland. Torch is pip-installed identically on top of both the hybrid and runpod/base, so it is excluded here — this compares the **userland layer**." +emit "" +emit "| image | compressed (download) | uncompressed |" +emit "| --- | --- | --- |" +emit "| **hybrid base-cuda** (nvidia base + Nix userland) | $(mb "$gz") MB | $(mb "$raw") MB |" +[ -n "$base_c" ] && emit "| runpod/base:1.0.7-cuda1281-ubuntu2404 (nvidia base + apt userland) | $(mb "$base_c") MB | — |" +[ -n "$nvidia_c" ] && emit "| nvidia/cuda:12.8.1-cudnn-devel-ubuntu24.04 (shared base) | $(mb "$nvidia_c") MB | — |" +[ -n "$pt_c" ] && emit "| runpod/pytorch:1.0.7-cu1281-torch280 (base + torch wheels) | $(mb "$pt_c") MB | — |" +emit "" +if [ -n "$base_c" ] && [ -n "$nvidia_c" ]; then + nix_layer=$((gz - nvidia_c)) + apt_layer=$((base_c - nvidia_c)) + emit "**Userland layer added on top of the shared CUDA base:**" + emit "- Nix: ~$(mb "$nix_layer") MB" + emit "- apt (5 pythons + build tooling): ~$(mb "$apt_layer") MB" + if [ "$apt_layer" -gt 0 ]; then + emit "" + emit "The Nix userland layer is **$((100 - nix_layer * 100 / apt_layer))% smaller** than the apt layer, cutting the base image from $(mb "$base_c") MB to $(mb "$gz") MB. The shared CUDA base ($(mb "$nvidia_c") MB) and torch wheels dominate the total, so the whole-image win is modest — the big further levers are a runtime (non-devel) base and a Nix-built torch." + fi +fi +emit "" +emit "_Determinism holds as in the CPU image (streamLayeredImage zeroes timestamps); base pinned by digest + Nix hash. See nix/images/README.md._" diff --git a/nix/images/compare-size.sh b/nix/images/compare-size.sh new file mode 100644 index 00000000..1c24f367 --- /dev/null +++ b/nix/images/compare-size.sh @@ -0,0 +1,65 @@ +#!/usr/bin/env bash +# Build-only comparison: the Nix-built base-cpu image vs the published +# runpod/base CPU image. Measures size + determinism and prints a table. +# No image is pushed anywhere. +# +# Usage: nix/images/compare-size.sh [baseline-tag] (default 1.0.7-ubuntu2404) +# Needs on PATH: nix, gzip, sha256sum, wc; and (for the baseline) skopeo + jq +# reachable via `nix run nixpkgs#...`. +set -euo pipefail + +BASELINE_TAG="${1:-1.0.7-ubuntu2404}" +BASELINE_REF="docker://docker.io/runpod/base:${BASELINE_TAG}" +SYSTEM="x86_64-linux" + +nixf() { nix --extra-experimental-features 'nix-command flakes' "$@"; } +mb() { echo $(($1 / 1048576)); } + +echo "==> building Nix base-cpu image (stream script, build-only)" +nixf build ".#packages.${SYSTEM}.base-cpu" -o result-base-cpu-image + +echo "==> measuring Nix image" +raw=$(./result-base-cpu-image | wc -c) +gz=$(./result-base-cpu-image | gzip -9 | wc -c) + +echo "==> determinism check: stream twice, compare sha256" +h1=$(./result-base-cpu-image | sha256sum | cut -d' ' -f1) +h2=$(./result-base-cpu-image | sha256sum | cut -d' ' -f1) +if [ "$h1" = "$h2" ]; then + det="identical (${h1:0:16}…)" +else + det="DIFFERENT ($h1 vs $h2)" +fi + +echo "==> baseline (compressed layer sizes via skopeo): $BASELINE_REF" +# Override any local (possibly v1) registries.conf so skopeo parses cleanly. +export CONTAINERS_REGISTRIES_CONF="${CONTAINERS_REGISTRIES_CONF:-/dev/null}" +base_c="" +if raw_manifest=$(nixf run nixpkgs#skopeo -- inspect --raw "$BASELINE_REF" 2>/dev/null); then + base_c=$(printf '%s' "$raw_manifest" | nixf run nixpkgs#jq -- -r '[.layers[].size] | add' 2>/dev/null || echo "") +fi + +emit() { + echo "$1" + if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then + echo "$1" >>"$GITHUB_STEP_SUMMARY" + fi +} + +emit "" +emit "## Nix base-cpu image — build-only size comparison" +emit "" +emit "| image | compressed (download) | uncompressed |" +emit "| --- | --- | --- |" +emit "| **nix base-cpu** (py3.14, single python) | $(mb "$gz") MB | $(mb "$raw") MB |" +if [ -n "$base_c" ]; then + emit "| runpod/base:${BASELINE_TAG} (5 pythons) | $(mb "$base_c") MB | — |" + emit "" + emit "Compressed download is **$((100 - gz * 100 / base_c))% smaller** ($(mb "$gz") MB vs $(mb "$base_c") MB)." +else + emit "| runpod/base:${BASELINE_TAG} | (manifest unavailable) | — |" +fi +emit "" +emit "Determinism (stream twice → sha256): **${det}**" +emit "" +emit "_Caveat: the published base ships Python 3.9–3.13 + a full apt userland; this PoC ships one Python (3.14). See nix/images/README.md._" diff --git a/nix/images/default.nix b/nix/images/default.nix new file mode 100644 index 00000000..5d9adabf --- /dev/null +++ b/nix/images/default.nix @@ -0,0 +1,22 @@ +# OCI images built with Nix (build-only, experimental — not published). +# Exposed as flake `packages` on Linux systems. +# +# `n2c` (the nix2container builder) is optional; when provided, the *-n2c +# variants are added alongside the dockerTools ones for comparison. + +{ + pkgs, + n2c ? null, +}: + +{ + base-cpu = import ./base-cpu.nix { inherit pkgs; }; + base-cuda = import ./base-cuda.nix { inherit pkgs; }; +} +// import ./family.nix { inherit pkgs; } +// ( + if n2c != null then + import ./family-n2c.nix { inherit pkgs n2c; } // import ./family-tiered.nix { inherit pkgs n2c; } + else + { } +) diff --git a/nix/images/family-n2c.nix b/nix/images/family-n2c.nix new file mode 100644 index 00000000..3b5c2400 --- /dev/null +++ b/nix/images/family-n2c.nix @@ -0,0 +1,74 @@ +# Same image family as family.nix, but built with nix2container instead of +# dockerTools.streamLayeredImage — kept as separate `*-n2c` targets so the two +# layering strategies can be compared head-to-head (see footprint.sh). +# +# nix2container lays out one layer per store path (up to maxLayers) without +# dockerTools' single "customisation layer" collapse, and references store +# paths lazily — so it can carry many layers cheaply and share them maximally +# across images. This is the candidate for closing the 34% -> 53% gap. + +{ pkgs, n2c }: + +let + userland = import ./userland.nix { inherit pkgs; }; + + # nix2container's copyToRoot strictly merges the paths into one root and + # errors on collisions (e.g. gawk's /include dir vs the python env's /include + # symlink). Pre-merge with buildEnv, which tolerates them (first-wins), to + # match how dockerTools assembled the same contents. + mkRoot = + name: contents: + pkgs.buildEnv { + name = "runpod-${name}-n2c-root"; + paths = contents; + ignoreCollisions = true; + }; + + mkImage = + name: contents: + n2c.buildImage { + name = "runpod-${name}-n2c"; + tag = "poc"; + copyToRoot = mkRoot name contents; + # High on purpose: nix2container references store paths lazily, so many + # layers are cheap to build (unlike dockerTools, which tars each). At 600 + # every store path gets its own layer, hitting the closure's store-path + # sharing ceiling (~53% here). At the default (100) it actually shares + # *less* than dockerTools — see footprint.sh / README. + maxLayers = 600; + config = { + Cmd = [ "/start.sh" ]; + WorkingDir = "/"; + Env = userland.baseEnv ++ [ "PATH=/bin:/usr/bin" ]; + }; + }; +in +{ + family-base-n2c = mkImage "family-base" (userland.mkContents { }); + + family-data-n2c = mkImage "family-data" ( + userland.mkContents { + extraPyPackages = + ps: with ps; [ + numpy + pandas + scikit-learn + matplotlib + scipy + ]; + } + ); + + family-serve-n2c = mkImage "family-serve" ( + userland.mkContents { + extraPyPackages = + ps: with ps; [ + fastapi + uvicorn + pydantic + pillow + requests + ]; + } + ); +} diff --git a/nix/images/family-tiered.nix b/nix/images/family-tiered.nix new file mode 100644 index 00000000..ee4a2da8 --- /dev/null +++ b/nix/images/family-tiered.nix @@ -0,0 +1,74 @@ +# Explicit-tiering family (nix2container): instead of relying on per-store-path +# auto-layering (which needs ~275 layers/image to hit the sharing ceiling), put +# the ENTIRE shared userland into one explicit `buildLayer` that every image +# reuses byte-for-byte, and give each flavor only a small delta on top. +# +# Goal: near-ceiling sharing at a LOW layer count. All three images reference +# the same `sharedLayer` derivation => identical digest => one cached blob; +# store paths already in it are not duplicated in a flavor's delta. +# +# Kept as separate `*-tiered` targets so the dockerTools and auto-layered n2c +# families remain for comparison (see footprint.sh). + +{ pkgs, n2c }: + +let + userland = import ./userland.nix { inherit pkgs; }; + + baseContents = userland.mkContents { }; + + # One shared layer holding the whole base-userland closure (tools + base + # python env). buildLayer defaults to a single layer; every image below + # includes this same derivation, so it dedupes to one blob across the family. + sharedLayer = n2c.buildLayer { deps = baseContents; }; + + mkImage = + name: contents: + n2c.buildImage { + name = "runpod-${name}-tiered"; + tag = "poc"; + layers = [ sharedLayer ]; + copyToRoot = pkgs.buildEnv { + name = "runpod-${name}-tiered-root"; + paths = contents; + ignoreCollisions = true; + }; + # Small delta budget: paths already in sharedLayer are not re-added, so + # only a flavor's extra packages land here. + maxLayers = 20; + config = { + Cmd = [ "/start.sh" ]; + WorkingDir = "/"; + Env = userland.baseEnv ++ [ "PATH=/bin:/usr/bin" ]; + }; + }; +in +{ + family-base-tiered = mkImage "family-base" baseContents; + + family-data-tiered = mkImage "family-data" ( + userland.mkContents { + extraPyPackages = + ps: with ps; [ + numpy + pandas + scikit-learn + matplotlib + scipy + ]; + } + ); + + family-serve-tiered = mkImage "family-serve" ( + userland.mkContents { + extraPyPackages = + ps: with ps; [ + fastapi + uvicorn + pydantic + pillow + requests + ]; + } + ); +} diff --git a/nix/images/family.nix b/nix/images/family.nix new file mode 100644 index 00000000..f1903fe7 --- /dev/null +++ b/nix/images/family.nix @@ -0,0 +1,58 @@ +# A small image *family* that all share the common userland, to demonstrate +# cross-image Docker layer sharing (the fleet-caching win). Each member reuses +# the exact same shared store paths for the userland + tools, and only adds its +# own Python stack on top — so those shared layers dedupe on any host that runs +# more than one of them. Measured by ./footprint.sh. +# +# Realistic RunPod-ish flavors, all CPU (so the family builds without GPU): +# family-base — the plain base userland +# family-data — + a data-science stack (numpy/pandas/scikit-learn/…) +# family-serve — + an inference-serving stack (fastapi/uvicorn/pillow/…) + +{ pkgs }: + +let + userland = import ./userland.nix { inherit pkgs; }; + + mkImage = + name: contents: + pkgs.dockerTools.streamLayeredImage { + name = "runpod-${name}-nix"; + tag = "poc"; + inherit contents; + config = { + Cmd = [ "/start.sh" ]; + WorkingDir = "/"; + Env = userland.baseEnv ++ [ "PATH=/bin:/usr/bin" ]; + }; + }; +in +{ + family-base = mkImage "family-base" (userland.mkContents { }); + + family-data = mkImage "family-data" ( + userland.mkContents { + extraPyPackages = + ps: with ps; [ + numpy + pandas + scikit-learn + matplotlib + scipy + ]; + } + ); + + family-serve = mkImage "family-serve" ( + userland.mkContents { + extraPyPackages = + ps: with ps; [ + fastapi + uvicorn + pydantic + pillow + requests + ]; + } + ); +} diff --git a/nix/images/fleet-footprint.sh b/nix/images/fleet-footprint.sh new file mode 100644 index 00000000..796dd6d3 --- /dev/null +++ b/nix/images/fleet-footprint.sh @@ -0,0 +1,78 @@ +#!/usr/bin/env bash +# Fleet-wide layer-sharing footprint of the *published* runpod images (all +# container types). Measures how much a host that ran the whole fleet would +# cache once shared Docker layers dedupe (unique blob digests) vs the naïve sum +# of every image — i.e. how much layer sharing the current Dockerfile fleet +# already achieves, and how much is still duplicated. +# +# Read-only: uses `skopeo inspect` (manifests only, no blob pulls). Anonymous +# Docker Hub manifest requests are rate-limited (~100/6h/IP); set +# DOCKERHUB_USERNAME + DOCKERHUB_TOKEN to authenticate and avoid throttling. +# +# Usage: nix/images/fleet-footprint.sh +set -euo pipefail + +REPOS=(base pytorch autoresearch nvidia-pytorch) +VER="1.0.7" +export CONTAINERS_REGISTRIES_CONF="${CONTAINERS_REGISTRIES_CONF:-/dev/null}" + +nixf() { nix --extra-experimental-features 'nix-command flakes' "$@"; } +sk() { nixf run nixpkgs#skopeo -- "$@"; } +jqr() { nixf run nixpkgs#jq -- "$@"; } +gb() { awk -v b="$1" 'BEGIN { printf "%.1f", b / 1073741824 }'; } + +creds=() +if [ -n "${DOCKERHUB_USERNAME:-}" ] && [ -n "${DOCKERHUB_TOKEN:-}" ]; then + creds=(--creds "${DOCKERHUB_USERNAME}:${DOCKERHUB_TOKEN}") +fi + +tmp=$(mktemp -d) +trap 'rm -rf "$tmp"' EXIT +: >"$tmp/layers.tsv" # digest \t size (one line per layer occurrence) +: >"$tmp/images.tsv" # repo \t ref \t imgsize + +emit() { + echo "$1" + if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then + echo "$1" >>"$GITHUB_STEP_SUMMARY" + fi +} + +for repo in "${REPOS[@]}"; do + tags=$(sk "${creds[@]}" list-tags "docker://docker.io/runpod/$repo" \ + | jqr -r '.Tags[]' | grep -E "^${VER}-" | grep -vE -- '-dev-|-rc\.' || true) + for t in $tags; do + ref="docker://docker.io/runpod/$repo:$t" + if ! j=$(sk "${creds[@]}" inspect --override-os linux --override-arch amd64 "$ref" 2>/dev/null); then + echo " skip (inspect failed / rate-limited): runpod/$repo:$t" >&2 + continue + fi + echo "$j" | jqr -r '.LayersData[] | "\(.Digest)\t\(.Size)"' >>"$tmp/layers.tsv" + isize=$(echo "$j" | jqr -r '[.LayersData[].Size] | add') + printf '%s\t%s\t%s\n' "$repo" "runpod/$repo:$t" "$isize" >>"$tmp/images.tsv" + done +done + +n=$(wc -l <"$tmp/images.tsv") +naive=$(awk -F'\t' '{s += $3} END {print s}' "$tmp/images.tsv") +unique=$(sort -u -k1,1 "$tmp/layers.tsv" | awk -F'\t' '{s += $2} END {print s}') +saved=$((naive - unique)) + +emit "" +emit "## Published fleet — current layer-sharing footprint" +emit "" +emit "All \`$VER\` release images across every container type (compressed / download sizes, amd64, via skopeo)." +emit "" +emit "| container type | images | naïve sum |" +emit "| --- | --- | --- |" +for repo in "${REPOS[@]}"; do + ri=$(awk -F'\t' -v r="$repo" '$1 == r {c++; s += $3} END {print c "\t" s}' "$tmp/images.tsv") + emit "| runpod/$repo | $(echo "$ri" | cut -f1) | $(gb "$(echo "$ri" | cut -f2)") GB |" +done +emit "" +emit "- **Fleet images:** $n" +emit "- **Naïve sum** (every image, no sharing): **$(gb "$naive") GB**" +emit "- **Unique cached on a host** (shared layers deduped by digest): **$(gb "$unique") GB**" +emit "- **Already shared by the current images:** $(gb "$saved") GB — $((saved * 100 / naive))% of the naïve total" +emit "" +emit "_The Dockerfile fleet already dedupes its common \`FROM\` layers (nvidia/cuda + runpod/base). Nix's additional lever is the userland tier (smaller + deterministic) and finer, guaranteed store-path sharing — see the family footprint above._" diff --git a/nix/images/footprint.sh b/nix/images/footprint.sh new file mode 100644 index 00000000..3e3df6af --- /dev/null +++ b/nix/images/footprint.sh @@ -0,0 +1,87 @@ +#!/usr/bin/env bash +# Cross-image layer-sharing footprint for the Nix image family, measured on the +# ACTUAL OCI layers each image emits (not the Nix closure), for BOTH builders: +# - dockerTools.streamLayeredImage (family-*) default maxLayers 100 +# - nix2container (family-*-n2c) maxLayers 600 +# +# Why real OCI layers: streamLayeredImage bundles the store paths beyond +# maxLayers into a single per-image customisation layer with a distinct digest, +# so store-path sharing overstates OCI sharing. We stream each image to a +# docker-archive, read real layer digests + sizes via `skopeo inspect`, and +# dedupe by digest across the family — what a host's Docker cache shares. +# +# Usage: nix/images/footprint.sh +set -euo pipefail + +SYSTEM="x86_64-linux" +export CONTAINERS_REGISTRIES_CONF="${CONTAINERS_REGISTRIES_CONF:-/dev/null}" + +nixf() { nix --extra-experimental-features 'nix-command flakes' "$@"; } +sk() { nixf run nixpkgs#skopeo -- "$@"; } +jqr() { nixf run nixpkgs#jq -- "$@"; } +mb() { echo "$(($1 / 1048576))"; } + +tmp=$(mktemp -d) +trap 'rm -rf "$tmp"' EXIT + +emit() { + echo "$1" + if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then + echo "$1" >>"$GITHUB_STEP_SUMMARY" + fi +} + +# Patched skopeo that reads nix2container's `nix:` transport. +skn2c="" +for p in $(nixf build --no-link --print-out-paths ".#packages.${SYSTEM}.skopeo-n2c"); do + [ -x "$p/bin/skopeo" ] && skn2c="$p/bin/skopeo" && break +done + +layers_of() { # $1 = docker-archive path -> "digest\tsize" lines + sk inspect "docker-archive:$1" | jqr -r '.LayersData[] | "\(.Digest)\t\(.Size)"' +} + +# Measure one family. $1 = builder label, $2.. = image attr names. +# Streams each image to a docker-archive, collects real OCI layer digests+sizes. +measure_family() { + local label="$1" + shift + local all="$tmp/$label.all.tsv" + : >"$all" + emit "" + emit "### $label" + emit "" + emit "| image | OCI layers | size |" + emit "| --- | --- | --- |" + for img in "$@"; do + echo "==> $label: $img" >&2 + local out + out=$(nixf build --no-link --print-out-paths ".#packages.${SYSTEM}.$img") + # nix2container images (image.json) need the patched skopeo + nix: transport; + # dockerTools images are a runnable stream script. + case "$img" in + *-n2c | *-tiered) "$skn2c" --insecure-policy copy "nix:$out" "docker-archive:$tmp/img.tar:r:t" >/dev/null 2>&1 ;; + *) "$out" >"$tmp/img.tar" ;; + esac + layers_of "$tmp/img.tar" >"$tmp/$img.layers" + rm -f "$tmp/img.tar" + cat "$tmp/$img.layers" >>"$all" + emit "| \`$img\` | $(wc -l <"$tmp/$img.layers") | $(mb "$(awk -F'\t' '{s+=$2} END{print s}' "$tmp/$img.layers")") MB |" + done + local naive unique saved + naive=$(awk -F'\t' '{s+=$2} END{print s}' "$all") + unique=$(sort -u -k1,1 "$all" | awk -F'\t' '{s+=$2} END{print s}') + saved=$((naive - unique)) + emit "" + emit "- naïve sum: $(mb "$naive") MB · unique cached: $(mb "$unique") MB · **shared: $((saved * 100 / naive))%** ($(mb "$saved") MB)" +} + +emit "" +emit "## Nix image family — real OCI layer-sharing footprint" + +measure_family "dockerTools (auto, maxLayers 100)" family-base family-data family-serve +measure_family "nix2container (auto, maxLayers 600)" family-base-n2c family-data-n2c family-serve-n2c +measure_family "nix2container (tiered buildLayer)" family-base-tiered family-data-tiered family-serve-tiered + +emit "" +emit "_Real OCI layer blobs (\`skopeo inspect docker-archive:\`), deduped by digest. Auto-layering hits the store-path ceiling (~53%) only at ~275+ layers/image. The **tiered** build puts the whole shared userland in one explicit \`buildLayer\` every image reuses, reaching ~the same sharing at ~15-20 layers/image — the practical answer. See README._" diff --git a/nix/images/userland.nix b/nix/images/userland.nix new file mode 100644 index 00000000..8c4438d3 --- /dev/null +++ b/nix/images/userland.nix @@ -0,0 +1,100 @@ +# Shared userland for the PoC images: a faithful subset of what +# official-templates/base installs, expressed as Nix store paths. Consumed by +# base-cpu.nix, base-cuda.nix (hybrid), and family.nix (shared-base family). +# +# `mkContents` lets a family member extend the common userland with extra +# Python packages / tools while reusing the exact same shared store paths — so +# those layers dedupe across every image (see ./footprint.sh, ./README.md). + +{ pkgs }: + +let + python = pkgs.python314; + + # Python packages every image gets. + basePyPackages = + ps: with ps; [ + jupyterlab + notebook + ipywidgets + hf-transfer + ]; + + # Non-Python userland, shared verbatim by every image (parity with the apt + # layer's common tools). + toolContents = [ + pkgs.uv + pkgs.filebrowser + pkgs.nginx + pkgs.openssh + pkgs.bashInteractive + pkgs.coreutils + pkgs.gnused + pkgs.gnugrep + pkgs.gawk + pkgs.findutils + pkgs.gnutar + pkgs.gzip + pkgs.zstd + pkgs.curl + pkgs.wget + pkgs.git + pkgs.cacert + pkgs.jq + pkgs.which + pkgs.openssl + pkgs.tmux + pkgs.vim + pkgs.nano + pkgs.rsync + pkgs.unzip + pkgs.zip + ]; + + # Stage the repo's runtime files (entrypoint + banner) into the image root. + runpodFiles = pkgs.runCommand "runpod-base-files" { } '' + mkdir -p "$out/etc" + cp ${../../container-template/start.sh} "$out/start.sh" + chmod +x "$out/start.sh" + cp ${../../container-template/runpod.txt} "$out/etc/runpod.txt" + ''; + + # Build a contents list = shared tools + (base + extra) Python env + extra + # tools. The shared pieces keep identical store paths across all callers. + mkContents = + { + extraPyPackages ? (_: [ ]), + extraTools ? [ ], + }: + toolContents + ++ extraTools + ++ [ + (python.withPackages (ps: basePyPackages ps ++ extraPyPackages ps)) + runpodFiles + ]; +in +{ + inherit + python + basePyPackages + toolContents + runpodFiles + mkContents + ; + + # Convenience: the plain base userland (no extras). + contents = mkContents { }; + + # Env shared by both images. base-cuda extends PATH/LD_LIBRARY_PATH with the + # CUDA directories from the nvidia base. + baseEnv = [ + "SHELL=/bin/bash" + "PYTHONUNBUFFERED=True" + "LANG=C.UTF-8" + "LC_ALL=C.UTF-8" + "SSL_CERT_FILE=${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt" + "TZ=Etc/UTC" + "RP_WORKSPACE=/workspace" + "HF_HUB_ENABLE_HF_TRANSFER=1" + ]; +} diff --git a/nix/lib.nix b/nix/lib.nix new file mode 100644 index 00000000..85592a7f --- /dev/null +++ b/nix/lib.nix @@ -0,0 +1,14 @@ +# Shared helpers for this repo's static-analysis flake. A slim "lib +# namespace root": specialized builders (`mk-lint.nix`, `mk-all-app.nix`, +# `mk-check.nix`, `maybe-tool.nix`) live alongside as `nix/lib/.nix` +# and are imported by their consumers. + +_: + +let + # Marker used by every lint app's banner. + repoName = "runpod-containers"; +in +{ + inherit repoName; +} diff --git a/nix/lib/maybe-tool.nix b/nix/lib/maybe-tool.nix new file mode 100644 index 00000000..472652b8 --- /dev/null +++ b/nix/lib/maybe-tool.nix @@ -0,0 +1,49 @@ +# `maybe-tool` — guard a lint app behind a tool that may be absent from +# the pinned nixpkgs. +# +# A few tools the flake wants may not be packaged in every nixpkgs +# revision. Referencing `pkgs.` directly aborts evaluation of +# the WHOLE flake (devShell, `nix flake show`, the composites), not just +# that one app. This helper lets a tool file pass `pkgs. or null`: +# when null, it yields a stand-in app that prints a visible "skipped" +# line and exits 0, so everything else keeps working. When present, it +# builds the real app via `build tool`. +# +# Usage (in nix/lints/.nix): +# { pkgs, lib }: +# import ../lib/maybe-tool.nix { inherit pkgs lib; } { +# name = "lint-foo"; +# tool = pkgs.foo or null; +# description = "Run foo against the tree."; +# build = foo: import ../lib/mk-lint.nix { inherit pkgs lib; } { +# name = "lint-foo"; +# inherit description; +# runtimeInputs = [ foo pkgs.git ]; +# command = ''foo .''; +# }; +# } + +{ pkgs, ... }: + +{ + name, + tool, + description, + build, +}: + +if tool != null then + build tool +else + pkgs.writeShellApplication { + inherit name; + runtimeInputs = [ ]; + text = '' + echo "==> ${name}" + echo " ${name}: tool not packaged in the pinned nixpkgs — skipped" + ''; + meta = { + description = "${description} (unavailable in the current nixpkgs pin)"; + mainProgram = name; + }; + } diff --git a/nix/lib/mk-all-app.nix b/nix/lib/mk-all-app.nix new file mode 100644 index 00000000..798c8216 --- /dev/null +++ b/nix/lib/mk-all-app.nix @@ -0,0 +1,49 @@ +# Builder for "fan-out" composite apps that run a list of sub-apps +# sequentially, accumulate pass/fail, and exit non-zero if any failed. +# Continues past failures so users get a full report instead of +# stopping at the first broken sub-app. + +{ pkgs, lib }: + +{ + name, + subs, + description, +}: + +let + runEach = lib.concatMapStringsSep "\n" ( + { binName, drv, ... }: + '' + echo + echo "============================================================" + echo "==> ${binName}" + echo "============================================================" + if ${drv}/bin/${binName}; then + passes=$((passes + 1)) + else + fails=$((fails + 1)) + failed_names="$failed_names ${binName}" + fi + '' + ) subs; +in +pkgs.writeShellApplication { + inherit name; + text = '' + set -uo pipefail + passes=0 + fails=0 + failed_names="" + ${runEach} + echo + echo "============================================================" + echo "Summary: $passes passed, $fails failed" + if [ "$fails" -gt 0 ]; then + echo "Failed:$failed_names" + fi + echo "============================================================" + [ "$fails" -eq 0 ] || exit 1 + ''; + meta.description = description; +} diff --git a/nix/lib/mk-check.nix b/nix/lib/mk-check.nix new file mode 100644 index 00000000..671d2c5f --- /dev/null +++ b/nix/lib/mk-check.nix @@ -0,0 +1,23 @@ +# Shared builder for `nix flake check` gated derivations. Each check +# stages `src` into a writable workdir, runs `script`, and writes +# `$out` on success. Tools that need network access (osv-scanner, +# trivy, lychee, trufflehog) cannot use this helper — expose them as +# `nix run .#lint-*` apps only. + +{ pkgs, src }: + +{ + name, + buildInputs ? [ ], + script, +}: + +pkgs.runCommand name { nativeBuildInputs = buildInputs; } '' + set -euo pipefail + workdir=$(mktemp -d) + cd "$workdir" + cp -r ${src}/. ./ + chmod -R u+w . + ${script} + touch $out +'' diff --git a/nix/lib/mk-lint.nix b/nix/lib/mk-lint.nix new file mode 100644 index 00000000..24039181 --- /dev/null +++ b/nix/lib/mk-lint.nix @@ -0,0 +1,53 @@ +# `mk-lint` — single helper used by every `nix/lints/.nix`. +# +# Two modes, selected by whether `globs` is provided: +# +# - **Tree-walk mode** (`globs` omitted or null): the tool walks the +# working tree itself (e.g. `statix check .`, `typos`, `gitleaks +# detect`). The body just runs `command` from the repo root. +# +# - **File-list mode** (`globs` is a list): the helper builds +# `files=( $(git ls-files ) )`, exits 0 cleanly when the +# list is empty, and the per-tool `command` references +# `"${files[@]}"`. +# +# Inside `command`, escape `$` for shell expansion as `''$` (Nix +# multi-line string convention). + +{ pkgs, lib }: + +{ + name, + description, + runtimeInputs, + header ? name, + globs ? null, + emptyMessage ? "(no matching files tracked)", + command, +}: + +let + cdRoot = ''cd "$(git rev-parse --show-toplevel)"''; + banner = ''echo "==> ${header}"''; + fileList = + if globs == null then + "" + else + '' + mapfile -t files < <(git ls-files ${lib.concatStringsSep " " (map (g: "'${g}'") globs)}) + if [ "''${#files[@]}" -eq 0 ]; then + echo " ${emptyMessage}" + exit 0 + fi + ''; +in +pkgs.writeShellApplication { + inherit name runtimeInputs; + text = '' + ${cdRoot} + ${banner} + ${fileList} + ${command} + ''; + meta.description = description; +} diff --git a/nix/lints/actionlint.nix b/nix/lints/actionlint.nix new file mode 100644 index 00000000..6719f298 --- /dev/null +++ b/nix/lints/actionlint.nix @@ -0,0 +1,20 @@ +# `lint-actionlint` — validate GitHub Actions workflow files. + +{ pkgs, lib }: + +import ../lib/mk-lint.nix { inherit pkgs lib; } { + name = "lint-actionlint"; + description = "Run actionlint against .github/workflows."; + runtimeInputs = [ + pkgs.actionlint + pkgs.git + ]; + header = "actionlint"; + command = '' + if [ -d .github/workflows ]; then + actionlint + else + echo " (no .github/workflows directory)" + fi + ''; +} diff --git a/nix/lints/bandit.nix b/nix/lints/bandit.nix new file mode 100644 index 00000000..c2899d85 --- /dev/null +++ b/nix/lints/bandit.nix @@ -0,0 +1,16 @@ +# `lint-bandit` — Python security linter (SAST). + +{ pkgs, lib }: + +import ../lib/mk-lint.nix { inherit pkgs lib; } { + name = "lint-bandit"; + description = "Run bandit security checks against tracked .py files."; + runtimeInputs = [ + pkgs.bandit + pkgs.git + ]; + header = "bandit"; + globs = [ "*.py" ]; + emptyMessage = "(no .py files tracked)"; + command = ''bandit "''${files[@]}"''; +} diff --git a/nix/lints/bashate.nix b/nix/lints/bashate.nix new file mode 100644 index 00000000..eae62d51 --- /dev/null +++ b/nix/lints/bashate.nix @@ -0,0 +1,27 @@ +# `lint-bashate` — style checks for bash (indentation, quoting, ...). +# Guarded via maybe-tool: not packaged in every nixpkgs pin. + +{ pkgs, lib }: + +import ../lib/maybe-tool.nix { inherit pkgs lib; } { + name = "lint-bashate"; + tool = pkgs.bashate or null; + description = "Run bashate against tracked shell files."; + build = + bashate: + import ../lib/mk-lint.nix { inherit pkgs lib; } { + name = "lint-bashate"; + description = "Run bashate against tracked shell files."; + runtimeInputs = [ + bashate + pkgs.git + ]; + header = "bashate"; + globs = [ + "*.sh" + "*.bash" + ]; + emptyMessage = "(no shell files tracked)"; + command = ''bashate "''${files[@]}"''; + }; +} diff --git a/nix/lints/biome.nix b/nix/lints/biome.nix new file mode 100644 index 00000000..8a7e230b --- /dev/null +++ b/nix/lints/biome.nix @@ -0,0 +1,16 @@ +# `lint-biome` — Biome as a fast JSON linter/format-checker (second +# opinion to jsonlint). Reads `biome.json` if present; otherwise runs +# with defaults over the tree. + +{ pkgs, lib }: + +import ../lib/mk-lint.nix { inherit pkgs lib; } { + name = "lint-biome"; + description = "Run biome check (JSON) against the tree."; + runtimeInputs = [ + pkgs.biome + pkgs.git + ]; + header = "biome check"; + command = "biome check --reporter=summary ."; +} diff --git a/nix/lints/checkbashisms.nix b/nix/lints/checkbashisms.nix new file mode 100644 index 00000000..c736c20b --- /dev/null +++ b/nix/lints/checkbashisms.nix @@ -0,0 +1,27 @@ +# `lint-checkbashisms` — flag bashisms in scripts that claim POSIX sh. +# Guarded via maybe-tool: not packaged in every nixpkgs pin. + +{ pkgs, lib }: + +import ../lib/maybe-tool.nix { inherit pkgs lib; } { + name = "lint-checkbashisms"; + tool = pkgs.checkbashisms or null; + description = "Run checkbashisms against tracked shell files."; + build = + checkbashisms: + import ../lib/mk-lint.nix { inherit pkgs lib; } { + name = "lint-checkbashisms"; + description = "Run checkbashisms against tracked shell files."; + runtimeInputs = [ + checkbashisms + pkgs.git + ]; + header = "checkbashisms"; + globs = [ + "*.sh" + "*.bash" + ]; + emptyMessage = "(no shell files tracked)"; + command = ''checkbashisms "''${files[@]}"''; + }; +} diff --git a/nix/lints/codespell.nix b/nix/lints/codespell.nix new file mode 100644 index 00000000..d8535554 --- /dev/null +++ b/nix/lints/codespell.nix @@ -0,0 +1,15 @@ +# `lint-codespell` — common-misspelling checker. Skips binary and +# generated paths. + +{ pkgs, lib }: + +import ../lib/mk-lint.nix { inherit pkgs lib; } { + name = "lint-codespell"; + description = "Run codespell against the tree."; + runtimeInputs = [ + pkgs.codespell + pkgs.git + ]; + header = "codespell"; + command = "codespell --skip='.git,*.svg,*.lock,node_modules,result,result-*' ."; +} diff --git a/nix/lints/cspell.nix b/nix/lints/cspell.nix new file mode 100644 index 00000000..2f31c55b --- /dev/null +++ b/nix/lints/cspell.nix @@ -0,0 +1,22 @@ +# `lint-cspell` — spell checker. Guarded via maybe-tool; reads +# `cspell.json` if present. + +{ pkgs, lib }: + +import ../lib/maybe-tool.nix { inherit pkgs lib; } { + name = "lint-cspell"; + tool = pkgs.cspell or null; + description = "Run cspell against the tree."; + build = + cspell: + import ../lib/mk-lint.nix { inherit pkgs lib; } { + name = "lint-cspell"; + description = "Run cspell against the tree."; + runtimeInputs = [ + cspell + pkgs.git + ]; + header = "cspell"; + command = ''cspell --no-progress --no-must-find-files "**/*.{md,py,sh,yml,yaml,json}"''; + }; +} diff --git a/nix/lints/deadnix.nix b/nix/lints/deadnix.nix new file mode 100644 index 00000000..1a0a0bad --- /dev/null +++ b/nix/lints/deadnix.nix @@ -0,0 +1,16 @@ +# `lint-deadnix` — find unused Nix bindings. + +{ pkgs, lib }: + +import ../lib/mk-lint.nix { inherit pkgs lib; } { + name = "lint-deadnix"; + description = "Run deadnix --fail against tracked .nix files."; + runtimeInputs = [ + pkgs.deadnix + pkgs.git + ]; + header = "deadnix --fail"; + globs = [ "*.nix" ]; + emptyMessage = "(no .nix files tracked)"; + command = ''deadnix --fail "''${files[@]}"''; +} diff --git a/nix/lints/default.nix b/nix/lints/default.nix new file mode 100644 index 00000000..3986cbe3 --- /dev/null +++ b/nix/lints/default.nix @@ -0,0 +1,198 @@ +# Lint registry. flake.nix imports this once per system; we return +# `apps = { ... }` ready for splicing into the flake outputs. +# +# Two layers of composites: +# - per-domain: lint-shell, lint-yaml, lint-json, lint-py, +# lint-nix-files, lint-docker, lint-md, lint-secrets, +# lint-sast, lint-supply, lint-spelling +# - whole-tree: lint-extreme (every per-tool app sequentially) +# everything (nix flake check + lint-extreme) +# +# Each per-domain composite re-uses the same `mk-all-app` builder so +# the pass/fail tally format stays consistent. + +{ pkgs, lib }: + +let + mkAllApp = import ../lib/mk-all-app.nix { inherit pkgs lib; }; + + perTool = lib.mapAttrs (_n: path: import path { inherit pkgs lib; }) { + # Shell + lint-shellcheck = ./shellcheck.nix; + lint-shfmt = ./shfmt.nix; + lint-bashate = ./bashate.nix; + lint-checkbashisms = ./checkbashisms.nix; + + # YAML / GitHub Actions + lint-yamllint = ./yamllint.nix; + lint-actionlint = ./actionlint.nix; + + # JSON / TOML + lint-jsonlint = ./jsonlint.nix; + lint-biome = ./biome.nix; + lint-taplo = ./taplo.nix; + + # Python + lint-ruff = ./ruff.nix; + lint-pyright = ./pyright.nix; + lint-mypy = ./mypy.nix; + lint-bandit = ./bandit.nix; + lint-pylint = ./pylint.nix; + lint-vulture = ./vulture.nix; + + # Nix + lint-nixfmt = ./nixfmt.nix; + lint-statix = ./statix.nix; + lint-deadnix = ./deadnix.nix; + lint-nil = ./nil.nix; + + # Docker + lint-hadolint = ./hadolint.nix; + + # Markdown / prose + lint-markdown = ./markdown.nix; + lint-mdformat = ./mdformat.nix; + lint-lychee = ./lychee.nix; + lint-vale = ./vale.nix; + + # Hygiene + lint-editorconfig = ./editorconfig-checker.nix; + lint-dotenv = ./dotenv-linter.nix; + + # Secrets + lint-gitleaks = ./gitleaks.nix; + lint-trufflehog = ./trufflehog.nix; + lint-detect-secrets = ./detect-secrets.nix; + + # SAST + lint-semgrep = ./semgrep.nix; + + # Supply chain + lint-trivy = ./trivy.nix; + lint-syft = ./syft.nix; + lint-osv = ./osv-scanner.nix; + + # Spelling + lint-cspell = ./cspell.nix; + lint-typos = ./typos.nix; + lint-codespell = ./codespell.nix; + }; + + pick = names: lib.attrValues (lib.filterAttrs (n: _v: lib.elem n names) perTool); + + mkSubs = + drvs: + map (drv: { + inherit drv; + binName = drv.meta.mainProgram or drv.name; + }) drvs; + + mkComposite = + name: description: drvs: + mkAllApp { + inherit name description; + subs = mkSubs drvs; + }; + + # Domain-grouped composites + lintShell = + mkComposite "lint-shell" "All shell lints: shellcheck, shfmt, bashate, checkbashisms." + (pick [ + "lint-shellcheck" + "lint-shfmt" + "lint-bashate" + "lint-checkbashisms" + ]); + lintYaml = mkComposite "lint-yaml" "YAML + Actions: yamllint, actionlint." (pick [ + "lint-yamllint" + "lint-actionlint" + ]); + lintJson = mkComposite "lint-json" "JSON/TOML: jsonlint (validate), biome, taplo." (pick [ + "lint-jsonlint" + "lint-biome" + "lint-taplo" + ]); + lintPy = + mkComposite "lint-py" "All Python lints: ruff, pyright, mypy, bandit, pylint, vulture." + (pick [ + "lint-ruff" + "lint-pyright" + "lint-mypy" + "lint-bandit" + "lint-pylint" + "lint-vulture" + ]); + lintNixFiles = mkComposite "lint-nix-files" "Nix lints: nixfmt, statix, deadnix, nil." (pick [ + "lint-nixfmt" + "lint-statix" + "lint-deadnix" + "lint-nil" + ]); + lintDocker = mkComposite "lint-docker" "Dockerfile lints: hadolint." (pick [ "lint-hadolint" ]); + lintMd = mkComposite "lint-md" "Markdown + prose: markdownlint, mdformat, vale, lychee." (pick [ + "lint-markdown" + "lint-mdformat" + "lint-vale" + "lint-lychee" + ]); + lintSecrets = mkComposite "lint-secrets" "Secrets: gitleaks, trufflehog, detect-secrets." (pick [ + "lint-gitleaks" + "lint-trufflehog" + "lint-detect-secrets" + ]); + lintSast = mkComposite "lint-sast" "SAST: semgrep, bandit." (pick [ + "lint-semgrep" + "lint-bandit" + ]); + lintSupply = mkComposite "lint-supply" "Supply chain: osv-scanner, trivy, syft." (pick [ + "lint-osv" + "lint-trivy" + "lint-syft" + ]); + lintSpelling = mkComposite "lint-spelling" "Spelling: cspell, typos, codespell." (pick [ + "lint-cspell" + "lint-typos" + "lint-codespell" + ]); + + lintExtreme = mkAllApp { + name = "lint-extreme"; + subs = lib.mapAttrsToList (n: drv: { + inherit drv; + binName = n; + }) perTool; + description = "Run every manual lint sequentially; tally pass/fail."; + }; + + everything = import ./everything.nix { inherit pkgs; }; + + composites = { + lint-shell = lintShell; + lint-yaml = lintYaml; + lint-json = lintJson; + lint-py = lintPy; + lint-nix-files = lintNixFiles; + lint-docker = lintDocker; + lint-md = lintMd; + lint-secrets = lintSecrets; + lint-sast = lintSast; + lint-supply = lintSupply; + lint-spelling = lintSpelling; + lint-extreme = lintExtreme; + inherit everything; + }; + + allLints = perTool // composites; + + drvsToApps = lib.mapAttrs ( + name: drv: { + type = "app"; + program = "${drv}/bin/${name}"; + meta = drv.meta or { }; + } + ); +in +{ + apps = drvsToApps allLints; + drvs = allLints; +} diff --git a/nix/lints/detect-secrets.nix b/nix/lints/detect-secrets.nix new file mode 100644 index 00000000..b3d49d30 --- /dev/null +++ b/nix/lints/detect-secrets.nix @@ -0,0 +1,27 @@ +# `lint-detect-secrets` — Yelp detect-secrets scan. Guarded via +# maybe-tool. + +{ pkgs, lib }: + +import ../lib/maybe-tool.nix { inherit pkgs lib; } { + name = "lint-detect-secrets"; + tool = pkgs.detect-secrets or pkgs.python3Packages.detect-secrets or null; + description = "Run detect-secrets scan against the tree."; + build = + detect-secrets: + import ../lib/mk-lint.nix { inherit pkgs lib; } { + name = "lint-detect-secrets"; + description = "Run detect-secrets scan against the tree."; + runtimeInputs = [ + detect-secrets + pkgs.git + ]; + header = "detect-secrets scan"; + command = '' + # Non-zero exit when any secret is detected. + found=$(detect-secrets scan --all-files | python3 -c 'import sys,json; d=json.load(sys.stdin); print(sum(len(v) for v in d.get("results",{}).values()))') + echo " detected candidate secrets: $found" + [ "$found" -eq 0 ] + ''; + }; +} diff --git a/nix/lints/dotenv-linter.nix b/nix/lints/dotenv-linter.nix new file mode 100644 index 00000000..e7782fdd --- /dev/null +++ b/nix/lints/dotenv-linter.nix @@ -0,0 +1,27 @@ +# `lint-dotenv` — lint .env files. Guarded via maybe-tool. + +{ pkgs, lib }: + +import ../lib/maybe-tool.nix { inherit pkgs lib; } { + name = "lint-dotenv"; + tool = pkgs.dotenv-linter or null; + description = "Run dotenv-linter against tracked .env files."; + build = + dotenv-linter: + import ../lib/mk-lint.nix { inherit pkgs lib; } { + name = "lint-dotenv"; + description = "Run dotenv-linter against tracked .env files."; + runtimeInputs = [ + dotenv-linter + pkgs.git + ]; + header = "dotenv-linter"; + globs = [ + "*.env" + ".env*" + "**/*.env" + ]; + emptyMessage = "(no .env files tracked)"; + command = ''dotenv-linter "''${files[@]}"''; + }; +} diff --git a/nix/lints/editorconfig-checker.nix b/nix/lints/editorconfig-checker.nix new file mode 100644 index 00000000..1b6416dc --- /dev/null +++ b/nix/lints/editorconfig-checker.nix @@ -0,0 +1,15 @@ +# `lint-editorconfig` — whitespace / EOL / charset hygiene. Honors a +# `.editorconfig` if present; otherwise applies its built-in defaults. + +{ pkgs, lib }: + +import ../lib/mk-lint.nix { inherit pkgs lib; } { + name = "lint-editorconfig"; + description = "Run editorconfig-checker against the tree."; + runtimeInputs = [ + pkgs.editorconfig-checker + pkgs.git + ]; + header = "editorconfig-checker"; + command = "editorconfig-checker"; +} diff --git a/nix/lints/everything.nix b/nix/lints/everything.nix new file mode 100644 index 00000000..185f0b0e --- /dev/null +++ b/nix/lints/everything.nix @@ -0,0 +1,53 @@ +# `everything` — single entry point that runs the full static-analysis +# pipeline: every gated derivation under `nix flake check`, then every +# manual lint via `lint-extreme`. + +{ pkgs }: + +pkgs.writeShellApplication { + name = "everything"; + runtimeInputs = [ + pkgs.nix + pkgs.git + ]; + text = '' + cd "$(git rev-parse --show-toplevel)" + + set -uo pipefail + passes=0 + fails=0 + failed_names="" + + echo + echo "============================================================" + echo "==> nix flake check (gated derivations)" + echo "============================================================" + if nix --extra-experimental-features 'nix-command flakes' flake check --print-build-logs; then + passes=$((passes + 1)) + else + fails=$((fails + 1)) + failed_names="$failed_names nix-flake-check" + fi + + echo + echo "============================================================" + echo "==> lint-extreme (manual lints)" + echo "============================================================" + if nix --extra-experimental-features 'nix-command flakes' run .#lint-extreme; then + passes=$((passes + 1)) + else + fails=$((fails + 1)) + failed_names="$failed_names lint-extreme" + fi + + echo + echo "============================================================" + echo "Overall: $passes passed, $fails failed" + if [ "$fails" -gt 0 ]; then + echo "Failed:$failed_names" + fi + echo "============================================================" + [ "$fails" -eq 0 ] || exit 1 + ''; + meta.description = "Run nix flake check + lint-extreme together; tally pass/fail."; +} diff --git a/nix/lints/gitleaks.nix b/nix/lints/gitleaks.nix new file mode 100644 index 00000000..18e7275c --- /dev/null +++ b/nix/lints/gitleaks.nix @@ -0,0 +1,15 @@ +# `lint-gitleaks` — scan git history + working tree for committed +# secrets. Reads `.gitleaks.toml` at the repo root for allowlists. + +{ pkgs, lib }: + +import ../lib/mk-lint.nix { inherit pkgs lib; } { + name = "lint-gitleaks"; + description = "Run gitleaks against the repo for committed secrets."; + runtimeInputs = [ + pkgs.gitleaks + pkgs.git + ]; + header = "gitleaks detect"; + command = "gitleaks detect --source=. --no-banner --redact --verbose"; +} diff --git a/nix/lints/hadolint.nix b/nix/lints/hadolint.nix new file mode 100644 index 00000000..93d516a9 --- /dev/null +++ b/nix/lints/hadolint.nix @@ -0,0 +1,22 @@ +# `lint-hadolint` — Dockerfile linter (default failure threshold). + +{ pkgs, lib }: + +import ../lib/mk-lint.nix { inherit pkgs lib; } { + name = "lint-hadolint"; + description = "Run hadolint against tracked Dockerfiles."; + runtimeInputs = [ + pkgs.hadolint + pkgs.git + ]; + header = "hadolint"; + globs = [ + "Dockerfile" + "Dockerfile.*" + "*.Dockerfile" + "**/Dockerfile" + "**/Dockerfile.*" + ]; + emptyMessage = "(no Dockerfile tracked)"; + command = ''hadolint "''${files[@]}"''; +} diff --git a/nix/lints/jsonlint.nix b/nix/lints/jsonlint.nix new file mode 100644 index 00000000..06c8ab8d --- /dev/null +++ b/nix/lints/jsonlint.nix @@ -0,0 +1,29 @@ +# `lint-jsonlint` — validate JSON well-formedness. Uses the Python +# stdlib json parser (always available) so it needs no extra package; +# useful for the large generated Grafana dashboard JSON in this repo. + +{ pkgs, lib }: + +import ../lib/mk-lint.nix { inherit pkgs lib; } { + name = "lint-jsonlint"; + description = "Validate tracked .json files parse cleanly."; + runtimeInputs = [ + pkgs.python3 + pkgs.git + ]; + header = "python -m json.tool (validate)"; + globs = [ "*.json" ]; + emptyMessage = "(no .json files tracked)"; + command = '' + failed=0 + for f in "''${files[@]}"; do + if ! python3 -m json.tool "$f" >/dev/null 2>err.txt; then + echo "=== $f ===" + cat err.txt + failed=1 + fi + done + rm -f err.txt + exit "$failed" + ''; +} diff --git a/nix/lints/lychee.nix b/nix/lints/lychee.nix new file mode 100644 index 00000000..6fd1174a --- /dev/null +++ b/nix/lints/lychee.nix @@ -0,0 +1,17 @@ +# `lint-lychee` — link checker for Markdown. Manual/network app: reaches +# out to external URLs, so it is NOT part of `nix flake check`. + +{ pkgs, lib }: + +import ../lib/mk-lint.nix { inherit pkgs lib; } { + name = "lint-lychee"; + description = "Run lychee link checker against tracked .md files."; + runtimeInputs = [ + pkgs.lychee + pkgs.git + ]; + header = "lychee (network)"; + globs = [ "*.md" ]; + emptyMessage = "(no .md files tracked)"; + command = ''lychee --no-progress "''${files[@]}"''; +} diff --git a/nix/lints/markdown.nix b/nix/lints/markdown.nix new file mode 100644 index 00000000..7f1d043a --- /dev/null +++ b/nix/lints/markdown.nix @@ -0,0 +1,21 @@ +# `lint-markdown` — markdownlint over tracked Markdown. Reads +# `.markdownlint.yaml` at the repo root. + +{ pkgs, lib }: + +import ../lib/mk-lint.nix { inherit pkgs lib; } { + name = "lint-markdown"; + description = "Run markdownlint-cli2 against tracked .md files."; + runtimeInputs = [ + pkgs.markdownlint-cli2 + pkgs.git + ]; + header = "markdownlint-cli2"; + command = '' + markdownlint-cli2 \ + '**/*.md' \ + '!node_modules/**' \ + '!dist/**' \ + '!build/**' + ''; +} diff --git a/nix/lints/mdformat.nix b/nix/lints/mdformat.nix new file mode 100644 index 00000000..e2151489 --- /dev/null +++ b/nix/lints/mdformat.nix @@ -0,0 +1,23 @@ +# `lint-mdformat` — check Markdown formatting. Guarded via maybe-tool. + +{ pkgs, lib }: + +import ../lib/maybe-tool.nix { inherit pkgs lib; } { + name = "lint-mdformat"; + tool = pkgs.python3Packages.mdformat or null; + description = "Run mdformat --check against tracked .md files."; + build = + mdformat: + import ../lib/mk-lint.nix { inherit pkgs lib; } { + name = "lint-mdformat"; + description = "Run mdformat --check against tracked .md files."; + runtimeInputs = [ + mdformat + pkgs.git + ]; + header = "mdformat --check"; + globs = [ "*.md" ]; + emptyMessage = "(no .md files tracked)"; + command = ''mdformat --check "''${files[@]}"''; + }; +} diff --git a/nix/lints/mypy.nix b/nix/lints/mypy.nix new file mode 100644 index 00000000..a32dc2b4 --- /dev/null +++ b/nix/lints/mypy.nix @@ -0,0 +1,17 @@ +# `lint-mypy` — Python type checker (non-strict, ignore-missing-imports +# so third-party deps that aren't installed don't drown the signal). + +{ pkgs, lib }: + +import ../lib/mk-lint.nix { inherit pkgs lib; } { + name = "lint-mypy"; + description = "Run mypy against tracked .py files."; + runtimeInputs = [ + pkgs.mypy + pkgs.git + ]; + header = "mypy --ignore-missing-imports"; + globs = [ "*.py" ]; + emptyMessage = "(no .py files tracked)"; + command = ''mypy --ignore-missing-imports "''${files[@]}"''; +} diff --git a/nix/lints/nil.nix b/nix/lints/nil.nix new file mode 100644 index 00000000..67fbd77c --- /dev/null +++ b/nix/lints/nil.nix @@ -0,0 +1,26 @@ +# `lint-nil` — nil language-server diagnostics over tracked Nix files. + +{ pkgs, lib }: + +import ../lib/mk-lint.nix { inherit pkgs lib; } { + name = "lint-nil"; + description = "Run nil diagnostics against tracked .nix files."; + runtimeInputs = [ + pkgs.nil + pkgs.git + ]; + header = "nil diagnostics"; + globs = [ "*.nix" ]; + emptyMessage = "(no .nix files tracked)"; + command = '' + failed=0 + for f in "''${files[@]}"; do + diag=$(nil diagnostics "$f" 2>&1) || failed=1 + if [ -n "$diag" ]; then + printf '=== %s ===\n%s\n' "$f" "$diag" + failed=1 + fi + done + exit "$failed" + ''; +} diff --git a/nix/lints/nixfmt.nix b/nix/lints/nixfmt.nix new file mode 100644 index 00000000..849743fd --- /dev/null +++ b/nix/lints/nixfmt.nix @@ -0,0 +1,16 @@ +# `lint-nixfmt` — check Nix formatting (RFC style). + +{ pkgs, lib }: + +import ../lib/mk-lint.nix { inherit pkgs lib; } { + name = "lint-nixfmt"; + description = "Run nixfmt --check against tracked .nix files."; + runtimeInputs = [ + pkgs.nixfmt-rfc-style + pkgs.git + ]; + header = "nixfmt --check"; + globs = [ "*.nix" ]; + emptyMessage = "(no .nix files tracked)"; + command = ''nixfmt --check "''${files[@]}"''; +} diff --git a/nix/lints/osv-scanner.nix b/nix/lints/osv-scanner.nix new file mode 100644 index 00000000..30928c06 --- /dev/null +++ b/nix/lints/osv-scanner.nix @@ -0,0 +1,15 @@ +# `lint-osv` — scan dependency manifests against the OSV database. +# Manual/network app. + +{ pkgs, lib }: + +import ../lib/mk-lint.nix { inherit pkgs lib; } { + name = "lint-osv"; + description = "Run osv-scanner against dependency manifests."; + runtimeInputs = [ + pkgs.osv-scanner + pkgs.git + ]; + header = "osv-scanner (network)"; + command = "osv-scanner scan --recursive ."; +} diff --git a/nix/lints/pylint.nix b/nix/lints/pylint.nix new file mode 100644 index 00000000..97d7d101 --- /dev/null +++ b/nix/lints/pylint.nix @@ -0,0 +1,17 @@ +# `lint-pylint` — thorough Python linter. Manual app: some checks want +# imports resolvable. `--exit-zero` is NOT used; treats findings as fail. + +{ pkgs, lib }: + +import ../lib/mk-lint.nix { inherit pkgs lib; } { + name = "lint-pylint"; + description = "Run pylint against tracked .py files."; + runtimeInputs = [ + pkgs.pylint + pkgs.git + ]; + header = "pylint"; + globs = [ "*.py" ]; + emptyMessage = "(no .py files tracked)"; + command = ''pylint "''${files[@]}"''; +} diff --git a/nix/lints/pyright.nix b/nix/lints/pyright.nix new file mode 100644 index 00000000..6b2437ec --- /dev/null +++ b/nix/lints/pyright.nix @@ -0,0 +1,17 @@ +# `lint-pyright` — Python type checker. Manual app: import resolution +# needs the project's dependencies available on the interpreter path. + +{ pkgs, lib }: + +import ../lib/mk-lint.nix { inherit pkgs lib; } { + name = "lint-pyright"; + description = "Run pyright against tracked .py files."; + runtimeInputs = [ + pkgs.pyright + pkgs.git + ]; + header = "pyright"; + globs = [ "*.py" ]; + emptyMessage = "(no .py files tracked)"; + command = ''pyright "''${files[@]}"''; +} diff --git a/nix/lints/ruff.nix b/nix/lints/ruff.nix new file mode 100644 index 00000000..93b57312 --- /dev/null +++ b/nix/lints/ruff.nix @@ -0,0 +1,17 @@ +# `lint-ruff` — fast Python linter (default rule set; reads +# `pyproject.toml` `[tool.ruff]` if present). + +{ pkgs, lib }: + +import ../lib/mk-lint.nix { inherit pkgs lib; } { + name = "lint-ruff"; + description = "Run ruff check against tracked .py files."; + runtimeInputs = [ + pkgs.ruff + pkgs.git + ]; + header = "ruff check"; + globs = [ "*.py" ]; + emptyMessage = "(no .py files tracked)"; + command = ''ruff check "''${files[@]}"''; +} diff --git a/nix/lints/semgrep.nix b/nix/lints/semgrep.nix new file mode 100644 index 00000000..45342c40 --- /dev/null +++ b/nix/lints/semgrep.nix @@ -0,0 +1,16 @@ +# `lint-semgrep` — multi-language SAST. Manual app: the default rule +# registry (`--config auto`) fetches rules over the network. Uses the +# offline `p/default` ruleset bundled with the package here. + +{ pkgs, lib }: + +import ../lib/mk-lint.nix { inherit pkgs lib; } { + name = "lint-semgrep"; + description = "Run semgrep SAST against the tree."; + runtimeInputs = [ + pkgs.semgrep + pkgs.git + ]; + header = "semgrep --config p/default"; + command = "semgrep --error --config p/default --quiet ."; +} diff --git a/nix/lints/shellcheck.nix b/nix/lints/shellcheck.nix new file mode 100644 index 00000000..db8070b6 --- /dev/null +++ b/nix/lints/shellcheck.nix @@ -0,0 +1,23 @@ +# `lint-shellcheck` — shellcheck over tracked shell scripts (default +# severity; `-x` follows `source` directives). + +{ pkgs, lib }: + +import ../lib/mk-lint.nix { inherit pkgs lib; } { + name = "lint-shellcheck"; + description = "Run shellcheck against tracked .sh / .bash files."; + runtimeInputs = [ + pkgs.shellcheck + pkgs.git + ]; + header = "shellcheck -x"; + globs = [ + "*.sh" + "*.bash" + ]; + emptyMessage = "(no .sh / .bash files tracked)"; + command = '' + printf ' %s\n' "''${files[@]}" + shellcheck -x "''${files[@]}" + ''; +} diff --git a/nix/lints/shfmt.nix b/nix/lints/shfmt.nix new file mode 100644 index 00000000..57ca7e0a --- /dev/null +++ b/nix/lints/shfmt.nix @@ -0,0 +1,19 @@ +# `lint-shfmt` — report shell-formatting diffs (does not rewrite files). + +{ pkgs, lib }: + +import ../lib/mk-lint.nix { inherit pkgs lib; } { + name = "lint-shfmt"; + description = "Run shfmt --diff against tracked shell files."; + runtimeInputs = [ + pkgs.shfmt + pkgs.git + ]; + header = "shfmt --diff"; + globs = [ + "*.sh" + "*.bash" + ]; + emptyMessage = "(no shell files tracked)"; + command = ''shfmt --diff "''${files[@]}"''; +} diff --git a/nix/lints/statix.nix b/nix/lints/statix.nix new file mode 100644 index 00000000..53e2483a --- /dev/null +++ b/nix/lints/statix.nix @@ -0,0 +1,14 @@ +# `lint-statix` — flag Nix antipatterns. + +{ pkgs, lib }: + +import ../lib/mk-lint.nix { inherit pkgs lib; } { + name = "lint-statix"; + description = "Run statix antipattern check against the tree."; + runtimeInputs = [ + pkgs.statix + pkgs.git + ]; + header = "statix check ."; + command = "statix check ."; +} diff --git a/nix/lints/syft.nix b/nix/lints/syft.nix new file mode 100644 index 00000000..c63271f7 --- /dev/null +++ b/nix/lints/syft.nix @@ -0,0 +1,15 @@ +# `lint-syft` — generate an SBOM of the tree. Informational; always +# exits 0 unless syft itself errors. + +{ pkgs, lib }: + +import ../lib/mk-lint.nix { inherit pkgs lib; } { + name = "lint-syft"; + description = "Generate an SBOM of the tree with syft."; + runtimeInputs = [ + pkgs.syft + pkgs.git + ]; + header = "syft (SBOM)"; + command = "syft dir:. --output table"; +} diff --git a/nix/lints/taplo.nix b/nix/lints/taplo.nix new file mode 100644 index 00000000..5be47065 --- /dev/null +++ b/nix/lints/taplo.nix @@ -0,0 +1,16 @@ +# `lint-taplo` — validate TOML files. + +{ pkgs, lib }: + +import ../lib/mk-lint.nix { inherit pkgs lib; } { + name = "lint-taplo"; + description = "Run taplo check against tracked .toml files."; + runtimeInputs = [ + pkgs.taplo + pkgs.git + ]; + header = "taplo check"; + globs = [ "*.toml" ]; + emptyMessage = "(no .toml files tracked)"; + command = ''taplo check "''${files[@]}"''; +} diff --git a/nix/lints/trivy.nix b/nix/lints/trivy.nix new file mode 100644 index 00000000..10de48e1 --- /dev/null +++ b/nix/lints/trivy.nix @@ -0,0 +1,15 @@ +# `lint-trivy` — filesystem vulnerability + misconfiguration scan. +# Manual/network app: pulls its vulnerability DB. + +{ pkgs, lib }: + +import ../lib/mk-lint.nix { inherit pkgs lib; } { + name = "lint-trivy"; + description = "Run trivy filesystem scan (vuln + misconfig)."; + runtimeInputs = [ + pkgs.trivy + pkgs.git + ]; + header = "trivy fs (network)"; + command = "trivy fs --scanners vuln,misconfig,secret --exit-code 1 ."; +} diff --git a/nix/lints/trufflehog.nix b/nix/lints/trufflehog.nix new file mode 100644 index 00000000..2ef7eb58 --- /dev/null +++ b/nix/lints/trufflehog.nix @@ -0,0 +1,15 @@ +# `lint-trufflehog` — verified-secret scanner. Manual app: it may reach +# out to validate candidate credentials, so keep it out of the sandbox. + +{ pkgs, lib }: + +import ../lib/mk-lint.nix { inherit pkgs lib; } { + name = "lint-trufflehog"; + description = "Run trufflehog filesystem scan for secrets."; + runtimeInputs = [ + pkgs.trufflehog + pkgs.git + ]; + header = "trufflehog filesystem"; + command = "trufflehog filesystem . --no-update --fail"; +} diff --git a/nix/lints/typos.nix b/nix/lints/typos.nix new file mode 100644 index 00000000..792e7607 --- /dev/null +++ b/nix/lints/typos.nix @@ -0,0 +1,15 @@ +# `lint-typos` — source-code spell checker (low false-positive). Reads +# `_typos.toml` / `typos.toml` if present. + +{ pkgs, lib }: + +import ../lib/mk-lint.nix { inherit pkgs lib; } { + name = "lint-typos"; + description = "Run typos spell checker against the tree."; + runtimeInputs = [ + pkgs.typos + pkgs.git + ]; + header = "typos"; + command = "typos"; +} diff --git a/nix/lints/vale.nix b/nix/lints/vale.nix new file mode 100644 index 00000000..b8672107 --- /dev/null +++ b/nix/lints/vale.nix @@ -0,0 +1,33 @@ +# `lint-vale` — prose linter. Guarded via maybe-tool; only meaningful +# with a `.vale.ini` in the repo, otherwise it no-ops cleanly. + +{ pkgs, lib }: + +import ../lib/maybe-tool.nix { inherit pkgs lib; } { + name = "lint-vale"; + tool = pkgs.vale or null; + description = "Run vale prose linter against tracked .md files."; + build = + vale: + import ../lib/mk-lint.nix { inherit pkgs lib; } { + name = "lint-vale"; + description = "Run vale prose linter against tracked .md files."; + runtimeInputs = [ + vale + pkgs.git + ]; + header = "vale"; + command = '' + if [ ! -f .vale.ini ]; then + echo " (no .vale.ini config — skipping)" + exit 0 + fi + mapfile -t files < <(git ls-files '*.md') + if [ "''${#files[@]}" -eq 0 ]; then + echo " (no .md files tracked)" + exit 0 + fi + vale "''${files[@]}" + ''; + }; +} diff --git a/nix/lints/vulture.nix b/nix/lints/vulture.nix new file mode 100644 index 00000000..78dc2538 --- /dev/null +++ b/nix/lints/vulture.nix @@ -0,0 +1,23 @@ +# `lint-vulture` — find dead Python code. Guarded via maybe-tool. + +{ pkgs, lib }: + +import ../lib/maybe-tool.nix { inherit pkgs lib; } { + name = "lint-vulture"; + tool = pkgs.vulture or pkgs.python3Packages.vulture or null; + description = "Run vulture (dead-code finder) against tracked .py files."; + build = + vulture: + import ../lib/mk-lint.nix { inherit pkgs lib; } { + name = "lint-vulture"; + description = "Run vulture (dead-code finder) against tracked .py files."; + runtimeInputs = [ + vulture + pkgs.git + ]; + header = "vulture"; + globs = [ "*.py" ]; + emptyMessage = "(no .py files tracked)"; + command = ''vulture "''${files[@]}"''; + }; +} diff --git a/nix/lints/yamllint.nix b/nix/lints/yamllint.nix new file mode 100644 index 00000000..2e5ca7d7 --- /dev/null +++ b/nix/lints/yamllint.nix @@ -0,0 +1,20 @@ +# `lint-yamllint` — yamllint over tracked YAML. Reads `.yamllint.yaml` +# at the repo root for the project's relaxed rule set. + +{ pkgs, lib }: + +import ../lib/mk-lint.nix { inherit pkgs lib; } { + name = "lint-yamllint"; + description = "Run yamllint against tracked .yml / .yaml files."; + runtimeInputs = [ + pkgs.yamllint + pkgs.git + ]; + header = "yamllint"; + globs = [ + "*.yml" + "*.yaml" + ]; + emptyMessage = "(no .yml / .yaml files tracked)"; + command = ''yamllint "''${files[@]}"''; +} diff --git a/nix/prod-images/README.md b/nix/prod-images/README.md new file mode 100644 index 00000000..ca53ecf0 --- /dev/null +++ b/nix/prod-images/README.md @@ -0,0 +1,90 @@ +# Native-parity Nix container images (`prod-images`) + +An attempt to rebuild the repo's containers **entirely from Nix** — replacing +every `apt`/`pip`/`uv` install with a Nix package (source-built, hash-pinned), +and CUDA with nixpkgs' own `cudaPackages` (no vendor `nvidia/cuda` base). Goal: +narHash-verified inputs (**security**), **bit-reproducibility**, and better +cross-image layer sharing. **Build-only — nothing is published.** + +Kept separate from the experimental PoC in `../images/` so both approaches +coexist and can be compared. Scope here: **base + pytorch** (CPU + CUDA). + +## What the current images install (review) + +| image | base | Python | key installs | +| --- | --- | --- | --- | +| base | ubuntu / nvidia-cuda devel | 3.9–3.13 (deadsnakes) | apt: build-essential, ffmpeg, cmake, …; pip: jupyterlab 4.5.9, notebook 7.5.6, ipywidgets 8.1.8, hf_transfer 0.1.9, jupyter-archive 3.4.0; uv; filebrowser 2.63.5; nginx | +| pytorch | runpod/base cuda | (from base) | pip torch/torchvision/torchaudio — matrix 2.6.0–2.9.1 × cu126/128/129/130 | + +Runtime contract (reproduced in `runtime.nix`): nginx proxy; `$PUBLIC_KEY`→sshd +(host-key gen); `$JUPYTER_PASSWORD`→`jupyter lab` on 8888 (`preferred_dir=/workspace`); +export env to `/etc/rp_environment`; `/pre_start.sh` + `/post_start.sh`; sleep. + +## The Nix rebuild + +- **`userland.nix`** — Python **3.14** env (jupyterlab/notebook/ipywidgets/hf-transfer + with our `jupyter-archive`) plus uv, filebrowser, nginx, openssh, and the apt-parity + dev toolchain (gcc, gnumake, cmake, gfortran, pkg-config, ffmpeg) — all Nix packages. +- **`pkgs/jupyter-archive.nix`** — the one `requirements.txt` entry missing from nixpkgs, + packaged from the PyPI **sdist** (hash-pinned, source-built via hatchling). +- **`runtime.nix`** — Nix-adapted `/start.sh` (the upstream uses Debian `service`, absent + here): runs `nginx`/`sshd`/`jupyter` directly, same env-var gating; stages the repo's + nginx proxy config + banner. +- **`base.nix`** — `prod-base-cpu`, `prod-base-cuda` (+ nixpkgs `cudaPackages` CUDA 12.9 / + cuDNN 9.22 on PATH/LD_LIBRARY_PATH; CUDA comes from NVIDIA **redistributables** = + hash-pinned fetches, not source builds). +- **`pytorch.nix`** — `prod-pytorch-cpu` (torch 2.12.0 CPU, cache-friendly) and + `prod-pytorch` (`cudaSupport=true`, a heavy from-source CUDA compile). + +## Measured (build-only, gzip ≈ registry download) + +| target | uncompressed | gzip | notes | +| --- | --- | --- | --- | +| `prod-base-cpu-lean` | 927 MB | **299 MB** | runtime userland only (no compilers) | +| `prod-base-cpu` | 2473 MB | **850 MB** | full dev toolchain + 1 python | +| `prod-pytorch-cpu` | 3967 MB | **1250 MB** | + torch 2.12.0 (CPU) | +| `prod-base-cuda` | 7231 MB | **3698 MB** | + nixpkgs CUDA 12.9 / cuDNN | +| `prod-pytorch` (CUDA) | 14772 MB | **7791 MB** | torch 2.12.0 source-built w/ CUDA 12.9 / cuDNN 9.22 | + +Reference points (compressed): `runpod/base:1.0.7-ubuntu2404` (apt) = **714 MB**; +`runpod/base:1.0.7-cuda1281-ubuntu2404` = **6359 MB**; +`runpod/pytorch:1.0.7-cu1281-torch280-ubuntu2404` = **11211 MB**. + +`torch` in the CUDA image is genuinely source-built against Nix `cudaPackages` +(verified: `torch.version.cuda == 12.9`, `cudnn 9.22`) — every input hash-pinned, +no download.pytorch.org wheel. + +- **CPU base — size is a *choice*, not a Nix limit.** The full parity base (850 MB) is a + touch *larger* than apt's 714 MB because it bundles the whole gcc/gfortran/cmake/ffmpeg + dev toolchain as Nix closures. Drop that build-time tooling (`prod-base-cpu-lean`) and the + runtime-only base is **299 MB — ~58% smaller** than apt. Most of `build-essential`/`ffmpeg` + is build-time, not needed in a running pod, so a lean runtime base wins comfortably; only + a "compile anything" base loses. Either way the determinism + hash-pinning is identical. +- **CUDA base**: full-Nix (3698 MB) is **~42% smaller** than the vendor-based image + (6359 MB) — it ships only the CUDA toolkit + cuDNN redistributables, not the ~5.6 GB + `nvidia/cuda:*-cudnn-devel` base. Plus layer sharing across the family (see + `../images/README.md`). +- **CUDA pytorch**: full-Nix (7791 MB) is **~30% smaller** than the vendor pytorch image + (11211 MB), and every byte — CUDA, cuDNN, torch — is source-built/hash-pinned rather than + a pip wheel. Cost: a long from-source torch compile (unfree ⇒ not in cache.nixos.org). + +## Honest caveats + +- **Version drift:** native nixpkgs gives **one** torch (2.12.0) + CUDA 12.9 — newer than + and not matching the repo's 2.6–2.9.1 × cu126–130 matrix. Inherent to source-from-nixpkgs. +- **CUDA torch build cost:** `prod-pytorch` is a long, RAM-heavy compile (unfree ⇒ absent from + cache.nixos.org). Build locally / on a big runner and push to a cache; GH-hosted CI can't. +- **Python 3.14** vs the repo's 3.9–3.13; jupyterlab 4.5.8 vs pinned 4.5.9 (nixpkgs' version). +- **Runtime not container-tested in the sandbox:** the image builds and contains everything; + full nginx/sshd/jupyter startup needs an actual pod/GPU host to validate. +- **Out of scope:** autoresearch, nvidia-pytorch (NGC proprietary), rocm. + +## Build & verify + +```sh +nix build .#packages.x86_64-linux.prod-base-cpu # + prod-pytorch-cpu, prod-base-cuda +nix run .#packages.x86_64-linux.prod-base-cpu | wc -c +# import check: +nix build --impure --expr '…userland.python.withPackages…' && python -c 'import jupyterlab, hf_transfer, jupyter_archive' +nix build .#packages.x86_64-linux.prod-pytorch # heavy: local/cache only +``` diff --git a/nix/prod-images/base.nix b/nix/prod-images/base.nix new file mode 100644 index 00000000..272c1700 --- /dev/null +++ b/nix/prod-images/base.nix @@ -0,0 +1,72 @@ +# Native-parity base images, built with dockerTools.streamLayeredImage. +# prod-base-cpu — userland + runtime, no GPU. +# prod-base-cuda — + full-Nix cudaPackages (CUDA toolkit + cuDNN from nixpkgs +# redistributables, hash-pinned) on PATH/LD_LIBRARY_PATH. +# +# No vendor nvidia/cuda base image — CUDA comes entirely from nixpkgs. + +{ pkgs }: + +let + inherit (pkgs) lib; + userland = import ./userland.nix { inherit pkgs; }; + runtime = import ./runtime.nix { inherit pkgs; }; + cuda = pkgs.cudaPackages; + + mkImage = + { + name, + contents ? userland.contents, + extraContents ? [ ], + pathDirs ? [ ], + env ? [ ], + }: + pkgs.dockerTools.streamLayeredImage { + inherit name; + tag = "poc"; + contents = contents ++ [ runtime ] ++ extraContents; + maxLayers = 100; + config = { + Cmd = [ "/start.sh" ]; + WorkingDir = "/"; + Env = + userland.baseEnv + ++ [ + "PATH=${ + lib.concatStringsSep ":" ( + [ + "/bin" + "/usr/bin" + ] + ++ pathDirs + ) + }" + ] + ++ env; + }; + }; +in +{ + prod-base-cpu = mkImage { name = "runpod-prod-base-cpu"; }; + + # Runtime-only CPU base: drops the gcc/gfortran/cmake/ffmpeg dev toolchain. + prod-base-cpu-lean = mkImage { + name = "runpod-prod-base-cpu-lean"; + contents = userland.mkContents { lean = true; }; + }; + + prod-base-cuda = mkImage { + name = "runpod-prod-base-cuda"; + extraContents = [ + cuda.cudatoolkit + cuda.cudnn + ]; + pathDirs = [ "${cuda.cudatoolkit}/bin" ]; + env = [ + "LD_LIBRARY_PATH=${cuda.cudatoolkit}/lib:${cuda.cudnn}/lib" + "CUDA_HOME=${cuda.cudatoolkit}" + "NVIDIA_VISIBLE_DEVICES=all" + "NVIDIA_DRIVER_CAPABILITIES=compute,utility" + ]; + }; +} diff --git a/nix/prod-images/default.nix b/nix/prod-images/default.nix new file mode 100644 index 00000000..c5a11386 --- /dev/null +++ b/nix/prod-images/default.nix @@ -0,0 +1,6 @@ +# Native-parity ("prod") Nix container images — build-only, not published. +# Kept separate from the experimental nix/images/* PoC so both can be compared. + +{ pkgs }: + +(import ./base.nix { inherit pkgs; }) // (import ./pytorch.nix { inherit pkgs; }) diff --git a/nix/prod-images/pkgs/jupyter-archive.nix b/nix/prod-images/pkgs/jupyter-archive.nix new file mode 100644 index 00000000..783748bc --- /dev/null +++ b/nix/prod-images/pkgs/jupyter-archive.nix @@ -0,0 +1,37 @@ +# jupyter-archive 3.4.0 — the one base `requirements.txt` entry not in nixpkgs. +# Built from the PyPI sdist (hash-pinned → narHash) via its hatchling backend. +# It's a JupyterLab extension whose sdist ships the prebuilt JS assets, so no +# node toolchain is needed at build time. If a future bump reintroduces an +# asset build, switch `src` to the wheel (format = "wheel"); the wheel hash is +# sha256-rIViwglxx0Gi3JxcFgoLXSelRwDbl3kGohVJ/S4pEJs=. + +{ python, fetchurl }: + +python.pkgs.buildPythonPackage { + pname = "jupyter-archive"; + version = "3.4.0"; + pyproject = true; + + src = fetchurl { + url = "https://files.pythonhosted.org/packages/6e/8d/2ab96674293cdb333dc90fa9b7a2f4787b3157006624698b44e09e33f3fd/jupyter_archive-3.4.0.tar.gz"; + hash = "sha256-m/AXDgtFrIP8Rsvxq8YPQM7Vqs8LWh2uRl6zvauvF+k="; + }; + + build-system = with python.pkgs; [ + hatchling + hatch-jupyter-builder + hatch-nodejs-version + jupyterlab + ]; + + dependencies = with python.pkgs; [ jupyter-server ]; + + # Extension ships prebuilt assets; don't let the jupyter builder try to + # rebuild them (would need node/jlpm). + env.HATCH_JUPYTER_BUILDER_SKIP = "1"; + + pythonImportsCheck = [ "jupyter_archive" ]; + doCheck = false; + + meta.description = "Jupyter server + lab extension to download folders as archives"; +} diff --git a/nix/prod-images/pytorch.nix b/nix/prod-images/pytorch.nix new file mode 100644 index 00000000..4ed4c15f --- /dev/null +++ b/nix/prod-images/pytorch.nix @@ -0,0 +1,83 @@ +# Native-parity pytorch images. torch/torchvision/torchaudio come from nixpkgs +# (source-built), extending the same base userland. +# prod-pytorch-cpu — torch 2.12.0 CPU (cache-friendly; validates the pipeline). +# prod-pytorch — torch with cudaSupport=true via a cudaSupport pkgs set; +# a heavy from-source CUDA compile (unfree ⇒ not cached). +# +# Single nixpkgs version (2.12.0 / CUDA 12.9), not the repo's wheel matrix — +# see README. + +{ pkgs }: + +let + inherit (pkgs) lib; + runtime = import ./runtime.nix { inherit pkgs; }; + + # A cudaSupport nixpkgs so torch + torchvision + torchaudio build against CUDA. + pkgsCuda = import pkgs.path { + inherit (pkgs.stdenv.hostPlatform) system; + config = { + allowUnfree = true; + cudaSupport = true; + }; + }; + + torchPkgs = + ps: with ps; [ + torch + torchvision + torchaudio + ]; + + mkPytorch = + { + name, + pkgs', + pathDirs ? [ ], + env ? [ ], + }: + let + userland = import ./userland.nix { pkgs = pkgs'; }; + in + pkgs.dockerTools.streamLayeredImage { + inherit name; + tag = "poc"; + contents = (userland.mkContents { extraPyPackages = torchPkgs; }) ++ [ runtime ]; + maxLayers = 100; + config = { + Cmd = [ "/start.sh" ]; + WorkingDir = "/"; + Env = + userland.baseEnv + ++ [ + "PATH=${ + lib.concatStringsSep ":" ( + [ + "/bin" + "/usr/bin" + ] + ++ pathDirs + ) + }" + ] + ++ env; + }; + }; +in +{ + prod-pytorch-cpu = mkPytorch { + name = "runpod-prod-pytorch-cpu"; + pkgs' = pkgs; + }; + + prod-pytorch = mkPytorch { + name = "runpod-prod-pytorch-cuda"; + pkgs' = pkgsCuda; + pathDirs = [ "${pkgsCuda.cudaPackages.cudatoolkit}/bin" ]; + env = [ + "CUDA_HOME=${pkgsCuda.cudaPackages.cudatoolkit}" + "NVIDIA_VISIBLE_DEVICES=all" + "NVIDIA_DRIVER_CAPABILITIES=compute,utility" + ]; + }; +} diff --git a/nix/prod-images/runtime.nix b/nix/prod-images/runtime.nix new file mode 100644 index 00000000..07e43b03 --- /dev/null +++ b/nix/prod-images/runtime.nix @@ -0,0 +1,122 @@ +# Runtime contract for the prod Nix images: a Nix-adapted /start.sh plus the +# staged nginx proxy config + banner. The upstream container-template/start.sh +# uses Debian `service nginx|ssh start`, which a Nix image has no sysvinit for; +# this reproduces the same behavior by invoking the daemons directly, preserving +# the env-var gating ($PUBLIC_KEY -> sshd, $JUPYTER_PASSWORD -> jupyter) and the +# /etc/rp_environment export. +# +# Returns a derivation that stages /start.sh, /etc/nginx/*, and +# /usr/share/nginx/html/* into the image root. + +{ pkgs }: + +let + # Adapted entrypoint. Runs from the image where the userland (nginx, sshd, + # ssh-keygen, python) is on PATH via /bin. + startScript = pkgs.writeShellScript "start.sh" '' + set -e + + start_nginx() { + echo "Starting Nginx service..." + # Nix has no `service`; run nginx directly. Prefix = /etc/nginx so the + # relative `include snippets/...` and temp dirs resolve; keep pid/logs in + # writable /run + /var/log (container fs is writable at runtime). + mkdir -p /run/nginx /var/log/nginx \ + /etc/nginx/client_body_temp /etc/nginx/proxy_temp \ + /etc/nginx/fastcgi_temp /etc/nginx/uwsgi_temp /etc/nginx/scgi_temp + nginx -p /etc/nginx -c /etc/nginx/nginx.conf \ + -g 'daemon on; pid /run/nginx/nginx.pid; error_log /var/log/nginx/error.log;' + } + + execute_script() { + local script_path=$1 script_msg=$2 + if [[ -f $script_path ]]; then + echo "$script_msg" + bash "$script_path" + fi + } + + setup_ssh() { + if [[ $PUBLIC_KEY ]]; then + echo "Setting up SSH..." + mkdir -p ~/.ssh + echo "$PUBLIC_KEY" >> ~/.ssh/authorized_keys + chmod 700 -R ~/.ssh + mkdir -p /etc/ssh /run/sshd + + local t + for t in rsa ecdsa ed25519; do + if [[ ! -f /etc/ssh/ssh_host_''${t}_key ]]; then + ssh-keygen -t "$t" -f "/etc/ssh/ssh_host_''${t}_key" -q -N "" + fi + done + + # Direct daemon instead of `service ssh start`. + sshd \ + -o PermitRootLogin=yes \ + -o AuthorizedKeysFile=/root/.ssh/authorized_keys \ + -o PidFile=/run/sshd/sshd.pid \ + -o UsePAM=no + + echo "SSH host keys:" + for key in /etc/ssh/*.pub; do + echo "Key: $key" + ssh-keygen -lf "$key" + done + fi + } + + export_env_vars() { + echo "Exporting environment variables..." + printenv | grep -E '^[A-Z_][A-Z0-9_]*=' | grep -v '^PUBLIC_KEY' \ + | awk -F = '{ val = $0; sub(/^[^=]*=/, "", val); print "export " $1 "=\"" val "\"" }' > /etc/rp_environment + if ! grep -q 'source /etc/rp_environment' ~/.bashrc 2>/dev/null; then + echo 'source /etc/rp_environment' >> ~/.bashrc + fi + } + + start_jupyter() { + if [[ $JUPYTER_PASSWORD ]]; then + echo "Starting Jupyter Lab..." + mkdir -p /workspace + cd / + nohup python -m jupyter lab --allow-root --no-browser --port=8888 --ip=* \ + --FileContentsManager.delete_to_trash=False \ + --ServerApp.terminado_settings='{"shell_command":["/bin/bash"]}' \ + --IdentityProvider.token="$JUPYTER_PASSWORD" \ + --ServerApp.allow_origin=* --ServerApp.preferred_dir=/workspace &> /jupyter.log & + local pid=$! + sleep 2 + if kill -0 "$pid" 2>/dev/null; then + echo "Jupyter Lab started (pid=$pid)" + else + echo "Jupyter Lab FAILED to start. /jupyter.log:" >&2 + cat /jupyter.log >&2 + return 1 + fi + fi + } + + start_nginx + execute_script "/pre_start.sh" "Running pre-start script..." + echo "Pod Started" + setup_ssh + start_jupyter + export_env_vars + echo "Start script(s) finished, Pod is ready to use." + execute_script "/post_start.sh" "Running post-start script..." + sleep infinity + ''; + + proxy = ../../container-template/proxy; +in +pkgs.runCommand "runpod-prod-runtime" { } '' + mkdir -p "$out/etc/nginx/snippets" "$out/usr/share/nginx/html" + cp ${startScript} "$out/start.sh" + chmod +x "$out/start.sh" + + cp ${proxy}/nginx.conf "$out/etc/nginx/nginx.conf" + cp ${proxy}/snippets/*.conf "$out/etc/nginx/snippets/" + cp ${proxy}/readme.html "$out/usr/share/nginx/html/readme.html" + cp ${../../README.md} "$out/usr/share/nginx/html/README.md" +'' diff --git a/nix/prod-images/userland.nix b/nix/prod-images/userland.nix new file mode 100644 index 00000000..6ee79378 --- /dev/null +++ b/nix/prod-images/userland.nix @@ -0,0 +1,128 @@ +# Native-parity userland for the prod Nix images. Everything the base image +# pip/apt-installs, expressed as Nix packages (source-built by nixpkgs, hash- +# pinned). Python 3.14 — the latest in nixpkgs. +# +# `mkContents { extraPyPackages, extraTools }` lets pytorch (and future images) +# extend the exact same shared base, so those store paths dedupe across images. + +{ pkgs }: + +let + python = pkgs.python314; + + jupyter-archive = import ./pkgs/jupyter-archive.nix { + inherit python; + inherit (pkgs) fetchurl; + }; + + # The base image's pip set: jupyterlab, notebook, ipywidgets, hf_transfer, + # jupyter-archive. + basePyPackages = + ps: with ps; [ + jupyterlab + notebook + ipywidgets + hf-transfer + jupyter-archive + ]; + + # Runtime userland: shell + services + everyday CLI tools. What a *running* + # pod needs — no compilers. Runtime .so deps of Python packages come in via + # their own closures. + coreTools = with pkgs; [ + uv + filebrowser + nginx + openssh + + bashInteractive + coreutils + gnugrep + gnused + gawk + findutils + gnutar + gzip + zstd + curl + wget + git + cacert + jq + which + openssl + tmux + vim + nano + rsync + unzip + zip + sudo + ]; + + # The apt base's dev/build toolchain (build-essential, cmake, gfortran, + # pkg-config, ffmpeg). Build-time tooling — heavy, and omitted by the `lean` + # variant. + devTools = with pkgs; [ + gcc + gnumake + cmake + gfortran + pkg-config + ffmpeg + ]; + + # Full apt-parity userland (core + dev toolchain). + toolContents = coreTools ++ devTools; + + runpodFiles = pkgs.runCommand "runpod-prod-files" { } '' + mkdir -p "$out/etc" + cp ${../../container-template/runpod.txt} "$out/etc/runpod.txt" + ''; + + # `lean` drops the dev/build toolchain — a runtime-only base. + mkContents = + { + extraPyPackages ? (_: [ ]), + extraTools ? [ ], + lean ? false, + }: + (if lean then coreTools else toolContents) + ++ extraTools + ++ [ + (python.withPackages (ps: basePyPackages ps ++ extraPyPackages ps)) + runpodFiles + ]; +in +{ + inherit + python + basePyPackages + coreTools + devTools + toolContents + runpodFiles + mkContents + jupyter-archive + ; + + contents = mkContents { }; + + baseEnv = [ + "SHELL=/bin/bash" + "PYTHONUNBUFFERED=True" + "LANG=C.UTF-8" + "LC_ALL=C.UTF-8" + "SSL_CERT_FILE=${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt" + "TZ=Etc/UTC" + "RP_WORKSPACE=/workspace" + "HF_HOME=/workspace/.cache/huggingface/" + "PIP_CACHE_DIR=/workspace/.cache/pip/" + "UV_CACHE_DIR=/workspace/.cache/uv/" + "VIRTUALENV_OVERRIDE_APP_DATA=/workspace/.cache/virtualenv/" + "HF_HUB_ENABLE_HF_TRANSFER=1" + "HF_XET_HIGH_PERFORMANCE=1" + "PIP_BREAK_SYSTEM_PACKAGES=1" + "PIP_ROOT_USER_ACTION=ignore" + ]; +} diff --git a/official-templates/autoresearch/README.md b/official-templates/autoresearch/README.md index df09a018..56a136a5 100644 --- a/official-templates/autoresearch/README.md +++ b/official-templates/autoresearch/README.md @@ -15,18 +15,19 @@ An AI coding agent autonomously runs ML experiments: it modifies `train.py`, tra 1. Launch a pod with this template 2. Connect your coding agent (Claude Code, Cursor, etc.) via SSH 3. Tell the agent: - ``` + + ```text Read /workspace/autoresearch/program.md and let's kick off a new experiment! ``` ### GPU recommendations -| GPU | VRAM | Notes | -|-----|------|-------| -| RTX 4090 | 24 GB | Budget option. Smaller optimal model size. | -| A40 | 48 GB | Good middle ground. | -| A100 80GB | 80 GB | Plenty of room for larger models. | -| H100 | 80 GB | Fastest. What Karpathy used. | +| GPU | VRAM | Notes | +| --------- | ----- | ------------------------------------------ | +| RTX 4090 | 24 GB | Budget option. Smaller optimal model size. | +| A40 | 48 GB | Good middle ground. | +| A100 80GB | 80 GB | Plenty of room for larger models. | +| H100 | 80 GB | Fastest. What Karpathy used. | The 5-minute fixed time budget means cheaper GPUs work fine — you get a different optimal model size. Results are comparable within the same GPU type. diff --git a/official-templates/base/README.md b/official-templates/base/README.md index c70a8a5a..56cb26ab 100644 --- a/official-templates/base/README.md +++ b/official-templates/base/README.md @@ -5,6 +5,7 @@ The Runpod Base images provide a clean, developer friendly environment for everything from quick experiments to production, supporting both GPU and CPU-only workloads. Use them standalone for a preconfigured workspace, or as the foundation for your own images. ### What's included + - **Multiple Python versions**: 3.9–3.13 preinstalled; 3.10 is the default. - **ML ready**: Essential libraries for scientific computing, computer vision, and machine learning, plus SLURM support. - **Developer friendly**: SSH server preconfigured for seamless remote development and debugging. @@ -13,6 +14,7 @@ The Runpod Base images provide a clean, developer friendly environment for every - **Jupyter ready (optional)**: Notebook and JupyterLab with widgets/extensions; enable by setting `JUPYTER_PASSWORD` (omit to disable). ### Available configurations + - **Ubuntu**: 22.04 (Jammy) and 24.04 (Noble) - **CUDA**: 12.8.0, 12.8.1, 12.9.0, and 13.0.0 @@ -22,21 +24,24 @@ Need something more specialized? Explore the templates in `official-templates` f ## Generated Images -### Base Images (CPU-Only, No GPU Drivers): +### Base Images (CPU-Only, No GPU Drivers) + - Ubuntu 22.04: `runpod/base:1.0.2-ubuntu2204` - Ubuntu 24.04: `runpod/base:1.0.2-ubuntu2404` -### CUDA Images (GPU Required) by Version: +### CUDA Images (GPU Required) by Version + - 12.8.0: - - Ubuntu 22.04: `runpod/base:1.0.2-cuda1280-ubuntu2204` - - Ubuntu 24.04: `runpod/base:1.0.2-cuda1280-ubuntu2404` + - Ubuntu 22.04: `runpod/base:1.0.2-cuda1280-ubuntu2204` + - Ubuntu 24.04: `runpod/base:1.0.2-cuda1280-ubuntu2404` - 12.8.1: - - Ubuntu 22.04: `runpod/base:1.0.2-cuda1281-ubuntu2204` - - Ubuntu 24.04: `runpod/base:1.0.2-cuda1281-ubuntu2404` + - Ubuntu 22.04: `runpod/base:1.0.2-cuda1281-ubuntu2204` + - Ubuntu 24.04: `runpod/base:1.0.2-cuda1281-ubuntu2404` - 12.9.0: - - Ubuntu 22.04: `runpod/base:1.0.2-cuda1290-ubuntu2204` - - Ubuntu 24.04: `runpod/base:1.0.2-cuda1290-ubuntu2404` + - Ubuntu 22.04: `runpod/base:1.0.2-cuda1290-ubuntu2204` + - Ubuntu 24.04: `runpod/base:1.0.2-cuda1290-ubuntu2404` - 13.0.0: - - Ubuntu 22.04: `runpod/base:1.0.2-cuda1300-ubuntu2204` - - Ubuntu 24.04: `runpod/base:1.0.2-cuda1300-ubuntu2404` + - Ubuntu 22.04: `runpod/base:1.0.2-cuda1300-ubuntu2204` + - Ubuntu 24.04: `runpod/base:1.0.2-cuda1300-ubuntu2404` + diff --git a/official-templates/nvidia-pytorch/README.md b/official-templates/nvidia-pytorch/README.md index 960bedae..7c45ab1c 100644 --- a/official-templates/nvidia-pytorch/README.md +++ b/official-templates/nvidia-pytorch/README.md @@ -1,8 +1,8 @@ # NVIDIA PyTorch Base Image -NVIDIA's PyTorch NGC Container (`nvcr.io/nvidia/pytorch`) built for easy deployment on Runpod. +NVIDIA's PyTorch NGC Container (`nvcr.io/nvidia/pytorch`) built for easy deployment on Runpod. -For more information on the NGC images visit https://catalog.ngc.nvidia.com/orgs/nvidia/containers/pytorch. +For more information on the NGC images visit . ## Deployment @@ -13,4 +13,3 @@ Please use `runpod/nvidia-pytorch:1.0.3-25.11` ```bash ./bake.sh nvidia-pytorch ``` - diff --git a/official-templates/pytorch/README.md b/official-templates/pytorch/README.md index 08026041..35ae5159 100644 --- a/official-templates/pytorch/README.md +++ b/official-templates/pytorch/README.md @@ -5,12 +5,14 @@ Built on our base images, these containers provide pre-configured PyTorch and CUDA combinations for immediate deep learning development. Skip the compatibility guesswork and setup time: just run, and start training. ### What's included + - **Version matched**: PyTorch and CUDA combinations tested for optimal compatibility. - **Zero setup**: PyTorch ready to import immediately, no additional installs required. - **GPU accelerated**: Full CUDA support enabled for immediate deep learning acceleration. - **Production ready**: Built on our stable base images with complete development toolchain. ### Available configurations + - **PyTorch**: 2.4.1, 2.5.0, 2.5.1, 2.6.0, 2.7.1, and 2.8.0 - **CUDA**: 12.4.1, 12.8.1, 12.9.0, and 13.0.0 (not available on Runpod) - **Ubuntu**: 22.04 (Jammy) and 24.04 (Noble) @@ -23,7 +25,8 @@ Please also see [../base/README.md](../base/README.md) ## Available PyTorch Images -### CUDA 12.8.1: +### CUDA 12.8.1 + - Torch 2.6.0: - Ubuntu 22.04: `runpod/pytorch:1.0.2-cu1281-torch260-ubuntu2204` - Ubuntu 24.04: `runpod/pytorch:1.0.2-cu1281-torch260-ubuntu2404` @@ -34,7 +37,8 @@ Please also see [../base/README.md](../base/README.md) - Ubuntu 22.04: `runpod/pytorch:1.0.2-cu1281-torch280-ubuntu2204` - Ubuntu 24.04: `runpod/pytorch:1.0.2-cu1281-torch280-ubuntu2404` -### CUDA 12.9.0: +### CUDA 12.9.0 + - Torch 2.6.0: - Ubuntu 22.04: `runpod/pytorch:1.0.2-cu1290-torch260-ubuntu2204` - Ubuntu 24.04: `runpod/pytorch:1.0.2-cu1290-torch260-ubuntu2404` @@ -45,7 +49,8 @@ Please also see [../base/README.md](../base/README.md) - Ubuntu 22.04: `runpod/pytorch:1.0.2-cu1290-torch280-ubuntu2204` - Ubuntu 24.04: `runpod/pytorch:1.0.2-cu1290-torch280-ubuntu2404` -### CUDA 13.0.0: +### CUDA 13.0.0 + - Torch 2.6.0: - Ubuntu 22.04: `runpod/pytorch:1.0.2-cu1300-torch260-ubuntu2204` - Ubuntu 24.04: `runpod/pytorch:1.0.2-cu1300-torch260-ubuntu2404` @@ -71,4 +76,4 @@ Please also see [../base/README.md](../base/README.md) - Ubuntu 20.04: `runpod/pytorch:0.7.0-cu1241-torch260-ubuntu2004` - Ubuntu 22.04: `runpod/pytorch:0.7.0-cu1241-torch260-ubuntu2204`
- \ No newline at end of file + diff --git a/official-templates/rocm/README.md b/official-templates/rocm/README.md index adef4f1a..20f15636 100644 --- a/official-templates/rocm/README.md +++ b/official-templates/rocm/README.md @@ -2,14 +2,16 @@ **AMD GPU-accelerated images for PyTorch workloads.** -Built on our base images, these containers provide AMD GPU acceleration through ROCm and PyTorch. +Built on our base images, these containers provide AMD GPU acceleration through ROCm and PyTorch. ### What's included + - **ROCm ready**: AMD GPU compute platform (6.4.4) preconfigured for immediate use. - **PyTorch accelerated**: Pre-installed and optimized for AMD GPU acceleration. - **Complete toolkit**: Full development environment from our base images with Jupyter, SSH, and ML libraries. ### Available configurations + - **ROCm**: 6.4.4 - **PyTorch**: 2.5.1, 2.6.0, and 2.7.1 - **Ubuntu**: 22.04 (Jammy) and 24.04 (Noble) @@ -22,15 +24,17 @@ Perfect for AMD GPU workloads with zero setup time.
-### ROCm 6.4.4 Images: +### ROCm 6.4.4 Images -**Ubuntu 22.04 (Python 3.10)** +**Ubuntu 22.04 (Python 3.10)** The venv must be activated with: `conda init && source ~/.bashrc && conda activate py_3.10` + - PyTorch 2.5.1: `runpod/base:1.0.2-rocm644-ubuntu2204-py310-pytorch251` - PyTorch 2.6.0: `runpod/base:1.0.2-rocm644-ubuntu2204-py310-pytorch260` -**Ubuntu 24.04 (Python 3.12)** +**Ubuntu 24.04 (Python 3.12)** The venv must be activated with: `conda init && source ~/.bashrc && conda activate py_3.12` + - PyTorch 2.6.0: `runpod/base:1.0.2-rocm644-ubuntu2404-py312-pytorch260` - PyTorch 2.7.1: `runpod/base:1.0.2-rocm644-ubuntu2404-py312-pytorch271` diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..cfd7dc6d --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,18 @@ +# Minimal project config to give the Nix static-analysis tools sane, +# pragmatic defaults. Not a packaging manifest. + +[tool.ruff] +line-length = 100 +target-version = "py310" + +[tool.ruff.lint] +# Default rule set (E, F, plus ruff's defaults). Add rules here to +# tighten over time. +select = ["E", "F", "W", "I", "UP", "B"] +ignore = [ + "E501", # line length handled by the formatter; not worth failing on +] + +[tool.mypy] +ignore_missing_imports = true +warn_unused_ignores = true diff --git a/tests/README.md b/tests/README.md index 40786b96..8fe7cf99 100644 --- a/tests/README.md +++ b/tests/README.md @@ -8,13 +8,13 @@ appear on a real GPU host** and that local `docker run` would miss: driver-version mismatches, broken NCCL/NVRTC, missing CUDA libs, `start.sh` regressions, JupyterLab proxy misconfiguration, etc. -``` +```text ./test_images.py [path/to/images.yaml] [group_filter] ``` The code is split into a small package next to the entry point: -``` +```text tests/ ├── README.md ← you are here ├── test_images.py ← entry point: main() + summary + CLI @@ -29,7 +29,6 @@ tests/ └── runner.py ← test_pair / test_image (per-image orchestration) ``` - ## Prerequisites 1. **Python ≥ 3.9** (stdlib only — no pip install needed). @@ -57,10 +56,9 @@ tests/ auth, parallel runs in particular will produce a wave of "image pull backoff" failures that look like image bugs but aren't. The script auto-discovers the first entry from `runpodctl registry - list`; pin a specific one with `REGISTRY_AUTH_ID` or +list`; pin a specific one with `REGISTRY_AUTH_ID` or `REGISTRY_AUTH_NAME`. - ## Quick start Smallest possible manifest — single CPU image: @@ -68,7 +66,7 @@ Smallest possible manifest — single CPU image: ```yaml # images-quickstart.yaml base_cpu: - images: + images: - runpod/base:1.0.6-dev-ubuntu2404 ``` @@ -103,54 +101,52 @@ To test a single group from a larger manifest: ./test_images.py images.yaml base_cpu ``` - ## Test lifecycle For every `(image, instance)` pair the manifest produces, the script runs this sequence and reports the outcome as soon as one step fails. -| # | Step | Failure → | -|---|------|---| -| 1 | `runpodctl pod create` (with `--gpu-id`, `--container-disk-in-gb`, `--ports`, registry auth, optional `--min-cuda-version`). Transient `5xx` / `Something went wrong` errors are retried silently up to `CREATE_RETRIES` with linear backoff. | `UNAVAILABLE` (no capacity for this instance type — try next) / `CREATE_FAIL` (bad image tag, registry auth, malformed request — any non-capacity, non-transient orchestrator error after retries are exhausted) | -| 2 | Poll `runpodctl pod get` until `ssh.ip` / `ssh.port` are assigned and one-shot `ssh root@ip -p port 'echo ready'` succeeds (the real readiness signal — `desiredStatus` is always `RUNNING` after create) | `STUCK` if no SSH endpoint within `CREATE_TIMEOUT` (almost always a bad host in the scheduler pool — try another instance type) | -| 3 | **CUDA functional check** over SSH — see [Functional check](#functional-check). Image-driven: pytorch ref → `torch.cuda` + matmul; cuda/rocm ref → `nvidia-smi` + `nvcc`; neither → skip | `FAIL` (image is broken — stop iterating; another GPU won't help) | -| 4 | **JupyterLab in-pod check** (only when `test_jupyter: true`) — see [Jupyter check](#jupyter-check-opt-in). SSH in, wait for `:8888` to bind, `jupyter server list`, `curl /api/status` with token | `FAIL` (`start.sh` didn't bring up Jupyter — usually wrong python interpreter) | -| 5 | **JupyterLab public-proxy check** (only when `test_jupyter: true`) — `GET https://-8888.proxy.runpod.net/api/status` from the test machine | `FAIL` (port not exposed as `8888/http`, or proxy never registered) | -| 6 | Sleep `DWELL_SEC`, re-probe SSH (catches "boots fine then crashes after 30s") | `FAIL` if SSH stops responding | -| 7 | `dump_pod_logs` — pull `uname`, `syslog`, `dmesg`, `/var/log/*.log`, `nvidia-smi` via SSH for the run log | _(diagnostic only)_ | -| 8 | `runpodctl pod delete` (always — even on Ctrl-C / exception via `atexit` + signal handlers) | _(diagnostic only)_ | +| # | Step | Failure → | +| --- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| 1 | `runpodctl pod create` (with `--gpu-id`, `--container-disk-in-gb`, `--ports`, registry auth, optional `--min-cuda-version`). Transient `5xx` / `Something went wrong` errors are retried silently up to `CREATE_RETRIES` with linear backoff. | `UNAVAILABLE` (no capacity for this instance type — try next) / `CREATE_FAIL` (bad image tag, registry auth, malformed request — any non-capacity, non-transient orchestrator error after retries are exhausted) | +| 2 | Poll `runpodctl pod get` until `ssh.ip` / `ssh.port` are assigned and one-shot `ssh root@ip -p port 'echo ready'` succeeds (the real readiness signal — `desiredStatus` is always `RUNNING` after create) | `STUCK` if no SSH endpoint within `CREATE_TIMEOUT` (almost always a bad host in the scheduler pool — try another instance type) | +| 3 | **CUDA functional check** over SSH — see [Functional check](#functional-check). Image-driven: pytorch ref → `torch.cuda` + matmul; cuda/rocm ref → `nvidia-smi` + `nvcc`; neither → skip | `FAIL` (image is broken — stop iterating; another GPU won't help) | +| 4 | **JupyterLab in-pod check** (only when `test_jupyter: true`) — see [Jupyter check](#jupyter-check-opt-in-via-manifest-test_jupyter-true). SSH in, wait for `:8888` to bind, `jupyter server list`, `curl /api/status` with token | `FAIL` (`start.sh` didn't bring up Jupyter — usually wrong python interpreter) | +| 5 | **JupyterLab public-proxy check** (only when `test_jupyter: true`) — `GET https://-8888.proxy.runpod.net/api/status` from the test machine | `FAIL` (port not exposed as `8888/http`, or proxy never registered) | +| 6 | Sleep `DWELL_SEC`, re-probe SSH (catches "boots fine then crashes after 30s") | `FAIL` if SSH stops responding | +| 7 | `dump_pod_logs` — pull `uname`, `syslog`, `dmesg`, `/var/log/*.log`, `nvidia-smi` via SSH for the run log | _(diagnostic only)_ | +| 8 | `runpodctl pod delete` (always — even on Ctrl-C / exception via `atexit` + signal handlers) | _(diagnostic only)_ | `test_image()` then iterates over the next instance candidate when the result was `UNAVAILABLE` or `STUCK`, and short-circuits on `PASS`, `FAIL`, or `CREATE_FAIL`. - ## Outcomes The summary at the end of every run groups results into three buckets. The granular per-pod outcomes below collapse into them: -| summary | per-pod outcome | what it means | what to do | -|---|---|---|---| -| `PASS` | `PASS` | Image booted, all checks passed, survived dwell. | nothing | -| `FAIL` | `FAIL` | Pod was created and the container itself proved broken (CUDA check failed, JupyterLab didn't start, crashed during dwell, etc.). Moving to another GPU won't help — the image is the problem. | fix the image | -| `FAIL` | `CREATE_FAIL` | Pod-create returned a non-capacity, non-transient orchestrator error (bad image tag, registry auth, malformed request, missing CUDA version). | fix the manifest / image ref / auth | -| `SKIP` | all `UNAVAILABLE` | RunPod had no capacity on **any** candidate instance type. | retry later, expand `instances:` list, or raise `max_price_per_hour` | -| `SKIP` | some `STUCK` + rest `UNAVAILABLE` | At least one instance was scheduled but RunPod never assigned an SSH endpoint within `CREATE_TIMEOUT` (slow pull / dead host). | retry later — usually transient | +| summary | per-pod outcome | what it means | what to do | +| ------- | --------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------- | +| `PASS` | `PASS` | Image booted, all checks passed, survived dwell. | nothing | +| `FAIL` | `FAIL` | Pod was created and the container itself proved broken (CUDA check failed, JupyterLab didn't start, crashed during dwell, etc.). Moving to another GPU won't help — the image is the problem. | fix the image | +| `FAIL` | `CREATE_FAIL` | Pod-create returned a non-capacity, non-transient orchestrator error (bad image tag, registry auth, malformed request, missing CUDA version). | fix the manifest / image ref / auth | +| `SKIP` | all `UNAVAILABLE` | RunPod had no capacity on **any** candidate instance type. | retry later, expand `instances:` list, or raise `max_price_per_hour` | +| `SKIP` | some `STUCK` + rest `UNAVAILABLE` | At least one instance was scheduled but RunPod never assigned an SSH endpoint within `CREATE_TIMEOUT` (slow pull / dead host). | retry later — usually transient | `FAIL` always exits `1`. `SKIP` is governed by `ON_SKIP` (env-var) / `on-skip` (CI input), one of: -* `fail` (default) — exit `1` + `::error::` annotation. Job goes red. -* `warn` — exit `0` + `::warning::` annotation. Job stays +- `fail` (default) — exit `1` + `::error::` annotation. Job goes red. +- `warn` — exit `0` + `::warning::` annotation. Job stays green; the run shows a yellow warning bubble in the PR check tab. -* `pass` — exit `0`, no annotation (legacy lenient mode). +- `pass` — exit `0`, no annotation (legacy lenient mode). Unknown values silently coerce to `fail` so a typo never disables the safer default. The summary lists the **instance that produced the outcome** in brackets, so you can tell whether a `FAIL` correlates with a specific GPU type: -``` +```text ================================== SUMMARY ================================= totals: 4 PASS, 1 FAIL, 1 SKIP @@ -162,7 +158,6 @@ totals: 4 PASS, 1 FAIL, 1 SKIP PASS runpod/base:…cuda1300-ubuntu2404 [RTX 4090] ``` - ## Common invocations ```bash @@ -203,36 +198,35 @@ every pod the script created. Any pod the script missed will still self-terminate within ~2 h via the `--terminate-after` clause set on every `pod create`. - ## Manifest schema ```yaml groupname: - images: # list of docker images to test (required) + images: # list of docker images to test (required) - imagename - instances: # explicit list of GPU display names, priority order + instances: # explicit list of GPU display names, priority order - "RTX A4000" - max_price_per_hour: 1.0 # OR budget filter — auto-pick cheapest first - min_vram_gb: 16 # extra filter for budget mode - manufacturer: Nvidia # 'Nvidia' or 'AMD' filter for budget mode - exclude_instances: # subtract fnmatch patterns from candidates + max_price_per_hour: 1.0 # OR budget filter — auto-pick cheapest first + min_vram_gb: 16 # extra filter for budget mode + manufacturer: Nvidia # 'Nvidia' or 'AMD' filter for budget mode + exclude_instances: # subtract fnmatch patterns from candidates - "*Blackwell*" - min_cuda_version: "13.0" # 'X.Y' string for --min-cuda-version (fallback only) - test_jupyter: true # opt-in JupyterLab in-pod + proxy check + min_cuda_version: "13.0" # 'X.Y' string for --min-cuda-version (fallback only) + test_jupyter: true # opt-in JupyterLab in-pod + proxy check ``` Field reference: -| field | description | -|---|---| -| `images` | Docker images to test. **Required.** | -| `instances` | Explicit list of GPU display names, tried in order. One of `instances:` or `max_price_per_hour:` is required (except for `base_cpu`). | -| `max_price_per_hour` | USD/hr budget — auto-pick any GPU at this price or below, cheapest first. Loses to explicit `instances:` if both are set. | -| `min_vram_gb` | Extra filter for budget mode (default 0). | -| `manufacturer` | `Nvidia` or `AMD` filter for budget mode (default: any). | -| `exclude_instances` | fnmatch-style patterns (case-insensitive) subtracted from the candidate list AFTER `instances:` or budget selection. Useful for blocking known-bad host pairings without rewriting the whole list — e.g. `"*Blackwell*"` skips every Blackwell GPU (sm\_100 / sm\_120 are not in the kernel set of PyTorch ≤ 2.6 wheels). | -| `min_cuda_version` | `X.Y` string passed to `runpodctl pod create --min-cuda-version`. Only used as a **fallback** when the image tag itself doesn't encode a CUDA version (e.g. NGC `nvidia-pytorch:25.11`). Image tags like `cu1281` / `cuda1281` always win. | -| `test_jupyter` | `true` / `false` — when true, the pod is created with `JUPYTER_PASSWORD=admin` in env and HTTP port 8888 exposed, then the script SSHes in and verifies JupyterLab is actually listening. Use for groups whose images use `container-template/start.sh` (`runpod/base`, `runpod/pytorch`, `runpod/autoresearch`, `rocm`). Skip for NGC `nvidia-pytorch` (different entrypoint). Default: `false`. | +| field | description | +| -------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `images` | Docker images to test. **Required.** | +| `instances` | Explicit list of GPU display names, tried in order. One of `instances:` or `max_price_per_hour:` is required (except for `base_cpu`). | +| `max_price_per_hour` | USD/hr budget — auto-pick any GPU at this price or below, cheapest first. Loses to explicit `instances:` if both are set. | +| `min_vram_gb` | Extra filter for budget mode (default 0). | +| `manufacturer` | `Nvidia` or `AMD` filter for budget mode (default: any). | +| `exclude_instances` | fnmatch-style patterns (case-insensitive) subtracted from the candidate list AFTER `instances:` or budget selection. Useful for blocking known-bad host pairings without rewriting the whole list — e.g. `"*Blackwell*"` skips every Blackwell GPU (sm_100 / sm_120 are not in the kernel set of PyTorch ≤ 2.6 wheels). | +| `min_cuda_version` | `X.Y` string passed to `runpodctl pod create --min-cuda-version`. Only used as a **fallback** when the image tag itself doesn't encode a CUDA version (e.g. NGC `nvidia-pytorch:25.11`). Image tags like `cu1281` / `cuda1281` always win. | +| `test_jupyter` | `true` / `false` — when true, the pod is created with `JUPYTER_PASSWORD=admin` in env and HTTP port 8888 exposed, then the script SSHes in and verifies JupyterLab is actually listening. Use for groups whose images use `container-template/start.sh` (`runpod/base`, `runpod/pytorch`, `runpod/autoresearch`, `rocm`). Skip for NGC `nvidia-pytorch` (different entrypoint). Default: `false`. | The `base_cpu` group is special: `runpodctl` 2.3.0 does not let us pick a specific CPU flavor (`--gpu-id` is rejected for `--compute-type CPU`), @@ -240,7 +234,6 @@ so the manifest needs ONLY an `images:` list for that group — no `instances:` / `max_price_per_hour:` / `min_vram_gb:`. RunPod picks a CPU flavor for us. - ## Example manifest (the real one used in this repo) Lives outside the repo at `~/tmp/runpod-scripts/testing/images` — the @@ -250,97 +243,95 @@ you're testing). Annotated example covering every supported pattern: ```yaml # CPU-only base image — no instances:, no budget, just images. base_cpu: - images: + images: - runpod/base:1.0.6-dev-ubuntu2204 - runpod/base:1.0.6-dev-ubuntu2404 - test_jupyter: true # base CPU image still ships JupyterLab + test_jupyter: true # base CPU image still ships JupyterLab # GPU base image, budget-selected. The CUDA functional check is auto- # applied because the tag contains 'cuda1281' / 'cuda1290' / 'cuda1300'. base_gpu: - images: + images: - runpod/base:1.0.6-dev-cuda1281-ubuntu2204 - runpod/base:1.0.6-dev-cuda1281-ubuntu2404 - runpod/base:1.0.6-dev-cuda1300-ubuntu2404 - max_price_per_hour: 1.0 - min_vram_gb: 16 - manufacturer: Nvidia - test_jupyter: true + max_price_per_hour: 1.0 + min_vram_gb: 16 + manufacturer: Nvidia + test_jupyter: true # autoresearch — torch lives in /opt/autoresearch/.venv, NOT importable # from system python. The image-driven check picks nvidia-smi (because # tag has 'cuda' but no 'pytorch' / 'torch\d' marker), which is what # we want — we'd never get a clean torch import over SSH otherwise. autoresearch: - images: + images: - runpod/autoresearch:1.0.6-dev-cuda1281-ubuntu2204 - runpod/autoresearch:1.0.6-dev-cuda1281-ubuntu2404 - max_price_per_hour: 1.0 - min_vram_gb: 16 - manufacturer: Nvidia - test_jupyter: true + max_price_per_hour: 1.0 + min_vram_gb: 16 + manufacturer: Nvidia + test_jupyter: true # NGC base image. Tag '25.11' encodes no CUDA version — without # min_cuda_version the scheduler picks any host and the container # fails at startup with `nvidia-container-cli: cuda>=13.0`. nvidia-pytorch: - images: + images: - runpod/nvidia-pytorch:1.0.6-dev-25.11 - max_price_per_hour: 1.0 - min_vram_gb: 16 - manufacturer: Nvidia - min_cuda_version: "13.0" # NGC 25.09+ PyTorch is built on cu13.0 - # No test_jupyter — NGC uses its own entrypoint, not our start.sh. + max_price_per_hour: 1.0 + min_vram_gb: 16 + manufacturer: Nvidia + min_cuda_version: "13.0" # NGC 25.09+ PyTorch is built on cu13.0 + # No test_jupyter — NGC uses its own entrypoint, not our start.sh. # AMD ROCm — explicit instance list because only MI300X carries ROCm. rocm: - images: + images: - runpod/base:1.0.6-dev-rocm644-ubuntu2204-py310-pytorch251 - runpod/base:1.0.6-dev-rocm644-ubuntu2404-py312-pytorch271 - instances: + instances: - MI300X - test_jupyter: true + test_jupyter: true # runpod/pytorch — torch in system python, full torch.cuda check runs. # PyTorch ≤ 2.6 wheels ship kernels only for sm_50…sm_90; Blackwell GPUs # are sm_100/sm_120, so booting on one of them gives "no kernel image # is available for execution on the device". Filter them out: pytorch: - images: + images: - runpod/pytorch:1.0.6-dev-cu1281-torch260-ubuntu2204 - runpod/pytorch:1.0.6-dev-cu1300-torch260-ubuntu2404 - max_price_per_hour: 1.0 - min_vram_gb: 16 - manufacturer: Nvidia - test_jupyter: true - exclude_instances: + max_price_per_hour: 1.0 + min_vram_gb: 16 + manufacturer: Nvidia + test_jupyter: true + exclude_instances: - "*Blackwell*" ``` - ## Environment variables -| var | default | description | -|---|---|---| -| `CLOUD_TYPE` | `SECURE` | `SECURE` or `COMMUNITY`. | -| `DISK_GB` | `100` | Container disk size for GPU pods. | -| `CPU_DISK_GB` | `20` | Container disk size for CPU pods. RunPod caps this per CPU flavor (20 GB on the cheapest, 30 GB on larger ones); 20 is the universal safe value. | -| `CPU_CANDIDATES` | `""` (uses `cpu-secure,cpu-community`) | CPU "instance candidates". `runpodctl pod create` doesn't accept `--vcpu` / `--mem` / `--cpu-flavor`, so we vary the axes it DOES expose for CPU: `--cloud-type` (SECURE vs COMMUNITY) and optional `--data-center-ids`. Each label becomes one candidate iterated by the same per-instance retry loop GPU groups use, so when SECURE is saturated COMMUNITY almost always has free CPU capacity. Format: `label:CLOUD[:DC1+DC2+…],label:CLOUD[:DC_CSV],…` (use `+` not `,` to separate DC ids inside one candidate so the outer csv stays unambiguous). CLOUD must be SECURE or COMMUNITY. Malformed entries are silently dropped; an empty/all-broken value falls back to the default 2-candidate list. | -| `RUNPOD_API_KEY` | _(from `~/.runpod/config.toml`)_ | Used for the GraphQL GPU pricing query. Set this in CI / containers without a config file. | -| `REGISTRY_AUTH_ID` | _(empty)_ | Explicit Docker Hub registry auth id to pass as `--registry-auth-id`. Overrides auto-discovery. | -| `REGISTRY_AUTH_NAME` | _(empty)_ | Display name to look up via `runpodctl registry list` when `REGISTRY_AUTH_ID` is not set. Falls back to the first entry. | -| `DWELL_SEC` | `60` | Extra seconds to wait after SSH becomes reachable, then re-probe SSH to catch containers that boot, accept SSH, then crash. Set 0 to skip the re-probe. | -| `CREATE_TIMEOUT` | `600` | Max seconds to wait for SSH to become reachable. Raise for ROCm workflows (`create-timeout: "1200"` on the action) — the official `rocm/pytorch:*` base images are 30-50GB and routinely take 8-15 minutes to pull. | -| `POLL_INTERVAL` | `10` | Poll cadence for SSH probes. | -| `MAX_PARALLEL` | `1` | How many images to smoke-test concurrently. Each worker holds at most one pod, so this caps simultaneous live pods. Keep modest to avoid RunPod rate limits and surprise bills. | -| `CREATE_RETRIES` | `3` | Retry pod-create up to N times on transient RunPod 5xx errors (`Something went wrong`, 502/503). Capacity shortages are NOT retried. | -| `CREATE_RETRY_BACKOFF` | `10` | Seconds between retries (linear backoff). | -| `STALL_HINT_AFTER` | `180` | Seconds without an SSH endpoint before the script prints a hint about slow pulls / possible Docker Hub rate limit. | -| `SSH_LOG_FETCH` | `1` | `1`/`0` — fetch container logs via direct SSH at PASS/FAIL. | -| `RUNPOD_SSH_KEY` | _(empty)_ | Path to private key matching the `PUBLIC_KEY` `runpodctl` injects into pods. Auto-discovered from common locations if not set. | -| `JUPYTER_WAIT_TIMEOUT` | `30` | Seconds the in-pod Jupyter probe waits for `:8888` to bind. | -| `JUPYTER_PROXY_TIMEOUT` | `60` | Seconds the proxy probe retries while RunPod's ingress registers the new pod. | - +| var | default | description | +| ----------------------- | -------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `CLOUD_TYPE` | `SECURE` | `SECURE` or `COMMUNITY`. | +| `DISK_GB` | `100` | Container disk size for GPU pods. | +| `CPU_DISK_GB` | `20` | Container disk size for CPU pods. RunPod caps this per CPU flavor (20 GB on the cheapest, 30 GB on larger ones); 20 is the universal safe value. | +| `CPU_CANDIDATES` | `""` (uses `cpu-secure,cpu-community`) | CPU "instance candidates". `runpodctl pod create` doesn't accept `--vcpu` / `--mem` / `--cpu-flavor`, so we vary the axes it DOES expose for CPU: `--cloud-type` (SECURE vs COMMUNITY) and optional `--data-center-ids`. Each label becomes one candidate iterated by the same per-instance retry loop GPU groups use, so when SECURE is saturated COMMUNITY almost always has free CPU capacity. Format: `label:CLOUD[:DC1+DC2+…],label:CLOUD[:DC_CSV],…` (use `+` not `,` to separate DC ids inside one candidate so the outer csv stays unambiguous). CLOUD must be SECURE or COMMUNITY. Malformed entries are silently dropped; an empty/all-broken value falls back to the default 2-candidate list. | +| `RUNPOD_API_KEY` | _(from `~/.runpod/config.toml`)_ | Used for the GraphQL GPU pricing query. Set this in CI / containers without a config file. | +| `REGISTRY_AUTH_ID` | _(empty)_ | Explicit Docker Hub registry auth id to pass as `--registry-auth-id`. Overrides auto-discovery. | +| `REGISTRY_AUTH_NAME` | _(empty)_ | Display name to look up via `runpodctl registry list` when `REGISTRY_AUTH_ID` is not set. Falls back to the first entry. | +| `DWELL_SEC` | `60` | Extra seconds to wait after SSH becomes reachable, then re-probe SSH to catch containers that boot, accept SSH, then crash. Set 0 to skip the re-probe. | +| `CREATE_TIMEOUT` | `600` | Max seconds to wait for SSH to become reachable. Raise for ROCm workflows (`create-timeout: "1200"` on the action) — the official `rocm/pytorch:*` base images are 30-50GB and routinely take 8-15 minutes to pull. | +| `POLL_INTERVAL` | `10` | Poll cadence for SSH probes. | +| `MAX_PARALLEL` | `1` | How many images to smoke-test concurrently. Each worker holds at most one pod, so this caps simultaneous live pods. Keep modest to avoid RunPod rate limits and surprise bills. | +| `CREATE_RETRIES` | `3` | Retry pod-create up to N times on transient RunPod 5xx errors (`Something went wrong`, 502/503). Capacity shortages are NOT retried. | +| `CREATE_RETRY_BACKOFF` | `10` | Seconds between retries (linear backoff). | +| `STALL_HINT_AFTER` | `180` | Seconds without an SSH endpoint before the script prints a hint about slow pulls / possible Docker Hub rate limit. | +| `SSH_LOG_FETCH` | `1` | `1`/`0` — fetch container logs via direct SSH at PASS/FAIL. | +| `RUNPOD_SSH_KEY` | _(empty)_ | Path to private key matching the `PUBLIC_KEY` `runpodctl` injects into pods. Auto-discovered from common locations if not set. | +| `JUPYTER_WAIT_TIMEOUT` | `30` | Seconds the in-pod Jupyter probe waits for `:8888` to bind. | +| `JUPYTER_PROXY_TIMEOUT` | `60` | Seconds the proxy probe retries while RunPod's ingress registers the new pod. | ## Functional check @@ -365,24 +356,22 @@ groups don't silently skip the check: - otherwise (no GPU markers) → no check. Pod must still boot and survive `DWELL_SEC`. - ## Jupyter check (opt-in via manifest `test_jupyter: true`) Two stages, both must pass: 1. **In-pod.** SSH into the pod and `curl http://127.0.0.1:8888/api/status` with our token. Catches silent `start.sh` failures (e.g. `python3 -m - jupyter` not finding the module on Ubuntu 22.04 — the kind of bug +jupyter` not finding the module on Ubuntu 22.04 — the kind of bug that prints `Jupyter Lab started` in the container log while no server is actually running). 2. **Public proxy.** From the test machine, `GET - https://-8888.proxy.runpod.net/api/status` with the token. +https://-8888.proxy.runpod.net/api/status` with the token. Catches port-type misconfigurations (`8888/tcp` instead of `8888/http` — the proxy never wires up non-http ports) and DNS / proxy registration issues that would prevent real users from reaching Jupyter from the RunPod console. - ## Running in CI The composite action at @@ -409,7 +398,7 @@ Typical caller (from a per-image-family build workflow): - uses: ./.github/actions/smoke-test with: image-refs: ${{ toJSON(steps.bake.outputs.image-refs) }} - profile: gpu # base = split CPU/GPU (only for runpod/base) | gpu = single base_gpu group (everything else) + profile: gpu # base = split CPU/GPU (only for runpod/base) | gpu = single base_gpu group (everything else) runpod-api-key: ${{ secrets.RUNPOD_API_KEY }} ssh-private-key: ${{ secrets.RUNPOD_SSH_KEY }} budget-usd-per-hour: "1.0" @@ -424,25 +413,23 @@ Typical caller (from a per-image-family build workflow): The full input reference lives in the action's own `description:` fields. - ## Troubleshooting -| symptom in logs | likely cause | fix | -|---|---|---| -| `runpodctl not found in PATH` | `runpodctl` binary missing | install from , put on `$PATH` | -| `runpodctl is not authenticated. Run 'runpodctl doctor'` | API key not configured or expired | `runpodctl config --apiKey ` | -| `warn: no GPU pricing data` | `RUNPOD_API_KEY` not set and no `~/.runpod/config.toml` | set `RUNPOD_API_KEY` or run `runpodctl config --apiKey` | -| `warn: no registry auth configured` | no Docker Hub auth registered | `runpodctl registry add` (paid Hub account strongly recommended for parallel runs) | -| every group says `no capacity on any of N candidate instance type(s)` | budget too low / VRAM too high / region saturated | raise `max_price_per_hour`, drop `min_vram_gb`, or set explicit `instances:` | -| only the `base_cpu` group says `no capacity` while GPU groups pass | the cloud(s) you target don't have CPU capacity right now | by default we already try SECURE then COMMUNITY. If both are full, add DC-pinned candidates: `CPU_CANDIDATES="cpu-secure:SECURE,cpu-community:COMMUNITY,cpu-eu:COMMUNITY:EU-RO-1+EU-NL-1,cpu-us:COMMUNITY:US-OR-1"` | -| pod stays in `ssh endpoint not assigned yet` past `STALL_HINT_AFTER` | slow image pull or Docker Hub `toomanyrequests` | add registry auth, reduce `MAX_PARALLEL`, or wait 6 h for the Hub rate limit to reset | -| `ssh_probe=FAIL — Permission denied (publickey)` | wrong SSH key | export `RUNPOD_SSH_KEY=/path/to/private/key` whose public half is on the RunPod account | -| `pod entered TIMEOUT state` repeatedly on Blackwell GPUs for a `pytorch` group | PyTorch ≤ 2.6 has no `sm_100`/`sm_120` kernels | add `exclude_instances: ["*Blackwell*"]` to the group | -| `nvidia-container-cli: requirement error: unsatisfied condition: cuda>=X.Y` in pod logs | image needs a newer driver than the host has | set `min_cuda_version: "X.Y"` in the manifest (only needed for tags without a `cuXYZW`/`cudaXYZW` marker) | -| `jupyter check (in-pod) FAILED -- start.sh did not bring up JupyterLab` | `start.sh` is launching Jupyter with the wrong Python interpreter (classic Ubuntu 22.04 `python3` → 3.10 vs `python` → 3.12) | fix `container-template/start.sh` to use `python -m jupyter lab` | -| `jupyter check (public proxy) FAILED` but in-pod check passed | port exposed as `8888/tcp` instead of `8888/http`, OR proxy hasn't registered the pod yet | check `pod create --ports` arg; bump `JUPYTER_PROXY_TIMEOUT` if proxy is just slow | -| script hangs at `Cleaning up N leftover pod(s)…` | RunPod API is slow to respond to delete | wait it out; `--terminate-after` (~2 h) is the backstop and will kill anything we missed | - +| symptom in logs | likely cause | fix | +| --------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `runpodctl not found in PATH` | `runpodctl` binary missing | install from , put on `$PATH` | +| `runpodctl is not authenticated. Run 'runpodctl doctor'` | API key not configured or expired | `runpodctl config --apiKey ` | +| `warn: no GPU pricing data` | `RUNPOD_API_KEY` not set and no `~/.runpod/config.toml` | set `RUNPOD_API_KEY` or run `runpodctl config --apiKey` | +| `warn: no registry auth configured` | no Docker Hub auth registered | `runpodctl registry add` (paid Hub account strongly recommended for parallel runs) | +| every group says `no capacity on any of N candidate instance type(s)` | budget too low / VRAM too high / region saturated | raise `max_price_per_hour`, drop `min_vram_gb`, or set explicit `instances:` | +| only the `base_cpu` group says `no capacity` while GPU groups pass | the cloud(s) you target don't have CPU capacity right now | by default we already try SECURE then COMMUNITY. If both are full, add DC-pinned candidates: `CPU_CANDIDATES="cpu-secure:SECURE,cpu-community:COMMUNITY,cpu-eu:COMMUNITY:EU-RO-1+EU-NL-1,cpu-us:COMMUNITY:US-OR-1"` | +| pod stays in `ssh endpoint not assigned yet` past `STALL_HINT_AFTER` | slow image pull or Docker Hub `toomanyrequests` | add registry auth, reduce `MAX_PARALLEL`, or wait 6 h for the Hub rate limit to reset | +| `ssh_probe=FAIL — Permission denied (publickey)` | wrong SSH key | export `RUNPOD_SSH_KEY=/path/to/private/key` whose public half is on the RunPod account | +| `pod entered TIMEOUT state` repeatedly on Blackwell GPUs for a `pytorch` group | PyTorch ≤ 2.6 has no `sm_100`/`sm_120` kernels | add `exclude_instances: ["*Blackwell*"]` to the group | +| `nvidia-container-cli: requirement error: unsatisfied condition: cuda>=X.Y` in pod logs | image needs a newer driver than the host has | set `min_cuda_version: "X.Y"` in the manifest (only needed for tags without a `cuXYZW`/`cudaXYZW` marker) | +| `jupyter check (in-pod) FAILED -- start.sh did not bring up JupyterLab` | `start.sh` is launching Jupyter with the wrong Python interpreter (classic Ubuntu 22.04 `python3` → 3.10 vs `python` → 3.12) | fix `container-template/start.sh` to use `python -m jupyter lab` | +| `jupyter check (public proxy) FAILED` but in-pod check passed | port exposed as `8888/tcp` instead of `8888/http`, OR proxy hasn't registered the pod yet | check `pod create --ports` arg; bump `JUPYTER_PROXY_TIMEOUT` if proxy is just slow | +| script hangs at `Cleaning up N leftover pod(s)…` | RunPod API is slow to respond to delete | wait it out; `--terminate-after` (~2 h) is the backstop and will kill anything we missed | ## Exit code @@ -455,8 +442,8 @@ capacity on every candidate, or every candidate landed on a stuck host) — that's effectively zero validation, so the default is strict. Override with: -* `ON_SKIP=warn` to keep the job green but get a GitHub Actions warning +- `ON_SKIP=warn` to keep the job green but get a GitHub Actions warning annotation in the PR check tab (visible signal without blocking the PR). -* `ON_SKIP=pass` to fully suppress the signal (no annotation at all). +- `ON_SKIP=pass` to fully suppress the signal (no annotation at all). Unknown values silently coerce to `fail`.