From 87d308dbb9a7f2c8bbd7389c97214a78bd6acd56 Mon Sep 17 00:00:00 2001 From: Matt Wozniski Date: Wed, 21 Oct 2026 18:24:10 -0700 Subject: [PATCH] Use python-build-standalone interpreters Instead of building Python interpreters ourselves, use the ones provided by the python-build-standalone project, installed via uv. Also, pre-compile `.pyc` files for each Python version we install. This makes snekbox start up about 80% faster, at the tradeoff of a modest increase in image size. --- .github/CONTRIBUTING.md | 9 +++++--- Dockerfile.pydis | 15 ++++++++----- LICENSE-THIRD-PARTY | 26 ---------------------- README.md | 2 +- scripts/build_python.sh | 37 ------------------------------- scripts/install_python.sh | 46 +++++++++++++++++++++++++++++++++++++++ tests/test_nsjail.py | 2 +- 7 files changed, 63 insertions(+), 74 deletions(-) delete mode 100755 scripts/build_python.sh create mode 100755 scripts/install_python.sh diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 67ad71b4..2534eb35 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -64,12 +64,15 @@ Other things to look out for are breaking changes to NsJail's config format, its ## Adding and Updating Python Interpreters -Python interpreters are built using pyenv via the `scripts/build_python.sh` helper script. This script accepts a pyenv version specifier (`pyenv install --list`) and builds the interpreter in a version-specific directory under `/snekbin/python`. In the image, each minor version of a Python interpreter should have its own build stage and the resulting `/snekbin/python` directory can be copied from that stage into the `base` stage. +Python interpreters are installed using [uv] via the `scripts/install_python.sh` helper script. This script installs a prebuilt [python-build-standalone] interpreter into a version-specific directory under `/snekbin/python`. It accepts a minor version (e.g. `3.14`), optionally followed by `t` for a free-threaded build or `j` for a build with the JIT enabled, and installs the latest patch release of that version known to uv. -When updating a patch version (e.g. 3.11.3 to 3.11.4), edit the existing build stage in the image for the minor version (3.11); do not add a new build stage. To have access to a new version, pyenv likely needs to be updated. To do so, change the tag in the `git clone` command in the image, but only for the build stage that needs access to the new version. Updating pyenv for all build stages will just cause unnecessary build cache invalidations. +[Dockerfile.pydis] uses the latest release of the `ghcr.io/astral-sh/uv` image, so new patch releases are picked up automatically once uv knows about them. Run `docker pull ghcr.io/astral-sh/uv:latest` before building to make sure a stale local copy of the uv image isn't used. -To change the default interpreter used by NsJail, update the target of the `/snekbin/python/default` symlink created in the `base` stage. +By default, NsJail uses the system Python of the `base` stage. [Dockerfile.pydis] changes the default by updating the target of the `/snekbin/python/default` symlink. [readme]: ../README.md [Dockerfile]: ../Dockerfile +[Dockerfile.pydis]: ../Dockerfile.pydis +[uv]: https://docs.astral.sh/uv/ +[python-build-standalone]: https://github.com/astral-sh/python-build-standalone [Compose v2]: https://docs.docker.com/compose/compose-v2/ diff --git a/Dockerfile.pydis b/Dockerfile.pydis index 46a51684..9383c34b 100644 --- a/Dockerfile.pydis +++ b/Dockerfile.pydis @@ -1,14 +1,17 @@ # This is a custom additional build of Snekbox that includes multiple Python versions. -# The Python versions are now pulled from pre-built images to reduce build time -# and complexity. The images are built in the python-builds repository. +# The Python versions are prebuilt python-build-standalone distributions, +# installed with uv to avoid having to build them from source. ARG SNEKBOX_IMAGE=ghcr.io/python-discord/snekbox:latest +FROM ghcr.io/astral-sh/uv:latest AS uv FROM ${SNEKBOX_IMAGE} -COPY --link --from=ghcr.io/python-discord/python-builds:3.13 /snekbin/ /snekbin/ -COPY --link --from=ghcr.io/python-discord/python-builds:3.14 /snekbin/ /snekbin/ -COPY --link --from=ghcr.io/python-discord/python-builds:3.14t /snekbin/ /snekbin/ -COPY --link --from=ghcr.io/python-discord/python-builds:3.14j /snekbin/ /snekbin/ +RUN --mount=from=uv,source=/uv,target=/usr/local/bin/uv \ + --mount=source=scripts/install_python.sh,target=/install_python.sh \ + /install_python.sh 3.13 \ + && /install_python.sh 3.14 \ + && /install_python.sh 3.14t \ + && /install_python.sh 3.14j RUN rm /snekbin/python/default && ln -s /snekbin/python/3.14/ /snekbin/python/default diff --git a/LICENSE-THIRD-PARTY b/LICENSE-THIRD-PARTY index 684f2dfd..58f4c469 100644 --- a/LICENSE-THIRD-PARTY +++ b/LICENSE-THIRD-PARTY @@ -1,29 +1,3 @@ --------------------------------------------------------------------------------- - MIT License -Applies to: - - Copyright (c) 2014 Docker, Inc. - - scripts/build_python.sh: find command for de-bloating Python install --------------------------------------------------------------------------------- - -Permission is hereby granted, free of charge, to any person obtaining -a copy of this software and associated documentation files (the -"Software"), to deal in the Software without restriction, including -without limitation the rights to use, copy, modify, merge, publish, -distribute, sublicense, and/or sell copies of the Software, and to -permit persons to whom the Software is furnished to do so, subject to -the following conditions: - -The above copyright notice and this permission notice shall be included -in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. -IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY -CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, -TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE -SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -------------------------------------------------------------------------------- Apache License, Version 2.0 Applies to: diff --git a/README.md b/README.md index 492c805f..2763dca5 100644 --- a/README.md +++ b/README.md @@ -121,7 +121,7 @@ You can use the `executable_path` parameter in `POST` requests to `/eval` to sel The default interpreter is at `/snekbin/python/default/bin/python`, you can symlink `/snekbin/python/default` to another interpreter such as `/snekbin/python/3.14` to change this default. -See [`Dockerfile.pydis`](Dockerfile.pydis) for an example using additional prebuilt interpreters and how to change the defaults.. This uses images built from [`python-discord/python-builds`](https://github.com/python-discord/python-builds). +See [`Dockerfile.pydis`](Dockerfile.pydis) for an example using additional prebuilt interpreters and how to change the defaults. This uses [`uv`](https://docs.astral.sh/uv/) to install prebuilt [python-build-standalone](https://github.com/astral-sh/python-build-standalone) interpreters. ## Third-party Packages diff --git a/scripts/build_python.sh b/scripts/build_python.sh deleted file mode 100755 index 7b1fce54..00000000 --- a/scripts/build_python.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/usr/bin/env bash -set -euxo pipefail -shopt -s inherit_errexit - -py_version="${1}" - -# Install Python interpreter under e.g. /snekbin/python/3.13/ (no patch version) -# By dropping everything after, and including, the last period or hyphen. -install_path="${py_version%[-.]*}" - -# Ensure the suffix letter is retained for free-threaded or JIT versions of Python. -if [[ $py_version == *t ]]; then - install_path+="t" -fi - -if [[ $py_version == *j ]]; then - # Enable JIT mode when passed a version that ends with a "j" - install_path+="j" - py_version="${py_version%j}" - PYTHON_CONFIGURE_OPTS+=" --enable-experimental-jit" -fi - -"${PYENV_ROOT}/plugins/python-build/bin/python-build" \ - "${py_version}" \ - "/snekbin/python/${install_path}" -"/snekbin/python/${install_path}/bin/python" -m pip install -U pip - -# Clean up some unnecessary files to reduce image size bloat. -find /snekbin/python/ -depth \ -\( \ - \( -type d -a \( \ - -name test -o -name tests -o -name idle_test \ - \) \) \ - -o \( -type f -a \( \ - -name '*.pyc' -o -name '*.pyo' -o -name 'libpython*.a' \ - \) \) \ -\) -exec rm -rf '{}' + diff --git a/scripts/install_python.sh b/scripts/install_python.sh new file mode 100755 index 00000000..52bce0e0 --- /dev/null +++ b/scripts/install_python.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash +set -euxo pipefail +shopt -s inherit_errexit + +py_version="${1}" + +# Install Python interpreter under /snekbin/python/3.X/ (no patch version). +# The version may have a `t` or `j` suffix for free-threading or JIT-enabled. +install_dir="/snekbin/python/${py_version}" + +if [[ $py_version == *j ]]; then + # Enable JIT mode when passed a version that ends with a "j". + # One standalone Python build is used for both with and without JIT. + # This requires setting an env var, which we set with a shell script that + # runs the non-JIT install, so that install must already exist. + real_exe="/snekbin/python/${py_version%j}/bin/python" + if [[ ! -x $real_exe ]]; then + echo "${real_exe} not found; install ${py_version%j} first" >&2 + exit 1 + fi + mkdir -p "${install_dir}/bin" + for exe in python python3; do + printf '#!/bin/sh\nPYTHON_JIT=1 exec %s "$@"\n' "${real_exe}" > "${install_dir}/bin/${exe}" + chmod +x "${install_dir}/bin/${exe}" + done + exit 0 +fi + +# Download a prebuilt interpreter with uv, then move it to its final location. +# The python-build-standalone distributions that uv installs are relocatable. +UV_PYTHON_INSTALL_DIR="$(mktemp -d)" +export UV_PYTHON_INSTALL_DIR +uv python install --no-bin "${py_version}" +py_exe="$(uv python find --no-project --managed-python "${py_version}")" +py_exe="$(readlink -f "${py_exe}")" + +# We install packages into the user-site directory. +rm -f "${py_exe%/bin/*}"/lib/python*/EXTERNALLY-MANAGED + +mv "${py_exe%/bin/*}" "${install_dir}" +rm -rf "${UV_PYTHON_INSTALL_DIR}" + +"${install_dir}/bin/python" -m pip install -U pip + +# Precompile bytecode, since it can't be written at runtime in the sandbox. +"${install_dir}/bin/python" -m compileall -q -j 0 "${install_dir}/lib" diff --git a/tests/test_nsjail.py b/tests/test_nsjail.py index 3594aa7b..323519b7 100644 --- a/tests/test_nsjail.py +++ b/tests/test_nsjail.py @@ -387,7 +387,7 @@ def test_unicode_env_erase_escape_fails(self): import os import sys os.unsetenv('PYTHONIOENCODING') - os.execl(sys.executable, 'python', '-c', 'print(chr(56550))') + os.execl(sys.executable, sys.executable, '-c', 'print(chr(56550))') """).strip()) self.assertEqual(result.returncode, None) self.assertEqual(result.stdout, "UnicodeDecodeError: invalid Unicode in output pipe")