From 81e3c212aaba66b842f7b604ed6ea7cdeb6178f0 Mon Sep 17 00:00:00 2001 From: FBumann <117816358+FBumann@users.noreply.github.com> Date: Sat, 13 Sep 2025 13:03:36 +0200 Subject: [PATCH 1/5] Add uv to CI --- .github/workflows/python-app.yaml | 95 ++++++++++++++++++++----------- 1 file changed, 61 insertions(+), 34 deletions(-) diff --git a/.github/workflows/python-app.yaml b/.github/workflows/python-app.yaml index 776737e41..ca26d14f9 100644 --- a/.github/workflows/python-app.yaml +++ b/.github/workflows/python-app.yaml @@ -32,18 +32,19 @@ jobs: - name: Check out code uses: actions/checkout@v5 + - name: Set up uv + uses: astral-sh/setup-uv@v6 + with: + version: "0.8.17" + - name: Set up Python uses: actions/setup-python@v6 with: python-version: ${{ env.PYTHON_VERSION }} - cache: 'pip' - cache-dependency-path: | - pyproject.toml - name: Install Ruff run: | - python -m pip install --upgrade pip - pip install "ruff==0.9.*" + uv pip install --system "ruff==0.9.*" ruff --version - name: Run Ruff Linting @@ -71,18 +72,19 @@ jobs: - name: Check out code uses: actions/checkout@v5 + - name: Set up uv + uses: astral-sh/setup-uv@v6 + with: + version: "0.8.17" + - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} - cache: 'pip' - cache-dependency-path: | - pyproject.toml - name: Install dependencies run: | - python -m pip install --upgrade pip - pip install .[dev] pytest-xdist + uv pip install --system .[dev] pytest-xdist - name: Run tests run: pytest -v -p no:warnings --numprocesses=auto @@ -95,18 +97,20 @@ jobs: - name: Check out code uses: actions/checkout@v5 + - name: Set up uv + uses: astral-sh/setup-uv@v6 + with: + version: "0.8.17" + enable-cache: true + - name: Set up Python uses: actions/setup-python@v6 with: python-version: ${{ env.PYTHON_VERSION }} - cache: 'pip' - cache-dependency-path: | - pyproject.toml - name: Install security tools run: | - python -m pip install --upgrade pip - pip install bandit[toml] + uv pip install --system bandit[toml] - name: Run Bandit security scan run: | @@ -137,6 +141,11 @@ jobs: with: fetch-depth: 0 + - name: Set up uv + uses: astral-sh/setup-uv@v6 + with: + version: "0.8.17" + - name: Set up Python uses: actions/setup-python@v6 with: @@ -171,18 +180,19 @@ jobs: - name: Checkout repository uses: actions/checkout@v5 + - name: Set up uv + uses: astral-sh/setup-uv@v6 + with: + version: "0.8.17" + - name: Set up Python uses: actions/setup-python@v6 with: python-version: ${{ env.PYTHON_VERSION }} - cache: 'pip' - cache-dependency-path: | - pyproject.toml - name: Install dependencies run: | - python -m pip install --upgrade pip - pip install build setuptools wheel twine + uv pip install --system build setuptools wheel twine - name: Build the distribution run: | @@ -196,10 +206,12 @@ jobs: TWINE_PASSWORD: ${{ secrets.TEST_PYPI_API_TOKEN }} - name: Test install from TestPyPI + if: env.SKIP_TESTPYPI_UPLOAD != 'true' run: | # Create a temporary environment to test installation - python -m venv test_env + uv venv test_env source test_env/bin/activate + # Get project name from pyproject.toml (PEP 621) PACKAGE_NAME=$(python - <<'PY' import sys, tomllib, pathlib @@ -207,15 +219,21 @@ jobs: print(data["project"]["name"]) PY ) + # Extract version from git tag VERSION=${GITHUB_REF#refs/tags/v} + + # Install packaging for version comparison + uv pip install packaging + # Wait and retry while TestPyPI indexes the package INSTALL_SUCCESS=false for d in 15 30 60 120 180 360 720 1080; do sleep "$d" echo "Attempting to install $PACKAGE_NAME==$VERSION from TestPyPI (retry after ${d}s)..." + # Install specific version and verify it matches - if pip install --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple/ "$PACKAGE_NAME==$VERSION" && \ + if uv pip install --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple/ "$PACKAGE_NAME==$VERSION" && \ python -c "from importlib.metadata import version; installed = version('$PACKAGE_NAME'); print(f'Installed: {installed}'); assert '$VERSION' == installed"; then INSTALL_SUCCESS=true break @@ -248,18 +266,19 @@ jobs: - name: Checkout repository uses: actions/checkout@v5 + - name: Set up uv + uses: astral-sh/setup-uv@v6 + with: + version: "0.8.17" + - name: Set up Python uses: actions/setup-python@v6 with: python-version: ${{ env.PYTHON_VERSION }} - cache: 'pip' - cache-dependency-path: | - pyproject.toml - name: Install dependencies run: | - python -m pip install --upgrade pip - pip install build setuptools wheel twine + uv pip install --system build twine - name: Build the distribution run: | @@ -275,8 +294,9 @@ jobs: - name: Verify PyPI installation run: | # Create a temporary environment to test installation - python -m venv prod_test_env + uv venv prod_test_env source prod_test_env/bin/activate + # Get project name from pyproject.toml (PEP 621) PACKAGE_NAME=$(python - <<'PY' import sys, tomllib, pathlib @@ -284,15 +304,21 @@ jobs: print(data["project"]["name"]) PY ) + # Extract version from git tag VERSION=${GITHUB_REF#refs/tags/v} + + # Install packaging for version comparison + uv pip install packaging + # Wait and retry while PyPI indexes the package INSTALL_SUCCESS=false for d in 5 10 15 30 60 120 180 360 720 1080; do sleep "$d" echo "Attempting to install $PACKAGE_NAME==$VERSION from PyPI (retry after ${d}s)..." + # Install specific version and verify it matches - if pip install "$PACKAGE_NAME==$VERSION" && \ + if uv pip install "$PACKAGE_NAME==$VERSION" && \ python -c "from importlib.metadata import version; installed = version('$PACKAGE_NAME'); print(f'Installed: {installed}'); assert '$VERSION' == installed"; then INSTALL_SUCCESS=true break @@ -326,13 +352,15 @@ jobs: with: fetch-depth: 0 # Fetch all history for proper versioning + - name: Set up uv + uses: astral-sh/setup-uv@v6 + with: + version: "0.8.17" + - name: Set up Python uses: actions/setup-python@v6 with: python-version: ${{ env.PYTHON_VERSION }} - cache: 'pip' - cache-dependency-path: | - pyproject.toml - name: Sync changelog to docs run: | @@ -341,8 +369,7 @@ jobs: - name: Install documentation dependencies run: | - python -m pip install --upgrade pip - pip install -e ".[docs]" + uv pip install --system -e ".[docs]" - name: Configure Git Credentials run: | From 663a973d0938e8c6b8da8f3e569877f1d1c7d1e6 Mon Sep 17 00:00:00 2001 From: FBumann <117816358+FBumann@users.noreply.github.com> Date: Sat, 13 Sep 2025 13:21:12 +0200 Subject: [PATCH 2/5] Use uvx and ensure caching --- .github/workflows/python-app.yaml | 45 +++++++++++++++++-------------- 1 file changed, 25 insertions(+), 20 deletions(-) diff --git a/.github/workflows/python-app.yaml b/.github/workflows/python-app.yaml index ca26d14f9..1e65e1816 100644 --- a/.github/workflows/python-app.yaml +++ b/.github/workflows/python-app.yaml @@ -36,6 +36,7 @@ jobs: uses: astral-sh/setup-uv@v6 with: version: "0.8.17" + enable-cache: true - name: Set up Python uses: actions/setup-python@v6 @@ -44,19 +45,18 @@ jobs: - name: Install Ruff run: | - uv pip install --system "ruff==0.9.*" - ruff --version + uvx ruff --version - name: Run Ruff Linting run: | echo "::group::Ruff Linting" - ruff check . --output-format=github + uvx ruff check . --output-format=github echo "::endgroup::" - name: Run Ruff Formatting Check run: | echo "::group::Ruff Formatting" - ruff format --check --diff . + uvx ruff format --check --diff . echo "::endgroup::" test: @@ -76,6 +76,7 @@ jobs: uses: astral-sh/setup-uv@v6 with: version: "0.8.17" + enable-cache: true - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v6 @@ -84,7 +85,7 @@ jobs: - name: Install dependencies run: | - uv pip install --system .[dev] pytest-xdist + uv pip install --system .[dev] "pytest-xdist==3.*" - name: Run tests run: pytest -v -p no:warnings --numprocesses=auto @@ -108,16 +109,12 @@ jobs: with: python-version: ${{ env.PYTHON_VERSION }} - - name: Install security tools - run: | - uv pip install --system bandit[toml] - - name: Run Bandit security scan run: | # Gate on HIGH severity & MEDIUM confidence; produce JSON artifact - bandit -r flixopt/ -c pyproject.toml -f json -o bandit-report.json -q -lll -ii + uvx bandit -r flixopt/ -c pyproject.toml -f json -o bandit-report.json -q -lll -ii # Human-readable output without affecting job status - bandit -r flixopt/ -c pyproject.toml -q --exit-zero + uvx bandit -r flixopt/ -c pyproject.toml -q --exit-zero - name: Upload security reports uses: actions/upload-artifact@v4 @@ -145,6 +142,7 @@ jobs: uses: astral-sh/setup-uv@v6 with: version: "0.8.17" + enable-cache: true - name: Set up Python uses: actions/setup-python@v6 @@ -184,6 +182,7 @@ jobs: uses: astral-sh/setup-uv@v6 with: version: "0.8.17" + enable-cache: true - name: Set up Python uses: actions/setup-python@v6 @@ -200,14 +199,16 @@ jobs: - name: Upload to TestPyPI run: | - twine upload --repository-url https://test.pypi.org/legacy/ dist/* --verbose + twine upload --repository-url https://test.pypi.org/legacy/ dist/* --verbose --skip-existing env: TWINE_USERNAME: __token__ TWINE_PASSWORD: ${{ secrets.TEST_PYPI_API_TOKEN }} + TWINE_NON_INTERACTIVE: "1" - name: Test install from TestPyPI if: env.SKIP_TESTPYPI_UPLOAD != 'true' run: | + set -Eeuo pipefail # Create a temporary environment to test installation uv venv test_env source test_env/bin/activate @@ -224,16 +225,16 @@ jobs: VERSION=${GITHUB_REF#refs/tags/v} # Install packaging for version comparison - uv pip install packaging + uv pip install "packaging==24.*" # Wait and retry while TestPyPI indexes the package INSTALL_SUCCESS=false - for d in 15 30 60 120 180 360 720 1080; do + for d in 15 30 60 120 240 360; do sleep "$d" echo "Attempting to install $PACKAGE_NAME==$VERSION from TestPyPI (retry after ${d}s)..." # Install specific version and verify it matches - if uv pip install --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple/ "$PACKAGE_NAME==$VERSION" && \ + if uv pip --timeout 60 install --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple/ "$PACKAGE_NAME==$VERSION" && \ python -c "from importlib.metadata import version; installed = version('$PACKAGE_NAME'); print(f'Installed: {installed}'); assert '$VERSION' == installed"; then INSTALL_SUCCESS=true break @@ -270,6 +271,7 @@ jobs: uses: astral-sh/setup-uv@v6 with: version: "0.8.17" + enable-cache: true - name: Set up Python uses: actions/setup-python@v6 @@ -286,13 +288,15 @@ jobs: - name: Upload to PyPI run: | - twine upload dist/* --verbose + twine upload dist/* --verbose --skip-existing env: TWINE_USERNAME: __token__ TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }} + TWINE_NON_INTERACTIVE: "1" - name: Verify PyPI installation run: | + set -Eeuo pipefail # Create a temporary environment to test installation uv venv prod_test_env source prod_test_env/bin/activate @@ -309,16 +313,16 @@ jobs: VERSION=${GITHUB_REF#refs/tags/v} # Install packaging for version comparison - uv pip install packaging + uv pip install "packaging==24.*" # Wait and retry while PyPI indexes the package INSTALL_SUCCESS=false - for d in 5 10 15 30 60 120 180 360 720 1080; do + for d in 5 10 15 30 60 120 240 360; do sleep "$d" echo "Attempting to install $PACKAGE_NAME==$VERSION from PyPI (retry after ${d}s)..." # Install specific version and verify it matches - if uv pip install "$PACKAGE_NAME==$VERSION" && \ + if uv pip --timeout 60 install "$PACKAGE_NAME==$VERSION" && \ python -c "from importlib.metadata import version; installed = version('$PACKAGE_NAME'); print(f'Installed: {installed}'); assert '$VERSION' == installed"; then INSTALL_SUCCESS=true break @@ -356,6 +360,7 @@ jobs: uses: astral-sh/setup-uv@v6 with: version: "0.8.17" + enable-cache: true - name: Set up Python uses: actions/setup-python@v6 @@ -369,7 +374,7 @@ jobs: - name: Install documentation dependencies run: | - uv pip install --system -e ".[docs]" + uv pip install --system ".[docs]" - name: Configure Git Credentials run: | From 32ed4b61bd054f345cd4eaed14b2edce11df9e34 Mon Sep 17 00:00:00 2001 From: FBumann <117816358+FBumann@users.noreply.github.com> Date: Sat, 13 Sep 2025 13:31:47 +0200 Subject: [PATCH 3/5] Fix ubinte version. Use verbose flags --- .github/workflows/python-app.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python-app.yaml b/.github/workflows/python-app.yaml index 1e65e1816..dc9dab75c 100644 --- a/.github/workflows/python-app.yaml +++ b/.github/workflows/python-app.yaml @@ -92,7 +92,7 @@ jobs: security: name: Security Scan - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 needs: lint steps: - name: Check out code @@ -112,7 +112,7 @@ jobs: - name: Run Bandit security scan run: | # Gate on HIGH severity & MEDIUM confidence; produce JSON artifact - uvx bandit -r flixopt/ -c pyproject.toml -f json -o bandit-report.json -q -lll -ii + uvx bandit -r flixopt/ -c pyproject.toml -f json -o bandit-report.json -q --severity-level HIGH --confidence-level MEDIUM # Human-readable output without affecting job status uvx bandit -r flixopt/ -c pyproject.toml -q --exit-zero From 205320b02e6cb21749ad7d37efd5f465869fe2b3 Mon Sep 17 00:00:00 2001 From: FBumann <117816358+FBumann@users.noreply.github.com> Date: Sat, 13 Sep 2025 13:38:13 +0200 Subject: [PATCH 4/5] Use uv build --- .github/workflows/python-app.yaml | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/.github/workflows/python-app.yaml b/.github/workflows/python-app.yaml index dc9dab75c..f51d0d061 100644 --- a/.github/workflows/python-app.yaml +++ b/.github/workflows/python-app.yaml @@ -173,6 +173,8 @@ jobs: environment: name: testpypi url: https://test.pypi.org/p/flixopt + env: + SKIP_TESTPYPI_UPLOAD: "false" steps: - name: Checkout repository @@ -191,11 +193,11 @@ jobs: - name: Install dependencies run: | - uv pip install --system build setuptools wheel twine + uv pip install --system twine - name: Build the distribution run: | - python -m build + uv build - name: Upload to TestPyPI run: | @@ -224,12 +226,9 @@ jobs: # Extract version from git tag VERSION=${GITHUB_REF#refs/tags/v} - # Install packaging for version comparison - uv pip install "packaging==24.*" - # Wait and retry while TestPyPI indexes the package INSTALL_SUCCESS=false - for d in 15 30 60 120 240 360; do + for d in 10 20 40 80 120; do sleep "$d" echo "Attempting to install $PACKAGE_NAME==$VERSION from TestPyPI (retry after ${d}s)..." @@ -280,11 +279,11 @@ jobs: - name: Install dependencies run: | - uv pip install --system build twine + uv pip install --system twine - name: Build the distribution run: | - python -m build + uv build - name: Upload to PyPI run: | @@ -312,12 +311,9 @@ jobs: # Extract version from git tag VERSION=${GITHUB_REF#refs/tags/v} - # Install packaging for version comparison - uv pip install "packaging==24.*" - # Wait and retry while PyPI indexes the package INSTALL_SUCCESS=false - for d in 5 10 15 30 60 120 240 360; do + for d in 10 20 40 80 120; do sleep "$d" echo "Attempting to install $PACKAGE_NAME==$VERSION from PyPI (retry after ${d}s)..." From 99a8d675518dc150e40bf00aad14c8cfcd260823 Mon Sep 17 00:00:00 2001 From: FBumann <117816358+FBumann@users.noreply.github.com> Date: Sat, 13 Sep 2025 17:49:12 +0200 Subject: [PATCH 5/5] Fix security level --- .github/workflows/python-app.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-app.yaml b/.github/workflows/python-app.yaml index f51d0d061..72463fa13 100644 --- a/.github/workflows/python-app.yaml +++ b/.github/workflows/python-app.yaml @@ -112,7 +112,7 @@ jobs: - name: Run Bandit security scan run: | # Gate on HIGH severity & MEDIUM confidence; produce JSON artifact - uvx bandit -r flixopt/ -c pyproject.toml -f json -o bandit-report.json -q --severity-level HIGH --confidence-level MEDIUM + uvx bandit -r flixopt/ -c pyproject.toml -f json -o bandit-report.json -q --severity-level high --confidence-level medium # Human-readable output without affecting job status uvx bandit -r flixopt/ -c pyproject.toml -q --exit-zero