diff --git a/.github/workflows/python-app.yaml b/.github/workflows/python-app.yaml index a60a2959a..0ff4fc0ff 100644 --- a/.github/workflows/python-app.yaml +++ b/.github/workflows/python-app.yaml @@ -2,14 +2,25 @@ name: Python Package CI/CD on: push: - branches: [main] # Run on pushes to main (after PR is merged) + branches: [main, dev] # Added develop branch tags: - - 'v*' # Trigger on version tags + - 'v*.*.*' # Trigger on semantic version tags pull_request: - branches: [main, next/*] # Run when PRs are created or updated + branches: [main, dev, 'feature/*', 'hotfix/*', 'fix/*'] types: [opened, synchronize, reopened] - release: - types: [created] # Trigger when a release is created + workflow_dispatch: # Allow manual triggering + +# Set permissions for security +permissions: + contents: read + +# Cancel previous runs on new push +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + PYTHON_VERSION: "3.11" jobs: lint: @@ -19,20 +30,34 @@ jobs: uses: actions/checkout@v4 - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: - python-version: "3.11" # Ruff tasks can run with a single Python version + python-version: ${{ env.PYTHON_VERSION }} + cache: 'pip' + cache-dependency-path: | + pyproject.toml - name: Install Ruff run: | python -m pip install --upgrade pip - pip install ruff + pip install "ruff==0.9.*" + ruff --version - name: Run Ruff Linting - run: ruff check . + run: | + echo "::group::Ruff Linting" + ruff check . --output-format=github + echo "::endgroup::" + + - name: Run Ruff Formatting Check + run: | + echo "::group::Ruff Formatting" + ruff format --check --diff . + echo "::endgroup::" test: runs-on: ubuntu-22.04 + timeout-minutes: 30 needs: lint # Run tests only after linting passes strategy: fail-fast: false # Continue testing other Python versions if one fails @@ -44,22 +69,63 @@ jobs: uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} + cache: 'pip' + cache-dependency-path: | + pyproject.toml - name: Install dependencies run: | python -m pip install --upgrade pip - pip install .[dev] + pip install .[dev] pytest-xdist - name: Run tests - run: pytest -v -p no:warnings + run: pytest -v -p no:warnings --numprocesses=auto + + security: + name: Security Scan + runs-on: ubuntu-latest + needs: lint + steps: + - name: Check out code + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + cache: 'pip' + cache-dependency-path: | + pyproject.toml + + - name: Install security tools + run: | + python -m pip install --upgrade pip + pip install bandit[toml] + + - name: Run Bandit security scan + run: | + # Gate on HIGH severity & MEDIUM confidence; produce JSON artifact + bandit -r flixopt/ -c pyproject.toml -f json -o bandit-report.json -q -lll -ii + # Human-readable output without affecting job status + bandit -r flixopt/ -c pyproject.toml -q --exit-zero + + - name: Upload security reports + uses: actions/upload-artifact@v4 + if: always() + with: + name: security-report + path: bandit-report.json + retention-days: 30 create-release: - name: Create Release with Changelog - runs-on: ubuntu-22.04 - needs: [test] + name: Create GitHub Release + runs-on: ubuntu-latest + permissions: + contents: write + needs: [lint, test, security] if: startsWith(github.ref, 'refs/tags/v') steps: @@ -69,14 +135,9 @@ jobs: fetch-depth: 0 - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: - python-version: "3.11" - - - name: Sync changelog to docs - run: | - cp CHANGELOG.md docs/changelog.md - echo "✅ Synced changelog to docs" + python-version: ${{ env.PYTHON_VERSION }} - name: Extract release notes run: | @@ -85,11 +146,12 @@ jobs: python scripts/extract_release_notes.py $VERSION > current_release_notes.md - name: Create GitHub Release - uses: softprops/action-gh-release@v1 + uses: softprops/action-gh-release@v2 with: body_path: current_release_notes.md draft: false prerelease: ${{ contains(github.ref, 'alpha') || contains(github.ref, 'beta') || contains(github.ref, 'rc') }} + generate_release_notes: true env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -98,15 +160,21 @@ jobs: runs-on: ubuntu-22.04 needs: [test, create-release] # Run after tests and release creation if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') # Only on tag push + environment: + name: testpypi + url: https://test.pypi.org/p/flixopt steps: - name: Checkout repository uses: actions/checkout@v4 - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: - python-version: "3.11" + python-version: ${{ env.PYTHON_VERSION }} + cache: 'pip' + cache-dependency-path: | + pyproject.toml - name: Install dependencies run: | @@ -119,7 +187,7 @@ jobs: - name: Upload to TestPyPI run: | - twine upload --repository testpypi dist/* --verbose + twine upload --repository-url https://test.pypi.org/legacy/ dist/* --verbose env: TWINE_USERNAME: __token__ TWINE_PASSWORD: ${{ secrets.TEST_PYPI_API_TOKEN }} @@ -129,29 +197,61 @@ jobs: # Create a temporary environment to test installation python -m venv test_env source test_env/bin/activate - # Get the package name from the built distribution - PACKAGE_NAME=$(ls dist/*.tar.gz | head -n 1 | sed 's/dist\///' | sed 's/-[0-9].*$//') - # Install from TestPyPI with retry (TestPyPI can be slow to index) - pip install --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple/ $PACKAGE_NAME || \ - (sleep 30 && pip install --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple/ $PACKAGE_NAME) || \ - (sleep 60 && pip install --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple/ $PACKAGE_NAME) - # Basic import test - python -c "import flixopt; print('Installation successful!')" + # Get project name from pyproject.toml (PEP 621) + PACKAGE_NAME=$(python - <<'PY' + import sys, tomllib, pathlib + data = tomllib.loads(pathlib.Path("pyproject.toml").read_text(encoding="utf-8")) + print(data["project"]["name"]) + PY + ) + # Extract version from git tag + VERSION=${GITHUB_REF#refs/tags/v} + # Wait and retry while TestPyPI indexes the package + INSTALL_SUCCESS=false + for d in 15 30 60 120 180 360 720 1080; do + sleep "$d" + echo "Attempting to install $PACKAGE_NAME==$VERSION from TestPyPI (retry after ${d}s)..." + # Install specific version and verify it matches + if pip install --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple/ "$PACKAGE_NAME==$VERSION" && \ + python -c "from importlib.metadata import version; installed = version('$PACKAGE_NAME'); print(f'Installed: {installed}'); assert '$VERSION' == installed"; then + INSTALL_SUCCESS=true + break + fi + done + + # Check if installation succeeded + if [ "$INSTALL_SUCCESS" = "false" ]; then + echo "ERROR: Failed to install $PACKAGE_NAME==$VERSION from TestPyPI after all retries" + echo "This could indicate:" + echo " - TestPyPI indexing issues" + echo " - Package upload problems" + echo " - Version mismatch between tag and package" + exit 1 + fi + + # Final success confirmation + python -c "import flixopt; print('TestPyPI installation successful!')" publish-pypi: name: Publish to PyPI runs-on: ubuntu-22.04 needs: [publish-testpypi] # Only run after TestPyPI publish succeeds if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') # Only on tag push + environment: + name: pypi + url: https://pypi.org/p/flixopt steps: - name: Checkout repository uses: actions/checkout@v4 - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: - python-version: "3.11" + python-version: ${{ env.PYTHON_VERSION }} + cache: 'pip' + cache-dependency-path: | + pyproject.toml - name: Install dependencies run: | @@ -174,18 +274,46 @@ jobs: # Create a temporary environment to test installation python -m venv prod_test_env source prod_test_env/bin/activate - # Get the package name from the built distribution - PACKAGE_NAME=$(ls dist/*.tar.gz | head -n 1 | sed 's/dist\///' | sed 's/-[0-9].*$//') - # Wait for PyPI to index the package - sleep 60 - # Install from PyPI - pip install $PACKAGE_NAME - # Basic import test + # Get project name from pyproject.toml (PEP 621) + PACKAGE_NAME=$(python - <<'PY' + import sys, tomllib, pathlib + data = tomllib.loads(pathlib.Path("pyproject.toml").read_text(encoding="utf-8")) + print(data["project"]["name"]) + PY + ) + # Extract version from git tag + VERSION=${GITHUB_REF#refs/tags/v} + # Wait and retry while PyPI indexes the package + INSTALL_SUCCESS=false + for d in 5 10 15 30 60 120 180 360 720 1080; do + sleep "$d" + echo "Attempting to install $PACKAGE_NAME==$VERSION from PyPI (retry after ${d}s)..." + # Install specific version and verify it matches + if pip install "$PACKAGE_NAME==$VERSION" && \ + python -c "from importlib.metadata import version; installed = version('$PACKAGE_NAME'); print(f'Installed: {installed}'); assert '$VERSION' == installed"; then + INSTALL_SUCCESS=true + break + fi + done + + # Check if installation succeeded + if [ "$INSTALL_SUCCESS" = "false" ]; then + echo "ERROR: Failed to install $PACKAGE_NAME==$VERSION from PyPI after all retries" + echo "This could indicate:" + echo " - PyPI indexing issues" + echo " - Package upload problems" + echo " - Version mismatch between tag and package" + exit 1 + fi + + # Final success confirmation python -c "import flixopt; print('PyPI installation successful!')" deploy-docs: name: Deploy Documentation runs-on: ubuntu-22.04 + permissions: + contents: write needs: [publish-pypi] # Deploy docs after successful PyPI publishing if: startsWith(github.ref, 'refs/tags/v') && !contains(github.ref, 'alpha') && !contains(github.ref, 'beta') && !contains(github.ref, 'rc') @@ -196,9 +324,12 @@ jobs: fetch-depth: 0 # Fetch all history for proper versioning - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: - python-version: "3.11" + python-version: ${{ env.PYTHON_VERSION }} + cache: 'pip' + cache-dependency-path: | + pyproject.toml - name: Sync changelog to docs run: | diff --git a/flixopt/calculation.py b/flixopt/calculation.py index c7367cad2..f22f09af6 100644 --- a/flixopt/calculation.py +++ b/flixopt/calculation.py @@ -62,13 +62,9 @@ def __init__( self.folder = pathlib.Path.cwd() / 'results' if folder is None else pathlib.Path(folder) self.results: Optional[CalculationResults] = None - if not self.folder.exists(): - try: - self.folder.mkdir(parents=False) - except FileNotFoundError as e: - raise FileNotFoundError( - f'Folder {self.folder} and its parent do not exist. Please create them first.' - ) from e + if self.folder.exists() and not self.folder.is_dir(): + raise NotADirectoryError(f'Path {self.folder} exists and is not a directory.') + self.folder.mkdir(parents=False, exist_ok=True) @property def main_results(self) -> Dict[str, Union[Scalar, Dict]]: diff --git a/pyproject.toml b/pyproject.toml index 99a1d061d..389d0ef4c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -150,3 +150,7 @@ extend-fixable = ["B"] # Enable fix for flake8-bugbear (`B`), on top of any ru quote-style = "single" indent-style = "space" docstring-code-format = true + +[tool.bandit] +skips = ["B101", "B506"] # assert_used and yaml_load +exclude_dirs = ["tests/"]