Feature/improve ci

.github/workflows/python-app.yaml

@@ -2,14 +2,25 @@ name: Python Package CI/CD
 
 on:
   push:
-    branches: [main]  # Run on pushes to main (after PR is merged)
+    branches: [main, dev]  # Added develop branch
     tags:
-      - 'v*'  # Trigger on version tags
+      - 'v*.*.*'  # Trigger on semantic version tags
   pull_request:
-    branches: [main, next/*]  # Run when PRs are created or updated
+    branches: [main, dev, 'feature/*', 'hotfix/*', 'fix/*']
     types: [opened, synchronize, reopened]
-  release:
-    types: [created]  # Trigger when a release is created
+  workflow_dispatch:  # Allow manual triggering
+
+# Set permissions for security
+permissions:
+  contents: read
+
+# Cancel previous runs on new push
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+env:
+  PYTHON_VERSION: "3.11"
 
 jobs:
   lint:
@@ -19,20 +30,34 @@ jobs:
         uses: actions/checkout@v4
 
       - name: Set up Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
-          python-version: "3.11"  # Ruff tasks can run with a single Python version
+          python-version: ${{ env.PYTHON_VERSION }}
+          cache: 'pip'
+          cache-dependency-path: |
+            pyproject.toml
 
       - name: Install Ruff
         run: |
           python -m pip install --upgrade pip
-          pip install ruff
+          pip install "ruff==0.9.*"
+          ruff --version
 
       - name: Run Ruff Linting
-        run: ruff check .
+        run: |
+          echo "::group::Ruff Linting"
+          ruff check . --output-format=github
+          echo "::endgroup::"
+
+      - name: Run Ruff Formatting Check
+        run: |
+          echo "::group::Ruff Formatting"
+          ruff format --check --diff .
+          echo "::endgroup::"
 
   test:
     runs-on: ubuntu-22.04
+    timeout-minutes: 30
     needs: lint  # Run tests only after linting passes
     strategy:
       fail-fast: false  # Continue testing other Python versions if one fails
@@ -44,22 +69,63 @@ jobs:
         uses: actions/checkout@v4
 
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
+          cache: 'pip'
+          cache-dependency-path: |
+            pyproject.toml
 
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install .[dev]
+          pip install .[dev] pytest-xdist
 
       - name: Run tests
-        run: pytest -v -p no:warnings
+        run: pytest -v -p no:warnings --numprocesses=auto
+
+  security:
+    name: Security Scan
+    runs-on: ubuntu-latest
+    needs: lint
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+          cache: 'pip'
+          cache-dependency-path: |
+            pyproject.toml
+
+      - name: Install security tools
+        run: |
+          python -m pip install --upgrade pip
+          pip install bandit[toml]
+
+      - name: Run Bandit security scan
+        run: |
+          # Gate on HIGH severity & MEDIUM confidence; produce JSON artifact
+          bandit -r flixopt/ -c pyproject.toml -f json -o bandit-report.json -q -lll -ii
+          # Human-readable output without affecting job status
+          bandit -r flixopt/ -c pyproject.toml -q --exit-zero
+
+      - name: Upload security reports
+        uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: security-report
+          path: bandit-report.json
+          retention-days: 30
 
   create-release:
-    name: Create Release with Changelog
-    runs-on: ubuntu-22.04
-    needs: [test]
+    name: Create GitHub Release
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    needs: [lint, test, security]
     if: startsWith(github.ref, 'refs/tags/v')
 
     steps:
@@ -69,14 +135,9 @@ jobs:
           fetch-depth: 0
 
       - name: Set up Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
-          python-version: "3.11"
-
-      - name: Sync changelog to docs
-        run: |
-          cp CHANGELOG.md docs/changelog.md
-          echo "✅ Synced changelog to docs"
+          python-version: ${{ env.PYTHON_VERSION }}
 
       - name: Extract release notes
         run: |
@@ -85,11 +146,12 @@ jobs:
           python scripts/extract_release_notes.py $VERSION > current_release_notes.md
 
       - name: Create GitHub Release
-        uses: softprops/action-gh-release@v1
+        uses: softprops/action-gh-release@v2
         with:
           body_path: current_release_notes.md
           draft: false
           prerelease: ${{ contains(github.ref, 'alpha') || contains(github.ref, 'beta') || contains(github.ref, 'rc') }}
+          generate_release_notes: true
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
@@ -98,15 +160,21 @@ jobs:
     runs-on: ubuntu-22.04
     needs: [test, create-release]  # Run after tests and release creation
     if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')  # Only on tag push
+    environment:
+      name: testpypi
+      url: https://test.pypi.org/p/flixopt
 
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
 
       - name: Set up Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
-          python-version: "3.11"
+          python-version: ${{ env.PYTHON_VERSION }}
+          cache: 'pip'
+          cache-dependency-path: |
+            pyproject.toml
 
       - name: Install dependencies
         run: |
@@ -119,7 +187,7 @@ jobs:
 
       - name: Upload to TestPyPI
         run: |
-          twine upload --repository testpypi dist/* --verbose
+          twine upload --repository-url https://test.pypi.org/legacy/ dist/* --verbose
         env:
           TWINE_USERNAME: __token__
           TWINE_PASSWORD: ${{ secrets.TEST_PYPI_API_TOKEN }}
@@ -129,29 +197,61 @@ jobs:
           # Create a temporary environment to test installation
           python -m venv test_env
           source test_env/bin/activate
-          # Get the package name from the built distribution
-          PACKAGE_NAME=$(ls dist/*.tar.gz | head -n 1 | sed 's/dist\///' | sed 's/-[0-9].*$//')
-          # Install from TestPyPI with retry (TestPyPI can be slow to index)
-          pip install --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple/ $PACKAGE_NAME || \
-          (sleep 30 && pip install --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple/ $PACKAGE_NAME) || \
-          (sleep 60 && pip install --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple/ $PACKAGE_NAME)
-          # Basic import test
-          python -c "import flixopt; print('Installation successful!')"
+          # Get project name from pyproject.toml (PEP 621)
+          PACKAGE_NAME=$(python - <<'PY'
+          import sys, tomllib, pathlib
+          data = tomllib.loads(pathlib.Path("pyproject.toml").read_text(encoding="utf-8"))
+          print(data["project"]["name"])
+          PY
+          )
+          # Extract version from git tag
+          VERSION=${GITHUB_REF#refs/tags/v}
+          # Wait and retry while TestPyPI indexes the package
+          INSTALL_SUCCESS=false
+          for d in 15 30 60 120 180 360 720 1080; do
+            sleep "$d"
+            echo "Attempting to install $PACKAGE_NAME==$VERSION from TestPyPI (retry after ${d}s)..."
+            # Install specific version and verify it matches
+            if pip install --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple/ "$PACKAGE_NAME==$VERSION" && \
+               python -c "from importlib.metadata import version; installed = version('$PACKAGE_NAME'); print(f'Installed: {installed}'); assert '$VERSION' == installed"; then
+              INSTALL_SUCCESS=true
+              break
+            fi
+          done
+
+          # Check if installation succeeded
+          if [ "$INSTALL_SUCCESS" = "false" ]; then
+            echo "ERROR: Failed to install $PACKAGE_NAME==$VERSION from TestPyPI after all retries"
+            echo "This could indicate:"
+            echo "  - TestPyPI indexing issues"
+            echo "  - Package upload problems"
+            echo "  - Version mismatch between tag and package"
+            exit 1
+          fi
+
+          # Final success confirmation
+          python -c "import flixopt; print('TestPyPI installation successful!')"
 
   publish-pypi:
     name: Publish to PyPI
     runs-on: ubuntu-22.04
     needs: [publish-testpypi]  # Only run after TestPyPI publish succeeds
     if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')  # Only on tag push
+    environment:
+      name: pypi
+      url: https://pypi.org/p/flixopt
 
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
 
       - name: Set up Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
-          python-version: "3.11"
+          python-version: ${{ env.PYTHON_VERSION }}
+          cache: 'pip'
+          cache-dependency-path: |
+            pyproject.toml
 
       - name: Install dependencies
         run: |
@@ -174,18 +274,46 @@ jobs:
           # Create a temporary environment to test installation
           python -m venv prod_test_env
           source prod_test_env/bin/activate
-          # Get the package name from the built distribution
-          PACKAGE_NAME=$(ls dist/*.tar.gz | head -n 1 | sed 's/dist\///' | sed 's/-[0-9].*$//')
-          # Wait for PyPI to index the package
-          sleep 60
-          # Install from PyPI
-          pip install $PACKAGE_NAME
-          # Basic import test
+          # Get project name from pyproject.toml (PEP 621)
+          PACKAGE_NAME=$(python - <<'PY'
+          import sys, tomllib, pathlib
+          data = tomllib.loads(pathlib.Path("pyproject.toml").read_text(encoding="utf-8"))
+          print(data["project"]["name"])
+          PY
+          )
+          # Extract version from git tag
+          VERSION=${GITHUB_REF#refs/tags/v}
+          # Wait and retry while PyPI indexes the package
+          INSTALL_SUCCESS=false
+          for d in 5 10 15 30 60 120 180 360 720 1080; do
+            sleep "$d"
+            echo "Attempting to install $PACKAGE_NAME==$VERSION from PyPI (retry after ${d}s)..."
+            # Install specific version and verify it matches
+            if pip install "$PACKAGE_NAME==$VERSION" && \
+               python -c "from importlib.metadata import version; installed = version('$PACKAGE_NAME'); print(f'Installed: {installed}'); assert '$VERSION' == installed"; then
+              INSTALL_SUCCESS=true
+              break
+            fi
+          done
+
+          # Check if installation succeeded
+          if [ "$INSTALL_SUCCESS" = "false" ]; then
+            echo "ERROR: Failed to install $PACKAGE_NAME==$VERSION from PyPI after all retries"
+            echo "This could indicate:"
+            echo "  - PyPI indexing issues"
+            echo "  - Package upload problems"
+            echo "  - Version mismatch between tag and package"
+            exit 1
+          fi
+
+          # Final success confirmation
           python -c "import flixopt; print('PyPI installation successful!')"
 
   deploy-docs:
     name: Deploy Documentation
     runs-on: ubuntu-22.04
+    permissions:
+      contents: write
     needs: [publish-pypi]  # Deploy docs after successful PyPI publishing
     if: startsWith(github.ref, 'refs/tags/v') && !contains(github.ref, 'alpha') && !contains(github.ref, 'beta') && !contains(github.ref, 'rc')
 
@@ -196,9 +324,12 @@ jobs:
           fetch-depth: 0  # Fetch all history for proper versioning
 
       - name: Set up Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
-          python-version: "3.11"
+          python-version: ${{ env.PYTHON_VERSION }}
+          cache: 'pip'
+          cache-dependency-path: |
+            pyproject.toml
 
       - name: Sync changelog to docs
         run: |

flixopt/calculation.py

@@ -62,13 +62,9 @@ def __init__(
         self.folder = pathlib.Path.cwd() / 'results' if folder is None else pathlib.Path(folder)
         self.results: Optional[CalculationResults] = None
 
-        if not self.folder.exists():
-            try:
-                self.folder.mkdir(parents=False)
-            except FileNotFoundError as e:
-                raise FileNotFoundError(
-                    f'Folder {self.folder} and its parent do not exist. Please create them first.'
-                ) from e
+        if self.folder.exists() and not self.folder.is_dir():
+            raise NotADirectoryError(f'Path {self.folder} exists and is not a directory.')
+        self.folder.mkdir(parents=False, exist_ok=True)
 
     @property
     def main_results(self) -> Dict[str, Union[Scalar, Dict]]:

pyproject.toml

@@ -150,3 +150,7 @@ extend-fixable = ["B"]   # Enable fix for flake8-bugbear (`B`), on top of any ru
 quote-style = "single"
 indent-style = "space"
 docstring-code-format = true
+
+[tool.bandit]
+skips = ["B101", "B506"]  # assert_used and yaml_load
+exclude_dirs = ["tests/"]