feat: misc changes for quicker build and deploy times (#236)

* feat: misc changes for quicker build and dpeloy times
* fix: fixture for user in frontend/unit tests
* fix: deploy issue
* fix: quicker frontend/unit tests - independent unit tests
* feat: pulling store refactor for specific components
* fix: in e2e-ready - pre-pull of python3.11 image
* fix: timeout check for liveness of kube in e2e-ready
* fix: event waiter
* fix: teardown improvement
* fix: removed xx.aclose in tests
* feat: deps groups in backend
* feat: conftest better fix

Author: HardMax71

.github/actions/e2e-ready/action.yml

@@ -20,7 +20,7 @@ runs:
         sudo k3s kubectl config view --raw > /home/runner/.kube/config
         sudo chmod 600 /home/runner/.kube/config
         export KUBECONFIG=/home/runner/.kube/config
-        timeout 90 bash -c 'until kubectl cluster-info 2>/dev/null; do sleep 3; done'
+        kubectl wait --for=condition=Ready node --all --timeout=90s
         kubectl create namespace integr8scode --dry-run=client -o yaml | kubectl apply -f -
         sed -E 's#https://(127\.0\.0\.1|0\.0\.0\.0):6443#https://host.docker.internal:6443#g' \
           /home/runner/.kube/config > backend/kubeconfig.yaml
@@ -32,9 +32,11 @@ runs:
         cp backend/config.test.toml backend/config.toml
         cp backend/secrets.example.toml backend/secrets.toml
 
-    - name: Pre-pull test runtime image into K3s
+    - name: Pre-pull test runtime images into K3s
       shell: bash
-      run: sudo k3s crictl pull docker.io/library/python:3.11-slim
+      run: |
+        sudo k3s crictl pull docker.io/library/python:3.11-slim
+        sudo k3s crictl pull docker.io/library/busybox:1.36
 
     - name: Wait for image pull and infra
       shell: bash

.github/workflows/docs.yml

@@ -40,7 +40,7 @@ jobs:
         run: uv tool install mkdocs --with mkdocs-material --with mkdocs-mermaid2-plugin --with mkdocs-swagger-ui-tag
 
       - name: Install backend dependencies
-        run: cd backend && uv sync --frozen
+        run: cd backend && uv sync --frozen --no-dev
 
       - name: Set up config for OpenAPI generation
         run: cp backend/secrets.example.toml backend/secrets.toml

.github/workflows/mypy.yml

@@ -24,7 +24,7 @@ jobs:
         run: |
           cd backend
           uv python install 3.12
-          uv sync --frozen
+          uv sync --frozen --group lint --no-dev
 
       - name: Run mypy
         env:

.github/workflows/ruff.yml

@@ -24,7 +24,7 @@ jobs:
         run: |
           cd backend
           uv python install 3.12
-          uv sync --frozen
+          uv sync --frozen --group lint --no-dev
 
       - name: Run ruff
         run: |

.github/workflows/stack-tests.yml

@@ -50,7 +50,7 @@ jobs:
         run: |
           cd backend
           uv python install 3.12
-          uv sync --frozen
+          uv sync --frozen --group test --no-dev
 
       - name: Set up config
         run: cp backend/secrets.example.toml backend/secrets.toml
@@ -170,7 +170,9 @@ jobs:
       # ── Backend (depends on local base image) ───────────────
       - name: Build backend image
         run: |
-          docker build -t integr8scode-backend:latest --build-context base=docker-image://integr8scode-base:latest -f ./backend/Dockerfile ./backend
+          docker build -t integr8scode-backend:latest \
+            --build-context base=docker-image://integr8scode-base:latest \
+            -f ./backend/Dockerfile ./backend
 
       # ── Utility images (GHA-cached, independent of base) ────────────
       - name: Build cert-generator image
@@ -248,11 +250,11 @@ jobs:
         timeout-minutes: 15
         run: |
           docker compose exec -T backend \
-            uv run pytest tests/e2e -v -rs \
+            sh -c 'uv sync --group test --no-dev --frozen --no-install-project && pytest tests/e2e -v -rs \
             --durations=0 \
             --cov=app \
             --cov-report=xml:coverage-e2e.xml \
-            --cov-report=term
+            --cov-report=term'
 
       - name: Copy coverage
         if: always()

.github/workflows/vulture.yml

@@ -24,7 +24,7 @@ jobs:
         run: |
           cd backend
           uv python install 3.12
-          uv sync --frozen
+          uv sync --frozen --group lint --no-dev
 
       - name: Run vulture
         run: |

backend/.dockerignore

@@ -23,3 +23,18 @@ htmlcov/
 # Local dev files
 *.log
 .DS_Store
+
+# Tests and docs (not needed in production image)
+tests/
+docs/
+*.md
+
+# Secrets and test/override configs (not needed in image)
+secrets.toml
+secrets.*.toml
+config.test.toml
+config.*.toml
+!config.toml
+
+# Dead code detection whitelist
+vulture_whitelist.py

backend/Dockerfile.base

@@ -1,26 +1,52 @@
 # Shared base image for all backend services
 # Contains: Python, system deps, uv, and all Python dependencies
-FROM python:3.12-slim
+# Multi-stage build: gcc + dev headers only in builder, not in final image
+
+FROM python:3.12-slim AS builder
 
 WORKDIR /app
 
-# Install OS security patches + system dependencies needed by any service
-RUN apt-get update && apt-get upgrade -y \
+# Install build-time dependencies (gcc + dev headers for C extensions)
+RUN apt-get update \
     && apt-get install -y --no-install-recommends \
     gcc \
-    curl \
     libsnappy-dev \
     liblzma-dev \
     && rm -rf /var/lib/apt/lists/*
 
 # Install uv (using Docker Hub mirror - ghcr.io has rate limiting issues)
 COPY --from=astral/uv:latest /uv /uvx /bin/
 
+# Pre-compile bytecode for faster startup; copy mode avoids symlink issues with cache mounts
+ENV UV_COMPILE_BYTECODE=1 UV_LINK_MODE=copy
+
 # Copy dependency files
 COPY pyproject.toml uv.lock ./
 
-# Install Python dependencies (production only)
-RUN uv sync --locked --no-dev --no-install-project
+# Install Python dependencies with BuildKit cache mount for faster rebuilds
+RUN --mount=type=cache,target=/root/.cache/uv \
+    uv sync --locked --no-dev --no-install-project
+
+FROM python:3.12-slim
+
+WORKDIR /app
+
+# Install only runtime dependencies (shared libs, no -dev headers, no gcc)
+RUN apt-get update && apt-get upgrade -y \
+    && apt-get install -y --no-install-recommends \
+    curl \
+    libsnappy1v5 \
+    liblzma5 \
+    && rm -rf /var/lib/apt/lists/*
+
+# Copy uv from builder (avoids second Docker Hub pull, guarantees version consistency)
+COPY --from=builder /bin/uv /bin/uvx /bin/
+
+# Copy pre-built virtual environment from builder stage
+COPY --from=builder /app/.venv /app/.venv
+
+# Copy dependency files (needed for uv to recognize the project)
+COPY pyproject.toml uv.lock ./
 
 # Set paths: PYTHONPATH for imports, PATH for venv binaries (no uv run needed at runtime)
 ENV PYTHONPATH=/app

backend/Dockerfile.test

@@ -6,6 +6,7 @@
 from dishka.integrations.fastapi import DishkaRoute
 from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException, Query
 from fastapi.responses import StreamingResponse
+from sse_starlette.sse import EventSourceResponse
 
 from app.api.dependencies import admin_user
 from app.domain.enums import EventType, ExportFormat
@@ -24,6 +25,16 @@
 )
 from app.schemas_pydantic.common import ErrorResponse
 from app.services.admin import AdminEventsService
+from app.services.sse import SSEService
+
+
+class _SSEResponse(EventSourceResponse):
+    """Workaround: sse-starlette sets media_type only in __init__, not as a
+    class attribute.  FastAPI reads the class attribute for OpenAPI generation,
+    so without this subclass every SSE endpoint shows application/json."""
+
+    media_type = "text/event-stream"
+
 
 router = APIRouter(
     prefix="/admin/events", tags=["admin-events"], route_class=DishkaRoute, dependencies=[Depends(admin_user)]
@@ -133,16 +144,25 @@ async def replay_events(
 
 @router.get(
     "/replay/{session_id}/status",
-    responses={404: {"model": ErrorResponse, "description": "Replay session not found"}},
+    response_class=_SSEResponse,
+    responses={
+        200: {"model": EventReplayStatusResponse},
+        404: {"model": ErrorResponse, "description": "Replay session not found"},
+    },
 )
-async def get_replay_status(session_id: str, service: FromDishka[AdminEventsService]) -> EventReplayStatusResponse:
-    """Get the status and progress of a replay session."""
-    status = await service.get_replay_status(session_id)
+async def stream_replay_status(
+    session_id: str,
+    service: FromDishka[AdminEventsService],
+    sse_service: FromDishka[SSEService],
+) -> EventSourceResponse:
+    """Stream the status and progress of a replay session via SSE."""
+    status = await service.get_replay_sse_status(session_id)
 
     if not status:
         raise HTTPException(status_code=404, detail="Replay session not found")
 
-    return EventReplayStatusResponse.model_validate(status)
+    stream = await sse_service.create_replay_stream(status)
+    return EventSourceResponse(stream, ping=15)
 
 
 @router.delete("/{event_id}", responses={404: {"model": ErrorResponse, "description": "Event not found"}})