diff --git a/.env.example b/.env.example index bb6f6b2..22ec4e4 100644 --- a/.env.example +++ b/.env.example @@ -41,7 +41,7 @@ totp_issuer=MultiAI GOOGLE_CLIENT_ID= GOOGLE_CLIENT_SECRET= -GOOGLE_REDIRECT_URI=http://127.0.0.1:8000/stuff/drive/callback +GOOGLE_REDIRECT_URI=http://127.0.0.1:8000/staff/drive/callback GOOGLE_OAUTH_SCOPES=https://www.googleapis.com/auth/drive.readonly openid email profile FACE_ENCRYPTION_KEY=hkbribvfirirbvivbibvib diff --git a/.env.staging.example b/.env.staging.example index ec25849..c33d0f7 100644 --- a/.env.staging.example +++ b/.env.staging.example @@ -49,4 +49,6 @@ GOOGLE_CLIENT_SECRET= GOOGLE_REDIRECT_URI=http://127.0.0.1:8000/staff/drive/callback GOOGLE_OAUTH_SCOPES=https://www.googleapis.com/auth/drive.readonly openid email profile -FACE_ENCRYPTION_KEY=base64-encoded-32-byte-key \ No newline at end of file +FACE_ENCRYPTION_KEY=base64-encoded-32-byte-key + +FIREBASE_CREDENTIALS_PATH=/app/firebase-credentials.json \ No newline at end of file diff --git a/.gitignore b/.gitignore index 1656658..ccfbe50 100644 --- a/.gitignore +++ b/.gitignore @@ -1,17 +1,16 @@ .env .env.staging +.env.mobile __pycache__ db/schema.sql - .vscode/settings.json multiai-c9380-firebase-adminsdk-fbsvc-cb6e5ce41b.json +firebase-credentials.json db.txt - .venv -multiai-c9380-firebase-adminsdk-fbsvc-cb6e5ce41b.json # Local Test & Debug Files .coverage dummy.json test_output.log -*.log +*.log \ No newline at end of file diff --git a/app/container.py b/app/container.py index 5311498..86e0d46 100644 --- a/app/container.py +++ b/app/container.py @@ -26,15 +26,14 @@ from db.generated import session as session_queries from db.generated import staff_drive_connections as staff_drive_queries from db.generated import staff_notifications as staff_notification_queries -from db.generated import stuff_user as staff_user_queries +from db.generated import staff_user as staff_user_queries from db.generated import upload_request_groups as upload_request_group_queries from db.generated import upload_request_photos as upload_request_photo_queries from db.generated import upload_requests as upload_request_queries from db.generated import user as user_queries from db.generated import events as event_queries -from db.generated import eventParticipant as participant_queries -from db.generated import stuff_user as staff_queries +from db.generated import event_participant as participant_queries from db.generated import notifications as notification_queries from db.generated import audit as audit_queries from db.generated import stats as stats_queries @@ -72,7 +71,6 @@ def __init__( self.audit_querier = audit_queries.AsyncQuerier(conn) self.event_querier = event_queries.AsyncQuerier(conn) self.participant_querier = participant_queries.AsyncQuerier(conn) - self.staff_querier = staff_queries.AsyncQuerier(conn) self.stats_querier = stats_queries.AsyncQuerier(conn) # services diff --git a/app/infra/nats.py b/app/infra/nats.py index 06e0f28..e17bd50 100644 --- a/app/infra/nats.py +++ b/app/infra/nats.py @@ -62,11 +62,12 @@ async def connect( @staticmethod async def close() -> None: if NatsClient._nc and not NatsClient._nc.is_closed: - await NatsClient._nc.drain() - await NatsClient._nc.close() - NatsClient._nc = None - NatsClient._js = None - + try: + await NatsClient._nc.drain() + await NatsClient._nc.close() + finally: + NatsClient._nc = None + NatsClient._js = None @staticmethod async def publish(subject: NatsSubjects | str, message: bytes) -> None: diff --git a/app/infra/redis.py b/app/infra/redis.py index d3116be..7e90101 100644 --- a/app/infra/redis.py +++ b/app/infra/redis.py @@ -77,3 +77,4 @@ async def srem(self, key: RedisKey | str, *values: str) -> int: async def close(self) -> None: await self._client.close() + type(self)._instance = None diff --git a/app/router/staff/uploads.py b/app/router/staff/uploads.py index 0ec0029..5ded556 100644 --- a/app/router/staff/uploads.py +++ b/app/router/staff/uploads.py @@ -14,9 +14,9 @@ RejectUploadRequestRequest, ) from app.schema.response.staff.upload_groups import ( - UploadRequestGroupListResponse, UploadRequestGroupPhotoListResponse, UploadRequestGroupSchema, + UploadRequestGroupSummaryListResponse, ) from app.schema.response.staff.uploads import ( UploadRequestListResponse, @@ -67,19 +67,19 @@ async def list_upload_requests( ) -@router.get("/groups", response_model=UploadRequestGroupListResponse) +@router.get("/groups", response_model=UploadRequestGroupSummaryListResponse) async def list_upload_request_groups( scope: Literal["my", "all"] = Query(default="my"), status: UploadRequestStatus | None = Query(default=None), current_staff_user: StaffUser = Depends(get_current_staff_user), container: Container = Depends(get_container), -) -> UploadRequestGroupListResponse: +) -> UploadRequestGroupSummaryListResponse: groups = await container.upload_requests_service.list_groups( current_staff_user=current_staff_user, scope=scope, status=status.value if status is not None else None, ) - return UploadRequestGroupListResponse.from_details_list(groups) + return UploadRequestGroupSummaryListResponse.from_groups(groups) @router.get("/groups/{group_id}", response_model=UploadRequestGroupSchema) diff --git a/app/schema/response/staff/upload_groups.py b/app/schema/response/staff/upload_groups.py index 02f111a..7a264c2 100644 --- a/app/schema/response/staff/upload_groups.py +++ b/app/schema/response/staff/upload_groups.py @@ -10,7 +10,7 @@ UploadRequestSchema, ) from app.service.upload_requests import UploadRequestGroupDetails -from db.generated.models import UploadRequestPhoto +from db.generated.models import UploadRequestGroup, UploadRequestPhoto class UploadRequestGroupSchema(BaseModel): @@ -67,3 +67,32 @@ def from_photos( return cls( items=[UploadRequestPhotoSchema.model_validate(p) for p in photos] ) + +class UploadRequestGroupSummarySchema(BaseModel): + model_config = ConfigDict(from_attributes=True) + id: UUID + event_id: UUID + folder_id: str + status: str + processing_status: str + total_photo_count: int + batch_count: int + processed_photo_count: int + failed_photo_count: int + created_at: datetime + approved_at: datetime | None + rejection_reason: str | None + error_message: str | None + + @field_validator("status", mode="before") + @classmethod + def coerce_status(cls, v: object) -> str: + return getattr(v, "value", str(v)) + + +class UploadRequestGroupSummaryListResponse(BaseModel): + items: list[UploadRequestGroupSummarySchema] + + @classmethod + def from_groups(cls, groups: list["UploadRequestGroup"]) -> "UploadRequestGroupSummaryListResponse": + return cls(items=[UploadRequestGroupSummarySchema.model_validate(g) for g in groups]) diff --git a/app/service/event.py b/app/service/event.py index 9723a06..e697ddd 100644 --- a/app/service/event.py +++ b/app/service/event.py @@ -13,7 +13,7 @@ ParticipantResponse ) from db.generated import events as event_queries -from db.generated import eventParticipant as participant_queries +from db.generated import event_participant as participant_queries from db.generated import models class EventService: diff --git a/app/service/staff_drive.py b/app/service/staff_drive.py index 9cb6ba9..f3b2ba0 100644 --- a/app/service/staff_drive.py +++ b/app/service/staff_drive.py @@ -17,7 +17,7 @@ from app.infra.minio import ImageBucket from app.infra.redis import RedisClient from db.generated import staff_drive_connections as drive_queries -from db.generated import stuff_user as staff_queries +from db.generated import staff_user as staff_queries from db.generated.models import StaffDriveConnection, StaffUser diff --git a/app/service/staff_user.py b/app/service/staff_user.py index 3a589e4..5675668 100644 --- a/app/service/staff_user.py +++ b/app/service/staff_user.py @@ -8,8 +8,8 @@ from app.core.exceptions import AppException, DBException, DBExceptionImpl from app.core.securite import create_access_staff_token, hash_password, verify_password from app.schema.response.web.auth import WebAuthResponse -from db.generated import stuff_user as staff_queries -from db.generated.stuff_user import ListStaffUsersParams +from db.generated import staff_user as staff_queries +from db.generated.staff_user import ListStaffUsersParams from db.generated.models import StaffUser, StaffRole diff --git a/app/service/upload_requests.py b/app/service/upload_requests.py index f794378..0ab404d 100644 --- a/app/service/upload_requests.py +++ b/app/service/upload_requests.py @@ -890,7 +890,7 @@ async def list_groups( current_staff_user: StaffUser, scope: Literal["my", "all"], status: str | None, - ) -> list[UploadRequestGroupDetails]: + ) -> list[UploadRequestGroup]: if scope == "all" and self._role_value(current_staff_user.role) != StaffRole.MULTI_TEAM_LEAD.value: raise AppException.forbidden("Multi team lead access required") @@ -915,15 +915,7 @@ async def list_groups( async for group in iterator: groups.append(group) - details: list[UploadRequestGroupDetails] = [] - for group in groups: - details.append( - await self.get_group_details( - group_id=group.id, - current_staff_user=current_staff_user, - ) - ) - return details + return groups async def list_group_photos( self, diff --git a/app/worker/notification/settings.py b/app/worker/notification/settings.py index 7bb2aa8..df8ca9e 100644 --- a/app/worker/notification/settings.py +++ b/app/worker/notification/settings.py @@ -1,24 +1,25 @@ from __future__ import annotations - from typing import ClassVar, Sequence - from pydantic import Field from pydantic_settings import BaseSettings, SettingsConfigDict - from app.schema.internal.notification import NotificationPriority, PRIORITY_ORDER class NotificationWorkerSettings(BaseSettings): subject_prefix: str = Field("notifications.delivery") queue_group: str | None = Field(None) - redis_host: str = Field("localhost") - redis_port: int = Field(6379) - redis_password: str = Field("") - nats_host: str = Field("localhost") - nats_port: int = Field(4222) - nats_user: str = Field("") - nats_password: str = Field("") + + redis_host: str + redis_port: int + redis_password: str = "" + + nats_host: str + nats_port: int + nats_user: str + nats_password: str + firebase_credentials_path: str | None = Field(None) + MAX_SEND_ATTEMPTS: ClassVar[int] = 5 BASE_RETRY_DELAY: ClassVar[int] = 2 TTL_SECONDS: ClassVar[int] = 30 * 24 * 3600 @@ -26,7 +27,7 @@ class NotificationWorkerSettings(BaseSettings): RATE_LIMIT: ClassVar[int] = 50 RATE_PERIOD: ClassVar[float] = 1.0 - model_config = SettingsConfigDict(env_prefix="NOTIFICATIONS_") + model_config = SettingsConfigDict(env_file=".env", extra="ignore") def subject_for(self, priority: NotificationPriority) -> str: return f"{self.subject_prefix}.{priority.value}" @@ -34,4 +35,5 @@ def subject_for(self, priority: NotificationPriority) -> str: def priority_subjects(self) -> Sequence[str]: return [self.subject_for(priority) for priority in PRIORITY_ORDER] + NotifSetting = NotificationWorkerSettings() # type: ignore diff --git a/app/worker/photo_worker/tests/test_photo_worker.py b/app/worker/photo_worker/tests/test_photo_worker.py index eecda3a..f113a70 100644 --- a/app/worker/photo_worker/tests/test_photo_worker.py +++ b/app/worker/photo_worker/tests/test_photo_worker.py @@ -5,9 +5,14 @@ import pytest -sys.modules["db.generated.user"] = MagicMock() -sys.modules["app.worker.notification.settings"] = MagicMock() -sys.modules["app.worker.notification.notification_queue"] = MagicMock() +_MOCKED_MODULES = ( + "db.generated.user", + "app.worker.notification.settings", + "app.worker.notification.notification_queue", +) +_original_modules = {name: sys.modules.get(name) for name in _MOCKED_MODULES} +for _name in _MOCKED_MODULES: + sys.modules[_name] = MagicMock() from app.service.face_embedding import DetectedFace, FaceEmbeddingService, FaceImagePayload # noqa: E402 from app.service.face_match import SingleFaceMatchService # noqa: E402 @@ -16,6 +21,13 @@ from app.worker.photo_worker.schema.event import PhotoProcessEvent # noqa: E402 from db.generated import photo_faces as photo_face_queries # noqa: E402 +# Restore sys.modules so other test files importing these modules +# (e.g. db.generated.user) get the real thing, not this leaked mock. +for _name, _original in _original_modules.items(): + if _original is None: + sys.modules.pop(_name, None) + else: + sys.modules[_name] = _original # ── fixtures ────────────────────────────────────────────────────────── diff --git a/app/worker/storage_cleaner/main.py b/app/worker/storage_cleaner/main.py index 344049c..eb723d2 100644 --- a/app/worker/storage_cleaner/main.py +++ b/app/worker/storage_cleaner/main.py @@ -3,9 +3,8 @@ import asyncio import json import uuid -from typing import Iterable, Optional, Set, Tuple +from typing import Iterable, Optional, Set -import sqlalchemy.ext.asyncio from fastapi import HTTPException from pydantic import BaseModel, ValidationError @@ -26,19 +25,6 @@ class FinalBucketCleanupPayload(BaseModel): storage_service = StagedUploadStorageService() -async def create_photo_querier() -> Tuple[ - sqlalchemy.ext.asyncio.AsyncConnection, - upload_request_photo_queries.AsyncQuerier, -]: - conn = await engine.connect() - querier = upload_request_photo_queries.AsyncQuerier(conn) - return conn, querier - - -async def close_connection(conn: sqlalchemy.ext.asyncio.AsyncConnection) -> None: - await conn.close() - - def _parse_payload(raw_data: bytes | str) -> Optional[FinalBucketCleanupPayload]: if isinstance(raw_data, bytes): try: @@ -131,11 +117,12 @@ async def _handle_cleanup_event( async def main() -> None: - conn, querier = await create_photo_querier() await NatsClient.connect() try: async def _jetstream_handler(data: bytes | str) -> None: - await _handle_cleanup_event(data, querier) + async with engine.begin() as conn: + querier = upload_request_photo_queries.AsyncQuerier(conn) + await _handle_cleanup_event(data, querier) await NatsClient.js_subscribe( subject=settings.subject_enum, @@ -150,7 +137,6 @@ async def _jetstream_handler(data: bytes | str) -> None: ) await asyncio.Event().wait() finally: - await close_connection(conn) await NatsClient.close() diff --git a/db/generated/eventParticipant.py b/db/generated/event_participant.py similarity index 99% rename from db/generated/eventParticipant.py rename to db/generated/event_participant.py index 0e6fd9c..65129c7 100644 --- a/db/generated/eventParticipant.py +++ b/db/generated/event_participant.py @@ -1,7 +1,7 @@ # Code generated by sqlc. DO NOT EDIT. # versions: # sqlc v1.31.1 -# source: eventParticipant.sql +# source: event_participant.sql import dataclasses import datetime from typing import Any, AsyncIterator, Optional diff --git a/db/generated/photo_approvals.py b/db/generated/photo_approvals.py index 13a5096..56ba3a3 100644 --- a/db/generated/photo_approvals.py +++ b/db/generated/photo_approvals.py @@ -11,25 +11,6 @@ from db.generated import models -EXPIRE_STALE_APPROVALS = """-- name: expire_stale_approvals \\:many -WITH stale_photos AS ( - SELECT id FROM photos - WHERE status = 'pending' - AND created_at < now() - make_interval(days => :p1::int) -), -_update_approvals AS ( - UPDATE photo_approvals - SET decision = 'approved', decided_at = now() - WHERE photo_id IN (SELECT id FROM stale_photos) - AND decision = 'pending' -) -UPDATE photos -SET status = 'approved' -WHERE id IN (SELECT id FROM stale_photos) -RETURNING id -""" - - CREATE_PHOTO_APPROVAL = """-- name: create_photo_approval \\:one INSERT INTO photo_approvals ( photo_id, @@ -87,11 +68,6 @@ class AsyncQuerier: def __init__(self, conn: sqlalchemy.ext.asyncio.AsyncConnection): self._conn = conn - async def expire_stale_approvals(self, *, timeout_days: int) -> AsyncIterator[uuid.UUID]: - result = await self._conn.stream(sqlalchemy.text(EXPIRE_STALE_APPROVALS), {"p1": timeout_days}) - async for row in result: - yield row[0] - async def create_photo_approval(self, *, photo_id: uuid.UUID, user_id: uuid.UUID, decision: str) -> Optional[models.PhotoApproval]: row = (await self._conn.execute(sqlalchemy.text(CREATE_PHOTO_APPROVAL), {"p1": photo_id, "p2": user_id, "p3": decision})).first() if row is None: diff --git a/db/generated/stuff_user.py b/db/generated/staff_user.py similarity index 99% rename from db/generated/stuff_user.py rename to db/generated/staff_user.py index d541078..74c85f9 100644 --- a/db/generated/stuff_user.py +++ b/db/generated/staff_user.py @@ -1,7 +1,7 @@ # Code generated by sqlc. DO NOT EDIT. # versions: # sqlc v1.31.1 -# source: stuff_user.sql +# source: staff_user.sql import dataclasses from typing import Any, AsyncIterator, Optional import uuid diff --git a/db/queries/eventParticipant.sql b/db/queries/event_participant.sql similarity index 100% rename from db/queries/eventParticipant.sql rename to db/queries/event_participant.sql diff --git a/db/queries/stuff_user.sql b/db/queries/staff_user.sql similarity index 100% rename from db/queries/stuff_user.sql rename to db/queries/staff_user.sql diff --git a/docker-compose.staging.local.yml b/docker-compose.staging.local.yml index 0d3c378..1a0a012 100644 --- a/docker-compose.staging.local.yml +++ b/docker-compose.staging.local.yml @@ -6,6 +6,7 @@ services: pull_policy: never volumes: - ./:/app + - /app/.venv - insightface_cache:/root/.insightface migrate: @@ -13,6 +14,9 @@ services: context: . dockerfile: Dockerfile pull_policy: never + volumes: + - ./:/app + - /app/.venv email-worker: build: @@ -21,6 +25,55 @@ services: pull_policy: never volumes: - ./:/app + - /app/.venv + + photo-worker: + build: + context: . + dockerfile: Dockerfile + pull_policy: never + volumes: + - ./:/app + - /app/.venv + - insightface_cache:/root/.insightface + + notification-worker: + profiles: ["firebase"] + build: + context: . + dockerfile: Dockerfile + pull_policy: never + volumes: + - ./:/app + - /app/.venv + - ./firebase-credentials.json:/app/firebase-credentials.json + + upload-group-worker: + build: + context: . + dockerfile: Dockerfile + pull_policy: never + volumes: + - ./:/app + - /app/.venv + + audit-worker: + build: + context: . + dockerfile: Dockerfile + pull_policy: never + volumes: + - ./:/app + - /app/.venv + + storage-cleaner: + build: + context: . + dockerfile: Dockerfile + pull_policy: never + volumes: + - ./:/app + - /app/.venv volumes: - insightface_cache: + insightface_cache: \ No newline at end of file diff --git a/docker-compose.staging.yml b/docker-compose.staging.yml index 93435d8..18b7e66 100644 --- a/docker-compose.staging.yml +++ b/docker-compose.staging.yml @@ -1,3 +1,4 @@ +name: multai-staging services: postgres: image: pgvector/pgvector:pg16 @@ -19,8 +20,10 @@ services: nats: image: nats:2.10-alpine container_name: multi_nats + restart: unless-stopped command: > -js + -sd /data -a 0.0.0.0 -m ${NATS_MONITOR_PORT} --user ${NATS_USER} @@ -28,6 +31,8 @@ services: ports: - "${NATS_PORT}:4222" - "${NATS_MONITOR_PORT}:8222" + volumes: + - nats_data:/data networks: - multi_network @@ -62,6 +67,8 @@ services: image: redis:7-alpine container_name: multi_redis restart: unless-stopped + env_file: + - .env.staging ports: - "${REDIS_PORT}:6379" networks: @@ -109,11 +116,82 @@ services: networks: - multi_network + photo-worker: + image: ghcr.io/microclub-usthb/multai-back:latest + container_name: multi_photo_worker + restart: unless-stopped + env_file: + - .env.staging + depends_on: + - postgres + - redis + - nats + - minio + command: ["uv", "run", "python", "-m", "app.worker.photo_worker.main"] + networks: + - multi_network + + notification-worker: + profiles: ["firebase"] + image: ghcr.io/microclub-usthb/multai-back:latest + container_name: multi_notification_worker + restart: unless-stopped + env_file: + - .env.staging + depends_on: + - nats + - redis + command: ["uv", "run", "python", "-m", "app.worker.notification.main"] + networks: + - multi_network + + upload-group-worker: + image: ghcr.io/microclub-usthb/multai-back:latest + container_name: multi_upload_group_worker + restart: unless-stopped + env_file: + - .env.staging + depends_on: + - postgres + - redis + - nats + - minio + command: ["uv", "run", "python", "-m", "app.worker.upload_group_worker.main"] + networks: + - multi_network + + audit-worker: + image: ghcr.io/microclub-usthb/multai-back:latest + container_name: multi_audit_worker + restart: unless-stopped + env_file: + - .env.staging + depends_on: + - postgres + - nats + command: ["uv", "run", "python", "-m", "app.worker.audit.main"] + networks: + - multi_network + + storage-cleaner: + image: ghcr.io/microclub-usthb/multai-back:latest + container_name: multi_storage_cleaner + restart: unless-stopped + env_file: + - .env.staging + depends_on: + - postgres + - nats + command: ["uv", "run", "python", "-m", "app.worker.storage_cleaner.main"] + networks: + - multi_network + volumes: postgres_data: minio_data: redis_data: + nats_data: networks: multi_network: - driver: bridge + driver: bridge \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 3be7bab..0056126 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,3 +1,4 @@ +name: multai-dev services: postgres: image: pgvector/pgvector:pg16 diff --git a/makefile b/makefile index 343f4c0..ce53087 100644 --- a/makefile +++ b/makefile @@ -64,8 +64,9 @@ run-workers: uv run python -m app.worker.upload_group_worker.main & \ uv run python -m app.worker.photo_worker.main & \ uv run python -m app.worker.storage_cleaner.main & \ + uv run python -m app.worker.email_worker.main & \ wait - + lint: uv run ruff check . diff --git a/mobile-quickstart/.env.mobile.example b/mobile-quickstart/.env.mobile.example index f3cecb5..95e55f2 100644 --- a/mobile-quickstart/.env.mobile.example +++ b/mobile-quickstart/.env.mobile.example @@ -58,4 +58,4 @@ CORS_ORIGINS=["http://localhost:3000", "http://localhost:5173", "http://127.0.0. # ========================= # Firebase # ========================= -FIREBASE_CREDENTIALS_PATH= +FIREBASE_CREDENTIALS_PATH=/app/firebase-credentials.json diff --git a/mobile-quickstart/docker-compose.mobile.yml b/mobile-quickstart/docker-compose.mobile.yml index c6474a8..88ceb22 100644 --- a/mobile-quickstart/docker-compose.mobile.yml +++ b/mobile-quickstart/docker-compose.mobile.yml @@ -2,6 +2,7 @@ name: multai-mobile-test services: postgres: image: pgvector/pgvector:pg16 + container_name: mobile_postgres restart: unless-stopped env_file: .env.mobile ports: @@ -17,14 +18,18 @@ services: redis: image: redis:7-alpine + container_name: mobile_redis restart: unless-stopped ports: - "6379:6379" + volumes: + - redis_data:/data networks: - multi_network minio: image: minio/minio:latest + container_name: mobile_minio restart: unless-stopped env_file: .env.mobile command: server /data --console-address ":9001" @@ -38,19 +43,26 @@ services: nats: image: nats:2.10-alpine + container_name: mobile_nats + restart: unless-stopped command: > -js + -sd /data -a 0.0.0.0 -m 8222 --user testuser --pass testpassword ports: - "4222:4222" + - "8222:8222" + volumes: + - nats_data:/data networks: - multi_network migrate: image: ghcr.io/microclub-usthb/multai-back:latest + container_name: mobile_migrate env_file: .env.mobile depends_on: postgres: @@ -61,6 +73,7 @@ services: fastapi: image: ghcr.io/microclub-usthb/multai-back:latest + container_name: mobile_fastapi restart: unless-stopped env_file: .env.mobile depends_on: @@ -77,9 +90,89 @@ services: networks: - multi_network + # ========================================== + # BACKGROUND WORKERS POOL (Added for full app feature completeness) + # ========================================== + email-worker: + image: ghcr.io/microclub-usthb/multai-back:latest + container_name: mobile_email_worker + restart: unless-stopped + env_file: .env.mobile + depends_on: + fastapi: + condition: service_started + command: ["uv", "run", "python", "-m", "app.worker.email_worker.main"] + networks: + - multi_network + + photo-worker: + image: ghcr.io/microclub-usthb/multai-back:latest + container_name: mobile_photo_worker + restart: unless-stopped + env_file: .env.mobile + depends_on: + fastapi: + condition: service_started + command: ["uv", "run", "python", "-m", "app.worker.photo_worker.main"] + networks: + - multi_network + + notification-worker: + image: ghcr.io/microclub-usthb/multai-back:latest + container_name: mobile_notification_worker + profiles: ["firebase"] + restart: unless-stopped + env_file: .env.mobile + volumes: + - ../firebase-credentials.json:/app/firebase-credentials.json:ro + depends_on: + fastapi: + condition: service_started + command: ["uv", "run", "python", "-m", "app.worker.notification.main"] + networks: + - multi_network + + upload-group-worker: + image: ghcr.io/microclub-usthb/multai-back:latest + container_name: mobile_upload_group_worker + restart: unless-stopped + env_file: .env.mobile + depends_on: + fastapi: + condition: service_started + command: ["uv", "run", "python", "-m", "app.worker.upload_group_worker.main"] + networks: + - multi_network + + audit-worker: + image: ghcr.io/microclub-usthb/multai-back:latest + container_name: mobile_audit_worker + restart: unless-stopped + env_file: .env.mobile + depends_on: + fastapi: + condition: service_started + command: ["uv", "run", "python", "-m", "app.worker.audit.main"] + networks: + - multi_network + + storage-cleaner: + image: ghcr.io/microclub-usthb/multai-back:latest + container_name: mobile_storage_cleaner + restart: unless-stopped + env_file: .env.mobile + depends_on: + fastapi: + condition: service_started + command: ["uv", "run", "python", "-m", "app.worker.storage_cleaner.main"] + networks: + - multi_network + volumes: postgres_data: minio_data: + redis_data: + nats_data: networks: multi_network: diff --git a/mobile-quickstart/readme.md b/mobile-quickstart/readme.md index 7232b0a..63717d5 100644 --- a/mobile-quickstart/readme.md +++ b/mobile-quickstart/readme.md @@ -15,7 +15,7 @@ This folder contains everything you need to run the multAI backend locally. You * `docker-compose.mobile.yml` — defines all backend services * `.env.mobile` — environment variables (copy from `.env.mobile.example`) -* `seed.py` — database seed script +* `seed.py` — database seed script (already baked into the backend image, no manual copying needed) * `.mypy_cache/` — local Python type-check cache (safe to ignore/delete) * `README.md` — this file @@ -44,21 +44,55 @@ This pulls and starts the following containers: * **migrate** — runs database migrations once then exits * **fastapi** — the API server -### 3. Copy the seed script into the container +### 3. Seed the database ```bash -docker cp seed.py multai-mobile-test-fastapi-1:/app/seed.py +docker compose -f docker-compose.mobile.yml exec -e PYTHONPATH=/app fastapi uv run python mobile-quickstart/seed.py ``` -> **Note:** Wait for all containers to show as running before doing this. +This creates all the test data you need: users, events, photos, and notifications. + +--- + +## Background workers + +In addition to `postgres`, `redis`, `minio`, `nats`, `migrate`, and `fastapi`, the stack also starts these background workers by default: + +* **email-worker** — sends transactional emails (OTP codes, notifications) +* **photo-worker** — processes uploaded photos and face recognition +* **audit-worker** — records audit log events +* **upload-group-worker** — manages grouped photo upload requests +* **storage-cleaner** — periodically cleans up orphaned files in MinIO + +You don't need to do anything special for these — they start automatically with `docker compose up -d` and don't require any extra configuration. -### 4. Seed the database +## Notification worker (optional) +`notification-worker` is different — it needs a real Firebase service account file to run, so it's **off by default**. The rest of the backend (including all workers above) works fine without it. + +If you need push notifications working locally: + +1. Get `firebase-credentials.json` from the backend team. +2. Place it in the folder that **contains** `mobile-quickstart/` — one level above wherever you put this folder, not inside it. + + For example, if you copied `mobile-quickstart/` into your own project at `my-app/mobile-quickstart/`, put the file at `my-app/firebase-credentials.json`: + + ``` + my-app/ + ├── firebase-credentials.json ✅ correct location + ├── src/ + └── mobile-quickstart/ (this folder) + ├── docker-compose.mobile.yml + ├── .env.mobile + └── seed.py + ``` + +3. Start with the profile enabled: ```bash -docker compose -f docker-compose.mobile.yml exec fastapi uv run python seed.py +docker compose -f docker-compose.mobile.yml --profile firebase up -d ``` -This creates all the test data you need: users, events, photos, and notifications. +If you forget the `--profile firebase` flag, `notification-worker` simply won't appear in `docker compose ps` — that's expected, not an error. --- @@ -127,10 +161,9 @@ docker compose -f docker-compose.mobile.yml down -v docker compose -f docker-compose.mobile.yml pull docker compose -f docker-compose.mobile.yml up -d docker compose -f docker-compose.mobile.yml ps -docker cp seed.py multai-mobile-test-fastapi-1:/app/seed.py -docker compose -f docker-compose.mobile.yml exec fastapi uv run python seed.py +docker compose -f docker-compose.mobile.yml exec -e PYTHONPATH=/app fastapi uv run python mobile-quickstart/seed.py ``` -> `down -v` removes all volumes. `pull` grabs the latest backend image before recreating containers. Check the `ps` output shows all containers as `Up`/`Healthy` before running the seed step — copying the seed script too early (before the fastapi container is ready) will fail. +> `down -v` removes all volumes. `pull` grabs the latest backend image before recreating containers. Check the `ps` output shows all containers as `Up`/`Healthy` before running the seed step — running it too early (before the fastapi container is ready) will fail. --- @@ -140,7 +173,6 @@ When the backend team pushes a new version: docker compose -f docker-compose.mobile.yml pull docker compose -f docker-compose.mobile.yml up -d docker compose -f docker-compose.mobile.yml ps -docker cp seed.py multai-mobile-test-fastapi-1:/app/seed.py -docker compose -f docker-compose.mobile.yml exec fastapi uv run python seed.py +docker compose -f docker-compose.mobile.yml exec -e PYTHONPATH=/app fastapi uv run python mobile-quickstart/seed.py ``` -> Re-copy the seed script after updating since the container is recreated. Run `ps` first to confirm the new container is actually up before copying/seeding. \ No newline at end of file +> Run `ps` first to confirm the new container is actually up before seeding — the seed script ships inside the image, so no manual copying is needed. \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 2fc41df..89d4ad8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,7 +20,6 @@ dependencies = [ "pyjwt>=2.11.0", "pyotp>=2.9.0", "redis>=7.2.1", - "setuptools>=82.0.0", "opencv-python-headless>=4.13.0.92", "numpy>=2.4.3", "insightface>=0.7.3", @@ -28,10 +27,10 @@ dependencies = [ "python-multipart>=0.0.22", "firebase-admin>=6.8.0", "pywebpush>=2.3.0", - "opencv-python>=4.13.0.92", "filetype>=1.2.0", "pillow-heif>=1.3.0", "sqlalchemy>=2.0.47", + "setuptools>=83.0.0", ] [tool.ruff] diff --git a/scripts/generate_drive_url.py b/scripts/generate_drive_url.py index 91a5802..319afdd 100644 --- a/scripts/generate_drive_url.py +++ b/scripts/generate_drive_url.py @@ -1,7 +1,7 @@ import asyncio from sqlalchemy.ext.asyncio import create_async_engine from app.core.config import settings -from db.generated import stuff_user as staff_queries +from db.generated import staff_user as staff_queries from app.infra.redis import RedisClient from app.service.staff_drive import StaffDriveService from db.generated import staff_drive_connections as drive_queries diff --git a/scripts/list_drive.py b/scripts/list_drive.py index d4940bf..8651bdd 100644 --- a/scripts/list_drive.py +++ b/scripts/list_drive.py @@ -1,7 +1,7 @@ import asyncio from sqlalchemy.ext.asyncio import create_async_engine from app.core.config import settings -from db.generated import stuff_user as staff_queries +from db.generated import staff_user as staff_queries from app.infra.redis import RedisClient from app.infra.google_drive import GoogleDriveClient from app.service.staff_drive import StaffDriveService diff --git a/scripts/trigger_import.py b/scripts/trigger_import.py index f309878..86bb226 100644 --- a/scripts/trigger_import.py +++ b/scripts/trigger_import.py @@ -1,7 +1,7 @@ import asyncio from sqlalchemy.ext.asyncio import create_async_engine from app.core.config import settings -from db.generated import stuff_user as staff_queries +from db.generated import staff_user as staff_queries from app.infra.redis import RedisClient from app.infra.google_drive import GoogleDriveClient from app.infra.minio import init_minio_client diff --git a/scripts/trigger_upload_request.py b/scripts/trigger_upload_request.py index 639579e..2b6361a 100644 --- a/scripts/trigger_upload_request.py +++ b/scripts/trigger_upload_request.py @@ -1,7 +1,7 @@ import asyncio from sqlalchemy.ext.asyncio import create_async_engine from app.core.config import settings -from db.generated import stuff_user as staff_queries +from db.generated import staff_user as staff_queries from db.generated import upload_request_groups as group_queries from db.generated import upload_requests as request_queries from db.generated import upload_request_photos as request_photo_queries diff --git a/tests/e2e/test_stats_endpoint.py b/tests/e2e/test_stats_endpoint.py index f16ca7b..787836d 100644 --- a/tests/e2e/test_stats_endpoint.py +++ b/tests/e2e/test_stats_endpoint.py @@ -9,6 +9,9 @@ @pytest.fixture(scope="module") def client() -> Generator[TestClient, None, None]: + import os + if os.getenv("MULTAI_RUN_E2E") != "1": + pytest.skip("set MULTAI_RUN_E2E=1 to run live e2e tests") # Override the dependency to bypass cookie auth mock_admin = StaffUser( id=uuid.uuid4(), diff --git a/tests/integration/test_photo_approval_flow.py b/tests/integration/test_photo_approval_flow.py index 2606df8..2cfb9f4 100644 --- a/tests/integration/test_photo_approval_flow.py +++ b/tests/integration/test_photo_approval_flow.py @@ -14,7 +14,7 @@ from app.service.photo_approval import PhotoApprovalService from db.generated import user as user_queries -from db.generated import stuff_user as staff_queries +from db.generated import staff_user as staff_queries from db.generated import events as event_queries from db.generated import photos as photo_queries from db.generated import photo_approvals as approval_queries diff --git a/uv.lock b/uv.lock index 87c2bf7..b403b75 100644 --- a/uv.lock +++ b/uv.lock @@ -313,7 +313,6 @@ dependencies = [ { name = "nats-py" }, { name = "numpy" }, { name = "onnxruntime" }, - { name = "opencv-python" }, { name = "opencv-python-headless" }, { name = "passlib", extra = ["bcrypt"] }, { name = "pillow-heif" }, @@ -353,7 +352,6 @@ requires-dist = [ { name = "nats-py", specifier = ">=2.14.0" }, { name = "numpy", specifier = ">=2.4.3" }, { name = "onnxruntime", specifier = ">=1.24.4" }, - { name = "opencv-python", specifier = ">=4.13.0.92" }, { name = "opencv-python-headless", specifier = ">=4.13.0.92" }, { name = "passlib", extras = ["bcrypt"], specifier = ">=1.7.4" }, { name = "pillow-heif", specifier = ">=1.3.0" }, @@ -365,7 +363,7 @@ requires-dist = [ { name = "python-multipart", specifier = ">=0.0.22" }, { name = "pywebpush", specifier = ">=2.3.0" }, { name = "redis", specifier = ">=7.2.1" }, - { name = "setuptools", specifier = ">=82.0.0" }, + { name = "setuptools", specifier = ">=83.0.0" }, { name = "sqlalchemy", specifier = ">=2.0.47" }, ] @@ -2196,24 +2194,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/6c/1d/1666dc64e78d8587d168fec4e3b7922b92eb286a2ddeebcf6acb55c7dc82/onnxruntime-1.24.4-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e1cc6a518255f012134bc791975a6294806be9a3b20c4a54cca25194c90cf731", size = 17247021, upload-time = "2026-03-17T22:04:52.377Z" }, ] -[[package]] -name = "opencv-python" -version = "4.13.0.92" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "numpy" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/fc/6f/5a28fef4c4a382be06afe3938c64cc168223016fa520c5abaf37e8862aa5/opencv_python-4.13.0.92-cp37-abi3-macosx_13_0_arm64.whl", hash = "sha256:caf60c071ec391ba51ed00a4a920f996d0b64e3e46068aac1f646b5de0326a19", size = 46247052, upload-time = "2026-02-05T07:01:25.046Z" }, - { url = "https://files.pythonhosted.org/packages/08/ac/6c98c44c650b8114a0fb901691351cfb3956d502e8e9b5cd27f4ee7fbf2f/opencv_python-4.13.0.92-cp37-abi3-macosx_14_0_x86_64.whl", hash = "sha256:5868a8c028a0b37561579bfb8ac1875babdc69546d236249fff296a8c010ccf9", size = 32568781, upload-time = "2026-02-05T07:01:41.379Z" }, - { url = "https://files.pythonhosted.org/packages/3e/51/82fed528b45173bf629fa44effb76dff8bc9f4eeaee759038362dfa60237/opencv_python-4.13.0.92-cp37-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:0bc2596e68f972ca452d80f444bc404e08807d021fbba40df26b61b18e01838a", size = 47685527, upload-time = "2026-02-05T06:59:11.24Z" }, - { url = "https://files.pythonhosted.org/packages/db/07/90b34a8e2cf9c50fe8ed25cac9011cde0676b4d9d9c973751ac7616223a2/opencv_python-4.13.0.92-cp37-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:402033cddf9d294693094de5ef532339f14ce821da3ad7df7c9f6e8316da32cf", size = 70460872, upload-time = "2026-02-05T06:59:19.162Z" }, - { url = "https://files.pythonhosted.org/packages/02/6d/7a9cc719b3eaf4377b9c2e3edeb7ed3a81de41f96421510c0a169ca3cfd4/opencv_python-4.13.0.92-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:bccaabf9eb7f897ca61880ce2869dcd9b25b72129c28478e7f2a5e8dee945616", size = 46708208, upload-time = "2026-02-05T06:59:15.419Z" }, - { url = "https://files.pythonhosted.org/packages/fd/55/b3b49a1b97aabcfbbd6c7326df9cb0b6fa0c0aefa8e89d500939e04aa229/opencv_python-4.13.0.92-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:620d602b8f7d8b8dab5f4b99c6eb353e78d3fb8b0f53db1bd258bb1aa001c1d5", size = 72927042, upload-time = "2026-02-05T06:59:23.389Z" }, - { url = "https://files.pythonhosted.org/packages/fb/17/de5458312bcb07ddf434d7bfcb24bb52c59635ad58c6e7c751b48949b009/opencv_python-4.13.0.92-cp37-abi3-win32.whl", hash = "sha256:372fe164a3148ac1ca51e5f3ad0541a4a276452273f503441d718fab9c5e5f59", size = 30932638, upload-time = "2026-02-05T07:02:14.98Z" }, - { url = "https://files.pythonhosted.org/packages/e9/a5/1be1516390333ff9be3a9cb648c9f33df79d5096e5884b5df71a588af463/opencv_python-4.13.0.92-cp37-abi3-win_amd64.whl", hash = "sha256:423d934c9fafb91aad38edf26efb46da91ffbc05f3f59c4b0c72e699720706f5", size = 40212062, upload-time = "2026-02-05T07:02:12.724Z" }, -] - [[package]] name = "opencv-python-headless" version = "4.13.0.92" @@ -3236,11 +3216,11 @@ wheels = [ [[package]] name = "setuptools" -version = "82.0.0" +version = "83.0.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/82/f3/748f4d6f65d1756b9ae577f329c951cda23fb900e4de9f70900ced962085/setuptools-82.0.0.tar.gz", hash = "sha256:22e0a2d69474c6ae4feb01951cb69d515ed23728cf96d05513d36e42b62b37cb", size = 1144893, upload-time = "2026-02-08T15:08:40.206Z" } +sdist = { url = "https://files.pythonhosted.org/packages/34/26/f5d29e25ffdb535afef2d35cdb55b325298f96debd670da4c325e08d70f4/setuptools-83.0.0.tar.gz", hash = "sha256:025bccbbf0fa05b6192bc64ae1e7b16e001fd6d6d4d5de03c97b1c1ade523bef", size = 1154254, upload-time = "2026-07-04T15:31:22.699Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/e1/c6/76dc613121b793286a3f91621d7b75a2b493e0390ddca50f11993eadf192/setuptools-82.0.0-py3-none-any.whl", hash = "sha256:70b18734b607bd1da571d097d236cfcfacaf01de45717d59e6e04b96877532e0", size = 1003468, upload-time = "2026-02-08T15:08:38.723Z" }, + { url = "https://files.pythonhosted.org/packages/5d/40/e1e72872c6354b306daef1703549e8e83b4d43cfea356311bf722a043752/setuptools-83.0.0-py3-none-any.whl", hash = "sha256:29b23c360f22f414dc7336bb39178cc7bcbf6021ed2733cde173f09dba19abb3", size = 1008090, upload-time = "2026-07-04T15:31:20.885Z" }, ] [[package]]