Skip to content
Merged

Fixs #63

Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .env.example
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ totp_issuer=MultiAI

GOOGLE_CLIENT_ID=
GOOGLE_CLIENT_SECRET=
GOOGLE_REDIRECT_URI=http://127.0.0.1:8000/stuff/drive/callback
GOOGLE_REDIRECT_URI=http://127.0.0.1:8000/staff/drive/callback
GOOGLE_OAUTH_SCOPES=https://www.googleapis.com/auth/drive.readonly openid email profile
FACE_ENCRYPTION_KEY=hkbribvfirirbvivbibvib

Expand Down
4 changes: 3 additions & 1 deletion .env.staging.example
Original file line number Diff line number Diff line change
Expand Up @@ -49,4 +49,6 @@ GOOGLE_CLIENT_SECRET=
GOOGLE_REDIRECT_URI=http://127.0.0.1:8000/staff/drive/callback
GOOGLE_OAUTH_SCOPES=https://www.googleapis.com/auth/drive.readonly openid email profile

FACE_ENCRYPTION_KEY=base64-encoded-32-byte-key
FACE_ENCRYPTION_KEY=base64-encoded-32-byte-key

FIREBASE_CREDENTIALS_PATH=/app/firebase-credentials.json
7 changes: 3 additions & 4 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -1,17 +1,16 @@
.env
.env.staging
.env.mobile
__pycache__
db/schema.sql

.vscode/settings.json
multiai-c9380-firebase-adminsdk-fbsvc-cb6e5ce41b.json
firebase-credentials.json
db.txt

.venv
multiai-c9380-firebase-adminsdk-fbsvc-cb6e5ce41b.json

# Local Test & Debug Files
.coverage
dummy.json
test_output.log
*.log
*.log
6 changes: 2 additions & 4 deletions app/container.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,15 +26,14 @@
from db.generated import session as session_queries
from db.generated import staff_drive_connections as staff_drive_queries
from db.generated import staff_notifications as staff_notification_queries
from db.generated import stuff_user as staff_user_queries
from db.generated import staff_user as staff_user_queries
from db.generated import upload_request_groups as upload_request_group_queries
from db.generated import upload_request_photos as upload_request_photo_queries
from db.generated import upload_requests as upload_request_queries
from db.generated import user as user_queries

from db.generated import events as event_queries
from db.generated import eventParticipant as participant_queries
from db.generated import stuff_user as staff_queries
from db.generated import event_participant as participant_queries
from db.generated import notifications as notification_queries
from db.generated import audit as audit_queries
from db.generated import stats as stats_queries
Expand Down Expand Up @@ -72,7 +71,6 @@ def __init__(
self.audit_querier = audit_queries.AsyncQuerier(conn)
self.event_querier = event_queries.AsyncQuerier(conn)
self.participant_querier = participant_queries.AsyncQuerier(conn)
self.staff_querier = staff_queries.AsyncQuerier(conn)
self.stats_querier = stats_queries.AsyncQuerier(conn)

# services
Expand Down
11 changes: 6 additions & 5 deletions app/infra/nats.py
Original file line number Diff line number Diff line change
Expand Up @@ -62,11 +62,12 @@ async def connect(
@staticmethod
async def close() -> None:
if NatsClient._nc and not NatsClient._nc.is_closed:
await NatsClient._nc.drain()
await NatsClient._nc.close()
NatsClient._nc = None
NatsClient._js = None

try:
await NatsClient._nc.drain()
await NatsClient._nc.close()
finally:
NatsClient._nc = None
NatsClient._js = None

@staticmethod
async def publish(subject: NatsSubjects | str, message: bytes) -> None:
Expand Down
1 change: 1 addition & 0 deletions app/infra/redis.py
Original file line number Diff line number Diff line change
Expand Up @@ -77,3 +77,4 @@ async def srem(self, key: RedisKey | str, *values: str) -> int:

async def close(self) -> None:
await self._client.close()
type(self)._instance = None
8 changes: 4 additions & 4 deletions app/router/staff/uploads.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,9 +14,9 @@
RejectUploadRequestRequest,
)
from app.schema.response.staff.upload_groups import (
UploadRequestGroupListResponse,
UploadRequestGroupPhotoListResponse,
UploadRequestGroupSchema,
UploadRequestGroupSummaryListResponse,
)
from app.schema.response.staff.uploads import (
UploadRequestListResponse,
Expand Down Expand Up @@ -67,19 +67,19 @@ async def list_upload_requests(
)


@router.get("/groups", response_model=UploadRequestGroupListResponse)
@router.get("/groups", response_model=UploadRequestGroupSummaryListResponse)
async def list_upload_request_groups(
scope: Literal["my", "all"] = Query(default="my"),
status: UploadRequestStatus | None = Query(default=None),
current_staff_user: StaffUser = Depends(get_current_staff_user),
container: Container = Depends(get_container),
) -> UploadRequestGroupListResponse:
) -> UploadRequestGroupSummaryListResponse:
groups = await container.upload_requests_service.list_groups(
current_staff_user=current_staff_user,
scope=scope,
status=status.value if status is not None else None,
)
return UploadRequestGroupListResponse.from_details_list(groups)
return UploadRequestGroupSummaryListResponse.from_groups(groups)


@router.get("/groups/{group_id}", response_model=UploadRequestGroupSchema)
Expand Down
31 changes: 30 additions & 1 deletion app/schema/response/staff/upload_groups.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
UploadRequestSchema,
)
from app.service.upload_requests import UploadRequestGroupDetails
from db.generated.models import UploadRequestPhoto
from db.generated.models import UploadRequestGroup, UploadRequestPhoto


class UploadRequestGroupSchema(BaseModel):
Expand Down Expand Up @@ -67,3 +67,32 @@ def from_photos(
return cls(
items=[UploadRequestPhotoSchema.model_validate(p) for p in photos]
)

class UploadRequestGroupSummarySchema(BaseModel):
model_config = ConfigDict(from_attributes=True)
id: UUID
event_id: UUID
folder_id: str
status: str
processing_status: str
total_photo_count: int
batch_count: int
processed_photo_count: int
failed_photo_count: int
created_at: datetime
approved_at: datetime | None
rejection_reason: str | None
error_message: str | None

@field_validator("status", mode="before")
@classmethod
def coerce_status(cls, v: object) -> str:
return getattr(v, "value", str(v))


class UploadRequestGroupSummaryListResponse(BaseModel):
items: list[UploadRequestGroupSummarySchema]

@classmethod
def from_groups(cls, groups: list["UploadRequestGroup"]) -> "UploadRequestGroupSummaryListResponse":
return cls(items=[UploadRequestGroupSummarySchema.model_validate(g) for g in groups])
2 changes: 1 addition & 1 deletion app/service/event.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
ParticipantResponse
)
from db.generated import events as event_queries
from db.generated import eventParticipant as participant_queries
from db.generated import event_participant as participant_queries
from db.generated import models

class EventService:
Expand Down
2 changes: 1 addition & 1 deletion app/service/staff_drive.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
from app.infra.minio import ImageBucket
from app.infra.redis import RedisClient
from db.generated import staff_drive_connections as drive_queries
from db.generated import stuff_user as staff_queries
from db.generated import staff_user as staff_queries
from db.generated.models import StaffDriveConnection, StaffUser


Expand Down
4 changes: 2 additions & 2 deletions app/service/staff_user.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,8 @@
from app.core.exceptions import AppException, DBException, DBExceptionImpl
from app.core.securite import create_access_staff_token, hash_password, verify_password
from app.schema.response.web.auth import WebAuthResponse
from db.generated import stuff_user as staff_queries
from db.generated.stuff_user import ListStaffUsersParams
from db.generated import staff_user as staff_queries
from db.generated.staff_user import ListStaffUsersParams
from db.generated.models import StaffUser, StaffRole


Expand Down
12 changes: 2 additions & 10 deletions app/service/upload_requests.py
Original file line number Diff line number Diff line change
Expand Up @@ -890,7 +890,7 @@ async def list_groups(
current_staff_user: StaffUser,
scope: Literal["my", "all"],
status: str | None,
) -> list[UploadRequestGroupDetails]:
) -> list[UploadRequestGroup]:
if scope == "all" and self._role_value(current_staff_user.role) != StaffRole.MULTI_TEAM_LEAD.value:
raise AppException.forbidden("Multi team lead access required")

Expand All @@ -915,15 +915,7 @@ async def list_groups(
async for group in iterator:
groups.append(group)

details: list[UploadRequestGroupDetails] = []
for group in groups:
details.append(
await self.get_group_details(
group_id=group.id,
current_staff_user=current_staff_user,
)
)
return details
return groups

async def list_group_photos(
self,
Expand Down
24 changes: 13 additions & 11 deletions app/worker/notification/settings.py
Original file line number Diff line number Diff line change
@@ -1,37 +1,39 @@
from __future__ import annotations

from typing import ClassVar, Sequence

from pydantic import Field
from pydantic_settings import BaseSettings, SettingsConfigDict

from app.schema.internal.notification import NotificationPriority, PRIORITY_ORDER


class NotificationWorkerSettings(BaseSettings):
subject_prefix: str = Field("notifications.delivery")
queue_group: str | None = Field(None)
redis_host: str = Field("localhost")
redis_port: int = Field(6379)
redis_password: str = Field("")
nats_host: str = Field("localhost")
nats_port: int = Field(4222)
nats_user: str = Field("")
nats_password: str = Field("")

redis_host: str
redis_port: int
redis_password: str = ""

nats_host: str
nats_port: int
nats_user: str
nats_password: str

firebase_credentials_path: str | None = Field(None)

MAX_SEND_ATTEMPTS: ClassVar[int] = 5
BASE_RETRY_DELAY: ClassVar[int] = 2
TTL_SECONDS: ClassVar[int] = 30 * 24 * 3600
CONCURRENCY: ClassVar[int] = 10
RATE_LIMIT: ClassVar[int] = 50
RATE_PERIOD: ClassVar[float] = 1.0

model_config = SettingsConfigDict(env_prefix="NOTIFICATIONS_")
model_config = SettingsConfigDict(env_file=".env", extra="ignore")

def subject_for(self, priority: NotificationPriority) -> str:
return f"{self.subject_prefix}.{priority.value}"

def priority_subjects(self) -> Sequence[str]:
return [self.subject_for(priority) for priority in PRIORITY_ORDER]


NotifSetting = NotificationWorkerSettings() # type: ignore
18 changes: 15 additions & 3 deletions app/worker/photo_worker/tests/test_photo_worker.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,14 @@

import pytest

sys.modules["db.generated.user"] = MagicMock()
sys.modules["app.worker.notification.settings"] = MagicMock()
sys.modules["app.worker.notification.notification_queue"] = MagicMock()
_MOCKED_MODULES = (
"db.generated.user",
"app.worker.notification.settings",
"app.worker.notification.notification_queue",
)
_original_modules = {name: sys.modules.get(name) for name in _MOCKED_MODULES}
for _name in _MOCKED_MODULES:
sys.modules[_name] = MagicMock()

from app.service.face_embedding import DetectedFace, FaceEmbeddingService, FaceImagePayload # noqa: E402
from app.service.face_match import SingleFaceMatchService # noqa: E402
Expand All @@ -16,6 +21,13 @@
from app.worker.photo_worker.schema.event import PhotoProcessEvent # noqa: E402
from db.generated import photo_faces as photo_face_queries # noqa: E402

# Restore sys.modules so other test files importing these modules
# (e.g. db.generated.user) get the real thing, not this leaked mock.
for _name, _original in _original_modules.items():
if _original is None:
sys.modules.pop(_name, None)
else:
sys.modules[_name] = _original

# ── fixtures ──────────────────────────────────────────────────────────

Expand Down
22 changes: 4 additions & 18 deletions app/worker/storage_cleaner/main.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,8 @@
import asyncio
import json
import uuid
from typing import Iterable, Optional, Set, Tuple
from typing import Iterable, Optional, Set

import sqlalchemy.ext.asyncio
from fastapi import HTTPException
from pydantic import BaseModel, ValidationError

Expand All @@ -26,19 +25,6 @@ class FinalBucketCleanupPayload(BaseModel):
storage_service = StagedUploadStorageService()


async def create_photo_querier() -> Tuple[
sqlalchemy.ext.asyncio.AsyncConnection,
upload_request_photo_queries.AsyncQuerier,
]:
conn = await engine.connect()
querier = upload_request_photo_queries.AsyncQuerier(conn)
return conn, querier


async def close_connection(conn: sqlalchemy.ext.asyncio.AsyncConnection) -> None:
await conn.close()


def _parse_payload(raw_data: bytes | str) -> Optional[FinalBucketCleanupPayload]:
if isinstance(raw_data, bytes):
try:
Expand Down Expand Up @@ -131,11 +117,12 @@ async def _handle_cleanup_event(


async def main() -> None:
conn, querier = await create_photo_querier()
await NatsClient.connect()
try:
async def _jetstream_handler(data: bytes | str) -> None:
await _handle_cleanup_event(data, querier)
async with engine.begin() as conn:
querier = upload_request_photo_queries.AsyncQuerier(conn)
await _handle_cleanup_event(data, querier)

await NatsClient.js_subscribe(
subject=settings.subject_enum,
Expand All @@ -150,7 +137,6 @@ async def _jetstream_handler(data: bytes | str) -> None:
)
await asyncio.Event().wait()
finally:
await close_connection(conn)
await NatsClient.close()


Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
# Code generated by sqlc. DO NOT EDIT.
# versions:
# sqlc v1.31.1
# source: eventParticipant.sql
# source: event_participant.sql
import dataclasses
import datetime
from typing import Any, AsyncIterator, Optional
Expand Down
Loading