From f18e7578a434551c9d115b18a6f09d68a9e86873 Mon Sep 17 00:00:00 2001 From: skyeslattery Date: Sun, 24 Aug 2025 14:12:44 -0400 Subject: [PATCH 1/3] fix n+1 query problem and add connection pooling --- src/category/views.py | 7 +++ src/eatery/views.py | 26 ++++++++- .../management/commands/populate_models.py | 25 ++++++++- src/eatery_blue_backend/settings.py | 8 +++ src/event/views.py | 7 +++ src/item/controllers/populate_item.py | 56 ++++++++++++++----- src/item/serializers.py | 43 +++++++------- src/item/views.py | 8 +++ 8 files changed, 141 insertions(+), 39 deletions(-) diff --git a/src/category/views.py b/src/category/views.py index bda65d2..3a16043 100644 --- a/src/category/views.py +++ b/src/category/views.py @@ -6,3 +6,10 @@ class CategoryViewSet(viewsets.ModelViewSet): queryset = Category.objects.all() serializer_class = CategorySerializer + + def get_queryset(self): + # prefetch items and their related fields + return Category.objects.select_related('event__eatery').prefetch_related( + 'items__dietary_preferences', + 'items__allergens' + ).all() \ No newline at end of file diff --git a/src/eatery/views.py b/src/eatery/views.py index 3c6f6db..7d71021 100644 --- a/src/eatery/views.py +++ b/src/eatery/views.py @@ -27,6 +27,18 @@ class EateryViewSet(viewsets.ModelViewSet): serializer_class = EaterySerializer permission_classes = [EateryPermission] + def get_queryset(self): + """ + Override to add prefetch_related for optimization + """ + queryset = super().get_queryset() + + # prefetch all related objects to avoid N+1 query problem + return queryset.prefetch_related( + 'events__menu__items__dietary_preferences', + 'events__menu__items__allergens' + ) + def retrieve(self, request, *args, **kwargs): instance = self.get_object() serializer = EaterySerializerOptimized(instance) @@ -98,7 +110,10 @@ class GetEateriesSimple(APIView): """ def get(self, request): - eateries = EaterySerializerSimple(Eatery.objects.all(), many=True) + eateries_queryset = Eatery.objects.prefetch_related( + 'events' + ).all() + eateries = EaterySerializerSimple(eateries_queryset, many=True) return Response(eateries.data) @@ -109,9 +124,14 @@ class GetEateriesByDay(APIView): @method_decorator(cache_page(60 * 60 * 2)) # cache for 2 hours def get(self, request, day): + eateries_queryset = Eatery.objects.prefetch_related( + 'events__menu__items__dietary_preferences', + 'events__menu__items__allergens' + ).exclude(events__event_description="Open") + eateries = EaterySerializerByDay( - Eatery.objects.exclude(events__event_description="Open"), + eateries_queryset, many=True, context={"day": day}, ) - return Response(eateries.data) + return Response(eateries.data) \ No newline at end of file diff --git a/src/eatery_blue_backend/management/commands/populate_models.py b/src/eatery_blue_backend/management/commands/populate_models.py index 89e1d13..87fc5fe 100644 --- a/src/eatery_blue_backend/management/commands/populate_models.py +++ b/src/eatery_blue_backend/management/commands/populate_models.py @@ -10,6 +10,7 @@ from category.controllers.populate_category import PopulateCategoryController import os import json +import shutil class Command(BaseCommand): @@ -33,18 +34,36 @@ def get_json(self): json_eateries = response["data"]["eateries"] return json_eateries + def ensure_external_eateries_exists(self): + external_path = "./static_sources/external_eateries.json" + static_path = "./static_sources/external_eateries_static.json" + + # if external_eateries.json doesn't exist, copy from static + if not os.path.exists(external_path): + if os.path.exists(static_path): + shutil.copy2(static_path, external_path) + print("Created external_eateries.json from static template") + else: + # create fallback file + data = {"eateries": []} + with open(external_path, "w") as f: + json.dump(data, f, indent=2) + print("Created minimal external_eateries.json") + def update_freedge_external_eatery(self): + self.ensure_external_eateries_exists() + GOOGLE_SHEETS_API_KEY = os.environ.get("GOOGLE_SHEETS_API_KEY") FREEDGE_SHEET_ID = os.environ.get("FREEDGE_SHEET_ID") FREEDGE_APPROVED_EMAILS = os.environ.get("FREEDGE_APPROVED_EMAILS") if not GOOGLE_SHEETS_API_KEY: - print("GOOGLE_SHEETS_API_KEY not set, cannot update freedge external eatery") + print("GOOGLE_SHEETS_API_KEY not set, skipping freedge update") return if not FREEDGE_SHEET_ID: - print("FREEDGE_SHEET_ID not set, cannot update freedge external eatery") + print("FREEDGE_SHEET_ID not set, skipping freedge update") return if not FREEDGE_APPROVED_EMAILS: - print("FREEDGE_APPROVED_EMAILS not set, cannot update freedge external eatery") + print("FREEDGE_APPROVED_EMAILS not set, skipping freedge update") return approved_emails = FREEDGE_APPROVED_EMAILS.split(",") diff --git a/src/eatery_blue_backend/settings.py b/src/eatery_blue_backend/settings.py index 27a1841..82ff7d4 100644 --- a/src/eatery_blue_backend/settings.py +++ b/src/eatery_blue_backend/settings.py @@ -108,6 +108,14 @@ "PASSWORD": os.getenv("POSTGRES_PASSWORD"), "HOST": os.getenv("POSTGRES_HOST"), "PORT": os.getenv("POSTGRES_PORT"), + "POOL_OPTIONS": { + "POOL_SIZE": 10, + "MAX_OVERFLOW": 10, + "RECYCLE": 300, + "PRE_PING": True, + "ECHO": False, + "TIMEOUT": 30, + } } } diff --git a/src/event/views.py b/src/event/views.py index 85b3022..da3135c 100644 --- a/src/event/views.py +++ b/src/event/views.py @@ -6,3 +6,10 @@ class EventViewSet(viewsets.ModelViewSet): queryset = Event.objects.all() serializer_class = EventSerializer + + def get_queryset(self): + # prefetch related menu categories and items + return Event.objects.select_related('eatery').prefetch_related( + 'menu__items__dietary_preferences', + 'menu__items__allergens' + ).all() \ No newline at end of file diff --git a/src/item/controllers/populate_item.py b/src/item/controllers/populate_item.py index 34a26a9..8a12d74 100644 --- a/src/item/controllers/populate_item.py +++ b/src/item/controllers/populate_item.py @@ -1,4 +1,4 @@ -from item.serializers import ItemSerializer +from item.models import Item, DietaryPreference, Allergen import json from util.constants import eatery_is_cafe @@ -7,6 +7,30 @@ class PopulateItemController: def __init__(self): self = self + def create_item_with_m2m(self, category_id, name, dietary_preferences=None, allergens=None): + # handle many to many relationships + item, created = Item.objects.get_or_create( + category_id=category_id, + name=name, + defaults={'base_price': 0.0} + ) + + # handle dietary preferences + if dietary_preferences: + for pref_name in dietary_preferences: + if pref_name: + pref, _ = DietaryPreference.objects.get_or_create(name=pref_name) + item.dietary_preferences.add(pref) + + # handle allergens + if allergens: + for allergen_name in allergens: + if allergen_name: + allergen, _ = Allergen.objects.get_or_create(name=allergen_name) + item.allergens.add(allergen) + + return item + def generate_cafe_items(self, menu, json_eatery): for json_item in json_eatery["diningItems"]: category_name = json_item["category"].strip() @@ -18,28 +42,32 @@ def generate_cafe_items(self, menu, json_eatery): dietary_preferences = json_item.get("dietaryPreferences", []) allergens = json_item.get("allergens", []) - data = {"category": category_id, "name": json_item["item"], "dietary_preferences": dietary_preferences, "allergens": allergens} - item = ItemSerializer(data=data) - if item.is_valid(): - item.save() - else: - print(item.errors) + self.create_item_with_m2m( + category_id=category_id, + name=json_item["item"], + dietary_preferences=dietary_preferences, + allergens=allergens + ) def generate_dining_hall_items(self, menu, json_event, json_eatery): json_menus = json_event["menu"] for json_menu in json_menus: category_name = json_menu["category"].strip() - category_id = menu[category_name] + try: + category_id = menu[category_name] + except KeyError: + continue for json_item in json_menu["items"]: dietary_preferences = json_item.get("dietaryPreferences", []) allergens = json_item.get("allergens", []) - data = {"category": category_id, "name": json_item["item"], "dietary_preferences": dietary_preferences, "allergens": allergens} - item = ItemSerializer(data=data) - if item.is_valid(): - item.save() - else: - print(item.errors) + + self.create_item_with_m2m( + category_id=category_id, + name=json_item["item"], + dietary_preferences=dietary_preferences, + allergens=allergens + ) def process(self, categories_dict, json_eateries): with open( diff --git a/src/item/serializers.py b/src/item/serializers.py index 769ce15..7bd513b 100644 --- a/src/item/serializers.py +++ b/src/item/serializers.py @@ -5,31 +5,36 @@ class ItemSerializer(serializers.ModelSerializer): id = serializers.IntegerField(read_only=True) name = serializers.CharField(default="Item") - dietary_preferences = serializers.ListField( - child=serializers.CharField(), allow_empty=True, default=[] - ) - allergens = serializers.ListField( - child=serializers.CharField(), allow_empty=True, default=[] - ) + dietary_preferences = serializers.SerializerMethodField() + allergens = serializers.SerializerMethodField() - def create(self, validated_data): - dietary_prefs = validated_data.pop('dietary_preferences', []) - allergens = validated_data.pop('allergens', []) - item, _ = Item.objects.get_or_create(**validated_data) - - for pref_name in dietary_prefs: - pref, _ = DietaryPreference.objects.get_or_create(name=pref_name) - item.dietary_preferences.add(pref) - - for allergen_name in allergens: - allergen, _ = Allergen.objects.get_or_create(name=allergen_name) - item.allergens.add(allergen) + def get_dietary_preferences(self, obj): + """Get list of dietary preference names""" + return list(obj.dietary_preferences.values_list('name', flat=True)) + + def get_allergens(self, obj): + """Get list of allergen names""" + return list(obj.allergens.values_list('name', flat=True)) + def create(self, validated_data): + # handle m2m fields differently + item, _ = Item.objects.get_or_create( + category=validated_data.get('category'), + name=validated_data.get('name'), + base_price=validated_data.get('base_price', 0.0) + ) return item + + def update(self, instance, validated_data): + instance.name = validated_data.get('name', instance.name) + instance.base_price = validated_data.get('base_price', instance.base_price) + instance.save() + return instance class Meta: model = Item - fields = ["id", "category", "name", "dietary_preferences", "allergens"] + fields = ["id", "category", "name", "base_price", "dietary_preferences", "allergens"] + read_only_fields = ["id"] class ItemSerializerOptimized(serializers.ModelSerializer): diff --git a/src/item/views.py b/src/item/views.py index 9ca7171..8de7fb7 100644 --- a/src/item/views.py +++ b/src/item/views.py @@ -6,3 +6,11 @@ class ItemViewSet(viewsets.ModelViewSet): queryset = Item.objects.all() serializer_class = ItemSerializer + + def get_queryset(self): + return Item.objects.select_related( + 'category__event__eatery' + ).prefetch_related( + 'dietary_preferences', + 'allergens' + ).all() \ No newline at end of file From f48323c398c9a07c2f3703ddb1f7e6f98098d881 Mon Sep 17 00:00:00 2001 From: skyeslattery Date: Sun, 24 Aug 2025 14:45:49 -0400 Subject: [PATCH 2/3] switch to dj_db_conn_pool.backends.postgresql engine --- src/eatery_blue_backend/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/eatery_blue_backend/settings.py b/src/eatery_blue_backend/settings.py index 82ff7d4..4d89281 100644 --- a/src/eatery_blue_backend/settings.py +++ b/src/eatery_blue_backend/settings.py @@ -102,7 +102,7 @@ DATABASES = { "default": { - "ENGINE": "django.db.backends.postgresql_psycopg2", + "ENGINE": "dj_db_conn_pool.backends.postgresql", "NAME": os.getenv("POSTGRES_NAME"), "USER": os.getenv("POSTGRES_USER"), "PASSWORD": os.getenv("POSTGRES_PASSWORD"), From 4e2c7c7b79979642a6c001e4f87a27ba53a11fde Mon Sep 17 00:00:00 2001 From: skyeslattery Date: Fri, 29 Aug 2025 13:20:25 -0400 Subject: [PATCH 3/3] update requirements.txt --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index f9c09d7..e654118 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,6 +8,7 @@ coreapi==2.3.3 coreschema==0.0.4 distlib==0.3.8 Django==4.0 +django-db-connection-pool==1.2.5 django-rest-swagger==2.2.0 djangorestframework==3.13.1 drf-yasg==1.21.7