From c05ed42e41351669b7ed2ae655b0bf46e0a703cb Mon Sep 17 00:00:00 2001 From: Aayush Agnihotri Date: Wed, 17 Sep 2025 20:15:14 -0400 Subject: [PATCH 1/6] Adding memory profiler --- .pre-commit-config.yaml | 24 +++++++++---------- requirements.txt | 5 ++-- src/eatery/controllers/populate_eatery.py | 2 ++ src/eatery/views.py | 12 ++++++---- .../management/commands/populate_models.py | 2 ++ 5 files changed, 27 insertions(+), 18 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index af043df..7c4724c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,25 +1,25 @@ repos: # Updates all subsequent hooks - - repo: https://gitlab.com/vojko.pribudic.foss/pre-commit-update - rev: v0.5.1 +- repo: https://gitlab.com/vojko.pribudic.foss/pre-commit-update + rev: v0.8.0 hooks: - - id: pre-commit-update + - id: pre-commit-update # Linter hook - - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.6.7 +- repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.13.0 hooks: - - id: ruff + - id: ruff args: [--fix] # Formatter hook - - repo: https://github.com/psf/black-pre-commit-mirror - rev: 24.8.0 +- repo: https://github.com/psf/black-pre-commit-mirror + rev: 25.1.0 hooks: - - id: black + - id: black # Static code analysis hook - - repo: https://github.com/PyCQA/bandit - rev: 1.7.10 +- repo: https://github.com/PyCQA/bandit + rev: 1.8.6 hooks: - - id: bandit + - id: bandit diff --git a/requirements.txt b/requirements.txt index e654118..20774da 100644 --- a/requirements.txt +++ b/requirements.txt @@ -26,6 +26,7 @@ inflection==0.5.1 itypes==1.2.0 Jinja2==3.1.4 MarkupSafe==2.1.5 +memory-profiler==0.61.0 mypy-extensions==0.4.3 nodeenv==1.9.1 oauthlib==3.1.1 @@ -35,6 +36,7 @@ pathspec==0.9.0 platformdirs==4.2.2 pre-commit==3.8.0 protobuf==3.19.1 +psutil==7.1.0 psycopg2-binary==2.9.3 pyasn1==0.4.8 pyasn1-modules==0.2.8 @@ -51,6 +53,5 @@ sqlparse==0.4.2 tomli==1.2.3 typing_extensions==4.0.1 uritemplate==4.1.1 -urllib3 +urllib3==1.26.20 virtualenv==20.26.3 -google-auth \ No newline at end of file diff --git a/src/eatery/controllers/populate_eatery.py b/src/eatery/controllers/populate_eatery.py index 13e365a..2191a2c 100644 --- a/src/eatery/controllers/populate_eatery.py +++ b/src/eatery/controllers/populate_eatery.py @@ -3,6 +3,7 @@ from eatery.serializers import EaterySerializer from eatery.models import Eatery from django.core.exceptions import ObjectDoesNotExist +from memory_profiler import profile class PopulateEateryController: @@ -88,6 +89,7 @@ def add_eatery_store(self): else: print(serialized.errors) + @profile def process(self, json_eateries): for json_eatery in json_eateries: self.generate_eatery(json_eatery) diff --git a/src/eatery/views.py b/src/eatery/views.py index 7d71021..ec8e8b9 100644 --- a/src/eatery/views.py +++ b/src/eatery/views.py @@ -7,8 +7,6 @@ from eatery.util.json import FieldType, error_json, success_json, verify_json_fields from django.http import JsonResponse from django.shortcuts import get_object_or_404 -from django.utils.decorators import method_decorator -from django.views.decorators.cache import cache_page from rest_framework.views import APIView from rest_framework.response import Response from rest_framework import viewsets @@ -16,6 +14,7 @@ from eatery.datatype.Eatery import EateryID from eatery.models import Eatery from .controllers.update_eatery import UpdateEateryController +from memory_profiler import profile class EateryViewSet(viewsets.ModelViewSet): @@ -39,17 +38,20 @@ def get_queryset(self): 'events__menu__items__allergens' ) + @profile def retrieve(self, request, *args, **kwargs): instance = self.get_object() serializer = EaterySerializerOptimized(instance) return Response(serializer.data) - @method_decorator(cache_page(60 * 60 * 2)) # cache for 2 hours + # @method_decorator(cache_page(60 * 60 * 2)) # cache for 2 hours + @profile def list(self, request, *args, **kwargs): queryset = self.filter_queryset(self.get_queryset()) serializer = EaterySerializerOptimized(queryset, many=True) return Response(serializer.data) + @profile def get_object(self): queryset = self.filter_queryset(self.get_queryset()) lookup_url_kwarg = self.lookup_url_kwarg or self.lookup_field @@ -109,6 +111,7 @@ class GetEateriesSimple(APIView): View all eateries with less information """ + @profile def get(self, request): eateries_queryset = Eatery.objects.prefetch_related( 'events' @@ -122,7 +125,8 @@ class GetEateriesByDay(APIView): Get all eatery information by day """ - @method_decorator(cache_page(60 * 60 * 2)) # cache for 2 hours + # @method_decorator(cache_page(60 * 60 * 2)) # cache for 2 hours + @profile def get(self, request, day): eateries_queryset = Eatery.objects.prefetch_related( 'events__menu__items__dietary_preferences', diff --git a/src/eatery_blue_backend/management/commands/populate_models.py b/src/eatery_blue_backend/management/commands/populate_models.py index 87fc5fe..2f2b823 100644 --- a/src/eatery_blue_backend/management/commands/populate_models.py +++ b/src/eatery_blue_backend/management/commands/populate_models.py @@ -11,6 +11,7 @@ import os import json import shutil +from memory_profiler import profile class Command(BaseCommand): @@ -121,6 +122,7 @@ def logger_wrapper(self, command_obj, log_title, args): print(f"Done ({int(datetime.now().timestamp()) - pre}s) ") return output + @profile def process(self): """ 1. Get JSON from API From 3a87ce70d2ef8d6e12b7e3564a1d6304a1792401 Mon Sep 17 00:00:00 2001 From: Aayush Agnihotri Date: Wed, 17 Sep 2025 20:23:14 -0400 Subject: [PATCH 2/6] Adding back caching --- src/eatery/views.py | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/src/eatery/views.py b/src/eatery/views.py index ec8e8b9..0b30b0a 100644 --- a/src/eatery/views.py +++ b/src/eatery/views.py @@ -7,6 +7,8 @@ from eatery.util.json import FieldType, error_json, success_json, verify_json_fields from django.http import JsonResponse from django.shortcuts import get_object_or_404 +from django.utils.decorators import method_decorator +from django.views.decorators.cache import cache_page from rest_framework.views import APIView from rest_framework.response import Response from rest_framework import viewsets @@ -31,11 +33,10 @@ def get_queryset(self): Override to add prefetch_related for optimization """ queryset = super().get_queryset() - + # prefetch all related objects to avoid N+1 query problem return queryset.prefetch_related( - 'events__menu__items__dietary_preferences', - 'events__menu__items__allergens' + "events__menu__items__dietary_preferences", "events__menu__items__allergens" ) @profile @@ -44,8 +45,8 @@ def retrieve(self, request, *args, **kwargs): serializer = EaterySerializerOptimized(instance) return Response(serializer.data) - # @method_decorator(cache_page(60 * 60 * 2)) # cache for 2 hours @profile + @method_decorator(cache_page(60 * 60 * 2)) # cache for 2 hours def list(self, request, *args, **kwargs): queryset = self.filter_queryset(self.get_queryset()) serializer = EaterySerializerOptimized(queryset, many=True) @@ -113,9 +114,7 @@ class GetEateriesSimple(APIView): @profile def get(self, request): - eateries_queryset = Eatery.objects.prefetch_related( - 'events' - ).all() + eateries_queryset = Eatery.objects.prefetch_related("events").all() eateries = EaterySerializerSimple(eateries_queryset, many=True) return Response(eateries.data) @@ -125,17 +124,16 @@ class GetEateriesByDay(APIView): Get all eatery information by day """ - # @method_decorator(cache_page(60 * 60 * 2)) # cache for 2 hours @profile + @method_decorator(cache_page(60 * 60 * 2)) # cache for 2 hours def get(self, request, day): eateries_queryset = Eatery.objects.prefetch_related( - 'events__menu__items__dietary_preferences', - 'events__menu__items__allergens' + "events__menu__items__dietary_preferences", "events__menu__items__allergens" ).exclude(events__event_description="Open") - + eateries = EaterySerializerByDay( eateries_queryset, many=True, context={"day": day}, ) - return Response(eateries.data) \ No newline at end of file + return Response(eateries.data) From 54dd212fc454bc39cd30332a575f338841a13b6c Mon Sep 17 00:00:00 2001 From: Aayush Agnihotri Date: Wed, 17 Sep 2025 21:22:08 -0400 Subject: [PATCH 3/6] Hopefully fixing n+1 issue with getEateriesByDay --- src/eatery/serializers.py | 23 +---------------------- src/eatery/views.py | 26 ++++++++++++++++++++++++-- 2 files changed, 25 insertions(+), 24 deletions(-) diff --git a/src/eatery/serializers.py b/src/eatery/serializers.py index 9f6dae0..0fa4e67 100644 --- a/src/eatery/serializers.py +++ b/src/eatery/serializers.py @@ -1,13 +1,10 @@ from rest_framework import serializers from eatery.models import Eatery -from event.models import Event from event.serializers import ( EventSerializer, EventSerializerSimple, EventSerializerOptimized, ) -from datetime import timedelta, datetime -from zoneinfo import ZoneInfo class EaterySerializer(serializers.ModelSerializer): @@ -107,25 +104,7 @@ class EaterySerializerByDay(serializers.ModelSerializer): allow_null=True, default="https://images-prod.healthline.com/hlcmsresource/images/AN_images/health-benefits-of-apples-1296x728-feature.jpg", ) - events = serializers.SerializerMethodField() - - def get_events(self, obj): - day_offset = self.context.get("day") - now = datetime.now(ZoneInfo("America/New_York")) - day = now.replace(hour=0, minute=0, second=0, microsecond=0) + timedelta( - days=day_offset - ) - day_unix = int(day.timestamp()) - day_end_unix = int((day + timedelta(days=1)).timestamp()) - print(f"Now: {now}") - print(f"Day: {day}") - print(f"Day Unix: {day_unix}") - print(f"Day End Unix: {day_end_unix}") - events = Event.objects.filter( - eatery=obj.id, start__gte=day_unix, start__lt=day_end_unix - ) - serializer = EventSerializerOptimized(instance=events, many=True) - return serializer.data + events = EventSerializerOptimized(many=True, source="filtered_events") class Meta: model = Eatery diff --git a/src/eatery/views.py b/src/eatery/views.py index 0b30b0a..0d1073a 100644 --- a/src/eatery/views.py +++ b/src/eatery/views.py @@ -4,6 +4,7 @@ EaterySerializerByDay, EaterySerializerOptimized, ) +from django.db.models import Prefetch from eatery.util.json import FieldType, error_json, success_json, verify_json_fields from django.http import JsonResponse from django.shortcuts import get_object_or_404 @@ -15,8 +16,11 @@ from .permissions import EateryPermission from eatery.datatype.Eatery import EateryID from eatery.models import Eatery +from event.models import Event from .controllers.update_eatery import UpdateEateryController from memory_profiler import profile +from datetime import timedelta, datetime +from zoneinfo import ZoneInfo class EateryViewSet(viewsets.ModelViewSet): @@ -127,13 +131,31 @@ class GetEateriesByDay(APIView): @profile @method_decorator(cache_page(60 * 60 * 2)) # cache for 2 hours def get(self, request, day): + now = datetime.now(ZoneInfo("America/New_York")) + start_date = now.replace(hour=0, minute=0, second=0, microsecond=0) + timedelta( + days=day + ) + end_date = start_date + timedelta(days=1) + + start_unix = int(start_date.timestamp()) + end_unix = int(end_date.timestamp()) + + filtered_events_prefetch = Prefetch( + "events", + queryset=Event.objects.filter( + start__gte=start_unix, start__lt=end_unix + ).prefetch_related( + "menu__items__dietary_preferences", "menu__items__allergens" + ), + to_attr="filtered_events", + ) + eateries_queryset = Eatery.objects.prefetch_related( - "events__menu__items__dietary_preferences", "events__menu__items__allergens" + filtered_events_prefetch ).exclude(events__event_description="Open") eateries = EaterySerializerByDay( eateries_queryset, many=True, - context={"day": day}, ) return Response(eateries.data) From ff32b975001b9956e50725daf1f11294892e478e Mon Sep 17 00:00:00 2001 From: Aayush Agnihotri Date: Wed, 17 Sep 2025 21:48:31 -0400 Subject: [PATCH 4/6] Implementing faster JSON renderers --- requirements.txt | 2 ++ src/eatery_blue_backend/settings.py | 7 ++++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 20774da..f8f1372 100644 --- a/requirements.txt +++ b/requirements.txt @@ -55,3 +55,5 @@ typing_extensions==4.0.1 uritemplate==4.1.1 urllib3==1.26.20 virtualenv==20.26.3 +orjson +djangorestframework-orjson diff --git a/src/eatery_blue_backend/settings.py b/src/eatery_blue_backend/settings.py index 4d89281..0e48ede 100644 --- a/src/eatery_blue_backend/settings.py +++ b/src/eatery_blue_backend/settings.py @@ -44,6 +44,11 @@ # Application definition +REST_FRAMEWORK = { + "DEFAULT_RENDERER_CLASSES": ("drf_orjson.renderers.ORJSONRenderer",), + "DEFAULT_PARSER_CLASSES": ("drf_orjson.parsers.ORJSONParser",), +} + INSTALLED_APPS = [ "django.contrib.admin", "django.contrib.auth", @@ -115,7 +120,7 @@ "PRE_PING": True, "ECHO": False, "TIMEOUT": 30, - } + }, } } From 1d820bd80e9cf877499b04007e94199148d20fbd Mon Sep 17 00:00:00 2001 From: Aayush Agnihotri Date: Wed, 17 Sep 2025 22:10:30 -0400 Subject: [PATCH 5/6] Adding drf-orjson dependency --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index f8f1372..c6b8f8b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -56,4 +56,5 @@ uritemplate==4.1.1 urllib3==1.26.20 virtualenv==20.26.3 orjson +drf-orjson djangorestframework-orjson From 60a9c68d15d4cab291d758ef2b9131a7b2770d04 Mon Sep 17 00:00:00 2001 From: Aayush Agnihotri Date: Wed, 17 Sep 2025 22:59:38 -0400 Subject: [PATCH 6/6] Fix retrieving individual eateries --- src/eatery/views.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/eatery/views.py b/src/eatery/views.py index 0d1073a..ee8e9b3 100644 --- a/src/eatery/views.py +++ b/src/eatery/views.py @@ -32,18 +32,24 @@ class EateryViewSet(viewsets.ModelViewSet): serializer_class = EaterySerializer permission_classes = [EateryPermission] + @profile def get_queryset(self): """ Override to add prefetch_related for optimization """ queryset = super().get_queryset() - # prefetch all related objects to avoid N+1 query problem - return queryset.prefetch_related( - "events__menu__items__dietary_preferences", "events__menu__items__allergens" - ) + # Prefetching only needed for list to prevent N+1 + if self.action == "list": + return queryset.prefetch_related( + "events__menu__items__dietary_preferences", + "events__menu__items__allergens", + ) + + return queryset @profile + @method_decorator(cache_page(60 * 60 * 2)) # cache for 2 hours def retrieve(self, request, *args, **kwargs): instance = self.get_object() serializer = EaterySerializerOptimized(instance)