Feat/tr/pagination (#123)

* feat: adding token based pagination + limit to 4 endpoints
* feat: updated backend models which now can accept a limit and token argument return tuples of data and if applicable, a pagination token
* feat: paginated endpoints return 'next' Link object that should be used to paginate through results.
* tests: creating pagination tester to isolate pagination checks into a uniform checker and make test logic more straightforward and readable
* feat: adding mock backend implementations for testing/demonstration purposes in applicaton.py
* feat: moved link object creation in paginated endpoints to separate functions to improve endpoint business logic readability

Author: theodorehreuter

.github/workflows/python-publish.yml

@@ -24,7 +24,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
-          python-version: "3.x"
+          python-version: "3.12.x"
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip

README.md

@@ -8,6 +8,18 @@ guaranteed to be not correct.
 STAPI FastAPI provides an `fastapi.APIRouter` which must be included in
 `fastapi.FastAPI` instance.
 
+### Pagination
+
+4 endpoints currently offer pagination:
+`GET`: `'/orders`, `/products`, `/orders/{order_id}/statuses`
+`POST`: `/opportunities`.
+
+Pagination is token based and follows recommendations in the [STAC API pagination].  Limit and token are passed in as query params for `GET` endpoints, and via the body aas separte key/value pairs for `POST` requests.
+
+If pagination is available and more records remain the response object will contain a `next` link object that can be used to get the next page of results.  No `next` `Link` returned indicates there are no further records available.
+
+`limit` defaults to 10 and maxes at 100.
+
 
 ## ADRs
 
@@ -59,3 +71,4 @@ With the `uvicorn` defaults the app should be accessible at
 
 [STAPI spec]: https://github.com/stapi-spec/stapi-spec
 [poetry]: https://python-poetry.org/
+[STAC API pagination]: https://github.com/radiantearth/stac-api-spec/blob/release/v1.0.0/item-search/examples.md#paging-examples

poetry.lock

@@ -3,6 +3,7 @@
 from typing import Protocol
 
 from fastapi import Request
+from returns.maybe import Maybe
 from returns.result import ResultE
 
 from stapi_fastapi.models.opportunity import Opportunity, OpportunityRequest
@@ -16,9 +17,11 @@ async def search_opportunities(
         product_router: ProductRouter,
         search: OpportunityRequest,
         request: Request,
-    ) -> ResultE[list[Opportunity]]:
+        next: str | None,
+        limit: int,
+    ) -> ResultE[tuple[list[Opportunity], Maybe[str]]]:
         """
-        Search for ordering opportunities for the  given search parameters.
+        Search for ordering opportunities for the  given search parameters and return pagination token if applicable.
 
         Backends must validate search constraints and return
         `stapi_fastapi.exceptions.ConstraintsException` if not valid.

src/stapi_fastapi/backends/product_backend.py

@@ -6,15 +6,16 @@
 
 from stapi_fastapi.models.order import (
     Order,
-    OrderCollection,
     OrderStatus,
 )
 
 
 class RootBackend[T: OrderStatus](Protocol):  # pragma: nocover
-    async def get_orders(self, request: Request) -> ResultE[OrderCollection]:
+    async def get_orders(
+        self, request: Request, next: str | None, limit: int
+    ) -> ResultE[tuple[list[Order], Maybe[str]]]:
         """
-        Return a list of existing orders.
+        Return a list of existing orders and pagination token if applicable.
         """
         ...
 
@@ -24,18 +25,18 @@ async def get_order(self, order_id: str, request: Request) -> ResultE[Maybe[Orde
 
         Should return returns.results.Success[Order] if order is found.
 
-        Should return returns.results.Failure[returns.maybe.Nothing] if the order is
-        not found or if access is denied.
+        Should return returns.results.Failure[returns.maybe.Nothing] if the
+        order is not found or if access is denied.
 
         A Failure[Exception] will result in a 500.
         """
         ...
 
     async def get_order_statuses(
-        self, order_id: str, request: Request
-    ) -> ResultE[list[T]]:
+        self, order_id: str, request: Request, next: str | None, limit: int
+    ) -> ResultE[tuple[list[T], Maybe[str]]]:
         """
-        Get statuses for order with `order_id`.
+        Get statuses for order with `order_id` and return pagination token if applicable
 
         Should return returns.results.Success[list[OrderStatus]] if order is found.
 

src/stapi_fastapi/backends/root_backend.py

@@ -2,10 +2,11 @@
 
 import logging
 import traceback
-from typing import TYPE_CHECKING, Self
+from typing import TYPE_CHECKING, Annotated, Self
 
-from fastapi import APIRouter, HTTPException, Request, Response, status
+from fastapi import APIRouter, Body, HTTPException, Request, Response, status
 from geojson_pydantic.geometries import Geometry
+from returns.maybe import Some
 from returns.result import Failure, Success
 
 from stapi_fastapi.constants import TYPE_JSON
@@ -161,27 +162,29 @@ def get_product(self, request: Request) -> Product:
         )
 
     async def search_opportunities(
-        self, search: OpportunityRequest, request: Request
+        self,
+        search: OpportunityRequest,
+        request: Request,
+        next: Annotated[str | None, Body()] = None,
+        limit: Annotated[int, Body()] = 10,
     ) -> OpportunityCollection:
         """
         Explore the opportunities available for a particular set of constraints
         """
-        match await self.product.backend.search_opportunities(self, search, request):
-            case Success(features):
-                return OpportunityCollection(
-                    features=features,
-                    links=[
-                        Link(
-                            href=str(
-                                request.url_for(
-                                    f"{self.root_router.name}:{self.product.id}:create-order",
-                                ),
-                            ),
-                            rel="create-order",
-                            type=TYPE_JSON,
-                        ),
-                    ],
-                )
+        links: list[Link] = []
+        match await self.product.backend.search_opportunities(
+            self, search, request, next, limit
+        ):
+            case Success((features, Some(pagination_token))):
+                links.append(self.order_link(request))
+                body = {
+                    "search": search.model_dump(mode="json"),
+                    "next": pagination_token,
+                    "limit": limit,
+                }
+                links.append(self.pagination_link(request, body))
+            case Success((features, Nothing)):  # noqa: F841
+                links.append(self.order_link(request))
             case Failure(e) if isinstance(e, ConstraintsException):
                 raise e
             case Failure(e):
@@ -195,6 +198,7 @@ async def search_opportunities(
                 )
             case x:
                 raise AssertionError(f"Expected code to be unreachable {x}")
+        return OpportunityCollection(features=features, links=links)
 
     def get_product_constraints(self: Self) -> JsonSchemaModel:
         """
@@ -237,3 +241,24 @@ async def create_order(
                 )
             case x:
                 raise AssertionError(f"Expected code to be unreachable {x}")
+
+    def order_link(self, request: Request):
+        return Link(
+            href=str(
+                request.url_for(
+                    f"{self.root_router.name}:{self.product.id}:create-order",
+                ),
+            ),
+            rel="create-order",
+            type=TYPE_JSON,
+            method="POST",
+        )
+
+    def pagination_link(self, request: Request, body: dict[str, str | dict]):
+        return Link(
+            href=str(request.url.remove_query_params(keys=["next", "limit"])),
+            rel="next",
+            type=TYPE_JSON,
+            method="POST",
+            body=body,
+        )

src/stapi_fastapi/routers/product_router.py

@@ -42,6 +42,7 @@ def __init__(
         self.conformances = conformances
         self.openapi_endpoint_name = openapi_endpoint_name
         self.docs_endpoint_name = docs_endpoint_name
+        self.product_ids: list[str] = []
 
         # A dict is used to track the product routers so we can ensure
         # idempotentcy in case a product is added multiple times, and also to
@@ -140,34 +141,52 @@ def get_root(self, request: Request) -> RootResponse:
     def get_conformance(self, request: Request) -> Conformance:
         return Conformance(conforms_to=self.conformances)
 
-    def get_products(self, request: Request) -> ProductsCollection:
+    def get_products(
+        self, request: Request, next: str | None = None, limit: int = 10
+    ) -> ProductsCollection:
+        start = 0
+        limit = min(limit, 100)
+        try:
+            if next:
+                start = self.product_ids.index(next)
+        except ValueError:
+            logging.exception("An error occurred while retrieving products")
+            raise NotFoundException(
+                detail="Error finding pagination token for products"
+            ) from None
+        end = start + limit
+        ids = self.product_ids[start:end]
+        links = [
+            Link(
+                href=str(request.url_for(f"{self.name}:list-products")),
+                rel="self",
+                type=TYPE_JSON,
+            ),
+        ]
+        if end > 0 and end < len(self.product_ids):
+            links.append(self.pagination_link(request, self.product_ids[end]))
         return ProductsCollection(
-            products=[pr.get_product(request) for pr in self.product_routers.values()],
-            links=[
-                Link(
-                    href=str(request.url_for(f"{self.name}:list-products")),
-                    rel="self",
-                    type=TYPE_JSON,
-                )
+            products=[
+                self.product_routers[product_id].get_product(request)
+                for product_id in ids
             ],
+            links=links,
         )
 
-    async def get_orders(self, request: Request) -> OrderCollection:
-        match await self.backend.get_orders(request):
-            case Success(orders):
+    async def get_orders(
+        self, request: Request, next: str | None = None, limit: int = 10
+    ) -> OrderCollection:
+        links: list[Link] = []
+        match await self.backend.get_orders(request, next, limit):
+            case Success((orders, Some(pagination_token))):
                 for order in orders:
-                    order.links.append(
-                        Link(
-                            href=str(
-                                request.url_for(
-                                    f"{self.name}:get-order", order_id=order.id
-                                )
-                            ),
-                            rel="self",
-                            type=TYPE_JSON,
-                        )
-                    )
-                return orders
+                    order.links.append(self.order_link(request, order))
+                links.append(self.pagination_link(request, pagination_token))
+            case Success((orders, Nothing)):  # noqa: F841
+                for order in orders:
+                    order.links.append(self.order_link(request, order))
+            case Failure(ValueError()):
+                raise NotFoundException(detail="Error finding pagination token")
             case Failure(e):
                 logger.error(
                     "An error occurred while retrieving orders: %s",
@@ -179,6 +198,7 @@ async def get_orders(self, request: Request) -> OrderCollection:
                 )
             case _:
                 raise AssertionError("Expected code to be unreachable")
+        return OrderCollection(features=orders, links=links)
 
     async def get_order(self: Self, order_id: str, request: Request) -> Order:
         """
@@ -204,25 +224,21 @@ async def get_order(self: Self, order_id: str, request: Request) -> Order:
                 raise AssertionError("Expected code to be unreachable")
 
     async def get_order_statuses(
-        self: Self, order_id: str, request: Request
+        self: Self,
+        order_id: str,
+        request: Request,
+        next: str | None = None,
+        limit: int = 10,
     ) -> OrderStatuses:
-        match await self.backend.get_order_statuses(order_id, request):
-            case Success(statuses):
-                return OrderStatuses(
-                    statuses=statuses,
-                    links=[
-                        Link(
-                            href=str(
-                                request.url_for(
-                                    f"{self.name}:list-order-statuses",
-                                    order_id=order_id,
-                                )
-                            ),
-                            rel="self",
-                            type=TYPE_JSON,
-                        )
-                    ],
-                )
+        links: list[Link] = []
+        match await self.backend.get_order_statuses(order_id, request, next, limit):
+            case Success((statuses, Some(pagination_token))):
+                links.append(self.order_statuses_link(request, order_id))
+                links.append(self.pagination_link(request, pagination_token))
+            case Success((statuses, Nothing)):  # noqa: F841
+                links.append(self.order_statuses_link(request, order_id))
+            case Failure(KeyError()):
+                raise NotFoundException("Error finding pagination token")
             case Failure(e):
                 logger.error(
                     "An error occurred while retrieving order statuses: %s",
@@ -234,12 +250,14 @@ async def get_order_statuses(
                 )
             case _:
                 raise AssertionError("Expected code to be unreachable")
+        return OrderStatuses(statuses=statuses, links=links)
 
     def add_product(self: Self, product: Product, *args, **kwargs) -> None:
         # Give the include a prefix from the product router
         product_router = ProductRouter(product, self, *args, **kwargs)
         self.include_router(product_router, prefix=f"/products/{product.id}")
         self.product_routers[product.id] = product_router
+        self.product_ids = [*self.product_routers.keys()]
 
     def generate_order_href(self: Self, request: Request, order_id: str) -> URL:
         return request.url_for(f"{self.name}:get-order", order_id=order_id)
@@ -264,3 +282,29 @@ def add_order_links(self, order: Order, request: Request):
                 type=TYPE_JSON,
             ),
         )
+
+    def order_link(self, request: Request, order: Order):
+        return Link(
+            href=str(request.url_for(f"{self.name}:get-order", order_id=order.id)),
+            rel="self",
+            type=TYPE_JSON,
+        )
+
+    def order_statuses_link(self, request: Request, order_id: str):
+        return Link(
+            href=str(
+                request.url_for(
+                    f"{self.name}:list-order-statuses",
+                    order_id=order_id,
+                )
+            ),
+            rel="self",
+            type=TYPE_JSON,
+        )
+
+    def pagination_link(self, request: Request, pagination_token: str):
+        return Link(
+            href=str(request.url.include_query_params(next=pagination_token)),
+            rel="next",
+            type=TYPE_JSON,
+        )