add find_page for paginated queries with envelope metadata

Author: purificant

src/bubble_data_api_client/__init__.py

@@ -28,6 +28,7 @@
     OnMultiple,
     OptionalBubbleUID,
     OptionalBubbleUIDs,
+    PageResult,
 )
 from bubble_data_api_client.validation import filter_bubble_uids, is_bubble_uid
 
@@ -56,6 +57,7 @@
     "OnMultiple",
     "OptionalBubbleUID",
     "OptionalBubbleUIDs",
+    "PageResult",
     # validation
     "filter_bubble_uids",
     "is_bubble_uid",

src/bubble_data_api_client/client/orm.py

@@ -29,13 +29,14 @@ class User(BubbleModel, typename="user"):
 from pydantic import BaseModel as PydanticBaseModel
 from pydantic import Field
 
-from bubble_data_api_client.client.raw_client import AdditionalSortField, RawClient
+from bubble_data_api_client.client.raw_client import (
+    _DEFAULT_PAGE_SIZE,
+    AdditionalSortField,
+    RawClient,
+)
 from bubble_data_api_client.constraints import Constraint, ConstraintType, constraint
 from bubble_data_api_client.exceptions import BubbleAPIError, UnknownFieldError
-from bubble_data_api_client.types import BUILTIN_FIELDS, BubbleField, OnMultiple
-
-# default page size for paginated requests.
-_DEFAULT_PAGE_SIZE: int = 100
+from bubble_data_api_client.types import BUILTIN_FIELDS, BubbleField, OnMultiple, PageResult
 
 # max UIDs per "in" constraint batch, matching the API's max page size.
 _MAX_IN_CONSTRAINT_SIZE: int = 100
@@ -219,6 +220,55 @@ async def find(
             )
             return [cls.model_validate(item) for item in response.json()["response"]["results"]]
 
+    @classmethod
+    async def find_page(
+        cls,
+        *,
+        constraints: list[Constraint] | None = None,
+        cursor: int = 0,
+        limit: int = _DEFAULT_PAGE_SIZE,
+        sort_field: str | None = None,
+        descending: bool | None = None,
+        additional_sort_fields: list[AdditionalSortField] | None = None,
+    ) -> PageResult[typing.Self]:
+        """Return one page of matching records with envelope metadata.
+
+        Unlike find(), this preserves Bubble's response envelope so callers
+        can display total counts and drive pagination UIs without issuing a
+        separate count() call.
+
+        See RawClient.find_page for important caveats about Bubble's
+        pagination limits: the 100-item silent cap on limit, the ~50,000
+        cursor cap on shared infrastructure, and the recommended keyset
+        pagination workaround for collections larger than the cursor cap.
+
+        Args:
+            constraints: Filter conditions (use constraint() helper to build).
+            cursor: Pagination offset (0-indexed).
+            limit: Maximum results to return on this page.
+            sort_field: Field name to sort by.
+            descending: Sort in descending order if True.
+            additional_sort_fields: Secondary sort fields after the primary.
+
+        Returns:
+            PageResult with typed model instances plus envelope metadata.
+        """
+        async with _get_client() as client:
+            page = await client.find_page(
+                cls._typename,
+                constraints=constraints,
+                cursor=cursor,
+                limit=limit,
+                sort_field=sort_field,
+                descending=descending,
+                additional_sort_fields=additional_sort_fields,
+            )
+            return PageResult(
+                items=[cls.model_validate(item) for item in page.items],
+                cursor=page.cursor,
+                remaining=page.remaining,
+            )
+
     @classmethod
     async def find_iter(
         cls,
@@ -229,7 +279,15 @@ async def find_iter(
         descending: bool | None = None,
         additional_sort_fields: list[AdditionalSortField] | None = None,
     ) -> AsyncIterator[typing.Self]:
-        """Iterate through all matching records with constant memory usage."""
+        """Iterate through all matching records with constant memory usage.
+
+        Offset pagination is capped at ~50,000 on Bubble's shared
+        infrastructure. For collections larger than that cap, prefer
+        keyset pagination (a Created Date constraint) instead of this
+        method, which will stop short at the cap. The empty-page guard
+        below prevents an infinite loop when the cursor reaches the cap
+        (Bubble continues to report a non-zero remaining past the cap).
+        """
         cursor: int = 0
         async with _get_client() as client:
             while True:
@@ -245,6 +303,11 @@ async def find_iter(
                 body = response.json()["response"]
                 for item in body["results"]:
                     yield cls.model_validate(item)
+                # stop on empty page first: past the cursor cap Bubble can
+                # return zero results while still reporting remaining > 0,
+                # which would otherwise cause an infinite loop.
+                if not body["results"]:
+                    break
                 if body["remaining"] == 0:
                     break
                 cursor += len(body["results"])

src/bubble_data_api_client/client/raw_client.py

@@ -38,8 +38,15 @@
     BulkCreateItemResult,
     CreateOrUpdateResult,
     OnMultiple,
+    PageResult,
 )
 
+# Bubble Data API returns 100 items per page by default and silently caps any
+# requested limit at 100 (no HTTP error, no warning). Verified empirically up
+# to limit=2,147,483,647 (int32 max).
+# https://manual.bubble.io/help-guides/integrations/api/the-bubble-api/the-data-api/data-api-requests
+_DEFAULT_PAGE_SIZE: int = 100
+
 
 # https://manual.bubble.io/core-resources/api/the-bubble-api/the-data-api/data-api-requests#sorting
 # in addition to 'sort_field' and 'descending', it is possible to have
@@ -182,6 +189,73 @@ async def find(
 
         return await self._transport.get(f"/{typename}", params=params)
 
+    async def find_page(
+        self,
+        typename: str,
+        *,
+        constraints: list[Constraint] | None = None,
+        cursor: int = 0,
+        limit: int = _DEFAULT_PAGE_SIZE,
+        sort_field: str | None = None,
+        descending: bool | None = None,
+        additional_sort_fields: list[AdditionalSortField] | None = None,
+    ) -> PageResult[dict[str, typing.Any]]:
+        """Return one page of results with envelope metadata.
+
+        Unlike find(), this parses Bubble's response envelope so callers can
+        display total counts and drive pagination UIs without issuing a separate
+        count() call. The returned PageResult exposes cursor, remaining, and a
+        computed total derived from len(items) (which is robust to the edge
+        case where Bubble returns count=-1 for limit=-1).
+
+        Bubble Data API pagination limits to be aware of:
+
+        - limit is silently capped at 100 items. Requesting more returns
+          exactly 100 with HTTP 200; PageResult does not echo the
+          requested limit, so there is no way to accidentally advance a
+          cursor by a capped value. Use len(page.items) to advance.
+        - cursor + page size is capped at roughly 50,000 on shared
+          infrastructure (higher on Enterprise plans). Beyond this offset,
+          Bubble returns zero results but continues to report a non-zero
+          remaining, making PageResult.total under-report past the cap.
+        - For collections larger than the cursor cap, do not use offset
+          pagination. Use keyset pagination: sort by a monotonic field (e.g.
+          Created Date) with a greater-than constraint on the last-seen
+          value, resetting cursor to 0 on each request.
+
+        Args:
+            typename: The Bubble data type to query.
+            constraints: Filter conditions (use constraint() helper to build).
+            cursor: Pagination offset (0-indexed).
+            limit: Maximum results to return on this page.
+            sort_field: Field name to sort by.
+            descending: Sort in descending order if True.
+            additional_sort_fields: Secondary sort fields after the primary.
+
+        Returns:
+            PageResult with items as raw dicts plus envelope metadata.
+        """
+        response = await self.find(
+            typename,
+            constraints=constraints,
+            cursor=cursor,
+            limit=limit,
+            sort_field=sort_field,
+            descending=descending,
+            additional_sort_fields=additional_sort_fields,
+        )
+        body = response.json()["response"]
+        # Prefer the server-reported cursor over the requested value so
+        # PageResult is a pure reflection of server state. Bubble echoes
+        # cursor on every observed response; if it ever normalizes the
+        # value (e.g. a negative cursor to 0), this surfaces the difference
+        # immediately rather than silently returning wrong data.
+        return PageResult(
+            items=body["results"],
+            cursor=body["cursor"],
+            remaining=body["remaining"],
+        )
+
     async def count(
         self,
         typename: str,

src/bubble_data_api_client/types.py

@@ -1,5 +1,6 @@
 """Bubble platform types for use with Pydantic models."""
 
+from dataclasses import dataclass
 from enum import StrEnum
 from typing import Annotated, Any, Literal, TypedDict
 
@@ -62,6 +63,45 @@ class BulkCreateItemResult(TypedDict):
     message: str | None
 
 
+@dataclass(frozen=True, slots=True)
+class PageResult[T]:
+    """One page of results from a Bubble find query, with envelope metadata.
+
+    To advance pagination, use cursor + len(items), not any notion of page
+    size. Bubble silently caps requested limits at 100, so a caller who
+    requested more than 100 items must not assume they got what they asked
+    for; len(items) is always the correct advancement step.
+
+    Attributes:
+        items: Results for this page, in Bubble's returned order.
+        cursor: The cursor (offset) that produced this page, as reported
+            by Bubble's response envelope.
+        remaining: Number of matching records after this page.
+    """
+
+    items: list[T]
+    cursor: int
+    remaining: int
+
+    @property
+    def total(self) -> int:
+        """Return total number of records matching the query.
+
+        Computed as cursor + len(items) + remaining. This under-reports
+        past Bubble's ~50,000 cursor cap on shared infrastructure, where
+        Bubble returns an empty page with a non-zero remaining value. For
+        collections larger than the cap, use keyset pagination (sort by a
+        monotonic field with a greater-than constraint) rather than
+        offset pagination.
+        """
+        return self.cursor + len(self.items) + self.remaining
+
+    @property
+    def has_more(self) -> bool:
+        """Return True if there are more pages after this one."""
+        return self.remaining > 0
+
+
 def _validate_bubble_uid(value: str) -> str:
     """Validate that a string is a valid Bubble UID."""
     if not is_bubble_uid(value):