Merge pull request #89

Issues/37 Add function for returning an iterator instead of sequence

Author: frankinspace

CHANGELOG.md

@@ -8,6 +8,12 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
 
+### Added
+- New function `get_iter` for returning results as an iterator instead of sequence ([#37](https://github.com/nasa/python_cmr/issues/37))
+
+### Deprecated
+- Function `get` has been marked as deprecated in favor of the new `get_iter` function. `get` will likely be removed for the 1.0.0 release. ([#37](https://github.com/nasa/python_cmr/issues/37))
+
 ## [0.13.0]
 
 ### Added

cmr/queries.py

@@ -7,6 +7,8 @@
 from datetime import date, datetime, timezone
 from inspect import getmembers, ismethod
 from re import search
+from typing import Iterator
+
 from typing_extensions import (
     Any,
     List,
@@ -20,7 +22,7 @@
     Tuple,
     TypeAlias,
     Union,
-    override,
+    override, deprecated,
 )
 from urllib.parse import quote
 
@@ -58,11 +60,12 @@ def __init__(self, route: str, mode: str = CMR_OPS):
         self.concept_id_chars: Set[str] = set()
         self.headers: MutableMapping[str, str] = {}
 
+    @deprecated("Use get_iter() instead")
     def get(self, limit: int = 2000) -> Sequence[Any]:
         """
         Get all results up to some limit, even if spanning multiple pages.
 
-        :limit: The number of results to return
+        :param limit: The number of results to return
         :returns: query results as a list
         """
 
@@ -117,14 +120,56 @@ def hits(self) -> int:
 
     def get_all(self) -> Sequence[Any]:
         """
-        Returns all of the results for the query. This will call hits() first to determine how many
-        results their are, and then calls get() with that number. This method could take quite
+        Returns all of the results for the query. This method could take quite
         awhile if many requests have to be made.
 
         :returns: query results as a list
         """
 
-        return self.get(self.hits())
+        return list(self.get_iter())
+
+    def get_iter(self, limit: int = -1, page_size: int = 2000) -> Iterator[Any]:
+        """
+        Returns all results for the query as an iterator (generator)
+
+        :param limit: The maximum number of results to return. Negative value means no limit.
+        :param page_size: The page size (min 0, max 2000) of results retrieved from CMR. Smaller page size means
+        fewer items in memory and more cmr queries. Larger page size means more items in memory and fewer cmr queries.
+        :returns: query results as an iterator (generator)
+        """
+
+        url = self._build_url()
+
+        headers = dict(self.headers or {})
+        more_results = True
+        page_size = min(max(0, page_size), 2000)
+        n_results = 0
+        if limit < 0:
+            limit = self.hits()
+
+        while more_results:
+            # Only get what we need on the last page.
+            page_size = min(limit - n_results, page_size)
+            response = requests.get(
+                url, headers=headers, params={"page_size": page_size}
+            )
+            response.raise_for_status()
+
+            # Explicitly track the number of results we have because the length
+            # of the results list will only match the number of entries fetched
+            # when the format is JSON.  Otherwise, the length of the results
+            # list is the number of *pages* fetched, not the number of *items*.
+            n_results += page_size
+
+            if self._format == "json":
+                yield from response.json()["feed"]["entry"]
+            else:
+                yield response.text
+
+            if cmr_search_after := response.headers.get("cmr-search-after"):
+                headers["cmr-search-after"] = cmr_search_after
+
+            more_results = n_results < limit and cmr_search_after is not None
 
     def parameters(self, **kwargs: Any) -> Self:
         """

poetry.lock

@@ -8,13 +8,11 @@ interactions:
       - gzip, deflate
       Connection:
       - keep-alive
-      User-Agent:
-      - python-requests/2.31.0
     method: GET
     uri: https://cmr.earthdata.nasa.gov/search/granules.json?short_name=TELLUS_GRAC_L3_JPL_RL06_LND_v04&page_size=0
   response:
     body:
-      string: '{"feed":{"updated":"2023-08-14T17:02:36.801Z","id":"https://cmr.earthdata.nasa.gov:443/search/granules.json?short_name=TELLUS_GRAC_L3_JPL_RL06_LND_v04&page_size=0","title":"ECHO
+      string: '{"feed":{"updated":"2024-09-24T00:24:58.509Z","id":"https://cmr.earthdata.nasa.gov:443/search/granules.json?short_name=TELLUS_GRAC_L3_JPL_RL06_LND_v04&page_size=0","title":"ECHO
         granule metadata","entry":[]}}'
     headers:
       Access-Control-Allow-Origin:
@@ -25,37 +23,41 @@ interactions:
       CMR-Hits:
       - '163'
       CMR-Request-Id:
-      - 5855d714-8aff-4d0f-b4cc-e556f02ef96a
+      - 64ce09ea-2037-48a1-b5e3-1e9706811229
       CMR-Took:
-      - '52'
+      - '131'
       Connection:
       - keep-alive
+      Content-MD5:
+      - 3c1bb7d108b84325434e60a36dda1159
+      Content-SHA1:
+      - 3d871ed3d2791fefc0fb58701b67811f121caa63
       Content-Type:
       - application/json;charset=utf-8
       Date:
-      - Mon, 14 Aug 2023 17:02:36 GMT
+      - Tue, 24 Sep 2024 00:24:58 GMT
       Server:
       - ServerTokens ProductOnly
       Strict-Transport-Security:
-      - max-age=31536000
+      - max-age=31536000; includeSubDomains; preload
       Transfer-Encoding:
       - chunked
       Vary:
       - Accept-Encoding, User-Agent
       Via:
-      - 1.1 cc58556a6e846289f4d3105969536e4c.cloudfront.net (CloudFront)
+      - 1.1 b837267595110a1135bf4fb036d71e1e.cloudfront.net (CloudFront)
       X-Amz-Cf-Id:
-      - qj9VuAc1JQu-rnMVDg3mGwstR-jGQA4rd7MKVRAEpXeTDbKZT5p5jg==
+      - nCF7mfer1omvbZi5CTMRTv9-9uPozEm7zBM8NhFZ8nJ_sXVz-tBAgw==
       X-Amz-Cf-Pop:
-      - SFO53-C1
+      - LAX50-C1
       X-Cache:
       - Miss from cloudfront
       X-Content-Type-Options:
       - nosniff
       X-Frame-Options:
       - SAMEORIGIN
       X-Request-Id:
-      - qj9VuAc1JQu-rnMVDg3mGwstR-jGQA4rd7MKVRAEpXeTDbKZT5p5jg==
+      - nCF7mfer1omvbZi5CTMRTv9-9uPozEm7zBM8NhFZ8nJ_sXVz-tBAgw==
       X-XSS-Protection:
       - 1; mode=block
       content-length:
@@ -72,13 +74,11 @@ interactions:
       - gzip, deflate
       Connection:
       - keep-alive
-      User-Agent:
-      - python-requests/2.31.0
     method: GET
     uri: https://cmr.earthdata.nasa.gov/search/granules.json?short_name=TELLUS_GRAC_L3_JPL_RL06_LND_v04&page_size=163
   response:
     body:
-      string: '{"feed":{"updated":"2023-08-14T17:02:40.416Z","id":"https://cmr.earthdata.nasa.gov:443/search/granules.json?short_name=TELLUS_GRAC_L3_JPL_RL06_LND_v04&page_size=163","title":"ECHO
+      string: '{"feed":{"updated":"2024-09-24T00:24:58.790Z","id":"https://cmr.earthdata.nasa.gov:443/search/granules.json?short_name=TELLUS_GRAC_L3_JPL_RL06_LND_v04&page_size=163","title":"ECHO
         granule metadata","entry":[{"boxes":["-89.5 0.5 89.5 180","-89.5 -180 89.5
         -0.5"],"time_start":"2002-04-04T00:00:00.000Z","updated":"2023-04-17T15:27:21.022Z","dataset_id":"JPL
         TELLUS GRACE Level-3 Monthly Land Water-Equivalent-Thickness Surface Mass
@@ -2045,39 +2045,43 @@ interactions:
       CMR-Hits:
       - '163'
       CMR-Request-Id:
-      - 60eb29b2-95e1-453c-8efe-6e59cf649eb5
+      - b82d5198-a729-42bd-b597-a1132b2652c3
       CMR-Search-After:
       - '["pocloud",1495497600000,2658328520]'
       CMR-Took:
-      - '4959'
+      - '125'
       Connection:
       - keep-alive
+      Content-MD5:
+      - 2f2981275f193e1579bea1c3e9f1acf5
+      Content-SHA1:
+      - 4fa7e296cc7b77f83dcf2bcda4237d29dc885fd1
       Content-Type:
       - application/json;charset=utf-8
       Date:
-      - Mon, 14 Aug 2023 17:02:42 GMT
+      - Tue, 24 Sep 2024 00:24:58 GMT
       Server:
       - ServerTokens ProductOnly
       Strict-Transport-Security:
-      - max-age=31536000
+      - max-age=31536000; includeSubDomains; preload
       Transfer-Encoding:
       - chunked
       Vary:
       - Accept-Encoding, User-Agent
       Via:
-      - 1.1 44933b72098305e9c31fc50b2e6554a0.cloudfront.net (CloudFront)
+      - 1.1 be66acbcc5d85e825abf1047b034d722.cloudfront.net (CloudFront)
       X-Amz-Cf-Id:
-      - 9TJ3JRMGc6mUxKegR4f2HSLC_1Cfwei5QHZuicg_aLsWEJS3T6XCNg==
+      - JIeDUJvd8TodeetWYvcK5xBnxBTh8jvsNt8if-ZsMjTUWW4sbZ9P2A==
       X-Amz-Cf-Pop:
-      - SFO53-C1
+      - LAX50-C1
       X-Cache:
       - Miss from cloudfront
       X-Content-Type-Options:
       - nosniff
       X-Frame-Options:
       - SAMEORIGIN
       X-Request-Id:
-      - 9TJ3JRMGc6mUxKegR4f2HSLC_1Cfwei5QHZuicg_aLsWEJS3T6XCNg==
+      - JIeDUJvd8TodeetWYvcK5xBnxBTh8jvsNt8if-ZsMjTUWW4sbZ9P2A==
       X-XSS-Protection:
       - 1; mode=block
       content-length: