[SYNPY-1590] Patching the async generator to return results as it gets them, and removing param passing to several submission related APIs which do not support it (#1305)

* Patching the async generator to return results as it gets them, and removing param passing to several submission related APIs which do not support it

---------

Co-authored-by: Jenny Medina <medina.stsci@gmail.com>

Author: BryanFauble

docs/tutorials/python/submission.md

@@ -14,6 +14,7 @@ Before working with Submissions, it's helpful to understand how they fit into Sy
 ## How Submissions Work
 
 When you submit an Entity to an Evaluation:
+
 - The Submission creates an immutable record linking your Entity to that Evaluation
 - The Evaluation owns this Submission record (not you as the submitter)
 - Organizers can add scores and feedback through a SubmissionStatus object
@@ -60,6 +61,13 @@ As an organizer of a Synapse challenge, you will
 * You have an existing entity with which to make a submission (can be a [File](./file.md) or Docker Repository)
 * You have the correct permissions on the Evaluation queue for your desired tutorial section (participant or organizer)
 
+
+Script setup:
+
+```python
+{!docs/tutorials/python/tutorial_scripts/submission_participant.py!lines=13-30}
+```
+
 ## 1. Participating in a Synapse challenge
 
 ### 1. Make a submission to an existing evaluation queue on Synapse
@@ -77,29 +85,36 @@ As an organizer of a Synapse challenge, you will
 ### 3. Count your submissions
 
 ```python
-{!docs/tutorials/python/tutorial_scripts/submission_participant.py!lines=72-88}
+{!docs/tutorials/python/tutorial_scripts/submission_participant.py!lines=72-82}
 ```
 
 ### 4. Fetch all of your submissions from an existing evaluation queue on Synapse
 
 ```python
-{!docs/tutorials/python/tutorial_scripts/submission_participant.py!lines=90-101}
+{!docs/tutorials/python/tutorial_scripts/submission_participant.py!lines=82-95}
 ```
 
 ### 5. Check the status of your submission
 
 ```python
-{!docs/tutorials/python/tutorial_scripts/submission_participant.py!lines=103-125}
+{!docs/tutorials/python/tutorial_scripts/submission_participant.py!lines=97-119}
 ```
 
 ### 6. Cancel your submission
 
 ```python
-{!docs/tutorials/python/tutorial_scripts/submission_participant.py!lines=126-143}
+{!docs/tutorials/python/tutorial_scripts/submission_participant.py!lines=120-137}
 ```
 
 ## 2. Organizing a Synapse challenge
 
+
+Script setup:
+
+```python
+{!docs/tutorials/python/tutorial_scripts/submission_organizer.py!lines=12-31}
+```
+
 ### 1. Annotate a submission to score it
 
 ```python

docs/tutorials/python/tutorial_scripts/submission_organizer.py

@@ -10,7 +10,7 @@
 """
 
 from synapseclient import Synapse
-from synapseclient.models import Submission, SubmissionBundle, SubmissionStatus
+from synapseclient.models import SubmissionBundle, SubmissionStatus
 
 syn = Synapse()
 syn.login()

docs/tutorials/python/tutorial_scripts/submission_participant.py

@@ -80,12 +80,6 @@
 
 print(f"Total submissions in evaluation: {submission_count}")
 
-# Get count of submissions with specific status (optional)
-scored_count = Submission.get_submission_count(
-    evaluation_id=EVALUATION_ID, status="SCORED"
-)
-
-print(f"SCORED submissions in evaluation: {scored_count}")
 
 # ==============================================================================
 # 4. Fetch all of your submissions from an existing evaluation queue

synapseclient/api/evaluation_services.py

@@ -476,20 +476,16 @@ async def get_evaluation_submissions(
     client = Synapse.get_client(synapse_client=synapse_client)
 
     uri = f"/evaluation/{evaluation_id}/submission/all"
-    query_params = {}
 
     if status:
-        query_params["status"] = status
+        uri = uri + "?status=" + status
 
-    async for item in rest_get_paginated_async(
-        uri=uri, params=query_params, synapse_client=client
-    ):
+    async for item in rest_get_paginated_async(uri=uri, synapse_client=client):
         yield item
 
 
 async def get_user_submissions(
     evaluation_id: str,
-    user_id: Optional[str] = None,
     *,
     synapse_client: Optional["Synapse"] = None,
 ) -> AsyncGenerator[dict[str, Any], None]:
@@ -501,8 +497,6 @@ async def get_user_submissions(
 
     Arguments:
         evaluation_id: The ID of the evaluation queue.
-        user_id: Optionally specify the ID of the user whose submissions will be returned.
-                If omitted, this returns the submissions of the caller.
         synapse_client: If not passed in and caching was not disabled by `Synapse.allow_client_caching(False)`
                         this will use the last created instance from the Synapse class constructor.
 
@@ -514,20 +508,13 @@ async def get_user_submissions(
     client = Synapse.get_client(synapse_client=synapse_client)
 
     uri = f"/evaluation/{evaluation_id}/submission"
-    query_params = {}
-
-    if user_id:
-        query_params["userId"] = user_id
 
-    async for item in rest_get_paginated_async(
-        uri=uri, params=query_params, synapse_client=client
-    ):
+    async for item in rest_get_paginated_async(uri=uri, synapse_client=client):
         yield item
 
 
 async def get_submission_count(
     evaluation_id: str,
-    status: Optional[str] = None,
     synapse_client: Optional["Synapse"] = None,
 ) -> dict:
     """
@@ -537,8 +524,6 @@ async def get_submission_count(
 
     Arguments:
         evaluation_id: The ID of the evaluation queue.
-        status: Optionally filter submissions by a submission status, such as SCORED, VALID,
-                INVALID, OPEN, CLOSED or EVALUATION_IN_PROGRESS.
         synapse_client: If not passed in and caching was not disabled by `Synapse.allow_client_caching(False)`
                         this will use the last created instance from the Synapse class constructor.
 
@@ -550,12 +535,8 @@ async def get_submission_count(
     client = Synapse.get_client(synapse_client=synapse_client)
 
     uri = f"/evaluation/{evaluation_id}/submission/count"
-    query_params = {}
 
-    if status:
-        query_params["status"] = status
-
-    response = await client.rest_get_async(uri, params=query_params)
+    response = await client.rest_get_async(uri)
 
     return response
 
@@ -812,14 +793,11 @@ async def get_evaluation_submission_bundles(
     client = Synapse.get_client(synapse_client=synapse_client)
 
     uri = f"/evaluation/{evaluation_id}/submission/bundle/all"
-    query_params = {}
 
     if status:
-        query_params["status"] = status
+        uri = uri + "?status=" + status
 
-    async for item in rest_get_paginated_async(
-        uri=uri, params=query_params, synapse_client=client
-    ):
+    async for item in rest_get_paginated_async(uri=uri, synapse_client=client):
         yield item
 
 
@@ -847,9 +825,6 @@ async def get_user_submission_bundles(
     client = Synapse.get_client(synapse_client=synapse_client)
 
     uri = f"/evaluation/{evaluation_id}/submission/bundle"
-    query_params = {}
 
-    async for item in rest_get_paginated_async(
-        uri=uri, params=query_params, synapse_client=client
-    ):
+    async for item in rest_get_paginated_async(uri=uri, synapse_client=client):
         yield item

synapseclient/client.py

@@ -294,17 +294,17 @@ class Synapse(object):
     Example: Getting started
         Logging in to Synapse using an authToken
 
-            ```python
-            import synapseclient
-            syn = synapseclient.login(authToken="authtoken")
-            ```
+        ```python
+        import synapseclient
+        syn = synapseclient.login(authToken="authtoken")
+        ```
 
         Using environment variable or `.synapseConfig`
 
-            ```python
-            import synapseclient
-            syn = synapseclient.login()
-            ```
+        ```python
+        import synapseclient
+        syn = synapseclient.login()
+        ```
 
     Example: Adding an additional `user_agent` value
         This example shows how to add an additional `user_agent` to the HTTP headers

synapseclient/core/async_utils.py

@@ -133,7 +133,8 @@ def wrap_async_generator_to_sync_generator(async_gen_func: Callable, *args, **kw
             f"For Jupyter/IPython notebooks: You can use 'async for' directly in cells.\n"
             f"For other async contexts: Ensure you're in an async function and use 'async for'."
         )
-    elif loop:
+
+    if loop:
         nest_asyncio.apply(loop=loop)
 
         # Create the async generator
@@ -143,7 +144,7 @@ def wrap_async_generator_to_sync_generator(async_gen_func: Callable, *args, **kw
         try:
             while True:
                 try:
-                    item = loop.run_until_complete(async_gen.__anext__())
+                    item = loop.run_until_complete(anext(async_gen))
                     yield item
                 except StopAsyncIteration:
                     break
@@ -154,23 +155,26 @@ def wrap_async_generator_to_sync_generator(async_gen_func: Callable, *args, **kw
             except (RuntimeError, StopAsyncIteration):
                 pass
     else:
-        # No running loop, create a new one
-        async def run_generator():
-            async_gen = async_gen_func(*args, **kwargs)
-            items = []
-            try:
-                async for item in async_gen:
-                    items.append(item)
-            finally:
+        # No running loop, create a new one and yield items one by one
+        async def async_wrapper():
+            async for item in async_gen_func(*args, **kwargs):
+                yield item
+
+        loop = asyncio.new_event_loop()
+        async_gen = async_wrapper()
+        try:
+            while True:
                 try:
-                    await async_gen.aclose()
-                except (RuntimeError, StopAsyncIteration):
-                    pass
-            return items
-
-        items = asyncio.run(run_generator())
-        for item in items:
-            yield item
+                    item = loop.run_until_complete(anext(async_gen))
+                    yield item
+                except StopAsyncIteration:
+                    break
+        finally:
+            try:
+                loop.run_until_complete(async_gen.aclose())
+            except (RuntimeError, StopAsyncIteration):
+                pass
+            loop.close()
 
 
 # Adapted from