Release v4.9.0: Merge pull request #1214 from Sage-Bionetworks/v4.9.0-rc-dev

Release v4.9.0

Author: BryanFauble

.env.example

@@ -0,0 +1,6 @@
+## Here are the environment variables you can set for the Synapse Python client to configure OpenTelemetry tracing and metrics:
+# You can copy this file to `.env` and fill in the values as needed.
+# OTEL_SERVICE_NAME=my-service-using-synapse-python-client
+# OTEL_EXPORTER_OTLP_ENDPOINT=http://fill-me-in
+# OTEL_SERVICE_INSTANCE_ID=local_development_testing
+# OTEL_EXPORTER_OTLP_HEADERS=# Authorization

.github/scripts/delete_evaluations.py

@@ -0,0 +1,70 @@
+import asyncio
+from typing import Set
+
+from synapseclient import Evaluation, Synapse
+
+syn = Synapse()
+syn.login()
+
+# Maximum number of concurrent deletion operations
+MAX_CONCURRENT_DELETIONS = 5
+
+
+async def delete_evaluation(eval_obj: Evaluation) -> str:
+    """Delete an evaluation asynchronously and return the status"""
+    try:
+        # Need to use run_in_executor since the delete function is synchronous
+        loop = asyncio.get_running_loop()
+        await loop.run_in_executor(None, lambda: syn.delete(eval_obj))
+        return f"Deleted evaluation {eval_obj.id}"
+    except Exception as e:
+        return f"Failed to delete evaluation {eval_obj.id}: {str(e)}"
+
+
+async def main():
+    # Create a semaphore to limit concurrent operations
+    semaphore = asyncio.Semaphore(MAX_CONCURRENT_DELETIONS)
+
+    # Set to track active tasks
+    pending_tasks: Set[asyncio.Task] = set()
+
+    # Track if we've processed any evaluations
+    processed_any = False
+
+    async def delete_with_semaphore(eval_obj: Evaluation):
+        """Helper function that uses the semaphore to limit concurrency"""
+        async with semaphore:
+            result = await delete_evaluation(eval_obj)
+            print(result)
+            return result
+
+    # Process evaluations as they come in from the paginated iterator
+    for result in syn._GET_paginated(
+        "/evaluation?accessType=DELETE", limit=200, offset=0
+    ):
+        processed_any = True
+        eval_obj = Evaluation(**result)
+
+        # Create a new task for this evaluation
+        task = asyncio.create_task(delete_with_semaphore(eval_obj))
+        pending_tasks.add(task)
+        task.add_done_callback(pending_tasks.discard)
+
+        # Process any completed tasks when we reach MAX_CONCURRENT_DELETIONS
+        if len(pending_tasks) >= MAX_CONCURRENT_DELETIONS:
+            # Wait for at least one task to complete before continuing
+            done, _ = await asyncio.wait(
+                pending_tasks, return_when=asyncio.FIRST_COMPLETED
+            )
+
+    # Wait for all remaining tasks to complete
+    if pending_tasks:
+        await asyncio.gather(*pending_tasks)
+
+    if not processed_any:
+        print("No evaluations found to delete")
+
+
+# Run the async main function
+if __name__ == "__main__":
+    asyncio.run(main())

.github/scripts/delete_projects.py

@@ -0,0 +1,67 @@
+import asyncio
+from typing import Dict, Set
+
+from synapseclient import Synapse
+
+syn = Synapse()
+syn.login()
+
+# Maximum number of concurrent deletion operations
+MAX_CONCURRENT_DELETIONS = 5
+
+
+async def delete_project(project_id: str) -> str:
+    """Delete a project asynchronously and return the result status"""
+    try:
+        # Need to use run_in_executor since the delete function is synchronous
+        loop = asyncio.get_running_loop()
+        await loop.run_in_executor(None, lambda: syn.delete(project_id))
+        return f"Deleted {project_id}"
+    except Exception as e:
+        return f"Failed to delete {project_id}: {str(e)}"
+
+
+async def main():
+    # Create a semaphore to limit concurrent operations
+    semaphore = asyncio.Semaphore(MAX_CONCURRENT_DELETIONS)
+
+    # Set to track active tasks
+    pending_tasks: Set[asyncio.Task] = set()
+
+    # Track if we've processed any projects
+    processed_any = False
+
+    async def delete_with_semaphore(project: Dict):
+        """Helper function that uses the semaphore to limit concurrency"""
+        async with semaphore:
+            result = await delete_project(project["id"])
+            print(result)
+            return result
+
+    # Process projects as they come in from the iterator
+    for project in syn.getChildren(parent=None, includeTypes=["project"]):
+        processed_any = True
+
+        # Create a new task for this project
+        task = asyncio.create_task(delete_with_semaphore(project))
+        pending_tasks.add(task)
+        task.add_done_callback(pending_tasks.discard)
+
+        # Process any completed tasks when we reach MAX_CONCURRENT_DELETIONS
+        if len(pending_tasks) >= MAX_CONCURRENT_DELETIONS:
+            # Wait for at least one task to complete before continuing
+            done, _ = await asyncio.wait(
+                pending_tasks, return_when=asyncio.FIRST_COMPLETED
+            )
+
+    # Wait for all remaining tasks to complete
+    if pending_tasks:
+        await asyncio.gather(*pending_tasks)
+
+    if not processed_any:
+        print("No projects found to delete")
+
+
+# Run the async main function
+if __name__ == "__main__":
+    asyncio.run(main())

.github/scripts/delete_teams.py

@@ -0,0 +1,70 @@
+import asyncio
+from typing import Dict, Set
+
+from synapseclient import Synapse
+
+syn = Synapse()
+syn.login()
+
+# Maximum number of concurrent team deletions
+MAX_CONCURRENT_DELETIONS = 5
+
+
+async def delete_team(team_id: str) -> str:
+    """Delete a team asynchronously and return the team_id"""
+    try:
+        # Need to use run_in_executor since the delete_team function is synchronous
+        loop = asyncio.get_running_loop()
+        await loop.run_in_executor(None, lambda: syn.delete_team(team_id))
+        return f"Deleted team {team_id}"
+    except Exception as e:
+        return f"Failed to delete team {team_id}: {str(e)}"
+
+
+async def main():
+    # Get all teams for the current user
+    teams = syn._find_teams_for_principal(principal_id=syn.credentials.owner_id)
+
+    # Create a semaphore to limit concurrent operations
+    semaphore = asyncio.Semaphore(MAX_CONCURRENT_DELETIONS)
+
+    # Set to track active tasks
+    pending_tasks: Set[asyncio.Task] = set()
+
+    # Track if we've processed any teams
+    processed_any = False
+
+    async def delete_with_semaphore(team: Dict):
+        """Helper function that uses the semaphore to limit concurrency"""
+        async with semaphore:
+            result = await delete_team(team["id"])
+            print(result)
+            return result
+
+    # Process teams as they come in from the iterator
+    for team in teams:
+        processed_any = True
+
+        # Create a new task for this team
+        task = asyncio.create_task(delete_with_semaphore(team))
+        pending_tasks.add(task)
+        task.add_done_callback(pending_tasks.discard)
+
+        # Process any completed tasks
+        if len(pending_tasks) >= MAX_CONCURRENT_DELETIONS:
+            # Wait for at least one task to complete before continuing
+            done, _ = await asyncio.wait(
+                pending_tasks, return_when=asyncio.FIRST_COMPLETED
+            )
+
+    # Wait for all remaining tasks to complete
+    if pending_tasks:
+        await asyncio.gather(*pending_tasks)
+
+    if not processed_any:
+        print("No teams found to delete")
+
+
+# Run the async main function
+if __name__ == "__main__":
+    asyncio.run(main())

.github/scripts/empty_trash.py

@@ -0,0 +1,70 @@
+import asyncio
+from typing import Set
+
+from synapseclient import Synapse
+
+syn = Synapse()
+syn.login()
+
+# Maximum number of concurrent deletion operations
+MAX_CONCURRENT_DELETIONS = 5
+
+
+async def purge_entity(entity_id: str) -> str:
+    """Purge an entity from trash asynchronously and return the status"""
+    try:
+        # Need to use run_in_executor since the restPUT function is synchronous
+        loop = asyncio.get_running_loop()
+        await loop.run_in_executor(
+            None, lambda: syn.restPUT(uri=f"/trashcan/purge/{entity_id}")
+        )
+        return f"Purged entity {entity_id} from trash"
+    except Exception as e:
+        return f"Failed to purge entity {entity_id}: {str(e)}"
+
+
+async def main():
+    # Create a semaphore to limit concurrent operations
+    semaphore = asyncio.Semaphore(MAX_CONCURRENT_DELETIONS)
+
+    # Set to track active tasks
+    pending_tasks: Set[asyncio.Task] = set()
+
+    # Track if we've processed any entities
+    processed_any = False
+
+    async def purge_with_semaphore(entity_id: str):
+        """Helper function that uses the semaphore to limit concurrency"""
+        async with semaphore:
+            result = await purge_entity(entity_id)
+            print(result)
+            return result
+
+    # Process entities as they come in from the paginated iterator
+    for result in syn._GET_paginated("/trashcan/view", limit=200, offset=0):
+        processed_any = True
+        entity_id = result["entityId"]
+
+        # Create a new task for this entity
+        task = asyncio.create_task(purge_with_semaphore(entity_id))
+        pending_tasks.add(task)
+        task.add_done_callback(pending_tasks.discard)
+
+        # Process any completed tasks when we reach MAX_CONCURRENT_DELETIONS
+        if len(pending_tasks) >= MAX_CONCURRENT_DELETIONS:
+            # Wait for at least one task to complete before continuing
+            done, _ = await asyncio.wait(
+                pending_tasks, return_when=asyncio.FIRST_COMPLETED
+            )
+
+    # Wait for all remaining tasks to complete
+    if pending_tasks:
+        await asyncio.gather(*pending_tasks)
+
+    if not processed_any:
+        print("No entities found in trash to purge")
+
+
+# Run the async main function
+if __name__ == "__main__":
+    asyncio.run(main())

.github/workflows/build.yml

@@ -83,12 +83,14 @@ jobs:
           path: |
             ${{ steps.get-dependencies.outputs.site_packages_loc }}
             ${{ steps.get-dependencies.outputs.site_bin_dir }}
-          key: ${{ runner.os }}-${{ matrix.python }}-build-${{ env.cache-name }}-${{ hashFiles('setup.py') }}-v22
+          key: ${{ runner.os }}-${{ matrix.python }}-build-${{ env.cache-name }}-${{ hashFiles('setup.py') }}-v24
 
       - name: Install py-dependencies
         if: steps.cache-dependencies.outputs.cache-hit != 'true'
         shell: bash
         run: |
+          python -m pip install --upgrade pip
+
           pip install -e ".[boto3,pandas,pysftp,tests]"
 
           # ensure that numpy c extensions are installed on windows
@@ -121,40 +123,6 @@ jobs:
           else
             echo "synapse_pat_available=true" >> $GITHUB_OUTPUT;
           fi
-      - name: OpenTelemtry pre-check
-        id: otel-check
-        if: ${{ steps.secret-check.outputs.secrets_available == 'true' && steps.secret-check.outputs.synapse_pat_available == 'true' }}
-        shell: bash
-        run: |
-          # Leave disabled during normal integration test runs - Enable when we want to
-          # collect the data.
-          # echo "run_opentelemetry=true" >> $GITHUB_OUTPUT;
-          echo "run_opentelemetry=false" >> $GITHUB_OUTPUT;
-
-          # AWS CLI is pre-installed on github hosted runners - Commented out for GH runs
-            # curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
-            # unzip awscliv2.zip
-            # sudo ./aws/install
-            # curl "https://s3.amazonaws.com/session-manager-downloads/plugin/latest/ubuntu_64bit/session-manager-plugin.deb" -o "session-manager-plugin.deb"
-            # sudo dpkg -i session-manager-plugin.deb
-      # - name: Create AWS Config
-      #   if: ${{ steps.otel-check.outputs.run_opentelemetry == 'true' }}
-      #   shell: bash
-      #   run: |
-      #     touch test.awsConfig
-      #     printf "[default]\nregion = us-east-1\ncredential_process = \"tests/integration/synapse_creds.sh\" \"https://sc.sageit.org\" \"${{ secrets.synapse_personal_access_token }}\"\n" >> test.awsConfig
-      #     chmod +x tests/integration/synapse_creds.sh
-      # If you are exporting data using `otlp` you can start a port forwading session
-      # - name: SSM Port Forward Start
-      #   if: ${{ steps.otel-check.outputs.run_opentelemetry == 'true' }}
-      #   shell: bash
-      #   env:
-      #     AWS_CONFIG_FILE: "test.awsConfig"
-      #   run: |
-      #     # Start a port-forwarding session in a non-interactive way. AWS will clean-up
-      #     # stale sessions after 20 minutes of inactivity
-      #     aws ssm start-session --target i-0ffcdecd1edf375ee --document-name AWS-StartPortForwardingSession  --parameters "portNumber"=["4318"],"localPortNumber"=["4318"] & disown
-      #     sleep 15
 
       # run integration tests iff the decryption keys for the test configuration are available.
       # they will not be available in pull requests from forks.
@@ -192,23 +160,31 @@ jobs:
           export EXTERNAL_S3_BUCKET_NAME="${{secrets.EXTERNAL_S3_BUCKET_NAME}}"
           export EXTERNAL_S3_BUCKET_AWS_ACCESS_KEY_ID="${{secrets.EXTERNAL_S3_BUCKET_AWS_ACCESS_KEY_ID}}"
           export EXTERNAL_S3_BUCKET_AWS_SECRET_ACCESS_KEY="${{secrets.EXTERNAL_S3_BUCKET_AWS_SECRET_ACCESS_KEY}}"
-          if [ ${{ steps.otel-check.outputs.run_opentelemetry }} == "true" ]; then
-            # Set to 'file' to enable OpenTelemetry export to file
-            export SYNAPSE_OTEL_INTEGRATION_TEST_EXPORTER="file"
+
+          # Set env vars for OTEL
+          export OTEL_EXPORTER_OTLP_ENDPOINT="${{ vars.OTEL_EXPORTER_OTLP_ENDPOINT }}"
+          export OTEL_SERVICE_INSTANCE_ID="${{ vars.OTEL_SERVICE_INSTANCE_ID }}"
+          export SYNAPSE_INTEGRATION_TEST_OTEL_ENABLED="${{ vars.SYNAPSE_INTEGRATION_TEST_OTEL_ENABLED }}"
+          export OTEL_EXPORTER_OTLP_HEADERS="${{ secrets.OTEL_EXPORTER_OTLP_HEADERS }}"
+
+          # Setup ignore patterns based on Python version
+          IGNORE_FLAGS="--ignore=tests/integration/synapseclient/test_command_line_client.py"
+
+          if [ "${{ matrix.python }}" == "3.9" ]; then
+            # For min Python version, ignore async tests
+            IGNORE_FLAGS="$IGNORE_FLAGS --ignore=tests/integration/synapseclient/models/async/"
+            echo "Running integration tests for Min Python version (3.9) - ignoring async tests"
+          elif [ "${{ matrix.python }}" == "3.13" ]; then
+            # For max Python version, ignore synchronous tests
+            IGNORE_FLAGS="$IGNORE_FLAGS --ignore=tests/integration/synapseclient/models/synchronous/"
+            echo "Running integration tests for Max Python version (3.13) - ignoring synchronous tests"
           fi
 
           # use loadscope to avoid issues running tests concurrently that share scoped fixtures
-          pytest -sv --reruns 3 --cov-append --cov=. --cov-report xml tests/integration -n 8 --ignore=tests/integration/synapseclient/test_command_line_client.py --dist loadscope
+          pytest -sv --reruns 3 --cov-append --cov=. --cov-report xml tests/integration -n 8 $IGNORE_FLAGS --dist loadscope
 
           # Execute the CLI tests in a non-dist way because they were causing some test instability when being run concurrently
           pytest -sv --reruns 3 --cov-append --cov=. --cov-report xml tests/integration/synapseclient/test_command_line_client.py
-      - name: Upload otel spans
-        uses: actions/upload-artifact@v4
-        if: always()
-        with:
-          name: otel_spans_integration_testing_${{ matrix.os }}
-          path: tests/integration/otel_spans_integration_testing_*.ndjson
-          if-no-files-found: ignore
       - name: Upload coverage report
         id: upload_coverage_report
         uses: actions/upload-artifact@v4