Skip to content

Commit 1250c82

Browse files
BryanFaubleBryanFauble
authored
[SYNPY-1618] Clean up test resources in Synapse created during integration test runs (#1209)
* Clean up test resources in Synapse created during integration test runs
1 parent 5aea88e commit 1250c82

5 files changed

Lines changed: 360 additions & 0 deletions

File tree

‎.github/scripts/delete_evaluations.py‎

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
import asyncio
2+
from typing import Set
3+
4+
from synapseclient import Evaluation, Synapse
5+
6+
syn = Synapse()
7+
syn.login()
8+
9+
# Maximum number of concurrent deletion operations
10+
MAX_CONCURRENT_DELETIONS = 5
11+
12+
13+
async def delete_evaluation(eval_obj: Evaluation) -> str:
14+
"""Delete an evaluation asynchronously and return the status"""
15+
try:
16+
# Need to use run_in_executor since the delete function is synchronous
17+
loop = asyncio.get_running_loop()
18+
await loop.run_in_executor(None, lambda: syn.delete(eval_obj))
19+
return f"Deleted evaluation {eval_obj.id}"
20+
except Exception as e:
21+
return f"Failed to delete evaluation {eval_obj.id}: {str(e)}"
22+
23+
24+
async def main():
25+
# Create a semaphore to limit concurrent operations
26+
semaphore = asyncio.Semaphore(MAX_CONCURRENT_DELETIONS)
27+
28+
# Set to track active tasks
29+
pending_tasks: Set[asyncio.Task] = set()
30+
31+
# Track if we've processed any evaluations
32+
processed_any = False
33+
34+
async def delete_with_semaphore(eval_obj: Evaluation):
35+
"""Helper function that uses the semaphore to limit concurrency"""
36+
async with semaphore:
37+
result = await delete_evaluation(eval_obj)
38+
print(result)
39+
return result
40+
41+
# Process evaluations as they come in from the paginated iterator
42+
for result in syn._GET_paginated(
43+
"/evaluation?accessType=DELETE", limit=200, offset=0
44+
):
45+
processed_any = True
46+
eval_obj = Evaluation(**result)
47+
48+
# Create a new task for this evaluation
49+
task = asyncio.create_task(delete_with_semaphore(eval_obj))
50+
pending_tasks.add(task)
51+
task.add_done_callback(pending_tasks.discard)
52+
53+
# Process any completed tasks when we reach MAX_CONCURRENT_DELETIONS
54+
if len(pending_tasks) >= MAX_CONCURRENT_DELETIONS:
55+
# Wait for at least one task to complete before continuing
56+
done, _ = await asyncio.wait(
57+
pending_tasks, return_when=asyncio.FIRST_COMPLETED
58+
)
59+
60+
# Wait for all remaining tasks to complete
61+
if pending_tasks:
62+
await asyncio.gather(*pending_tasks)
63+
64+
if not processed_any:
65+
print("No evaluations found to delete")
66+
67+
68+
# Run the async main function
69+
if __name__ == "__main__":
70+
asyncio.run(main())

‎.github/scripts/delete_projects.py‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
import asyncio
2+
from typing import Dict, Set
3+
4+
from synapseclient import Synapse
5+
6+
syn = Synapse()
7+
syn.login()
8+
9+
# Maximum number of concurrent deletion operations
10+
MAX_CONCURRENT_DELETIONS = 5
11+
12+
13+
async def delete_project(project_id: str) -> str:
14+
"""Delete a project asynchronously and return the result status"""
15+
try:
16+
# Need to use run_in_executor since the delete function is synchronous
17+
loop = asyncio.get_running_loop()
18+
await loop.run_in_executor(None, lambda: syn.delete(project_id))
19+
return f"Deleted {project_id}"
20+
except Exception as e:
21+
return f"Failed to delete {project_id}: {str(e)}"
22+
23+
24+
async def main():
25+
# Create a semaphore to limit concurrent operations
26+
semaphore = asyncio.Semaphore(MAX_CONCURRENT_DELETIONS)
27+
28+
# Set to track active tasks
29+
pending_tasks: Set[asyncio.Task] = set()
30+
31+
# Track if we've processed any projects
32+
processed_any = False
33+
34+
async def delete_with_semaphore(project: Dict):
35+
"""Helper function that uses the semaphore to limit concurrency"""
36+
async with semaphore:
37+
result = await delete_project(project["id"])
38+
print(result)
39+
return result
40+
41+
# Process projects as they come in from the iterator
42+
for project in syn.getChildren(parent=None, includeTypes=["project"]):
43+
processed_any = True
44+
45+
# Create a new task for this project
46+
task = asyncio.create_task(delete_with_semaphore(project))
47+
pending_tasks.add(task)
48+
task.add_done_callback(pending_tasks.discard)
49+
50+
# Process any completed tasks when we reach MAX_CONCURRENT_DELETIONS
51+
if len(pending_tasks) >= MAX_CONCURRENT_DELETIONS:
52+
# Wait for at least one task to complete before continuing
53+
done, _ = await asyncio.wait(
54+
pending_tasks, return_when=asyncio.FIRST_COMPLETED
55+
)
56+
57+
# Wait for all remaining tasks to complete
58+
if pending_tasks:
59+
await asyncio.gather(*pending_tasks)
60+
61+
if not processed_any:
62+
print("No projects found to delete")
63+
64+
65+
# Run the async main function
66+
if __name__ == "__main__":
67+
asyncio.run(main())

‎.github/scripts/delete_teams.py‎

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
import asyncio
2+
from typing import Dict, Set
3+
4+
from synapseclient import Synapse
5+
6+
syn = Synapse()
7+
syn.login()
8+
9+
# Maximum number of concurrent team deletions
10+
MAX_CONCURRENT_DELETIONS = 5
11+
12+
13+
async def delete_team(team_id: str) -> str:
14+
"""Delete a team asynchronously and return the team_id"""
15+
try:
16+
# Need to use run_in_executor since the delete_team function is synchronous
17+
loop = asyncio.get_running_loop()
18+
await loop.run_in_executor(None, lambda: syn.delete_team(team_id))
19+
return f"Deleted team {team_id}"
20+
except Exception as e:
21+
return f"Failed to delete team {team_id}: {str(e)}"
22+
23+
24+
async def main():
25+
# Get all teams for the current user
26+
teams = syn._find_teams_for_principal(principal_id=syn.credentials.owner_id)
27+
28+
# Create a semaphore to limit concurrent operations
29+
semaphore = asyncio.Semaphore(MAX_CONCURRENT_DELETIONS)
30+
31+
# Set to track active tasks
32+
pending_tasks: Set[asyncio.Task] = set()
33+
34+
# Track if we've processed any teams
35+
processed_any = False
36+
37+
async def delete_with_semaphore(team: Dict):
38+
"""Helper function that uses the semaphore to limit concurrency"""
39+
async with semaphore:
40+
result = await delete_team(team["id"])
41+
print(result)
42+
return result
43+
44+
# Process teams as they come in from the iterator
45+
for team in teams:
46+
processed_any = True
47+
48+
# Create a new task for this team
49+
task = asyncio.create_task(delete_with_semaphore(team))
50+
pending_tasks.add(task)
51+
task.add_done_callback(pending_tasks.discard)
52+
53+
# Process any completed tasks
54+
if len(pending_tasks) >= MAX_CONCURRENT_DELETIONS:
55+
# Wait for at least one task to complete before continuing
56+
done, _ = await asyncio.wait(
57+
pending_tasks, return_when=asyncio.FIRST_COMPLETED
58+
)
59+
60+
# Wait for all remaining tasks to complete
61+
if pending_tasks:
62+
await asyncio.gather(*pending_tasks)
63+
64+
if not processed_any:
65+
print("No teams found to delete")
66+
67+
68+
# Run the async main function
69+
if __name__ == "__main__":
70+
asyncio.run(main())

‎.github/scripts/empty_trash.py‎

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
import asyncio
2+
from typing import Set
3+
4+
from synapseclient import Synapse
5+
6+
syn = Synapse()
7+
syn.login()
8+
9+
# Maximum number of concurrent deletion operations
10+
MAX_CONCURRENT_DELETIONS = 5
11+
12+
13+
async def purge_entity(entity_id: str) -> str:
14+
"""Purge an entity from trash asynchronously and return the status"""
15+
try:
16+
# Need to use run_in_executor since the restPUT function is synchronous
17+
loop = asyncio.get_running_loop()
18+
await loop.run_in_executor(
19+
None, lambda: syn.restPUT(uri=f"/trashcan/purge/{entity_id}")
20+
)
21+
return f"Purged entity {entity_id} from trash"
22+
except Exception as e:
23+
return f"Failed to purge entity {entity_id}: {str(e)}"
24+
25+
26+
async def main():
27+
# Create a semaphore to limit concurrent operations
28+
semaphore = asyncio.Semaphore(MAX_CONCURRENT_DELETIONS)
29+
30+
# Set to track active tasks
31+
pending_tasks: Set[asyncio.Task] = set()
32+
33+
# Track if we've processed any entities
34+
processed_any = False
35+
36+
async def purge_with_semaphore(entity_id: str):
37+
"""Helper function that uses the semaphore to limit concurrency"""
38+
async with semaphore:
39+
result = await purge_entity(entity_id)
40+
print(result)
41+
return result
42+
43+
# Process entities as they come in from the paginated iterator
44+
for result in syn._GET_paginated("/trashcan/view", limit=200, offset=0):
45+
processed_any = True
46+
entity_id = result["entityId"]
47+
48+
# Create a new task for this entity
49+
task = asyncio.create_task(purge_with_semaphore(entity_id))
50+
pending_tasks.add(task)
51+
task.add_done_callback(pending_tasks.discard)
52+
53+
# Process any completed tasks when we reach MAX_CONCURRENT_DELETIONS
54+
if len(pending_tasks) >= MAX_CONCURRENT_DELETIONS:
55+
# Wait for at least one task to complete before continuing
56+
done, _ = await asyncio.wait(
57+
pending_tasks, return_when=asyncio.FIRST_COMPLETED
58+
)
59+
60+
# Wait for all remaining tasks to complete
61+
if pending_tasks:
62+
await asyncio.gather(*pending_tasks)
63+
64+
if not processed_any:
65+
print("No entities found in trash to purge")
66+
67+
68+
# Run the async main function
69+
if __name__ == "__main__":
70+
asyncio.run(main())

‎.github/workflows/test-cleanup.yml‎

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
name: "Integration test cleanup"
2+
3+
on:
4+
schedule:
5+
# Run every Friday at 10:00 PM Eastern Time (3:00 AM UTC Saturday)
6+
- cron: '0 3 * * 6'
7+
workflow_dispatch: # Allow manual triggering
8+
9+
jobs:
10+
cleanup:
11+
name: Delete resources
12+
runs-on: ubuntu-latest
13+
permissions:
14+
contents: read
15+
16+
steps:
17+
- name: Checkout repository
18+
uses: actions/checkout@v4
19+
20+
- name: Set up Python
21+
uses: actions/setup-python@v4
22+
with:
23+
python-version: '3.13'
24+
25+
- name: get-dependencies-location
26+
shell: bash
27+
run: |
28+
SITE_PACKAGES_LOCATION=$(python -c "from sysconfig import get_path; print(get_path('purelib'))")
29+
SITE_BIN_DIR=$(python3 -c "import os; import platform; import sysconfig; pre = sysconfig.get_config_var('prefix'); bindir = os.path.join(pre, 'Scripts' if platform.system() == 'Windows' else 'bin'); print(bindir)")
30+
echo "site_packages_loc=$SITE_PACKAGES_LOCATION" >> $GITHUB_OUTPUT
31+
echo "site_bin_dir=$SITE_BIN_DIR" >> $GITHUB_OUTPUT
32+
id: get-dependencies
33+
34+
- name: Cache py-dependencies
35+
id: cache-dependencies
36+
uses: actions/cache@v4
37+
env:
38+
cache-name: cache-py-dependencies-cleanup
39+
with:
40+
path: |
41+
${{ steps.get-dependencies.outputs.site_packages_loc }}
42+
${{ steps.get-dependencies.outputs.site_bin_dir }}
43+
key: ${{ runner.os }}-3.13-build-${{ env.cache-name }}-${{ hashFiles('setup.py') }}-v1
44+
45+
- name: Install py-dependencies
46+
if: steps.cache-dependencies.outputs.cache-hit != 'true'
47+
shell: bash
48+
run: |
49+
python -m pip install --upgrade pip
50+
51+
pip install -e ".[boto3,pandas,pysftp,tests]"
52+
53+
# ensure that numpy c extensions are installed on windows
54+
# https://stackoverflow.com/a/59346525
55+
if [ "${{startsWith(runner.os, 'Windows')}}" == "true" ]; then
56+
pip uninstall -y numpy
57+
pip uninstall -y setuptools
58+
pip install setuptools
59+
pip install numpy
60+
fi
61+
62+
- name: Set up Synapse credentials
63+
shell: bash
64+
run: |
65+
# decrypt the encrypted test synapse configuration
66+
openssl aes-256-cbc -K ${{ secrets.encrypted_d17283647768_key }} -iv ${{ secrets.encrypted_d17283647768_iv }} -in test.synapseConfig.enc -out test.synapseConfig -d
67+
mv test.synapseConfig ~/.synapseConfig
68+
69+
- name: Run evaluation deletion script
70+
run: |
71+
python .github/scripts/delete_evaluations.py
72+
73+
- name: Run project deletion script
74+
run: |
75+
python .github/scripts/delete_projects.py
76+
77+
- name: Run team deletion script
78+
run: |
79+
python .github/scripts/delete_teams.py
80+
81+
- name: Run empty trash script
82+
run: |
83+
python .github/scripts/empty_trash.py

0 commit comments

Comments
 (0)