Skip to content

Commit 8ce4467

Browse files
scottgigante-immunaiscottgigante-immunai
authored
Dispatch CI to tower (#398)
* try using nf-core/tower-action * shortcut * fix cancel * faster * compute_env * try a different ID * use name not ID * try curl * comment cleanup * syntax * POST * set run_name * fix name * fix * typo * add process_results workflow * install deps * just fetch the data from aws * cp * argv * fix workflow * cp quiet * full ci * typo * fix logic * return s3fs and cli * pass github_pat * fix cancel * Revert "fix cancel" This reverts commit fd54e7d. * more logic fixes
1 parent 2c2d785 commit 8ce4467

3 files changed

Lines changed: 140 additions & 82 deletions

File tree

‎.github/workflows/process_results.yml‎

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,96 @@
1+
name: Process Nextflow results
2+
3+
on:
4+
repository_dispatch:
5+
types: [benchmark_complete]
6+
7+
jobs:
8+
process_results:
9+
runs-on: ubuntu-latest
10+
11+
env:
12+
UPDATE_BRANCH_NAME: "auto_update_benchmark_${{ github.run_number }}"
13+
14+
steps:
15+
- uses: actions/checkout@v2
16+
with:
17+
fetch-depth: 0
18+
19+
- name: Set up Python
20+
uses: actions/setup-python@v2
21+
with:
22+
python-version: ${{ matrix.config.python }}
23+
24+
- name: Install AWS CLI
25+
run: |
26+
mkdir /tmp/awscli
27+
cd /tmp/awscli
28+
curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
29+
unzip -q awscliv2.zip
30+
sudo ./aws/install || sudo ./aws/install --update
31+
aws --version
32+
33+
- name: Install package & dependencies
34+
run: |
35+
python -m pip install --upgrade pip
36+
pip install -U wheel setuptools
37+
pip install -U --editable .[process]
38+
python -c "import openproblems"
39+
40+
- name: Parse results
41+
env:
42+
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
43+
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
44+
AWS_DEFAULT_REGION: us-west-2
45+
run: |
46+
aws s3 cp --quiet --recursive "s3://openproblems-nextflow/cwd_main" results/
47+
python workflow/parse_nextflow.py $(pwd)
48+
49+
- name: AWS S3 cleanup
50+
env:
51+
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
52+
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
53+
AWS_DEFAULT_REGION: us-west-2
54+
run: |
55+
aws s3 rm --recursive "s3://openproblems-nextflow/work_main"
56+
aws s3 rm --recursive "s3://openproblems-nextflow/cwd_main"
57+
58+
- name: Remove untagged images
59+
continue-on-error: true
60+
env:
61+
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
62+
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
63+
AWS_DEFAULT_REGION: us-west-2
64+
run: |
65+
IMAGES_TO_DELETE=$( aws ecr list-images --region $AWS_DEFAULT_REGION --repository-name openproblems --filter "tagStatus=UNTAGGED" --query 'imageIds[*]' --max-items 100 --output json )
66+
aws ecr batch-delete-image --region $AWS_DEFAULT_REGION --repository-name openproblems --image-ids "$IMAGES_TO_DELETE"
67+
68+
- name: Pre-commit
69+
uses: pre-commit/action@v2.0.0
70+
continue-on-error: true
71+
env:
72+
SKIP: style-files,lintr
73+
74+
- name: Set up Git branch
75+
run: |
76+
git checkout -b $UPDATE_BRANCH_NAME
77+
git push -u origin $UPDATE_BRANCH_NAME
78+
79+
- name: Commit result
80+
uses: EndBug/add-and-commit@v6
81+
with:
82+
author_name: SingleCellOpenProblems
83+
author_email: singlecellopenproblems@protonmail.com
84+
message: 'Update benchmark results # ci skip'
85+
add: "['website/data/results', 'results.json']"
86+
branch: ${{ env.UPDATE_BRANCH_NAME }}
87+
push: false
88+
89+
- name: Create Pull Request
90+
uses: peter-evans/create-pull-request@v3
91+
with:
92+
branch: ${{ env.UPDATE_BRANCH_NAME }}
93+
delete-branch: true
94+
base: main
95+
title: '[auto] Update benchmark results'
96+
reviewers: scottgigante, dburkhardt

‎.github/workflows/run_tests.yml‎

Lines changed: 40 additions & 81 deletions
Original file line numberDiff line numberDiff line change
@@ -125,6 +125,7 @@ jobs:
125125
!endsWith(github.event.head_commit.message, '# ci skip') &&
126126
(
127127
startsWith(github.ref, 'refs/heads') ||
128+
startsWith(github.ref, 'refs/tags') ||
128129
github.event.pull_request.draft == false &&
129130
!startsWith(github.ref, 'refs/heads/test_docker')
130131
)
@@ -199,14 +200,10 @@ jobs:
199200
runs-on: ${{ matrix.config.os }}
200201
if: >-
201202
always() &&
202-
!contains(github.event.head_commit.message, 'ci skip') &&
203+
!endsWith(github.event.head_commit.message, '# ci skip') &&
203204
github.event_name == 'push' &&
204205
needs.run_tester.result == 'success'
205206
206-
env:
207-
UPDATE_BRANCH_NAME: "auto_update_benchmark_${{ github.run_number }}"
208-
RESULTS_PATH: "/tmp/${{ github.ref_name }}_results"
209-
210207
strategy:
211208
fail-fast: false
212209
matrix:
@@ -247,21 +244,13 @@ jobs:
247244
echo "UBUNTU_VERSION=`grep DISTRIB_RELEASE /etc/lsb-release | sed 's/.*=//g'`" >> $GITHUB_ENV
248245
# If not on the base repository, append first 6 characters of username to the image name
249246
# to avoid clashes on ECR
250-
if [[ "${{ github.repository }}" == "openproblems-bio/openproblems" ]]; \
251-
then echo "BRANCH=`echo ${{ github.ref }} | sed 's:refs/[a-z]*/::' | sed 's:[/_]:-:g'| head -c 40`"; \
252-
else echo "BRANCH=`echo ${{ github.repository }} | awk '{print $1}' FS=/ | head -c 6`-`echo ${{ github.ref }} | sed 's:refs/[a-z]*/::' | sed 's:[/_]:-:g'| head -c 33`"; \
247+
if [[ "${{ github.repository }}" == "openproblems-bio/openproblems" ]]; then
248+
echo "BRANCH=`echo ${{ github.ref }} | sed 's:refs/[a-z]*/::' | sed 's:[/_]:-:g'| head -c 40`"; \
249+
else
250+
echo "BRANCH=`echo ${{ github.repository }} | awk '{print $1}' FS=/ | head -c 6`-`echo ${{ github.ref }} | sed 's:refs/[a-z]*/::' | sed 's:[/_]:-:g'| head -c 33`"; \
253251
fi >> $GITHUB_ENV
254252
# don't use tower if the env var is not set
255253
echo "WITH_TOWER=`if [ $TOWER_ACCESS_TOKEN ]; then echo "-with-tower"; else echo; fi`" >> $GITHUB_ENV
256-
if [ $(echo ${{ github.ref }} | grep -E -e "^refs/heads/(main|test_benchmark)" -e "^refs/tags") ]; then
257-
PROFILE="aws"
258-
RESUME=""
259-
else
260-
PROFILE="test,aws"
261-
RESUME="-resume"
262-
fi
263-
echo "PROFILE=$PROFILE" >> $GITHUB_ENV
264-
echo "RESUME=$RESUME" >> $GITHUB_ENV
265254
266255
- name: Cache Python packages
267256
uses: actions/cache@v2
@@ -344,7 +333,7 @@ jobs:
344333
docker push --quiet ${ECR_ENDPOINT}/openproblems:${BRANCH}-${image}
345334
done
346335
347-
- name: Run benchmark
336+
- name: Run test benchmark
348337
env:
349338
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
350339
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
@@ -356,31 +345,26 @@ jobs:
356345
RUN_NAME="$(echo "$BRANCH" | sed "s/[^a-z]//g")_$(git rev-parse --short HEAD)_${GITHUB_RUN_ATTEMPT}"
357346
cd /mnt/openproblems-nextflow/cwd/${BRANCH}
358347
nextflow run $WITH_TOWER \
359-
-ansi-log false $RESUME \
360-
-profile $PROFILE \
348+
-ansi-log false -resume \
349+
-profile aws,test \
361350
-work-dir "/mnt/openproblems-nextflow/work/${BRANCH}" \
362351
-bucket-dir "s3://openproblems-nextflow/bucket/${BRANCH}" \
363352
-name "${RUN_NAME}" \
364353
-e.PYTHONPATH="${PYTHONPATH}" \
365354
openproblems-bio/nf-openproblems \
366355
--branch ${BRANCH} | \
367356
tee >(grep --color=never --line-buffered "Monitor the execution with Nextflow Tower using this url" >> $GITHUB_STEP_SUMMARY)
368-
shell: /bin/bash -exou pipefail {0}
357+
shell: /bin/bash -eou pipefail {0}
369358

370-
- name: Copy results
371-
run: |
372-
mkdir -p "${RESULTS_PATH}"
373-
cp -r /mnt/openproblems-nextflow/cwd/${{ env.BRANCH }}/results "${RESULTS_PATH}"
374-
375-
- name: Parse results
376-
# There's a bug with the results getting pulled from Nextflow with caching, but
377-
# this doesn't happen on main, which is the only place `Parse results` is needed
378-
if: startsWith(github.ref, 'refs/heads/main') || startsWith(github.ref, 'refs/tags')
379-
run: |
380-
python workflow/parse_nextflow.py "${RESULTS_PATH}"
359+
- name: Upload nextflow log
360+
if: always()
361+
uses: actions/upload-artifact@main
362+
with:
363+
name: nextflow.log
364+
path: /mnt/openproblems-nextflow/cwd/${{ env.BRANCH }}/.nextflow.log
381365

382-
- name: AWS cleanup
383-
if: success() || startsWith(github.ref, 'refs/heads/main') || startsWith(github.ref, 'refs/tags')
366+
- name: AWS ECR cleanup
367+
if: success() && !startsWith(github.ref, 'refs/heads/main')
384368
env:
385369
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
386370
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
@@ -389,58 +373,33 @@ jobs:
389373
for image in $(cd docker && ls -1d */ | tr -d '/'); do
390374
aws ecr batch-delete-image --region $AWS_DEFAULT_REGION --repository-name openproblems --image-ids "imageTag=${BRANCH}-${image}"
391375
done
392-
aws s3 rm --recursive "s3://openproblems-nextflow/work/${{ env.BRANCH }}"
393-
aws s3 rm --recursive "s3://openproblems-nextflow/bucket/${{ env.BRANCH }}"
394-
aws s3 rm --recursive "s3://openproblems-nextflow/cwd/${{ env.BRANCH }}"
395376
396-
- name: Remove untagged images
397-
if: startsWith(github.ref, 'refs/heads/main')
398-
continue-on-error: true
377+
- name: AWS S3 cleanup
378+
if: success() || startsWith(github.ref, 'refs/heads/main') || startsWith(github.ref, 'refs/tags')
399379
env:
400380
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
401381
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
402382
AWS_DEFAULT_REGION: us-west-2
403383
run: |
404-
IMAGES_TO_DELETE=$( aws ecr list-images --region $AWS_DEFAULT_REGION --repository-name openproblems --filter "tagStatus=UNTAGGED" --query 'imageIds[*]' --max-items 100 --output json )
405-
aws ecr batch-delete-image --region $AWS_DEFAULT_REGION --repository-name openproblems --image-ids "$IMAGES_TO_DELETE"
406-
407-
- name: Pre-commit
408-
if: startsWith(github.ref, 'refs/tags')
409-
uses: pre-commit/action@v2.0.0
410-
continue-on-error: true
411-
env:
412-
SKIP: style-files,lintr
384+
aws s3 rm --recursive "s3://openproblems-nextflow/work/${{ env.BRANCH }}"
385+
aws s3 rm --recursive "s3://openproblems-nextflow/bucket/${{ env.BRANCH }}"
386+
aws s3 rm --recursive "s3://openproblems-nextflow/cwd/${{ env.BRANCH }}"
413387
414-
- name: Set up Git branch
415-
if: startsWith(github.ref, 'refs/tags')
388+
- name: Run full benchmark
389+
if: |
390+
(startsWith(github.ref, 'refs/heads/test_benchmark') || startsWith(github.ref, 'refs/tags')) &&
391+
github.repository == 'openproblems-bio/openproblems'
416392
run: |
417-
git checkout -b $UPDATE_BRANCH_NAME
418-
git push -u origin $UPDATE_BRANCH_NAME
419-
420-
- name: Commit result
421-
if: startsWith(github.ref, 'refs/tags')
422-
uses: EndBug/add-and-commit@v6
423-
with:
424-
author_name: SingleCellOpenProblems
425-
author_email: singlecellopenproblems@protonmail.com
426-
message: 'Update benchmark results # ci skip'
427-
add: "['website/data/results', 'results.json']"
428-
branch: ${{ env.UPDATE_BRANCH_NAME }}
429-
push: false
430-
431-
- name: Create Pull Request
432-
if: startsWith(github.ref, 'refs/tags')
433-
uses: peter-evans/create-pull-request@v3
434-
with:
435-
branch: ${{ env.UPDATE_BRANCH_NAME }}
436-
delete-branch: true
437-
base: main
438-
title: '[auto] Update benchmark results'
439-
reviewers: scottgigante, dburkhardt
440-
441-
- name: Upload results
442-
if: always()
443-
uses: actions/upload-artifact@main
444-
with:
445-
name: ${{ matrix.config.name }}_results
446-
path: ${{ env.RESULTS_PATH }}
393+
generate_parameters()
394+
{
395+
cat <<EOF
396+
{
397+
"branch": "main",
398+
"github_pat": "${{ secrets.GH_WORKFLOW_PAT }}"
399+
}
400+
EOF
401+
}
402+
curl -H "Content-Type: application/json" \
403+
-H "Authorization: Bearer ${{ secrets.TOWER_ACCESS_KEY }}" \
404+
https://api.tower.nf/actions/2kxpi4Q0zNCy6wPExD8yD5/launch?workspaceId=53907369739130 \
405+
--data "$(generate_parameters)"

‎setup.py‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,9 @@
2323
"anndata2ri>=1.0.6",
2424
]
2525

26-
evaluate_requires = ["snakemake", "numpyencoder"]
26+
evaluate_requires = ["snakemake"]
27+
28+
process_requires = ["numpyencoder"]
2729

2830
test_requires = [
2931
"pytest",
@@ -59,6 +61,7 @@
5961
"doc": doc_requires,
6062
"r": r_requires,
6163
"evaluate": evaluate_requires,
64+
"process": process_requires,
6265
},
6366
entry_points={
6467
"console_scripts": ["openproblems-cli=openproblems.api.main:main"],

0 commit comments

Comments
 (0)