Build docker images before testing (#396)

* label docker images

* fix syntax

* build images before testing

* try something different

* needs

* fewer linebreaks

* try as string

* move the if

* remove one condition

* fix

* cancel more quickly

* run benchmark

* don't build on main in run_benchmark

* add GITHUB_STEP_SUMMARY

* don't need stderr redirected

* better step summary grep

Author: scottgigante-immunai

.github/workflows/build_docker.yml

@@ -4,12 +4,110 @@ on:
   push:
     tags:
       - '*'
-    branches-ignore:
-      - 'test_docker'
+    branches:
+      - '**'
   pull_request:
     types: [opened, synchronize, reopened, ready_for_review]
 
 jobs:
+  cancel_previous_runs:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Cancel Previous Runs
+        uses: styfle/cancel-workflow-action@0.6.0
+        with:
+          access_token: ${{ github.token }}
+
+  build_images:
+
+    needs: cancel_previous_runs
+    runs-on: ${{ matrix.config.os }}
+    if: |
+      !endsWith(github.event.head_commit.message, '# ci skip') &&
+      (
+        startsWith(github.ref, 'refs/heads/test_docker') ||
+        (
+          startsWith(github.ref, 'refs/heads/main') &&
+          github.repository == 'openproblems-bio/openproblems'
+        )
+      )
+
+    env:
+      BRANCH_NAME: "auto_update_docker_${{ github.run_number }}"
+
+    strategy:
+      fail-fast: false
+      matrix:
+        config:
+        - {name: 'current', os: ubuntu-18.04, python: '3.8' }
+
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        fetch-depth: 1000
+
+    - name: Set up Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.config.python }}
+
+    - name: Install package & dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -U wheel setuptools
+        pip install --editable .[evaluate]
+        python -c "import openproblems"
+
+    - name: Build Docker images
+      run: |
+        cd workflow
+        snakemake -j $(grep -c processor /proc/cpuinfo) docker_build
+        cd ..
+
+    - name: Push Docker images
+      if: startsWith(github.ref, 'refs/heads/main')
+      env:
+        DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
+      run: |
+        cd workflow
+        snakemake -j $(grep -c processor /proc/cpuinfo) docker_push
+        cd ..
+        echo "CHANGED=`git diff --exit-code > /dev/null && echo false || echo true`" >> $GITHUB_ENV
+
+    - name: Set up Git branch
+      if: env.CHANGED == 'true' && startsWith(github.ref, 'refs/heads/main')
+      run: |
+        git checkout -b $BRANCH_NAME
+        git push -u origin $BRANCH_NAME
+
+    - name: Commit result
+      if: env.CHANGED == 'true' && startsWith(github.ref, 'refs/heads/main')
+      uses: EndBug/add-and-commit@v6
+      with:
+        author_name: SingleCellOpenProblems
+        author_email: singlecellopenproblems@protonmail.com
+        message: 'Update docker version # ci skip'
+        add: "docker/.version"
+        branch: ${{ env.BRANCH_NAME }}
+        push: false
+
+    - name: Create Pull Request
+      if: env.CHANGED == 'true' && startsWith(github.ref, 'refs/heads/main')
+      uses: peter-evans/create-pull-request@v3
+      with:
+        branch: ${{ env.BRANCH_NAME }}
+        delete-branch: true
+        base: main
+        title: '[auto] Update docker version'
+        reviewers: scottgigante, dburkhardt
+
+    - name: Upload check results on fail
+      if: failure()
+      uses: actions/upload-artifact@main
+      with:
+        name: ${{ matrix.config.name }}_results
+        path: check
+
   run_tester:
     runs-on: ubuntu-latest
 
@@ -20,18 +118,18 @@ jobs:
         - /tmp:/tmp
       options: --user root
 
-    if: >-
+    needs: build_images
+    if: |
+      always() &&
+      (needs.build_images.result == 'success' || needs.build_images.result == 'skipped') &&
       !endsWith(github.event.head_commit.message, '# ci skip') &&
       (
         startsWith(github.ref, 'refs/heads') ||
-        github.event.pull_request.draft == false
+        github.event.pull_request.draft == false &&
+        !startsWith(github.ref, 'refs/heads/test_docker')
       )
 
     steps:
-    - name: Cancel Previous Runs
-      uses: styfle/cancel-workflow-action@0.6.0
-      with:
-        access_token: ${{ github.token }}
 
     - uses: actions/checkout@v2
       with:
@@ -74,10 +172,11 @@ jobs:
         install_renv("docker/openproblems-r-base/r_requirements.txt")
       shell: Rscript {0}
 
-    - name: Build Docker images
+    - name: Update Docker images
+      if: "!(startsWith(github.ref, 'refs/heads/main') && github.repository == 'openproblems-bio/openproblems')"
       run: |
         cd workflow
-        snakemake -j 4 docker
+        snakemake -j $(nproc) docker
         cd ..
 
     - name: Run tests
@@ -99,8 +198,10 @@ jobs:
     needs: run_tester
     runs-on: ${{ matrix.config.os }}
     if: >-
+      always() &&
       !contains(github.event.head_commit.message, 'ci skip') &&
-      github.event_name == 'push'
+      github.event_name == 'push' &&
+      needs.run_tester.result == 'success'
 
     env:
       UPDATE_BRANCH_NAME: "auto_update_benchmark_${{ github.run_number }}"
@@ -215,7 +316,15 @@ jobs:
         openproblems-cli --version
         openproblems-cli --test-hash
 
+    - name: Pull Docker images
+      if: "(startsWith(github.ref, 'refs/heads/main') && github.repository == 'openproblems-bio/openproblems')"
+      run: |
+        cd workflow
+        snakemake -j $(nproc) docker_pull
+        cd ..
+
     - name: Build Docker images
+      if: "!(startsWith(github.ref, 'refs/heads/main') && github.repository == 'openproblems-bio/openproblems')"
       run: |
         cd workflow
         snakemake -j $(nproc) docker
@@ -254,8 +363,8 @@ jobs:
         -name "${RUN_NAME}" \
         -e.PYTHONPATH="${PYTHONPATH}" \
         openproblems-bio/nf-openproblems \
-        --branch ${BRANCH} 2>&1 | \
-        grep -v "^WARN: Unable to retrieve AWS batch instance type.*Invalid identifier"
+        --branch ${BRANCH} | \
+        tee >(grep --color=never --line-buffered "Monitor the execution with Nextflow Tower using this url" >> $GITHUB_STEP_SUMMARY)
       shell: /bin/bash -exou pipefail {0}
 
     - name: Copy results