release: docs dev to main

.copier-answers.yml

@@ -1,7 +1,7 @@
 # Changes here will be overwritten by Copier; NEVER EDIT MANUALLY
 _commit: main
 _src_path: https://github.com/ONEcampaign/bblocks-projects.git
-author_email: jorge.rivera@one.org
+author_email: data@one.org
 author_name: ONE Campaign
 include_notebooks: false
 include_pytest: true

.github/workflows/build.yml

@@ -25,28 +25,20 @@
 # if, which bash does not treat as a failure, captures the build's three-way exit status
 # and lychee's, below.
 #
-# Private sources: the build clones with a short-lived, read-only installation token for the
-# `dfo-reader` GitHub App, minted each run from the `READER_APP_CLIENT_ID` variable and the
-# `READER_APP_PRIVATE_KEY` secret. The token reaches git through a credential helper set
-# in the environment of the build step alone. Git asks a helper only when the server
-# challenges, so public sources still clone anonymously. To admit a private repository,
-# install `dfo-reader` on it and add its name to the `repositories:` input of the "Mint a
-# read-only token for private sources" step below. A missing secret fails the run at that
-# step, and nothing falls back to an anonymous clone.
+# Sources: every registered repository is public, so the build clones them all anonymously
+# and the job holds no secret beyond the workflow token that publishes the `corpus` branch.
 #
-# Every run, pull requests included, is ONEcampaign-only. The job mints a `dfo-reader`
-# token, which the DevelopmentFinanceObservatory copy of the repository keeps only in
-# main-limited environments, and the `corpus` branch it publishes is consumed only by
-# ONEcampaign. The copy skips the workflow.
+# Branches: `dev` is the integration branch and the only one that publishes. A pull request
+# into `dev` or `main` runs everything except the publish step.
 #
 # A draft pull request skips the build; marking it ready starts the run.
 name: Build and publish corpus
 
 on:
   push:
-    branches: [main]
+    branches: [dev]
   pull_request:
-    branches: [main]
+    branches: [dev, main]
     types: [opened, synchronize, reopened, ready_for_review]
   workflow_dispatch:
     inputs:
@@ -75,12 +67,7 @@ permissions:
 
 jobs:
   build:
-    # Mints a dfo-reader token, which prod keeps only in main-limited
-    # environments, so this runs in ONEcampaign only. The identical tree
-    # ran it here before it was synced.
-    if: >-
-      github.repository_owner == 'ONEcampaign' &&
-      github.event.pull_request.draft != true
+    if: github.event.pull_request.draft != true
     runs-on: ubuntu-latest
     env:
       # Every place the publish branch's name is needed reads this rather than a literal
@@ -103,51 +90,10 @@ jobs:
       - name: Run tests
         run: uv run pytest -q
 
-      # An unset variable or secret expands to an empty string, so this step names which
-      # one is missing before the token action runs.
-      - name: Check the dfo-reader credentials are configured
-        env:
-          READER_APP_CLIENT_ID: ${{ vars.READER_APP_CLIENT_ID }}
-          READER_APP_PRIVATE_KEY: ${{ secrets.READER_APP_PRIVATE_KEY }}
-        run: |
-          missing=0
-          if [ -z "$READER_APP_CLIENT_ID" ]; then
-            echo "::error::The READER_APP_CLIENT_ID variable is not set or not available to this run; private sources cannot be cloned."
-            missing=1
-          fi
-          if [ -z "$READER_APP_PRIVATE_KEY" ]; then
-            echo "::error::The READER_APP_PRIVATE_KEY secret is not set or not available to this run; private sources cannot be cloned."
-            missing=1
-          fi
-          exit "$missing"
-
-      # `permission-contents: read` narrows the token below what the App itself holds, so
-      # the build can read these repositories and write nothing to them.
-      - name: Mint a read-only token for private sources
-        id: reader
-        uses: actions/create-github-app-token@v3
-        with:
-          client-id: ${{ vars.READER_APP_CLIENT_ID }}
-          private-key: ${{ secrets.READER_APP_PRIVATE_KEY }}
-          owner: ONEcampaign
-          repositories: dfo-apis,dfo-ai,dfo-skills
-          permission-contents: read
-
       - name: Build the roadmap and the corpus
         id: build
         # This step runs under bash -e, so the build's exit code (0, 1, or 2; see
         # src/dfo_docs/cli.py's main()) is read from an if rather than from a bare $?.
-        #
-        # GIT_CONFIG_COUNT and its KEY/VALUE pairs add a credential helper for
-        # github.com to every git command in this step and in no other. The helper
-        # answers only `get`, reads the token from the environment, and so never
-        # writes it to a config file or a remote URL.
-        env:
-          READER_APP_TOKEN: ${{ steps.reader.outputs.token }}
-          GIT_CONFIG_COUNT: "1"
-          GIT_CONFIG_KEY_0: credential.https://github.com.helper
-          GIT_CONFIG_VALUE_0: >-
-            !f() { test "$1" = get || return 0; echo username=x-access-token; echo "password=$READER_APP_TOKEN"; }; f
         run: |
           set -uo pipefail
           if uv run python scripts/build.py --out published; then
@@ -184,9 +130,10 @@ jobs:
 
       # A pull request build runs every step above this one, so the build, the guards and
       # the link check all gate the merge. This step is the only one that writes, and a
-      # pull request must not move the branch the site pins, so it stops here.
+      # pull request or a run from another branch must not move the branch the site pins,
+      # so it stops there.
       - name: Publish
-        if: steps.build.outputs.status != '1' && github.event_name != 'pull_request'
+        if: steps.build.outputs.status != '1' && github.event_name != 'pull_request' && github.ref == 'refs/heads/dev'
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |

.github/workflows/ci.yml

@@ -1,12 +1,12 @@
 name: CI
 
 # A draft pull request runs nothing; marking it ready starts the run. The push to
-# main is the only test of the merged tree.
+# dev or main is the only test of the merged tree.
 on:
   push:
-    branches: [main]
+    branches: [dev, main]
   pull_request:
-    branches: [main]
+    branches: [dev, main]
     types: [opened, synchronize, reopened, ready_for_review]
 
 jobs:

.github/workflows/docsync.yml

@@ -2,37 +2,40 @@
 # requests, by running `dfo-docsync` (see src/dfo_docs/docsync/cli.py's module docstring
 # for what each command does).
 #
-# A `workflow_dispatch` run takes one command. `init` creates the tracking Sheet, `push`
-# publishes the pages, and `pull` opens pull requests from Doc edits. The daily `schedule`
-# run, at 04:00 UTC before editors start work, runs `pull` and then `push`, so edits reach
-# the repositories before a push replaces the Docs. It reads the source repositories
-# directly, so it does not wait for the build workflow. A page that fails does not stop
-# the run. The job keeps each command's exit code and fails if any command exited
-# non-zero (1 when a page or repository failed, 2 when the run could not start). A pull
-# that exits 2 skips the push. When its state save failed, the unsaved-states file must be
-# restored before another run saves state over it.
+# Only a `workflow_dispatch` run starts it, with one command. `init` creates the tracking
+# Sheet, `push` publishes the pages, and `pull` opens pull requests from Doc edits. Run
+# `pull` before `push`, so edits reach the repositories before a push replaces the Docs.
+# It reads the source repositories directly, so it does not wait for the build workflow.
+# A page that fails does not stop the run. The job fails if the command exited non-zero
+# (1 when a page or repository failed, 2 when the run could not start). When its state
+# save failed, the unsaved-states file must be restored before another run saves state
+# over it.
+#
+# The job runs from `dev` in the `docsync` environment, which holds everything it needs:
+# the variables `GCP_WIF_PROVIDER`, `DOCSYNC_SERVICE_ACCOUNT` and `DOCSYNC_APP_CLIENT_ID`,
+# and the secret `DOCSYNC_APP_PRIVATE_KEY`. The environment admits only `dev`.
 #
 # Google access: `google-github-actions/auth` exchanges the run's OIDC token for the
-# `DOCSYNC_SERVICE_ACCOUNT` service account through the workload identity provider
-# `GCP_WIF_PROVIDER`, and writes a credentials file whose path it exports as
-# GOOGLE_APPLICATION_CREDENTIALS. `dfo-docsync` finds it through Application Default
-# Credentials, so no Google token is passed explicitly.
+# `DOCSYNC_SERVICE_ACCOUNT` service account (dfo-docs-sync) through the workload identity
+# provider `GCP_WIF_PROVIDER`, and writes a credentials file whose path it exports as
+# GOOGLE_APPLICATION_CREDENTIALS. The service account's binding in infra accepts
+# `refs/heads/dev` together with the `docsync` environment. `dfo-docsync` finds the
+# credentials through Application Default Credentials, so no Google token is passed
+# explicitly.
 #
-# GitHub access: the `dfo-bot` GitHub App token is minted each run from the
-# `DFO_BOT_CLIENT_ID` repository variable and the `DFO_BOT_PRIVATE_KEY` secret, scoped to
+# GitHub access: the `dfo-docsync` GitHub App token is minted each run from the
+# `DOCSYNC_APP_CLIENT_ID` variable and the `DOCSYNC_APP_PRIVATE_KEY` secret, scoped to
 # contents and pull-requests write on the repositories docsync opens pull requests against
-# and keeps state in. To admit another source repository, install `dfo-bot` on it and add
-# its name to the `repositories:` input of the "Mint a token for dfo-bot" step. The token
-# reaches `dfo-docsync` as DFO_BOT_TOKEN, and the commits it makes are authored by the
+# and keeps state in, which are the `dev` branches of docs, apis, ai and skills. To admit
+# another source repository, install `dfo-docsync` on it and add its name to the
+# `repositories:` input of the "Mint a token for dfo-docsync" step. The token reaches the
+# `dfo-docsync` command as DFO_DOCSYNC_TOKEN, and the commits it makes are authored by the
 # App's bot user. `push` and `pull` list open docsync pull requests by the author
 # `app/<slug>`, with the slug the token step reports, so the listing follows the App's
 # real slug whatever `pr_author` in docsync.yaml says. The checkout does not persist the
 # workflow token, because git would send its read-only credential ahead of the App token.
 # The Run step gives git the App token through a credential helper in its environment.
 #
-# Only ONEcampaign runs this workflow, so only ONEcampaign writes to Google Docs. A copy
-# of the repository in another organisation skips the job.
-#
 # A missing variable or secret fails the run at the first step that names it.
 name: Sync docs with Google Docs
 
@@ -45,8 +48,6 @@ on:
         options: [init, push, pull]
         required: false
         default: push
-  schedule:
-    - cron: "0 4 * * *"
 
 # Runs queue behind each other. A cancelled push or pull could stop half way through
 # writing Docs, the Sheet or the state branch.
@@ -60,10 +61,11 @@ permissions:
 
 jobs:
   sync:
-    # The WIF binding in dfo-infra admits only main, and the guard keeps the dfo-bot write
-    # token off branch runs. The owner check keeps Google Docs writes in ONEcampaign.
-    if: github.repository_owner == 'ONEcampaign' && github.ref == 'refs/heads/main'
+    # The WIF binding in infra and the environment's branch policy both admit only dev,
+    # and the guard keeps the dfo-docsync write token off branch runs.
+    if: github.ref == 'refs/heads/dev'
     runs-on: ubuntu-latest
+    environment: docsync
     timeout-minutes: 60
     steps:
       # Full history, because hub pages are read from git (`git show`) at their baseline.
@@ -88,24 +90,24 @@ jobs:
         env:
           GCP_WIF_PROVIDER: ${{ vars.GCP_WIF_PROVIDER }}
           DOCSYNC_SERVICE_ACCOUNT: ${{ vars.DOCSYNC_SERVICE_ACCOUNT }}
-          DFO_BOT_CLIENT_ID: ${{ vars.DFO_BOT_CLIENT_ID }}
-          DFO_BOT_PRIVATE_KEY: ${{ secrets.DFO_BOT_PRIVATE_KEY }}
+          DOCSYNC_APP_CLIENT_ID: ${{ vars.DOCSYNC_APP_CLIENT_ID }}
+          DOCSYNC_APP_PRIVATE_KEY: ${{ secrets.DOCSYNC_APP_PRIVATE_KEY }}
         run: |
           missing=0
           if [ -z "$GCP_WIF_PROVIDER" ]; then
-            echo "::error::The GCP_WIF_PROVIDER repository variable is not set; the run cannot authenticate to Google."
+            echo "::error::The GCP_WIF_PROVIDER variable is not set; the run cannot authenticate to Google."
             missing=1
           fi
           if [ -z "$DOCSYNC_SERVICE_ACCOUNT" ]; then
-            echo "::error::The DOCSYNC_SERVICE_ACCOUNT repository variable is not set; the run cannot authenticate to Google."
+            echo "::error::The DOCSYNC_SERVICE_ACCOUNT variable is not set; the run cannot authenticate to Google."
             missing=1
           fi
-          if [ -z "$DFO_BOT_CLIENT_ID" ]; then
-            echo "::error::The DFO_BOT_CLIENT_ID repository variable is not set; the run cannot open pull requests or write state."
+          if [ -z "$DOCSYNC_APP_CLIENT_ID" ]; then
+            echo "::error::The DOCSYNC_APP_CLIENT_ID variable is not set; the run cannot open pull requests or write state."
             missing=1
           fi
-          if [ -z "$DFO_BOT_PRIVATE_KEY" ]; then
-            echo "::error::The DFO_BOT_PRIVATE_KEY secret is not set or not available to this run; the run cannot open pull requests or write state."
+          if [ -z "$DOCSYNC_APP_PRIVATE_KEY" ]; then
+            echo "::error::The DOCSYNC_APP_PRIVATE_KEY secret is not set or not available to this run; the run cannot open pull requests or write state."
             missing=1
           fi
           exit "$missing"
@@ -118,24 +120,23 @@ jobs:
           create_credentials_file: true
 
       # The repositories are the ones docsync reads sources from and opens pull requests
-      # against, plus dfo-docs, which also holds the `docsync-state` branch.
-      - name: Mint a token for dfo-bot
-        id: dfo-bot
+      # against, plus docs, which also holds the `docsync-state` branch.
+      - name: Mint a token for dfo-docsync
+        id: dfo-docsync
         uses: actions/create-github-app-token@v3
         with:
-          client-id: ${{ vars.DFO_BOT_CLIENT_ID }}
-          private-key: ${{ secrets.DFO_BOT_PRIVATE_KEY }}
-          owner: ONEcampaign
-          repositories: dfo-docs,dfo-apis,dfo-ai,dfo-skills
+          client-id: ${{ vars.DOCSYNC_APP_CLIENT_ID }}
+          private-key: ${{ secrets.DOCSYNC_APP_PRIVATE_KEY }}
+          repositories: docs,apis,ai,skills
           permission-contents: write
           permission-pull-requests: write
 
       # The App's bot user is `<slug>[bot]`, and its noreply address carries the user id.
       # git reads the identity from the global config for every commit docsync makes.
-      - name: Set the git identity to the dfo-bot user
+      - name: Set the git identity to the dfo-docsync user
         env:
-          GH_TOKEN: ${{ steps.dfo-bot.outputs.token }}
-          APP_SLUG: ${{ steps.dfo-bot.outputs.app-slug }}
+          GH_TOKEN: ${{ steps.dfo-docsync.outputs.token }}
+          APP_SLUG: ${{ steps.dfo-docsync.outputs.app-slug }}
         run: |
           set -euo pipefail
           bot="${APP_SLUG}[bot]"
@@ -144,54 +145,25 @@ jobs:
           git config --global user.email "${bot_id}+${bot}@users.noreply.github.com"
 
       # GIT_CONFIG_COUNT and its KEY/VALUE pair add a credential helper for github.com to
-      # every git command in this step and in no other, as in build.yml's build step. The
-      # sources are private repositories, so the clones and the lazy blob fetches that
-      # `git show` makes in the partial clones need it. The helper answers only `get` and
-      # reads the token from the environment, so the token stays out of config files and
-      # remote URLs.
+      # every git command in this step and in no other. The pushes of the edit branches and
+      # the `docsync-state` branch need it. The helper answers only `get` and reads the
+      # token from the environment, so the token stays out of config files and remote URLs.
       - name: Run dfo-docsync
         env:
-          DFO_BOT_TOKEN: ${{ steps.dfo-bot.outputs.token }}
-          APP_SLUG: ${{ steps.dfo-bot.outputs.app-slug }}
+          DFO_DOCSYNC_TOKEN: ${{ steps.dfo-docsync.outputs.token }}
+          APP_SLUG: ${{ steps.dfo-docsync.outputs.app-slug }}
           COMMAND: ${{ inputs.command }}
           GIT_CONFIG_COUNT: "1"
           GIT_CONFIG_KEY_0: credential.https://github.com.helper
           GIT_CONFIG_VALUE_0: >-
-            !f() { test "$1" = get || return 0; echo username=x-access-token; echo "password=$DFO_BOT_TOKEN"; }; f
+            !f() { test "$1" = get || return 0; echo username=x-access-token; echo "password=$DFO_DOCSYNC_TOKEN"; }; f
         run: |
           set -uo pipefail
-          if [ "$GITHUB_EVENT_NAME" = "schedule" ]; then
-            commands=(pull push)
-          else
-            commands=("$COMMAND")
+          args=()
+          if [ "$COMMAND" != "init" ]; then
+            args+=(--pr-author "app/$APP_SLUG")
           fi
-
-          # Each command's exit status is read from an if, because the step runs under
-          # bash -e. A pull that exits 1 still lets the push run, one that exits 2 stops
-          # the loop, and the worst status wins.
-          worst=0
-          for command in "${commands[@]}"; do
-            echo "::group::dfo-docsync $command"
-            args=()
-            if [ "$command" != "init" ]; then
-              args+=(--pr-author "app/$APP_SLUG")
-            fi
-            if uv run dfo-docsync "$command" "${args[@]}"; then
-              status=0
-            else
-              status=$?
-            fi
-            echo "::endgroup::"
-            echo "dfo-docsync $command exited $status."
-            if [ "$status" -gt "$worst" ]; then
-              worst=$status
-            fi
-            if [ "$command" = "pull" ] && [ "$status" -gt 1 ]; then
-              echo "::error::dfo-docsync pull exited $status, so push is skipped."
-              break
-            fi
-          done
-          exit "$worst"
+          uv run dfo-docsync "$COMMAND" "${args[@]}"
 
       # Baselines and Doc ids of unsaved states, with no tokens. Restore a file with
       # `uv run dfo-docsync restore FILE`, adding --state-dir DIR for a local state directory.