diff --git a/.copier-answers.yml b/.copier-answers.yml index 626d96d..4bca36a 100644 --- a/.copier-answers.yml +++ b/.copier-answers.yml @@ -1,7 +1,7 @@ # Changes here will be overwritten by Copier; NEVER EDIT MANUALLY _commit: main _src_path: https://github.com/ONEcampaign/bblocks-projects.git -author_email: jorge.rivera@one.org +author_email: data@one.org author_name: ONE Campaign include_notebooks: false include_pytest: true diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 0a03183..87ecca9 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -25,28 +25,20 @@ # if, which bash does not treat as a failure, captures the build's three-way exit status # and lychee's, below. # -# Private sources: the build clones with a short-lived, read-only installation token for the -# `dfo-reader` GitHub App, minted each run from the `READER_APP_CLIENT_ID` variable and the -# `READER_APP_PRIVATE_KEY` secret. The token reaches git through a credential helper set -# in the environment of the build step alone. Git asks a helper only when the server -# challenges, so public sources still clone anonymously. To admit a private repository, -# install `dfo-reader` on it and add its name to the `repositories:` input of the "Mint a -# read-only token for private sources" step below. A missing secret fails the run at that -# step, and nothing falls back to an anonymous clone. +# Sources: every registered repository is public, so the build clones them all anonymously +# and the job holds no secret beyond the workflow token that publishes the `corpus` branch. # -# Every run, pull requests included, is ONEcampaign-only. The job mints a `dfo-reader` -# token, which the DevelopmentFinanceObservatory copy of the repository keeps only in -# main-limited environments, and the `corpus` branch it publishes is consumed only by -# ONEcampaign. The copy skips the workflow. +# Branches: `dev` is the integration branch and the only one that publishes. A pull request +# into `dev` or `main` runs everything except the publish step. # # A draft pull request skips the build; marking it ready starts the run. name: Build and publish corpus on: push: - branches: [main] + branches: [dev] pull_request: - branches: [main] + branches: [dev, main] types: [opened, synchronize, reopened, ready_for_review] workflow_dispatch: inputs: @@ -75,12 +67,7 @@ permissions: jobs: build: - # Mints a dfo-reader token, which prod keeps only in main-limited - # environments, so this runs in ONEcampaign only. The identical tree - # ran it here before it was synced. - if: >- - github.repository_owner == 'ONEcampaign' && - github.event.pull_request.draft != true + if: github.event.pull_request.draft != true runs-on: ubuntu-latest env: # Every place the publish branch's name is needed reads this rather than a literal @@ -103,51 +90,10 @@ jobs: - name: Run tests run: uv run pytest -q - # An unset variable or secret expands to an empty string, so this step names which - # one is missing before the token action runs. - - name: Check the dfo-reader credentials are configured - env: - READER_APP_CLIENT_ID: ${{ vars.READER_APP_CLIENT_ID }} - READER_APP_PRIVATE_KEY: ${{ secrets.READER_APP_PRIVATE_KEY }} - run: | - missing=0 - if [ -z "$READER_APP_CLIENT_ID" ]; then - echo "::error::The READER_APP_CLIENT_ID variable is not set or not available to this run; private sources cannot be cloned." - missing=1 - fi - if [ -z "$READER_APP_PRIVATE_KEY" ]; then - echo "::error::The READER_APP_PRIVATE_KEY secret is not set or not available to this run; private sources cannot be cloned." - missing=1 - fi - exit "$missing" - - # `permission-contents: read` narrows the token below what the App itself holds, so - # the build can read these repositories and write nothing to them. - - name: Mint a read-only token for private sources - id: reader - uses: actions/create-github-app-token@v3 - with: - client-id: ${{ vars.READER_APP_CLIENT_ID }} - private-key: ${{ secrets.READER_APP_PRIVATE_KEY }} - owner: ONEcampaign - repositories: dfo-apis,dfo-ai,dfo-skills - permission-contents: read - - name: Build the roadmap and the corpus id: build # This step runs under bash -e, so the build's exit code (0, 1, or 2; see # src/dfo_docs/cli.py's main()) is read from an if rather than from a bare $?. - # - # GIT_CONFIG_COUNT and its KEY/VALUE pairs add a credential helper for - # github.com to every git command in this step and in no other. The helper - # answers only `get`, reads the token from the environment, and so never - # writes it to a config file or a remote URL. - env: - READER_APP_TOKEN: ${{ steps.reader.outputs.token }} - GIT_CONFIG_COUNT: "1" - GIT_CONFIG_KEY_0: credential.https://github.com.helper - GIT_CONFIG_VALUE_0: >- - !f() { test "$1" = get || return 0; echo username=x-access-token; echo "password=$READER_APP_TOKEN"; }; f run: | set -uo pipefail if uv run python scripts/build.py --out published; then @@ -184,9 +130,10 @@ jobs: # A pull request build runs every step above this one, so the build, the guards and # the link check all gate the merge. This step is the only one that writes, and a - # pull request must not move the branch the site pins, so it stops here. + # pull request or a run from another branch must not move the branch the site pins, + # so it stops there. - name: Publish - if: steps.build.outputs.status != '1' && github.event_name != 'pull_request' + if: steps.build.outputs.status != '1' && github.event_name != 'pull_request' && github.ref == 'refs/heads/dev' env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index da8c2d7..f4e5540 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,12 +1,12 @@ name: CI # A draft pull request runs nothing; marking it ready starts the run. The push to -# main is the only test of the merged tree. +# dev or main is the only test of the merged tree. on: push: - branches: [main] + branches: [dev, main] pull_request: - branches: [main] + branches: [dev, main] types: [opened, synchronize, reopened, ready_for_review] jobs: diff --git a/.github/workflows/docsync.yml b/.github/workflows/docsync.yml index 5d3ee61..d29e268 100644 --- a/.github/workflows/docsync.yml +++ b/.github/workflows/docsync.yml @@ -2,37 +2,40 @@ # requests, by running `dfo-docsync` (see src/dfo_docs/docsync/cli.py's module docstring # for what each command does). # -# A `workflow_dispatch` run takes one command. `init` creates the tracking Sheet, `push` -# publishes the pages, and `pull` opens pull requests from Doc edits. The daily `schedule` -# run, at 04:00 UTC before editors start work, runs `pull` and then `push`, so edits reach -# the repositories before a push replaces the Docs. It reads the source repositories -# directly, so it does not wait for the build workflow. A page that fails does not stop -# the run. The job keeps each command's exit code and fails if any command exited -# non-zero (1 when a page or repository failed, 2 when the run could not start). A pull -# that exits 2 skips the push. When its state save failed, the unsaved-states file must be -# restored before another run saves state over it. +# Only a `workflow_dispatch` run starts it, with one command. `init` creates the tracking +# Sheet, `push` publishes the pages, and `pull` opens pull requests from Doc edits. Run +# `pull` before `push`, so edits reach the repositories before a push replaces the Docs. +# It reads the source repositories directly, so it does not wait for the build workflow. +# A page that fails does not stop the run. The job fails if the command exited non-zero +# (1 when a page or repository failed, 2 when the run could not start). When its state +# save failed, the unsaved-states file must be restored before another run saves state +# over it. +# +# The job runs from `dev` in the `docsync` environment, which holds everything it needs: +# the variables `GCP_WIF_PROVIDER`, `DOCSYNC_SERVICE_ACCOUNT` and `DOCSYNC_APP_CLIENT_ID`, +# and the secret `DOCSYNC_APP_PRIVATE_KEY`. The environment admits only `dev`. # # Google access: `google-github-actions/auth` exchanges the run's OIDC token for the -# `DOCSYNC_SERVICE_ACCOUNT` service account through the workload identity provider -# `GCP_WIF_PROVIDER`, and writes a credentials file whose path it exports as -# GOOGLE_APPLICATION_CREDENTIALS. `dfo-docsync` finds it through Application Default -# Credentials, so no Google token is passed explicitly. +# `DOCSYNC_SERVICE_ACCOUNT` service account (dfo-docs-sync) through the workload identity +# provider `GCP_WIF_PROVIDER`, and writes a credentials file whose path it exports as +# GOOGLE_APPLICATION_CREDENTIALS. The service account's binding in infra accepts +# `refs/heads/dev` together with the `docsync` environment. `dfo-docsync` finds the +# credentials through Application Default Credentials, so no Google token is passed +# explicitly. # -# GitHub access: the `dfo-bot` GitHub App token is minted each run from the -# `DFO_BOT_CLIENT_ID` repository variable and the `DFO_BOT_PRIVATE_KEY` secret, scoped to +# GitHub access: the `dfo-docsync` GitHub App token is minted each run from the +# `DOCSYNC_APP_CLIENT_ID` variable and the `DOCSYNC_APP_PRIVATE_KEY` secret, scoped to # contents and pull-requests write on the repositories docsync opens pull requests against -# and keeps state in. To admit another source repository, install `dfo-bot` on it and add -# its name to the `repositories:` input of the "Mint a token for dfo-bot" step. The token -# reaches `dfo-docsync` as DFO_BOT_TOKEN, and the commits it makes are authored by the +# and keeps state in, which are the `dev` branches of docs, apis, ai and skills. To admit +# another source repository, install `dfo-docsync` on it and add its name to the +# `repositories:` input of the "Mint a token for dfo-docsync" step. The token reaches the +# `dfo-docsync` command as DFO_DOCSYNC_TOKEN, and the commits it makes are authored by the # App's bot user. `push` and `pull` list open docsync pull requests by the author # `app/`, with the slug the token step reports, so the listing follows the App's # real slug whatever `pr_author` in docsync.yaml says. The checkout does not persist the # workflow token, because git would send its read-only credential ahead of the App token. # The Run step gives git the App token through a credential helper in its environment. # -# Only ONEcampaign runs this workflow, so only ONEcampaign writes to Google Docs. A copy -# of the repository in another organisation skips the job. -# # A missing variable or secret fails the run at the first step that names it. name: Sync docs with Google Docs @@ -45,8 +48,6 @@ on: options: [init, push, pull] required: false default: push - schedule: - - cron: "0 4 * * *" # Runs queue behind each other. A cancelled push or pull could stop half way through # writing Docs, the Sheet or the state branch. @@ -60,10 +61,11 @@ permissions: jobs: sync: - # The WIF binding in dfo-infra admits only main, and the guard keeps the dfo-bot write - # token off branch runs. The owner check keeps Google Docs writes in ONEcampaign. - if: github.repository_owner == 'ONEcampaign' && github.ref == 'refs/heads/main' + # The WIF binding in infra and the environment's branch policy both admit only dev, + # and the guard keeps the dfo-docsync write token off branch runs. + if: github.ref == 'refs/heads/dev' runs-on: ubuntu-latest + environment: docsync timeout-minutes: 60 steps: # Full history, because hub pages are read from git (`git show`) at their baseline. @@ -88,24 +90,24 @@ jobs: env: GCP_WIF_PROVIDER: ${{ vars.GCP_WIF_PROVIDER }} DOCSYNC_SERVICE_ACCOUNT: ${{ vars.DOCSYNC_SERVICE_ACCOUNT }} - DFO_BOT_CLIENT_ID: ${{ vars.DFO_BOT_CLIENT_ID }} - DFO_BOT_PRIVATE_KEY: ${{ secrets.DFO_BOT_PRIVATE_KEY }} + DOCSYNC_APP_CLIENT_ID: ${{ vars.DOCSYNC_APP_CLIENT_ID }} + DOCSYNC_APP_PRIVATE_KEY: ${{ secrets.DOCSYNC_APP_PRIVATE_KEY }} run: | missing=0 if [ -z "$GCP_WIF_PROVIDER" ]; then - echo "::error::The GCP_WIF_PROVIDER repository variable is not set; the run cannot authenticate to Google." + echo "::error::The GCP_WIF_PROVIDER variable is not set; the run cannot authenticate to Google." missing=1 fi if [ -z "$DOCSYNC_SERVICE_ACCOUNT" ]; then - echo "::error::The DOCSYNC_SERVICE_ACCOUNT repository variable is not set; the run cannot authenticate to Google." + echo "::error::The DOCSYNC_SERVICE_ACCOUNT variable is not set; the run cannot authenticate to Google." missing=1 fi - if [ -z "$DFO_BOT_CLIENT_ID" ]; then - echo "::error::The DFO_BOT_CLIENT_ID repository variable is not set; the run cannot open pull requests or write state." + if [ -z "$DOCSYNC_APP_CLIENT_ID" ]; then + echo "::error::The DOCSYNC_APP_CLIENT_ID variable is not set; the run cannot open pull requests or write state." missing=1 fi - if [ -z "$DFO_BOT_PRIVATE_KEY" ]; then - echo "::error::The DFO_BOT_PRIVATE_KEY secret is not set or not available to this run; the run cannot open pull requests or write state." + if [ -z "$DOCSYNC_APP_PRIVATE_KEY" ]; then + echo "::error::The DOCSYNC_APP_PRIVATE_KEY secret is not set or not available to this run; the run cannot open pull requests or write state." missing=1 fi exit "$missing" @@ -118,24 +120,23 @@ jobs: create_credentials_file: true # The repositories are the ones docsync reads sources from and opens pull requests - # against, plus dfo-docs, which also holds the `docsync-state` branch. - - name: Mint a token for dfo-bot - id: dfo-bot + # against, plus docs, which also holds the `docsync-state` branch. + - name: Mint a token for dfo-docsync + id: dfo-docsync uses: actions/create-github-app-token@v3 with: - client-id: ${{ vars.DFO_BOT_CLIENT_ID }} - private-key: ${{ secrets.DFO_BOT_PRIVATE_KEY }} - owner: ONEcampaign - repositories: dfo-docs,dfo-apis,dfo-ai,dfo-skills + client-id: ${{ vars.DOCSYNC_APP_CLIENT_ID }} + private-key: ${{ secrets.DOCSYNC_APP_PRIVATE_KEY }} + repositories: docs,apis,ai,skills permission-contents: write permission-pull-requests: write # The App's bot user is `[bot]`, and its noreply address carries the user id. # git reads the identity from the global config for every commit docsync makes. - - name: Set the git identity to the dfo-bot user + - name: Set the git identity to the dfo-docsync user env: - GH_TOKEN: ${{ steps.dfo-bot.outputs.token }} - APP_SLUG: ${{ steps.dfo-bot.outputs.app-slug }} + GH_TOKEN: ${{ steps.dfo-docsync.outputs.token }} + APP_SLUG: ${{ steps.dfo-docsync.outputs.app-slug }} run: | set -euo pipefail bot="${APP_SLUG}[bot]" @@ -144,54 +145,25 @@ jobs: git config --global user.email "${bot_id}+${bot}@users.noreply.github.com" # GIT_CONFIG_COUNT and its KEY/VALUE pair add a credential helper for github.com to - # every git command in this step and in no other, as in build.yml's build step. The - # sources are private repositories, so the clones and the lazy blob fetches that - # `git show` makes in the partial clones need it. The helper answers only `get` and - # reads the token from the environment, so the token stays out of config files and - # remote URLs. + # every git command in this step and in no other. The pushes of the edit branches and + # the `docsync-state` branch need it. The helper answers only `get` and reads the + # token from the environment, so the token stays out of config files and remote URLs. - name: Run dfo-docsync env: - DFO_BOT_TOKEN: ${{ steps.dfo-bot.outputs.token }} - APP_SLUG: ${{ steps.dfo-bot.outputs.app-slug }} + DFO_DOCSYNC_TOKEN: ${{ steps.dfo-docsync.outputs.token }} + APP_SLUG: ${{ steps.dfo-docsync.outputs.app-slug }} COMMAND: ${{ inputs.command }} GIT_CONFIG_COUNT: "1" GIT_CONFIG_KEY_0: credential.https://github.com.helper GIT_CONFIG_VALUE_0: >- - !f() { test "$1" = get || return 0; echo username=x-access-token; echo "password=$DFO_BOT_TOKEN"; }; f + !f() { test "$1" = get || return 0; echo username=x-access-token; echo "password=$DFO_DOCSYNC_TOKEN"; }; f run: | set -uo pipefail - if [ "$GITHUB_EVENT_NAME" = "schedule" ]; then - commands=(pull push) - else - commands=("$COMMAND") + args=() + if [ "$COMMAND" != "init" ]; then + args+=(--pr-author "app/$APP_SLUG") fi - - # Each command's exit status is read from an if, because the step runs under - # bash -e. A pull that exits 1 still lets the push run, one that exits 2 stops - # the loop, and the worst status wins. - worst=0 - for command in "${commands[@]}"; do - echo "::group::dfo-docsync $command" - args=() - if [ "$command" != "init" ]; then - args+=(--pr-author "app/$APP_SLUG") - fi - if uv run dfo-docsync "$command" "${args[@]}"; then - status=0 - else - status=$? - fi - echo "::endgroup::" - echo "dfo-docsync $command exited $status." - if [ "$status" -gt "$worst" ]; then - worst=$status - fi - if [ "$command" = "pull" ] && [ "$status" -gt 1 ]; then - echo "::error::dfo-docsync pull exited $status, so push is skipped." - break - fi - done - exit "$worst" + uv run dfo-docsync "$COMMAND" "${args[@]}" # Baselines and Doc ids of unsaved states, with no tokens. Restore a file with # `uv run dfo-docsync restore FILE`, adding --state-dir DIR for a local state directory. diff --git a/.github/workflows/registry-health.yml b/.github/workflows/registry-health.yml index 36a6b53..af5fdf4 100644 --- a/.github/workflows/registry-health.yml +++ b/.github/workflows/registry-health.yml @@ -8,15 +8,7 @@ # (see lychee.toml's `offline = true`). This job checks the corpus branch, the tree # the site actually reads, without that restriction. # -# The default GITHUB_TOKEN cannot read a private repository outside this one, so the rename -# and archival check reads the private sources with a read-only `dfo-reader` installation -# token, minted the same way as in build.yml. That token covers only the repositories named -# in READER_REPOSITORIES below; every other source keeps using GITHUB_TOKEN, which reads any -# public repository. Admitting a private source means adding it to READER_REPOSITORIES here -# as well as to the build workflow's `repositories:` list. -# -# Only ONEcampaign runs this workflow. It checks the `corpus` branch that build.yml -# publishes there, and a copy in another organisation publishes none. +# Every registered repository is public, so the default GITHUB_TOKEN reads them all. # # This opens a GitHub issue rather than failing the workflow run. A red X on a # scheduled job is easy to miss for months. Whoever inherits this repository reads @@ -37,11 +29,7 @@ permissions: jobs: registry-health: - if: github.repository_owner == 'ONEcampaign' runs-on: ubuntu-latest - env: - # The ONEcampaign repositories the dfo-reader token is minted for, comma-separated. - READER_REPOSITORIES: dfo-apis,dfo-ai,dfo-skills steps: - name: Check out dfo-docs uses: actions/checkout@v7 @@ -55,38 +43,10 @@ jobs: - name: Install dependencies run: uv sync --frozen - - name: Check the dfo-reader credentials are configured - env: - READER_APP_CLIENT_ID: ${{ vars.READER_APP_CLIENT_ID }} - READER_APP_PRIVATE_KEY: ${{ secrets.READER_APP_PRIVATE_KEY }} - run: | - missing=0 - if [ -z "$READER_APP_CLIENT_ID" ]; then - echo "::error::The READER_APP_CLIENT_ID variable is not set or not available to this run; private sources cannot be checked." - missing=1 - fi - if [ -z "$READER_APP_PRIVATE_KEY" ]; then - echo "::error::The READER_APP_PRIVATE_KEY secret is not set or not available to this run; private sources cannot be checked." - missing=1 - fi - exit "$missing" - - # `GET /repos/{repo}` needs metadata read and nothing more. - - name: Mint a read-only token for private sources - id: reader - uses: actions/create-github-app-token@v3 - with: - client-id: ${{ vars.READER_APP_CLIENT_ID }} - private-key: ${{ secrets.READER_APP_PRIVATE_KEY }} - owner: ONEcampaign - repositories: ${{ env.READER_REPOSITORIES }} - permission-metadata: read - - name: Check registered repositories for a rename or archival id: registry env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - READER_APP_TOKEN: ${{ steps.reader.outputs.token }} run: | uv run python - <<'PY' import json @@ -96,10 +56,7 @@ jobs: from dfo_docs.registry import load_sources - reader_repos = { - f"ONEcampaign/{name.strip()}" - for name in os.environ["READER_REPOSITORIES"].split(",") - } + token = os.environ["GITHUB_TOKEN"] sources = load_sources("sources.yaml") findings = [] @@ -107,9 +64,6 @@ jobs: if not source.enabled or "/" not in source.repo: continue # a disabled row, or a non-GitHub repo shorthand, has nothing to check - token = os.environ[ - "READER_APP_TOKEN" if source.repo in reader_repos else "GITHUB_TOKEN" - ] request = urllib.request.Request( f"https://api.github.com/repos/{source.repo}", headers={ diff --git a/.github/workflows/sync-guard.yml b/.github/workflows/sync-guard.yml deleted file mode 100644 index 6635731..0000000 --- a/.github/workflows/sync-guard.yml +++ /dev/null @@ -1,45 +0,0 @@ -# sync-guard: the required check on main in every DevelopmentFinanceObservatory -# repo. It passes only a sync PR whose tree matches its ONEcampaign source -# commit, or a release App image pin. The rules live in -# DevelopmentFinanceObservatory/infra, in .github/actions/sync-guard and -# scripts/sync-guard.sh. -# -# pull_request_target runs this file and the action from the default branch, -# never from the PR, so a PR cannot edit the guard it faces. The job is -# prod-only. In ONEcampaign it skips before the private action is ever -# fetched. -# -# The job posts the sync-guard check run on the PR's head commit with the -# release App's token, and the ruleset accepts sync-guard only from that -# App. The token's secrets live in the sync-guard environment, which only -# main may use, so a workflow a PR adds cannot mint it. -name: sync-guard - -on: - pull_request_target: - types: [opened, synchronize, reopened] - branches: [main] - -permissions: - contents: read - -# A force-push cancels the run for the old head. Each run judges the head -# commit its own event names. -concurrency: - group: sync-guard-${{ github.event.pull_request.number }} - cancel-in-progress: true - -jobs: - sync-guard: - # Named apart from the check run it posts, which is the required one. - name: sync-guard runner - if: github.repository_owner == 'DevelopmentFinanceObservatory' - runs-on: ubuntu-latest - environment: sync-guard - steps: - - uses: DevelopmentFinanceObservatory/infra/.github/actions/sync-guard@main - with: - reader-client-id: ${{ vars.READER_APP_CLIENT_ID }} - reader-private-key: ${{ secrets.READER_APP_PRIVATE_KEY }} - release-client-id: ${{ vars.RELEASE_APP_CLIENT_ID }} - release-private-key: ${{ secrets.RELEASE_APP_PRIVATE_KEY }} diff --git a/CODEOWNERS b/CODEOWNERS deleted file mode 100644 index dc1b3be..0000000 --- a/CODEOWNERS +++ /dev/null @@ -1,11 +0,0 @@ -# The hub's own content is authored directly in this repository. Everything else is fetched -# or generated, and review happens where the site pins a commit. - -/content/hub/ @jorge-rivera - -# Each roadmap track publishes as its own document and is authored, reviewed, and owned -# separately, which is why it is one file per track rather than one shared file. - -/roadmap/data.yaml @jorge-rivera -/roadmap/platform.yaml @jorge-rivera -/roadmap/ai-tools.yaml @jorge-rivera diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index cc4460b..523912a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,12 +1,14 @@ # Contributing to the DFO docs -The DFO docs serve analysts who know the DAC and the CRS, write some Python or R, and need data they can trust in their own work. AI assistants read the same pages. This guide covers where pages live, how to write one, and how a source repository publishes its own pages into the corpus. +Issues and pull requests are welcome. The ONE Data team maintains this repository. Open an issue to discuss a large change, such as a new section or a new source repository, before you write it. Contributors follow the [Code of Conduct](https://github.com/DevelopmentFinanceObservatory/.github/blob/main/CODE_OF_CONDUCT.md). Report security issues through the "Report a vulnerability" button under the repository's Security tab, as the [security policy](https://github.com/DevelopmentFinanceObservatory/.github/blob/main/SECURITY.md) describes. + +The DFO docs serve analysts who know the DAC and the CRS, write some Python or R, and need data they can trust in their own work. AI assistants read the same pages. ## Where pages live The site publishes hub pages from Markdown files under `content/hub/` in this repository. Source pages live in each source repository's docs folder, and the build fetches them through a row in `sources.yaml`. -``` +```text dfo-docs/ ├── content/hub/ hub pages; a page's URL is its path here │ ├── index.md Home, published at /docs @@ -21,7 +23,7 @@ dfo-docs/ A hub page publishes at its path under `content/hub/` without the `.md` suffix, so `content/hub/data/glossary.md` publishes at `/docs/data/glossary`. Its place in the nav comes from `hub.yaml`, so a page can move in the nav without changing its URL. Only pages named in `hub.yaml` publish. -A source repository with several pages groups them by mode: `tutorials/`, `how-to/` and `about/`, as `tossd-reader` does. +A source repository with several pages groups them by mode in `tutorials/`, `how-to/` and `about/`, as `tossd-reader` does. ## Writing a page @@ -48,11 +50,11 @@ Use DFO values in examples, including `country/NGA`, CRS purpose 12220, finance End every page with a `## Next` section of one or two links, each with a sentence saying what the reader gets there. Leave out boilerplate headings such as Overview, Summary or Conclusion. -Use only three callouts: `!!! note`, `!!! tip` and `!!! warning`. Leave them untitled and indent the body four spaces. The build renders them as site alerts. A title becomes a bold lead sentence, so put the point in the body. Use callouts for optional material and keep required information in the page body. +Use only three callouts, `!!! note`, `!!! tip` and `!!! warning`. Leave them untitled and indent the body four spaces. The build renders them as site alerts. A title becomes a bold lead sentence, so put the point in the body. Use callouts for optional material and keep required information in the page body. ```markdown !!! warning - Studio links are always snapshots. +Studio links are always snapshots. ``` The build passes collapsible `???` blocks through unchanged, where they render as raw text. Leave them out. @@ -96,9 +98,9 @@ Only published pages are valid link targets. The link check fails for a page abs ## Publishing a source repository's docs -A source repository keeps its pages in a docs folder and declares them in a nav sidecar. The build looks for the sidecar in three places, in this order: `docs/mkdocs.yml`, a root `mkdocs.yml`, and a root `zensical.toml`. The docs root is the sidecar's own `docs_dir`, resolved from the folder the sidecar sits in. +A source repository keeps its pages in a docs folder and declares them in a nav sidecar. The build looks for the sidecar at `docs/mkdocs.yml`, then a root `mkdocs.yml`, then a root `zensical.toml`. The docs root is the sidecar's own `docs_dir`, resolved from the folder the sidecar sits in. -Use a root `mkdocs.yml` when user-facing pages sit in a subfolder separate from a repository's internal docs. dfo-apis publishes `docs/guide/` this way: +Use a root `mkdocs.yml` when user-facing pages sit in a subfolder separate from a repository's internal docs. `apis` publishes `docs/guide/` this way: ```yaml site_name: API @@ -119,30 +121,40 @@ nav: Then register the repository in `sources.yaml`: ```yaml - - name: dfo-apis - repo: ONEcampaign/dfo-apis - ref: dev - docs_dir: docs/guide - path: api - title: API +- name: dfo-apis + repo: DevelopmentFinanceObservatory/apis + ref: dev + docs_dir: docs/guide + path: api + title: API ``` -In `sources.yaml`, `docs_dir` sets which folder the build checks out. Match it to the sidecar's `docs_dir`. Set `path` to `extra.dfo.path`. The build fails if they differ. Add a `mount` entry in `hub.yaml` that names the source row. To register a repository before publication, set `enabled: false` with a `reason` and leave it out of `hub.yaml`. A mount for a disabled source fails the build. A private repository also needs the `dfo-reader` GitHub App installed on it and its name added to the token steps in the build and registry-health workflows, as the README describes. The README covers `exclude`, `kind: external`, `kind: skill`, and each build failure. +In `sources.yaml`, `docs_dir` sets which folder the build checks out. Match it to the sidecar's `docs_dir`. Set `path` to `extra.dfo.path`. The build fails if they differ. Add a `mount` entry in `hub.yaml` that names the source row. To register a repository before publication, set `enabled: false` with a `reason` and leave it out of `hub.yaml`. A mount for a disabled source fails the build. Every registered repository must be public. [docs/development.md](docs/development.md) covers `exclude`, `kind: external`, `kind: skill`, and each build failure. -Before opening a pull request in the source repository, check its nav against its pages from a dfo-docs checkout. Then check links from the source repository's docs folder: +Before opening a pull request in the source repository, check its nav against its pages from a `docs` checkout. Then check links from the source repository's docs folder: -``` +```bash uv run python scripts/check_nav.py /path/to/source-repo lychee --offline /path/to/source-repo/docs/guide/ ``` ## Checking a change here -``` +```bash uv run ruff check . +uv run ruff format --check . +uv run ty check src/dfo_docs uv run pytest uv run python scripts/build.py --out /tmp/dfo-docs-build lychee --config lychee.toml /tmp/dfo-docs-build/corpus/ ``` -The build clones every enabled source over HTTPS. Public sources need no credentials, and private ones use whatever GitHub credentials your git already has, so you need read access to them. +The build clones every enabled source over HTTPS without credentials, so none of this needs ONE's cloud access. Run `uv run pre-commit install` once, and pre-commit then runs the formatters and hygiene checks on each commit. + +## Pull requests + +Branch from `dev` and open the pull request into `dev`. A push to `dev` rebuilds the corpus the site reads, and a pull request runs the build without publishing. A draft pull request runs no CI, so mark it ready when you want the checks. CI runs ruff, ty, the tests on Python 3.13 and 3.14, the build, and the link check. A page change needs a description in its front matter and working links. Code changes need tests and must pass these checks. + +Commit messages follow Conventional Commits, such as `docs(docsync): describe the first run without state`. + +Maintainer internals are in [docs/development.md](docs/development.md). diff --git a/README.md b/README.md index 9703fdf..da191ca 100644 --- a/README.md +++ b/README.md @@ -1,121 +1,46 @@ -# dfo-docs +# DFO docs -dfo-docs fetches documentation from the DFO's source repositories, normalises it, and publishes a markdown corpus with a manifest that the DFO site renders. Each source repository keeps its own `docs/` folder and owns what is in it. +This repository holds the documentation of the [Development Finance Observatory](https://github.com/DevelopmentFinanceObservatory) (DFO), an initiative of [ONE Data](https://data.one.org) built on [Google Data Commons](https://datacommons.org). It also holds the build that gathers pages from the DFO's other repositories into one corpus for the DFO website. -To write a page or publish a repository's docs, read [CONTRIBUTING.md](CONTRIBUTING.md). +The published docs are at . Public launch is expected in late 2026. -## Adding a repository +This repository is under active development. Interfaces, schemas, and data may change without notice. -Add a row to `sources.yaml` with `name`, `repo`, `ref`, `docs_dir`, and the `path` the documentation should publish at. A row with `kind: external` points at a package that publishes its own documentation elsewhere, and carries a `url` instead of a `docs_dir`. - -A row with `kind: skill` publishes a repository of agent skills, such as dfo-skills. It needs no sidecar. The build walks `docs_dir` and publishes each directory that holds a `SKILL.md` as a skill page at `/`, with each file in its `references/` as a page at `//references/`. Directories whose names start with `_` stay out. The row must set `docs_dir`, `path`, and `title`, and it may not set `exclude`. It may set `internal_dir`, a second directory the build walks the same way. Skills under `docs_dir` are offered for install. Skills under `internal_dir` are published at the same `/` and are not offered. A directory name may appear under only one of the two. Until the repository has the `internal_dir` directory, the build warns and reads `docs_dir` alone. Its `hub.yaml` mount becomes one group titled by `title`. A `categories.yaml` in `docs_dir` lists skills under `sources` or under a domain in `domains`, and each category becomes a group inside the mount, titled in sentence case: Sources, and Domains with a group such as Sovereign debt inside it. Categories and the skills in them follow the file's order. Skills the file leaves out come first, in directory order, and without the file that is every skill. Each skill page opens with a header the build writes from the front matter: the skill's name and the first sentence of its `description`. For a skill under `docs_dir`, whatever its category, the build writes a zip to `corpus//.zip`, holding `/SKILL.md` and `/references/*.md` as committed, records the zip on the page's manifest entry as `bundle_path`, and adds a line to the header that links to the hub page on using skills in your own agent and to the zip. A skill under `internal_dir` gets no zip, no `bundle_path`, and no such line. - -If the repository needs to sit under a grouping heading, or nest alongside another package, add it to `hub.yaml`'s nav as a `mount` (or `external`) entry. A grouping heading such as `bblocks` or `ODA` lives only in `hub.yaml`, since no source repository declares it. - -A private repository in the ONEcampaign organisation clones with a read-only token for the `dfo-reader` GitHub App. To admit one, install `dfo-reader` on the repository, then add its name to the `repositories:` input of the token step in `.github/workflows/build.yml` and to `READER_REPOSITORIES` in `.github/workflows/registry-health.yml`. Both workflows read the App's client id from the `READER_APP_CLIENT_ID` repository variable and its private key from the `READER_APP_PRIVATE_KEY` repository secret, and fail if either is missing. - -To register a repository without building it yet, set `enabled: false` and a `reason`. A disabled row still loads and is reported by name. Turning it on means enabling the row and adding its `hub.yaml` mount, since a mount whose source is disabled fails the build. - -To take most of a repository but hold some pages back, list glob patterns under `exclude`. They match the docs-root-relative path, so `reference/*` keeps a whole section out and `changelog.md` keeps one page out. Use it for pages that cannot render outside their own site, such as an mkdocstrings reference section. A pattern matching no nav entry fails the build, so a pattern left behind by an upstream rename is reported rather than quietly letting those pages in. - -## Running the build locally - -``` -uv sync -uv run python scripts/build.py -``` - -The build clones every enabled repository over HTTPS, normalises what it finds, and writes `corpus/` and `manifest.json`. Public repositories need no credentials. The private ones clone with whatever GitHub credentials your git already uses, so a local build needs read access to them. - -`corpus/` and `manifest.json` are generated output that lives on the `corpus` branch. A push to `main` runs the same build in CI and commits the result there, and that branch is what the DFO site pins and reads. - -## Editing in Google Docs - -`dfo-docsync` publishes the pages of the sources listed in `docsync.yaml` as Google Docs in the "DFO docs" folder of the ONE Data shared drive, so editors who do not use GitHub can revise them. It tracks the pages in a Sheet and turns the edits made in the Docs into pull requests, one for each source repository. The script splices the words an editor changed into the source file one word at a time, so untouched text keeps its exact form, and keeps the structure, code blocks, and tabs as they are. Editors read the guide in the Sheet's "How to edit" tab, which `init` writes from `src/dfo_docs/docsync/how_to_edit.md`. - -``` -uv run dfo-docsync init -uv run dfo-docsync push -uv run dfo-docsync pull -``` - -- `init` creates the tracking Sheet in the folder. Run it once. -- `push` creates a Doc for each page, or replaces it with the page's current source, and updates the Sheet. It refuses to run when the state store is empty but Docs exist under the folder (`StatelessDocsError`). Restore the state, or pass `--adopt` to adopt the Docs that match their source. -- `pull` reads every Doc whose Status is "Ready for review" and opens a new pull request for each source repository, on its own branch `docsync/edits-`. docsync never pushes to that branch or edits the pull request again, so reviewers may edit it or commit to it. A page is in at most one open pull request. `pull` skips a page while its pull request is open, and fails a Ready page that an open docsync pull request carries but its state does not name, with the message "merge or close that PR, or revert the file on its branch, before sending this page". Edits made in the Doc while the pull request is open go in the next one. -- Both `push` and `pull` first settle each page's pull request. A merged pull request settles once the merge is in the branch the page's source is read from, and until then the page waits. The page then shows "Merged", or "Editing" when the Doc holds later edits, and the next `push` refreshes the Doc from GitHub. A pull request that is closed, or merged without the page, returns the page to "Editing" with the reason in the "Not applied" column. A page set to "Discard my edits" stays on it and is discarded. A "Ready for review" page whose Doc holds nothing new to send becomes "Merged" when its pull request has just merged and "Editing" otherwise, with "Nothing to send" in "Not applied". -- To drop one page from a multi-page pull request, revert that file on the pull request branch before merging. The page returns to "Editing". -- "Discard my edits" is a Status an editor sets. The next `push` resets the page's Doc from GitHub, even with open comments, and sets the page to "Editing". It refuses while the Doc has open suggestions, and waits while the page's pull request is open. When that pull request merged into a branch the page is not read from, the discard goes ahead. The page's state records the discard until a `push` reads another Status, so when the Sheet write after a discard fails, the next `push` only sets the page to "Editing" and keeps what the editor typed since. -- A Doc that was trashed or deleted is recreated on the next `push`, unless the page's pull request is open. - -`--page ` limits `push` or `pull` to one page and can be repeated. A page id is the manifest's, such as `hub/build/share`. `pull --dry-run ` writes the edited files and pull request bodies under `` and changes nothing in git, the Sheet, or the state. `docsync.yaml` sets `pr_author`, the author `gh` lists open docsync pull requests by; it is `app/dfo-bot` for the pull requests CI opens. For a local run that opens pull requests under your own login, pass `--pr-author @me` to `push` and `pull`. - -Hub pages (`hub/...`) run only from a checkout of dfo-docs `main` that is at `origin/main`. - -A maintainer runs a sync from a `main` checkout at `origin/main`, signed in to Google with a one.org account. `.github/workflows/docsync.yml` (daily `pull` then `push`, or one command on dispatch) is disabled: a Markdown upload made by its service account loses code blocks, and every page with one then fails to align. Its job runs only in the ONEcampaign organisation, so a copy of the repository elsewhere never writes to Google Docs. The token lasts about an hour. Without it, the command uses Application Default Credentials. +## What's inside +```text +content/hub/ hub pages, written here: Get started, Build, Data, Skills, About +roadmap/ the DFO roadmap, one YAML file per track +hub.yaml the site nav: hub pages, grouping headings, and source mounts +sources.yaml the registry of repositories whose docs the build fetches +docsync.yaml the sources published as Google Docs for editors who do not use GitHub +src/dfo_docs/ the build and the docsync tool +scripts/ build.py and check_nav.py +tests/ the test suite ``` -git switch main && git pull -gcloud auth login --enable-gdrive-access -export DOCSYNC_ACCESS_TOKEN=$(gcloud auth print-access-token) -uv run dfo-docsync pull --pr-author @me -uv run dfo-docsync push --pr-author @me -``` - -`pull` opens a pull request for each source repository with pages set to Ready for review. `push` refreshes each Doc whose page changed in its source repository, the pages of merged docsync pull requests included, and adds a Doc for each new page. Run the sequence again after docsync pull requests merge. A merge to dfo-docs `main` rebuilds the corpus the site reads at once. A merge in another source repository reaches it at the 06:00 UTC build, or sooner with `gh workflow run build.yml --repo ONEcampaign/dfo-docs`. - -Each page's state (Doc, source commit, baseline export, pull request) is a `state/.json` file on the orphan branch `docsync-state`, with one commit for each run. These guards hold a page back, and the run reports the reason and carries on: - -- `push` leaves a Doc alone when it has edits that have not reached GitHub ("edited in Docs; set Status to Ready for review to send"), open comments, or open suggestions, because replacing a Doc discards or detaches them. "Discard my edits" overrides the first two. It checks the Doc's revision again just before replacing it and holds the Doc if the revision changed. -- `pull` skips a Doc with open suggestions. A skipped or failed page does not stop the other pages of its repository from going in the pull request. -- `pull` leaves out an edit the source cannot take, such as a changed code block or code block language, a new table row, a folded description, or a deleted label line, and lists it in the pull request body. When that leaves nothing to send for the page (the edits are all reported, or already in GitHub), the reasons go to the Sheet's "Not applied" column, the page returns to "Editing", and the next `push` refreshes the Doc from GitHub. The script never writes the Notes column. -- A line break an editor adds inside a paragraph starts a new paragraph in the source. A new list item goes after the last block of the item above it, its note included, at that item's marker, and a new list opens after a top-level paragraph, so adding items and links to a "Next" list applies. A bullet or number added to or removed from a paragraph applies only its text, and the report says so. -- Author notes follow the Doc. An Author note whose label an editor removed or changed becomes page text and so appears on the site. A paragraph that gains the label becomes an Author note and leaves the site. The pull request reports each change ("an Author note became page text; check it belongs on the site", "a paragraph became an Author note and leaves the site"), and the reviewer decides. When a note or the Description is changed, added or deleted and page text next to it also changes, with no unchanged paragraph between them, the edit applies and the pull request flags it once ("an Author note and the page text beside it changed together; check no text moved between them", and the same with "the Description"), unless one of the two flags above already covers it. A report of an applied edit, an anchor change or an Author note flag, appears in the pull request that carries it and not in the next one. A report of an edit left out appears each time. An Author note emptied down to its label is deleted, and a new paragraph that starts with the label becomes a note. -- A label belongs at the start of its own Doc paragraph. A label typed mid-paragraph is refused. A note joined onto the paragraph before or after it, or Enter pressed in the middle of a note, is refused, or applied with that flag when the page then matches the Doc. A refused region is left unchanged, and the report quotes the editor's text of it. Text crosses a note's edge when a note's words join another paragraph or split off it; a paragraph typed with Enter next to a note is a new paragraph and applies. A paragraph typed with Enter next to a note in a list item is refused and held when the source gives it no list item to join. A refusal also covers a label that is unbolded, a Description or Tab label removed or changed with its text kept, a Note, Tip or Warning label deleted together with a change to its text, a tab line deleted together with a change to the text beside it, a note added or labelled inside a plain `>` quote, a new note that holds `-->`, a heading or list item given the note label, an added paragraph the source has no place for, and an edit beside repeated text where docsync cannot tell which copy changed. Changing Note to Tip or Warning, deleting a callout's label with its text unchanged, and deleting a whole tab line or Description line are reported as not applied and not refused, so the page records its export as applied and the next `push` refreshes the Doc. A refusal is reported as the message, then the Doc's text in a fence, byte for byte. A note inside a list item whose label is removed becomes text in that item, flagged. A line typed above the Description is page text and the Description stays front matter. -- A pull that sends nothing for a page and has a refusal leaves the Doc alone. The page returns to "Editing", the Not applied column holds each refusal's message and quoted Doc text and says to rework the edit or set "Discard my edits", and the page's state records no new applied export, so `push` finds the Doc edited and holds it. Every other report kind records the export as applied, and the next `push` refreshes the Doc from GitHub. When the pull sends a pull request for the page, the refusals go in its body, and the refused text leaves the Doc when that pull request merges and `push` refreshes it. The Not applied cell holds at most 45,000 characters and the pull request body 60,000. Refusal quotes are cut to fit. A cut quote ends with the line "(Shortened here. The full text was in the Doc when this pull ran.)". -- `pull` checks that the merged file reads in the source dialect, as `push` reads it. When it does not, nothing is sent for the page, the run exits 1, and the page is held like a refusal: the Not applied column says "the edits would leave a file docsync cannot read: ; nothing sent" and the Doc keeps the editor's text. -- When upstream and an editor changed different words of one paragraph, both changes stay. When they changed the same words, the pull request applies the editor's text and shows the upstream text beside it. - -The exit code is 1 when a page fails. It is 2 when the run stops: no credentials, folder, or Sheet; a git, `gh`, or Google error outside a page; a Page id listed twice, or a missing Page id or Status column, in the Sheet; or a state save that fails. A failed save writes the changed page states to `docsync-unsaved-states-.json` in the working directory. In CI the workflow uploads it as the `docsync-unsaved-states` artifact. Run `uv run dfo-docsync restore ` to save those states over the stored ones before the next run. `--state-dir ` restores into a local state directory. `restore` refuses when the state has moved since the failed run, because a later run may have saved newer states. Pass `--page ` (repeatable) to restore chosen pages, or `--force` to overwrite every page in the file. - -To add a source, append its name to `sources` in `docsync.yaml`. It must be an enabled `docs` or `skill` row in `sources.yaml` that `hub.yaml` mounts. For another repository, install `dfo-bot` on it and add its name to the `repositories:` input of the "Mint a token for dfo-bot" step in `.github/workflows/docsync.yml`. Then run `push`. [CONTRIBUTING.md](CONTRIBUTING.md) covers writing the pages. -To try a change away from the real folder, make a scratch folder in the shared drive, share it with your account, and point each command at it. A Doc pushed under one folder belongs to that folder alone. +Every other page lives in the `docs/` folder of its own repository, such as [apis](https://github.com/DevelopmentFinanceObservatory/apis), [ai](https://github.com/DevelopmentFinanceObservatory/ai), and [skills](https://github.com/DevelopmentFinanceObservatory/skills), or in a package repository under [ONEcampaign](https://github.com/ONEcampaign), and `sources.yaml` points the build at it. To change one of those pages, open a pull request in the repository that holds it. The [org profile](https://github.com/DevelopmentFinanceObservatory) links every DFO repository. -``` -uv run dfo-docsync init --folder -uv run dfo-docsync push --folder --state-dir ./scratch-state -uv run dfo-docsync pull --folder --state-dir ./scratch-state --dry-run ./scratch-out -``` - -Without `--state-dir`, `push` and `pull` read and commit to the `docsync-state` branch on `origin`, so pass it whenever you experiment. +## Getting started -## Working on this repo +Everything here runs without access to ONE's Google Cloud project, because the build clones public repositories anonymously. You need [uv](https://docs.astral.sh/uv/) and Python 3.13. +```bash +uv sync +uv run python scripts/build.py --out /tmp/dfo-docs-build ``` -uv run ruff check . -uv run ruff format . -uv run ty check src/dfo_docs -uv run pytest -``` - -Run `uv run pre-commit install` once. Pre-commit then blocks a commit that fails these checks. - -## When the build fails - -The build checks every enabled repository before it exits, so the log names every failure in one run. - -**A source repository cannot be cloned.** The repository was renamed, made private, or deleted, or it is private and `dfo-reader` cannot read it. Fix the `repo` field in `sources.yaml`, admit the private repository as described under [Adding a repository](#adding-a-repository), or disable the row. -**An empty docs directory.** The source repository renamed its `docs_dir`. Update `docs_dir` in `sources.yaml` to match. +The build writes `corpus/` and `manifest.json` under the output directory. The Google Docs sync (`dfo-docsync`) needs ONE credentials and is for maintainers. -**A leaked `::: ` directive.** A repository is publishing mkdocstrings-generated reference through the normal build path. Exclude the pages that carry it, or move the whole repository to the self-published tier with `kind: external`. +## Documentation -**A sidecar declaring no nav.** The source would publish nothing. In a `zensical.toml` this is usually a `nav` array written after a `[project.]` header, which TOML scopes into that table instead of into `[project]`. Move the array above every sub-table header. +- [CONTRIBUTING.md](CONTRIBUTING.md) explains how to edit a hub page, page modes, style, the link rules, and how a repository publishes its docs. +- [docs/development.md](docs/development.md) holds maintainer notes: the source registry, build failures, and the Google Docs sync. -**A skill directory without a `SKILL.md`, or with front matter that does not parse.** The build names the directory or file. Add the `SKILL.md`, fix its front matter so it carries a `name` and a `description`, or move the directory out of the row's `docs_dir` and `internal_dir` if it is not a skill. +## Contributing -**A skill directory of the same name under both `docs_dir` and `internal_dir`.** The build names both. A skill's directory name is its page address, so keep one. +Issues and pull requests are welcome. See [CONTRIBUTING.md](CONTRIBUTING.md). -**A `categories.yaml` that does not parse, or lists a skill twice.** The build names the file. Give `sources` a list of skill directories and `domains` a mapping from each domain to such a list, and list each skill once. +## Licence -**An exclude pattern matching nothing.** A page an `exclude` row named was renamed or removed upstream. Update the pattern in `sources.yaml`, or drop it if the page is gone for good. +MIT. See [LICENSE](LICENSE). diff --git a/content/hub/data/where-the-numbers-come-from.md b/content/hub/data/where-the-numbers-come-from.md index 26ff88d..c382e77 100644 --- a/content/hub/data/where-the-numbers-come-from.md +++ b/content/hub/data/where-the-numbers-come-from.md @@ -75,7 +75,7 @@ The rebuilt files are then loaded into the graph. A [live link](../build/share.m The ontology is public. Every concept the graph uses, from a purpose code to a type of finance, is an item in the [development finance Wikibase](https://developmentfinance.wikibase.cloud/wiki/Main_Page). Each item gives the concept's definition, its place in the hierarchy and the source code it represents. [The development finance ontology](ontology.md) explains how the items are organised. -The packages the pipelines use to fetch sources are open source. The OECD's DAC statistics, including the CRS, come through the [ODA reader](../../oda_reader/index.md), and you can run the same `bulk_download_crs` call yourself. The IMF's World Economic Outlook comes through the [IMF reader](../../imf-reader/index.md). [DCP tools](../../dcp-tools/index.md) writes the files the graph loads. +The pipelines that produce these numbers are open source. You can read their code, transformation notes, and tests in [DevelopmentFinanceObservatory/pipelines](https://github.com/DevelopmentFinanceObservatory/pipelines). The packages they use are open source too. The OECD's DAC statistics, including the CRS, come through the [ODA reader](../../oda_reader/index.md), and you can run the same `bulk_download_crs` call yourself. The IMF's World Economic Outlook comes through the [IMF reader](../../imf-reader/index.md). [DCP tools](../../dcp-tools/index.md) writes the files the graph loads. ## Next diff --git a/content/hub/get-started/use-studio.md b/content/hub/get-started/use-studio.md index 8caa60d..3a275a4 100644 --- a/content/hub/get-started/use-studio.md +++ b/content/hub/get-started/use-studio.md @@ -10,7 +10,7 @@ This example asks for an overview of health spending in Kenya and produces a pin The walkthrough takes about ten minutes and uses two of the five queries available without an account. Sign in first if you want to save your canvas or ask more than five queries. - + ## Step 1: Ask a question diff --git a/content/hub/skills/use-in-your-agent.md b/content/hub/skills/use-in-your-agent.md index 6f0c45a..df76531 100644 --- a/content/hub/skills/use-in-your-agent.md +++ b/content/hub/skills/use-in-your-agent.md @@ -44,7 +44,7 @@ These steps cover Claude on the web, the Claude desktop app, and Cowork. To install one skill on its own, download its zip, then open Customize, then Skills, click +, and choose "Upload a skill". Uploaded skills need code execution switched on in Claude's settings. - + ::tab{label="ChatGPT"} @@ -56,7 +56,7 @@ Whether you can upload a skill depends on your ChatGPT plan, and in a ChatGPT wo 1. Wait for ChatGPT to finish scanning the skill. 1. Start a new chat. - + @@ -75,7 +75,7 @@ Gemini takes skills on a personal Google Account, for people aged 18 or over, wi - + ::tab{label="Claude Code"} @@ -114,7 +114,7 @@ An agent that reads the Agent Skills format loads a skill from a folder on your 1. Start a new session. - + Check your agent's documentation if it reads a different folder. Keep the unzipped folder whole, because `SKILL.md` points to the files in `references/` by relative path. @@ -162,7 +162,7 @@ Download the zip again from the skill's page. In Claude, ChatGPT, or Gemini, del ::: - + ## Next diff --git a/content/hub/skills/what-a-skill-is.md b/content/hub/skills/what-a-skill-is.md index 63907af..1fda52e 100644 --- a/content/hub/skills/what-a-skill-is.md +++ b/content/hub/skills/what-a-skill-is.md @@ -32,7 +32,7 @@ oda-measures/ `SKILL.md` holds YAML front matter with three fields. `name` matches the directory name. `description` says what the skill covers and when to load it, and it's the text an agent matches a question against. `license` names the licence, `CC-BY-4.0` for every DFO skill. Below the front matter, the body lists the reference files and says when to read each one. -`references/` holds the long material: definitions, method notes, defaults. The dfo-skills repository adds two rules of its own, checked in CI by `scripts/validate_skills.py`: +`references/` holds the long material: definitions, method notes, defaults. The `skills` repository adds two rules of its own, checked in CI by `scripts/validate_skills.py`: - Pointers stay one level deep. `SKILL.md` names every reference file, and no reference file sends the agent on to another. - Keep each paragraph and list item on one line. @@ -59,19 +59,19 @@ Some skills point to others. [`oda-dataset-selection`](../../dfo-skills/oda-data ## Where skills live in the DFO -The dfo-skills repository holds each skill in its own directory, and each `description` says what its skill covers. `skills/` holds the skills offered for install. `internal/skills/` holds the skills the DFO's own assistant uses and that are not offered for install, such as `dfo-faq`. Each skill also has a page in this section, which shows its `SKILL.md` with its reference files as pages beneath it. Sources lists the skills that each explain one dataset, such as [`oda-dac1`](../../dfo-skills/oda-dac1/SKILL.md). Domains lists the skills that explain a subject across datasets, grouped by domain, such as Sovereign debt. +The `skills` repository holds each skill in its own directory, and each `description` says what its skill covers. `skills/` holds the skills offered for install. `internal/skills/` holds the skills the DFO's own assistant uses and that are not offered for install, such as `dfo-faq`. Each skill also has a page in this section, which shows its `SKILL.md` with its reference files as pages beneath it. Sources lists the skills that each explain one dataset, such as [`oda-dac1`](../../dfo-skills/oda-dac1/SKILL.md). Domains lists the skills that explain a subject across datasets, grouped by domain, such as Sovereign debt. -dfo-pipelines checks dfo-skills for new commits and publishes each skill to the [knowledge graph](../data/knowledge-graph.md) as an `AgentSkill` node. Each reference file is an `AgentSkillReference` node. +The `pipelines` repository checks the `skills` repository for new commits and publishes each skill to the [knowledge graph](../data/knowledge-graph.md) as an `AgentSkill` node. Each reference file is an `AgentSkillReference` node. Install the skills under `skills/` in your own assistant to use them. ## How the DFO assistant uses skills -The DFO assistant is an agent too, and it reads the same skills from the knowledge graph. dfo-apis serves them through its Skills API, and the DFO MCP server gives the assistant two tools over it: `get_skill` loads a skill's `SKILL.md` and lists its reference files, and `get_skill_file` reads one of those files. +The DFO assistant is an agent too, and it reads the same skills from the knowledge graph. The `apis` repository serves them through its Skills API, and the DFO MCP server gives the assistant two tools over it: `get_skill` loads a skill's `SKILL.md` and lists its reference files, and `get_skill_file` reads one of those files. The descriptions the assistant matches against come from its data search. Skills are attached to datasets, and the search result names up to five skills attached to the datasets it found. A question about DFO itself goes straight to [`dfo-faq`](../../dfo-skills/dfo-faq/SKILL.md). -An edit to a skill reaches the assistant after dfo-pipelines publishes it to the graph and the Skills API refreshes its index, which it does every 15 minutes. +An edit to a skill reaches the assistant after `pipelines` publishes it to the graph and the Skills API refreshes its index, which it does every 15 minutes. ## Next diff --git a/docs/development.md b/docs/development.md new file mode 100644 index 0000000..b65c54e --- /dev/null +++ b/docs/development.md @@ -0,0 +1,128 @@ +# Maintainer notes + +Detail for the ONE Data maintainers of this repository, covering the source registry, the build and its branches, the Google Docs sync, and build failures. Contributors writing pages need only [CONTRIBUTING.md](../CONTRIBUTING.md). + +## Adding a repository + +Add a row to `sources.yaml` with `name`, `repo`, `ref`, `docs_dir`, and the `path` the documentation should publish at. A row with `kind: external` points at a package that publishes its own documentation elsewhere, and carries a `url` instead of a `docs_dir`. + +A row with `kind: skill` publishes a repository of agent skills, such as `skills`. It needs no sidecar. The build walks `docs_dir` and publishes each directory that holds a `SKILL.md` as a skill page at `/`, with each file in its `references/` as a page at `//references/`. Directories whose names start with `_` stay out. The row must set `docs_dir`, `path`, and `title`, and it may not set `exclude`. It may set `internal_dir`, a second directory the build walks the same way. Skills under `docs_dir` are offered for install. Skills under `internal_dir` are published at the same `/` and are not offered. A directory name may appear under only one of the two. Until the repository has the `internal_dir` directory, the build warns and reads `docs_dir` alone. Its `hub.yaml` mount becomes one group titled by `title`. A `categories.yaml` in `docs_dir` lists skills under `sources` or under a domain in `domains`, and each category becomes a group inside the mount, titled in sentence case: Sources, and Domains with a group such as Sovereign debt inside it. Categories and the skills in them follow the file's order. Skills the file leaves out come first, in directory order, and without the file that is every skill. Each skill page opens with a header the build writes from the front matter: the skill's name and the first sentence of its `description`. For a skill under `docs_dir`, whatever its category, the build writes a zip to `corpus//.zip`, holding `/SKILL.md` and `/references/*.md` as committed, records the zip on the page's manifest entry as `bundle_path`, and adds a line to the header that links to the hub page on using skills in your own agent and to the zip. A skill under `internal_dir` gets no zip, no `bundle_path`, and no such line. + +If the repository needs to sit under a grouping heading, or nest alongside another package, add it to `hub.yaml`'s nav as a `mount` (or `external`) entry. A grouping heading such as `bblocks` or `ODA` lives only in `hub.yaml`, since no source repository declares it. + +Every source repository is public, and the build and the registry-health workflow clone and query them without credentials. A private repository cannot be registered. + +To register a repository without building it yet, set `enabled: false` and a `reason`. A disabled row still loads and is reported by name. Turning it on means enabling the row and adding its `hub.yaml` mount, since a mount whose source is disabled fails the build. + +To take most of a repository but hold some pages back, list glob patterns under `exclude`. They match the docs-root-relative path, so `reference/*` keeps a whole section out and `changelog.md` keeps one page out. Use it for pages that cannot render outside their own site, such as an mkdocstrings reference section. A pattern matching no nav entry fails the build, so a pattern left behind by an upstream rename is reported. + +## Building and publishing + +```bash +uv sync +uv run python scripts/build.py +``` + +The build clones every enabled repository over HTTPS without credentials, normalises what it finds, and writes `corpus/` and `manifest.json`. + +`corpus/` and `manifest.json` are generated output that lives on the `corpus` branch. A push to `dev` runs the same build in CI and commits the result there, and that branch is what the DFO site pins and reads. A pull request into `dev` or `main` runs the build and publishes nothing. + +The build also runs in CI every day at 06:00 UTC. + +## Editing in Google Docs + +`dfo-docsync` publishes the pages of the sources listed in `docsync.yaml` as Google Docs in the "DFO docs" folder of the ONE Data shared drive, so editors who do not use GitHub can revise them. It tracks the pages in a Sheet and turns the edits made in the Docs into pull requests, one for each source repository. The script splices the words an editor changed into the source file one word at a time, so untouched text keeps its exact form, and keeps the structure, code blocks, and tabs as they are. Editors read the guide in the Sheet's "How to edit" tab, which `init` writes from `src/dfo_docs/docsync/how_to_edit.md`. + +```bash +uv run dfo-docsync init +uv run dfo-docsync push +uv run dfo-docsync pull +``` + +- `init` creates the tracking Sheet in the folder. Run it once. +- `push` creates a Doc for each page, or replaces it with the page's current source, and updates the Sheet. It refuses to run when the state store is empty but Docs exist under the folder (`StatelessDocsError`). Restore the state, or pass `--adopt` to adopt the Docs that match their source. +- `pull` reads every Doc whose Status is "Ready for review" and opens a new pull request for each source repository, on its own branch `docsync/edits-`. docsync never pushes to that branch or edits the pull request again, so reviewers may edit it or commit to it. A page is in at most one open pull request. `pull` skips a page while its pull request is open, and fails a Ready page that an open docsync pull request carries but its state does not name, with the message "merge or close that PR, or revert the file on its branch, before sending this page". Edits made in the Doc while the pull request is open go in the next one. +- Both `push` and `pull` first settle each page's pull request. A merged pull request settles once the merge is in the branch the page's source is read from, and until then the page waits. The page then shows "Merged", or "Editing" when the Doc holds later edits, and the next `push` refreshes the Doc from GitHub. A pull request that is closed, or merged without the page, returns the page to "Editing" with the reason in the "Not applied" column. A page set to "Discard my edits" stays on it and is discarded. A "Ready for review" page whose Doc holds nothing new to send becomes "Merged" when its pull request has just merged and "Editing" otherwise, with "Nothing to send" in "Not applied". +- To drop one page from a multi-page pull request, revert that file on the pull request branch before merging. The page returns to "Editing". +- "Discard my edits" is a Status an editor sets. The next `push` resets the page's Doc from GitHub, even with open comments, and sets the page to "Editing". It refuses while the Doc has open suggestions, and waits while the page's pull request is open. When that pull request merged into a branch the page is not read from, the discard goes ahead. The page's state records the discard until a `push` reads another Status, so when the Sheet write after a discard fails, the next `push` only sets the page to "Editing" and keeps what the editor typed since. +- A Doc that was trashed or deleted is recreated on the next `push`, unless the page's pull request is open. + +`--page ` limits `push` or `pull` to one page and can be repeated. A page id is the manifest's, such as `hub/build/share`. `pull --dry-run ` writes the edited files and pull request bodies under `` and changes nothing in git, the Sheet, or the state. `docsync.yaml` sets `pr_author`, the author `gh` lists open docsync pull requests by; it is `app/dfo-docsync` for the pull requests CI opens. For a local run that opens pull requests under your own login, pass `--pr-author @me` to `push` and `pull`. + +Hub pages (`hub/...`) run only from a checkout of this repository's `dev` that is at `origin/dev`. + +A maintainer runs a sync from a `dev` checkout at `origin/dev`, signed in to Google with a one.org account. `.github/workflows/docsync.yml` runs one command on dispatch, with no schedule. Maintainers do not use it, because a Markdown upload made by its service account loses code blocks, and every page with one then fails to align. Its job runs only on `dev`, in the `docsync` environment, which holds the `dfo-docsync` App key and the Google workload identity settings (see the workflow's header). To run locally, pass a Google access token, which lasts about an hour. Without one, the command uses Application Default Credentials. + +```bash +git switch dev && git pull +gcloud auth login --enable-gdrive-access +export DOCSYNC_ACCESS_TOKEN=$(gcloud auth print-access-token) +uv run dfo-docsync pull --pr-author @me +uv run dfo-docsync push --pr-author @me +``` + +`pull` opens a pull request for each source repository with pages set to Ready for review. `push` refreshes each Doc whose page changed in its source repository, the pages of merged docsync pull requests included, and adds a Doc for each new page. Run the sequence again after docsync pull requests merge. A merge to this repository's `dev` rebuilds the corpus the site reads at once. A merge in another source repository reaches it at the 06:00 UTC build, or sooner with `gh workflow run build.yml --repo DevelopmentFinanceObservatory/docs --ref dev`. + +When the `docsync-state` branch is empty but Docs already exist under the folder, `push` stops with `StatelessDocsError`. Run it once as `uv run dfo-docsync push --adopt --pr-author @me` from a `dev` checkout at `origin/dev`. `--adopt` takes over each Doc that carries a page id and whose settled export aligns with the source, and records the current `dev` commit as its baseline, so it needs no earlier commit. A Doc that differs from the source holds an editor's unsent work, so its page fails and the Doc stays. Resolve those by hand, then rerun `push --adopt`. Later runs need no flag. + +Each page's state (Doc, source commit, baseline export, pull request) is a `state/.json` file on the orphan branch `docsync-state`, with one commit for each run. These guards hold a page back, and the run reports the reason and carries on: + +- `push` leaves a Doc alone when it has edits that have not reached GitHub ("edited in Docs; set Status to Ready for review to send"), open comments, or open suggestions, because replacing a Doc discards or detaches them. "Discard my edits" overrides the first two. It checks the Doc's revision again just before replacing it and holds the Doc if the revision changed. +- `pull` skips a Doc with open suggestions. A skipped or failed page does not stop the other pages of its repository from going in the pull request. +- `pull` leaves out an edit the source cannot take, such as a changed code block or code block language, a new table row, a folded description, or a deleted label line, and lists it in the pull request body. When that leaves nothing to send for the page (the edits are all reported, or already in GitHub), the reasons go to the Sheet's "Not applied" column, the page returns to "Editing", and the next `push` refreshes the Doc from GitHub. The script never writes the Notes column. +- A line break an editor adds inside a paragraph starts a new paragraph in the source. A new list item goes after the last block of the item above it, its note included, at that item's marker, and a new list opens after a top-level paragraph, so adding items and links to a "Next" list applies. A bullet or number added to or removed from a paragraph applies only its text, and the report says so. +- Author notes follow the Doc. An Author note whose label an editor removed or changed becomes page text and so appears on the site. A paragraph that gains the label becomes an Author note and leaves the site. The pull request reports each change ("an Author note became page text; check it belongs on the site", "a paragraph became an Author note and leaves the site"), and the reviewer decides. When a note or the Description is changed, added or deleted and page text next to it also changes, with no unchanged paragraph between them, the edit applies and the pull request flags it once ("an Author note and the page text beside it changed together; check no text moved between them", and the same with "the Description"), unless one of the two flags above already covers it. A report of an applied edit, an anchor change or an Author note flag, appears in the pull request that carries it and not in the next one. A report of an edit left out appears each time. An Author note emptied down to its label is deleted, and a new paragraph that starts with the label becomes a note. +- A label belongs at the start of its own Doc paragraph. A label typed mid-paragraph is refused. A note joined onto the paragraph before or after it, or Enter pressed in the middle of a note, is refused, or applied with that flag when the page then matches the Doc. A refused region is left unchanged, and the report quotes the editor's text of it. Text crosses a note's edge when a note's words join another paragraph or split off it; a paragraph typed with Enter next to a note is a new paragraph and applies. A paragraph typed with Enter next to a note in a list item is refused and held when the source gives it no list item to join. A refusal also covers a label that is unbolded, a Description or Tab label removed or changed with its text kept, a Note, Tip or Warning label deleted together with a change to its text, a tab line deleted together with a change to the text beside it, a note added or labelled inside a plain `>` quote, a new note that holds `-->`, a heading or list item given the note label, an added paragraph the source has no place for, and an edit beside repeated text where docsync cannot tell which copy changed. Changing Note to Tip or Warning, deleting a callout's label with its text unchanged, and deleting a whole tab line or Description line are reported as not applied and not refused, so the page records its export as applied and the next `push` refreshes the Doc. A refusal is reported as the message, then the Doc's text in a fence, byte for byte. A note inside a list item whose label is removed becomes text in that item, flagged. A line typed above the Description is page text and the Description stays front matter. +- A pull that sends nothing for a page and has a refusal leaves the Doc alone. The page returns to "Editing", the Not applied column holds each refusal's message and quoted Doc text and says to rework the edit or set "Discard my edits", and the page's state records no new applied export, so `push` finds the Doc edited and holds it. Every other report kind records the export as applied, and the next `push` refreshes the Doc from GitHub. When the pull sends a pull request for the page, the refusals go in its body, and the refused text leaves the Doc when that pull request merges and `push` refreshes it. The Not applied cell holds at most 45,000 characters and the pull request body 60,000. Refusal quotes are cut to fit. A cut quote ends with the line "(Shortened here. The full text was in the Doc when this pull ran.)". +- `pull` checks that the merged file reads in the source dialect, as `push` reads it. When it does not, nothing is sent for the page, the run exits 1, and the page is held like a refusal: the Not applied column says "the edits would leave a file docsync cannot read: ; nothing sent" and the Doc keeps the editor's text. +- When upstream and an editor changed different words of one paragraph, both changes stay. When they changed the same words, the pull request applies the editor's text and shows the upstream text beside it. + +The exit code is 1 when a page fails. It is 2 when the run stops: no credentials, folder, or Sheet; a git, `gh`, or Google error outside a page; a Page id listed twice, or a missing Page id or Status column, in the Sheet; or a state save that fails. A failed save writes the changed page states to `docsync-unsaved-states-.json` in the working directory. In CI the workflow uploads it as the `docsync-unsaved-states` artifact. Run `uv run dfo-docsync restore ` to save those states over the stored ones before the next run. `--state-dir ` restores into a local state directory. `restore` refuses when the state has moved since the failed run, because a later run may have saved newer states. Pass `--page ` (repeatable) to restore chosen pages, or `--force` to overwrite every page in the file. + +To add a source, append its name to `sources` in `docsync.yaml`. It must be an enabled `docs` or `skill` row in `sources.yaml` that `hub.yaml` mounts. For another repository, install the `dfo-docsync` GitHub App on it and add its name to the `repositories:` input of the "Mint a token for dfo-docsync" step in `.github/workflows/docsync.yml`. Then run `push`. [CONTRIBUTING.md](../CONTRIBUTING.md) covers writing the pages. + +To try a change away from the real folder, make a scratch folder in the shared drive, share it with your account, and point each command at it. A Doc pushed under one folder belongs to that folder alone. + +```bash +uv run dfo-docsync init --folder +uv run dfo-docsync push --folder --state-dir ./scratch-state +uv run dfo-docsync pull --folder --state-dir ./scratch-state --dry-run ./scratch-out +``` + +Without `--state-dir`, `push` and `pull` read and commit to the `docsync-state` branch on `origin`, so pass it whenever you experiment. + +## When the build fails + +The build checks every enabled repository before it exits, so the log names every failure in one run. + +### A source repository cannot be cloned + +The repository was renamed, made private, or deleted. Fix the `repo` field in `sources.yaml`, or disable the row. + +### An empty docs directory + +The source repository renamed its `docs_dir`. Update `docs_dir` in `sources.yaml` to match. + +### A leaked `::: ` directive + +A repository is publishing mkdocstrings-generated reference through the normal build path. Exclude the pages that carry it, or move the whole repository to the self-published tier with `kind: external`. + +### A sidecar declaring no nav + +The source would publish nothing. In a `zensical.toml` this is usually a `nav` array written after a `[project.
]` header, which TOML scopes into that table instead of into `[project]`. Move the array above every sub-table header. + +### A skill directory without a `SKILL.md`, or with front matter that does not parse + +The build names the directory or file. Add the `SKILL.md`, fix its front matter so it carries a `name` and a `description`, or move the directory out of the row's `docs_dir` and `internal_dir` if it is not a skill. + +### A skill directory of the same name under both `docs_dir` and `internal_dir` + +The build names both. A skill's directory name is its page address, so keep one. + +### A `categories.yaml` that does not parse, or lists a skill twice + +The build names the file. Give `sources` a list of skill directories and `domains` a mapping from each domain to such a list, and list each skill once. + +### An exclude pattern matching nothing + +A page an `exclude` row named was renamed or removed upstream. Update the pattern in `sources.yaml`, or drop it if the page is gone for good. diff --git a/docsync.yaml b/docsync.yaml index ea8fc31..3f98f81 100644 --- a/docsync.yaml +++ b/docsync.yaml @@ -9,7 +9,7 @@ # in it, created by `dfo-docsync init` and found by its Drive property dfo_role=tracking-sheet. # # `pr_author` is the author docsync lists its open pull requests by, as gh's `--author` takes -# it. The value `app/dfo-bot` names the dfo-bot GitHub App that opens them in CI. A local run +# it. The value `app/dfo-docsync` names the dfo-docsync GitHub App that opens them in CI. A local run # that opens pull requests with your own gh login passes `--pr-author @me`. sources: @@ -20,4 +20,4 @@ sources: drive_folder_id: 1DL_BWEeo92oksJEr28xlXFD46xQxFQG1 -pr_author: app/dfo-bot +pr_author: app/dfo-docsync diff --git a/pyproject.toml b/pyproject.toml index bf255c1..b3314c5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ version = "0.1.0" description = "Fetches, normalises, and publishes documentation aggregated from DFO source repositories." authors = [ - { name = "ONE Campaign", email = "jorge.rivera@one.org" } + { name = "ONE Campaign", email = "data@one.org" }, ] readme = "README.md" diff --git a/sources.yaml b/sources.yaml index 0393e3f..5a52ee5 100644 --- a/sources.yaml +++ b/sources.yaml @@ -69,25 +69,26 @@ sources: exclude: - reference/* - # Private repositories, cloned with the dfo-reader GitHub App token the build workflow mints. + # The DFO's own repositories. Their integration branch is `dev`, which docsync opens its pull + # requests against. - name: dfo-apis - repo: ONEcampaign/dfo-apis + repo: DevelopmentFinanceObservatory/apis ref: dev docs_dir: docs/guide path: api title: API - name: dfo-ai - repo: ONEcampaign/dfo-ai + repo: DevelopmentFinanceObservatory/ai ref: dev docs_dir: docs/guide path: mcp title: MCP server - name: dfo-skills - repo: ONEcampaign/dfo-skills + repo: DevelopmentFinanceObservatory/skills kind: skill - ref: main + ref: dev docs_dir: skills internal_dir: internal/skills path: skills diff --git a/src/dfo_docs/docsync/cli.py b/src/dfo_docs/docsync/cli.py index be168d6..7f14a41 100644 --- a/src/dfo_docs/docsync/cli.py +++ b/src/dfo_docs/docsync/cli.py @@ -136,7 +136,7 @@ ROOT_PROPERTY = "dfo_root" PR_TITLE = "Apply edits from Google Docs" -HUB_BASE = "main" +HUB_BASE = "dev" HUB_REMOTE = "origin" DOC_URL = "https://docs.google.com/document/d/{doc_id}/edit" @@ -399,7 +399,7 @@ def _run(args: argparse.Namespace) -> int: sheet_id = find_sheet(session, drive_id=drive_id, folder_id=folder_id) if HUB in config.sources: - require_hub_at_main(Git(root=repo_root, env=command_env(github_token))) + require_hub_at_base(Git(root=repo_root, env=command_env(github_token))) pr_author = args.pr_author or config.pr_author ctx = Context( session=session, @@ -465,23 +465,23 @@ def open_session() -> requests.Session: ) from exc -def require_hub_at_main(git: Git) -> None: - """Refuse unless the dfo-docs checkout's `HEAD` is the remote's `main`. +def require_hub_at_base(git: Git) -> None: + """Refuse unless the dfo-docs checkout's `HEAD` is the remote's `HUB_BASE` branch. - Hub pages are read at `HEAD` and their edits proposed against `main`, so any other + Hub pages are read at `HEAD` and their edits proposed against `HUB_BASE`, so any other commit would publish unmerged text or carry unrelated changes into the pull request. Raises: - HubCheckoutError: `HEAD` differs from the freshly fetched `origin/main`. + HubCheckoutError: `HEAD` differs from the freshly fetched `origin/dev`. GitError: the fetch failed. """ git.run("fetch", "-q", HUB_REMOTE, f"refs/heads/{HUB_BASE}") - main = git.run("rev-parse", "--verify", "FETCH_HEAD^{commit}").strip() + base = git.run("rev-parse", "--verify", "FETCH_HEAD^{commit}").strip() head = git.run("rev-parse", "--verify", "HEAD^{commit}").strip() - if head != main: + if head != base: raise HubCheckoutError( f"{git.root}: HEAD {head[:12]} is not {HUB_REMOTE}/{HUB_BASE}" - f" {main[:12]}; check out {HUB_BASE} and pull before syncing hub pages" + f" {base[:12]}; check out {HUB_BASE} and pull before syncing hub pages" ) diff --git a/src/dfo_docs/docsync/edits.py b/src/dfo_docs/docsync/edits.py index 16688dd..e61ba94 100644 --- a/src/dfo_docs/docsync/edits.py +++ b/src/dfo_docs/docsync/edits.py @@ -1884,9 +1884,10 @@ def _same_list_kind(block: Block, other: Block) -> bool: def heading_anchor(heading: str) -> str: """The anchor the docs site gives a heading with Markdown text `heading`. - The site (dfo-website `docsSlug`) takes the heading's rendered text, code spans - included, lowercases and trims it, drops every character but ASCII letters, - digits, `_`, `-` and whitespace, and turns each run of whitespace into `-`. + The site (`docsSlug` in the website repository) takes the heading's rendered + text, code spans included, lowercases and trims it, drops every character but + ASCII letters, digits, `_`, `-` and whitespace, and turns each run of whitespace + into `-`. """ text = "".join(_text_content(_MARKDOWN.parseInline(heading)[0].children or [])) dropped = _SLUG_DROP_RE.sub("", text.lower().strip()) diff --git a/src/dfo_docs/docsync/git.py b/src/dfo_docs/docsync/git.py index 6ee0473..1c63657 100644 --- a/src/dfo_docs/docsync/git.py +++ b/src/dfo_docs/docsync/git.py @@ -2,7 +2,7 @@ `Git` runs one git command in a checkout with a fixed environment. `command_env` builds that environment: the process environment plus, when a token is given, a credential -helper for github.com that reads the token from `DFO_BOT_TOKEN`, so the token never +helper for github.com that reads the token from `DFO_DOCSYNC_TOKEN`, so the token never reaches a config file or a remote URL. """ @@ -16,9 +16,9 @@ # The environment variable the credential helper reads the token from, and the one # `prs.propose_edits` reads the push token from when none is passed. CI sets it to a -# dfo-bot installation token; local runs leave it unset and use the user's own git and +# dfo-docsync installation token; local runs leave it unset and use the user's own git and # gh credentials. -TOKEN_ENV = "DFO_BOT_TOKEN" +TOKEN_ENV = "DFO_DOCSYNC_TOKEN" _TOKEN_HELPER = ( '!f() { test "$1" = get || return 0; echo username=x-access-token; ' @@ -83,9 +83,9 @@ def run( def command_env(token: str | None) -> dict[str, str]: """The environment for git and gh, carrying `token` when one is given. - The token goes in `DFO_BOT_TOKEN` and `GH_TOKEN`. GIT_CONFIG_COUNT entries, appended + The token goes in `DFO_DOCSYNC_TOKEN` and `GH_TOKEN`. GIT_CONFIG_COUNT entries, appended after any the caller already set, first clear the helpers git reads from config - files and then add one for github.com that answers `get` from `DFO_BOT_TOKEN`. + files and then add one for github.com that answers `get` from `DFO_DOCSYNC_TOKEN`. """ env = {**os.environ, "GIT_TERMINAL_PROMPT": "0", "GH_PROMPT_DISABLED": "1"} if token is None: diff --git a/src/dfo_docs/docsync/pages.py b/src/dfo_docs/docsync/pages.py index 7823656..c9cb00a 100644 --- a/src/dfo_docs/docsync/pages.py +++ b/src/dfo_docs/docsync/pages.py @@ -42,7 +42,7 @@ class HubCheckoutError(RuntimeError): """The dfo-docs checkout cannot supply hub pages. Its hub content has uncommitted changes, it has no branch checked out, or its - `HEAD` is not the remote's `main`. + `HEAD` is not the remote's `dev`. """ diff --git a/src/dfo_docs/docsync/prs.py b/src/dfo_docs/docsync/prs.py index 1cbdde1..647b422 100644 --- a/src/dfo_docs/docsync/prs.py +++ b/src/dfo_docs/docsync/prs.py @@ -274,7 +274,7 @@ def propose_edits( ) -> str: """Push `files` as one commit on the new branch `branch` and open a PR from it. - The commit sits on the tip of `base` on `remote`. For hub pages pass `main`, the + The commit sits on the tip of `base` on `remote`. For hub pages pass `dev`, the branch edits are proposed against, whatever `clone_root` has checked out. The push is a plain push of a branch that does not exist yet, so it can never replace commits on an existing branch. If `branch` exists, the remote rejects it. @@ -300,7 +300,7 @@ def propose_edits( `list_open_pull_requests`. identity: Author and committer of the commit. When `None`, git resolves them from its own environment variables and config. - token: A GitHub token for the push and for gh. When `None`, the `DFO_BOT_TOKEN` + token: A GitHub token for the push and for gh. When `None`, the `DFO_DOCSYNC_TOKEN` environment variable is used if set, else the user's own git and gh credentials. The token reaches git through a credential helper set in the command environment, and gh through `GH_TOKEN`, so it is never written to @@ -382,7 +382,7 @@ def list_open_pull_requests( Args: repo: The GitHub repository as `owner/name`. - author: The gh `--author` value: `app/dfo-bot` for the bot, or a login. + author: The gh `--author` value: `app/dfo-docsync` for the bot, or a login. token: A GitHub token for gh, handled as in `propose_edits`. Raises: diff --git a/src/dfo_docs/manifest.py b/src/dfo_docs/manifest.py index 26aaf1c..83ab1b4 100644 --- a/src/dfo_docs/manifest.py +++ b/src/dfo_docs/manifest.py @@ -46,7 +46,7 @@ # The hub's own content has no registered Source, so it is given a stand-in name and # repo shorthand consistent with the "owner/name" shape every other source_repo carries. _HUB_SOURCE_NAME = "hub" -_HUB_SOURCE_REPO = "ONEcampaign/dfo-docs" +_HUB_SOURCE_REPO = "DevelopmentFinanceObservatory/docs" # The hub page each generated skill header links to, relative to the hub's content root. _SKILL_INSTALL_PAGE = "skills/use-in-your-agent.md" diff --git a/src/dfo_docs/schema/manifest.schema.json b/src/dfo_docs/schema/manifest.schema.json index 0e00e02..4fd843f 100644 --- a/src/dfo_docs/schema/manifest.schema.json +++ b/src/dfo_docs/schema/manifest.schema.json @@ -1,6 +1,6 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://raw.githubusercontent.com/ONEcampaign/dfo-docs/corpus/manifest.schema.json", + "$id": "https://raw.githubusercontent.com/DevelopmentFinanceObservatory/docs/corpus/manifest.schema.json", "title": "dfo-docs manifest envelope", "description": "The published manifest.json envelope: a versioned nav tree and the pages it points at. Generate types from this schema rather than reverse-engineering shapes from an example; the two do not always agree, and this file is the one that is kept honest by a test that validates a real generator run against it.", "type": "object", diff --git a/src/dfo_docs/schema/roadmap-track.schema.json b/src/dfo_docs/schema/roadmap-track.schema.json index b0aad60..8511424 100644 --- a/src/dfo_docs/schema/roadmap-track.schema.json +++ b/src/dfo_docs/schema/roadmap-track.schema.json @@ -1,6 +1,6 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://raw.githubusercontent.com/ONEcampaign/dfo-docs/corpus/roadmap-track.schema.json", + "$id": "https://raw.githubusercontent.com/DevelopmentFinanceObservatory/docs/corpus/roadmap-track.schema.json", "title": "dfo-docs roadmap track", "description": "One roadmap track, published at the path roadmap.json gives for it, such as roadmap/data.json. Holds that track's own category vocabulary and every entry in it, upcoming and released. Each track is authored and reviewed by the team that owns it, and a track publishes even when another track's document fails to build. Generate types from this schema rather than from an example. Votes are live, per-viewer state that changes without a rebuild, so they live in the site's own store keyed by entry id rather than in this file.", "type": "object", diff --git a/src/dfo_docs/schema/roadmap.schema.json b/src/dfo_docs/schema/roadmap.schema.json index 763f332..9f9b4b5 100644 --- a/src/dfo_docs/schema/roadmap.schema.json +++ b/src/dfo_docs/schema/roadmap.schema.json @@ -1,6 +1,6 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://raw.githubusercontent.com/ONEcampaign/dfo-docs/corpus/roadmap.schema.json", + "$id": "https://raw.githubusercontent.com/DevelopmentFinanceObservatory/docs/corpus/roadmap.schema.json", "title": "dfo-docs roadmap index", "description": "The published roadmap.json names the roadmap tracks and says where to fetch each one. It carries no entries. Each track is a separate document validated by roadmap-track.schema.json and owned by the team that owns that part of the product, so a track is edited, reviewed, and published without touching the others. Fetch this file first, render the tabs from it in array order, and load a track's own document when its tab is opened.", "type": "object", diff --git a/tests/docsync/fixtures/edits/sampler.edited.md b/tests/docsync/fixtures/edits/sampler.edited.md index 7b685de..a7abb1d 100644 --- a/tests/docsync/fixtures/edits/sampler.edited.md +++ b/tests/docsync/fixtures/edits/sampler.edited.md @@ -43,7 +43,7 @@ and its second line. 1. Clone the skills repo. ```sh -git clone --depth 1 https://github.com/ONEcampaign/dfo-skills.git +git clone --depth 1 https://github.com/DevelopmentFinanceObservatory/skills.git ``` 2. Copy the skills into a skills folder. diff --git a/tests/docsync/fixtures/edits/sampler.expected.md b/tests/docsync/fixtures/edits/sampler.expected.md index f17dd94..40194c1 100644 --- a/tests/docsync/fixtures/edits/sampler.expected.md +++ b/tests/docsync/fixtures/edits/sampler.expected.md @@ -45,7 +45,7 @@ An unindented callout body. 1. Clone the skills repo. ```bash - git clone --depth 1 https://github.com/ONEcampaign/dfo-skills.git + git clone --depth 1 https://github.com/DevelopmentFinanceObservatory/skills.git ``` 1. Copy the skills into a skills folder. diff --git a/tests/docsync/fixtures/pages/conventions-live.baseline.md b/tests/docsync/fixtures/pages/conventions-live.baseline.md index 4560720..67465d5 100644 --- a/tests/docsync/fixtures/pages/conventions-live.baseline.md +++ b/tests/docsync/fixtures/pages/conventions-live.baseline.md @@ -40,7 +40,7 @@ A proxy in front of every route enforces three limits before the request reaches | `413` | Request body too large. | | `415` | The media type is unsupported. Send `Content-Type: application/json`. | -**Author note:** TODO: rate-limit thresholds; set in proxy config, not recorded in dfo-apis. +**Author note:** TODO: rate-limit thresholds; set in proxy config, not recorded in the apis repository. **Author note:** TODO: byte ceiling; set in proxy config. diff --git a/tests/docsync/fixtures/pages/conventions-live.edited.md b/tests/docsync/fixtures/pages/conventions-live.edited.md index aa597d8..a07e90d 100644 --- a/tests/docsync/fixtures/pages/conventions-live.edited.md +++ b/tests/docsync/fixtures/pages/conventions-live.edited.md @@ -41,7 +41,7 @@ A proxy in front of every route enforces three limits before the request reaches | `413` | Request body too large. | | `415` | The media type is unsupported. Send `Content-Type: application/json`. | -**Author note:** TODO: rate-limit thresholds; set in proxy config, not recorded in dfo-apis. +**Author note:** TODO: rate-limit thresholds; set in proxy config, not recorded in the apis repository. **Author note:** TODO: byte ceiling; set in proxy config. diff --git a/tests/docsync/fixtures/pages/conventions-live.source.md b/tests/docsync/fixtures/pages/conventions-live.source.md index ced0545..67fd7c1 100644 --- a/tests/docsync/fixtures/pages/conventions-live.source.md +++ b/tests/docsync/fixtures/pages/conventions-live.source.md @@ -38,7 +38,7 @@ A proxy in front of every route enforces three limits before the request reaches | Status | Meaning | | ------ | ------------------------------------------------------------------------------------------------------------------------------- | -| `429` | Too many requests. | +| `429` | Too many requests. | | `413` | Request body too large. | | `415` | The media type is unsupported. Send `Content-Type: application/json`. | diff --git a/tests/docsync/fixtures/pages/conventions.baseline.md b/tests/docsync/fixtures/pages/conventions.baseline.md index 4560720..67465d5 100644 --- a/tests/docsync/fixtures/pages/conventions.baseline.md +++ b/tests/docsync/fixtures/pages/conventions.baseline.md @@ -40,7 +40,7 @@ A proxy in front of every route enforces three limits before the request reaches | `413` | Request body too large. | | `415` | The media type is unsupported. Send `Content-Type: application/json`. | -**Author note:** TODO: rate-limit thresholds; set in proxy config, not recorded in dfo-apis. +**Author note:** TODO: rate-limit thresholds; set in proxy config, not recorded in the apis repository. **Author note:** TODO: byte ceiling; set in proxy config. diff --git a/tests/docsync/fixtures/pages/conventions.source.md b/tests/docsync/fixtures/pages/conventions.source.md index ced0545..67fd7c1 100644 --- a/tests/docsync/fixtures/pages/conventions.source.md +++ b/tests/docsync/fixtures/pages/conventions.source.md @@ -38,7 +38,7 @@ A proxy in front of every route enforces three limits before the request reaches | Status | Meaning | | ------ | ------------------------------------------------------------------------------------------------------------------------------- | -| `429` | Too many requests. | +| `429` | Too many requests. | | `413` | Request body too large. | | `415` | The media type is unsupported. Send `Content-Type: application/json`. | diff --git a/tests/docsync/fixtures/pages/use-in-your-agent.baseline.md b/tests/docsync/fixtures/pages/use-in-your-agent.baseline.md index fd190ed..6283071 100644 --- a/tests/docsync/fixtures/pages/use-in-your-agent.baseline.md +++ b/tests/docsync/fixtures/pages/use-in-your-agent.baseline.md @@ -4,7 +4,7 @@ Copy the DFO skills into an agent you run to give it the DFO team's methods for ODA questions. -**Author note:** TODO: add a route for the DFO assistant in DFO Studio once dfo-ai loads dfo-skills (dfo-ai docs/upstream-asks.md item 8\) and SKILLS\_ENABLED is on in production. The assistant and the MCP server's `search` tool do not load skills today. State then whether the reader needs to do anything. +**Author note:** TODO: add a route for the DFO assistant in DFO Studio once the assistant loads skills and SKILLS\_ENABLED is on in production. The assistant and the MCP server's `search` tool do not load skills today. State then whether the reader needs to do anything. ## In Claude Code @@ -13,16 +13,13 @@ Claude Code reads skills from two folders. Skills in `~/.claude/skills/` load in ### Before you start - Claude Code installed. -- Read access to `ONEcampaign/dfo-skills` on GitHub. The repository is private, so this route works only for ONE staff. - -**Author note:** TODO: dfo-skills must be public, with a licence, before this route works for readers outside ONE. Remove the access line above when it is. ### Steps 1. Clone the skills repository. ```sh -git clone --depth 1 https://github.com/ONEcampaign/dfo-skills.git +git clone --depth 1 https://github.com/DevelopmentFinanceObservatory/skills.git ``` 2. Copy the skills into a skills folder. @@ -31,7 +28,7 @@ git clone --depth 1 https://github.com/ONEcampaign/dfo-skills.git ```sh mkdir -p ~/.claude/skills -rsync -a --exclude _template dfo-skills/skills/ ~/.claude/skills/ +rsync -a --exclude _template skills/skills/ ~/.claude/skills/ ``` **Tab: For one project** @@ -42,13 +39,13 @@ rsync -a --exclude _template dfo-skills/skills/ ~/.claude/skills/ ```sh mkdir -p .claude/skills -rsync -a --exclude _template path/to/dfo-skills/skills/ .claude/skills/ +rsync -a --exclude _template path/to/skills/skills/ .claude/skills/ ``` The `_template` directory holds the files for creating a new skill. Exclude it when copying skills. Copy a single skill by naming its directory without a trailing slash. On macOS, a trailing slash copies `SKILL.md` and `references/` directly into `skills/`. ```sh -cp -R dfo-skills/skills/oda-measures ~/.claude/skills/ +cp -R skills/skills/oda-measures ~/.claude/skills/ ``` 3. Start a new Claude Code session. @@ -75,17 +72,17 @@ Ask a question one of the skills covers, such as "Should I use grant equivalents Pull the repository, then sync the skills. `rsync` overwrites changed files and leaves your other skills alone. ```sh -git -C dfo-skills pull -rsync -a --exclude _template dfo-skills/skills/ ~/.claude/skills/ +git -C skills pull +rsync -a --exclude _template skills/skills/ ~/.claude/skills/ ``` -Delete a skill's directory when dfo-skills removes it. `rsync` leaves the local copy in place. +Delete a skill's directory when the skills repository removes it. `rsync` leaves the local copy in place. ### Other agents An agent that supports the Agent Skills format can read these directories. Copy each skill directory into its skills folder, keeping `SKILL.md` and `references/` together because `SKILL.md` points to its reference files by relative path. -**Author note:** TODO: add a "Programmatically" route. dfo-apis includes a Skills API that lists skills and serves each skill's body and reference files, with no public deployment yet. Document the Skills API (GET /api/skills, GET /api/skills/{skill\_id}, GET /api/skills/{skill\_id}/resources/{path}) and link the API pages once a public deployment serves it with DFO\_SKILLS\_SURFACE on. +**Author note:** TODO: add a "Programmatically" route. `apis` includes a Skills API that lists skills and serves each skill's body and reference files, with no public deployment yet. Document the Skills API (GET /api/skills, GET /api/skills/{skill\_id}, GET /api/skills/{skill\_id}/resources/{path}) and link the API pages once a public deployment serves it with DFO\_SKILLS\_SURFACE on. ## Next diff --git a/tests/docsync/fixtures/pages/use-in-your-agent.raw-export.md b/tests/docsync/fixtures/pages/use-in-your-agent.raw-export.md index ad66efd..b0a7098 100644 --- a/tests/docsync/fixtures/pages/use-in-your-agent.raw-export.md +++ b/tests/docsync/fixtures/pages/use-in-your-agent.raw-export.md @@ -13,14 +13,13 @@ Claude Code reads skills from two folders. Skills in `~/.claude/skills/` load in ### Before you start - Claude Code installed. -- Read access to `ONEcampaign/dfo-skills` on GitHub. The repository is private, so this route works only for ONE staff. ### Steps 1. Clone the skills repository. ```sh -git clone --depth 1 https://github.com/ONEcampaign/dfo-skills.git +git clone --depth 1 https://github.com/DevelopmentFinanceObservatory/skills.git ``` 2. Copy the skills into a skills folder. @@ -31,7 +30,7 @@ git clone --depth 1 https://github.com/ONEcampaign/dfo-skills.git ```sh mkdir -p ~/.claude/skills -rsync -a --exclude _template dfo-skills/skills/ ~/.claude/skills/ +rsync -a --exclude _template skills/skills/ ~/.claude/skills/ ``` ::tab{label="For one project"} @@ -42,7 +41,7 @@ rsync -a --exclude _template dfo-skills/skills/ ~/.claude/skills/ ```sh mkdir -p .claude/skills -rsync -a --exclude _template path/to/dfo-skills/skills/ .claude/skills/ +rsync -a --exclude _template path/to/skills/skills/ .claude/skills/ ``` ::: @@ -52,7 +51,7 @@ rsync -a --exclude _template path/to/dfo-skills/skills/ .claude/skills/ The `_template` directory holds the files for creating a new skill. Exclude it when copying skills. Copy a single skill by naming its directory without a trailing slash. On macOS, a trailing slash copies `SKILL.md` and `references/` directly into `skills/`. ```sh -cp -R dfo-skills/skills/oda-measures ~/.claude/skills/ +cp -R skills/skills/oda-measures ~/.claude/skills/ ``` 3. Start a new Claude Code session. @@ -79,11 +78,11 @@ Ask a question one of the skills covers, such as "Should I use grant equivalents Pull the repository, then sync the skills. `rsync` overwrites changed files and leaves your other skills alone. ```sh -git -C dfo-skills pull -rsync -a --exclude _template dfo-skills/skills/ ~/.claude/skills/ +git -C skills pull +rsync -a --exclude _template skills/skills/ ~/.claude/skills/ ``` -Delete a skill's directory when dfo-skills removes it. `rsync` leaves the local copy in place. +Delete a skill's directory when the skills repository removes it. `rsync` leaves the local copy in place. ### Other agents diff --git a/tests/docsync/fixtures/pages/use-in-your-agent.source.md b/tests/docsync/fixtures/pages/use-in-your-agent.source.md index 39c383a..1f6bf45 100644 --- a/tests/docsync/fixtures/pages/use-in-your-agent.source.md +++ b/tests/docsync/fixtures/pages/use-in-your-agent.source.md @@ -6,7 +6,7 @@ description: Copy the DFO skills into Claude Code or another Agent Skills agent. Copy the DFO skills into an agent you run to give it the DFO team's methods for ODA questions. - + ## In Claude Code @@ -15,16 +15,13 @@ Claude Code reads skills from two folders. Skills in `~/.claude/skills/` load in ### Before you start - Claude Code installed. -- Read access to `ONEcampaign/dfo-skills` on GitHub. The repository is private, so this route works only for ONE staff. - - ### Steps 1. Clone the skills repository. ```bash - git clone --depth 1 https://github.com/ONEcampaign/dfo-skills.git + git clone --depth 1 https://github.com/DevelopmentFinanceObservatory/skills.git ``` 1. Copy the skills into a skills folder. @@ -35,7 +32,7 @@ Claude Code reads skills from two folders. Skills in `~/.claude/skills/` load in ```bash mkdir -p ~/.claude/skills - rsync -a --exclude _template dfo-skills/skills/ ~/.claude/skills/ + rsync -a --exclude _template skills/skills/ ~/.claude/skills/ ``` ::tab{label="For one project"} @@ -44,7 +41,7 @@ Claude Code reads skills from two folders. Skills in `~/.claude/skills/` load in ```bash mkdir -p .claude/skills - rsync -a --exclude _template path/to/dfo-skills/skills/ .claude/skills/ + rsync -a --exclude _template path/to/skills/skills/ .claude/skills/ ``` ::: @@ -52,7 +49,7 @@ Claude Code reads skills from two folders. Skills in `~/.claude/skills/` load in The `_template` directory holds the files for creating a new skill. Exclude it when copying skills. Copy a single skill by naming its directory without a trailing slash. On macOS, a trailing slash copies `SKILL.md` and `references/` directly into `skills/`. ```bash - cp -R dfo-skills/skills/oda-measures ~/.claude/skills/ + cp -R skills/skills/oda-measures ~/.claude/skills/ ``` 1. Start a new Claude Code session. @@ -79,17 +76,17 @@ Ask a question one of the skills covers, such as "Should I use grant equivalents Pull the repository, then sync the skills. `rsync` overwrites changed files and leaves your other skills alone. ```bash -git -C dfo-skills pull -rsync -a --exclude _template dfo-skills/skills/ ~/.claude/skills/ +git -C skills pull +rsync -a --exclude _template skills/skills/ ~/.claude/skills/ ``` -Delete a skill's directory when dfo-skills removes it. `rsync` leaves the local copy in place. +Delete a skill's directory when the skills repository removes it. `rsync` leaves the local copy in place. ### Other agents An agent that supports the Agent Skills format can read these directories. Copy each skill directory into its skills folder, keeping `SKILL.md` and `references/` together because `SKILL.md` points to its reference files by relative path. - + ## Next diff --git a/tests/docsync/fixtures/pages/use-studio.baseline.md b/tests/docsync/fixtures/pages/use-studio.baseline.md index 42240c2..2888e88 100644 --- a/tests/docsync/fixtures/pages/use-studio.baseline.md +++ b/tests/docsync/fixtures/pages/use-studio.baseline.md @@ -12,7 +12,7 @@ A canvas with one pinned line chart, "Current Health Expenditure Per Capita in K Studio generates a new plan, title, and answer for each run. The chart uses the same data. -**Author note:** TODO: verify the chart title, plan and answer against the staging site; they come from the recorded Studio session dfo-ai docs/transcripts/studio\_transcript\_plan.jsonl. +**Author note:** TODO: verify the chart title, plan and answer against the staging site; they come from the recorded Studio session in the ai repository, docs/transcripts/studio\_transcript\_plan.jsonl. ## What you'll learn diff --git a/tests/docsync/fixtures/pages/use-studio.source.md b/tests/docsync/fixtures/pages/use-studio.source.md index 0501347..590bd8b 100644 --- a/tests/docsync/fixtures/pages/use-studio.source.md +++ b/tests/docsync/fixtures/pages/use-studio.source.md @@ -14,7 +14,7 @@ A canvas with one pinned line chart, "Current Health Expenditure Per Capita in K Studio generates a new plan, title, and answer for each run. The chart uses the same data. - + ## What you'll learn diff --git a/tests/docsync/fixtures/probe/sampler.baseline.md b/tests/docsync/fixtures/probe/sampler.baseline.md index 663d502..6e76c53 100644 --- a/tests/docsync/fixtures/probe/sampler.baseline.md +++ b/tests/docsync/fixtures/probe/sampler.baseline.md @@ -25,7 +25,7 @@ A paragraph with a [relative link](../about/cite-and-licence.md), a [bare link]( 1. Clone the skills repository. ```sh -git clone --depth 1 https://github.com/ONEcampaign/dfo-skills.git +git clone --depth 1 https://github.com/DevelopmentFinanceObservatory/skills.git ``` 1. Copy the skills into a skills folder. diff --git a/tests/docsync/fixtures/probe/sampler.doc.md b/tests/docsync/fixtures/probe/sampler.doc.md index 2c92d03..0488d65 100644 --- a/tests/docsync/fixtures/probe/sampler.doc.md +++ b/tests/docsync/fixtures/probe/sampler.doc.md @@ -22,7 +22,7 @@ A paragraph with a [relative link](../about/cite-and-licence.md), a [bare link]( 1. Clone the skills repository. ```bash -git clone --depth 1 https://github.com/ONEcampaign/dfo-skills.git +git clone --depth 1 https://github.com/DevelopmentFinanceObservatory/skills.git ``` 1. Copy the skills into a skills folder. diff --git a/tests/docsync/fixtures/probe/sampler.edited.md b/tests/docsync/fixtures/probe/sampler.edited.md index 5ec5929..355c85d 100644 --- a/tests/docsync/fixtures/probe/sampler.edited.md +++ b/tests/docsync/fixtures/probe/sampler.edited.md @@ -26,7 +26,7 @@ An inserted paragraph. 1. Clone the skills repo. ```sh -git clone --depth 1 https://github.com/ONEcampaign/dfo-skills.git +git clone --depth 1 https://github.com/DevelopmentFinanceObservatory/skills.git ``` 1. Copy the skills into a skills folder. diff --git a/tests/docsync/fixtures/probe/sampler.export.md b/tests/docsync/fixtures/probe/sampler.export.md index 663d502..6e76c53 100644 --- a/tests/docsync/fixtures/probe/sampler.export.md +++ b/tests/docsync/fixtures/probe/sampler.export.md @@ -25,7 +25,7 @@ A paragraph with a [relative link](../about/cite-and-licence.md), a [bare link]( 1. Clone the skills repository. ```sh -git clone --depth 1 https://github.com/ONEcampaign/dfo-skills.git +git clone --depth 1 https://github.com/DevelopmentFinanceObservatory/skills.git ``` 1. Copy the skills into a skills folder. diff --git a/tests/docsync/fixtures/sampler.baseline.md b/tests/docsync/fixtures/sampler.baseline.md index 3faf9db..7432548 100644 --- a/tests/docsync/fixtures/sampler.baseline.md +++ b/tests/docsync/fixtures/sampler.baseline.md @@ -45,7 +45,7 @@ and its second line. 1. Clone the skills repository. ```sh -git clone --depth 1 https://github.com/ONEcampaign/dfo-skills.git +git clone --depth 1 https://github.com/DevelopmentFinanceObservatory/skills.git ``` 2. Copy the skills into a skills folder. diff --git a/tests/docsync/fixtures/sampler.source.md b/tests/docsync/fixtures/sampler.source.md index 983d40b..b3f6db9 100644 --- a/tests/docsync/fixtures/sampler.source.md +++ b/tests/docsync/fixtures/sampler.source.md @@ -46,7 +46,7 @@ An unindented callout body. 1. Clone the skills repository. ```bash - git clone --depth 1 https://github.com/ONEcampaign/dfo-skills.git + git clone --depth 1 https://github.com/DevelopmentFinanceObservatory/skills.git ``` 1. Copy the skills into a skills folder. diff --git a/tests/docsync/test_cli.py b/tests/docsync/test_cli.py index 7fd07a0..3cad2ba 100644 --- a/tests/docsync/test_cli.py +++ b/tests/docsync/test_cli.py @@ -40,9 +40,9 @@ ROOT = "root-folder" DRIVE = "drive-1" SHEET = "sheet-1" -REPO = "ONEcampaign/dfo-apis" -PR_URL = "https://github.com/ONEcampaign/dfo-apis/pull/7" -PR_URL_2 = "https://github.com/ONEcampaign/dfo-apis/pull/8" +REPO = "DevelopmentFinanceObservatory/apis" +PR_URL = "https://github.com/DevelopmentFinanceObservatory/apis/pull/7" +PR_URL_2 = "https://github.com/DevelopmentFinanceObservatory/apis/pull/8" PAGE_TEXT = """\ --- @@ -411,7 +411,7 @@ def world(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> World: sheet_id=SHEET, pull_request=github.pull_request, open_pull_requests=github.open_pull_requests, - pr_author="app/dfo-bot", + pr_author="app/dfo-docsync", now="2026-10-01 09:00 UTC", ) return World(fake, MemoryStore(), proposals, source, github, ctx) @@ -808,9 +808,11 @@ def test_pull_skips_rows_not_ready(world: World): assert world.proposals.calls == [] -def test_pull_hub_pages_target_main(world: World): +def test_pull_hub_pages_target_dev(world: World): page, state = _pushed_page(world) - hub = replace(page, page_id="hub/call", repo="ONEcampaign/dfo-docs", ref="feat") + hub = replace( + page, page_id="hub/call", repo="DevelopmentFinanceObservatory/docs", ref="feat" + ) world.store.states["hub/call"] = replace(state, page_id="hub/call") world.google.rows["hub/call"] = { "Page id": "hub/call", @@ -820,7 +822,7 @@ def test_pull_hub_pages_target_main(world: World): world.pull([hub]) - assert world.proposals.calls[0]["base"] == "main" + assert world.proposals.calls[0]["base"] == "dev" def test_pull_refuses_doc_with_open_suggestions( @@ -1263,9 +1265,12 @@ def test_dry_run_writes_files_and_no_state_or_sheet( log = world.pull([page], dry_run_dir=out) assert log.failures == 0 - written = out / "ONEcampaign__dfo-apis" / "files" / "docs" / "call.md" + written = out / "DevelopmentFinanceObservatory__apis" / "files" / "docs" / "call.md" assert written.read_text() == PAGE_TEXT.replace("your token", "your API token") - assert "Owner: `Ana`" in (out / "ONEcampaign__dfo-apis" / "pr.md").read_text() + assert ( + "Owner: `Ana`" + in (out / "DevelopmentFinanceObservatory__apis" / "pr.md").read_text() + ) assert world.store.states[page.page_id] == state assert world.store.saves == ["docsync push: 1 pages"] assert len(world.google.upserts) == upserts @@ -1697,7 +1702,9 @@ def test_main_exits_2_on_missing_folder( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture ): config = tmp_path / "docsync.yaml" - config.write_text("sources: [hub]\ndrive_folder_id: nope\npr_author: app/dfo-bot\n") + config.write_text( + "sources: [hub]\ndrive_folder_id: nope\npr_author: app/dfo-docsync\n" + ) monkeypatch.setenv(cli.ACCESS_TOKEN_ENV, "ya29.token") class NoFolder: @@ -1867,7 +1874,7 @@ def crash_on_second(session, file_id): GitError(["git", "push"], "rejected"), StateConflictError("docsync-state moved"), cli.GoogleApiError(status=500, message="boom", method="GET", url="u"), - HubCheckoutError("HEAD is not origin/main"), + HubCheckoutError("HEAD is not origin/dev"), ], ) def test_main_exits_2_on_named_errors( @@ -2119,9 +2126,9 @@ def test_restore_refuses_a_malformed_file_and_saves_nothing( # Hub checkout, page selection, Sheet layout -def test_hub_pages_need_head_at_origin_main(tmp_path: Path): +def test_hub_pages_need_head_at_origin_dev(tmp_path: Path): origin = Source(tmp_path / "origin") - origin.git("checkout", "-q", "-b", "main") + origin.git("checkout", "-q", "-b", "dev") origin.commit("content/hub/index.md", "# Home\n") clone = tmp_path / "clone" subprocess.run( @@ -2131,7 +2138,7 @@ def test_hub_pages_need_head_at_origin_main(tmp_path: Path): ) git = Git(root=clone, env=command_env(None)) - cli.require_hub_at_main(git) + cli.require_hub_at_base(git) git.run( "-c", @@ -2144,8 +2151,8 @@ def test_hub_pages_need_head_at_origin_main(tmp_path: Path): "-m", "local", ) - with pytest.raises(HubCheckoutError, match="is not origin/main"): - cli.require_hub_at_main(git) + with pytest.raises(HubCheckoutError, match="is not origin/dev"): + cli.require_hub_at_base(git) @pytest.mark.parametrize( @@ -2216,7 +2223,7 @@ class Wired: def wired(world: World, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Wired: config = tmp_path / "docsync.yaml" config.write_text( - "sources: [dfo-apis]\ndrive_folder_id: unused\npr_author: app/dfo-bot\n" + "sources: [dfo-apis]\ndrive_folder_id: unused\npr_author: app/dfo-docsync\n" ) monkeypatch.setenv(cli.TOKEN_ENV, "bot-token") monkeypatch.setattr(cli, "open_session", lambda: None) @@ -2310,7 +2317,9 @@ def test_pull_command_dry_run_writes_files_only( assert cli.main(["pull", *wired.args, "--dry-run", str(tmp_path / "dry")]) == 0 (call,) = world.proposals.calls - assert call["dry_run_dir"] == tmp_path / "dry" / "ONEcampaign__dfo-apis" + assert ( + call["dry_run_dir"] == tmp_path / "dry" / "DevelopmentFinanceObservatory__apis" + ) assert len(world.store.saves) == saves assert world.status(page) == "Ready for review" diff --git a/tests/docsync/test_edits.py b/tests/docsync/test_edits.py index 773ed75..54909d9 100644 --- a/tests/docsync/test_edits.py +++ b/tests/docsync/test_edits.py @@ -5,11 +5,12 @@ deleteContentRange); its baseline is the recorded `sampler.baseline.md` or `pages/.baseline.md`, which the upload reproduced byte for byte. `.expected.md` is the source with exactly those edits made by hand. -`pages/conventions.*` is a dfo-apis page and its recorded baseline, which holds -Author notes on lines of their own and in table cells. `pages/use-studio.*`, -`pages/connect.*` and `pages/methods.*` are hub and dfo-ai pages with their recorded -baselines, holding Author notes beside paragraphs, tabs and code blocks. -`pages/conventions-live.*` is an earlier version of the dfo-apis page, its recorded +`pages/conventions.*` is a page from the apis repository and its recorded baseline, +which holds Author notes on lines of their own and in table cells. +`pages/use-studio.*`, `pages/connect.*` and `pages/methods.*` are pages from the hub +and the ai repository with their recorded baselines, holding Author notes beside +paragraphs, tabs and code blocks. +`pages/conventions-live.*` is an earlier version of that page, its recorded baseline, and Google's export after an editor resolved a TODO note, pressed Enter at the end of a paragraph and typed a sentence, and typed a paragraph below the page's final list. @@ -273,7 +274,7 @@ def test_a_note_resolved_in_place_becomes_page_text(note: str, answer: str) -> N def test_a_table_cell_note_resolved_in_place_follows_the_table() -> None: - note = "TODO: rate-limit thresholds; set in proxy config, not recorded in dfo-apis." + note = "TODO: rate-limit thresholds; set in proxy config, not recorded in the apis repository." answer = "The rate limit is 10 calls per second, set in the proxy config." source = _read("pages/conventions.source.md") table_end = ( @@ -736,13 +737,13 @@ def test_each_note_a_region_turns_into_text_is_flagged() -> None: ) STUDIO_NOTE = ( "**Author note:** TODO: verify the chart title, plan and answer against the" - " staging site; they come from the recorded Studio session dfo-ai" - " docs/transcripts/studio\\_transcript\\_plan.jsonl." + " staging site; they come from the recorded Studio session in the ai" + " repository, docs/transcripts/studio\\_transcript\\_plan.jsonl." ) STUDIO_COMMENT = ( "" + " they come from the recorded Studio session in the ai" + " repository, docs/transcripts/studio_transcript_plan.jsonl. -->" ) ANSWER = "The staging site shows the same title." diff --git a/tests/docsync/test_pages.py b/tests/docsync/test_pages.py index 44e98cf..effcb68 100644 --- a/tests/docsync/test_pages.py +++ b/tests/docsync/test_pages.py @@ -146,7 +146,7 @@ def _site(tmp_path: Path) -> Path: def _config(*sources: str) -> DocsyncConfig: return DocsyncConfig( - sources=sources, drive_folder_id="folder", pr_author="app/dfo-bot" + sources=sources, drive_folder_id="folder", pr_author="app/dfo-docsync" ) @@ -210,7 +210,7 @@ def test_each_page_carries_raw_text_at_its_commit_in_a_live_clone(tmp_path): hub = pages["hub/build/share"] assert hub.clone_root == site - assert hub.repo == "ONEcampaign/dfo-docs" + assert hub.repo == "DevelopmentFinanceObservatory/docs" assert hub.ref == "main" assert hub.commit == _git(site, "rev-parse", "HEAD").strip() assert hub.path == "content/hub/build/share.md" @@ -280,7 +280,7 @@ def test_load_config_reads_the_repository_docsync_yaml(): config = load_config(DFO_DOCS_ROOT / "docsync.yaml") assert config.sources == ("hub", "dfo-apis", "dfo-ai", "dfo-skills") assert config.drive_folder_id == "1DL_BWEeo92oksJEr28xlXFD46xQxFQG1" - assert config.pr_author == "app/dfo-bot" + assert config.pr_author == "app/dfo-docsync" def test_the_repository_allowlist_names_publishable_sources(): diff --git a/tests/docsync/test_prs.py b/tests/docsync/test_prs.py index e470223..f6fdc82 100644 --- a/tests/docsync/test_prs.py +++ b/tests/docsync/test_prs.py @@ -38,12 +38,12 @@ view_pull_request, ) -REPO = "ONEcampaign/dfo-apis" -PR_URL = "https://github.com/ONEcampaign/dfo-apis/pull/7" -BOT = GitIdentity(name="dfo-bot[bot]", email="dfo-bot@users.noreply.github.com") +REPO = "DevelopmentFinanceObservatory/apis" +PR_URL = "https://github.com/DevelopmentFinanceObservatory/apis/pull/7" +BOT = GitIdentity(name="dfo-docsync[bot]", email="dfo-docsync@users.noreply.github.com") BRANCH = edits_branch("20261001T090000Z") WJ = "\u2060" -AUTHOR = "app/dfo-bot" +AUTHOR = "app/dfo-docsync" _FAKE_GH = f"""#!{sys.executable} import json, os, sys @@ -198,7 +198,7 @@ def test_creates_branch_one_commit_over_base_and_opens_pr( ] author = _git(remote, "log", "-1", "--format=%an <%ae>|%cn|%s", BRANCH) assert author == ( - "dfo-bot[bot] |dfo-bot[bot]|" + "dfo-docsync[bot] |dfo-docsync[bot]|" "Docs edits from Google Docs" ) @@ -509,7 +509,7 @@ def test_list_open_pull_requests_keeps_the_authors_docsync_branches( ) ) - listed = list_open_pull_requests(REPO, author="app/dfo-bot", token="ghs_secret") + listed = list_open_pull_requests(REPO, author="app/dfo-docsync", token="ghs_secret") assert listed == ( OpenPullRequest( @@ -521,14 +521,14 @@ def test_list_open_pull_requests_keeps_the_authors_docsync_branches( assert args[:2] == ["pr", "list"] assert args[args.index("--repo") + 1] == REPO assert args[args.index("--state") + 1] == "open" - assert args[args.index("--author") + 1] == "app/dfo-bot" + assert args[args.index("--author") + 1] == "app/dfo-docsync" assert call["gh_token"] == "ghs_secret" def test_list_open_pull_requests_refuses_output_that_is_not_a_list(gh: Path) -> None: (gh / "reply.json").write_text("{}") with pytest.raises(GhError, match="no JSON list"): - list_open_pull_requests(REPO, author="app/dfo-bot") + list_open_pull_requests(REPO, author="app/dfo-docsync") def test_view_pull_request_reads_state_files_and_merge_commit(gh: Path) -> None: @@ -639,7 +639,7 @@ def test_render_pr_body_neutralises_sheet_and_doc_text() -> None: def test_render_pr_body_fences_clash_text_unchanged() -> None: text = ( - "Fixes https://github.com/o/r/issues/1, closes ONEcampaign/dfo-docs#42, " + "Fixes https://github.com/o/r/issues/1, closes DevelopmentFinanceObservatory/docs#42, " "cc @ONEcampaign/data" ) body = render_pr_body( diff --git a/tests/docsync/test_state.py b/tests/docsync/test_state.py index ea5ff68..03dbf39 100644 --- a/tests/docsync/test_state.py +++ b/tests/docsync/test_state.py @@ -30,7 +30,7 @@ GUIDE = PageState( page_id="hub/guide/index", doc_id="doc-guide", - repo="ONEcampaign/dfo-docs", + repo="DevelopmentFinanceObservatory/docs", ref="main", path="docs/guide/index.md", source_commit="a" * 40, @@ -39,14 +39,14 @@ API = PageState( page_id="apis/use", doc_id="doc-api", - repo="ONEcampaign/dfo-apis", + repo="DevelopmentFinanceObservatory/apis", ref="main", path="docs/use.md", source_commit="b" * 40, baseline="# Use\n", applied_export="# Use\n\nFirst edit.\n", pulled_export="# Use\n\nEdited.\n", - pr_url="https://github.com/ONEcampaign/dfo-apis/pull/7", + pr_url="https://github.com/DevelopmentFinanceObservatory/apis/pull/7", ) diff --git a/tests/test_manifest.py b/tests/test_manifest.py index fede5fa..aeba756 100644 --- a/tests/test_manifest.py +++ b/tests/test_manifest.py @@ -373,7 +373,7 @@ def test_hub_pages_land_at_corpus_root(tmp_path): assert home.title == "Home" assert home.description == "The hub landing page." assert home.corpus_path == "corpus/hub/index.md" - assert home.source_repo == "ONEcampaign/dfo-docs" + assert home.source_repo == "DevelopmentFinanceObservatory/docs" assert home.source_commit is None assert home.last_modified is None assert (corpus_root / "hub" / "index.md").is_file() @@ -875,7 +875,7 @@ def test_skill_corpus_files_carry_the_header_and_keep_links(tmp_path): assert not (corpus_root / "dfo-skills" / "_template").exists() -# The page entry and the download link that dfo-website's sync and link resolver are +# The page entry and the download link that the website's sync and link resolver are # tested against. Both repositories hold these literals. _BUNDLE_PAGE = { "path": "skills/oda-crs", diff --git a/tests/test_registry.py b/tests/test_registry.py index a786b4d..eb8197d 100644 --- a/tests/test_registry.py +++ b/tests/test_registry.py @@ -188,7 +188,7 @@ def test_external_row_may_not_exclude(tmp_path: Path): _SKILL_ROW = ( "sources:\n" " - name: dfo-skills\n" - " repo: ONEcampaign/dfo-skills\n" + " repo: DevelopmentFinanceObservatory/skills\n" " kind: skill\n" " docs_dir: skills\n" " path: skills\n" diff --git a/tests/test_skills.py b/tests/test_skills.py index 7a5a8e0..e0acb6c 100644 --- a/tests/test_skills.py +++ b/tests/test_skills.py @@ -21,7 +21,7 @@ write_skill_bundle, ) -# The seven published dfo-skills descriptions, as YAML parses them, and the lead each gives. +# The seven published skill descriptions, as YAML parses them, and the lead each gives. REAL_DESCRIPTIONS = [ ( "dfo-faq", @@ -84,7 +84,7 @@ def _fetched(tmp_path: Path, *, internal_dir: str | None = None) -> Fetched: docs_root.mkdir(parents=True) source = Source( name="dfo-skills", - repo="ONEcampaign/dfo-skills", + repo="DevelopmentFinanceObservatory/skills", kind=SourceKind.SKILL, docs_dir="skills", path="skills",