Allow updating multiple docs versions at once

.github/workflows/update-docs-base-bun.yml

@@ -1,47 +1,62 @@
 name: Update Docs
 run-name: >-
   ${{ github.event_name == 'schedule'
-      && 'Nightly docs build - latest'
-      || format('Manual docs build - v{0}', inputs.major_version) }}
+      && 'Nightly docs build'
+      || format('Manual docs build - v{0}', inputs.major_versions) }}
 
 on:
-  # This triggers the auto docs build every night and uses the hard-coded LATEST_VERSION
+  # This triggers the auto docs build every night and uses the hard-coded DOC_VERSIONS
   schedule:
     - cron: "0 2 * * *"
-  # This triggers a build of any numeric docs version
+  # This triggers a build of any set of numeric docs versions
   workflow_dispatch:
     inputs:
-      major_version:
-        description: 'Major version to pull docs from metabase/metabase release branch'
+      major_versions:
+        description: 'Comma separated major versions to pull docs from metabase/metabase release branches, eg. "63,62"'
         required: true
-        type: number
-        default: 999
+        type: string
+        default: "999"
 
 env:
-  LATEST_VERSION: 63
+  # Every version listed here is pulled in and published by a single nightly build.
+  DOC_VERSIONS: "63,62"
 
 jobs:
   params:
     name: Prep params
     runs-on: ubuntu-latest
     timeout-minutes: 5
     outputs:
-      major_version: ${{ steps.set-major-version.outputs.major_version }}
+      versions: ${{ steps.set-versions.outputs.versions }}
     steps:
-      - name: Set major version
-        id: set-major-version
+      - name: Resolve and validate versions
+        id: set-versions
+        env:
+          RAW_VERSIONS: ${{ inputs.major_versions || env.DOC_VERSIONS }}
         run: |
-          echo "major_version=${{ inputs.major_version || env.LATEST_VERSION }}" >> $GITHUB_OUTPUT
+          versions="$(echo "$RAW_VERSIONS" | tr -d '[:space:]')"
+          if ! echo "$versions" | grep -Eq '^[0-9]+(,[0-9]+)*$'; then
+            echo "::error::Expected a comma separated list of major versions (eg. \"63,62\"), got: '$RAW_VERSIONS'"
+            exit 1
+          fi
+          case ",$versions," in
+            *,999,*)
+              echo "::error::999 is the placeholder value - pick real major versions (eg. \"63,62\")."
+              exit 1 ;;
+          esac
+          echo "versions=$versions" >> "$GITHUB_OUTPUT"
+          echo "Building docs for versions: $versions"
 
   build:
     needs: params
-    name: Build docs v${{ needs.params.outputs.major_version }}
+    name: Build docs v${{ needs.params.outputs.versions }}
     runs-on: ubuntu-latest
-    timeout-minutes: 60
+    # The docs pull step can retry up to 3x30 minutes on its own, so leave room for
+    # that plus the jekyll build and htmlproofer.
+    timeout-minutes: 150
     env:
       GH_TOKEN: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}
-      MAIN_REPO_SOURCE_BRANCH: release-x.${{ needs.params.outputs.major_version }}.x
-      MAIN_REPO_TARGET_BRANCH: release-x.${{ needs.params.outputs.major_version }}.x
+      DOC_VERSIONS: ${{ needs.params.outputs.versions }}
     steps:
       - uses: actions/checkout@v4
 
@@ -63,7 +78,9 @@ jobs:
 
       - name: Filter non-documented branches
         run: |
-          bb script/check_incoming_branchname.clj --target-branch "$MAIN_REPO_TARGET_BRANCH"
+          for v in ${DOC_VERSIONS//,/ }; do
+            bb script/check_incoming_branchname.clj --target-branch "release-x.$v.x"
+          done
 
       - name: Setup Node.js
         uses: actions/setup-node@v4
@@ -89,14 +106,20 @@ jobs:
           git remote set-url origin "https://x-access-token:${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}@github.com/metabase/docs.metabase.github.io.git"
 
       # observed this failing during sdk docs generation deps downloads, so adding retry logic
+      # note: a retry re-pulls every version, since the whole loop is the retried unit
       - name: Update docs for branchname
         uses: nick-fields/retry@v3
         with:
-          timeout_minutes: 20
+          timeout_minutes: 30
           max_attempts: 3
           retry_on: error
+          shell: bash
           command: |
-            bb script/update_docs_for_branchname.clj --source-branch "$MAIN_REPO_SOURCE_BRANCH" --target-branch "$MAIN_REPO_TARGET_BRANCH"
+            for v in ${DOC_VERSIONS//,/ }; do
+              echo "::group::Pulling docs from release-x.$v.x"
+              bb script/update_docs_for_branchname.clj --source-branch "release-x.$v.x" --target-branch "release-x.$v.x"
+              echo "::endgroup::"
+            done
 
       - name: Cleanup cloud docs (move them to latest, remove from all other versions)
         run: |
@@ -151,11 +174,10 @@ jobs:
       - name: Update or Create the Pull Request
         run: |
           bb script/update_or_create_pr.clj \
-          --source-branch "$MAIN_REPO_SOURCE_BRANCH" \
-          --target-branch "$MAIN_REPO_TARGET_BRANCH" \
+          --versions "$DOC_VERSIONS" \
           --pr-number "$PR_NUMBER" \
           --update-dirs "$UPDATE_DIRS"
 
       - name: Merge Updated Docs
         run: |
-          bb script/merge.clj --source-branch "$MAIN_REPO_SOURCE_BRANCH" --target-branch "$MAIN_REPO_TARGET_BRANCH"
+          bb script/merge.clj --versions "$DOC_VERSIONS"

script/_test/all.clj

@@ -63,6 +63,36 @@
     (is (integer? docs-version)
         (str "Expected config version to be an integer, got: " docs-version))))
 
+(deftest parse-versions-test
+  (is (= [63 62] (u/parse-versions "63,62")))
+  (is (= [63 62] (u/parse-versions " 62 , 63 "))
+      "whitespace is trimmed and versions are sorted newest first")
+  (is (= [63] (u/parse-versions "63,63")) "duplicates collapse")
+  (is (= [63] (u/parse-versions "63")))
+  (is (= [63] (u/parse-versions 63)) "babashka.cli may hand us a number for a single version")
+  (doseq [bad ["" "," "63,x" "v63" "63,-1" "63,0"]]
+    (is (thrown? clojure.lang.ExceptionInfo (u/parse-versions bad))
+        (str "Expected " (pr-str bad) " to be rejected"))))
+
+(deftest versions->head-ref-name-test
+  (is (= "docs-update-v63-v62" (u/versions->head-ref-name [63 62])))
+  (is (= "docs-update-v63-v62" (u/versions->head-ref-name [62 63]))
+      "the branch name depends on the set of versions, not the order they were listed in"))
+
+(deftest versions->artifacts-test
+  (let [current   (u/config-docs-version)
+        previous  (dec current)
+        artifacts (u/versions->artifacts [current previous])]
+    (is (= (count artifacts) (count (distinct artifacts)))
+        (str "Expected no duplicate paths, got: " (pr-str artifacts)))
+    (is (some #{"_docs/latest"} artifacts)
+        "the current version also publishes to _docs/latest")
+    (is (some #{(str "_docs/v0." current)} artifacts))
+    (is (some #{(str "_docs/v0." previous)} artifacts))
+    (is (some #{"_data/shared_chrome.json"} artifacts))
+    (is (not (some #{"_docs/master"} artifacts))
+        "master docs are never published")))
+
 (deftest categorize-branchname-test
   (doseq [branchname branches
           :let [[category release-num] (u/categorize-branchname branchname)]]

script/_test/find_broken_workflows.clj

@@ -22,13 +22,13 @@
     true))
 
 (defn runs-for-head-ref-name [head-ref-name all-runs]
-  (let [[from to] (str/split head-ref-name #"->")]
+  ;; Head refs from before the multi-version build look like "<source>-><target>";
+  ;; newer ones ("docs-update-v63-v62") have no arrow, so match on whichever parts we get.
+  (let [parts (remove str/blank? (str/split head-ref-name #"->"))]
     (into []
           (filter
             (fn [{:keys [name]}]
-              (let [[from-name to-name] (str/split name #"->")]
-                (and (str/includes? from-name from)
-                     (str/includes? to-name to)))))
+              (every? #(str/includes? (str name) %) parts)))
           all-runs)))
 
 (defn- ->epoch [time-str]

script/merge.clj

@@ -10,19 +10,15 @@
 
 (def cli-spec
   {:spec
-   {:target-branch {:ref "<target-branch>"
-                    :desc "The target branch of the triggering PR."
-                    :alias :t
-                    :require true}
-    :source-branch {:ref "<source-branch>"
-                    :desc "The source branch of the triggering PR."
-                    :alias :r
-                    :require true}}
+   {:versions {:ref "<versions>"
+               :desc "Comma separated major versions included in this build, eg. \"63,62\"."
+               :alias :v
+               :require true}}
    :error-fn u/cli-error-fn})
 
-(defn- find-pr-list [source-branch target-branch]
+(defn- find-pr-list [head-ref]
   (let [pr (-> (p/sh "gh" "pr" "list"
-                     "--head" (u/head-ref-name source-branch target-branch)
+                     "--head" head-ref
                      "--json" "number,headRefName")
                :out
                (json/parse-string true)
@@ -31,13 +27,12 @@
       (do (ice/p [:green "Found PR #" (:number pr)])
           (:number pr))
       (throw (ex-info
-               (str "No PR found for " (u/head-ref-name source-branch target-branch))
-               {:source-branch source-branch
-                :target-branch target-branch
+               (str "No PR found for " head-ref)
+               {:head-ref head-ref
                 :babashka/exit 1})))))
 
-(defn- find-pr-view [source-branch target-branch]
-  (let [pr-num (-> (p/sh "gh" "pr" "view" (u/head-ref-name source-branch target-branch)
+(defn- find-pr-view [head-ref]
+  (let [pr-num (-> (p/sh "gh" "pr" "view" head-ref
                          "--json" "number"
                          "--jq" ".number")
                    :out
@@ -79,44 +74,42 @@
                   (resolve-conflicts-for-file file strat))))
             (resolve-conflicts-for-file artifact strat)))))))
 
-(defn- update-and-merge-pr [source-branch target-branch pr-number merge-strategy]
-  (let [head-ref-name (u/head-ref-name source-branch target-branch)]
-
-
-    (ice/p [:blue "Updating PR branch..."])
-    (ice/p [:blue "Attempting merge with origin/master..."])
-    (let [merge-result (p/sh {:continue true} "git" "merge" "origin/master")]
-      (when-not (zero? (:exit merge-result))
-        (ice/p [:red "✗ Merge failed: " (:err merge-result)])
-        (let [winner (if (= merge-strategy :ours) "PR" "master")]
-          (ice/p [:yellow "Attempting to resolve conflicts with git, preferring changes from " winner "..."])
-          (resolve-conflicts (u/->artifacts target-branch) merge-strategy)
-          ;; Do the commit, now that we've resolved conflicts
-          (pr-str (p/sh "git" "commit" "--no-edit" "-m"
-                        (str "Merge " target-branch " for PR #(" pr-number ")"
-                             ", preferring changes from " winner)))))
-
-      (ice/p [:blue "Pushing changes to PR branch..."])
-      (ice/p "Result: " (pr-str (p/sh "git" "push" "origin" head-ref-name))))
-
-    ;; Wait a bit for GitHub to process to avoid a race condition
-    (Thread/sleep 5000)
-
-    ;; Merge the PR
-    (ice/p [:blue "Merging PR #" pr-number "..."])
-    (let [merge-result (p/sh {:continue true}
-                             "gh" "pr" "merge" (str pr-number)
-                             "--squash" "--delete-branch"
-                             "--repo" "metabase/docs.metabase.github.io")]
-      (if (zero? (:exit merge-result))
-        (ice/p [:green "✓ PR merged successfully!"])
-        (ice/p [:red "✗ Merge failed: " [:bold (:err merge-result)]])))))
+(defn- update-and-merge-pr [head-ref-name versions pr-number merge-strategy]
+  (ice/p [:blue "Updating PR branch..."])
+  (ice/p [:blue "Attempting merge with origin/master..."])
+  (let [merge-result (p/sh {:continue true} "git" "merge" "origin/master")]
+    (when-not (zero? (:exit merge-result))
+      (ice/p [:red "✗ Merge failed: " (:err merge-result)])
+      (let [winner (if (= merge-strategy :ours) "PR" "master")]
+        (ice/p [:yellow "Attempting to resolve conflicts with git, preferring changes from " winner "..."])
+        (resolve-conflicts (u/versions->artifacts versions) merge-strategy)
+        ;; Do the commit, now that we've resolved conflicts
+        (pr-str (p/sh "git" "commit" "--no-edit" "-m"
+                      (str "Merge master into " head-ref-name " for PR #(" pr-number ")"
+                           ", preferring changes from " winner)))))
+
+    (ice/p [:blue "Pushing changes to PR branch..."])
+    (ice/p "Result: " (pr-str (p/sh "git" "push" "origin" head-ref-name))))
+
+  ;; Wait a bit for GitHub to process to avoid a race condition
+  (Thread/sleep 5000)
+
+  ;; Merge the PR
+  (ice/p [:blue "Merging PR #" pr-number "..."])
+  (let [merge-result (p/sh {:continue true}
+                           "gh" "pr" "merge" (str pr-number)
+                           "--squash" "--delete-branch"
+                           "--repo" "metabase/docs.metabase.github.io")]
+    (if (zero? (:exit merge-result))
+      (ice/p [:green "✓ PR merged successfully!"])
+      (ice/p [:red "✗ Merge failed: " [:bold (:err merge-result)]]))))
 
 (defn- should-pr-win?
-  "Determine if the current PR should win conflicts based on PR number comparison"
-  [current-pr-number target-branch]
+  "Determine if the current PR should win conflicts based on PR number comparison.
+   Compares against master, which is what these PRs are always based on."
+  [current-pr-number]
   (let [_ (p/sh "git" "fetch" "origin")
-        latest-master-commit (-> (p/sh "git" "log" "--oneline" "-1" (str "origin/" target-branch))
+        latest-master-commit (-> (p/sh "git" "log" "--oneline" "-1" "origin/master")
                                  :out
                                  str/trim)
         ;; Extract PR number from commit message like "[auto] adding content to docs-rc-notes->master (#380)"
@@ -165,26 +158,26 @@
 
 (defn -main [& args]
   (println "Merge opertaion running at: " (str (java.time.Instant/now)))
-  (let [{:keys [source-branch target-branch]} (cli/parse-opts args cli-spec)
-        [source-branch target-branch] (mapv str/trim [source-branch target-branch])
-        head-ref-name (u/head-ref-name source-branch target-branch)]
+  (let [{:keys [versions]} (cli/parse-opts args cli-spec)
+        versions      (u/parse-versions versions)
+        head-ref-name (u/versions->head-ref-name versions)]
 
     ;; Ensure we're working with the latest remote state
     (ice/p [:blue "Fetching latest from origin..."]) (p/sh "git" "fetch" "origin")
     (ice/p [:blue "Checking out branch: " head-ref-name]) (checkout-branch! head-ref-name)
 
     (let [current-branch (:out (p/sh "git" "branch" "--show-current"))
           _ (ice/p [:green "Currently on branch: " (str/trim current-branch)])
-          pr-number-view (try (find-pr-view source-branch target-branch)
+          pr-number-view (try (find-pr-view head-ref-name)
                               (catch Exception e
                                 (ice/p [:red "Error finding pr-number via view: " (ex-message e)])))
-          pr-number-list (try (find-pr-list source-branch target-branch)
+          pr-number-list (try (find-pr-list head-ref-name)
                               (catch Exception e
                                 (ice/p [:red "Error finding pr-number via list: " (ex-message e)])))
           pr-number (or pr-number-view pr-number-list)
-          merge-strategy (if (should-pr-win? pr-number target-branch) :ours :theirs)]
-      (ice/p [:green "Merging PR #" pr-number ": " (u/head-ref-name source-branch target-branch) " | with strategy: " [:blue merge-strategy]])
-      (update-and-merge-pr source-branch target-branch pr-number merge-strategy))))
+          merge-strategy (if (should-pr-win? pr-number) :ours :theirs)]
+      (ice/p [:green "Merging PR #" pr-number ": " head-ref-name " | with strategy: " [:blue merge-strategy]])
+      (update-and-merge-pr head-ref-name versions pr-number merge-strategy))))
 
 (when (= *file* (System/getProperty "babashka.file"))
   (apply -main *command-line-args*))