From e577a67ebfdb31699f421c545173b3b933117474 Mon Sep 17 00:00:00 2001 From: Ryan Laurie Date: Thu, 30 Jul 2026 14:17:26 -0600 Subject: [PATCH] allow multiple version updates --- .github/workflows/update-docs-base-bun.yml | 68 ++++++++----- script/_test/all.clj | 30 ++++++ script/_test/find_broken_workflows.clj | 8 +- script/merge.clj | 109 ++++++++++----------- script/update_or_create_pr.clj | 87 +++++++--------- script/util.clj | 32 +++++- 6 files changed, 197 insertions(+), 137 deletions(-) diff --git a/.github/workflows/update-docs-base-bun.yml b/.github/workflows/update-docs-base-bun.yml index 15feeea1ce..6fccc6a00c 100644 --- a/.github/workflows/update-docs-base-bun.yml +++ b/.github/workflows/update-docs-base-bun.yml @@ -1,24 +1,25 @@ name: Update Docs run-name: >- ${{ github.event_name == 'schedule' - && 'Nightly docs build - latest' - || format('Manual docs build - v{0}', inputs.major_version) }} + && 'Nightly docs build' + || format('Manual docs build - v{0}', inputs.major_versions) }} on: - # This triggers the auto docs build every night and uses the hard-coded LATEST_VERSION + # This triggers the auto docs build every night and uses the hard-coded DOC_VERSIONS schedule: - cron: "0 2 * * *" - # This triggers a build of any numeric docs version + # This triggers a build of any set of numeric docs versions workflow_dispatch: inputs: - major_version: - description: 'Major version to pull docs from metabase/metabase release branch' + major_versions: + description: 'Comma separated major versions to pull docs from metabase/metabase release branches, eg. "63,62"' required: true - type: number - default: 999 + type: string + default: "999" env: - LATEST_VERSION: 63 + # Every version listed here is pulled in and published by a single nightly build. + DOC_VERSIONS: "63,62" jobs: params: @@ -26,22 +27,36 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 5 outputs: - major_version: ${{ steps.set-major-version.outputs.major_version }} + versions: ${{ steps.set-versions.outputs.versions }} steps: - - name: Set major version - id: set-major-version + - name: Resolve and validate versions + id: set-versions + env: + RAW_VERSIONS: ${{ inputs.major_versions || env.DOC_VERSIONS }} run: | - echo "major_version=${{ inputs.major_version || env.LATEST_VERSION }}" >> $GITHUB_OUTPUT + versions="$(echo "$RAW_VERSIONS" | tr -d '[:space:]')" + if ! echo "$versions" | grep -Eq '^[0-9]+(,[0-9]+)*$'; then + echo "::error::Expected a comma separated list of major versions (eg. \"63,62\"), got: '$RAW_VERSIONS'" + exit 1 + fi + case ",$versions," in + *,999,*) + echo "::error::999 is the placeholder value - pick real major versions (eg. \"63,62\")." + exit 1 ;; + esac + echo "versions=$versions" >> "$GITHUB_OUTPUT" + echo "Building docs for versions: $versions" build: needs: params - name: Build docs v${{ needs.params.outputs.major_version }} + name: Build docs v${{ needs.params.outputs.versions }} runs-on: ubuntu-latest - timeout-minutes: 60 + # The docs pull step can retry up to 3x30 minutes on its own, so leave room for + # that plus the jekyll build and htmlproofer. + timeout-minutes: 150 env: GH_TOKEN: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }} - MAIN_REPO_SOURCE_BRANCH: release-x.${{ needs.params.outputs.major_version }}.x - MAIN_REPO_TARGET_BRANCH: release-x.${{ needs.params.outputs.major_version }}.x + DOC_VERSIONS: ${{ needs.params.outputs.versions }} steps: - uses: actions/checkout@v4 @@ -63,7 +78,9 @@ jobs: - name: Filter non-documented branches run: | - bb script/check_incoming_branchname.clj --target-branch "$MAIN_REPO_TARGET_BRANCH" + for v in ${DOC_VERSIONS//,/ }; do + bb script/check_incoming_branchname.clj --target-branch "release-x.$v.x" + done - name: Setup Node.js uses: actions/setup-node@v4 @@ -89,14 +106,20 @@ jobs: git remote set-url origin "https://x-access-token:${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}@github.com/metabase/docs.metabase.github.io.git" # observed this failing during sdk docs generation deps downloads, so adding retry logic + # note: a retry re-pulls every version, since the whole loop is the retried unit - name: Update docs for branchname uses: nick-fields/retry@v3 with: - timeout_minutes: 20 + timeout_minutes: 30 max_attempts: 3 retry_on: error + shell: bash command: | - bb script/update_docs_for_branchname.clj --source-branch "$MAIN_REPO_SOURCE_BRANCH" --target-branch "$MAIN_REPO_TARGET_BRANCH" + for v in ${DOC_VERSIONS//,/ }; do + echo "::group::Pulling docs from release-x.$v.x" + bb script/update_docs_for_branchname.clj --source-branch "release-x.$v.x" --target-branch "release-x.$v.x" + echo "::endgroup::" + done - name: Cleanup cloud docs (move them to latest, remove from all other versions) run: | @@ -151,11 +174,10 @@ jobs: - name: Update or Create the Pull Request run: | bb script/update_or_create_pr.clj \ - --source-branch "$MAIN_REPO_SOURCE_BRANCH" \ - --target-branch "$MAIN_REPO_TARGET_BRANCH" \ + --versions "$DOC_VERSIONS" \ --pr-number "$PR_NUMBER" \ --update-dirs "$UPDATE_DIRS" - name: Merge Updated Docs run: | - bb script/merge.clj --source-branch "$MAIN_REPO_SOURCE_BRANCH" --target-branch "$MAIN_REPO_TARGET_BRANCH" + bb script/merge.clj --versions "$DOC_VERSIONS" diff --git a/script/_test/all.clj b/script/_test/all.clj index 896332b424..45812e2d7a 100644 --- a/script/_test/all.clj +++ b/script/_test/all.clj @@ -63,6 +63,36 @@ (is (integer? docs-version) (str "Expected config version to be an integer, got: " docs-version)))) +(deftest parse-versions-test + (is (= [63 62] (u/parse-versions "63,62"))) + (is (= [63 62] (u/parse-versions " 62 , 63 ")) + "whitespace is trimmed and versions are sorted newest first") + (is (= [63] (u/parse-versions "63,63")) "duplicates collapse") + (is (= [63] (u/parse-versions "63"))) + (is (= [63] (u/parse-versions 63)) "babashka.cli may hand us a number for a single version") + (doseq [bad ["" "," "63,x" "v63" "63,-1" "63,0"]] + (is (thrown? clojure.lang.ExceptionInfo (u/parse-versions bad)) + (str "Expected " (pr-str bad) " to be rejected")))) + +(deftest versions->head-ref-name-test + (is (= "docs-update-v63-v62" (u/versions->head-ref-name [63 62]))) + (is (= "docs-update-v63-v62" (u/versions->head-ref-name [62 63])) + "the branch name depends on the set of versions, not the order they were listed in")) + +(deftest versions->artifacts-test + (let [current (u/config-docs-version) + previous (dec current) + artifacts (u/versions->artifacts [current previous])] + (is (= (count artifacts) (count (distinct artifacts))) + (str "Expected no duplicate paths, got: " (pr-str artifacts))) + (is (some #{"_docs/latest"} artifacts) + "the current version also publishes to _docs/latest") + (is (some #{(str "_docs/v0." current)} artifacts)) + (is (some #{(str "_docs/v0." previous)} artifacts)) + (is (some #{"_data/shared_chrome.json"} artifacts)) + (is (not (some #{"_docs/master"} artifacts)) + "master docs are never published"))) + (deftest categorize-branchname-test (doseq [branchname branches :let [[category release-num] (u/categorize-branchname branchname)]] diff --git a/script/_test/find_broken_workflows.clj b/script/_test/find_broken_workflows.clj index 038c159109..8851b44644 100644 --- a/script/_test/find_broken_workflows.clj +++ b/script/_test/find_broken_workflows.clj @@ -22,13 +22,13 @@ true)) (defn runs-for-head-ref-name [head-ref-name all-runs] - (let [[from to] (str/split head-ref-name #"->")] + ;; Head refs from before the multi-version build look like "->"; + ;; newer ones ("docs-update-v63-v62") have no arrow, so match on whichever parts we get. + (let [parts (remove str/blank? (str/split head-ref-name #"->"))] (into [] (filter (fn [{:keys [name]}] - (let [[from-name to-name] (str/split name #"->")] - (and (str/includes? from-name from) - (str/includes? to-name to))))) + (every? #(str/includes? (str name) %) parts))) all-runs))) (defn- ->epoch [time-str] diff --git a/script/merge.clj b/script/merge.clj index 3621d24c2a..4ffa9c28d6 100644 --- a/script/merge.clj +++ b/script/merge.clj @@ -10,19 +10,15 @@ (def cli-spec {:spec - {:target-branch {:ref "" - :desc "The target branch of the triggering PR." - :alias :t - :require true} - :source-branch {:ref "" - :desc "The source branch of the triggering PR." - :alias :r - :require true}} + {:versions {:ref "" + :desc "Comma separated major versions included in this build, eg. \"63,62\"." + :alias :v + :require true}} :error-fn u/cli-error-fn}) -(defn- find-pr-list [source-branch target-branch] +(defn- find-pr-list [head-ref] (let [pr (-> (p/sh "gh" "pr" "list" - "--head" (u/head-ref-name source-branch target-branch) + "--head" head-ref "--json" "number,headRefName") :out (json/parse-string true) @@ -31,13 +27,12 @@ (do (ice/p [:green "Found PR #" (:number pr)]) (:number pr)) (throw (ex-info - (str "No PR found for " (u/head-ref-name source-branch target-branch)) - {:source-branch source-branch - :target-branch target-branch + (str "No PR found for " head-ref) + {:head-ref head-ref :babashka/exit 1}))))) -(defn- find-pr-view [source-branch target-branch] - (let [pr-num (-> (p/sh "gh" "pr" "view" (u/head-ref-name source-branch target-branch) +(defn- find-pr-view [head-ref] + (let [pr-num (-> (p/sh "gh" "pr" "view" head-ref "--json" "number" "--jq" ".number") :out @@ -79,44 +74,42 @@ (resolve-conflicts-for-file file strat)))) (resolve-conflicts-for-file artifact strat))))))) -(defn- update-and-merge-pr [source-branch target-branch pr-number merge-strategy] - (let [head-ref-name (u/head-ref-name source-branch target-branch)] - - - (ice/p [:blue "Updating PR branch..."]) - (ice/p [:blue "Attempting merge with origin/master..."]) - (let [merge-result (p/sh {:continue true} "git" "merge" "origin/master")] - (when-not (zero? (:exit merge-result)) - (ice/p [:red "✗ Merge failed: " (:err merge-result)]) - (let [winner (if (= merge-strategy :ours) "PR" "master")] - (ice/p [:yellow "Attempting to resolve conflicts with git, preferring changes from " winner "..."]) - (resolve-conflicts (u/->artifacts target-branch) merge-strategy) - ;; Do the commit, now that we've resolved conflicts - (pr-str (p/sh "git" "commit" "--no-edit" "-m" - (str "Merge " target-branch " for PR #(" pr-number ")" - ", preferring changes from " winner))))) - - (ice/p [:blue "Pushing changes to PR branch..."]) - (ice/p "Result: " (pr-str (p/sh "git" "push" "origin" head-ref-name)))) - - ;; Wait a bit for GitHub to process to avoid a race condition - (Thread/sleep 5000) - - ;; Merge the PR - (ice/p [:blue "Merging PR #" pr-number "..."]) - (let [merge-result (p/sh {:continue true} - "gh" "pr" "merge" (str pr-number) - "--squash" "--delete-branch" - "--repo" "metabase/docs.metabase.github.io")] - (if (zero? (:exit merge-result)) - (ice/p [:green "✓ PR merged successfully!"]) - (ice/p [:red "✗ Merge failed: " [:bold (:err merge-result)]]))))) +(defn- update-and-merge-pr [head-ref-name versions pr-number merge-strategy] + (ice/p [:blue "Updating PR branch..."]) + (ice/p [:blue "Attempting merge with origin/master..."]) + (let [merge-result (p/sh {:continue true} "git" "merge" "origin/master")] + (when-not (zero? (:exit merge-result)) + (ice/p [:red "✗ Merge failed: " (:err merge-result)]) + (let [winner (if (= merge-strategy :ours) "PR" "master")] + (ice/p [:yellow "Attempting to resolve conflicts with git, preferring changes from " winner "..."]) + (resolve-conflicts (u/versions->artifacts versions) merge-strategy) + ;; Do the commit, now that we've resolved conflicts + (pr-str (p/sh "git" "commit" "--no-edit" "-m" + (str "Merge master into " head-ref-name " for PR #(" pr-number ")" + ", preferring changes from " winner))))) + + (ice/p [:blue "Pushing changes to PR branch..."]) + (ice/p "Result: " (pr-str (p/sh "git" "push" "origin" head-ref-name)))) + + ;; Wait a bit for GitHub to process to avoid a race condition + (Thread/sleep 5000) + + ;; Merge the PR + (ice/p [:blue "Merging PR #" pr-number "..."]) + (let [merge-result (p/sh {:continue true} + "gh" "pr" "merge" (str pr-number) + "--squash" "--delete-branch" + "--repo" "metabase/docs.metabase.github.io")] + (if (zero? (:exit merge-result)) + (ice/p [:green "✓ PR merged successfully!"]) + (ice/p [:red "✗ Merge failed: " [:bold (:err merge-result)]])))) (defn- should-pr-win? - "Determine if the current PR should win conflicts based on PR number comparison" - [current-pr-number target-branch] + "Determine if the current PR should win conflicts based on PR number comparison. + Compares against master, which is what these PRs are always based on." + [current-pr-number] (let [_ (p/sh "git" "fetch" "origin") - latest-master-commit (-> (p/sh "git" "log" "--oneline" "-1" (str "origin/" target-branch)) + latest-master-commit (-> (p/sh "git" "log" "--oneline" "-1" "origin/master") :out str/trim) ;; Extract PR number from commit message like "[auto] adding content to docs-rc-notes->master (#380)" @@ -165,9 +158,9 @@ (defn -main [& args] (println "Merge opertaion running at: " (str (java.time.Instant/now))) - (let [{:keys [source-branch target-branch]} (cli/parse-opts args cli-spec) - [source-branch target-branch] (mapv str/trim [source-branch target-branch]) - head-ref-name (u/head-ref-name source-branch target-branch)] + (let [{:keys [versions]} (cli/parse-opts args cli-spec) + versions (u/parse-versions versions) + head-ref-name (u/versions->head-ref-name versions)] ;; Ensure we're working with the latest remote state (ice/p [:blue "Fetching latest from origin..."]) (p/sh "git" "fetch" "origin") @@ -175,16 +168,16 @@ (let [current-branch (:out (p/sh "git" "branch" "--show-current")) _ (ice/p [:green "Currently on branch: " (str/trim current-branch)]) - pr-number-view (try (find-pr-view source-branch target-branch) + pr-number-view (try (find-pr-view head-ref-name) (catch Exception e (ice/p [:red "Error finding pr-number via view: " (ex-message e)]))) - pr-number-list (try (find-pr-list source-branch target-branch) + pr-number-list (try (find-pr-list head-ref-name) (catch Exception e (ice/p [:red "Error finding pr-number via list: " (ex-message e)]))) pr-number (or pr-number-view pr-number-list) - merge-strategy (if (should-pr-win? pr-number target-branch) :ours :theirs)] - (ice/p [:green "Merging PR #" pr-number ": " (u/head-ref-name source-branch target-branch) " | with strategy: " [:blue merge-strategy]]) - (update-and-merge-pr source-branch target-branch pr-number merge-strategy)))) + merge-strategy (if (should-pr-win? pr-number) :ours :theirs)] + (ice/p [:green "Merging PR #" pr-number ": " head-ref-name " | with strategy: " [:blue merge-strategy]]) + (update-and-merge-pr head-ref-name versions pr-number merge-strategy)))) (when (= *file* (System/getProperty "babashka.file")) (apply -main *command-line-args*)) diff --git a/script/update_or_create_pr.clj b/script/update_or_create_pr.clj index 33786d7548..1a79f3c76d 100644 --- a/script/update_or_create_pr.clj +++ b/script/update_or_create_pr.clj @@ -9,14 +9,10 @@ (def cli-spec {:spec - {:target-branch {:ref "" - :desc "The target branch of the triggering PR." - :alias :t - :require true} - :source-branch {:ref "" - :desc "The source branch of the triggering PR." - :alias :r - :require true} + {:versions {:ref "" + :desc "Comma separated major versions included in this build, eg. \"63,62\"." + :alias :v + :require true} :annotation {:ref "" :desc "The annotation to add to the PR." :default "auto-build"} @@ -24,27 +20,26 @@ :desc "The PR number to update, if it exists." :default nil} :update-dirs {:ref "" - :desc "The directories to update in the PR, smart defaults based on the target branch." + :desc "The directories to update in the PR, smart defaults based on the versions." :default ""}} :error-fn u/cli-error-fn}) -(defn existing-pr-num-by-source+target - "Checks if a PR already exists for the given target branch name." - [source target] +(defn existing-pr-num-by-head + "Checks if a PR already exists for the given head branch name." + [head-ref] (let [raw-data (p/sh {:out :string :continue true} "gh" "pr" "list" "--repo" "metabase/docs.metabase.github.io" "--json" "title,number,state,baseRefName,headRefName") - ;; _ (println "→ Curl data: " (pr-str curl-data)) pr-data (-> raw-data :out (json/parse-string true)) _ (println "→ PR data: " (pr-str pr-data)) - pr-info (first (filter #(= (:headRefName %) (u/head-ref-name source target)) - pr-data))] + pr-info (first (filter #(= (:headRefName %) head-ref) pr-data))] (println "→ PR info:" pr-info) (:number pr-info))) -(defn- report-pr-body [source-branch target-branch artifact-dirs pr-number] +(defn- report-pr-body [versions artifact-dirs pr-number] (str/join "\n" - [(str "`" source-branch "` -> `" target-branch "`") + [(str "Docs pulled from: " + (str/join ", " (map #(str "`release-x." % ".x`") versions))) "" "## Updated Directories:" (str/join "\n" (map #(str "- `" % "`") artifact-dirs)) @@ -57,62 +52,54 @@ (defn -main "Main function to update or create a PR. " [& args] - (let [{:keys [source-branch target-branch annotation pr-number update-dirs] + (let [{:keys [versions annotation pr-number update-dirs] :as opts} (cli/parse-opts args cli-spec) _ (when (or (:help opts) (:h opts)) (u/pp ["recieved options:" opts]) (u/show-usage-and-exit cli-spec)) - [category - release-num] (u/categorize-branchname target-branch) - _ (do (println "→ Target Branch info: " - (case category - :master "master" - :release (str "Release version:" release-num) - (throw (ex-info (str "Unpublishable branchname: " target-branch) - {:babashka/exit 1})))) - (println "→ Source Branch info: " source-branch)) + versions (u/parse-versions versions) + _ (println "→ Versions in this build: " (str/join ", " versions)) - target-branch-name (u/head-ref-name source-branch target-branch) - _ (p/shell "git" "checkout" "-B" target-branch-name) + head-ref (u/versions->head-ref-name versions) + _ (p/shell "git" "checkout" "-B" head-ref) update-dirs (remove str/blank? (str/split update-dirs #",")) _ (u/pp ["update-dirs" update-dirs]) - artifact-dirs (concat - update-dirs - (u/->artifacts category release-num)) - _ (doseq [ad artifact-dirs] - (println "Adding" ad "...") - (p/sh {:continue true} "git" "add" ad)) - {diff-exit :exit} (p/shell {:continue true} "git" "diff" "--cached" "--quiet") - target-branch-title (str "[" annotation "] " source-branch " -> " target-branch)] + artifact-dirs (concat + update-dirs + (u/versions->artifacts versions)) + _ (doseq [ad artifact-dirs] + (println "Adding" ad "...") + (p/sh {:continue true} "git" "add" ad)) + {diff-exit :exit} (p/shell {:continue true} "git" "diff" "--cached" "--quiet") + pr-title (str "[" annotation "] docs update: " + (str/join ", " (map #(str "v0." %) versions)))] (if (zero? diff-exit) (println "→ No changes to commit.") (do (println "→ Changes detected, committing...") - (p/shell "git" "commit" "-m" (str "[auto] adding content to " target-branch-name)) - (p/shell "git" "push" "--force" "origin" target-branch-name) - (println (str "→ Target Branch '" target-branch-name "' updated successfully.")) + (p/shell "git" "commit" "-m" (str "[auto] adding content to " head-ref)) + (p/shell "git" "push" "--force" "origin" head-ref) + (println (str "→ Branch '" head-ref "' updated successfully.")) (println "→ Checking for existing PR...") - (if-let [pr-info (existing-pr-num-by-source+target source-branch target-branch)] + (if-let [pr-info (existing-pr-num-by-head head-ref)] (println "✓ PR already exists: #" pr-info) (do (println "→ Creating new PR...") (let [args ["gh" "pr" "create" "--repo" "metabase/docs.metabase.github.io" - "--title" target-branch-title - "--body" (report-pr-body source-branch target-branch artifact-dirs pr-number) - "--head" target-branch-name]] + "--title" pr-title + "--body" (report-pr-body versions artifact-dirs pr-number) + "--head" head-ref]] (println "running: " (str/join " " args)) (apply p/shell {:continue true} args)))))) - (prn {:category category - :release release-num - :source-branch source-branch - :target-branch target-branch - :target-branch-title target-branch-title - :artifact-dirs artifact-dirs}))) + (prn {:versions versions + :head-ref head-ref + :pr-title pr-title + :artifact-dirs artifact-dirs}))) (when (= *file* (System/getProperty "babashka.file")) (apply -main *command-line-args*)) diff --git a/script/util.clj b/script/util.clj index 2913b86741..d8d4d95c35 100644 --- a/script/util.clj +++ b/script/util.clj @@ -84,5 +84,33 @@ :else []) (concat artifacts-to-include)))) -(defn head-ref-name [source-branch target-branch] - (str source-branch "->" target-branch)) +(defn parse-versions + "Parse a comma separated list of major versions (eg. \"63,62\") into a seq of + longs, sorted newest first and deduped. Throws on anything that isn't a + positive integer so a typo fails here instead of halfway through a build." + [s] + (let [parts (->> (str/split (str s) #",") + (map str/trim) + (remove str/blank?)) + nums (mapv (fn [part] + (let [n (parse-long part)] + (when-not (and n (pos? n)) + (throw (ex-info (str "Not a major version number: " (pr-str part)) + {:babashka/exit 1 :versions s}))) + n)) + parts)] + (when (empty? nums) + (throw (ex-info "No versions given, expected something like \"63,62\"" + {:babashka/exit 1 :versions s}))) + (vec (distinct (sort > nums))))) + +(defn versions->artifacts + "The union of artifact paths across every release number in `versions`." + [versions] + (into [] (distinct) (mapcat #(->artifacts :release %) versions))) + +(defn versions->head-ref-name + "The PR head branch for a docs update covering `versions`. Sorted newest first + so the name depends on the set of versions, not the order they were listed in." + [versions] + (str "docs-update-" (str/join "-" (map #(str "v" %) (sort > versions)))))