From b774a980a3db0ba999ae4fd21c9d8c3cc52a0ab7 Mon Sep 17 00:00:00 2001 From: Chris Smith Date: Fri, 22 May 2026 23:05:09 +0000 Subject: [PATCH 1/4] chore: add Rakefile as alternative CI tooling Offer a Rakefile as an alternative to the current Toys-based CI tooling to improve simplicity and maintainability in this relatively small repository. * Encountered a failure in `toys spec` in CI due to dependency resolution issues with the local gem (see #570). * Added Rakefile with equivalents for all Toys tasks. * Added a separate verification workflow `.github/workflows/rake_ci.yml` for Ruby 3.4. * Added `vendor/**/*` to RuboCop excludes to prevent inspecting gem dependencies in CI (which Toys handled implicitly). Closes #573 --- .github/workflows/rake_ci.yml | 33 +++++++++++ .rubocop.yml | 1 + .tool-versions | 2 + Gemfile | 2 + Rakefile | 105 ++++++++++++++++++++++++++++++++++ 5 files changed, 143 insertions(+) create mode 100644 .github/workflows/rake_ci.yml create mode 100644 .tool-versions create mode 100644 Rakefile diff --git a/.github/workflows/rake_ci.yml b/.github/workflows/rake_ci.yml new file mode 100644 index 00000000..d7f05f2f --- /dev/null +++ b/.github/workflows/rake_ci.yml @@ -0,0 +1,33 @@ +name: Rake CI +on: + pull_request: + branches: + - main + push: + branches: + - main + workflow_dispatch: + +jobs: + Rake-CI: + strategy: + matrix: + os: [ubuntu-latest] + ruby: ["3.4"] + fail-fast: false + runs-on: ${{ matrix.os }} + steps: + - name: Checkout repo + uses: actions/checkout@v4 + - name: Install Ruby ${{ matrix.ruby }} + uses: ruby/setup-ruby@v1 + with: + ruby-version: "${{ matrix.ruby }}" + bundler-cache: true + - name: Install NodeJS 18.x + uses: actions/setup-node@v4 + with: + node-version: "18.x" + - name: Run all Rake tasks + shell: bash + run: bundle exec rake ci diff --git a/.rubocop.yml b/.rubocop.yml index 4afcc011..31e706eb 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -6,6 +6,7 @@ AllCops: - "integration/**/*" - "spec/**/*" - "test/**/*" + - "vendor/**/*" Metrics/BlockLength: Exclude: - "googleauth.gemspec" diff --git a/.tool-versions b/.tool-versions new file mode 100644 index 00000000..e9ba0338 --- /dev/null +++ b/.tool-versions @@ -0,0 +1,2 @@ +ruby 3.4.9 +nodejs 25.9.0 diff --git a/Gemfile b/Gemfile index 9bc3ae2d..d23f87c3 100755 --- a/Gemfile +++ b/Gemfile @@ -11,8 +11,10 @@ gem "logging", "~> 2.0" gem "minitest", "~> 5.14" gem "minitest-focus", "~> 1.1" gem "rack-test", "~> 2.0" +gem "rake", "~> 13.0" gem "redcarpet", "~> 3.0" gem "redis", ">= 4.0", "< 6" gem "rspec", "~> 3.0" +gem "rubocop-rspec" gem "webmock", "~> 3.8" gem "yard", "~> 0.9" diff --git a/Rakefile b/Rakefile new file mode 100644 index 00000000..1d44b5a5 --- /dev/null +++ b/Rakefile @@ -0,0 +1,105 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require "rake/testtask" +require "rspec/core/rake_task" +require "rubocop/rake_task" +require "yard" +require "multi_json" + +# Helper to print files +def print_files pattern + files = Dir.glob pattern + puts "Matched #{files.size} files." + files.each { |f| puts " #{f}" } if Rake.application.options.trace +end + +Rake::TestTask.new :minitest_run do |t| + t.libs << "test" + t.pattern = "test/**/*_test.rb" + t.warning = true +end + +desc "Run Minitest tests" +task :test do + puts "--- Starting Minitest tests ---" + print_files "test/**/*_test.rb" + Rake::Task[:minitest_run].invoke + puts "--- Finished Minitest tests ---" +end + +RSpec::Core::RakeTask.new :rspec_run do |t| + t.rspec_opts = "-Ilib -Ispec" +end + +desc "Run RSpec specs" +task :spec do + puts "--- Starting RSpec specs ---" + print_files "spec/**/*_spec.rb" + Rake::Task[:rspec_run].invoke + puts "--- Finished RSpec specs ---" +end + +RuboCop::RakeTask.new :rubocop_run + +desc "Run RuboCop checks" +task :rubocop do + puts "--- Starting RuboCop checks ---" + Rake::Task[:rubocop_run].invoke + puts "--- Finished RuboCop checks ---" +end + +Rake::TestTask.new :integration_run do |t| + t.libs << "integration" + t.pattern = "integration/**/*_test.rb" + t.warning = true +end + +desc "Run integration tests" +task :integration do + puts "--- Starting integration tests ---" + print_files "integration/**/*_test.rb" + Rake::Task[:integration_run].invoke + puts "--- Finished integration tests ---" +end + +desc "Build the gem" +task :build do + puts "--- Starting gem build ---" + sh "gem build googleauth.gemspec" + puts "--- Finished gem build ---" +end + +YARD::Config.options[:generate_output_flag] = true +YARD::Rake::YardocTask.new :yardoc_run + +desc "Generate documentation" +task :yardoc do + puts "--- Starting documentation generation ---" + Rake::Task[:yardoc_run].invoke + puts "--- Finished documentation generation ---" +end + +desc "Run Link checks" +task linkinator: :yardoc do + puts "--- Starting link checks ---" + sh "npx -y linkinator ./doc --skip stackoverflow.com" + puts "--- Finished link checks ---" +end + +desc "Run all CI tasks" +task ci: [:test, :spec, :rubocop, :integration, :build, :yardoc, :linkinator] + +# Default task +task default: [:test, :spec, :rubocop] From 9203391ffbab16849778117d49a5b74ba0cd8db5 Mon Sep 17 00:00:00 2001 From: Chris Smith Date: Tue, 26 May 2026 22:24:30 +0000 Subject: [PATCH 2/4] chore: replace Toys with Rake in CI Replace Toys task runner with Rake for CI checks. Update .github/workflows/ci.yml to use Rake tasks and remove Ruby 3.1. Remove .toys directory and rake_ci.yml. Update .kokoro/samples.sh to use rake samples instead of toys. * Kept toys dependency in .kokoro/release.sh for performing releases, as it is part of Google's automated release infrastructure. * Kept toys dependency in samples/Gemfile and helper.rb for fetching sample_loader.rb. * Added rake samples task to Rakefile. * Invoke presubmit/samples with exec rake samples load_kokoro_context=true. * Implemented load_kokoro_env in Rakefile to load service account credentials in CI. * Set GOOGLE_APPLICATION_CREDENTIALS to path from secret manager. * Set GOOGLE_CLOUD_PROJECT from credentials if available. closes: #573 closes: #574 --- .github/workflows/ci.yml | 25 ++--- .github/workflows/rake_ci.yml | 33 ------- .kokoro/samples.sh | 5 +- .toys/.lib/repo_context.rb | 34 ------- .toys/.toys.rb | 48 ---------- .toys/ci.rb | 45 --------- .toys/linkinator.rb | 45 --------- .toys/release.rb | 24 ----- .toys/samples.rb | 36 -------- Rakefile | 92 +++++++++++++++++++ .../authenticate_implicit_with_adc_test.rb | 4 +- 11 files changed, 108 insertions(+), 283 deletions(-) delete mode 100644 .github/workflows/rake_ci.yml delete mode 100644 .toys/.lib/repo_context.rb delete mode 100644 .toys/.toys.rb delete mode 100644 .toys/ci.rb delete mode 100644 .toys/linkinator.rb delete mode 100644 .toys/release.rb delete mode 100644 .toys/samples.rb diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 898ff513..de5a565f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,27 +13,24 @@ jobs: strategy: matrix: include: - - os: ubuntu-22.04 - ruby: "3.1" - task: test , spec - os: ubuntu-22.04 ruby: "3.2" - task: test , spec + task: "test , spec" - os: ubuntu-latest ruby: "3.3" - task: test , spec + task: "test , spec" - os: ubuntu-latest ruby: "3.4" - task: test , spec + task: "test , spec" - os: macos-latest ruby: "3.4" - task: test , spec + task: "test , spec" - os: windows-latest ruby: "3.4" - task: test , spec + task: "test , spec" - os: ubuntu-latest ruby: "3.4" - task: rubocop , integration , build , yardoc , linkinator + task: "rubocop , integration , build , yardoc , linkinator" fail-fast: false runs-on: ${{ matrix.os }} steps: @@ -43,15 +40,13 @@ jobs: uses: ruby/setup-ruby@v1 with: ruby-version: "${{ matrix.ruby }}" + bundler-cache: true - name: Install NodeJS 18.x uses: actions/setup-node@v4 with: node-version: "18.x" - - name: Install dependencies + - name: Run Rake tasks shell: bash run: | - gem install --no-document toys - bundle install - - name: Test ${{ matrix.task }} - shell: bash - run: toys do ${{ matrix.task }} < /dev/null + CLEAN_TASK=$(echo "${{ matrix.task }}" | tr -d ',') + bundle exec rake $CLEAN_TASK diff --git a/.github/workflows/rake_ci.yml b/.github/workflows/rake_ci.yml deleted file mode 100644 index d7f05f2f..00000000 --- a/.github/workflows/rake_ci.yml +++ /dev/null @@ -1,33 +0,0 @@ -name: Rake CI -on: - pull_request: - branches: - - main - push: - branches: - - main - workflow_dispatch: - -jobs: - Rake-CI: - strategy: - matrix: - os: [ubuntu-latest] - ruby: ["3.4"] - fail-fast: false - runs-on: ${{ matrix.os }} - steps: - - name: Checkout repo - uses: actions/checkout@v4 - - name: Install Ruby ${{ matrix.ruby }} - uses: ruby/setup-ruby@v1 - with: - ruby-version: "${{ matrix.ruby }}" - bundler-cache: true - - name: Install NodeJS 18.x - uses: actions/setup-node@v4 - with: - node-version: "18.x" - - name: Run all Rake tasks - shell: bash - run: bundle exec rake ci diff --git a/.kokoro/samples.sh b/.kokoro/samples.sh index a8b79b98..dfda1b86 100755 --- a/.kokoro/samples.sh +++ b/.kokoro/samples.sh @@ -7,6 +7,7 @@ set -eo pipefail export GEM_HOME=$HOME/.gem export PATH=$GEM_HOME/bin:$PATH -gem install --no-document toys +bundle install +BUNDLE_GEMFILE=samples/Gemfile bundle install -toys samples < /dev/null +bundle exec rake samples load_kokoro_context=true diff --git a/.toys/.lib/repo_context.rb b/.toys/.lib/repo_context.rb deleted file mode 100644 index 9ffe6776..00000000 --- a/.toys/.lib/repo_context.rb +++ /dev/null @@ -1,34 +0,0 @@ -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -class RepoContext - @loaded_env = false - - def self.load_kokoro_env - return if @loaded_env - @loaded_env = true - - gfile_dir = ::ENV["KOKORO_GFILE_DIR"] - return unless gfile_dir - - filename = "#{gfile_dir}/ruby_env_vars.json" - raise "#{filename} is not a file" unless ::File.file? filename - env_vars = ::JSON.parse ::File.read filename - env_vars.each { |k, v| ::ENV[k] ||= v } - - filename = "#{gfile_dir}/secret_manager/ruby-main-ci-service-account" - raise "#{filename} is not a file" unless ::File.file? filename - ::ENV["GOOGLE_APPLICATION_CREDENTIALS"] = filename - end -end diff --git a/.toys/.toys.rb b/.toys/.toys.rb deleted file mode 100644 index 28395037..00000000 --- a/.toys/.toys.rb +++ /dev/null @@ -1,48 +0,0 @@ -# frozen_string_literal: true - -# Copyright 2021 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -expand :clean, paths: :gitignore - -expand :rspec do |t| - t.libs = ["lib", "spec"] - t.use_bundler -end - -expand :minitest do |t| - t.libs = ["lib", "test"] - t.use_bundler - t.files = "test/**/*_test.rb" -end - -expand :minitest do |t| - t.name = "integration" - t.libs = ["lib", "integration"] - t.use_bundler - t.files = "integration/**/*_test.rb" -end - -expand :rubocop, bundler: true - -expand :yardoc do |t| - t.generate_output_flag = true - # t.fail_on_warning = true - t.use_bundler -end -alias_tool :yard, :yardoc - -expand :gem_build - -expand :gem_build, name: "install", install_gem: true diff --git a/.toys/ci.rb b/.toys/ci.rb deleted file mode 100644 index 2f7263fa..00000000 --- a/.toys/ci.rb +++ /dev/null @@ -1,45 +0,0 @@ -# frozen_string_literal: true - -# Copyright 2021 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -desc "Run CI checks" - -TESTS = ["test", "integration", "spec", "rubocop", "yardoc", "build", "linkinator"] - -flag :only -TESTS.each do |name| - flag "include_#{name}".to_sym, "--[no-]include-#{name}" -end - -include :exec, result_callback: :handle_result -include :terminal - -def handle_result result - if result.success? - puts "** #{result.name} passed\n\n", :green, :bold - else - puts "** CI terminated: #{result.name} failed!", :red, :bold - exit 1 - end -end - -def run - ::Dir.chdir context_directory - TESTS.each do |name| - setting = get "include_#{name}".to_sym - setting = !only if setting.nil? - exec ["toys", name], name: name.capitalize if setting - end -end diff --git a/.toys/linkinator.rb b/.toys/linkinator.rb deleted file mode 100644 index 66b06c0c..00000000 --- a/.toys/linkinator.rb +++ /dev/null @@ -1,45 +0,0 @@ -# frozen_string_literal: true - -# Copyright 2021 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -desc "Run Link checks" - -flag :install, desc: "Install linkinator instead of running checks" - -include :exec, e: true -include :terminal - -def run - ::Dir.chdir context_directory - if install - Kernel.exec "npm install linkinator" - else - exec_tool ["yardoc"] - check_links - end -end - -def check_links - result = exec ["npx", "linkinator", "./doc", "--skip", "stackoverflow.com"], out: :capture - puts result.captured_out - checked_links = result.captured_out.split "\n" - checked_links.select! { |link| link =~ /^\[(\d+)\]/ && ::Regexp.last_match[1] != "200" } - unless checked_links.empty? - checked_links.each do |link| - puts link, :yellow - end - exit 1 - end -end diff --git a/.toys/release.rb b/.toys/release.rb deleted file mode 100644 index c2679e4f..00000000 --- a/.toys/release.rb +++ /dev/null @@ -1,24 +0,0 @@ -# frozen_string_literal: true - -# Copyright 2021 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -if ENV["RUBY_COMMON_TOOLS"] - common_tools_dir = File.expand_path ENV["RUBY_COMMON_TOOLS"] - load File.join(common_tools_dir, "toys", "release") -else - load_git remote: "https://github.com/googleapis/ruby-common-tools.git", - path: "toys/release", - update: true -end diff --git a/.toys/samples.rb b/.toys/samples.rb deleted file mode 100644 index dbee686e..00000000 --- a/.toys/samples.rb +++ /dev/null @@ -1,36 +0,0 @@ -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -expand :minitest do |t| - t.name = "test" - t.libs = ["lib", "samples"] - t.use_bundler on_missing: :install, gemfile_path: "samples/Gemfile" - t.files = "samples/acceptance/*_test.rb" -end - -desc "Run samples tests" - -include :exec -include :terminal, styled: true - -def run - require "json" - require "repo_context" - RepoContext.load_kokoro_env - - Dir.chdir context_directory - - puts "Samples tests ...", :bold, :cyan - exec_tool ["samples", "test"], name: "Samples tests" -end diff --git a/Rakefile b/Rakefile index 1d44b5a5..b81e41ea 100644 --- a/Rakefile +++ b/Rakefile @@ -98,6 +98,98 @@ task linkinator: :yardoc do puts "--- Finished link checks ---" end +# @private +# Guard variable to ensure Kokoro environment is only loaded once. +@loaded_kokoro_env = false + +## +# Loads environment variables and service account credentials from the Kokoro +# environment. +# +# This method replicates the mechanics used in `google-cloud-ruby` via Toys +# to prepare the environment for integration and samples tests in CI. +# +# It expects the following resources to be present in the directory pointed to +# by the `KOKORO_GFILE_DIR` environment variable: +# +# 1. `ruby_env_vars.json`: A JSON file containing a flat hash of key-value +# pairs representing environment variables. This file is REQUIRED if +# `KOKORO_GFILE_DIR` is set, as it is the primary mechanism for passing +# environment config in Kokoro. +# 2. `secret_manager/ruby-main-ci-service-account`: A file containing the +# JSON key for the service account used for testing. This file is optional, +# but required for tests that need live cloud access. +# +# @raise [RuntimeError] if `KOKORO_GFILE_DIR` is set but `ruby_env_vars.json` +# is missing. +# +# @example Usage in Rake task: +# # To trigger this via command line: +# # bundle exec rake samples load_kokoro_context=true +# if ENV["load_kokoro_context"] == "true" +# load_kokoro_env +# end +# +def load_kokoro_env + return if @loaded_kokoro_env + @loaded_kokoro_env = true + + gfile_dir = ENV["KOKORO_GFILE_DIR"] + return unless gfile_dir + + load_ruby_env_vars gfile_dir + load_sa_credentials gfile_dir +end + +# @private +def load_ruby_env_vars gfile_dir + env_vars_file = File.join gfile_dir, "ruby_env_vars.json" + unless File.file? env_vars_file + raise "Kokoro environment file missing: #{env_vars_file}. " \ + "Expected to be populated by populate-secrets.sh." + end + + puts "Loading environment variables from #{env_vars_file}" + require "json" + env_vars = JSON.parse File.read env_vars_file + env_vars.each { |k, v| ENV[k] ||= v } +end + +# @private +def load_sa_credentials gfile_dir + keyfile = File.join gfile_dir, "secret_manager", "ruby-main-ci-service-account" + if File.file? keyfile + ENV["GOOGLE_APPLICATION_CREDENTIALS"] = keyfile + + # Extract project_id from the key file if available + require "json" + key_data = JSON.parse File.read keyfile + if key_data["project_id"] + ENV["GOOGLE_CLOUD_PROJECT"] ||= key_data["project_id"] + end + + ENV["GCLOUD_TEST_KEYFILE_JSON"] = File.read keyfile + else + puts "Warning: Secret file not found at #{keyfile}." + puts "Falling back to ambient credentials (e.g., default Compute Engine service account)." + end +end + +desc "Run samples tests. Usage: bundle exec rake samples load_kokoro_context=true" +task :samples do + puts "--- Starting samples tests ---" + + # Load Kokoro environment if requested via command line override + if ENV["load_kokoro_context"] == "true" + load_kokoro_env + end + + cmd = "BUNDLE_GEMFILE=samples/Gemfile bundle exec ruby -Ilib -Isamples -e " \ + "'Dir.glob(\"samples/acceptance/*_test.rb\").each{|f| require File.expand_path(f)}'" + sh cmd + puts "--- Finished samples tests ---" +end + desc "Run all CI tasks" task ci: [:test, :spec, :rubocop, :integration, :build, :yardoc, :linkinator] diff --git a/samples/acceptance/authenticate_implicit_with_adc_test.rb b/samples/acceptance/authenticate_implicit_with_adc_test.rb index e67dc193..437b151b 100644 --- a/samples/acceptance/authenticate_implicit_with_adc_test.rb +++ b/samples/acceptance/authenticate_implicit_with_adc_test.rb @@ -23,8 +23,10 @@ # list_buckets sample = SampleLoader.load "authenticate_implicit_with_adc.rb" + test_project = ENV["GOOGLE_CLOUD_PROJECT"] || storage_client.project + assert_output(/Plaintext: Listed all storage buckets./) do - sample.run project_id: storage_client.project + sample.run project_id: test_project end end end From f6f722c20b52031e1dbd0140840b6bcb6732434a Mon Sep 17 00:00:00 2001 From: Chris Smith Date: Fri, 29 May 2026 16:22:19 +0000 Subject: [PATCH 3/4] chore: revert Rake migration and apply targeted Toys fix Revert the migration to Rake for CI tasks to maintain consistency with the upcoming monorepo migration and release pipeline. Apply targeted fixes in Toys instead: * Restore `.toys` directory from `upstream/main`. * Replace `expand :rspec` and `expand :minitest` in `.toys/.toys.rb` with custom tools that call `bundle exec` directly. * This preserves the Minitest and RSpec execution experience while avoiding the Toys Gemfile bug that caused `GemNotFound` errors in clean CI environments. * Delete `Rakefile`. * Restore `.github/workflows/ci.yml` and remove Ruby 3.1 again. * Revert changes to `Gemfile` and `.rubocop.yml`. * Restore `.kokoro/samples.sh`. --- .github/workflows/ci.yml | 22 +++-- .kokoro/samples.sh | 5 +- .rubocop.yml | 1 - .toys/.lib/repo_context.rb | 34 +++++++ .toys/.toys.rb | 54 ++++++++++ .toys/ci.rb | 45 +++++++++ .toys/linkinator.rb | 45 +++++++++ .toys/release.rb | 24 +++++ .toys/samples.rb | 36 +++++++ Gemfile | 2 - Rakefile | 197 ------------------------------------- 11 files changed, 252 insertions(+), 213 deletions(-) create mode 100644 .toys/.lib/repo_context.rb create mode 100644 .toys/.toys.rb create mode 100644 .toys/ci.rb create mode 100644 .toys/linkinator.rb create mode 100644 .toys/release.rb create mode 100644 .toys/samples.rb delete mode 100644 Rakefile diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index de5a565f..0ad0c501 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,22 +15,22 @@ jobs: include: - os: ubuntu-22.04 ruby: "3.2" - task: "test , spec" + task: test , spec - os: ubuntu-latest ruby: "3.3" - task: "test , spec" + task: test , spec - os: ubuntu-latest ruby: "3.4" - task: "test , spec" + task: test , spec - os: macos-latest ruby: "3.4" - task: "test , spec" + task: test , spec - os: windows-latest ruby: "3.4" - task: "test , spec" + task: test , spec - os: ubuntu-latest ruby: "3.4" - task: "rubocop , integration , build , yardoc , linkinator" + task: rubocop , integration , build , yardoc , linkinator fail-fast: false runs-on: ${{ matrix.os }} steps: @@ -40,13 +40,15 @@ jobs: uses: ruby/setup-ruby@v1 with: ruby-version: "${{ matrix.ruby }}" - bundler-cache: true - name: Install NodeJS 18.x uses: actions/setup-node@v4 with: node-version: "18.x" - - name: Run Rake tasks + - name: Install dependencies shell: bash run: | - CLEAN_TASK=$(echo "${{ matrix.task }}" | tr -d ',') - bundle exec rake $CLEAN_TASK + gem install --no-document toys + bundle install + - name: Test ${{ matrix.task }} + shell: bash + run: toys do ${{ matrix.task }} < /dev/null diff --git a/.kokoro/samples.sh b/.kokoro/samples.sh index dfda1b86..a8b79b98 100755 --- a/.kokoro/samples.sh +++ b/.kokoro/samples.sh @@ -7,7 +7,6 @@ set -eo pipefail export GEM_HOME=$HOME/.gem export PATH=$GEM_HOME/bin:$PATH -bundle install -BUNDLE_GEMFILE=samples/Gemfile bundle install +gem install --no-document toys -bundle exec rake samples load_kokoro_context=true +toys samples < /dev/null diff --git a/.rubocop.yml b/.rubocop.yml index 31e706eb..4afcc011 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -6,7 +6,6 @@ AllCops: - "integration/**/*" - "spec/**/*" - "test/**/*" - - "vendor/**/*" Metrics/BlockLength: Exclude: - "googleauth.gemspec" diff --git a/.toys/.lib/repo_context.rb b/.toys/.lib/repo_context.rb new file mode 100644 index 00000000..9ffe6776 --- /dev/null +++ b/.toys/.lib/repo_context.rb @@ -0,0 +1,34 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +class RepoContext + @loaded_env = false + + def self.load_kokoro_env + return if @loaded_env + @loaded_env = true + + gfile_dir = ::ENV["KOKORO_GFILE_DIR"] + return unless gfile_dir + + filename = "#{gfile_dir}/ruby_env_vars.json" + raise "#{filename} is not a file" unless ::File.file? filename + env_vars = ::JSON.parse ::File.read filename + env_vars.each { |k, v| ::ENV[k] ||= v } + + filename = "#{gfile_dir}/secret_manager/ruby-main-ci-service-account" + raise "#{filename} is not a file" unless ::File.file? filename + ::ENV["GOOGLE_APPLICATION_CREDENTIALS"] = filename + end +end diff --git a/.toys/.toys.rb b/.toys/.toys.rb new file mode 100644 index 00000000..8e60c3ad --- /dev/null +++ b/.toys/.toys.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true + +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +expand :clean, paths: :gitignore + +tool "spec" do + desc "Run RSpec tests" + include :exec + def run + exec ["bundle", "exec", "rspec"] + end +end + +tool "test" do + desc "Run unit tests" + include :exec + def run + exec ["bundle", "exec", "ruby", "-Ilib", "-Itest", "-e", "Dir.glob('test/**/*_test.rb').each{|f| require File.expand_path(f)}"] + end +end + +tool "integration" do + desc "Run integration tests" + include :exec + def run + exec ["bundle", "exec", "ruby", "-Ilib", "-Iintegration", "-e", "Dir.glob('integration/**/*_test.rb').each{|f| require File.expand_path(f)}"] + end +end + +expand :rubocop, bundler: true + +expand :yardoc do |t| + t.generate_output_flag = true + # t.fail_on_warning = true + t.use_bundler +end +alias_tool :yard, :yardoc + +expand :gem_build + +expand :gem_build, name: "install", install_gem: true diff --git a/.toys/ci.rb b/.toys/ci.rb new file mode 100644 index 00000000..2f7263fa --- /dev/null +++ b/.toys/ci.rb @@ -0,0 +1,45 @@ +# frozen_string_literal: true + +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +desc "Run CI checks" + +TESTS = ["test", "integration", "spec", "rubocop", "yardoc", "build", "linkinator"] + +flag :only +TESTS.each do |name| + flag "include_#{name}".to_sym, "--[no-]include-#{name}" +end + +include :exec, result_callback: :handle_result +include :terminal + +def handle_result result + if result.success? + puts "** #{result.name} passed\n\n", :green, :bold + else + puts "** CI terminated: #{result.name} failed!", :red, :bold + exit 1 + end +end + +def run + ::Dir.chdir context_directory + TESTS.each do |name| + setting = get "include_#{name}".to_sym + setting = !only if setting.nil? + exec ["toys", name], name: name.capitalize if setting + end +end diff --git a/.toys/linkinator.rb b/.toys/linkinator.rb new file mode 100644 index 00000000..66b06c0c --- /dev/null +++ b/.toys/linkinator.rb @@ -0,0 +1,45 @@ +# frozen_string_literal: true + +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +desc "Run Link checks" + +flag :install, desc: "Install linkinator instead of running checks" + +include :exec, e: true +include :terminal + +def run + ::Dir.chdir context_directory + if install + Kernel.exec "npm install linkinator" + else + exec_tool ["yardoc"] + check_links + end +end + +def check_links + result = exec ["npx", "linkinator", "./doc", "--skip", "stackoverflow.com"], out: :capture + puts result.captured_out + checked_links = result.captured_out.split "\n" + checked_links.select! { |link| link =~ /^\[(\d+)\]/ && ::Regexp.last_match[1] != "200" } + unless checked_links.empty? + checked_links.each do |link| + puts link, :yellow + end + exit 1 + end +end diff --git a/.toys/release.rb b/.toys/release.rb new file mode 100644 index 00000000..c2679e4f --- /dev/null +++ b/.toys/release.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if ENV["RUBY_COMMON_TOOLS"] + common_tools_dir = File.expand_path ENV["RUBY_COMMON_TOOLS"] + load File.join(common_tools_dir, "toys", "release") +else + load_git remote: "https://github.com/googleapis/ruby-common-tools.git", + path: "toys/release", + update: true +end diff --git a/.toys/samples.rb b/.toys/samples.rb new file mode 100644 index 00000000..dbee686e --- /dev/null +++ b/.toys/samples.rb @@ -0,0 +1,36 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +expand :minitest do |t| + t.name = "test" + t.libs = ["lib", "samples"] + t.use_bundler on_missing: :install, gemfile_path: "samples/Gemfile" + t.files = "samples/acceptance/*_test.rb" +end + +desc "Run samples tests" + +include :exec +include :terminal, styled: true + +def run + require "json" + require "repo_context" + RepoContext.load_kokoro_env + + Dir.chdir context_directory + + puts "Samples tests ...", :bold, :cyan + exec_tool ["samples", "test"], name: "Samples tests" +end diff --git a/Gemfile b/Gemfile index d23f87c3..9bc3ae2d 100755 --- a/Gemfile +++ b/Gemfile @@ -11,10 +11,8 @@ gem "logging", "~> 2.0" gem "minitest", "~> 5.14" gem "minitest-focus", "~> 1.1" gem "rack-test", "~> 2.0" -gem "rake", "~> 13.0" gem "redcarpet", "~> 3.0" gem "redis", ">= 4.0", "< 6" gem "rspec", "~> 3.0" -gem "rubocop-rspec" gem "webmock", "~> 3.8" gem "yard", "~> 0.9" diff --git a/Rakefile b/Rakefile deleted file mode 100644 index b81e41ea..00000000 --- a/Rakefile +++ /dev/null @@ -1,197 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -require "rake/testtask" -require "rspec/core/rake_task" -require "rubocop/rake_task" -require "yard" -require "multi_json" - -# Helper to print files -def print_files pattern - files = Dir.glob pattern - puts "Matched #{files.size} files." - files.each { |f| puts " #{f}" } if Rake.application.options.trace -end - -Rake::TestTask.new :minitest_run do |t| - t.libs << "test" - t.pattern = "test/**/*_test.rb" - t.warning = true -end - -desc "Run Minitest tests" -task :test do - puts "--- Starting Minitest tests ---" - print_files "test/**/*_test.rb" - Rake::Task[:minitest_run].invoke - puts "--- Finished Minitest tests ---" -end - -RSpec::Core::RakeTask.new :rspec_run do |t| - t.rspec_opts = "-Ilib -Ispec" -end - -desc "Run RSpec specs" -task :spec do - puts "--- Starting RSpec specs ---" - print_files "spec/**/*_spec.rb" - Rake::Task[:rspec_run].invoke - puts "--- Finished RSpec specs ---" -end - -RuboCop::RakeTask.new :rubocop_run - -desc "Run RuboCop checks" -task :rubocop do - puts "--- Starting RuboCop checks ---" - Rake::Task[:rubocop_run].invoke - puts "--- Finished RuboCop checks ---" -end - -Rake::TestTask.new :integration_run do |t| - t.libs << "integration" - t.pattern = "integration/**/*_test.rb" - t.warning = true -end - -desc "Run integration tests" -task :integration do - puts "--- Starting integration tests ---" - print_files "integration/**/*_test.rb" - Rake::Task[:integration_run].invoke - puts "--- Finished integration tests ---" -end - -desc "Build the gem" -task :build do - puts "--- Starting gem build ---" - sh "gem build googleauth.gemspec" - puts "--- Finished gem build ---" -end - -YARD::Config.options[:generate_output_flag] = true -YARD::Rake::YardocTask.new :yardoc_run - -desc "Generate documentation" -task :yardoc do - puts "--- Starting documentation generation ---" - Rake::Task[:yardoc_run].invoke - puts "--- Finished documentation generation ---" -end - -desc "Run Link checks" -task linkinator: :yardoc do - puts "--- Starting link checks ---" - sh "npx -y linkinator ./doc --skip stackoverflow.com" - puts "--- Finished link checks ---" -end - -# @private -# Guard variable to ensure Kokoro environment is only loaded once. -@loaded_kokoro_env = false - -## -# Loads environment variables and service account credentials from the Kokoro -# environment. -# -# This method replicates the mechanics used in `google-cloud-ruby` via Toys -# to prepare the environment for integration and samples tests in CI. -# -# It expects the following resources to be present in the directory pointed to -# by the `KOKORO_GFILE_DIR` environment variable: -# -# 1. `ruby_env_vars.json`: A JSON file containing a flat hash of key-value -# pairs representing environment variables. This file is REQUIRED if -# `KOKORO_GFILE_DIR` is set, as it is the primary mechanism for passing -# environment config in Kokoro. -# 2. `secret_manager/ruby-main-ci-service-account`: A file containing the -# JSON key for the service account used for testing. This file is optional, -# but required for tests that need live cloud access. -# -# @raise [RuntimeError] if `KOKORO_GFILE_DIR` is set but `ruby_env_vars.json` -# is missing. -# -# @example Usage in Rake task: -# # To trigger this via command line: -# # bundle exec rake samples load_kokoro_context=true -# if ENV["load_kokoro_context"] == "true" -# load_kokoro_env -# end -# -def load_kokoro_env - return if @loaded_kokoro_env - @loaded_kokoro_env = true - - gfile_dir = ENV["KOKORO_GFILE_DIR"] - return unless gfile_dir - - load_ruby_env_vars gfile_dir - load_sa_credentials gfile_dir -end - -# @private -def load_ruby_env_vars gfile_dir - env_vars_file = File.join gfile_dir, "ruby_env_vars.json" - unless File.file? env_vars_file - raise "Kokoro environment file missing: #{env_vars_file}. " \ - "Expected to be populated by populate-secrets.sh." - end - - puts "Loading environment variables from #{env_vars_file}" - require "json" - env_vars = JSON.parse File.read env_vars_file - env_vars.each { |k, v| ENV[k] ||= v } -end - -# @private -def load_sa_credentials gfile_dir - keyfile = File.join gfile_dir, "secret_manager", "ruby-main-ci-service-account" - if File.file? keyfile - ENV["GOOGLE_APPLICATION_CREDENTIALS"] = keyfile - - # Extract project_id from the key file if available - require "json" - key_data = JSON.parse File.read keyfile - if key_data["project_id"] - ENV["GOOGLE_CLOUD_PROJECT"] ||= key_data["project_id"] - end - - ENV["GCLOUD_TEST_KEYFILE_JSON"] = File.read keyfile - else - puts "Warning: Secret file not found at #{keyfile}." - puts "Falling back to ambient credentials (e.g., default Compute Engine service account)." - end -end - -desc "Run samples tests. Usage: bundle exec rake samples load_kokoro_context=true" -task :samples do - puts "--- Starting samples tests ---" - - # Load Kokoro environment if requested via command line override - if ENV["load_kokoro_context"] == "true" - load_kokoro_env - end - - cmd = "BUNDLE_GEMFILE=samples/Gemfile bundle exec ruby -Ilib -Isamples -e " \ - "'Dir.glob(\"samples/acceptance/*_test.rb\").each{|f| require File.expand_path(f)}'" - sh cmd - puts "--- Finished samples tests ---" -end - -desc "Run all CI tasks" -task ci: [:test, :spec, :rubocop, :integration, :build, :yardoc, :linkinator] - -# Default task -task default: [:test, :spec, :rubocop] From 75ee93593abc3fea48dfa7098d2d94ce891627ab Mon Sep 17 00:00:00 2001 From: Chris Smith Date: Fri, 29 May 2026 16:57:43 +0000 Subject: [PATCH 4/4] chore: revert Rake migration and apply targeted Toys fix Revert the migration to Rake for CI tasks to maintain consistency with the upcoming monorepo migration and release pipeline. Apply targeted fixes in Toys instead: * Restore `.toys` directory from `upstream/main`. * Replace `expand :rspec` and `expand :minitest` in `.toys/.toys.rb` with custom tools that call `bundle exec` directly. * This preserves the Minitest and RSpec execution experience while avoiding the Toys Gemfile bug that caused `GemNotFound` errors in clean CI environments. * Delete `Rakefile`. * Restore `.github/workflows/ci.yml` and remove Ruby 3.1 again. * Revert changes to `Gemfile` and `.rubocop.yml`. * Restore `.kokoro/samples.sh`. * Download `sample_loader.rb` explicitly in `.kokoro/samples.sh` to fix `LoadError` for `toys/utils/git_cache`. * Convert `.toys/samples.rb` to a self-contained custom tool that runs `bundle exec ruby` in the `samples` directory to fix `LoadError` for `google/cloud/storage`. * Extract `project_id` from service account key in `.toys/.lib/repo_context.rb` to automatically set `GOOGLE_CLOUD_PROJECT`. --- .kokoro/samples.sh | 14 ++++++++++++++ .toys/.lib/repo_context.rb | 8 ++++++++ .toys/.toys.rb | 10 ++++++---- .toys/samples.rb | 21 ++++++++++----------- samples/acceptance/helper.rb | 4 +--- 5 files changed, 39 insertions(+), 18 deletions(-) diff --git a/.kokoro/samples.sh b/.kokoro/samples.sh index a8b79b98..50ae3929 100755 --- a/.kokoro/samples.sh +++ b/.kokoro/samples.sh @@ -9,4 +9,18 @@ export PATH=$GEM_HOME/bin:$PATH gem install --no-document toys +# To run acceptance tests for samples, we need the `sample_loader.rb` helper +# from the `googleapis/ruby-common-tools` repository. +# +# Previously, this file was downloaded dynamically at runtime in `helper.rb` +# using `Toys::Utils::GitCache`. However, that approach creates a dependency +# on Toys internal utilities during test execution, which fails when running +# within an isolated bundle (e.g., via Bundler). +# +# To make the tests more robust and independent of the task runner's internal +# state, we explicitly download the helper file here before executing the tests. +# This does not change the testing infrastructure; it only makes the download +# step explicit and reliable. +curl -sSL https://raw.githubusercontent.com/googleapis/ruby-common-tools/main/lib/sample_loader.rb -o samples/acceptance/sample_loader.rb + toys samples < /dev/null diff --git a/.toys/.lib/repo_context.rb b/.toys/.lib/repo_context.rb index 9ffe6776..bde035e6 100644 --- a/.toys/.lib/repo_context.rb +++ b/.toys/.lib/repo_context.rb @@ -30,5 +30,13 @@ def self.load_kokoro_env filename = "#{gfile_dir}/secret_manager/ruby-main-ci-service-account" raise "#{filename} is not a file" unless ::File.file? filename ::ENV["GOOGLE_APPLICATION_CREDENTIALS"] = filename + + # Extract project_id from the key file if available + require "json" + key_data = JSON.parse File.read filename + if key_data["project_id"] + puts "Inferred project_id from keyfile: #{key_data['project_id']}" + ::ENV["GOOGLE_CLOUD_PROJECT"] ||= key_data["project_id"] + end end end diff --git a/.toys/.toys.rb b/.toys/.toys.rb index 8e60c3ad..9f76ff2a 100644 --- a/.toys/.toys.rb +++ b/.toys/.toys.rb @@ -42,10 +42,12 @@ def run expand :rubocop, bundler: true -expand :yardoc do |t| - t.generate_output_flag = true - # t.fail_on_warning = true - t.use_bundler +tool "yardoc" do + desc "Generate documentation" + include :exec + def run + exec ["bundle", "exec", "yard", "doc"] + end end alias_tool :yard, :yardoc diff --git a/.toys/samples.rb b/.toys/samples.rb index dbee686e..bb0fa013 100644 --- a/.toys/samples.rb +++ b/.toys/samples.rb @@ -12,13 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -expand :minitest do |t| - t.name = "test" - t.libs = ["lib", "samples"] - t.use_bundler on_missing: :install, gemfile_path: "samples/Gemfile" - t.files = "samples/acceptance/*_test.rb" -end - desc "Run samples tests" include :exec @@ -29,8 +22,14 @@ def run require "repo_context" RepoContext.load_kokoro_env - Dir.chdir context_directory - - puts "Samples tests ...", :bold, :cyan - exec_tool ["samples", "test"], name: "Samples tests" + Dir.chdir File.join(context_directory, "samples") do + puts "Updating samples bundle ...", :bold, :cyan + exec ["bundle", "install"] + + puts "Samples tests ...", :bold, :cyan + cmd = ["bundle", "exec", "ruby", "-I../lib", "-Iacceptance", "-e", + "Dir.glob('acceptance/*_test.rb').each{|f| require File.expand_path(f)}"] + + exec cmd + end end diff --git a/samples/acceptance/helper.rb b/samples/acceptance/helper.rb index 443d7a55..486341ac 100644 --- a/samples/acceptance/helper.rb +++ b/samples/acceptance/helper.rb @@ -14,6 +14,4 @@ require "minitest/autorun" require "minitest/focus" -require "toys/utils/git_cache" -require Toys::Utils::GitCache.new.get "https://github.com/googleapis/ruby-common-tools.git", - path: "lib/sample_loader.rb", update: 300 +require_relative "sample_loader"