From e49795bc436d73e0bf647e3d99802ad24630d3ac Mon Sep 17 00:00:00 2001 From: Hiroshi SHIBATA Date: Wed, 10 Jun 2026 14:14:00 +0900 Subject: [PATCH 01/19] Add Gem::CompactIndexClient::CacheFile First piece of a RubyGems-side compact index client, ported from Bundler::CompactIndexClient. The two implementations are intentionally kept separate so that gem commands can adopt the compact index without touching Bundler. Co-Authored-By: Claude Fable 5 --- lib/rubygems/compact_index_client.rb | 19 +++ .../compact_index_client/cache_file.rb | 141 ++++++++++++++++++ ...est_gem_compact_index_client_cache_file.rb | 135 +++++++++++++++++ 3 files changed, 295 insertions(+) create mode 100644 lib/rubygems/compact_index_client.rb create mode 100644 lib/rubygems/compact_index_client/cache_file.rb create mode 100644 test/rubygems/test_gem_compact_index_client_cache_file.rb diff --git a/lib/rubygems/compact_index_client.rb b/lib/rubygems/compact_index_client.rb new file mode 100644 index 000000000000..7de0d620add4 --- /dev/null +++ b/lib/rubygems/compact_index_client.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +## +# The CompactIndexClient fetches and parses the compact index files +# (names, versions and info/[gem]) served by a gem server, keeping a +# local cache so subsequent fetches only transfer what changed. +# +# This is an independent RubyGems port of Bundler::CompactIndexClient. +# Both implementations are intentionally kept separate so that changes +# on either side cannot affect the other; this one only depends on +# RubyGems itself. + +class Gem::CompactIndexClient + SUPPORTED_DIGESTS = { "sha-256" => :SHA256 }.freeze + + class Error < StandardError; end + + require_relative "compact_index_client/cache_file" +end diff --git a/lib/rubygems/compact_index_client/cache_file.rb b/lib/rubygems/compact_index_client/cache_file.rb new file mode 100644 index 000000000000..3656645d8220 --- /dev/null +++ b/lib/rubygems/compact_index_client/cache_file.rb @@ -0,0 +1,141 @@ +# frozen_string_literal: true + +require "digest" +require "fileutils" +require_relative "../package" + +class Gem::CompactIndexClient + # write cache files in a way that is robust to concurrent modifications + # if digests are given, the checksums will be verified + class CacheFile + DEFAULT_FILE_MODE = 0o644 + private_constant :DEFAULT_FILE_MODE + + class Error < RuntimeError; end + class ClosedError < Error; end + + class DigestMismatchError < Error + def initialize(digests, expected_digests) + super "Calculated checksums #{digests.inspect} did not match expected #{expected_digests.inspect}." + end + end + + # Initialize with a copy of the original file, then yield the instance. + def self.copy(path, &block) + new(path) do |file| + file.initialize_digests + + path.open("rb") do |s| + file.open {|f| IO.copy_stream(s, f) } + end + + yield file + end + end + + # Write data to a temp file, then replace the original file with it verifying the digests if given. + def self.write(path, data, digests = nil) + return unless data + new(path) do |file| + file.digests = digests + file.write(data) + end + end + + attr_reader :original_path, :path + + def initialize(original_path, &block) + @original_path = original_path + @perm = original_path.file? ? original_path.stat.mode : DEFAULT_FILE_MODE + @path = original_path.sub(/$/, ".#{$$}.tmp") + return unless block_given? + begin + yield self + ensure + close + end + end + + def size + path.size + end + + # initialize the digests using CompactIndexClient::SUPPORTED_DIGESTS, or a subset based on keys. + def initialize_digests(keys = nil) + @digests = keys ? SUPPORTED_DIGESTS.slice(*keys) : SUPPORTED_DIGESTS.dup + @digests.transform_values! {|algo_class| Digest(algo_class).new } + end + + # reset the digests so they don't contain any previously read data + def reset_digests + @digests&.each_value(&:reset) + end + + # set the digests that will be verified at the end + def digests=(expected_digests) + @expected_digests = expected_digests + + if @expected_digests.nil? + @digests = nil + elsif @digests + @digests = @digests.slice(*@expected_digests.keys) + else + initialize_digests(@expected_digests.keys) + end + end + + def digests? + @digests&.any? + end + + # Open the temp file for writing, reusing original permissions, yielding the IO object. + def open(write_mode = "wb", perm = @perm, &block) + raise ClosedError, "Cannot reopen closed file" if @closed + path.open(write_mode, perm) do |f| + yield digests? ? Gem::Package::DigestIO.new(f, @digests) : f + end + end + + # Returns false without appending when no digests since appending is too error prone to do without digests. + def append(data) + return false unless digests? + open("a") {|f| f.write data } + verify && commit + end + + def write(data) + reset_digests + open {|f| f.write data } + commit! + end + + def commit! + verify || raise(DigestMismatchError.new(@base64digests, @expected_digests)) + commit + end + + # Verify the digests, returning true on match, false on mismatch. + def verify + return true unless @expected_digests && digests? + @base64digests = @digests.transform_values!(&:base64digest) + @digests = nil + @base64digests.all? {|algo, digest| @expected_digests[algo] == digest } + end + + # Replace the original file with the temp file without verifying digests. + # The file is permanently closed. + def commit + raise ClosedError, "Cannot commit closed file" if @closed + FileUtils.mv(path, original_path) + @closed = true + end + + # Remove the temp file without replacing the original file. + # The file is permanently closed. + def close + return if @closed + FileUtils.remove_file(path) if @path&.file? + @closed = true + end + end +end diff --git a/test/rubygems/test_gem_compact_index_client_cache_file.rb b/test/rubygems/test_gem_compact_index_client_cache_file.rb new file mode 100644 index 000000000000..749d6eba404a --- /dev/null +++ b/test/rubygems/test_gem_compact_index_client_cache_file.rb @@ -0,0 +1,135 @@ +# frozen_string_literal: true + +require_relative "helper" +require "pathname" +require "rubygems/compact_index_client" + +class TestGemCompactIndexClientCacheFile < Gem::TestCase + CacheFile = Gem::CompactIndexClient::CacheFile + + def setup + super + + @path = Pathname(@tempdir).join("versions") + end + + def sha256(data) + { "sha-256" => Digest::SHA256.base64digest(data) } + end + + def test_write_creates_file + CacheFile.write(@path, "created_at: 2026-06-10\n---\nrake 13.0.0\n") + + assert_equal "created_at: 2026-06-10\n---\nrake 13.0.0\n", @path.read + end + + def test_write_replaces_original_file + @path.write "old" + + CacheFile.write(@path, "new") + + assert_equal "new", @path.read + end + + def test_write_removes_temp_file + CacheFile.write(@path, "data") + + assert_empty Dir.glob("#{@path}.*.tmp") + end + + def test_write_nil_data_does_nothing + CacheFile.write(@path, nil) + + refute @path.exist? + end + + def test_write_with_matching_digests + CacheFile.write(@path, "data", sha256("data")) + + assert_equal "data", @path.read + end + + def test_write_with_mismatched_digests + @path.write "old" + + assert_raise CacheFile::DigestMismatchError do + CacheFile.write(@path, "data", sha256("other data")) + end + + assert_equal "old", @path.read + end + + def test_append_without_digests_returns_false + @path.write "abc" + + appended = nil + CacheFile.new(@path) {|file| appended = file.append("def") } + + refute appended + assert_equal "abc", @path.read + end + + def test_append_with_matching_digests + @path.write "abc" + + appended = nil + CacheFile.copy(@path) do |file| + file.digests = sha256("abcdef") + appended = file.append("def") + end + + assert appended + assert_equal "abcdef", @path.read + end + + def test_append_with_mismatched_digests_keeps_original + @path.write "abc" + + appended = nil + CacheFile.copy(@path) do |file| + file.digests = sha256("abcxyz") + appended = file.append("def") + end + + refute appended + assert_equal "abc", @path.read + end + + def test_close_removes_temp_file + file = CacheFile.new(@path) + file.open {|f| f.write "data" } + file.close + + assert_empty Dir.glob("#{@path}.*.tmp") + refute @path.exist? + end + + def test_open_after_close_raises + file = CacheFile.new(@path) + file.close + + assert_raise CacheFile::ClosedError do + file.open {|f| f.write "data" } + end + end + + def test_commit_after_close_raises + file = CacheFile.new(@path) + file.close + + assert_raise CacheFile::ClosedError do + file.commit + end + end + + def test_write_preserves_permissions + pend "chmod is unreliable on Windows" if Gem.win_platform? + + @path.write "old" + @path.chmod 0o600 + + CacheFile.write(@path, "new") + + assert_equal 0o600, @path.stat.mode & 0o777 + end +end From a49de1b4afe94ca1b6d50d0ad095aea8fb460951 Mon Sep 17 00:00:00 2001 From: Hiroshi SHIBATA Date: Wed, 10 Jun 2026 14:16:20 +0900 Subject: [PATCH 02/19] Add Gem::CompactIndexClient::Updater Keeps cached compact index files in sync with the server using ETag conditional requests and ranged requests, verifying Repr-Digest checksums. Unlike the Bundler version, checksum and gzip failures raise Gem::CompactIndexClient errors instead of Bundler::HTTPError. Co-Authored-By: Claude Fable 5 --- lib/rubygems/compact_index_client.rb | 1 + lib/rubygems/compact_index_client/updater.rb | 104 +++++++++++ .../test_gem_compact_index_client_updater.rb | 170 ++++++++++++++++++ 3 files changed, 275 insertions(+) create mode 100644 lib/rubygems/compact_index_client/updater.rb create mode 100644 test/rubygems/test_gem_compact_index_client_updater.rb diff --git a/lib/rubygems/compact_index_client.rb b/lib/rubygems/compact_index_client.rb index 7de0d620add4..3f6da232de40 100644 --- a/lib/rubygems/compact_index_client.rb +++ b/lib/rubygems/compact_index_client.rb @@ -16,4 +16,5 @@ class Gem::CompactIndexClient class Error < StandardError; end require_relative "compact_index_client/cache_file" + require_relative "compact_index_client/updater" end diff --git a/lib/rubygems/compact_index_client/updater.rb b/lib/rubygems/compact_index_client/updater.rb new file mode 100644 index 000000000000..eef094b611d9 --- /dev/null +++ b/lib/rubygems/compact_index_client/updater.rb @@ -0,0 +1,104 @@ +# frozen_string_literal: true + +require "zlib" +require_relative "../vendored_net_http" + +class Gem::CompactIndexClient + # Updates the cached files on disk, keeping them in sync with the server + # using conditional requests (ETag) and ranged requests where possible. + class Updater + class MismatchedChecksumError < Error + def initialize(path, message) + super "The checksum of /#{path} does not match the checksum provided by the server! Something is wrong. #{message}" + end + end + + def initialize(fetcher) + @fetcher = fetcher + end + + def update(remote_path, local_path, etag_path) + append(remote_path, local_path, etag_path) || replace(remote_path, local_path, etag_path) + rescue CacheFile::DigestMismatchError => e + raise MismatchedChecksumError.new(remote_path, e.message) + rescue Zlib::GzipFile::Error + raise Error, "invalid gzip response while fetching /#{remote_path}" + end + + private + + def append(remote_path, local_path, etag_path) + return false unless local_path.file? && local_path.size.nonzero? + + CacheFile.copy(local_path) do |file| + etag = etag_path.read.tap(&:chomp!) if etag_path.file? + + # Subtract a byte to ensure the range won't be empty. + # Avoids 416 (Range Not Satisfiable) responses. + response = @fetcher.call(remote_path, request_headers(etag, file.size - 1)) + break true if response.is_a?(Gem::Net::HTTPNotModified) + + file.digests = parse_digests(response) + # server may ignore Range and return the full response + if response.is_a?(Gem::Net::HTTPPartialContent) + tail = response.body.byteslice(1..-1) + break false unless tail && file.append(tail) + else + file.write(response.body) + end + CacheFile.write(etag_path, etag_from_response(response)) + true + end + end + + # request without range header to get the full file or a 304 Not Modified + def replace(remote_path, local_path, etag_path) + etag = etag_path.read.tap(&:chomp!) if etag_path.file? + response = @fetcher.call(remote_path, request_headers(etag)) + return true if response.is_a?(Gem::Net::HTTPNotModified) + CacheFile.write(local_path, response.body, parse_digests(response)) + CacheFile.write(etag_path, etag_from_response(response)) + end + + def request_headers(etag, range_start = nil) + headers = {} + headers["Range"] = "bytes=#{range_start}-" if range_start + headers["If-None-Match"] = %("#{etag}") if etag + headers + end + + def etag_from_response(response) + return unless response["ETag"] + etag = response["ETag"].delete_prefix("W/") + return if etag.delete_prefix!('"') && !etag.delete_suffix!('"') + etag + end + + # Unwraps and returns a Hash of digest algorithms and base64 values + # according to RFC 8941 Structured Field Values for HTTP. + # https://www.rfc-editor.org/rfc/rfc8941#name-parsing-a-byte-sequence + # Ignores unsupported algorithms. + def parse_digests(response) + return unless header = response["Repr-Digest"] || response["Digest"] + digests = {} + header.split(",") do |param| + algorithm, value = param.split("=", 2) + algorithm.strip! + algorithm.downcase! + next unless SUPPORTED_DIGESTS.key?(algorithm) + next unless value = byte_sequence(value) + digests[algorithm] = value + end + digests.empty? ? nil : digests + end + + # Unwrap surrounding colons (byte sequence) + # The wrapping characters must be matched or we return nil. + # Also handles quotes because right now rubygems.org sends them. + def byte_sequence(value) + return if value.delete_prefix!(":") && !value.delete_suffix!(":") + return if value.delete_prefix!('"') && !value.delete_suffix!('"') + value + end + end +end diff --git a/test/rubygems/test_gem_compact_index_client_updater.rb b/test/rubygems/test_gem_compact_index_client_updater.rb new file mode 100644 index 000000000000..04aa1965b782 --- /dev/null +++ b/test/rubygems/test_gem_compact_index_client_updater.rb @@ -0,0 +1,170 @@ +# frozen_string_literal: true + +require_relative "helper" +require "pathname" +require "rubygems/compact_index_client" + +class TestGemCompactIndexClientUpdater < Gem::TestCase + class FakeFetcher + attr_reader :requests + + def initialize(*responses) + @responses = responses + @requests = [] + end + + def call(path, headers) + @requests << [path, headers] + @responses.shift + end + end + + class FakeResponse < Gem::Net::HTTPOK + def initialize(body, headers = {}) + super("1.1", "200", "OK") + @fake_body = body + headers.each {|name, value| self[name] = value } + end + + attr_reader :fake_body + alias_method :body, :fake_body + end + + class FakePartialResponse < Gem::Net::HTTPPartialContent + def initialize(body, headers = {}) + super("1.1", "206", "Partial Content") + @fake_body = body + headers.each {|name, value| self[name] = value } + end + + attr_reader :fake_body + alias_method :body, :fake_body + end + + class FakeNotModified < Gem::Net::HTTPNotModified + def initialize + super("1.1", "304", "Not Modified") + end + end + + def setup + super + + @local_path = Pathname(@tempdir).join("versions") + @etag_path = Pathname(@tempdir).join("versions.etag") + end + + def digest_header(data) + "sha-256=:#{Digest::SHA256.base64digest(data)}:" + end + + def test_update_fetches_full_file_when_no_local_copy + fetcher = FakeFetcher.new(FakeResponse.new("a 1.0.0\n", "ETag" => '"abc123"')) + updater = Gem::CompactIndexClient::Updater.new(fetcher) + + updater.update("versions", @local_path, @etag_path) + + assert_equal "a 1.0.0\n", @local_path.read + assert_equal "abc123", @etag_path.read + + path, headers = fetcher.requests.first + assert_equal "versions", path + assert_empty headers + end + + def test_update_sends_etag_and_keeps_file_on_not_modified + @local_path.write "a 1.0.0\n" + @etag_path.write "abc123" + fetcher = FakeFetcher.new(FakeNotModified.new) + updater = Gem::CompactIndexClient::Updater.new(fetcher) + + updater.update("versions", @local_path, @etag_path) + + assert_equal "a 1.0.0\n", @local_path.read + + _, headers = fetcher.requests.first + assert_equal "bytes=7-", headers["Range"] + assert_equal '"abc123"', headers["If-None-Match"] + end + + def test_update_appends_ranged_response + @local_path.write "a 1.0.0\n" + body = "\na 1.1.0\n" + response = FakePartialResponse.new(body, + "ETag" => '"def456"', + "Repr-Digest" => digest_header("a 1.0.0\na 1.1.0\n")) + fetcher = FakeFetcher.new(response) + updater = Gem::CompactIndexClient::Updater.new(fetcher) + + updater.update("versions", @local_path, @etag_path) + + assert_equal "a 1.0.0\na 1.1.0\n", @local_path.read + assert_equal "def456", @etag_path.read + + _, headers = fetcher.requests.first + assert_equal "bytes=7-", headers["Range"] + end + + def test_update_replaces_file_when_server_ignores_range + @local_path.write "stale data" + response = FakeResponse.new("a 1.0.0\n", + "ETag" => '"def456"', + "Repr-Digest" => digest_header("a 1.0.0\n")) + fetcher = FakeFetcher.new(response) + updater = Gem::CompactIndexClient::Updater.new(fetcher) + + updater.update("versions", @local_path, @etag_path) + + assert_equal "a 1.0.0\n", @local_path.read + end + + def test_update_retries_with_full_request_on_bad_ranged_response + @local_path.write "a 1.0.0\n" + bad_append = FakePartialResponse.new("\nb 1.0.0\n", + "Repr-Digest" => digest_header("something else entirely")) + full = FakeResponse.new("a 1.0.0\nb 1.0.0\n", + "ETag" => '"def456"', + "Repr-Digest" => digest_header("a 1.0.0\nb 1.0.0\n")) + fetcher = FakeFetcher.new(bad_append, full) + updater = Gem::CompactIndexClient::Updater.new(fetcher) + + updater.update("versions", @local_path, @etag_path) + + assert_equal "a 1.0.0\nb 1.0.0\n", @local_path.read + assert_equal 2, fetcher.requests.size + end + + def test_update_raises_on_full_response_checksum_mismatch + response = FakeResponse.new("a 1.0.0\n", + "Repr-Digest" => digest_header("tampered")) + fetcher = FakeFetcher.new(response) + updater = Gem::CompactIndexClient::Updater.new(fetcher) + + assert_raise Gem::CompactIndexClient::Updater::MismatchedChecksumError do + updater.update("versions", @local_path, @etag_path) + end + + refute @local_path.exist? + end + + def test_update_parses_weak_etag + response = FakeResponse.new("a 1.0.0\n", "ETag" => 'W/"weak1"') + fetcher = FakeFetcher.new(response) + updater = Gem::CompactIndexClient::Updater.new(fetcher) + + updater.update("versions", @local_path, @etag_path) + + assert_equal "weak1", @etag_path.read + end + + def test_update_ignores_unsupported_digest_algorithms + response = FakeResponse.new("a 1.0.0\n", + "Repr-Digest" => "md5=:#{Digest::MD5.base64digest("bogus")}:") + fetcher = FakeFetcher.new(response) + updater = Gem::CompactIndexClient::Updater.new(fetcher) + + updater.update("versions", @local_path, @etag_path) + + assert_equal "a 1.0.0\n", @local_path.read + end +end From d02c8b2d65234a797dca09865fc533370845e4c3 Mon Sep 17 00:00:00 2001 From: Hiroshi SHIBATA Date: Wed, 10 Jun 2026 14:18:34 +0900 Subject: [PATCH 03/19] Add Gem::CompactIndexClient::Cache Manages the on-disk cache layout (versions, names, info/*, etags) and delegates fetching to the Updater, deduplicating endpoint fetches per process. Also adds the DEBUG_COMPACT_INDEX debug logger to the client. Co-Authored-By: Claude Fable 5 --- lib/rubygems/compact_index_client.rb | 7 ++ lib/rubygems/compact_index_client/cache.rb | 101 +++++++++++++++ .../test_gem_compact_index_client_cache.rb | 117 ++++++++++++++++++ 3 files changed, 225 insertions(+) create mode 100644 lib/rubygems/compact_index_client/cache.rb create mode 100644 test/rubygems/test_gem_compact_index_client_cache.rb diff --git a/lib/rubygems/compact_index_client.rb b/lib/rubygems/compact_index_client.rb index 3f6da232de40..00e79a5d9489 100644 --- a/lib/rubygems/compact_index_client.rb +++ b/lib/rubygems/compact_index_client.rb @@ -12,9 +12,16 @@ class Gem::CompactIndexClient SUPPORTED_DIGESTS = { "sha-256" => :SHA256 }.freeze + DEBUG_MUTEX = Thread::Mutex.new + + def self.debug + return unless ENV["DEBUG_COMPACT_INDEX"] + DEBUG_MUTEX.synchronize { warn("[#{self}] #{yield}") } + end class Error < StandardError; end + require_relative "compact_index_client/cache" require_relative "compact_index_client/cache_file" require_relative "compact_index_client/updater" end diff --git a/lib/rubygems/compact_index_client/cache.rb b/lib/rubygems/compact_index_client/cache.rb new file mode 100644 index 000000000000..8992b1bd80bb --- /dev/null +++ b/lib/rubygems/compact_index_client/cache.rb @@ -0,0 +1,101 @@ +# frozen_string_literal: true + +require "digest" +require "fileutils" +require "pathname" +require "set" + +class Gem::CompactIndexClient + # Calls the Updater to update the cached files on disk, reads the + # cached files and returns their contents. + class Cache + attr_reader :directory + + def initialize(directory, fetcher = nil) + @directory = Pathname.new(directory).expand_path + @updater = Updater.new(fetcher) if fetcher + @mutex = Thread::Mutex.new + @endpoints = Set.new + + @info_root = mkdir("info") + @special_characters_info_root = mkdir("info-special-characters") + @info_etag_root = mkdir("info-etags") + end + + def names + fetch("names", names_path, names_etag_path) + end + + def versions + fetch("versions", versions_path, versions_etag_path) + end + + def info(name, remote_checksum = nil) + path = info_path(name) + + if remote_checksum && remote_checksum != checksum_for_file(path) + fetch("info/#{name}", path, info_etag_path(name)) + else + Gem::CompactIndexClient.debug { "update skipped info/#{name} (#{remote_checksum ? "versions index checksum matches local" : "versions index checksum is nil"})" } + read(path) + end + end + + def reset! + @mutex.synchronize { @endpoints.clear } + end + + private + + def names_path = directory.join("names") + def names_etag_path = directory.join("names.etag") + def versions_path = directory.join("versions") + def versions_etag_path = directory.join("versions.etag") + + def info_path(name) + name = name.to_s + if /[^a-z0-9_-]/.match?(name) + name += "-#{Digest::MD5.hexdigest(name).downcase}" + @special_characters_info_root.join(name) + else + @info_root.join(name) + end + end + + def info_etag_path(name) + name = name.to_s + @info_etag_root.join("#{name}-#{Digest::MD5.hexdigest(name).downcase}") + end + + def checksum_for_file(path) + return unless path.file? + Digest::MD5.file(path).hexdigest + end + + def mkdir(name) + directory.join(name).tap do |dir| + FileUtils.mkdir_p(dir) + end + end + + def fetch(remote_path, path, etag_path) + if already_fetched?(remote_path) + Gem::CompactIndexClient.debug { "already fetched #{remote_path}" } + else + Gem::CompactIndexClient.debug { "fetching #{remote_path}" } + @updater&.update(remote_path, path, etag_path) + end + + read(path) + end + + def already_fetched?(remote_path) + @mutex.synchronize { !@endpoints.add?(remote_path) } + end + + def read(path) + return unless path.file? + path.read + end + end +end diff --git a/test/rubygems/test_gem_compact_index_client_cache.rb b/test/rubygems/test_gem_compact_index_client_cache.rb new file mode 100644 index 000000000000..82a7dd1baab2 --- /dev/null +++ b/test/rubygems/test_gem_compact_index_client_cache.rb @@ -0,0 +1,117 @@ +# frozen_string_literal: true + +require_relative "helper" +require "pathname" +require "rubygems/compact_index_client" + +class TestGemCompactIndexClientCache < Gem::TestCase + class FakeResponse < Gem::Net::HTTPOK + def initialize(body) + super("1.1", "200", "OK") + @fake_body = body + end + + attr_reader :fake_body + alias_method :body, :fake_body + end + + class FakeFetcher + attr_reader :requests + + def initialize(body) + @body = body + @requests = [] + end + + def call(path, headers) + @requests << path + FakeResponse.new(@body) + end + end + + def setup + super + + @dir = Pathname(@tempdir).join("compact_index") + end + + def test_initialize_creates_cache_directories + Gem::CompactIndexClient::Cache.new(@dir) + + assert @dir.join("info").directory? + assert @dir.join("info-special-characters").directory? + assert @dir.join("info-etags").directory? + end + + def test_reads_cached_files_without_fetcher + cache = Gem::CompactIndexClient::Cache.new(@dir) + @dir.join("versions").write "a 1.0.0\n" + + assert_equal "a 1.0.0\n", cache.versions + assert_nil cache.names + end + + def test_versions_fetches_once + fetcher = FakeFetcher.new("a 1.0.0\n") + cache = Gem::CompactIndexClient::Cache.new(@dir, fetcher) + + assert_equal "a 1.0.0\n", cache.versions + assert_equal "a 1.0.0\n", cache.versions + assert_equal ["versions"], fetcher.requests + assert_equal "a 1.0.0\n", @dir.join("versions").read + end + + def test_reset_allows_fetching_again + fetcher = FakeFetcher.new("a 1.0.0\n") + cache = Gem::CompactIndexClient::Cache.new(@dir, fetcher) + + cache.versions + cache.reset! + cache.versions + + assert_equal %w[versions versions], fetcher.requests + end + + def test_info_skips_fetch_when_checksum_matches + fetcher = FakeFetcher.new("a 1.0.0\n") + cache = Gem::CompactIndexClient::Cache.new(@dir, fetcher) + @dir.join("info", "a").write "a 1.0.0\n" + + content = cache.info("a", Digest::MD5.hexdigest("a 1.0.0\n")) + + assert_equal "a 1.0.0\n", content + assert_empty fetcher.requests + end + + def test_info_fetches_when_checksum_differs + fetcher = FakeFetcher.new("a 1.0.0\na 1.1.0\n") + cache = Gem::CompactIndexClient::Cache.new(@dir, fetcher) + @dir.join("info", "a").write "a 1.0.0\n" + + content = cache.info("a", Digest::MD5.hexdigest("a 1.0.0\na 1.1.0\n")) + + assert_equal "a 1.0.0\na 1.1.0\n", content + assert_equal ["info/a"], fetcher.requests + assert_equal "a 1.0.0\na 1.1.0\n", @dir.join("info", "a").read + end + + def test_info_without_checksum_reads_cached_file + fetcher = FakeFetcher.new("a 1.0.0\n") + cache = Gem::CompactIndexClient::Cache.new(@dir, fetcher) + @dir.join("info", "a").write "a 1.0.0\n" + + assert_equal "a 1.0.0\n", cache.info("a") + assert_empty fetcher.requests + end + + def test_info_with_special_characters_uses_hashed_path + fetcher = FakeFetcher.new("1.0.0\n") + cache = Gem::CompactIndexClient::Cache.new(@dir, fetcher) + + cache.info("Rails", "no-match") + + hashed = "Rails-#{Digest::MD5.hexdigest("Rails").downcase}" + assert_equal "1.0.0\n", @dir.join("info-special-characters", hashed).read + refute @dir.join("info", "Rails").exist? + end +end From e400dd5fd7ee0e867ba34519c1b907b7465a4072 Mon Sep 17 00:00:00 2001 From: Hiroshi SHIBATA Date: Wed, 10 Jun 2026 14:20:56 +0900 Subject: [PATCH 04/19] Add Gem::CompactIndexClient::Parser Parses the versions index (including deletion lines and per-gem info checksums) and info files. Reuses Gem::Resolver::APISet::GemParser for info lines, which already preserves compact index v2 metadata such as created_at. Co-Authored-By: Claude Fable 5 --- lib/rubygems/compact_index_client.rb | 1 + lib/rubygems/compact_index_client/parser.rb | 87 +++++++++++++ .../test_gem_compact_index_client_parser.rb | 117 ++++++++++++++++++ 3 files changed, 205 insertions(+) create mode 100644 lib/rubygems/compact_index_client/parser.rb create mode 100644 test/rubygems/test_gem_compact_index_client_parser.rb diff --git a/lib/rubygems/compact_index_client.rb b/lib/rubygems/compact_index_client.rb index 00e79a5d9489..e1e8c02d5bde 100644 --- a/lib/rubygems/compact_index_client.rb +++ b/lib/rubygems/compact_index_client.rb @@ -23,5 +23,6 @@ class Error < StandardError; end require_relative "compact_index_client/cache" require_relative "compact_index_client/cache_file" + require_relative "compact_index_client/parser" require_relative "compact_index_client/updater" end diff --git a/lib/rubygems/compact_index_client/parser.rb b/lib/rubygems/compact_index_client/parser.rb new file mode 100644 index 000000000000..e36164cb2454 --- /dev/null +++ b/lib/rubygems/compact_index_client/parser.rb @@ -0,0 +1,87 @@ +# frozen_string_literal: true + +require_relative "../resolver/api_set/gem_parser" + +class Gem::CompactIndexClient + class Parser + # `compact_index` - an object responding to #names, #versions, #info(name, checksum), + # returning the file contents as a string + def initialize(compact_index) + @compact_index = compact_index + @info_checksums = nil + @versions_by_name = nil + @available = nil + @gem_parser = nil + end + + def names + lines(@compact_index.names) + end + + def versions + @versions_by_name ||= Hash.new {|hash, key| hash[key] = [] } + @info_checksums = {} + + lines(@compact_index.versions).each do |line| + name, versions_string, checksum = line.split(" ", 3) + @info_checksums[name] = checksum || "" + versions_string.split(",") do |version| + delete = version.delete_prefix!("-") + version = version.split("-", 2).unshift(name) + if delete + @versions_by_name[name].delete(version) + else + @versions_by_name[name] << version + end + end + end + + @versions_by_name + end + + def info(name) + data = @compact_index.info(name, info_checksums[name]) + lines(data).map {|line| gem_parser.parse(line).unshift(name) } + end + + def available? + return @available unless @available.nil? + @available = !info_checksums.empty? + end + + private + + def info_checksums + @info_checksums ||= lines(@compact_index.versions).each_with_object({}) do |line, checksums| + parse_version_checksum(line, checksums) + end + end + + def lines(data) + return [] if data.nil? || data.empty? + lines = data.split("\n") + header = lines.index("---") + header ? lines[header + 1..-1] : lines + end + + def gem_parser + @gem_parser ||= Gem::Resolver::APISet::GemParser.new + end + + # This is mostly the same as `split(" ", 3)` but it avoids allocating extra objects. + # This method gets called at least once for every gem when parsing versions. + def parse_version_checksum(line, checksums) + return unless (name_end = line.index(" ")) # Artifactory bug causes blank lines in artifactory index files + checksum_start = line.index(" ", name_end + 1) + return unless checksum_start + checksum_start += 1 + + checksum_end = line.size - checksum_start + + line.freeze # allows slicing into the string to not allocate a copy of the line + name = line[0, name_end] + checksum = line[checksum_start, checksum_end] + checksums[name.freeze] = checksum # freeze name since it is used as a hash key + end + end +end diff --git a/test/rubygems/test_gem_compact_index_client_parser.rb b/test/rubygems/test_gem_compact_index_client_parser.rb new file mode 100644 index 000000000000..590cb102cb99 --- /dev/null +++ b/test/rubygems/test_gem_compact_index_client_parser.rb @@ -0,0 +1,117 @@ +# frozen_string_literal: true + +require_relative "helper" +require "rubygems/compact_index_client" + +class TestGemCompactIndexClientParser < Gem::TestCase + class FakeIndex + attr_reader :info_requests + + def initialize(names: nil, versions: nil, info: {}) + @names = names + @versions = versions + @info = info + @info_requests = {} + end + + attr_reader :names, :versions + + def info(name, checksum) + @info_requests[name] = checksum + @info[name] + end + end + + def test_names_strips_header + index = FakeIndex.new(names: "---\na\nb\n") + parser = Gem::CompactIndexClient::Parser.new(index) + + assert_equal %w[a b], parser.names + end + + def test_versions_parses_versions_and_platforms + index = FakeIndex.new(versions: <<~VERSIONS) + created_at: 2026-06-10T00:00:00Z + --- + a 1.0.0,1.1.0 aaa111 + b 1.0.0-java bbb222 + VERSIONS + parser = Gem::CompactIndexClient::Parser.new(index) + + versions = parser.versions + + assert_equal [["a", "1.0.0"], ["a", "1.1.0"]], versions["a"] + assert_equal [["b", "1.0.0", "java"]], versions["b"] + end + + def test_versions_applies_deletions + index = FakeIndex.new(versions: <<~VERSIONS) + --- + a 1.0.0,1.1.0 aaa111 + a -1.1.0 aaa222 + VERSIONS + parser = Gem::CompactIndexClient::Parser.new(index) + + assert_equal [["a", "1.0.0"]], parser.versions["a"] + end + + def test_info_passes_checksum_from_versions_index + index = FakeIndex.new(versions: "---\na 1.0.0 aaa111\na 1.1.0 aaa222\n", + info: { "a" => "---\n1.0.0 |checksum:abc\n" }) + parser = Gem::CompactIndexClient::Parser.new(index) + + parser.info("a") + + assert_equal({ "a" => "aaa222" }, index.info_requests) + end + + def test_info_parses_dependencies_and_requirements + line = "1.2.0 b:>= 1.0&< 2.0,c:= 0.5" \ + "|checksum:abc123,ruby:>= 3.0,rubygems:>= 3.3.3,created_at:2026-06-05T10:30:45Z" + index = FakeIndex.new(versions: "---\na 1.2.0 aaa111\n", + info: { "a" => "---\n#{line}\n" }) + parser = Gem::CompactIndexClient::Parser.new(index) + + info = parser.info("a") + + name, version, platform, dependencies, requirements = info.first + assert_equal "a", name + assert_equal "1.2.0", version + assert_nil platform + assert_equal [["b", [">= 1.0", "< 2.0"]], ["c", ["= 0.5"]]], dependencies + assert_equal [["checksum", ["abc123"]], ["ruby", [">= 3.0"]], + ["rubygems", [">= 3.3.3"]], ["created_at", ["2026-06-05T10:30:45Z"]]], requirements + end + + def test_info_parses_platform_version + index = FakeIndex.new(versions: "---\na 1.0.0-java aaa111\n", + info: { "a" => "---\n1.0.0-java |checksum:abc\n" }) + parser = Gem::CompactIndexClient::Parser.new(index) + + _, version, platform, = parser.info("a").first + + assert_equal "1.0.0", version + assert_equal "java", platform + end + + def test_available_with_versions_data + parser = Gem::CompactIndexClient::Parser.new(FakeIndex.new(versions: "---\na 1.0.0 aaa111\n")) + + assert parser.available? + end + + def test_available_with_no_data + parser = Gem::CompactIndexClient::Parser.new(FakeIndex.new) + + refute parser.available? + end + + def test_skips_blank_lines_in_versions_index + index = FakeIndex.new(versions: "---\na 1.0.0 aaa111\n\n", + info: { "a" => "1.0.0 |checksum:abc\n" }) + parser = Gem::CompactIndexClient::Parser.new(index) + + assert parser.available? + assert_equal 1, parser.info("a").size + end +end From 130c0f8632ebed6a9a8fa50db89d1e64644128f2 Mon Sep 17 00:00:00 2001 From: Hiroshi SHIBATA Date: Wed, 10 Jun 2026 14:22:42 +0900 Subject: [PATCH 05/19] Add Gem::CompactIndexClient public API Completes the client facade: names, versions, info, dependencies, latest_version, available? and reset! on top of Cache and Parser. Co-Authored-By: Claude Fable 5 --- lib/rubygems/compact_index_client.rb | 52 +++++++++ .../rubygems/test_gem_compact_index_client.rb | 108 ++++++++++++++++++ 2 files changed, 160 insertions(+) create mode 100644 test/rubygems/test_gem_compact_index_client.rb diff --git a/lib/rubygems/compact_index_client.rb b/lib/rubygems/compact_index_client.rb index e1e8c02d5bde..44759ec2e7e8 100644 --- a/lib/rubygems/compact_index_client.rb +++ b/lib/rubygems/compact_index_client.rb @@ -14,6 +14,13 @@ class Gem::CompactIndexClient SUPPORTED_DIGESTS = { "sha-256" => :SHA256 }.freeze DEBUG_MUTEX = Thread::Mutex.new + # info returns an Array of INFO Arrays. Each INFO Array has the following indices: + INFO_NAME = 0 + INFO_VERSION = 1 + INFO_PLATFORM = 2 + INFO_DEPS = 3 + INFO_REQS = 4 + def self.debug return unless ENV["DEBUG_COMPACT_INDEX"] DEBUG_MUTEX.synchronize { warn("[#{self}] #{yield}") } @@ -25,4 +32,49 @@ class Error < StandardError; end require_relative "compact_index_client/cache_file" require_relative "compact_index_client/parser" require_relative "compact_index_client/updater" + + # The client is instantiated with: + # - `directory`: the root directory where the cache files are stored. + # - `fetcher`: (optional) an object that responds to #call(uri_path, headers) + # and returns a Gem::Net::HTTP response. When the fetcher is not provided, + # the client only reads cached files from disk. + def initialize(directory, fetcher = nil) + @cache = Cache.new(directory, fetcher) + @parser = Parser.new(@cache) + end + + def names + Gem::CompactIndexClient.debug { "names" } + @parser.names + end + + def versions + Gem::CompactIndexClient.debug { "versions" } + @parser.versions + end + + def dependencies(names) + Gem::CompactIndexClient.debug { "dependencies(#{names})" } + names.map {|name| info(name) } + end + + def info(name) + Gem::CompactIndexClient.debug { "info(#{name})" } + @parser.info(name) + end + + def latest_version(name) + Gem::CompactIndexClient.debug { "latest_version(#{name})" } + @parser.info(name).map {|d| Gem::Version.new(d[INFO_VERSION]) }.max + end + + def available? + Gem::CompactIndexClient.debug { "available?" } + @parser.available? + end + + def reset! + Gem::CompactIndexClient.debug { "reset!" } + @cache.reset! + end end diff --git a/test/rubygems/test_gem_compact_index_client.rb b/test/rubygems/test_gem_compact_index_client.rb new file mode 100644 index 000000000000..e877c938fb48 --- /dev/null +++ b/test/rubygems/test_gem_compact_index_client.rb @@ -0,0 +1,108 @@ +# frozen_string_literal: true + +require_relative "helper" +require "rubygems/compact_index_client" + +class TestGemCompactIndexClient < Gem::TestCase + class FakeResponse < Gem::Net::HTTPOK + def initialize(body) + super("1.1", "200", "OK") + @fake_body = body + end + + attr_reader :fake_body + alias_method :body, :fake_body + end + + class FakeFetcher + FILES = { + "names" => "---\na\nb\n", + "versions" => "created_at: 2026-06-10T00:00:00Z\n---\n" \ + "a 1.0.0,1.1.0 #{Digest::MD5.hexdigest("---\n1.0.0 |checksum:c1\n1.1.0 |checksum:c2,created_at:2026-06-05T10:30:45Z\n")}\n" \ + "b 1.0.0-java #{Digest::MD5.hexdigest("---\n1.0.0-java |checksum:c3\n")}\n", + "info/a" => "---\n1.0.0 |checksum:c1\n1.1.0 |checksum:c2,created_at:2026-06-05T10:30:45Z\n", + "info/b" => "---\n1.0.0-java |checksum:c3\n", + }.freeze + + attr_reader :requests + + def initialize + @requests = [] + end + + def call(path, headers) + @requests << path + FakeResponse.new(FILES.fetch(path)) + end + end + + def setup + super + + @fetcher = FakeFetcher.new + @client = Gem::CompactIndexClient.new(File.join(@tempdir, "compact_index"), @fetcher) + end + + def test_names + assert_equal %w[a b], @client.names + end + + def test_versions + versions = @client.versions + + assert_equal [["a", "1.0.0"], ["a", "1.1.0"]], versions["a"] + assert_equal [["b", "1.0.0", "java"]], versions["b"] + end + + def test_info_returns_parsed_info_arrays + info = @client.info("a") + + assert_equal 2, info.size + assert_equal "a", info.last[Gem::CompactIndexClient::INFO_NAME] + assert_equal "1.1.0", info.last[Gem::CompactIndexClient::INFO_VERSION] + assert_nil info.last[Gem::CompactIndexClient::INFO_PLATFORM] + assert_includes info.last[Gem::CompactIndexClient::INFO_REQS], ["created_at", ["2026-06-05T10:30:45Z"]] + end + + def test_dependencies + dependencies = @client.dependencies(%w[a b]) + + assert_equal 2, dependencies.size + assert_equal "b", dependencies.last.first[Gem::CompactIndexClient::INFO_NAME] + assert_equal "java", dependencies.last.first[Gem::CompactIndexClient::INFO_PLATFORM] + end + + def test_latest_version + assert_equal Gem::Version.new("1.1.0"), @client.latest_version("a") + end + + def test_available + assert @client.available? + end + + def test_not_available_without_data + client = Gem::CompactIndexClient.new(File.join(@tempdir, "empty_index")) + + refute client.available? + end + + def test_reset_refetches_versions + @client.versions + @client.reset! + @client.versions + + assert_equal %w[versions versions], @fetcher.requests + end + + def test_info_uses_local_cache_when_checksum_matches + @client.versions # prime info checksums and write cache + @client.info("a") + + requests_before = @fetcher.requests.dup + fresh = Gem::CompactIndexClient.new(File.join(@tempdir, "compact_index"), @fetcher) + fresh.versions + fresh.info("a") + + assert_equal requests_before + ["versions"], @fetcher.requests + end +end From 31f618898a43ea797a0fa1a09c253d48c4a338e8 Mon Sep 17 00:00:00 2001 From: Hiroshi SHIBATA Date: Wed, 10 Jun 2026 14:24:45 +0900 Subject: [PATCH 06/19] Add Gem::CompactIndexClient::HTTPFetcher Adapts Gem::RemoteFetcher to the fetcher interface expected by the compact index client. RemoteFetcher#fetch_path only supports If-Modified-Since, while the compact index needs ETag conditional requests and ranged requests, so this issues requests directly through RemoteFetcher#request. Co-Authored-By: Claude Fable 5 --- lib/rubygems/compact_index_client.rb | 1 + .../compact_index_client/http_fetcher.rb | 49 ++++++++ ...t_gem_compact_index_client_http_fetcher.rb | 112 ++++++++++++++++++ 3 files changed, 162 insertions(+) create mode 100644 lib/rubygems/compact_index_client/http_fetcher.rb create mode 100644 test/rubygems/test_gem_compact_index_client_http_fetcher.rb diff --git a/lib/rubygems/compact_index_client.rb b/lib/rubygems/compact_index_client.rb index 44759ec2e7e8..3fc38684722e 100644 --- a/lib/rubygems/compact_index_client.rb +++ b/lib/rubygems/compact_index_client.rb @@ -30,6 +30,7 @@ class Error < StandardError; end require_relative "compact_index_client/cache" require_relative "compact_index_client/cache_file" + require_relative "compact_index_client/http_fetcher" require_relative "compact_index_client/parser" require_relative "compact_index_client/updater" diff --git a/lib/rubygems/compact_index_client/http_fetcher.rb b/lib/rubygems/compact_index_client/http_fetcher.rb new file mode 100644 index 000000000000..daef0c686faf --- /dev/null +++ b/lib/rubygems/compact_index_client/http_fetcher.rb @@ -0,0 +1,49 @@ +# frozen_string_literal: true + +require_relative "../remote_fetcher" + +class Gem::CompactIndexClient + # Fetches compact index files relative to +base_uri+ using + # Gem::RemoteFetcher's connection infrastructure (proxy, TLS, + # connection pooling). Implements the fetcher interface expected by + # Gem::CompactIndexClient: #call(path, headers) returning a + # Gem::Net::HTTP response. + class HTTPFetcher + REDIRECT_LIMIT = 10 + private_constant :REDIRECT_LIMIT + + def initialize(base_uri, remote_fetcher = Gem::RemoteFetcher.fetcher) + base_uri = base_uri.to_s + base_uri += "/" unless base_uri.end_with?("/") + @base_uri = Gem::URI(base_uri) + @remote_fetcher = remote_fetcher + end + + def call(path, headers = {}) + fetch(@base_uri + path, headers, REDIRECT_LIMIT) + end + + private + + def fetch(uri, headers, redirects_remaining) + response = @remote_fetcher.request(uri, Gem::Net::HTTP::Get) do |req| + headers.each {|name, value| req[name] = value } + end + + case response + when Gem::Net::HTTPSuccess, Gem::Net::HTTPNotModified + response + when Gem::Net::HTTPMovedPermanently, Gem::Net::HTTPFound, Gem::Net::HTTPSeeOther, + Gem::Net::HTTPTemporaryRedirect + raise Gem::RemoteFetcher::FetchError.new("too many redirects", uri) if redirects_remaining.zero? + + location = response["Location"] + raise Gem::RemoteFetcher::FetchError.new("redirecting but no redirect location was given", uri) unless location + + fetch(uri + location, headers, redirects_remaining - 1) + else + raise Gem::RemoteFetcher::FetchError.new("bad response #{response.message} #{response.code}", uri) + end + end + end +end diff --git a/test/rubygems/test_gem_compact_index_client_http_fetcher.rb b/test/rubygems/test_gem_compact_index_client_http_fetcher.rb new file mode 100644 index 000000000000..c2c948df651d --- /dev/null +++ b/test/rubygems/test_gem_compact_index_client_http_fetcher.rb @@ -0,0 +1,112 @@ +# frozen_string_literal: true + +require_relative "helper" +require "rubygems/compact_index_client" + +class TestGemCompactIndexClientHTTPFetcher < Gem::TestCase + class FakeResponse < Gem::Net::HTTPOK + def initialize(body) + super("1.1", "200", "OK") + @fake_body = body + end + + attr_reader :fake_body + alias_method :body, :fake_body + end + + class FakeRedirect < Gem::Net::HTTPFound + def initialize(location) + super("1.1", "302", "Found") + self["Location"] = location + end + end + + class FakeNotFound < Gem::Net::HTTPNotFound + def initialize + super("1.1", "404", "Not Found") + end + end + + class FakeRemoteFetcher + attr_reader :requests + + def initialize(responses) + @responses = responses + @requests = [] + end + + def request(uri, request_class) + request = request_class.new(uri) + yield request if block_given? + @requests << [uri, request] + @responses.fetch(uri.to_s) + end + end + + def fetcher_for(responses) + remote = FakeRemoteFetcher.new(responses) + [Gem::CompactIndexClient::HTTPFetcher.new("https://index.example", remote), remote] + end + + def test_call_joins_path_with_base_uri + fetcher, remote = fetcher_for("https://index.example/info/a" => FakeResponse.new("data")) + + response = fetcher.call("info/a") + + assert_equal "data", response.body + assert_equal Gem::URI("https://index.example/info/a"), remote.requests.first.first + end + + def test_call_applies_request_headers + fetcher, remote = fetcher_for("https://index.example/versions" => FakeResponse.new("data")) + + fetcher.call("versions", "If-None-Match" => '"abc"', "Range" => "bytes=10-") + + _, request = remote.requests.first + assert_equal '"abc"', request["If-None-Match"] + assert_equal "bytes=10-", request["Range"] + end + + def test_call_follows_redirects + fetcher, remote = fetcher_for( + "https://index.example/versions" => FakeRedirect.new("https://mirror.example/versions"), + "https://mirror.example/versions" => FakeResponse.new("data") + ) + + response = fetcher.call("versions") + + assert_equal "data", response.body + assert_equal 2, remote.requests.size + end + + def test_call_resolves_relative_redirect_location + fetcher, _remote = fetcher_for( + "https://index.example/versions" => FakeRedirect.new("/v2/versions"), + "https://index.example/v2/versions" => FakeResponse.new("data") + ) + + assert_equal "data", fetcher.call("versions").body + end + + def test_call_raises_after_too_many_redirects + fetcher, _remote = fetcher_for( + "https://index.example/versions" => FakeRedirect.new("https://index.example/versions") + ) + + error = assert_raise Gem::RemoteFetcher::FetchError do + fetcher.call("versions") + end + + assert_match(/too many redirects/, error.message) + end + + def test_call_raises_fetch_error_on_failure_response + fetcher, _remote = fetcher_for("https://index.example/versions" => FakeNotFound.new) + + error = assert_raise Gem::RemoteFetcher::FetchError do + fetcher.call("versions") + end + + assert_match(/bad response Not Found 404/, error.message) + end +end From 7a569dbb869c364edd2fe60ec1a4ce84b51d3da3 Mon Sep 17 00:00:00 2001 From: Hiroshi SHIBATA Date: Wed, 10 Jun 2026 14:29:40 +0900 Subject: [PATCH 07/19] Add compact index stub helpers for tests util_setup_compact_index serves versions, names and info/NAME on the FakeFetcher with consistent MD5/SHA-256 checksums, ETags and optional created_at v2 metadata, so functional tests can drive gem commands against a stubbed compact index. Verified against the real Gem::CompactIndexClient. Co-Authored-By: Claude Fable 5 --- test/rubygems/helper.rb | 71 +++++++++++++++ test/rubygems/test_gem_compact_index_stub.rb | 94 ++++++++++++++++++++ 2 files changed, 165 insertions(+) create mode 100644 test/rubygems/test_gem_compact_index_stub.rb diff --git a/test/rubygems/helper.rb b/test/rubygems/helper.rb index 2411dbc6495c..e5158ff0a9ff 100644 --- a/test/rubygems/helper.rb +++ b/test/rubygems/helper.rb @@ -35,6 +35,7 @@ require "test/unit" +require "digest" require "fileutils" require "pathname" require "pp" @@ -1173,6 +1174,76 @@ def util_setup_spec_fetcher(*specs) nil # force errors end + ## + # Sets up the compact index API endpoints (versions, names and + # info/NAME) for +specs+ on +@fetcher+. +created_at+ maps spec + # original names to ISO8601 timestamps emitted as compact index v2 + # metadata. + + def util_setup_compact_index(*specs, created_at: {}) + by_name = Hash.new {|hash, name| hash[name] = [] } + specs.each {|spec| by_name[spec.name] << spec } + + names_body = +"---\n" + versions_body = +"created_at: 2026-01-01T00:00:00Z\n---\n" + + by_name.keys.sort.each do |name| + info_body = +"---\n" + by_name[name].each do |spec| + info_body << util_compact_index_info_line(spec, created_at[spec.original_name]) << "\n" + end + + versions_list = by_name[name].map {|spec| spec.original_name.delete_prefix("#{spec.name}-") }.join(",") + versions_body << "#{name} #{versions_list} #{Digest::MD5.hexdigest(info_body)}\n" + names_body << "#{name}\n" + + @fetcher.data["#{@gem_repo}info/#{name}"] = util_compact_index_response(info_body) + end + + @fetcher.data["#{@gem_repo}versions"] = util_compact_index_response(versions_body) + @fetcher.data["#{@gem_repo}names"] = util_compact_index_response(names_body) + + nil + end + + ## + # A compact index info file line for +spec+, including v2 metadata. + + def util_compact_index_info_line(spec, created_at = nil) + version = spec.original_name.delete_prefix("#{spec.name}-") + + dependencies = spec.runtime_dependencies.map do |dependency| + "#{dependency.name}:#{util_compact_index_requirement(dependency.requirement)}" + end.join(",") + + metadata = +"checksum:#{Digest::SHA256.hexdigest(spec.original_name)}" + unless spec.required_ruby_version.nil? || spec.required_ruby_version.none? + metadata << ",ruby:#{util_compact_index_requirement(spec.required_ruby_version)}" + end + unless spec.required_rubygems_version.nil? || spec.required_rubygems_version.none? + metadata << ",rubygems:#{util_compact_index_requirement(spec.required_rubygems_version)}" + end + metadata << ",created_at:#{created_at}" if created_at + + "#{version} #{dependencies}|#{metadata}" + end + + def util_compact_index_requirement(requirement) + requirement.as_list.join("&") + end + + def util_compact_index_response(body) + Gem::HTTPResponseFactory.create( + body: body, + code: 200, + msg: "OK", + headers: { + "ETag" => %("#{Digest::MD5.hexdigest(body)}"), + "Repr-Digest" => "sha-256=:#{Digest::SHA256.base64digest(body)}:", + } + ) + end + def write_marshalled_gemspecs(*all_specs) v = Gem.marshal_version diff --git a/test/rubygems/test_gem_compact_index_stub.rb b/test/rubygems/test_gem_compact_index_stub.rb new file mode 100644 index 000000000000..dc9980a021c9 --- /dev/null +++ b/test/rubygems/test_gem_compact_index_stub.rb @@ -0,0 +1,94 @@ +# frozen_string_literal: true + +require_relative "helper" +require "rubygems/compact_index_client" + +## +# Exercises the util_setup_compact_index test helper against the real +# Gem::CompactIndexClient to ensure the stubbed endpoints speak the +# compact index protocol correctly. + +class TestGemCompactIndexStub < Gem::TestCase + def setup + super + + @fetcher = Gem::FakeFetcher.new + Gem::RemoteFetcher.fetcher = @fetcher + + @a1 = util_spec "a", "1.0.0", "b" => ">= 1.0" + @a2 = util_spec "a", "1.1.0" + @b1 = util_spec "b", "1.0.0" + @c1 = util_spec "c", "1.0.0" do |s| + s.platform = "java" + s.required_ruby_version = ">= 3.0" + end + end + + def client + Gem::CompactIndexClient.new( + File.join(@tempdir, "compact_index"), + Gem::CompactIndexClient::HTTPFetcher.new(@gem_repo, @fetcher) + ) + end + + def test_serves_names + util_setup_compact_index @a1, @a2, @b1 + + assert_equal %w[a b], client.names + end + + def test_serves_versions_with_platforms + util_setup_compact_index @a1, @a2, @c1 + + versions = client.versions + + assert_equal [["a", "1.0.0"], ["a", "1.1.0"]], versions["a"] + assert_equal [["c", "1.0.0", "java"]], versions["c"] + end + + def test_serves_info_with_dependencies_and_requirements + util_setup_compact_index @a1, @a2, @b1, @c1 + + info = client.info("a") + + assert_equal "1.0.0", info.first[Gem::CompactIndexClient::INFO_VERSION] + assert_equal [["b", [">= 1.0"]]], info.first[Gem::CompactIndexClient::INFO_DEPS] + + ruby_req = client.info("c").first[Gem::CompactIndexClient::INFO_REQS].assoc("ruby") + assert_equal ["ruby", [">= 3.0"]], ruby_req + end + + def test_serves_created_at_metadata + util_setup_compact_index @a1, @a2, created_at: { "a-1.1.0" => "2026-06-05T10:30:45Z" } + + info = client.info("a") + + assert_nil info.first[Gem::CompactIndexClient::INFO_REQS].assoc("created_at") + assert_equal ["created_at", ["2026-06-05T10:30:45Z"]], + info.last[Gem::CompactIndexClient::INFO_REQS].assoc("created_at") + end + + def test_versions_checksums_match_info_files + util_setup_compact_index @a1, @a2, @b1 + + c = client + c.versions + c.info("a") + + # checksum from the versions index matches the cached info file, so a + # fresh client only refetches the versions index + fresh = client + fresh.versions + fresh.info("a") + + info_requests = @fetcher.requests.count {|req| req.path.end_with?("info/a") } + assert_equal 1, info_requests + end + + def test_repr_digest_verification_round_trip + util_setup_compact_index @a1 + + assert_equal [["a", "1.0.0"]], client.versions["a"] + assert @tempdir && File.file?(File.join(@tempdir, "compact_index", "versions")) + end +end From c5b424f39a9db19b07a0c2326be7fafa4f3dd709 Mon Sep 17 00:00:00 2001 From: Hiroshi SHIBATA Date: Wed, 10 Jun 2026 14:32:04 +0900 Subject: [PATCH 08/19] Add Gem::CompactIndexClient#fetch_info Fetches a single gem's info file with an ETag conditional request, without consulting the versions index. The versions index download only pays off when most gems are needed; for gem install, fetching just the required info files keeps the first-use cost at the current level while still benefiting from the disk cache. Co-Authored-By: Claude Fable 5 --- lib/rubygems/compact_index_client.rb | 9 +++++++++ lib/rubygems/compact_index_client/cache.rb | 6 ++++++ lib/rubygems/compact_index_client/parser.rb | 5 ++++- test/rubygems/test_gem_compact_index_client.rb | 15 +++++++++++++++ .../test_gem_compact_index_client_cache.rb | 9 +++++++++ 5 files changed, 43 insertions(+), 1 deletion(-) diff --git a/lib/rubygems/compact_index_client.rb b/lib/rubygems/compact_index_client.rb index 3fc38684722e..84e629d8dd5b 100644 --- a/lib/rubygems/compact_index_client.rb +++ b/lib/rubygems/compact_index_client.rb @@ -64,6 +64,15 @@ def info(name) @parser.info(name) end + # Fetches a single gem's info without consulting the versions index, + # using a conditional request to refresh the cached file. Useful when + # only a few gems are needed and the versions index download would + # dominate, as in gem install. + def fetch_info(name) + Gem::CompactIndexClient.debug { "fetch_info(#{name})" } + @parser.parse_info(@cache.fetch_info(name), name) + end + def latest_version(name) Gem::CompactIndexClient.debug { "latest_version(#{name})" } @parser.info(name).map {|d| Gem::Version.new(d[INFO_VERSION]) }.max diff --git a/lib/rubygems/compact_index_client/cache.rb b/lib/rubygems/compact_index_client/cache.rb index 8992b1bd80bb..98f946a73bd4 100644 --- a/lib/rubygems/compact_index_client/cache.rb +++ b/lib/rubygems/compact_index_client/cache.rb @@ -41,6 +41,12 @@ def info(name, remote_checksum = nil) end end + # Fetch a single gem's info file without consulting the versions + # index, refreshing the cached file with a conditional request. + def fetch_info(name) + fetch("info/#{name}", info_path(name), info_etag_path(name)) + end + def reset! @mutex.synchronize { @endpoints.clear } end diff --git a/lib/rubygems/compact_index_client/parser.rb b/lib/rubygems/compact_index_client/parser.rb index e36164cb2454..dac3ccd6e0fb 100644 --- a/lib/rubygems/compact_index_client/parser.rb +++ b/lib/rubygems/compact_index_client/parser.rb @@ -40,7 +40,10 @@ def versions end def info(name) - data = @compact_index.info(name, info_checksums[name]) + parse_info(@compact_index.info(name, info_checksums[name]), name) + end + + def parse_info(data, name) lines(data).map {|line| gem_parser.parse(line).unshift(name) } end diff --git a/test/rubygems/test_gem_compact_index_client.rb b/test/rubygems/test_gem_compact_index_client.rb index e877c938fb48..2a05cfd6397e 100644 --- a/test/rubygems/test_gem_compact_index_client.rb +++ b/test/rubygems/test_gem_compact_index_client.rb @@ -86,6 +86,21 @@ def test_not_available_without_data refute client.available? end + def test_fetch_info_does_not_fetch_versions_index + info = @client.fetch_info("a") + + assert_equal %w[info/a], @fetcher.requests + assert_equal "1.1.0", info.last[Gem::CompactIndexClient::INFO_VERSION] + assert_includes info.last[Gem::CompactIndexClient::INFO_REQS], ["created_at", ["2026-06-05T10:30:45Z"]] + end + + def test_fetch_info_fetches_once_per_process + @client.fetch_info("a") + @client.fetch_info("a") + + assert_equal %w[info/a], @fetcher.requests + end + def test_reset_refetches_versions @client.versions @client.reset! diff --git a/test/rubygems/test_gem_compact_index_client_cache.rb b/test/rubygems/test_gem_compact_index_client_cache.rb index 82a7dd1baab2..0e79139b6141 100644 --- a/test/rubygems/test_gem_compact_index_client_cache.rb +++ b/test/rubygems/test_gem_compact_index_client_cache.rb @@ -104,6 +104,15 @@ def test_info_without_checksum_reads_cached_file assert_empty fetcher.requests end + def test_fetch_info_fetches_without_checksum + fetcher = FakeFetcher.new("a 1.0.0\n") + cache = Gem::CompactIndexClient::Cache.new(@dir, fetcher) + + assert_equal "a 1.0.0\n", cache.fetch_info("a") + assert_equal ["info/a"], fetcher.requests + assert_equal "a 1.0.0\n", @dir.join("info", "a").read + end + def test_info_with_special_characters_uses_hashed_path fetcher = FakeFetcher.new("1.0.0\n") cache = Gem::CompactIndexClient::Cache.new(@dir, fetcher) From 00f7205401aa1a7465ebbbf2b11e50e7e75a63b1 Mon Sep 17 00:00:00 2001 From: Hiroshi SHIBATA Date: Wed, 10 Jun 2026 14:43:26 +0900 Subject: [PATCH 09/19] Update Manifest.txt for Gem::CompactIndexClient Co-Authored-By: Claude Fable 5 --- Manifest.txt | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Manifest.txt b/Manifest.txt index fed1f772c476..c6110e44e382 100644 --- a/Manifest.txt +++ b/Manifest.txt @@ -404,6 +404,12 @@ lib/rubygems/commands/unpack_command.rb lib/rubygems/commands/update_command.rb lib/rubygems/commands/which_command.rb lib/rubygems/commands/yank_command.rb +lib/rubygems/compact_index_client.rb +lib/rubygems/compact_index_client/cache.rb +lib/rubygems/compact_index_client/cache_file.rb +lib/rubygems/compact_index_client/http_fetcher.rb +lib/rubygems/compact_index_client/parser.rb +lib/rubygems/compact_index_client/updater.rb lib/rubygems/config_file.rb lib/rubygems/core_ext/kernel_gem.rb lib/rubygems/core_ext/kernel_require.rb From b8907a8aee61a01cdc2b7216c017f31ee19e49c4 Mon Sep 17 00:00:00 2001 From: Hiroshi SHIBATA Date: Wed, 10 Jun 2026 14:43:43 +0900 Subject: [PATCH 10/19] Fetch the compact index in APISet through Gem::CompactIndexClient Info files are now cached on disk under Gem.spec_cache_dir and refreshed with ETag conditional requests instead of being downloaded in full on every resolution. APISet keeps fetching only the info files it needs (via fetch_info) rather than the whole versions index, so the first-use cost stays at the current level. When the user's home is not safely writable the cache falls back to a temporary directory. Co-Authored-By: Claude Fable 5 --- lib/rubygems/resolver/api_set.rb | 73 ++++++++++++++------- test/rubygems/test_gem_resolver_api_set.rb | 24 +++---- test/rubygems/test_gem_resolver_best_set.rb | 2 +- 3 files changed, 62 insertions(+), 37 deletions(-) diff --git a/lib/rubygems/resolver/api_set.rb b/lib/rubygems/resolver/api_set.rb index 3f443519d80f..c1b115f97a78 100644 --- a/lib/rubygems/resolver/api_set.rb +++ b/lib/rubygems/resolver/api_set.rb @@ -35,6 +35,7 @@ def initialize(dep_uri = "https://index.rubygems.org/info/") @dep_uri = dep_uri @uri = dep_uri + ".." + @client = nil @data = Hash.new {|h,k| h[k] = [] } @source = Gem::Source.new @uri @@ -99,26 +100,20 @@ def pretty_print(q) # :nodoc: # Return data for all versions of the gem +name+. def versions(name) # :nodoc: - if @data.key?(name) - return @data[name] - end - - uri = @dep_uri + name + return @data[name] if @data.key?(name) - begin - str = Gem::RemoteFetcher.fetcher.fetch_path uri - rescue Gem::RemoteFetcher::FetchError - @data[name] = [] - else - lines(str).each do |ver| - number, platform, dependencies, requirements = parse_gem(ver) + infos = begin + client.fetch_info(name) + rescue Gem::RemoteFetcher::FetchError, Gem::CompactIndexClient::Error + [] + end - platform ||= "ruby" - dependencies = dependencies.map {|dep_name, reqs| [dep_name, reqs.join(", ")] } - requirements = requirements.map {|req_name, reqs| [req_name.to_sym, reqs] }.to_h + infos.each do |_, number, platform, dependencies, requirements| + platform ||= "ruby" + dependencies = dependencies.map {|dep_name, reqs| [dep_name, reqs.join(", ")] } + requirements = requirements.map {|req_name, reqs| [req_name.to_sym, reqs] }.to_h - @data[name] << { name: name, number: number, platform: platform, dependencies: dependencies, requirements: requirements } - end + @data[name] << { name: name, number: number, platform: platform, dependencies: dependencies, requirements: requirements } end @data[name] @@ -126,14 +121,44 @@ def versions(name) # :nodoc: private - def lines(str) - lines = str.split("\n") - header = lines.index("---") - header ? lines[header + 1..-1] : lines + def client # :nodoc: + @client ||= begin + # Loaded lazily because resolver.rb requires this file at load time + # and the client transitively references resolver constants. + require_relative "../compact_index_client" + + Gem::CompactIndexClient.new(cache_dir, Gem::CompactIndexClient::HTTPFetcher.new(@uri)) + end + end + + ## + # The compact index cache directory for this source. Falls back to a + # temporary directory when the user's cache directory cannot safely be + # written to. + + def cache_dir + if update_cache? + # Correct for windows paths + escaped_path = @uri.path.sub(%r{^/([a-z]):/}i, '/\\1-/') + + File.join Gem.spec_cache_dir, "compact_index", + "#{@uri.host}%#{@uri.port}", *escaped_path.split("/").reject(&:empty?) + else + require "tmpdir" + Dir.mktmpdir "gem_compact_index" + end end - def parse_gem(string) - @gem_parser ||= GemParser.new - @gem_parser.parse(string) + ## + # Returns true when it is possible and safe to update the cache directory. + + def update_cache? + return @update_cache unless @update_cache.nil? + @update_cache = + begin + File.stat(Gem.user_home).uid == Process.uid + rescue Errno::ENOENT + false + end end end diff --git a/test/rubygems/test_gem_resolver_api_set.rb b/test/rubygems/test_gem_resolver_api_set.rb index b0b4943beafa..af855f1692ad 100644 --- a/test/rubygems/test_gem_resolver_api_set.rb +++ b/test/rubygems/test_gem_resolver_api_set.rb @@ -42,7 +42,7 @@ def test_find_all dependencies: [] }, ] - @fetcher.data["#{@dep_uri}a"] = "---\n1 " + @fetcher.data["#{@dep_uri}a"] = util_compact_index_response("---\n1 ") set = Gem::Resolver::APISet.new @dep_uri @@ -69,7 +69,7 @@ def test_find_all_prereleases dependencies: [] }, ] - @fetcher.data["#{@dep_uri}a"] = "---\n1\n2.a" + @fetcher.data["#{@dep_uri}a"] = util_compact_index_response("---\n1\n2.a") set = Gem::Resolver::APISet.new @dep_uri set.prerelease = true @@ -94,7 +94,7 @@ def test_find_all_cache dependencies: [] }, ] - @fetcher.data["#{@dep_uri}a"] = "---\n1 " + @fetcher.data["#{@dep_uri}a"] = util_compact_index_response("---\n1 ") set = Gem::Resolver::APISet.new @dep_uri @@ -123,7 +123,7 @@ def test_find_all_local def test_find_all_missing spec_fetcher - @fetcher.data["#{@dep_uri}a"] = "---" + @fetcher.data["#{@dep_uri}a"] = util_compact_index_response("---") set = Gem::Resolver::APISet.new @dep_uri @@ -158,8 +158,8 @@ def test_find_all_not_found def test_prefetch spec_fetcher - @fetcher.data["#{@dep_uri}a"] = "---\n1 \n" - @fetcher.data["#{@dep_uri}b"] = "---" + @fetcher.data["#{@dep_uri}a"] = util_compact_index_response("---\n1 \n") + @fetcher.data["#{@dep_uri}b"] = util_compact_index_response("---") set = Gem::Resolver::APISet.new @dep_uri @@ -175,7 +175,7 @@ def test_prefetch def test_prefetch_cache spec_fetcher - @fetcher.data["#{@dep_uri}a"] = "---\n1 \n" + @fetcher.data["#{@dep_uri}a"] = util_compact_index_response("---\n1 \n") set = Gem::Resolver::APISet.new @dep_uri @@ -185,7 +185,7 @@ def test_prefetch_cache set.prefetch [a_dep] @fetcher.data.delete "#{@dep_uri}a" - @fetcher.data["#{@dep_uri}?b"] = "---" + @fetcher.data["#{@dep_uri}?b"] = util_compact_index_response("---") set.prefetch [a_dep, b_dep] end @@ -193,8 +193,8 @@ def test_prefetch_cache def test_prefetch_cache_missing spec_fetcher - @fetcher.data["#{@dep_uri}a"] = "---\n1 \n" - @fetcher.data["#{@dep_uri}b"] = "---" + @fetcher.data["#{@dep_uri}a"] = util_compact_index_response("---\n1 \n") + @fetcher.data["#{@dep_uri}b"] = util_compact_index_response("---") set = Gem::Resolver::APISet.new @dep_uri @@ -212,8 +212,8 @@ def test_prefetch_cache_missing def test_prefetch_local spec_fetcher - @fetcher.data["#{@dep_uri}a"] = "---\n1 \n" - @fetcher.data["#{@dep_uri}b"] = "---" + @fetcher.data["#{@dep_uri}a"] = util_compact_index_response("---\n1 \n") + @fetcher.data["#{@dep_uri}b"] = util_compact_index_response("---") set = Gem::Resolver::APISet.new @dep_uri set.remote = false diff --git a/test/rubygems/test_gem_resolver_best_set.rb b/test/rubygems/test_gem_resolver_best_set.rb index ac186884d15b..dfe50f02d76a 100644 --- a/test/rubygems/test_gem_resolver_best_set.rb +++ b/test/rubygems/test_gem_resolver_best_set.rb @@ -16,7 +16,7 @@ def test_find_all api_uri = Gem::URI "#{@gem_repo}info/" - @fetcher.data["#{api_uri}a"] = "---\n1 " + @fetcher.data["#{api_uri}a"] = util_compact_index_response("---\n1 ") set = Gem::Resolver::BestSet.new From 2cb9f6363f35791086e604da42ba76e3ffd2f302 Mon Sep 17 00:00:00 2001 From: Hiroshi SHIBATA Date: Wed, 10 Jun 2026 14:45:59 +0900 Subject: [PATCH 11/19] Expose created_at on Gem::Resolver::APISpecification The compact index v2 publishes per-version creation timestamps which the parser already preserves. Keep them on resolver specifications so features like cooldown can consult publish dates during resolution. Sources without timestamps leave created_at nil. Co-Authored-By: Claude Fable 5 --- lib/rubygems/resolver/api_specification.rb | 14 +++++++++ lib/rubygems/resolver/specification.rb | 7 +++++ .../test_gem_resolver_api_specification.rb | 31 +++++++++++++++++++ 3 files changed, 52 insertions(+) diff --git a/lib/rubygems/resolver/api_specification.rb b/lib/rubygems/resolver/api_specification.rb index ccfd6fe0843b..d630fb484fe6 100644 --- a/lib/rubygems/resolver/api_specification.rb +++ b/lib/rubygems/resolver/api_specification.rb @@ -38,6 +38,7 @@ def initialize(set, api_data) end.freeze @required_ruby_version = Gem::Requirement.new(api_data.dig(:requirements, :ruby)).freeze @required_rubygems_version = Gem::Requirement.new(api_data.dig(:requirements, :rubygems)).freeze + @created_at = parse_created_at(api_data.dig(:requirements, :created_at))&.freeze end def ==(other) # :nodoc: @@ -102,4 +103,17 @@ def spec # :nodoc: def source # :nodoc: @set.source end + + private + + def parse_created_at(value) + value = value.first if value.is_a?(Array) + return unless value.is_a?(String) + + begin + Time.new(value) + rescue ArgumentError + nil + end + end end diff --git a/lib/rubygems/resolver/specification.rb b/lib/rubygems/resolver/specification.rb index d2098ef0e2aa..986fa7c9ae82 100644 --- a/lib/rubygems/resolver/specification.rb +++ b/lib/rubygems/resolver/specification.rb @@ -54,10 +54,17 @@ class Gem::Resolver::Specification attr_reader :required_rubygems_version + ## + # The time this gem version was published, when the source provides it + # (compact index v2), nil otherwise. + + attr_reader :created_at + ## # Sets default instance variables for the specification. def initialize + @created_at = nil @dependencies = nil @name = nil @platform = nil diff --git a/test/rubygems/test_gem_resolver_api_specification.rb b/test/rubygems/test_gem_resolver_api_specification.rb index 2119d734780b..ed6e6615c353 100644 --- a/test/rubygems/test_gem_resolver_api_specification.rb +++ b/test/rubygems/test_gem_resolver_api_specification.rb @@ -27,6 +27,37 @@ def test_initialize ] assert_equal expected, spec.dependencies + assert_nil spec.created_at + end + + def test_initialize_created_at + set = Gem::Resolver::APISet.new + data = { + name: "rails", + number: "3.0.3", + platform: "ruby", + dependencies: [], + requirements: { created_at: ["2026-06-05T10:30:45Z"] }, + } + + spec = Gem::Resolver::APISpecification.new set, data + + assert_equal Time.new("2026-06-05T10:30:45Z"), spec.created_at + end + + def test_initialize_created_at_invalid + set = Gem::Resolver::APISet.new + data = { + name: "rails", + number: "3.0.3", + platform: "ruby", + dependencies: [], + requirements: { created_at: ["not a timestamp"] }, + } + + spec = Gem::Resolver::APISpecification.new set, data + + assert_nil spec.created_at end def test_fetch_development_dependencies From ce7ae2814e09617aba6910a569714afe54c54b4a Mon Sep 17 00:00:00 2001 From: Hiroshi SHIBATA Date: Wed, 10 Jun 2026 14:51:32 +0900 Subject: [PATCH 12/19] Build APISpecification#spec from compact index data The compact index info file carries everything needed to download and install a gem, so materializing the resolved specification no longer fetches the Marshal gemspec from /quick/. Development dependencies are not part of the info file; fetch_development_dependencies still fetches the full gemspec for --development installs. Co-Authored-By: Claude Fable 5 --- lib/rubygems/resolver/api_specification.rb | 26 ++++++++++++------- .../rubygems/test_gem_dependency_installer.rb | 23 ++++++++++++++++ 2 files changed, 39 insertions(+), 10 deletions(-) diff --git a/lib/rubygems/resolver/api_specification.rb b/lib/rubygems/resolver/api_specification.rb index d630fb484fe6..070877ac7653 100644 --- a/lib/rubygems/resolver/api_specification.rb +++ b/lib/rubygems/resolver/api_specification.rb @@ -85,19 +85,25 @@ def pretty_print(q) # :nodoc: end ## - # Fetches a Gem::Specification for this APISpecification. + # A Gem::Specification stub built from the Compact Index data for this + # specification. The compact index carries everything needed to + # download and install the gem, so the Marshal gemspec is not fetched. + # Development dependencies are not included; see + # #fetch_development_dependencies. def spec # :nodoc: - @spec ||= - begin - tuple = Gem::NameTuple.new @name, @version, @platform - source.fetch_spec tuple - rescue Gem::RemoteFetcher::FetchError - raise if @original_platform == @platform - - tuple = Gem::NameTuple.new @name, @version, @original_platform - source.fetch_spec tuple + @spec ||= Gem::Specification.new do |s| + s.name = @name + s.version = @version + s.platform = @platform + s.original_platform = @original_platform + s.required_ruby_version = @required_ruby_version + s.required_rubygems_version = @required_rubygems_version + + @dependencies.each do |dependency| + s.add_runtime_dependency dependency.name, *dependency.requirement.as_list end + end end def source # :nodoc: diff --git a/test/rubygems/test_gem_dependency_installer.rb b/test/rubygems/test_gem_dependency_installer.rb index c2fb6f264b92..27fad9513536 100644 --- a/test/rubygems/test_gem_dependency_installer.rb +++ b/test/rubygems/test_gem_dependency_installer.rb @@ -506,6 +506,29 @@ def test_install_local_dependency_no_network_for_target_gem assert_equal %w[a-1 b-1], inst.installed_gems.map(&:full_name) end + def test_install_compact_index_api + a1, a1_gem = util_gem "a", 1, "b" => ">= 1" + b1, b1_gem = util_gem "b", 1 + + util_setup_compact_index a1, b1 + + add_to_fetcher a1, a1_gem + add_to_fetcher b1, b1_gem + + # the compact index probe succeeds, so resolution goes through APISet + response = Gem::HTTPResponseFactory.create(body: "", code: 200, msg: "OK") + response.uri = Gem::URI("#{@gem_repo}versions") + @fetcher.data["#{@gem_repo}versions"] = response + + inst = Gem::DependencyInstaller.new + inst.install "a" + + assert_equal %w[a-1 b-1], inst.installed_gems.map(&:full_name) + + quick_gemspec_fetches = @fetcher.paths.grep(/gemspec\.rz/) + assert_empty quick_gemspec_fetches + end + def test_install_local_subdir util_setup_gems From 625a5a772d5d5858179404f74a0025c0e1385965 Mon Sep 17 00:00:00 2001 From: Hiroshi SHIBATA Date: Wed, 10 Jun 2026 15:00:42 +0900 Subject: [PATCH 13/19] Load spec name tuples from the compact index in Gem::Source Gem::Source#load_specs now builds the released, latest and prerelease name tuple lists from the compact index versions file, falling back to the Marshal spec indexes when the source does not provide a usable compact index. This moves gem update, outdated, list and search off Marshal data for compact index sources. The client construction moves to Gem::Source so APISet and load_specs share one client and disk cache per source. Co-Authored-By: Claude Fable 5 --- lib/rubygems/resolver/api_set.rb | 39 +--------- lib/rubygems/source.rb | 126 ++++++++++++++++++++++++++----- test/rubygems/test_gem_source.rb | 54 +++++++++++++ 3 files changed, 161 insertions(+), 58 deletions(-) diff --git a/lib/rubygems/resolver/api_set.rb b/lib/rubygems/resolver/api_set.rb index c1b115f97a78..c3dc4c0bcabc 100644 --- a/lib/rubygems/resolver/api_set.rb +++ b/lib/rubygems/resolver/api_set.rb @@ -122,43 +122,6 @@ def versions(name) # :nodoc: private def client # :nodoc: - @client ||= begin - # Loaded lazily because resolver.rb requires this file at load time - # and the client transitively references resolver constants. - require_relative "../compact_index_client" - - Gem::CompactIndexClient.new(cache_dir, Gem::CompactIndexClient::HTTPFetcher.new(@uri)) - end - end - - ## - # The compact index cache directory for this source. Falls back to a - # temporary directory when the user's cache directory cannot safely be - # written to. - - def cache_dir - if update_cache? - # Correct for windows paths - escaped_path = @uri.path.sub(%r{^/([a-z]):/}i, '/\\1-/') - - File.join Gem.spec_cache_dir, "compact_index", - "#{@uri.host}%#{@uri.port}", *escaped_path.split("/").reject(&:empty?) - else - require "tmpdir" - Dir.mktmpdir "gem_compact_index" - end - end - - ## - # Returns true when it is possible and safe to update the cache directory. - - def update_cache? - return @update_cache unless @update_cache.nil? - @update_cache = - begin - File.stat(Gem.user_home).uid == Process.uid - rescue Errno::ENOENT - false - end + @client ||= @source.compact_index_client end end diff --git a/lib/rubygems/source.rb b/lib/rubygems/source.rb index 86717e3e7178..aa6d232d3fb5 100644 --- a/lib/rubygems/source.rb +++ b/lib/rubygems/source.rb @@ -155,8 +155,60 @@ def fetch_spec(name_tuple) # :latest => Return the list of only the highest version of each gem # :prerelease => Return the list of all prerelease only specs # + # The compact index is used when the source provides it, falling back + # to the Marshal spec indexes. def load_specs(type) + load_compact_index_specs(type) || load_marshal_specs(type) + end + + ## + # The compact index client for this source, caching under + # Gem.spec_cache_dir. Also used by Gem::Resolver::APISet. + + def compact_index_client # :nodoc: + @compact_index_client ||= begin + require_relative "compact_index_client" + + index_uri = compact_index_uri + + Gem::CompactIndexClient.new(compact_index_cache_dir(index_uri), + Gem::CompactIndexClient::HTTPFetcher.new(index_uri)) + end + end + + ## + # Downloads +spec+ and writes it to +dir+. See also + # Gem::RemoteFetcher#download. + + def download(spec, dir = Dir.pwd) + fetcher = Gem::RemoteFetcher.fetcher + fetcher.download spec, uri.to_s, dir + end + + def pretty_print(q) # :nodoc: + q.object_group(self) do + q.group 2, "[Remote:", "]" do + q.breakable + q.text @uri.to_s + + if api = uri + q.breakable + q.text "API URI: " + q.text api.to_s + end + end + end + end + + def typo_squatting?(host, distance_threshold = 4) + return if @uri.host.nil? + levenshtein_distance(@uri.host, host).between? 1, distance_threshold + end + + private + + def load_marshal_specs(type) file = FILES[type] fetcher = Gem::RemoteFetcher.fetcher file_name = "#{file}.#{Gem.marshal_version}" @@ -187,35 +239,69 @@ def load_specs(type) end ## - # Downloads +spec+ and writes it to +dir+. See also - # Gem::RemoteFetcher#download. + # Builds the name tuple list for +type+ from the compact index versions + # file. Returns nil when the source does not provide a usable compact + # index, so the caller can fall back to the Marshal spec indexes. - def download(spec, dir = Dir.pwd) - fetcher = Gem::RemoteFetcher.fetcher - fetcher.download spec, uri.to_s, dir - end + def load_compact_index_specs(type) + return unless %w[http https].include?(uri.scheme) - def pretty_print(q) # :nodoc: - q.object_group(self) do - q.group 2, "[Remote:", "]" do - q.breakable - q.text @uri.to_s + versions = compact_index_versions + return if versions.nil? || versions.empty? - if api = uri - q.breakable - q.text "API URI: " - q.text api.to_s - end + tuples = [] + + versions.each_value do |rows| + gem_tuples = rows.filter_map do |name, version_string, platform| + version = Gem::Version.new(version_string) + next if version.prerelease? != (type == :prerelease) + + Gem::NameTuple.new(name, version, platform || "ruby") end + + gem_tuples = max_versions_by_platform(gem_tuples) if type == :latest + + tuples.concat(gem_tuples) end + + tuples + rescue ArgumentError + nil end - def typo_squatting?(host, distance_threshold = 4) - return if @uri.host.nil? - levenshtein_distance(@uri.host, host).between? 1, distance_threshold + def compact_index_versions + @compact_index_versions ||= compact_index_client.versions + rescue Gem::RemoteFetcher::FetchError, Gem::CompactIndexClient::Error + @compact_index_versions = {} + nil end - private + def max_versions_by_platform(tuples) + tuples.group_by(&:platform).map {|_, platform_tuples| platform_tuples.max_by(&:version) } + end + + def compact_index_uri + if uri.host == "rubygems.org" + index_uri = uri.dup + index_uri.host = "index.rubygems.org" + index_uri + else + uri + end + end + + def compact_index_cache_dir(index_uri) + if update_cache? + # Correct for windows paths + escaped_path = index_uri.path.sub(%r{^/([a-z]):/}i, '/\\1-/') + + File.join Gem.spec_cache_dir, "compact_index", + "#{index_uri.host}%#{index_uri.port}", *escaped_path.split("/").reject(&:empty?) + else + require "tmpdir" + Dir.mktmpdir "gem_compact_index" + end + end def new_dependency_resolver_set return Gem::Resolver::IndexSet.new self if uri.scheme == "file" diff --git a/test/rubygems/test_gem_source.rb b/test/rubygems/test_gem_source.rb index 423abd6dd20a..cd9f3398532b 100644 --- a/test/rubygems/test_gem_source.rb +++ b/test/rubygems/test_gem_source.rb @@ -132,6 +132,60 @@ def test_load_specs assert File.exist?(cache_file) end + def test_load_specs_compact_index + a1 = util_spec "a", "1" + a2 = util_spec "a", "2" + a3_pre = util_spec "a", "3.a" + b2_java = util_spec "b", "2" do |s| + s.platform = "java" + end + + util_setup_compact_index a1, a2, a3_pre, b2_java + + released = @source.load_specs(:released).map(&:full_name) + assert_equal %w[a-1 a-2 b-2-java], released + + latest = @source.load_specs(:latest).map(&:full_name) + assert_equal %w[a-2 b-2-java], latest + + prerelease = @source.load_specs(:prerelease).map(&:full_name) + assert_equal %w[a-3.a], prerelease + + cache_dir = File.join Gem.spec_cache_dir, "compact_index", "gems.example.com%80" + assert File.exist?(File.join(cache_dir, "versions")), "versions cache file does not exist" + end + + def test_load_specs_compact_index_latest_per_platform + a1 = util_spec "a", "1" + a2_java = util_spec "a", "2" do |s| + s.platform = "java" + end + + util_setup_compact_index a1, a2_java + + latest = @source.load_specs(:latest).map(&:full_name) + assert_equal %w[a-1 a-2-java], latest + end + + def test_load_specs_compact_index_fetched_once + a1 = util_spec "a", "1" + + util_setup_compact_index a1 + + @source.load_specs :released + @source.load_specs :prerelease + + versions_requests = @fetcher.requests.count {|req| req.path.end_with?("/versions") } + assert_equal 1, versions_requests + end + + def test_load_specs_falls_back_to_marshal_index + # no compact index data set up, only the Marshal indexes from setup + released = @source.load_specs(:released).map(&:full_name) + + assert_equal %W[a-2 a-1 b-2], released + end + def test_load_specs_cached latest_specs = @source.load_specs :latest From 4d1aa45dc3d6ec07da36f30396d1e56f719cf49f Mon Sep 17 00:00:00 2001 From: Hiroshi SHIBATA Date: Wed, 10 Jun 2026 15:08:23 +0900 Subject: [PATCH 14/19] Add compact index coverage for gem update and gem outdated End-to-end tests driving both commands against a stubbed compact index with no usable Marshal data, proving the SpecFetcher path works through Gem::Source#load_specs. The stubbed versions response now carries a response uri so the dependency_resolver_set probe and the compact index fetch share one endpoint, as on real servers. Co-Authored-By: Claude Fable 5 --- test/rubygems/helper.rb | 7 ++++- .../test_gem_commands_outdated_command.rb | 20 +++++++++++++ .../test_gem_commands_update_command.rb | 28 +++++++++++++++++++ 3 files changed, 54 insertions(+), 1 deletion(-) diff --git a/test/rubygems/helper.rb b/test/rubygems/helper.rb index e5158ff0a9ff..4c47d46835e3 100644 --- a/test/rubygems/helper.rb +++ b/test/rubygems/helper.rb @@ -1200,7 +1200,12 @@ def util_setup_compact_index(*specs, created_at: {}) @fetcher.data["#{@gem_repo}info/#{name}"] = util_compact_index_response(info_body) end - @fetcher.data["#{@gem_repo}versions"] = util_compact_index_response(versions_body) + versions_response = util_compact_index_response(versions_body) + # Gem::Source#dependency_resolver_set probes this URL via fetch_path and + # builds the info URL from the response uri + versions_response.uri = Gem::URI("#{@gem_repo}versions") + + @fetcher.data["#{@gem_repo}versions"] = versions_response @fetcher.data["#{@gem_repo}names"] = util_compact_index_response(names_body) nil diff --git a/test/rubygems/test_gem_commands_outdated_command.rb b/test/rubygems/test_gem_commands_outdated_command.rb index 3e61033af366..07289d821fba 100644 --- a/test/rubygems/test_gem_commands_outdated_command.rb +++ b/test/rubygems/test_gem_commands_outdated_command.rb @@ -30,6 +30,26 @@ def test_execute assert_equal "", @ui.error end + def test_execute_compact_index + spec_fetcher do |fetcher| + fetcher.gem "foo", "0.2" + end + + foo2 = util_spec "foo", "2.0" + util_setup_compact_index foo2 + + # drop the in-memory tuples spec_fetcher pre-populated so the lookup + # goes through Gem::Source#load_specs + Gem::SpecFetcher.fetcher = nil + + use_ui @ui do + @cmd.execute + end + + assert_equal "foo (0.2 < 2.0)\n", @ui.output + assert_equal "", @ui.error + end + def test_execute_with_up_to_date_platform_specific_gem spec_fetcher do |fetcher| fetcher.download "foo", "2.0" diff --git a/test/rubygems/test_gem_commands_update_command.rb b/test/rubygems/test_gem_commands_update_command.rb index 5ed12ad48140..5bc36d61035e 100644 --- a/test/rubygems/test_gem_commands_update_command.rb +++ b/test/rubygems/test_gem_commands_update_command.rb @@ -42,6 +42,34 @@ def test_execute assert_empty out end + def test_execute_compact_index + spec_fetcher do |fetcher| + fetcher.gem "b", 1 + end + + b2, b2_gem = util_gem "b", 2 + util_setup_compact_index b2 + add_to_fetcher b2, b2_gem + + # drop the in-memory tuples spec_fetcher pre-populated so the lookup + # goes through Gem::Source#load_specs + Gem::SpecFetcher.fetcher = nil + + @cmd.options[:args] = [] + + use_ui @ui do + @cmd.execute + end + + out = @ui.output.split "\n" + assert_equal "Updating installed gems", out.shift + assert_equal "Updating b", out.shift + assert_equal "Gems updated: b", out.shift + assert_empty out + + assert_path_exist File.join(@gemhome, "specifications", "b-2.gemspec") + end + def test_execute_multiple spec_fetcher do |fetcher| fetcher.download "a", 2 From 24f03011930c53f92dc83a64c3fb9ae75bbdb94d Mon Sep 17 00:00:00 2001 From: Hiroshi SHIBATA Date: Wed, 10 Jun 2026 16:19:38 +0900 Subject: [PATCH 15/19] Skip malformed digest header parameters A Digest or Repr-Digest parameter without a value (no "=") made byte_sequence raise NoMethodError on nil, failing the whole fetch when a mirror or proxy sends a broken header. The same flaw exists in the Bundler implementation this was ported from. Co-Authored-By: Claude Fable 5 --- lib/rubygems/compact_index_client/updater.rb | 1 + test/rubygems/test_gem_compact_index_client_updater.rb | 10 ++++++++++ 2 files changed, 11 insertions(+) diff --git a/lib/rubygems/compact_index_client/updater.rb b/lib/rubygems/compact_index_client/updater.rb index eef094b611d9..79a6cdfe7d82 100644 --- a/lib/rubygems/compact_index_client/updater.rb +++ b/lib/rubygems/compact_index_client/updater.rb @@ -86,6 +86,7 @@ def parse_digests(response) algorithm.strip! algorithm.downcase! next unless SUPPORTED_DIGESTS.key?(algorithm) + next unless value next unless value = byte_sequence(value) digests[algorithm] = value end diff --git a/test/rubygems/test_gem_compact_index_client_updater.rb b/test/rubygems/test_gem_compact_index_client_updater.rb index 04aa1965b782..b3f8d277540a 100644 --- a/test/rubygems/test_gem_compact_index_client_updater.rb +++ b/test/rubygems/test_gem_compact_index_client_updater.rb @@ -157,6 +157,16 @@ def test_update_parses_weak_etag assert_equal "weak1", @etag_path.read end + def test_update_ignores_malformed_digest_header + response = FakeResponse.new("a 1.0.0\n", "Repr-Digest" => "sha-256") + fetcher = FakeFetcher.new(response) + updater = Gem::CompactIndexClient::Updater.new(fetcher) + + updater.update("versions", @local_path, @etag_path) + + assert_equal "a 1.0.0\n", @local_path.read + end + def test_update_ignores_unsupported_digest_algorithms response = FakeResponse.new("a 1.0.0\n", "Repr-Digest" => "md5=:#{Digest::MD5.base64digest("bogus")}:") From 3d643fc60fd4a9bc6be54575eb27c1d8bddb65bf Mon Sep 17 00:00:00 2001 From: Hiroshi SHIBATA Date: Wed, 10 Jun 2026 16:19:51 +0900 Subject: [PATCH 16/19] Follow 308 redirects in compact index HTTPFetcher Co-Authored-By: Claude Fable 5 --- .../compact_index_client/http_fetcher.rb | 2 +- ...test_gem_compact_index_client_http_fetcher.rb | 16 ++++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/lib/rubygems/compact_index_client/http_fetcher.rb b/lib/rubygems/compact_index_client/http_fetcher.rb index daef0c686faf..2d439df6d17b 100644 --- a/lib/rubygems/compact_index_client/http_fetcher.rb +++ b/lib/rubygems/compact_index_client/http_fetcher.rb @@ -34,7 +34,7 @@ def fetch(uri, headers, redirects_remaining) when Gem::Net::HTTPSuccess, Gem::Net::HTTPNotModified response when Gem::Net::HTTPMovedPermanently, Gem::Net::HTTPFound, Gem::Net::HTTPSeeOther, - Gem::Net::HTTPTemporaryRedirect + Gem::Net::HTTPTemporaryRedirect, Gem::Net::HTTPPermanentRedirect raise Gem::RemoteFetcher::FetchError.new("too many redirects", uri) if redirects_remaining.zero? location = response["Location"] diff --git a/test/rubygems/test_gem_compact_index_client_http_fetcher.rb b/test/rubygems/test_gem_compact_index_client_http_fetcher.rb index c2c948df651d..c563482dad89 100644 --- a/test/rubygems/test_gem_compact_index_client_http_fetcher.rb +++ b/test/rubygems/test_gem_compact_index_client_http_fetcher.rb @@ -21,6 +21,13 @@ def initialize(location) end end + class FakePermanentRedirect < Gem::Net::HTTPPermanentRedirect + def initialize(location) + super("1.1", "308", "Permanent Redirect") + self["Location"] = location + end + end + class FakeNotFound < Gem::Net::HTTPNotFound def initialize super("1.1", "404", "Not Found") @@ -79,6 +86,15 @@ def test_call_follows_redirects assert_equal 2, remote.requests.size end + def test_call_follows_permanent_redirects + fetcher, _remote = fetcher_for( + "https://index.example/versions" => FakePermanentRedirect.new("https://mirror.example/versions"), + "https://mirror.example/versions" => FakeResponse.new("data") + ) + + assert_equal "data", fetcher.call("versions").body + end + def test_call_resolves_relative_redirect_location fetcher, _remote = fetcher_for( "https://index.example/versions" => FakeRedirect.new("/v2/versions"), From 66e843130b5e13e23c553b3e3d22b27355a8e270 Mon Sep 17 00:00:00 2001 From: Hiroshi SHIBATA Date: Wed, 10 Jun 2026 16:19:51 +0900 Subject: [PATCH 17/19] Reuse compact_index_uri in new_dependency_resolver_set Both methods computed the rubygems.org to index.rubygems.org rewrite independently; keep the logic in one place. Co-Authored-By: Claude Fable 5 --- lib/rubygems/source.rb | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/lib/rubygems/source.rb b/lib/rubygems/source.rb index aa6d232d3fb5..91f242e22a20 100644 --- a/lib/rubygems/source.rb +++ b/lib/rubygems/source.rb @@ -306,15 +306,7 @@ def compact_index_cache_dir(index_uri) def new_dependency_resolver_set return Gem::Resolver::IndexSet.new self if uri.scheme == "file" - fetch_uri = if uri.host == "rubygems.org" - index_uri = uri.dup - index_uri.host = "index.rubygems.org" - index_uri - else - uri - end - - bundler_api_uri = enforce_trailing_slash(fetch_uri) + "versions" + bundler_api_uri = enforce_trailing_slash(compact_index_uri) + "versions" begin fetcher = Gem::RemoteFetcher.fetcher From c6359875bf81d780f59a823356750e6bbe370278 Mon Sep 17 00:00:00 2001 From: Hiroshi SHIBATA Date: Wed, 10 Jun 2026 16:26:14 +0900 Subject: [PATCH 18/19] Guard pathname require with defined?(Pathname) Pathname is built into Ruby 4.0+, so only require the library when the constant is not already available. Co-Authored-By: Claude Fable 5 --- lib/rubygems/compact_index_client/cache.rb | 2 +- test/rubygems/test_gem_compact_index_client_cache.rb | 2 +- test/rubygems/test_gem_compact_index_client_cache_file.rb | 2 +- test/rubygems/test_gem_compact_index_client_updater.rb | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/rubygems/compact_index_client/cache.rb b/lib/rubygems/compact_index_client/cache.rb index 98f946a73bd4..34e20dfea0b8 100644 --- a/lib/rubygems/compact_index_client/cache.rb +++ b/lib/rubygems/compact_index_client/cache.rb @@ -2,7 +2,7 @@ require "digest" require "fileutils" -require "pathname" +require "pathname" unless defined?(Pathname) require "set" class Gem::CompactIndexClient diff --git a/test/rubygems/test_gem_compact_index_client_cache.rb b/test/rubygems/test_gem_compact_index_client_cache.rb index 0e79139b6141..f6ee2008b0f4 100644 --- a/test/rubygems/test_gem_compact_index_client_cache.rb +++ b/test/rubygems/test_gem_compact_index_client_cache.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true require_relative "helper" -require "pathname" +require "pathname" unless defined?(Pathname) require "rubygems/compact_index_client" class TestGemCompactIndexClientCache < Gem::TestCase diff --git a/test/rubygems/test_gem_compact_index_client_cache_file.rb b/test/rubygems/test_gem_compact_index_client_cache_file.rb index 749d6eba404a..6b28ea8bae46 100644 --- a/test/rubygems/test_gem_compact_index_client_cache_file.rb +++ b/test/rubygems/test_gem_compact_index_client_cache_file.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true require_relative "helper" -require "pathname" +require "pathname" unless defined?(Pathname) require "rubygems/compact_index_client" class TestGemCompactIndexClientCacheFile < Gem::TestCase diff --git a/test/rubygems/test_gem_compact_index_client_updater.rb b/test/rubygems/test_gem_compact_index_client_updater.rb index b3f8d277540a..3218faa5f5e1 100644 --- a/test/rubygems/test_gem_compact_index_client_updater.rb +++ b/test/rubygems/test_gem_compact_index_client_updater.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true require_relative "helper" -require "pathname" +require "pathname" unless defined?(Pathname) require "rubygems/compact_index_client" class TestGemCompactIndexClientUpdater < Gem::TestCase From a29a32f564ec818ed432d0dde37c1a270308f475 Mon Sep 17 00:00:00 2001 From: Hiroshi SHIBATA Date: Wed, 10 Jun 2026 18:26:21 +0900 Subject: [PATCH 19/19] Write compact index test fixtures in binary mode Pathname#write uses text mode, so on Windows the LF in fixture data became CRLF, shifting the file size the Range header is computed from and breaking the MD5/SHA-256 checksums. The client itself is unaffected since CacheFile always writes in binary mode. Co-Authored-By: Claude Fable 5 --- test/rubygems/test_gem_compact_index_client_cache.rb | 8 ++++---- .../test_gem_compact_index_client_cache_file.rb | 12 ++++++------ .../test_gem_compact_index_client_updater.rb | 10 +++++----- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/test/rubygems/test_gem_compact_index_client_cache.rb b/test/rubygems/test_gem_compact_index_client_cache.rb index f6ee2008b0f4..e34ddce6f633 100644 --- a/test/rubygems/test_gem_compact_index_client_cache.rb +++ b/test/rubygems/test_gem_compact_index_client_cache.rb @@ -45,7 +45,7 @@ def test_initialize_creates_cache_directories def test_reads_cached_files_without_fetcher cache = Gem::CompactIndexClient::Cache.new(@dir) - @dir.join("versions").write "a 1.0.0\n" + @dir.join("versions").binwrite "a 1.0.0\n" assert_equal "a 1.0.0\n", cache.versions assert_nil cache.names @@ -75,7 +75,7 @@ def test_reset_allows_fetching_again def test_info_skips_fetch_when_checksum_matches fetcher = FakeFetcher.new("a 1.0.0\n") cache = Gem::CompactIndexClient::Cache.new(@dir, fetcher) - @dir.join("info", "a").write "a 1.0.0\n" + @dir.join("info", "a").binwrite "a 1.0.0\n" content = cache.info("a", Digest::MD5.hexdigest("a 1.0.0\n")) @@ -86,7 +86,7 @@ def test_info_skips_fetch_when_checksum_matches def test_info_fetches_when_checksum_differs fetcher = FakeFetcher.new("a 1.0.0\na 1.1.0\n") cache = Gem::CompactIndexClient::Cache.new(@dir, fetcher) - @dir.join("info", "a").write "a 1.0.0\n" + @dir.join("info", "a").binwrite "a 1.0.0\n" content = cache.info("a", Digest::MD5.hexdigest("a 1.0.0\na 1.1.0\n")) @@ -98,7 +98,7 @@ def test_info_fetches_when_checksum_differs def test_info_without_checksum_reads_cached_file fetcher = FakeFetcher.new("a 1.0.0\n") cache = Gem::CompactIndexClient::Cache.new(@dir, fetcher) - @dir.join("info", "a").write "a 1.0.0\n" + @dir.join("info", "a").binwrite "a 1.0.0\n" assert_equal "a 1.0.0\n", cache.info("a") assert_empty fetcher.requests diff --git a/test/rubygems/test_gem_compact_index_client_cache_file.rb b/test/rubygems/test_gem_compact_index_client_cache_file.rb index 6b28ea8bae46..c782f6336165 100644 --- a/test/rubygems/test_gem_compact_index_client_cache_file.rb +++ b/test/rubygems/test_gem_compact_index_client_cache_file.rb @@ -24,7 +24,7 @@ def test_write_creates_file end def test_write_replaces_original_file - @path.write "old" + @path.binwrite "old" CacheFile.write(@path, "new") @@ -50,7 +50,7 @@ def test_write_with_matching_digests end def test_write_with_mismatched_digests - @path.write "old" + @path.binwrite "old" assert_raise CacheFile::DigestMismatchError do CacheFile.write(@path, "data", sha256("other data")) @@ -60,7 +60,7 @@ def test_write_with_mismatched_digests end def test_append_without_digests_returns_false - @path.write "abc" + @path.binwrite "abc" appended = nil CacheFile.new(@path) {|file| appended = file.append("def") } @@ -70,7 +70,7 @@ def test_append_without_digests_returns_false end def test_append_with_matching_digests - @path.write "abc" + @path.binwrite "abc" appended = nil CacheFile.copy(@path) do |file| @@ -83,7 +83,7 @@ def test_append_with_matching_digests end def test_append_with_mismatched_digests_keeps_original - @path.write "abc" + @path.binwrite "abc" appended = nil CacheFile.copy(@path) do |file| @@ -125,7 +125,7 @@ def test_commit_after_close_raises def test_write_preserves_permissions pend "chmod is unreliable on Windows" if Gem.win_platform? - @path.write "old" + @path.binwrite "old" @path.chmod 0o600 CacheFile.write(@path, "new") diff --git a/test/rubygems/test_gem_compact_index_client_updater.rb b/test/rubygems/test_gem_compact_index_client_updater.rb index 3218faa5f5e1..2660f0df24c5 100644 --- a/test/rubygems/test_gem_compact_index_client_updater.rb +++ b/test/rubygems/test_gem_compact_index_client_updater.rb @@ -73,8 +73,8 @@ def test_update_fetches_full_file_when_no_local_copy end def test_update_sends_etag_and_keeps_file_on_not_modified - @local_path.write "a 1.0.0\n" - @etag_path.write "abc123" + @local_path.binwrite "a 1.0.0\n" + @etag_path.binwrite "abc123" fetcher = FakeFetcher.new(FakeNotModified.new) updater = Gem::CompactIndexClient::Updater.new(fetcher) @@ -88,7 +88,7 @@ def test_update_sends_etag_and_keeps_file_on_not_modified end def test_update_appends_ranged_response - @local_path.write "a 1.0.0\n" + @local_path.binwrite "a 1.0.0\n" body = "\na 1.1.0\n" response = FakePartialResponse.new(body, "ETag" => '"def456"', @@ -106,7 +106,7 @@ def test_update_appends_ranged_response end def test_update_replaces_file_when_server_ignores_range - @local_path.write "stale data" + @local_path.binwrite "stale data" response = FakeResponse.new("a 1.0.0\n", "ETag" => '"def456"', "Repr-Digest" => digest_header("a 1.0.0\n")) @@ -119,7 +119,7 @@ def test_update_replaces_file_when_server_ignores_range end def test_update_retries_with_full_request_on_bad_ranged_response - @local_path.write "a 1.0.0\n" + @local_path.binwrite "a 1.0.0\n" bad_append = FakePartialResponse.new("\nb 1.0.0\n", "Repr-Digest" => digest_header("something else entirely")) full = FakeResponse.new("a 1.0.0\nb 1.0.0\n",