diff --git a/Manifest.txt b/Manifest.txt index fed1f772c476..c6110e44e382 100644 --- a/Manifest.txt +++ b/Manifest.txt @@ -404,6 +404,12 @@ lib/rubygems/commands/unpack_command.rb lib/rubygems/commands/update_command.rb lib/rubygems/commands/which_command.rb lib/rubygems/commands/yank_command.rb +lib/rubygems/compact_index_client.rb +lib/rubygems/compact_index_client/cache.rb +lib/rubygems/compact_index_client/cache_file.rb +lib/rubygems/compact_index_client/http_fetcher.rb +lib/rubygems/compact_index_client/parser.rb +lib/rubygems/compact_index_client/updater.rb lib/rubygems/config_file.rb lib/rubygems/core_ext/kernel_gem.rb lib/rubygems/core_ext/kernel_require.rb diff --git a/lib/rubygems/compact_index_client.rb b/lib/rubygems/compact_index_client.rb new file mode 100644 index 000000000000..84e629d8dd5b --- /dev/null +++ b/lib/rubygems/compact_index_client.rb @@ -0,0 +1,90 @@ +# frozen_string_literal: true + +## +# The CompactIndexClient fetches and parses the compact index files +# (names, versions and info/[gem]) served by a gem server, keeping a +# local cache so subsequent fetches only transfer what changed. +# +# This is an independent RubyGems port of Bundler::CompactIndexClient. +# Both implementations are intentionally kept separate so that changes +# on either side cannot affect the other; this one only depends on +# RubyGems itself. + +class Gem::CompactIndexClient + SUPPORTED_DIGESTS = { "sha-256" => :SHA256 }.freeze + DEBUG_MUTEX = Thread::Mutex.new + + # info returns an Array of INFO Arrays. Each INFO Array has the following indices: + INFO_NAME = 0 + INFO_VERSION = 1 + INFO_PLATFORM = 2 + INFO_DEPS = 3 + INFO_REQS = 4 + + def self.debug + return unless ENV["DEBUG_COMPACT_INDEX"] + DEBUG_MUTEX.synchronize { warn("[#{self}] #{yield}") } + end + + class Error < StandardError; end + + require_relative "compact_index_client/cache" + require_relative "compact_index_client/cache_file" + require_relative "compact_index_client/http_fetcher" + require_relative "compact_index_client/parser" + require_relative "compact_index_client/updater" + + # The client is instantiated with: + # - `directory`: the root directory where the cache files are stored. + # - `fetcher`: (optional) an object that responds to #call(uri_path, headers) + # and returns a Gem::Net::HTTP response. When the fetcher is not provided, + # the client only reads cached files from disk. + def initialize(directory, fetcher = nil) + @cache = Cache.new(directory, fetcher) + @parser = Parser.new(@cache) + end + + def names + Gem::CompactIndexClient.debug { "names" } + @parser.names + end + + def versions + Gem::CompactIndexClient.debug { "versions" } + @parser.versions + end + + def dependencies(names) + Gem::CompactIndexClient.debug { "dependencies(#{names})" } + names.map {|name| info(name) } + end + + def info(name) + Gem::CompactIndexClient.debug { "info(#{name})" } + @parser.info(name) + end + + # Fetches a single gem's info without consulting the versions index, + # using a conditional request to refresh the cached file. Useful when + # only a few gems are needed and the versions index download would + # dominate, as in gem install. + def fetch_info(name) + Gem::CompactIndexClient.debug { "fetch_info(#{name})" } + @parser.parse_info(@cache.fetch_info(name), name) + end + + def latest_version(name) + Gem::CompactIndexClient.debug { "latest_version(#{name})" } + @parser.info(name).map {|d| Gem::Version.new(d[INFO_VERSION]) }.max + end + + def available? + Gem::CompactIndexClient.debug { "available?" } + @parser.available? + end + + def reset! + Gem::CompactIndexClient.debug { "reset!" } + @cache.reset! + end +end diff --git a/lib/rubygems/compact_index_client/cache.rb b/lib/rubygems/compact_index_client/cache.rb new file mode 100644 index 000000000000..34e20dfea0b8 --- /dev/null +++ b/lib/rubygems/compact_index_client/cache.rb @@ -0,0 +1,107 @@ +# frozen_string_literal: true + +require "digest" +require "fileutils" +require "pathname" unless defined?(Pathname) +require "set" + +class Gem::CompactIndexClient + # Calls the Updater to update the cached files on disk, reads the + # cached files and returns their contents. + class Cache + attr_reader :directory + + def initialize(directory, fetcher = nil) + @directory = Pathname.new(directory).expand_path + @updater = Updater.new(fetcher) if fetcher + @mutex = Thread::Mutex.new + @endpoints = Set.new + + @info_root = mkdir("info") + @special_characters_info_root = mkdir("info-special-characters") + @info_etag_root = mkdir("info-etags") + end + + def names + fetch("names", names_path, names_etag_path) + end + + def versions + fetch("versions", versions_path, versions_etag_path) + end + + def info(name, remote_checksum = nil) + path = info_path(name) + + if remote_checksum && remote_checksum != checksum_for_file(path) + fetch("info/#{name}", path, info_etag_path(name)) + else + Gem::CompactIndexClient.debug { "update skipped info/#{name} (#{remote_checksum ? "versions index checksum matches local" : "versions index checksum is nil"})" } + read(path) + end + end + + # Fetch a single gem's info file without consulting the versions + # index, refreshing the cached file with a conditional request. + def fetch_info(name) + fetch("info/#{name}", info_path(name), info_etag_path(name)) + end + + def reset! + @mutex.synchronize { @endpoints.clear } + end + + private + + def names_path = directory.join("names") + def names_etag_path = directory.join("names.etag") + def versions_path = directory.join("versions") + def versions_etag_path = directory.join("versions.etag") + + def info_path(name) + name = name.to_s + if /[^a-z0-9_-]/.match?(name) + name += "-#{Digest::MD5.hexdigest(name).downcase}" + @special_characters_info_root.join(name) + else + @info_root.join(name) + end + end + + def info_etag_path(name) + name = name.to_s + @info_etag_root.join("#{name}-#{Digest::MD5.hexdigest(name).downcase}") + end + + def checksum_for_file(path) + return unless path.file? + Digest::MD5.file(path).hexdigest + end + + def mkdir(name) + directory.join(name).tap do |dir| + FileUtils.mkdir_p(dir) + end + end + + def fetch(remote_path, path, etag_path) + if already_fetched?(remote_path) + Gem::CompactIndexClient.debug { "already fetched #{remote_path}" } + else + Gem::CompactIndexClient.debug { "fetching #{remote_path}" } + @updater&.update(remote_path, path, etag_path) + end + + read(path) + end + + def already_fetched?(remote_path) + @mutex.synchronize { !@endpoints.add?(remote_path) } + end + + def read(path) + return unless path.file? + path.read + end + end +end diff --git a/lib/rubygems/compact_index_client/cache_file.rb b/lib/rubygems/compact_index_client/cache_file.rb new file mode 100644 index 000000000000..3656645d8220 --- /dev/null +++ b/lib/rubygems/compact_index_client/cache_file.rb @@ -0,0 +1,141 @@ +# frozen_string_literal: true + +require "digest" +require "fileutils" +require_relative "../package" + +class Gem::CompactIndexClient + # write cache files in a way that is robust to concurrent modifications + # if digests are given, the checksums will be verified + class CacheFile + DEFAULT_FILE_MODE = 0o644 + private_constant :DEFAULT_FILE_MODE + + class Error < RuntimeError; end + class ClosedError < Error; end + + class DigestMismatchError < Error + def initialize(digests, expected_digests) + super "Calculated checksums #{digests.inspect} did not match expected #{expected_digests.inspect}." + end + end + + # Initialize with a copy of the original file, then yield the instance. + def self.copy(path, &block) + new(path) do |file| + file.initialize_digests + + path.open("rb") do |s| + file.open {|f| IO.copy_stream(s, f) } + end + + yield file + end + end + + # Write data to a temp file, then replace the original file with it verifying the digests if given. + def self.write(path, data, digests = nil) + return unless data + new(path) do |file| + file.digests = digests + file.write(data) + end + end + + attr_reader :original_path, :path + + def initialize(original_path, &block) + @original_path = original_path + @perm = original_path.file? ? original_path.stat.mode : DEFAULT_FILE_MODE + @path = original_path.sub(/$/, ".#{$$}.tmp") + return unless block_given? + begin + yield self + ensure + close + end + end + + def size + path.size + end + + # initialize the digests using CompactIndexClient::SUPPORTED_DIGESTS, or a subset based on keys. + def initialize_digests(keys = nil) + @digests = keys ? SUPPORTED_DIGESTS.slice(*keys) : SUPPORTED_DIGESTS.dup + @digests.transform_values! {|algo_class| Digest(algo_class).new } + end + + # reset the digests so they don't contain any previously read data + def reset_digests + @digests&.each_value(&:reset) + end + + # set the digests that will be verified at the end + def digests=(expected_digests) + @expected_digests = expected_digests + + if @expected_digests.nil? + @digests = nil + elsif @digests + @digests = @digests.slice(*@expected_digests.keys) + else + initialize_digests(@expected_digests.keys) + end + end + + def digests? + @digests&.any? + end + + # Open the temp file for writing, reusing original permissions, yielding the IO object. + def open(write_mode = "wb", perm = @perm, &block) + raise ClosedError, "Cannot reopen closed file" if @closed + path.open(write_mode, perm) do |f| + yield digests? ? Gem::Package::DigestIO.new(f, @digests) : f + end + end + + # Returns false without appending when no digests since appending is too error prone to do without digests. + def append(data) + return false unless digests? + open("a") {|f| f.write data } + verify && commit + end + + def write(data) + reset_digests + open {|f| f.write data } + commit! + end + + def commit! + verify || raise(DigestMismatchError.new(@base64digests, @expected_digests)) + commit + end + + # Verify the digests, returning true on match, false on mismatch. + def verify + return true unless @expected_digests && digests? + @base64digests = @digests.transform_values!(&:base64digest) + @digests = nil + @base64digests.all? {|algo, digest| @expected_digests[algo] == digest } + end + + # Replace the original file with the temp file without verifying digests. + # The file is permanently closed. + def commit + raise ClosedError, "Cannot commit closed file" if @closed + FileUtils.mv(path, original_path) + @closed = true + end + + # Remove the temp file without replacing the original file. + # The file is permanently closed. + def close + return if @closed + FileUtils.remove_file(path) if @path&.file? + @closed = true + end + end +end diff --git a/lib/rubygems/compact_index_client/http_fetcher.rb b/lib/rubygems/compact_index_client/http_fetcher.rb new file mode 100644 index 000000000000..2d439df6d17b --- /dev/null +++ b/lib/rubygems/compact_index_client/http_fetcher.rb @@ -0,0 +1,49 @@ +# frozen_string_literal: true + +require_relative "../remote_fetcher" + +class Gem::CompactIndexClient + # Fetches compact index files relative to +base_uri+ using + # Gem::RemoteFetcher's connection infrastructure (proxy, TLS, + # connection pooling). Implements the fetcher interface expected by + # Gem::CompactIndexClient: #call(path, headers) returning a + # Gem::Net::HTTP response. + class HTTPFetcher + REDIRECT_LIMIT = 10 + private_constant :REDIRECT_LIMIT + + def initialize(base_uri, remote_fetcher = Gem::RemoteFetcher.fetcher) + base_uri = base_uri.to_s + base_uri += "/" unless base_uri.end_with?("/") + @base_uri = Gem::URI(base_uri) + @remote_fetcher = remote_fetcher + end + + def call(path, headers = {}) + fetch(@base_uri + path, headers, REDIRECT_LIMIT) + end + + private + + def fetch(uri, headers, redirects_remaining) + response = @remote_fetcher.request(uri, Gem::Net::HTTP::Get) do |req| + headers.each {|name, value| req[name] = value } + end + + case response + when Gem::Net::HTTPSuccess, Gem::Net::HTTPNotModified + response + when Gem::Net::HTTPMovedPermanently, Gem::Net::HTTPFound, Gem::Net::HTTPSeeOther, + Gem::Net::HTTPTemporaryRedirect, Gem::Net::HTTPPermanentRedirect + raise Gem::RemoteFetcher::FetchError.new("too many redirects", uri) if redirects_remaining.zero? + + location = response["Location"] + raise Gem::RemoteFetcher::FetchError.new("redirecting but no redirect location was given", uri) unless location + + fetch(uri + location, headers, redirects_remaining - 1) + else + raise Gem::RemoteFetcher::FetchError.new("bad response #{response.message} #{response.code}", uri) + end + end + end +end diff --git a/lib/rubygems/compact_index_client/parser.rb b/lib/rubygems/compact_index_client/parser.rb new file mode 100644 index 000000000000..dac3ccd6e0fb --- /dev/null +++ b/lib/rubygems/compact_index_client/parser.rb @@ -0,0 +1,90 @@ +# frozen_string_literal: true + +require_relative "../resolver/api_set/gem_parser" + +class Gem::CompactIndexClient + class Parser + # `compact_index` - an object responding to #names, #versions, #info(name, checksum), + # returning the file contents as a string + def initialize(compact_index) + @compact_index = compact_index + @info_checksums = nil + @versions_by_name = nil + @available = nil + @gem_parser = nil + end + + def names + lines(@compact_index.names) + end + + def versions + @versions_by_name ||= Hash.new {|hash, key| hash[key] = [] } + @info_checksums = {} + + lines(@compact_index.versions).each do |line| + name, versions_string, checksum = line.split(" ", 3) + @info_checksums[name] = checksum || "" + versions_string.split(",") do |version| + delete = version.delete_prefix!("-") + version = version.split("-", 2).unshift(name) + if delete + @versions_by_name[name].delete(version) + else + @versions_by_name[name] << version + end + end + end + + @versions_by_name + end + + def info(name) + parse_info(@compact_index.info(name, info_checksums[name]), name) + end + + def parse_info(data, name) + lines(data).map {|line| gem_parser.parse(line).unshift(name) } + end + + def available? + return @available unless @available.nil? + @available = !info_checksums.empty? + end + + private + + def info_checksums + @info_checksums ||= lines(@compact_index.versions).each_with_object({}) do |line, checksums| + parse_version_checksum(line, checksums) + end + end + + def lines(data) + return [] if data.nil? || data.empty? + lines = data.split("\n") + header = lines.index("---") + header ? lines[header + 1..-1] : lines + end + + def gem_parser + @gem_parser ||= Gem::Resolver::APISet::GemParser.new + end + + # This is mostly the same as `split(" ", 3)` but it avoids allocating extra objects. + # This method gets called at least once for every gem when parsing versions. + def parse_version_checksum(line, checksums) + return unless (name_end = line.index(" ")) # Artifactory bug causes blank lines in artifactory index files + checksum_start = line.index(" ", name_end + 1) + return unless checksum_start + checksum_start += 1 + + checksum_end = line.size - checksum_start + + line.freeze # allows slicing into the string to not allocate a copy of the line + name = line[0, name_end] + checksum = line[checksum_start, checksum_end] + checksums[name.freeze] = checksum # freeze name since it is used as a hash key + end + end +end diff --git a/lib/rubygems/compact_index_client/updater.rb b/lib/rubygems/compact_index_client/updater.rb new file mode 100644 index 000000000000..79a6cdfe7d82 --- /dev/null +++ b/lib/rubygems/compact_index_client/updater.rb @@ -0,0 +1,105 @@ +# frozen_string_literal: true + +require "zlib" +require_relative "../vendored_net_http" + +class Gem::CompactIndexClient + # Updates the cached files on disk, keeping them in sync with the server + # using conditional requests (ETag) and ranged requests where possible. + class Updater + class MismatchedChecksumError < Error + def initialize(path, message) + super "The checksum of /#{path} does not match the checksum provided by the server! Something is wrong. #{message}" + end + end + + def initialize(fetcher) + @fetcher = fetcher + end + + def update(remote_path, local_path, etag_path) + append(remote_path, local_path, etag_path) || replace(remote_path, local_path, etag_path) + rescue CacheFile::DigestMismatchError => e + raise MismatchedChecksumError.new(remote_path, e.message) + rescue Zlib::GzipFile::Error + raise Error, "invalid gzip response while fetching /#{remote_path}" + end + + private + + def append(remote_path, local_path, etag_path) + return false unless local_path.file? && local_path.size.nonzero? + + CacheFile.copy(local_path) do |file| + etag = etag_path.read.tap(&:chomp!) if etag_path.file? + + # Subtract a byte to ensure the range won't be empty. + # Avoids 416 (Range Not Satisfiable) responses. + response = @fetcher.call(remote_path, request_headers(etag, file.size - 1)) + break true if response.is_a?(Gem::Net::HTTPNotModified) + + file.digests = parse_digests(response) + # server may ignore Range and return the full response + if response.is_a?(Gem::Net::HTTPPartialContent) + tail = response.body.byteslice(1..-1) + break false unless tail && file.append(tail) + else + file.write(response.body) + end + CacheFile.write(etag_path, etag_from_response(response)) + true + end + end + + # request without range header to get the full file or a 304 Not Modified + def replace(remote_path, local_path, etag_path) + etag = etag_path.read.tap(&:chomp!) if etag_path.file? + response = @fetcher.call(remote_path, request_headers(etag)) + return true if response.is_a?(Gem::Net::HTTPNotModified) + CacheFile.write(local_path, response.body, parse_digests(response)) + CacheFile.write(etag_path, etag_from_response(response)) + end + + def request_headers(etag, range_start = nil) + headers = {} + headers["Range"] = "bytes=#{range_start}-" if range_start + headers["If-None-Match"] = %("#{etag}") if etag + headers + end + + def etag_from_response(response) + return unless response["ETag"] + etag = response["ETag"].delete_prefix("W/") + return if etag.delete_prefix!('"') && !etag.delete_suffix!('"') + etag + end + + # Unwraps and returns a Hash of digest algorithms and base64 values + # according to RFC 8941 Structured Field Values for HTTP. + # https://www.rfc-editor.org/rfc/rfc8941#name-parsing-a-byte-sequence + # Ignores unsupported algorithms. + def parse_digests(response) + return unless header = response["Repr-Digest"] || response["Digest"] + digests = {} + header.split(",") do |param| + algorithm, value = param.split("=", 2) + algorithm.strip! + algorithm.downcase! + next unless SUPPORTED_DIGESTS.key?(algorithm) + next unless value + next unless value = byte_sequence(value) + digests[algorithm] = value + end + digests.empty? ? nil : digests + end + + # Unwrap surrounding colons (byte sequence) + # The wrapping characters must be matched or we return nil. + # Also handles quotes because right now rubygems.org sends them. + def byte_sequence(value) + return if value.delete_prefix!(":") && !value.delete_suffix!(":") + return if value.delete_prefix!('"') && !value.delete_suffix!('"') + value + end + end +end diff --git a/lib/rubygems/resolver/api_set.rb b/lib/rubygems/resolver/api_set.rb index 3f443519d80f..c3dc4c0bcabc 100644 --- a/lib/rubygems/resolver/api_set.rb +++ b/lib/rubygems/resolver/api_set.rb @@ -35,6 +35,7 @@ def initialize(dep_uri = "https://index.rubygems.org/info/") @dep_uri = dep_uri @uri = dep_uri + ".." + @client = nil @data = Hash.new {|h,k| h[k] = [] } @source = Gem::Source.new @uri @@ -99,26 +100,20 @@ def pretty_print(q) # :nodoc: # Return data for all versions of the gem +name+. def versions(name) # :nodoc: - if @data.key?(name) - return @data[name] - end - - uri = @dep_uri + name + return @data[name] if @data.key?(name) - begin - str = Gem::RemoteFetcher.fetcher.fetch_path uri - rescue Gem::RemoteFetcher::FetchError - @data[name] = [] - else - lines(str).each do |ver| - number, platform, dependencies, requirements = parse_gem(ver) + infos = begin + client.fetch_info(name) + rescue Gem::RemoteFetcher::FetchError, Gem::CompactIndexClient::Error + [] + end - platform ||= "ruby" - dependencies = dependencies.map {|dep_name, reqs| [dep_name, reqs.join(", ")] } - requirements = requirements.map {|req_name, reqs| [req_name.to_sym, reqs] }.to_h + infos.each do |_, number, platform, dependencies, requirements| + platform ||= "ruby" + dependencies = dependencies.map {|dep_name, reqs| [dep_name, reqs.join(", ")] } + requirements = requirements.map {|req_name, reqs| [req_name.to_sym, reqs] }.to_h - @data[name] << { name: name, number: number, platform: platform, dependencies: dependencies, requirements: requirements } - end + @data[name] << { name: name, number: number, platform: platform, dependencies: dependencies, requirements: requirements } end @data[name] @@ -126,14 +121,7 @@ def versions(name) # :nodoc: private - def lines(str) - lines = str.split("\n") - header = lines.index("---") - header ? lines[header + 1..-1] : lines - end - - def parse_gem(string) - @gem_parser ||= GemParser.new - @gem_parser.parse(string) + def client # :nodoc: + @client ||= @source.compact_index_client end end diff --git a/lib/rubygems/resolver/api_specification.rb b/lib/rubygems/resolver/api_specification.rb index ccfd6fe0843b..070877ac7653 100644 --- a/lib/rubygems/resolver/api_specification.rb +++ b/lib/rubygems/resolver/api_specification.rb @@ -38,6 +38,7 @@ def initialize(set, api_data) end.freeze @required_ruby_version = Gem::Requirement.new(api_data.dig(:requirements, :ruby)).freeze @required_rubygems_version = Gem::Requirement.new(api_data.dig(:requirements, :rubygems)).freeze + @created_at = parse_created_at(api_data.dig(:requirements, :created_at))&.freeze end def ==(other) # :nodoc: @@ -84,22 +85,41 @@ def pretty_print(q) # :nodoc: end ## - # Fetches a Gem::Specification for this APISpecification. + # A Gem::Specification stub built from the Compact Index data for this + # specification. The compact index carries everything needed to + # download and install the gem, so the Marshal gemspec is not fetched. + # Development dependencies are not included; see + # #fetch_development_dependencies. def spec # :nodoc: - @spec ||= - begin - tuple = Gem::NameTuple.new @name, @version, @platform - source.fetch_spec tuple - rescue Gem::RemoteFetcher::FetchError - raise if @original_platform == @platform - - tuple = Gem::NameTuple.new @name, @version, @original_platform - source.fetch_spec tuple + @spec ||= Gem::Specification.new do |s| + s.name = @name + s.version = @version + s.platform = @platform + s.original_platform = @original_platform + s.required_ruby_version = @required_ruby_version + s.required_rubygems_version = @required_rubygems_version + + @dependencies.each do |dependency| + s.add_runtime_dependency dependency.name, *dependency.requirement.as_list end + end end def source # :nodoc: @set.source end + + private + + def parse_created_at(value) + value = value.first if value.is_a?(Array) + return unless value.is_a?(String) + + begin + Time.new(value) + rescue ArgumentError + nil + end + end end diff --git a/lib/rubygems/resolver/specification.rb b/lib/rubygems/resolver/specification.rb index d2098ef0e2aa..986fa7c9ae82 100644 --- a/lib/rubygems/resolver/specification.rb +++ b/lib/rubygems/resolver/specification.rb @@ -54,10 +54,17 @@ class Gem::Resolver::Specification attr_reader :required_rubygems_version + ## + # The time this gem version was published, when the source provides it + # (compact index v2), nil otherwise. + + attr_reader :created_at + ## # Sets default instance variables for the specification. def initialize + @created_at = nil @dependencies = nil @name = nil @platform = nil diff --git a/lib/rubygems/source.rb b/lib/rubygems/source.rb index 86717e3e7178..91f242e22a20 100644 --- a/lib/rubygems/source.rb +++ b/lib/rubygems/source.rb @@ -155,8 +155,60 @@ def fetch_spec(name_tuple) # :latest => Return the list of only the highest version of each gem # :prerelease => Return the list of all prerelease only specs # + # The compact index is used when the source provides it, falling back + # to the Marshal spec indexes. def load_specs(type) + load_compact_index_specs(type) || load_marshal_specs(type) + end + + ## + # The compact index client for this source, caching under + # Gem.spec_cache_dir. Also used by Gem::Resolver::APISet. + + def compact_index_client # :nodoc: + @compact_index_client ||= begin + require_relative "compact_index_client" + + index_uri = compact_index_uri + + Gem::CompactIndexClient.new(compact_index_cache_dir(index_uri), + Gem::CompactIndexClient::HTTPFetcher.new(index_uri)) + end + end + + ## + # Downloads +spec+ and writes it to +dir+. See also + # Gem::RemoteFetcher#download. + + def download(spec, dir = Dir.pwd) + fetcher = Gem::RemoteFetcher.fetcher + fetcher.download spec, uri.to_s, dir + end + + def pretty_print(q) # :nodoc: + q.object_group(self) do + q.group 2, "[Remote:", "]" do + q.breakable + q.text @uri.to_s + + if api = uri + q.breakable + q.text "API URI: " + q.text api.to_s + end + end + end + end + + def typo_squatting?(host, distance_threshold = 4) + return if @uri.host.nil? + levenshtein_distance(@uri.host, host).between? 1, distance_threshold + end + + private + + def load_marshal_specs(type) file = FILES[type] fetcher = Gem::RemoteFetcher.fetcher file_name = "#{file}.#{Gem.marshal_version}" @@ -187,48 +239,74 @@ def load_specs(type) end ## - # Downloads +spec+ and writes it to +dir+. See also - # Gem::RemoteFetcher#download. + # Builds the name tuple list for +type+ from the compact index versions + # file. Returns nil when the source does not provide a usable compact + # index, so the caller can fall back to the Marshal spec indexes. - def download(spec, dir = Dir.pwd) - fetcher = Gem::RemoteFetcher.fetcher - fetcher.download spec, uri.to_s, dir - end + def load_compact_index_specs(type) + return unless %w[http https].include?(uri.scheme) - def pretty_print(q) # :nodoc: - q.object_group(self) do - q.group 2, "[Remote:", "]" do - q.breakable - q.text @uri.to_s + versions = compact_index_versions + return if versions.nil? || versions.empty? - if api = uri - q.breakable - q.text "API URI: " - q.text api.to_s - end + tuples = [] + + versions.each_value do |rows| + gem_tuples = rows.filter_map do |name, version_string, platform| + version = Gem::Version.new(version_string) + next if version.prerelease? != (type == :prerelease) + + Gem::NameTuple.new(name, version, platform || "ruby") end + + gem_tuples = max_versions_by_platform(gem_tuples) if type == :latest + + tuples.concat(gem_tuples) end - end - def typo_squatting?(host, distance_threshold = 4) - return if @uri.host.nil? - levenshtein_distance(@uri.host, host).between? 1, distance_threshold + tuples + rescue ArgumentError + nil end - private + def compact_index_versions + @compact_index_versions ||= compact_index_client.versions + rescue Gem::RemoteFetcher::FetchError, Gem::CompactIndexClient::Error + @compact_index_versions = {} + nil + end - def new_dependency_resolver_set - return Gem::Resolver::IndexSet.new self if uri.scheme == "file" + def max_versions_by_platform(tuples) + tuples.group_by(&:platform).map {|_, platform_tuples| platform_tuples.max_by(&:version) } + end - fetch_uri = if uri.host == "rubygems.org" + def compact_index_uri + if uri.host == "rubygems.org" index_uri = uri.dup index_uri.host = "index.rubygems.org" index_uri else uri end + end + + def compact_index_cache_dir(index_uri) + if update_cache? + # Correct for windows paths + escaped_path = index_uri.path.sub(%r{^/([a-z]):/}i, '/\\1-/') + + File.join Gem.spec_cache_dir, "compact_index", + "#{index_uri.host}%#{index_uri.port}", *escaped_path.split("/").reject(&:empty?) + else + require "tmpdir" + Dir.mktmpdir "gem_compact_index" + end + end + + def new_dependency_resolver_set + return Gem::Resolver::IndexSet.new self if uri.scheme == "file" - bundler_api_uri = enforce_trailing_slash(fetch_uri) + "versions" + bundler_api_uri = enforce_trailing_slash(compact_index_uri) + "versions" begin fetcher = Gem::RemoteFetcher.fetcher diff --git a/test/rubygems/helper.rb b/test/rubygems/helper.rb index 2411dbc6495c..4c47d46835e3 100644 --- a/test/rubygems/helper.rb +++ b/test/rubygems/helper.rb @@ -35,6 +35,7 @@ require "test/unit" +require "digest" require "fileutils" require "pathname" require "pp" @@ -1173,6 +1174,81 @@ def util_setup_spec_fetcher(*specs) nil # force errors end + ## + # Sets up the compact index API endpoints (versions, names and + # info/NAME) for +specs+ on +@fetcher+. +created_at+ maps spec + # original names to ISO8601 timestamps emitted as compact index v2 + # metadata. + + def util_setup_compact_index(*specs, created_at: {}) + by_name = Hash.new {|hash, name| hash[name] = [] } + specs.each {|spec| by_name[spec.name] << spec } + + names_body = +"---\n" + versions_body = +"created_at: 2026-01-01T00:00:00Z\n---\n" + + by_name.keys.sort.each do |name| + info_body = +"---\n" + by_name[name].each do |spec| + info_body << util_compact_index_info_line(spec, created_at[spec.original_name]) << "\n" + end + + versions_list = by_name[name].map {|spec| spec.original_name.delete_prefix("#{spec.name}-") }.join(",") + versions_body << "#{name} #{versions_list} #{Digest::MD5.hexdigest(info_body)}\n" + names_body << "#{name}\n" + + @fetcher.data["#{@gem_repo}info/#{name}"] = util_compact_index_response(info_body) + end + + versions_response = util_compact_index_response(versions_body) + # Gem::Source#dependency_resolver_set probes this URL via fetch_path and + # builds the info URL from the response uri + versions_response.uri = Gem::URI("#{@gem_repo}versions") + + @fetcher.data["#{@gem_repo}versions"] = versions_response + @fetcher.data["#{@gem_repo}names"] = util_compact_index_response(names_body) + + nil + end + + ## + # A compact index info file line for +spec+, including v2 metadata. + + def util_compact_index_info_line(spec, created_at = nil) + version = spec.original_name.delete_prefix("#{spec.name}-") + + dependencies = spec.runtime_dependencies.map do |dependency| + "#{dependency.name}:#{util_compact_index_requirement(dependency.requirement)}" + end.join(",") + + metadata = +"checksum:#{Digest::SHA256.hexdigest(spec.original_name)}" + unless spec.required_ruby_version.nil? || spec.required_ruby_version.none? + metadata << ",ruby:#{util_compact_index_requirement(spec.required_ruby_version)}" + end + unless spec.required_rubygems_version.nil? || spec.required_rubygems_version.none? + metadata << ",rubygems:#{util_compact_index_requirement(spec.required_rubygems_version)}" + end + metadata << ",created_at:#{created_at}" if created_at + + "#{version} #{dependencies}|#{metadata}" + end + + def util_compact_index_requirement(requirement) + requirement.as_list.join("&") + end + + def util_compact_index_response(body) + Gem::HTTPResponseFactory.create( + body: body, + code: 200, + msg: "OK", + headers: { + "ETag" => %("#{Digest::MD5.hexdigest(body)}"), + "Repr-Digest" => "sha-256=:#{Digest::SHA256.base64digest(body)}:", + } + ) + end + def write_marshalled_gemspecs(*all_specs) v = Gem.marshal_version diff --git a/test/rubygems/test_gem_commands_outdated_command.rb b/test/rubygems/test_gem_commands_outdated_command.rb index 3e61033af366..07289d821fba 100644 --- a/test/rubygems/test_gem_commands_outdated_command.rb +++ b/test/rubygems/test_gem_commands_outdated_command.rb @@ -30,6 +30,26 @@ def test_execute assert_equal "", @ui.error end + def test_execute_compact_index + spec_fetcher do |fetcher| + fetcher.gem "foo", "0.2" + end + + foo2 = util_spec "foo", "2.0" + util_setup_compact_index foo2 + + # drop the in-memory tuples spec_fetcher pre-populated so the lookup + # goes through Gem::Source#load_specs + Gem::SpecFetcher.fetcher = nil + + use_ui @ui do + @cmd.execute + end + + assert_equal "foo (0.2 < 2.0)\n", @ui.output + assert_equal "", @ui.error + end + def test_execute_with_up_to_date_platform_specific_gem spec_fetcher do |fetcher| fetcher.download "foo", "2.0" diff --git a/test/rubygems/test_gem_commands_update_command.rb b/test/rubygems/test_gem_commands_update_command.rb index 5ed12ad48140..5bc36d61035e 100644 --- a/test/rubygems/test_gem_commands_update_command.rb +++ b/test/rubygems/test_gem_commands_update_command.rb @@ -42,6 +42,34 @@ def test_execute assert_empty out end + def test_execute_compact_index + spec_fetcher do |fetcher| + fetcher.gem "b", 1 + end + + b2, b2_gem = util_gem "b", 2 + util_setup_compact_index b2 + add_to_fetcher b2, b2_gem + + # drop the in-memory tuples spec_fetcher pre-populated so the lookup + # goes through Gem::Source#load_specs + Gem::SpecFetcher.fetcher = nil + + @cmd.options[:args] = [] + + use_ui @ui do + @cmd.execute + end + + out = @ui.output.split "\n" + assert_equal "Updating installed gems", out.shift + assert_equal "Updating b", out.shift + assert_equal "Gems updated: b", out.shift + assert_empty out + + assert_path_exist File.join(@gemhome, "specifications", "b-2.gemspec") + end + def test_execute_multiple spec_fetcher do |fetcher| fetcher.download "a", 2 diff --git a/test/rubygems/test_gem_compact_index_client.rb b/test/rubygems/test_gem_compact_index_client.rb new file mode 100644 index 000000000000..2a05cfd6397e --- /dev/null +++ b/test/rubygems/test_gem_compact_index_client.rb @@ -0,0 +1,123 @@ +# frozen_string_literal: true + +require_relative "helper" +require "rubygems/compact_index_client" + +class TestGemCompactIndexClient < Gem::TestCase + class FakeResponse < Gem::Net::HTTPOK + def initialize(body) + super("1.1", "200", "OK") + @fake_body = body + end + + attr_reader :fake_body + alias_method :body, :fake_body + end + + class FakeFetcher + FILES = { + "names" => "---\na\nb\n", + "versions" => "created_at: 2026-06-10T00:00:00Z\n---\n" \ + "a 1.0.0,1.1.0 #{Digest::MD5.hexdigest("---\n1.0.0 |checksum:c1\n1.1.0 |checksum:c2,created_at:2026-06-05T10:30:45Z\n")}\n" \ + "b 1.0.0-java #{Digest::MD5.hexdigest("---\n1.0.0-java |checksum:c3\n")}\n", + "info/a" => "---\n1.0.0 |checksum:c1\n1.1.0 |checksum:c2,created_at:2026-06-05T10:30:45Z\n", + "info/b" => "---\n1.0.0-java |checksum:c3\n", + }.freeze + + attr_reader :requests + + def initialize + @requests = [] + end + + def call(path, headers) + @requests << path + FakeResponse.new(FILES.fetch(path)) + end + end + + def setup + super + + @fetcher = FakeFetcher.new + @client = Gem::CompactIndexClient.new(File.join(@tempdir, "compact_index"), @fetcher) + end + + def test_names + assert_equal %w[a b], @client.names + end + + def test_versions + versions = @client.versions + + assert_equal [["a", "1.0.0"], ["a", "1.1.0"]], versions["a"] + assert_equal [["b", "1.0.0", "java"]], versions["b"] + end + + def test_info_returns_parsed_info_arrays + info = @client.info("a") + + assert_equal 2, info.size + assert_equal "a", info.last[Gem::CompactIndexClient::INFO_NAME] + assert_equal "1.1.0", info.last[Gem::CompactIndexClient::INFO_VERSION] + assert_nil info.last[Gem::CompactIndexClient::INFO_PLATFORM] + assert_includes info.last[Gem::CompactIndexClient::INFO_REQS], ["created_at", ["2026-06-05T10:30:45Z"]] + end + + def test_dependencies + dependencies = @client.dependencies(%w[a b]) + + assert_equal 2, dependencies.size + assert_equal "b", dependencies.last.first[Gem::CompactIndexClient::INFO_NAME] + assert_equal "java", dependencies.last.first[Gem::CompactIndexClient::INFO_PLATFORM] + end + + def test_latest_version + assert_equal Gem::Version.new("1.1.0"), @client.latest_version("a") + end + + def test_available + assert @client.available? + end + + def test_not_available_without_data + client = Gem::CompactIndexClient.new(File.join(@tempdir, "empty_index")) + + refute client.available? + end + + def test_fetch_info_does_not_fetch_versions_index + info = @client.fetch_info("a") + + assert_equal %w[info/a], @fetcher.requests + assert_equal "1.1.0", info.last[Gem::CompactIndexClient::INFO_VERSION] + assert_includes info.last[Gem::CompactIndexClient::INFO_REQS], ["created_at", ["2026-06-05T10:30:45Z"]] + end + + def test_fetch_info_fetches_once_per_process + @client.fetch_info("a") + @client.fetch_info("a") + + assert_equal %w[info/a], @fetcher.requests + end + + def test_reset_refetches_versions + @client.versions + @client.reset! + @client.versions + + assert_equal %w[versions versions], @fetcher.requests + end + + def test_info_uses_local_cache_when_checksum_matches + @client.versions # prime info checksums and write cache + @client.info("a") + + requests_before = @fetcher.requests.dup + fresh = Gem::CompactIndexClient.new(File.join(@tempdir, "compact_index"), @fetcher) + fresh.versions + fresh.info("a") + + assert_equal requests_before + ["versions"], @fetcher.requests + end +end diff --git a/test/rubygems/test_gem_compact_index_client_cache.rb b/test/rubygems/test_gem_compact_index_client_cache.rb new file mode 100644 index 000000000000..e34ddce6f633 --- /dev/null +++ b/test/rubygems/test_gem_compact_index_client_cache.rb @@ -0,0 +1,126 @@ +# frozen_string_literal: true + +require_relative "helper" +require "pathname" unless defined?(Pathname) +require "rubygems/compact_index_client" + +class TestGemCompactIndexClientCache < Gem::TestCase + class FakeResponse < Gem::Net::HTTPOK + def initialize(body) + super("1.1", "200", "OK") + @fake_body = body + end + + attr_reader :fake_body + alias_method :body, :fake_body + end + + class FakeFetcher + attr_reader :requests + + def initialize(body) + @body = body + @requests = [] + end + + def call(path, headers) + @requests << path + FakeResponse.new(@body) + end + end + + def setup + super + + @dir = Pathname(@tempdir).join("compact_index") + end + + def test_initialize_creates_cache_directories + Gem::CompactIndexClient::Cache.new(@dir) + + assert @dir.join("info").directory? + assert @dir.join("info-special-characters").directory? + assert @dir.join("info-etags").directory? + end + + def test_reads_cached_files_without_fetcher + cache = Gem::CompactIndexClient::Cache.new(@dir) + @dir.join("versions").binwrite "a 1.0.0\n" + + assert_equal "a 1.0.0\n", cache.versions + assert_nil cache.names + end + + def test_versions_fetches_once + fetcher = FakeFetcher.new("a 1.0.0\n") + cache = Gem::CompactIndexClient::Cache.new(@dir, fetcher) + + assert_equal "a 1.0.0\n", cache.versions + assert_equal "a 1.0.0\n", cache.versions + assert_equal ["versions"], fetcher.requests + assert_equal "a 1.0.0\n", @dir.join("versions").read + end + + def test_reset_allows_fetching_again + fetcher = FakeFetcher.new("a 1.0.0\n") + cache = Gem::CompactIndexClient::Cache.new(@dir, fetcher) + + cache.versions + cache.reset! + cache.versions + + assert_equal %w[versions versions], fetcher.requests + end + + def test_info_skips_fetch_when_checksum_matches + fetcher = FakeFetcher.new("a 1.0.0\n") + cache = Gem::CompactIndexClient::Cache.new(@dir, fetcher) + @dir.join("info", "a").binwrite "a 1.0.0\n" + + content = cache.info("a", Digest::MD5.hexdigest("a 1.0.0\n")) + + assert_equal "a 1.0.0\n", content + assert_empty fetcher.requests + end + + def test_info_fetches_when_checksum_differs + fetcher = FakeFetcher.new("a 1.0.0\na 1.1.0\n") + cache = Gem::CompactIndexClient::Cache.new(@dir, fetcher) + @dir.join("info", "a").binwrite "a 1.0.0\n" + + content = cache.info("a", Digest::MD5.hexdigest("a 1.0.0\na 1.1.0\n")) + + assert_equal "a 1.0.0\na 1.1.0\n", content + assert_equal ["info/a"], fetcher.requests + assert_equal "a 1.0.0\na 1.1.0\n", @dir.join("info", "a").read + end + + def test_info_without_checksum_reads_cached_file + fetcher = FakeFetcher.new("a 1.0.0\n") + cache = Gem::CompactIndexClient::Cache.new(@dir, fetcher) + @dir.join("info", "a").binwrite "a 1.0.0\n" + + assert_equal "a 1.0.0\n", cache.info("a") + assert_empty fetcher.requests + end + + def test_fetch_info_fetches_without_checksum + fetcher = FakeFetcher.new("a 1.0.0\n") + cache = Gem::CompactIndexClient::Cache.new(@dir, fetcher) + + assert_equal "a 1.0.0\n", cache.fetch_info("a") + assert_equal ["info/a"], fetcher.requests + assert_equal "a 1.0.0\n", @dir.join("info", "a").read + end + + def test_info_with_special_characters_uses_hashed_path + fetcher = FakeFetcher.new("1.0.0\n") + cache = Gem::CompactIndexClient::Cache.new(@dir, fetcher) + + cache.info("Rails", "no-match") + + hashed = "Rails-#{Digest::MD5.hexdigest("Rails").downcase}" + assert_equal "1.0.0\n", @dir.join("info-special-characters", hashed).read + refute @dir.join("info", "Rails").exist? + end +end diff --git a/test/rubygems/test_gem_compact_index_client_cache_file.rb b/test/rubygems/test_gem_compact_index_client_cache_file.rb new file mode 100644 index 000000000000..c782f6336165 --- /dev/null +++ b/test/rubygems/test_gem_compact_index_client_cache_file.rb @@ -0,0 +1,135 @@ +# frozen_string_literal: true + +require_relative "helper" +require "pathname" unless defined?(Pathname) +require "rubygems/compact_index_client" + +class TestGemCompactIndexClientCacheFile < Gem::TestCase + CacheFile = Gem::CompactIndexClient::CacheFile + + def setup + super + + @path = Pathname(@tempdir).join("versions") + end + + def sha256(data) + { "sha-256" => Digest::SHA256.base64digest(data) } + end + + def test_write_creates_file + CacheFile.write(@path, "created_at: 2026-06-10\n---\nrake 13.0.0\n") + + assert_equal "created_at: 2026-06-10\n---\nrake 13.0.0\n", @path.read + end + + def test_write_replaces_original_file + @path.binwrite "old" + + CacheFile.write(@path, "new") + + assert_equal "new", @path.read + end + + def test_write_removes_temp_file + CacheFile.write(@path, "data") + + assert_empty Dir.glob("#{@path}.*.tmp") + end + + def test_write_nil_data_does_nothing + CacheFile.write(@path, nil) + + refute @path.exist? + end + + def test_write_with_matching_digests + CacheFile.write(@path, "data", sha256("data")) + + assert_equal "data", @path.read + end + + def test_write_with_mismatched_digests + @path.binwrite "old" + + assert_raise CacheFile::DigestMismatchError do + CacheFile.write(@path, "data", sha256("other data")) + end + + assert_equal "old", @path.read + end + + def test_append_without_digests_returns_false + @path.binwrite "abc" + + appended = nil + CacheFile.new(@path) {|file| appended = file.append("def") } + + refute appended + assert_equal "abc", @path.read + end + + def test_append_with_matching_digests + @path.binwrite "abc" + + appended = nil + CacheFile.copy(@path) do |file| + file.digests = sha256("abcdef") + appended = file.append("def") + end + + assert appended + assert_equal "abcdef", @path.read + end + + def test_append_with_mismatched_digests_keeps_original + @path.binwrite "abc" + + appended = nil + CacheFile.copy(@path) do |file| + file.digests = sha256("abcxyz") + appended = file.append("def") + end + + refute appended + assert_equal "abc", @path.read + end + + def test_close_removes_temp_file + file = CacheFile.new(@path) + file.open {|f| f.write "data" } + file.close + + assert_empty Dir.glob("#{@path}.*.tmp") + refute @path.exist? + end + + def test_open_after_close_raises + file = CacheFile.new(@path) + file.close + + assert_raise CacheFile::ClosedError do + file.open {|f| f.write "data" } + end + end + + def test_commit_after_close_raises + file = CacheFile.new(@path) + file.close + + assert_raise CacheFile::ClosedError do + file.commit + end + end + + def test_write_preserves_permissions + pend "chmod is unreliable on Windows" if Gem.win_platform? + + @path.binwrite "old" + @path.chmod 0o600 + + CacheFile.write(@path, "new") + + assert_equal 0o600, @path.stat.mode & 0o777 + end +end diff --git a/test/rubygems/test_gem_compact_index_client_http_fetcher.rb b/test/rubygems/test_gem_compact_index_client_http_fetcher.rb new file mode 100644 index 000000000000..c563482dad89 --- /dev/null +++ b/test/rubygems/test_gem_compact_index_client_http_fetcher.rb @@ -0,0 +1,128 @@ +# frozen_string_literal: true + +require_relative "helper" +require "rubygems/compact_index_client" + +class TestGemCompactIndexClientHTTPFetcher < Gem::TestCase + class FakeResponse < Gem::Net::HTTPOK + def initialize(body) + super("1.1", "200", "OK") + @fake_body = body + end + + attr_reader :fake_body + alias_method :body, :fake_body + end + + class FakeRedirect < Gem::Net::HTTPFound + def initialize(location) + super("1.1", "302", "Found") + self["Location"] = location + end + end + + class FakePermanentRedirect < Gem::Net::HTTPPermanentRedirect + def initialize(location) + super("1.1", "308", "Permanent Redirect") + self["Location"] = location + end + end + + class FakeNotFound < Gem::Net::HTTPNotFound + def initialize + super("1.1", "404", "Not Found") + end + end + + class FakeRemoteFetcher + attr_reader :requests + + def initialize(responses) + @responses = responses + @requests = [] + end + + def request(uri, request_class) + request = request_class.new(uri) + yield request if block_given? + @requests << [uri, request] + @responses.fetch(uri.to_s) + end + end + + def fetcher_for(responses) + remote = FakeRemoteFetcher.new(responses) + [Gem::CompactIndexClient::HTTPFetcher.new("https://index.example", remote), remote] + end + + def test_call_joins_path_with_base_uri + fetcher, remote = fetcher_for("https://index.example/info/a" => FakeResponse.new("data")) + + response = fetcher.call("info/a") + + assert_equal "data", response.body + assert_equal Gem::URI("https://index.example/info/a"), remote.requests.first.first + end + + def test_call_applies_request_headers + fetcher, remote = fetcher_for("https://index.example/versions" => FakeResponse.new("data")) + + fetcher.call("versions", "If-None-Match" => '"abc"', "Range" => "bytes=10-") + + _, request = remote.requests.first + assert_equal '"abc"', request["If-None-Match"] + assert_equal "bytes=10-", request["Range"] + end + + def test_call_follows_redirects + fetcher, remote = fetcher_for( + "https://index.example/versions" => FakeRedirect.new("https://mirror.example/versions"), + "https://mirror.example/versions" => FakeResponse.new("data") + ) + + response = fetcher.call("versions") + + assert_equal "data", response.body + assert_equal 2, remote.requests.size + end + + def test_call_follows_permanent_redirects + fetcher, _remote = fetcher_for( + "https://index.example/versions" => FakePermanentRedirect.new("https://mirror.example/versions"), + "https://mirror.example/versions" => FakeResponse.new("data") + ) + + assert_equal "data", fetcher.call("versions").body + end + + def test_call_resolves_relative_redirect_location + fetcher, _remote = fetcher_for( + "https://index.example/versions" => FakeRedirect.new("/v2/versions"), + "https://index.example/v2/versions" => FakeResponse.new("data") + ) + + assert_equal "data", fetcher.call("versions").body + end + + def test_call_raises_after_too_many_redirects + fetcher, _remote = fetcher_for( + "https://index.example/versions" => FakeRedirect.new("https://index.example/versions") + ) + + error = assert_raise Gem::RemoteFetcher::FetchError do + fetcher.call("versions") + end + + assert_match(/too many redirects/, error.message) + end + + def test_call_raises_fetch_error_on_failure_response + fetcher, _remote = fetcher_for("https://index.example/versions" => FakeNotFound.new) + + error = assert_raise Gem::RemoteFetcher::FetchError do + fetcher.call("versions") + end + + assert_match(/bad response Not Found 404/, error.message) + end +end diff --git a/test/rubygems/test_gem_compact_index_client_parser.rb b/test/rubygems/test_gem_compact_index_client_parser.rb new file mode 100644 index 000000000000..590cb102cb99 --- /dev/null +++ b/test/rubygems/test_gem_compact_index_client_parser.rb @@ -0,0 +1,117 @@ +# frozen_string_literal: true + +require_relative "helper" +require "rubygems/compact_index_client" + +class TestGemCompactIndexClientParser < Gem::TestCase + class FakeIndex + attr_reader :info_requests + + def initialize(names: nil, versions: nil, info: {}) + @names = names + @versions = versions + @info = info + @info_requests = {} + end + + attr_reader :names, :versions + + def info(name, checksum) + @info_requests[name] = checksum + @info[name] + end + end + + def test_names_strips_header + index = FakeIndex.new(names: "---\na\nb\n") + parser = Gem::CompactIndexClient::Parser.new(index) + + assert_equal %w[a b], parser.names + end + + def test_versions_parses_versions_and_platforms + index = FakeIndex.new(versions: <<~VERSIONS) + created_at: 2026-06-10T00:00:00Z + --- + a 1.0.0,1.1.0 aaa111 + b 1.0.0-java bbb222 + VERSIONS + parser = Gem::CompactIndexClient::Parser.new(index) + + versions = parser.versions + + assert_equal [["a", "1.0.0"], ["a", "1.1.0"]], versions["a"] + assert_equal [["b", "1.0.0", "java"]], versions["b"] + end + + def test_versions_applies_deletions + index = FakeIndex.new(versions: <<~VERSIONS) + --- + a 1.0.0,1.1.0 aaa111 + a -1.1.0 aaa222 + VERSIONS + parser = Gem::CompactIndexClient::Parser.new(index) + + assert_equal [["a", "1.0.0"]], parser.versions["a"] + end + + def test_info_passes_checksum_from_versions_index + index = FakeIndex.new(versions: "---\na 1.0.0 aaa111\na 1.1.0 aaa222\n", + info: { "a" => "---\n1.0.0 |checksum:abc\n" }) + parser = Gem::CompactIndexClient::Parser.new(index) + + parser.info("a") + + assert_equal({ "a" => "aaa222" }, index.info_requests) + end + + def test_info_parses_dependencies_and_requirements + line = "1.2.0 b:>= 1.0&< 2.0,c:= 0.5" \ + "|checksum:abc123,ruby:>= 3.0,rubygems:>= 3.3.3,created_at:2026-06-05T10:30:45Z" + index = FakeIndex.new(versions: "---\na 1.2.0 aaa111\n", + info: { "a" => "---\n#{line}\n" }) + parser = Gem::CompactIndexClient::Parser.new(index) + + info = parser.info("a") + + name, version, platform, dependencies, requirements = info.first + assert_equal "a", name + assert_equal "1.2.0", version + assert_nil platform + assert_equal [["b", [">= 1.0", "< 2.0"]], ["c", ["= 0.5"]]], dependencies + assert_equal [["checksum", ["abc123"]], ["ruby", [">= 3.0"]], + ["rubygems", [">= 3.3.3"]], ["created_at", ["2026-06-05T10:30:45Z"]]], requirements + end + + def test_info_parses_platform_version + index = FakeIndex.new(versions: "---\na 1.0.0-java aaa111\n", + info: { "a" => "---\n1.0.0-java |checksum:abc\n" }) + parser = Gem::CompactIndexClient::Parser.new(index) + + _, version, platform, = parser.info("a").first + + assert_equal "1.0.0", version + assert_equal "java", platform + end + + def test_available_with_versions_data + parser = Gem::CompactIndexClient::Parser.new(FakeIndex.new(versions: "---\na 1.0.0 aaa111\n")) + + assert parser.available? + end + + def test_available_with_no_data + parser = Gem::CompactIndexClient::Parser.new(FakeIndex.new) + + refute parser.available? + end + + def test_skips_blank_lines_in_versions_index + index = FakeIndex.new(versions: "---\na 1.0.0 aaa111\n\n", + info: { "a" => "1.0.0 |checksum:abc\n" }) + parser = Gem::CompactIndexClient::Parser.new(index) + + assert parser.available? + assert_equal 1, parser.info("a").size + end +end diff --git a/test/rubygems/test_gem_compact_index_client_updater.rb b/test/rubygems/test_gem_compact_index_client_updater.rb new file mode 100644 index 000000000000..2660f0df24c5 --- /dev/null +++ b/test/rubygems/test_gem_compact_index_client_updater.rb @@ -0,0 +1,180 @@ +# frozen_string_literal: true + +require_relative "helper" +require "pathname" unless defined?(Pathname) +require "rubygems/compact_index_client" + +class TestGemCompactIndexClientUpdater < Gem::TestCase + class FakeFetcher + attr_reader :requests + + def initialize(*responses) + @responses = responses + @requests = [] + end + + def call(path, headers) + @requests << [path, headers] + @responses.shift + end + end + + class FakeResponse < Gem::Net::HTTPOK + def initialize(body, headers = {}) + super("1.1", "200", "OK") + @fake_body = body + headers.each {|name, value| self[name] = value } + end + + attr_reader :fake_body + alias_method :body, :fake_body + end + + class FakePartialResponse < Gem::Net::HTTPPartialContent + def initialize(body, headers = {}) + super("1.1", "206", "Partial Content") + @fake_body = body + headers.each {|name, value| self[name] = value } + end + + attr_reader :fake_body + alias_method :body, :fake_body + end + + class FakeNotModified < Gem::Net::HTTPNotModified + def initialize + super("1.1", "304", "Not Modified") + end + end + + def setup + super + + @local_path = Pathname(@tempdir).join("versions") + @etag_path = Pathname(@tempdir).join("versions.etag") + end + + def digest_header(data) + "sha-256=:#{Digest::SHA256.base64digest(data)}:" + end + + def test_update_fetches_full_file_when_no_local_copy + fetcher = FakeFetcher.new(FakeResponse.new("a 1.0.0\n", "ETag" => '"abc123"')) + updater = Gem::CompactIndexClient::Updater.new(fetcher) + + updater.update("versions", @local_path, @etag_path) + + assert_equal "a 1.0.0\n", @local_path.read + assert_equal "abc123", @etag_path.read + + path, headers = fetcher.requests.first + assert_equal "versions", path + assert_empty headers + end + + def test_update_sends_etag_and_keeps_file_on_not_modified + @local_path.binwrite "a 1.0.0\n" + @etag_path.binwrite "abc123" + fetcher = FakeFetcher.new(FakeNotModified.new) + updater = Gem::CompactIndexClient::Updater.new(fetcher) + + updater.update("versions", @local_path, @etag_path) + + assert_equal "a 1.0.0\n", @local_path.read + + _, headers = fetcher.requests.first + assert_equal "bytes=7-", headers["Range"] + assert_equal '"abc123"', headers["If-None-Match"] + end + + def test_update_appends_ranged_response + @local_path.binwrite "a 1.0.0\n" + body = "\na 1.1.0\n" + response = FakePartialResponse.new(body, + "ETag" => '"def456"', + "Repr-Digest" => digest_header("a 1.0.0\na 1.1.0\n")) + fetcher = FakeFetcher.new(response) + updater = Gem::CompactIndexClient::Updater.new(fetcher) + + updater.update("versions", @local_path, @etag_path) + + assert_equal "a 1.0.0\na 1.1.0\n", @local_path.read + assert_equal "def456", @etag_path.read + + _, headers = fetcher.requests.first + assert_equal "bytes=7-", headers["Range"] + end + + def test_update_replaces_file_when_server_ignores_range + @local_path.binwrite "stale data" + response = FakeResponse.new("a 1.0.0\n", + "ETag" => '"def456"', + "Repr-Digest" => digest_header("a 1.0.0\n")) + fetcher = FakeFetcher.new(response) + updater = Gem::CompactIndexClient::Updater.new(fetcher) + + updater.update("versions", @local_path, @etag_path) + + assert_equal "a 1.0.0\n", @local_path.read + end + + def test_update_retries_with_full_request_on_bad_ranged_response + @local_path.binwrite "a 1.0.0\n" + bad_append = FakePartialResponse.new("\nb 1.0.0\n", + "Repr-Digest" => digest_header("something else entirely")) + full = FakeResponse.new("a 1.0.0\nb 1.0.0\n", + "ETag" => '"def456"', + "Repr-Digest" => digest_header("a 1.0.0\nb 1.0.0\n")) + fetcher = FakeFetcher.new(bad_append, full) + updater = Gem::CompactIndexClient::Updater.new(fetcher) + + updater.update("versions", @local_path, @etag_path) + + assert_equal "a 1.0.0\nb 1.0.0\n", @local_path.read + assert_equal 2, fetcher.requests.size + end + + def test_update_raises_on_full_response_checksum_mismatch + response = FakeResponse.new("a 1.0.0\n", + "Repr-Digest" => digest_header("tampered")) + fetcher = FakeFetcher.new(response) + updater = Gem::CompactIndexClient::Updater.new(fetcher) + + assert_raise Gem::CompactIndexClient::Updater::MismatchedChecksumError do + updater.update("versions", @local_path, @etag_path) + end + + refute @local_path.exist? + end + + def test_update_parses_weak_etag + response = FakeResponse.new("a 1.0.0\n", "ETag" => 'W/"weak1"') + fetcher = FakeFetcher.new(response) + updater = Gem::CompactIndexClient::Updater.new(fetcher) + + updater.update("versions", @local_path, @etag_path) + + assert_equal "weak1", @etag_path.read + end + + def test_update_ignores_malformed_digest_header + response = FakeResponse.new("a 1.0.0\n", "Repr-Digest" => "sha-256") + fetcher = FakeFetcher.new(response) + updater = Gem::CompactIndexClient::Updater.new(fetcher) + + updater.update("versions", @local_path, @etag_path) + + assert_equal "a 1.0.0\n", @local_path.read + end + + def test_update_ignores_unsupported_digest_algorithms + response = FakeResponse.new("a 1.0.0\n", + "Repr-Digest" => "md5=:#{Digest::MD5.base64digest("bogus")}:") + fetcher = FakeFetcher.new(response) + updater = Gem::CompactIndexClient::Updater.new(fetcher) + + updater.update("versions", @local_path, @etag_path) + + assert_equal "a 1.0.0\n", @local_path.read + end +end diff --git a/test/rubygems/test_gem_compact_index_stub.rb b/test/rubygems/test_gem_compact_index_stub.rb new file mode 100644 index 000000000000..dc9980a021c9 --- /dev/null +++ b/test/rubygems/test_gem_compact_index_stub.rb @@ -0,0 +1,94 @@ +# frozen_string_literal: true + +require_relative "helper" +require "rubygems/compact_index_client" + +## +# Exercises the util_setup_compact_index test helper against the real +# Gem::CompactIndexClient to ensure the stubbed endpoints speak the +# compact index protocol correctly. + +class TestGemCompactIndexStub < Gem::TestCase + def setup + super + + @fetcher = Gem::FakeFetcher.new + Gem::RemoteFetcher.fetcher = @fetcher + + @a1 = util_spec "a", "1.0.0", "b" => ">= 1.0" + @a2 = util_spec "a", "1.1.0" + @b1 = util_spec "b", "1.0.0" + @c1 = util_spec "c", "1.0.0" do |s| + s.platform = "java" + s.required_ruby_version = ">= 3.0" + end + end + + def client + Gem::CompactIndexClient.new( + File.join(@tempdir, "compact_index"), + Gem::CompactIndexClient::HTTPFetcher.new(@gem_repo, @fetcher) + ) + end + + def test_serves_names + util_setup_compact_index @a1, @a2, @b1 + + assert_equal %w[a b], client.names + end + + def test_serves_versions_with_platforms + util_setup_compact_index @a1, @a2, @c1 + + versions = client.versions + + assert_equal [["a", "1.0.0"], ["a", "1.1.0"]], versions["a"] + assert_equal [["c", "1.0.0", "java"]], versions["c"] + end + + def test_serves_info_with_dependencies_and_requirements + util_setup_compact_index @a1, @a2, @b1, @c1 + + info = client.info("a") + + assert_equal "1.0.0", info.first[Gem::CompactIndexClient::INFO_VERSION] + assert_equal [["b", [">= 1.0"]]], info.first[Gem::CompactIndexClient::INFO_DEPS] + + ruby_req = client.info("c").first[Gem::CompactIndexClient::INFO_REQS].assoc("ruby") + assert_equal ["ruby", [">= 3.0"]], ruby_req + end + + def test_serves_created_at_metadata + util_setup_compact_index @a1, @a2, created_at: { "a-1.1.0" => "2026-06-05T10:30:45Z" } + + info = client.info("a") + + assert_nil info.first[Gem::CompactIndexClient::INFO_REQS].assoc("created_at") + assert_equal ["created_at", ["2026-06-05T10:30:45Z"]], + info.last[Gem::CompactIndexClient::INFO_REQS].assoc("created_at") + end + + def test_versions_checksums_match_info_files + util_setup_compact_index @a1, @a2, @b1 + + c = client + c.versions + c.info("a") + + # checksum from the versions index matches the cached info file, so a + # fresh client only refetches the versions index + fresh = client + fresh.versions + fresh.info("a") + + info_requests = @fetcher.requests.count {|req| req.path.end_with?("info/a") } + assert_equal 1, info_requests + end + + def test_repr_digest_verification_round_trip + util_setup_compact_index @a1 + + assert_equal [["a", "1.0.0"]], client.versions["a"] + assert @tempdir && File.file?(File.join(@tempdir, "compact_index", "versions")) + end +end diff --git a/test/rubygems/test_gem_dependency_installer.rb b/test/rubygems/test_gem_dependency_installer.rb index c2fb6f264b92..27fad9513536 100644 --- a/test/rubygems/test_gem_dependency_installer.rb +++ b/test/rubygems/test_gem_dependency_installer.rb @@ -506,6 +506,29 @@ def test_install_local_dependency_no_network_for_target_gem assert_equal %w[a-1 b-1], inst.installed_gems.map(&:full_name) end + def test_install_compact_index_api + a1, a1_gem = util_gem "a", 1, "b" => ">= 1" + b1, b1_gem = util_gem "b", 1 + + util_setup_compact_index a1, b1 + + add_to_fetcher a1, a1_gem + add_to_fetcher b1, b1_gem + + # the compact index probe succeeds, so resolution goes through APISet + response = Gem::HTTPResponseFactory.create(body: "", code: 200, msg: "OK") + response.uri = Gem::URI("#{@gem_repo}versions") + @fetcher.data["#{@gem_repo}versions"] = response + + inst = Gem::DependencyInstaller.new + inst.install "a" + + assert_equal %w[a-1 b-1], inst.installed_gems.map(&:full_name) + + quick_gemspec_fetches = @fetcher.paths.grep(/gemspec\.rz/) + assert_empty quick_gemspec_fetches + end + def test_install_local_subdir util_setup_gems diff --git a/test/rubygems/test_gem_resolver_api_set.rb b/test/rubygems/test_gem_resolver_api_set.rb index b0b4943beafa..af855f1692ad 100644 --- a/test/rubygems/test_gem_resolver_api_set.rb +++ b/test/rubygems/test_gem_resolver_api_set.rb @@ -42,7 +42,7 @@ def test_find_all dependencies: [] }, ] - @fetcher.data["#{@dep_uri}a"] = "---\n1 " + @fetcher.data["#{@dep_uri}a"] = util_compact_index_response("---\n1 ") set = Gem::Resolver::APISet.new @dep_uri @@ -69,7 +69,7 @@ def test_find_all_prereleases dependencies: [] }, ] - @fetcher.data["#{@dep_uri}a"] = "---\n1\n2.a" + @fetcher.data["#{@dep_uri}a"] = util_compact_index_response("---\n1\n2.a") set = Gem::Resolver::APISet.new @dep_uri set.prerelease = true @@ -94,7 +94,7 @@ def test_find_all_cache dependencies: [] }, ] - @fetcher.data["#{@dep_uri}a"] = "---\n1 " + @fetcher.data["#{@dep_uri}a"] = util_compact_index_response("---\n1 ") set = Gem::Resolver::APISet.new @dep_uri @@ -123,7 +123,7 @@ def test_find_all_local def test_find_all_missing spec_fetcher - @fetcher.data["#{@dep_uri}a"] = "---" + @fetcher.data["#{@dep_uri}a"] = util_compact_index_response("---") set = Gem::Resolver::APISet.new @dep_uri @@ -158,8 +158,8 @@ def test_find_all_not_found def test_prefetch spec_fetcher - @fetcher.data["#{@dep_uri}a"] = "---\n1 \n" - @fetcher.data["#{@dep_uri}b"] = "---" + @fetcher.data["#{@dep_uri}a"] = util_compact_index_response("---\n1 \n") + @fetcher.data["#{@dep_uri}b"] = util_compact_index_response("---") set = Gem::Resolver::APISet.new @dep_uri @@ -175,7 +175,7 @@ def test_prefetch def test_prefetch_cache spec_fetcher - @fetcher.data["#{@dep_uri}a"] = "---\n1 \n" + @fetcher.data["#{@dep_uri}a"] = util_compact_index_response("---\n1 \n") set = Gem::Resolver::APISet.new @dep_uri @@ -185,7 +185,7 @@ def test_prefetch_cache set.prefetch [a_dep] @fetcher.data.delete "#{@dep_uri}a" - @fetcher.data["#{@dep_uri}?b"] = "---" + @fetcher.data["#{@dep_uri}?b"] = util_compact_index_response("---") set.prefetch [a_dep, b_dep] end @@ -193,8 +193,8 @@ def test_prefetch_cache def test_prefetch_cache_missing spec_fetcher - @fetcher.data["#{@dep_uri}a"] = "---\n1 \n" - @fetcher.data["#{@dep_uri}b"] = "---" + @fetcher.data["#{@dep_uri}a"] = util_compact_index_response("---\n1 \n") + @fetcher.data["#{@dep_uri}b"] = util_compact_index_response("---") set = Gem::Resolver::APISet.new @dep_uri @@ -212,8 +212,8 @@ def test_prefetch_cache_missing def test_prefetch_local spec_fetcher - @fetcher.data["#{@dep_uri}a"] = "---\n1 \n" - @fetcher.data["#{@dep_uri}b"] = "---" + @fetcher.data["#{@dep_uri}a"] = util_compact_index_response("---\n1 \n") + @fetcher.data["#{@dep_uri}b"] = util_compact_index_response("---") set = Gem::Resolver::APISet.new @dep_uri set.remote = false diff --git a/test/rubygems/test_gem_resolver_api_specification.rb b/test/rubygems/test_gem_resolver_api_specification.rb index 2119d734780b..ed6e6615c353 100644 --- a/test/rubygems/test_gem_resolver_api_specification.rb +++ b/test/rubygems/test_gem_resolver_api_specification.rb @@ -27,6 +27,37 @@ def test_initialize ] assert_equal expected, spec.dependencies + assert_nil spec.created_at + end + + def test_initialize_created_at + set = Gem::Resolver::APISet.new + data = { + name: "rails", + number: "3.0.3", + platform: "ruby", + dependencies: [], + requirements: { created_at: ["2026-06-05T10:30:45Z"] }, + } + + spec = Gem::Resolver::APISpecification.new set, data + + assert_equal Time.new("2026-06-05T10:30:45Z"), spec.created_at + end + + def test_initialize_created_at_invalid + set = Gem::Resolver::APISet.new + data = { + name: "rails", + number: "3.0.3", + platform: "ruby", + dependencies: [], + requirements: { created_at: ["not a timestamp"] }, + } + + spec = Gem::Resolver::APISpecification.new set, data + + assert_nil spec.created_at end def test_fetch_development_dependencies diff --git a/test/rubygems/test_gem_resolver_best_set.rb b/test/rubygems/test_gem_resolver_best_set.rb index ac186884d15b..dfe50f02d76a 100644 --- a/test/rubygems/test_gem_resolver_best_set.rb +++ b/test/rubygems/test_gem_resolver_best_set.rb @@ -16,7 +16,7 @@ def test_find_all api_uri = Gem::URI "#{@gem_repo}info/" - @fetcher.data["#{api_uri}a"] = "---\n1 " + @fetcher.data["#{api_uri}a"] = util_compact_index_response("---\n1 ") set = Gem::Resolver::BestSet.new diff --git a/test/rubygems/test_gem_source.rb b/test/rubygems/test_gem_source.rb index 423abd6dd20a..cd9f3398532b 100644 --- a/test/rubygems/test_gem_source.rb +++ b/test/rubygems/test_gem_source.rb @@ -132,6 +132,60 @@ def test_load_specs assert File.exist?(cache_file) end + def test_load_specs_compact_index + a1 = util_spec "a", "1" + a2 = util_spec "a", "2" + a3_pre = util_spec "a", "3.a" + b2_java = util_spec "b", "2" do |s| + s.platform = "java" + end + + util_setup_compact_index a1, a2, a3_pre, b2_java + + released = @source.load_specs(:released).map(&:full_name) + assert_equal %w[a-1 a-2 b-2-java], released + + latest = @source.load_specs(:latest).map(&:full_name) + assert_equal %w[a-2 b-2-java], latest + + prerelease = @source.load_specs(:prerelease).map(&:full_name) + assert_equal %w[a-3.a], prerelease + + cache_dir = File.join Gem.spec_cache_dir, "compact_index", "gems.example.com%80" + assert File.exist?(File.join(cache_dir, "versions")), "versions cache file does not exist" + end + + def test_load_specs_compact_index_latest_per_platform + a1 = util_spec "a", "1" + a2_java = util_spec "a", "2" do |s| + s.platform = "java" + end + + util_setup_compact_index a1, a2_java + + latest = @source.load_specs(:latest).map(&:full_name) + assert_equal %w[a-1 a-2-java], latest + end + + def test_load_specs_compact_index_fetched_once + a1 = util_spec "a", "1" + + util_setup_compact_index a1 + + @source.load_specs :released + @source.load_specs :prerelease + + versions_requests = @fetcher.requests.count {|req| req.path.end_with?("/versions") } + assert_equal 1, versions_requests + end + + def test_load_specs_falls_back_to_marshal_index + # no compact index data set up, only the Marshal indexes from setup + released = @source.load_specs(:released).map(&:full_name) + + assert_equal %W[a-2 a-1 b-2], released + end + def test_load_specs_cached latest_specs = @source.load_specs :latest