Rebuild enrichment branch on top of 0.6.0 changes

Author: andrew

lib/git/pkgs.rb

@@ -11,13 +11,15 @@
 require_relative "pkgs/ecosystems"
 require_relative "pkgs/osv_client"
 
+require_relative "pkgs/purl_helper"
 require_relative "pkgs/models/branch"
 require_relative "pkgs/models/branch_commit"
 require_relative "pkgs/models/commit"
 require_relative "pkgs/models/manifest"
 require_relative "pkgs/models/dependency_change"
 require_relative "pkgs/models/dependency_snapshot"
 require_relative "pkgs/models/package"
+require_relative "pkgs/models/version"
 require_relative "pkgs/models/vulnerability"
 require_relative "pkgs/models/vulnerability_package"
 

lib/git/pkgs/database.rb

@@ -84,6 +84,7 @@ def self.refresh_models
           Git::Pkgs::Models::DependencyChange,
           Git::Pkgs::Models::DependencySnapshot,
           Git::Pkgs::Models::Package,
+          Git::Pkgs::Models::Version,
           Git::Pkgs::Models::Vulnerability,
           Git::Pkgs::Models::VulnerabilityPackage
         ].each do |model|
@@ -201,6 +202,21 @@ def self.create_schema(with_indexes: true)
           index [:ecosystem, :name]
         end
 
+        @db.create_table?(:versions) do
+          primary_key :id
+          String :purl, null: false
+          String :package_purl, null: false
+          String :license
+          DateTime :published_at
+          String :integrity, text: true
+          String :source
+          DateTime :enriched_at
+          DateTime :created_at
+          DateTime :updated_at
+          index :purl, unique: true
+          index :package_purl
+        end
+
         # Core vulnerability data (one row per CVE/GHSA)
         @db.create_table?(:vulnerabilities) do
           String :id, primary_key: true          # CVE-2024-1234, GHSA-xxxx, etc.

lib/git/pkgs/models/dependency_change.rb

@@ -28,6 +28,14 @@ def for_platform(platform)
             where(ecosystem: platform)
           end
         end
+
+        def purl(with_version: true)
+          version = nil
+          if with_version && manifest&.kind == "lockfile"
+            version = requirement
+          end
+          PurlHelper.build_purl(ecosystem: ecosystem, name: name, version: version)
+        end
       end
     end
   end

lib/git/pkgs/models/dependency_snapshot.rb

@@ -29,6 +29,14 @@ def self.current_for_branch(branch)
 
           where(commit: commit)
         end
+
+        def purl(with_version: true)
+          version = nil
+          if with_version && manifest&.kind == "lockfile"
+            version = requirement
+          end
+          PurlHelper.build_purl(ecosystem: ecosystem, name: name, version: version)
+        end
       end
     end
   end

lib/git/pkgs/models/package.rb

@@ -6,6 +6,8 @@ module Models
       class Package < Sequel::Model
         STALE_THRESHOLD = 86400 # 24 hours
 
+        one_to_many :versions, key: :package_purl, primary_key: :purl
+
         dataset_module do
           def by_ecosystem(ecosystem)
             where(ecosystem: ecosystem)
@@ -20,6 +22,18 @@ def synced
           end
         end
 
+        def parsed_purl
+          @parsed_purl ||= Purl.parse(purl)
+        end
+
+        def registry_url
+          parsed_purl.registry_url
+        end
+
+        def enriched?
+          !enriched_at.nil?
+        end
+
         def needs_vuln_sync?
           vulns_synced_at.nil? || vulns_synced_at < Time.now - STALE_THRESHOLD
         end

lib/git/pkgs/models/version.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Git
+  module Pkgs
+    module Models
+      class Version < Sequel::Model
+        many_to_one :package, key: :package_purl, primary_key: :purl
+
+        def parsed_purl
+          @parsed_purl ||= Purl.parse(purl)
+        end
+
+        def version_string
+          parsed_purl.version
+        end
+
+        def registry_url
+          parsed_purl.registry_url
+        end
+
+        def enriched?
+          !enriched_at.nil?
+        end
+      end
+    end
+  end
+end

lib/git/pkgs/purl_helper.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+require "purl"
+
+module Git
+  module Pkgs
+    module PurlHelper
+      # Mapping from Bibliothecary/ecosyste.ms ecosystem names to PURL types
+      # Source: https://packages.ecosyste.ms/api/v1/registries/
+      ECOSYSTEM_TO_PURL_TYPE = {
+        "npm" => "npm",
+        "go" => "golang",
+        "docker" => "docker",
+        "pypi" => "pypi",
+        "nuget" => "nuget",
+        "maven" => "maven",
+        "packagist" => "composer",
+        "cargo" => "cargo",
+        "rubygems" => "gem",
+        "cocoapods" => "cocoapods",
+        "pub" => "pub",
+        "bower" => "bower",
+        "cpan" => "cpan",
+        "alpine" => "alpine",
+        "actions" => "githubactions",
+        "cran" => "cran",
+        "clojars" => "clojars",
+        "conda" => "conda",
+        "hex" => "hex",
+        "hackage" => "hackage",
+        "julia" => "julia",
+        "swiftpm" => "swift",
+        "openvsx" => "openvsx",
+        "spack" => "spack",
+        "homebrew" => "brew",
+        "puppet" => "puppet",
+        "deno" => "deno",
+        "elm" => "elm",
+        "vcpkg" => "vcpkg",
+        "racket" => "racket",
+        "bioconductor" => "bioconductor",
+        "carthage" => "carthage",
+        "elpa" => "melpa"
+      }.freeze
+
+      def self.purl_type_for(ecosystem)
+        ECOSYSTEM_TO_PURL_TYPE.fetch(ecosystem, ecosystem)
+      end
+
+      def self.build_purl(ecosystem:, name:, version: nil)
+        type = purl_type_for(ecosystem)
+        Purl::PackageURL.new(type: type, name: name, version: version)
+      end
+    end
+  end
+end

test/git/pkgs/test_models.rb

@@ -111,4 +111,150 @@ def test_branch_commit_associations
     assert_includes branch.commits, commit
     assert_includes commit.branches, branch
   end
+
+  def test_dependency_change_purl_from_lockfile
+    repo = Git::Pkgs::Repository.new(@test_dir)
+    rugged_commit = repo.walk("main").first
+    commit = Git::Pkgs::Models::Commit.find_or_create_from_rugged(rugged_commit)
+
+    manifest = Git::Pkgs::Models::Manifest.find_or_create(
+      path: "Gemfile.lock",
+      ecosystem: "rubygems",
+      kind: "lockfile"
+    )
+
+    change = Git::Pkgs::Models::DependencyChange.create(
+      commit: commit,
+      manifest: manifest,
+      name: "rails",
+      ecosystem: "rubygems",
+      change_type: "added",
+      requirement: "7.0.0"
+    )
+
+    assert_equal "pkg:gem/rails@7.0.0", change.purl.to_s
+    assert_equal "pkg:gem/rails", change.purl(with_version: false).to_s
+  end
+
+  def test_dependency_change_purl_from_manifest_omits_version
+    repo = Git::Pkgs::Repository.new(@test_dir)
+    rugged_commit = repo.walk("main").first
+    commit = Git::Pkgs::Models::Commit.find_or_create_from_rugged(rugged_commit)
+
+    manifest = Git::Pkgs::Models::Manifest.find_or_create(
+      path: "Gemfile",
+      ecosystem: "rubygems",
+      kind: "manifest"
+    )
+
+    change = Git::Pkgs::Models::DependencyChange.create(
+      commit: commit,
+      manifest: manifest,
+      name: "rails",
+      ecosystem: "rubygems",
+      change_type: "added",
+      requirement: "~> 7.0"
+    )
+
+    assert_equal "pkg:gem/rails", change.purl.to_s
+  end
+
+  def test_dependency_snapshot_purl_from_lockfile
+    repo = Git::Pkgs::Repository.new(@test_dir)
+    rugged_commit = repo.walk("main").first
+    commit = Git::Pkgs::Models::Commit.find_or_create_from_rugged(rugged_commit)
+
+    manifest = Git::Pkgs::Models::Manifest.find_or_create(
+      path: "package-lock.json",
+      ecosystem: "npm",
+      kind: "lockfile"
+    )
+
+    snapshot = Git::Pkgs::Models::DependencySnapshot.create(
+      commit: commit,
+      manifest: manifest,
+      name: "lodash",
+      ecosystem: "npm",
+      requirement: "4.17.21"
+    )
+
+    assert_equal "pkg:npm/lodash@4.17.21", snapshot.purl.to_s
+    assert_equal "pkg:npm/lodash", snapshot.purl(with_version: false).to_s
+  end
+
+  def test_package_creation
+    package = Git::Pkgs::Models::Package.create(
+      purl: "pkg:gem/rails",
+      latest_version: "7.1.0",
+      license: "MIT",
+      description: "Full-stack web framework",
+      source: "ecosystems"
+    )
+
+    assert_equal "pkg:gem/rails", package.purl
+    assert_equal "7.1.0", package.latest_version
+    assert_equal "MIT", package.license
+    assert_equal "ecosystems", package.source
+  end
+
+  def test_package_parsed_purl
+    package = Git::Pkgs::Models::Package.create(purl: "pkg:gem/rails")
+
+    assert_equal "gem", package.parsed_purl.type
+    assert_equal "rails", package.parsed_purl.name
+  end
+
+  def test_package_enriched
+    package = Git::Pkgs::Models::Package.create(purl: "pkg:gem/rails")
+    refute package.enriched?
+
+    package.update(enriched_at: Time.now)
+    assert package.enriched?
+  end
+
+  def test_version_creation
+    Git::Pkgs::Models::Package.create(purl: "pkg:gem/rails")
+
+    version = Git::Pkgs::Models::Version.create(
+      purl: "pkg:gem/rails@7.0.0",
+      package_purl: "pkg:gem/rails",
+      license: "MIT",
+      published_at: Time.parse("2021-12-15"),
+      integrity: "sha256:abc123",
+      source: "ecosystems"
+    )
+
+    assert_equal "pkg:gem/rails@7.0.0", version.purl
+    assert_equal "pkg:gem/rails", version.package_purl
+    assert_equal "7.0.0", version.version_string
+  end
+
+  def test_version_belongs_to_package
+    package = Git::Pkgs::Models::Package.create(purl: "pkg:gem/rails")
+
+    version = Git::Pkgs::Models::Version.create(
+      purl: "pkg:gem/rails@7.0.0",
+      package_purl: "pkg:gem/rails"
+    )
+
+    assert_equal package.id, version.package.id
+    assert_includes package.versions.map(&:id), version.id
+  end
+
+  def test_package_purl_uniqueness
+    Git::Pkgs::Models::Package.create(purl: "pkg:gem/rails")
+
+    assert_raises(Sequel::UniqueConstraintViolation) do
+      Git::Pkgs::Models::Package.create(purl: "pkg:gem/rails")
+    end
+  end
+
+  def test_version_purl_uniqueness
+    Git::Pkgs::Models::Package.create(purl: "pkg:gem/rails")
+    Git::Pkgs::Models::Version.create(purl: "pkg:gem/rails@7.0.0", package_purl: "pkg:gem/rails")
+
+    assert_raises(Sequel::UniqueConstraintViolation) do
+      Git::Pkgs::Models::Version.create(purl: "pkg:gem/rails@7.0.0", package_purl: "pkg:gem/rails")
+    end
+  end
 end

test/git/pkgs/test_purl_helper.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+require "test_helper"
+
+class Git::Pkgs::TestPurlHelper < Minitest::Test
+  def test_purl_type_for_rubygems
+    assert_equal "gem", Git::Pkgs::PurlHelper.purl_type_for("rubygems")
+  end
+
+  def test_purl_type_for_npm
+    assert_equal "npm", Git::Pkgs::PurlHelper.purl_type_for("npm")
+  end
+
+  def test_purl_type_for_go
+    assert_equal "golang", Git::Pkgs::PurlHelper.purl_type_for("go")
+  end
+
+  def test_purl_type_for_packagist
+    assert_equal "composer", Git::Pkgs::PurlHelper.purl_type_for("packagist")
+  end
+
+  def test_purl_type_for_unknown_falls_back_to_ecosystem
+    assert_equal "unknown", Git::Pkgs::PurlHelper.purl_type_for("unknown")
+  end
+
+  def test_build_purl_without_version
+    purl = Git::Pkgs::PurlHelper.build_purl(ecosystem: "rubygems", name: "rails")
+    assert_equal "pkg:gem/rails", purl.to_s
+  end
+
+  def test_build_purl_with_version
+    purl = Git::Pkgs::PurlHelper.build_purl(ecosystem: "rubygems", name: "rails", version: "7.0.0")
+    assert_equal "pkg:gem/rails@7.0.0", purl.to_s
+  end
+
+  def test_build_purl_for_npm
+    purl = Git::Pkgs::PurlHelper.build_purl(ecosystem: "npm", name: "lodash", version: "4.17.21")
+    assert_equal "pkg:npm/lodash@4.17.21", purl.to_s
+  end
+
+  def test_build_purl_for_go
+    purl = Git::Pkgs::PurlHelper.build_purl(ecosystem: "go", name: "github.com/gorilla/mux", version: "1.8.0")
+    assert_equal "golang", purl.type
+    assert_equal "github.com/gorilla/mux", purl.name
+    assert_equal "1.8.0", purl.version
+  end
+end