diff --git a/docs/architecture.md b/docs/architecture.md index f9325b25..76b2156f 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -338,7 +338,7 @@ Version age filtering for supply chain attack mitigation, provided by [github.co ### `internal/enrichment` -Package metadata enrichment. Fetches license, description, homepage, repository URL, and vulnerability data from upstream registries. Powers the `/api/` endpoints and the web UI's package detail pages. +Package metadata enrichment, built on [github.com/git-pkgs/enrichment](https://github.com/git-pkgs/enrichment). Fetches license, description, homepage, repository URL, and vulnerability data from upstream registries. Powers the `/api/` endpoints and the web UI's package detail pages. ### `internal/mirror` diff --git a/go.mod b/go.mod index 7d17bfde..bf5d5525 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/git-pkgs/archives v0.9.0 github.com/git-pkgs/artifacts v0.2.1 github.com/git-pkgs/cooldown v0.3.0 - github.com/git-pkgs/enrichment v0.7.3 + github.com/git-pkgs/enrichment v0.7.4 github.com/git-pkgs/gcs v0.1.0 github.com/git-pkgs/integrity v0.2.0 github.com/git-pkgs/magic v0.5.0 @@ -20,8 +20,6 @@ require ( github.com/git-pkgs/registries v0.11.0 github.com/git-pkgs/sbom v0.1.7 github.com/git-pkgs/spdx v0.3.3 - github.com/git-pkgs/vers v0.7.2 - github.com/git-pkgs/vulns v0.2.4 github.com/go-chi/chi/v5 v5.5.0 github.com/jmoiron/sqlx v1.4.0 github.com/klauspost/compress v1.20.1 @@ -78,6 +76,8 @@ require ( github.com/ecosyste-ms/ecosystems-go v0.5.0 // indirect github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a // indirect github.com/git-pkgs/pom v0.1.8 // indirect + github.com/git-pkgs/vers v0.7.2 // indirect + github.com/git-pkgs/vulns v0.2.4 // indirect github.com/github/go-spdx/v2 v2.7.0 // indirect github.com/go-logr/logr v1.4.4 // indirect github.com/go-logr/stdr v1.2.2 // indirect diff --git a/go.sum b/go.sum index ee376526..6fb0b93b 100644 --- a/go.sum +++ b/go.sum @@ -123,8 +123,8 @@ github.com/git-pkgs/artifacts v0.2.1 h1:VwdxR4yTDaqBZ34h0slxQBVyr2Xfa+QGOKCKviMx github.com/git-pkgs/artifacts v0.2.1/go.mod h1:Otosgq52pXT5UNN7lh6s/lszpWKVDO8XrOjN1M70/IA= github.com/git-pkgs/cooldown v0.3.0 h1:eAl2CQFwqXxluoVP9hHmropAf1VoV3gSxLMkN+NEu8A= github.com/git-pkgs/cooldown v0.3.0/go.mod h1:FnTg8wufEALATC9Sm/9AHjLdA+u9eVn96e9BqXXjGB0= -github.com/git-pkgs/enrichment v0.7.3 h1:l8bGVUP2Ncg3uyoZz7Gve+pyojUVcIQ3oJOFbYhWSOM= -github.com/git-pkgs/enrichment v0.7.3/go.mod h1:ZN/cH4PnmzsapEVPvmAwREP7+ox+RX0yfR6KvoB8xKE= +github.com/git-pkgs/enrichment v0.7.4 h1:owiCKjITG8tNxBikcyUCDuDf1i3C/+Yoa5yJseuqIjA= +github.com/git-pkgs/enrichment v0.7.4/go.mod h1:ZN/cH4PnmzsapEVPvmAwREP7+ox+RX0yfR6KvoB8xKE= github.com/git-pkgs/gcs v0.1.0 h1:E3awGtsO0xZyHT9FUfEwMHjMkRxw41Bh+c7lgTmPvBo= github.com/git-pkgs/gcs v0.1.0/go.mod h1:bdkCFD66ryaWnU8MBhokVA3WkJfyAEchm9qec5woBpE= github.com/git-pkgs/integrity v0.2.0 h1:sxpAk7EUllZnTYttOoPTW/Yfe5J2imglpuJiuBUZOxM= diff --git a/internal/enrichment/enrichment.go b/internal/enrichment/enrichment.go index db8cc139..4d847af5 100644 --- a/internal/enrichment/enrichment.go +++ b/internal/enrichment/enrichment.go @@ -1,106 +1,59 @@ -// Package enrichment provides package metadata enrichment using external data sources. -// It fetches license information, vulnerability data, and version information -// from package registries and vulnerability databases. +// Package enrichment adapts github.com/git-pkgs/enrichment to the proxy's +// enrichment API. It builds package URLs from the proxy's ecosystem names, +// normalizes licenses for display and treats retracted versions as yanked. package enrichment import ( "context" + "errors" "log/slog" "sync" - "time" + shared "github.com/git-pkgs/enrichment" "github.com/git-pkgs/proxy/internal/packageurl" - "github.com/git-pkgs/purl" "github.com/git-pkgs/registries" - _ "github.com/git-pkgs/registries/all" // Import all registry implementations "github.com/git-pkgs/spdx" - "github.com/git-pkgs/vers" - "github.com/git-pkgs/vulns" - "github.com/git-pkgs/vulns/osv" ) -// Service provides package enrichment capabilities. +// Service looks up package, version and vulnerability data. type Service struct { logger *slog.Logger - regClient *registries.Client - vulnSource vulns.Source + registries *shared.RegistriesClient + vulns *shared.VulnerabilityClient } // New creates a new enrichment service. func New(logger *slog.Logger) *Service { return &Service{ logger: logger, - regClient: registries.DefaultClient(), - vulnSource: osv.New(), + registries: shared.NewRegistriesClient(), + vulns: shared.NewVulnerabilityClient(), } } -// PackageInfo contains enriched package metadata. -type PackageInfo struct { - Ecosystem string - Name string - LatestVersion string - License string - Description string - Homepage string - Repository string - RegistryURL string -} - -// VersionInfo contains enriched version metadata. -type VersionInfo struct { - Number string - License string - PublishedAt time.Time - Integrity string - Yanked bool -} - -// VulnInfo contains vulnerability information for a package version. -type VulnInfo struct { - ID string - Summary string - Severity string - CVSSScore float64 - FixedVersion string - References []string -} +type ( + PackageInfo = shared.PackageInfo + VersionInfo = shared.VersionInfo + VulnInfo = shared.VulnInfo +) -// EnrichPackage fetches metadata for a package from registry APIs. +// EnrichPackage fetches metadata for a package from its registry. It returns +// nil without an error when the registry doesn't have the package or the name +// can't be expressed as a PURL. func (s *Service) EnrichPackage(ctx context.Context, ecosystem, name string) (*PackageInfo, error) { purlStr := packageurl.MakeString(ecosystem, name, "") if purlStr == "" { return nil, nil } - - pkg, err := registries.FetchPackageFromPURL(ctx, purlStr, s.regClient) - if err != nil { - return nil, err - } - - if pkg == nil { + info, err := s.registries.GetPackage(ctx, purlStr) + if errors.Is(err, registries.ErrNotFound) { return nil, nil } - - info := &PackageInfo{ - Ecosystem: ecosystem, - Name: pkg.Name, - LatestVersion: pkg.LatestVersion, - Description: pkg.Description, - Homepage: pkg.Homepage, - Repository: pkg.Repository, - RegistryURL: registries.DefaultURL(ecosystem), - } - - // Normalize license to SPDX format - if pkg.Licenses != "" { - if normalized, err := spdx.NormalizeExpressionLax(pkg.Licenses); err == nil { - info.License = normalized - } else { - info.License = pkg.Licenses - } + if err != nil || info == nil { + return nil, err } - + info.Ecosystem = ecosystem + info.License = normalizeLicense(info.License) return info, nil } @@ -110,36 +63,24 @@ func (s *Service) EnrichVersion(ctx context.Context, ecosystem, name, version st if purlStr == "" { return nil, nil } - - ver, err := registries.FetchVersionFromPURL(ctx, purlStr, s.regClient) - if err != nil { + info, err := s.registries.GetVersion(ctx, purlStr) + if err != nil || info == nil { return nil, err } + adaptVersion(info) + return info, nil +} - if ver == nil { - return nil, nil - } - - info := &VersionInfo{ - Number: ver.Number, - PublishedAt: ver.PublishedAt, - Integrity: ver.Integrity, - Yanked: ver.Status == registries.StatusYanked || ver.Status == registries.StatusRetracted, - } - - // Normalize license - if ver.Licenses != "" { - if normalized, err := spdx.NormalizeExpressionLax(ver.Licenses); err == nil { - info.License = normalized - } else { - info.License = ver.Licenses - } +// adaptVersion reports retracted versions as yanked and normalizes the license. +func adaptVersion(info *VersionInfo) { + if info.Status == string(registries.StatusRetracted) { + info.Yanked = true } - - return info, nil + info.License = normalizeLicense(info.License) } -// BulkEnrichPackages fetches metadata for multiple packages in parallel. +// BulkEnrichPackages fetches metadata for multiple packages in parallel, +// keyed by PURL. Packages that can't be fetched are left out. func (s *Service) BulkEnrichPackages(ctx context.Context, packages []struct{ Ecosystem, Name string }) map[string]*PackageInfo { purls := make([]string, 0, len(packages)) for _, pkg := range packages { @@ -150,126 +91,40 @@ func (s *Service) BulkEnrichPackages(ctx context.Context, packages []struct{ Eco if len(purls) == 0 { return map[string]*PackageInfo{} } - - pkgData := registries.BulkFetchPackages(ctx, purls, s.regClient) - - result := make(map[string]*PackageInfo, len(pkgData)) - for purlStr, pkg := range pkgData { - if pkg == nil { - continue - } - - p, err := purl.Parse(purlStr) - if err != nil { - continue - } - info := &PackageInfo{ - Ecosystem: p.Type, - Name: pkg.Name, - LatestVersion: pkg.LatestVersion, - Description: pkg.Description, - Homepage: pkg.Homepage, - Repository: pkg.Repository, - RegistryURL: registries.DefaultURL(p.Type), - } - - if pkg.Licenses != "" { - if normalized, err := spdx.NormalizeExpressionLax(pkg.Licenses); err == nil { - info.License = normalized - } else { - info.License = pkg.Licenses - } - } - - result[purlStr] = info + result, _ := s.registries.BulkLookup(ctx, purls) + for _, info := range result { + info.License = normalizeLicense(info.License) } - return result } // CheckVulnerabilities queries for vulnerabilities affecting a package version. func (s *Service) CheckVulnerabilities(ctx context.Context, ecosystem, name, version string) ([]VulnInfo, error) { - p := packageurl.Make(ecosystem, name, version) - if p == nil { + if packageurl.Make(ecosystem, name, version) == nil { return nil, nil } - - vulnList, err := s.vulnSource.Query(ctx, p) - if err != nil { - return nil, err - } - - results := make([]VulnInfo, 0, len(vulnList)) - for _, v := range vulnList { - info := VulnInfo{ - ID: v.ID, - Summary: v.Summary, - Severity: v.SeverityLevel(), - CVSSScore: v.CVSSScore(), - FixedVersion: v.FixedVersion(ecosystem, name), - } - - for _, ref := range v.References { - info.References = append(info.References, ref.URL) - } - - results = append(results, info) - } - - return results, nil -} - -// IsOutdated checks if a version is older than the latest version. -func (s *Service) IsOutdated(currentVersion, latestVersion string) bool { - if latestVersion == "" || currentVersion == "" { - return false - } - return vers.Compare(currentVersion, latestVersion) < 0 + return s.vulns.Check(ctx, ecosystem, name, version) } // GetLatestVersion fetches the latest version for a package. func (s *Service) GetLatestVersion(ctx context.Context, ecosystem, name string) (string, error) { - purlStr := packageurl.MakeString(ecosystem, name, "") - if purlStr == "" { - return "", nil - } - - latest, err := registries.FetchLatestVersionFromPURL(ctx, purlStr, s.regClient) - if err != nil { + info, err := s.EnrichPackage(ctx, ecosystem, name) + if err != nil || info == nil { return "", err } - - if latest == nil { - return "", nil - } - - return latest.Number, nil + return info.LatestVersion, nil } -// LicenseCategory represents the category of a license. -type LicenseCategory string - -const ( - LicensePermissive LicenseCategory = "permissive" - LicenseCopyleft LicenseCategory = "copyleft" - LicenseUnknown LicenseCategory = "unknown" -) - -// CategorizeLicense returns the category of a license. -func (s *Service) CategorizeLicense(license string) LicenseCategory { - if license == "" || license == "Unknown" { - return LicenseUnknown +// normalizeLicense returns license as an SPDX expression when it can be +// parsed as one, and unchanged otherwise. +func normalizeLicense(license string) string { + if license == "" { + return "" } - - if spdx.HasCopyleft(license) { - return LicenseCopyleft - } - - if spdx.IsFullyPermissive(license) { - return LicensePermissive + if normalized, err := spdx.NormalizeExpressionLax(license); err == nil { + return normalized } - - return LicenseUnknown + return license } // EnrichmentResult contains all enrichment data for a package version. @@ -278,40 +133,21 @@ type EnrichmentResult struct { Version *VersionInfo Vulnerabilities []VulnInfo IsOutdated bool - LicenseCategory LicenseCategory + LicenseCategory shared.LicenseCategory } -// EnrichFull performs full enrichment for a package version. +// EnrichFull fetches package, version and vulnerability data for a package +// version concurrently. Failed lookups are logged and left empty. func (s *Service) EnrichFull(ctx context.Context, ecosystem, name, version string) (*EnrichmentResult, error) { result := &EnrichmentResult{} var wg sync.WaitGroup var pkgErr, verErr, vulnErr error - - // Fetch package info - wg.Add(1) - go func() { - defer wg.Done() - result.Package, pkgErr = s.EnrichPackage(ctx, ecosystem, name) - }() - - // Fetch version info - wg.Add(1) - go func() { - defer wg.Done() - result.Version, verErr = s.EnrichVersion(ctx, ecosystem, name, version) - }() - - // Check vulnerabilities - wg.Add(1) - go func() { - defer wg.Done() - result.Vulnerabilities, vulnErr = s.CheckVulnerabilities(ctx, ecosystem, name, version) - }() - + wg.Go(func() { result.Package, pkgErr = s.EnrichPackage(ctx, ecosystem, name) }) + wg.Go(func() { result.Version, verErr = s.EnrichVersion(ctx, ecosystem, name, version) }) + wg.Go(func() { result.Vulnerabilities, vulnErr = s.CheckVulnerabilities(ctx, ecosystem, name, version) }) wg.Wait() - // Log errors but don't fail if pkgErr != nil { s.logger.Debug("failed to enrich package", "ecosystem", ecosystem, "name", name, "error", pkgErr) } @@ -322,19 +158,17 @@ func (s *Service) EnrichFull(ctx context.Context, ecosystem, name, version strin s.logger.Debug("failed to check vulnerabilities", "ecosystem", ecosystem, "name", name, "version", version, "error", vulnErr) } - // Determine outdated status - if result.Package != nil && result.Package.LatestVersion != "" { - result.IsOutdated = s.IsOutdated(version, result.Package.LatestVersion) + if result.Package != nil { + result.IsOutdated = shared.IsOutdated(version, result.Package.LatestVersion) } - // Categorize license license := "" if result.Version != nil && result.Version.License != "" { license = result.Version.License - } else if result.Package != nil && result.Package.License != "" { + } else if result.Package != nil { license = result.Package.License } - result.LicenseCategory = s.CategorizeLicense(license) + result.LicenseCategory = shared.CategorizeLicense(license) return result, nil } diff --git a/internal/enrichment/enrichment_test.go b/internal/enrichment/enrichment_test.go index 3952eb25..a854e5a8 100644 --- a/internal/enrichment/enrichment_test.go +++ b/internal/enrichment/enrichment_test.go @@ -5,6 +5,8 @@ import ( "log/slog" "os" "testing" + + "github.com/git-pkgs/registries" ) func TestNew(t *testing.T) { @@ -15,12 +17,12 @@ func TestNew(t *testing.T) { t.Fatal("New() returned nil") } - if svc.regClient == nil { - t.Error("regClient is nil") + if svc.registries == nil { + t.Error("registries is nil") } - if svc.vulnSource == nil { - t.Error("vulnSource is nil") + if svc.vulns == nil { + t.Error("vulns is nil") } } @@ -54,54 +56,23 @@ func TestSwiftRegistryIdentitySkipsPURLDependentLookups(t *testing.T) { } } -func TestIsOutdated(t *testing.T) { - logger := slog.New(slog.NewTextHandler(os.Stdout, nil)) - svc := New(logger) - +func TestAdaptVersion(t *testing.T) { tests := []struct { - current string - latest string - expected bool + status registries.VersionStatus + yanked bool + license string + wantYanked bool + wantLicense string }{ - {"1.0.0", "2.0.0", true}, - {"2.0.0", "2.0.0", false}, - {"2.0.0", "1.0.0", false}, - {"1.0.0", "", false}, - {"", "2.0.0", false}, - {"1.2.3", "1.2.4", true}, - {"1.2.4", "1.2.3", false}, + {status: registries.StatusRetracted, license: "mit", wantYanked: true, wantLicense: "MIT"}, + {status: registries.StatusYanked, yanked: true, wantYanked: true}, + {status: registries.StatusDeprecated, license: "Custom terms", wantLicense: "Custom terms"}, } - - for _, tc := range tests { - result := svc.IsOutdated(tc.current, tc.latest) - if result != tc.expected { - t.Errorf("IsOutdated(%q, %q) = %v, want %v", tc.current, tc.latest, result, tc.expected) - } - } -} - -func TestCategorizeLicense(t *testing.T) { - logger := slog.New(slog.NewTextHandler(os.Stdout, nil)) - svc := New(logger) - - tests := []struct { - license string - expected LicenseCategory - }{ - {"MIT", LicensePermissive}, - {"Apache-2.0", LicensePermissive}, - {"BSD-3-Clause", LicensePermissive}, - {"GPL-3.0", LicenseCopyleft}, - {"AGPL-3.0", LicenseCopyleft}, - {"LGPL-2.1", LicenseCopyleft}, - {"", LicenseUnknown}, - {"Unknown", LicenseUnknown}, - } - for _, tc := range tests { - result := svc.CategorizeLicense(tc.license) - if result != tc.expected { - t.Errorf("CategorizeLicense(%q) = %v, want %v", tc.license, result, tc.expected) + info := &VersionInfo{Status: string(tc.status), Yanked: tc.yanked, License: tc.license} + adaptVersion(info) + if info.Yanked != tc.wantYanked || info.License != tc.wantLicense { + t.Errorf("adaptVersion(%s, %q) = yanked %v, license %q; want %v, %q", tc.status, tc.license, info.Yanked, info.License, tc.wantYanked, tc.wantLicense) } } } diff --git a/internal/server/api.go b/internal/server/api.go index 992d7366..b2043672 100644 --- a/internal/server/api.go +++ b/internal/server/api.go @@ -174,7 +174,7 @@ func (h *APIHandler) HandlePackagePath(w http.ResponseWriter, r *http.Request) { Name: info.Name, LatestVersion: info.LatestVersion, License: info.License, - LicenseCategory: string(h.enrichment.CategorizeLicense(info.License)), + LicenseCategory: string(shared.CategorizeLicense(info.License)), Description: info.Description, Homepage: info.Homepage, Repository: info.Repository, @@ -207,7 +207,7 @@ func (h *APIHandler) getPackage(w http.ResponseWriter, r *http.Request, ecosyste Name: info.Name, LatestVersion: info.LatestVersion, License: info.License, - LicenseCategory: string(h.enrichment.CategorizeLicense(info.License)), + LicenseCategory: string(shared.CategorizeLicense(info.License)), Description: info.Description, Homepage: info.Homepage, Repository: info.Repository, @@ -235,7 +235,7 @@ func (h *APIHandler) getVersion(w http.ResponseWriter, r *http.Request, ecosyste Name: result.Package.Name, LatestVersion: result.Package.LatestVersion, License: result.Package.License, - LicenseCategory: string(h.enrichment.CategorizeLicense(result.Package.License)), + LicenseCategory: string(shared.CategorizeLicense(result.Package.License)), Description: result.Package.Description, Homepage: result.Package.Homepage, Repository: result.Package.Repository, @@ -366,7 +366,7 @@ func (h *APIHandler) HandleOutdated(w http.ResponseWriter, r *http.Request) { latest, err := h.enrichment.GetLatestVersion(r.Context(), pkg.Ecosystem, pkg.Name) if err == nil && latest != "" { result.LatestVersion = latest - result.IsOutdated = h.enrichment.IsOutdated(pkg.Version, latest) + result.IsOutdated = shared.IsOutdated(pkg.Version, latest) } resp.Results = append(resp.Results, result) @@ -413,7 +413,7 @@ func (h *APIHandler) HandleBulkLookup(w http.ResponseWriter, r *http.Request) { Name: info.Name, LatestVersion: info.LatestVersion, License: info.License, - LicenseCategory: string(h.enrichment.CategorizeLicense(info.License)), + LicenseCategory: string(shared.CategorizeLicense(info.License)), Description: info.Description, Homepage: info.Homepage, Repository: info.Repository, @@ -444,7 +444,7 @@ func (h *APIHandler) HandleBulkLookup(w http.ResponseWriter, r *http.Request) { Name: info.Name, LatestVersion: info.LatestVersion, License: info.License, - LicenseCategory: string(h.enrichment.CategorizeLicense(info.License)), + LicenseCategory: string(shared.CategorizeLicense(info.License)), Description: info.Description, Homepage: info.Homepage, Repository: info.Repository, @@ -635,9 +635,7 @@ func (h *APIHandler) HandlePackagesList(w http.ResponseWriter, r *http.Request) licenseCategory := licenseCategoryUnknown if pkg.License.Valid { license = pkg.License.String - if h.enrichment != nil { - licenseCategory = string(h.enrichment.CategorizeLicense(license)) - } + licenseCategory = string(shared.CategorizeLicense(license)) } cachedAt := "" if pkg.CachedAt.Valid { diff --git a/internal/server/server.go b/internal/server/server.go index b332f849..bd2f287c 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -70,6 +70,7 @@ import ( "github.com/git-pkgs/archives" "github.com/git-pkgs/cooldown" + shared "github.com/git-pkgs/enrichment" swaggerdoc "github.com/git-pkgs/proxy/docs/swagger" "github.com/git-pkgs/proxy/internal/accesslog" "github.com/git-pkgs/proxy/internal/config" @@ -87,7 +88,6 @@ import ( registryclient "github.com/git-pkgs/registries/client" "github.com/git-pkgs/registries/fetch" "github.com/git-pkgs/registries/safehttp" - "github.com/git-pkgs/spdx" "github.com/go-chi/chi/v5" "github.com/go-chi/chi/v5/middleware" ) @@ -1470,21 +1470,9 @@ func formatTimeAgo(t time.Time) string { } } -// categorizeLicenseCSS returns the CSS class suffix for a license category using the spdx module. +// categorizeLicenseCSS returns the CSS class suffix for a license category. func categorizeLicenseCSS(license string) string { - if license == "" { - return licenseCategoryUnknown - } - - if spdx.HasCopyleft(license) { - return "copyleft" - } - - if spdx.IsFullyPermissive(license) { - return "permissive" - } - - return licenseCategoryUnknown + return string(shared.CategorizeLicense(license)) } // categorizeLicense is a helper that handles sql.NullString. diff --git a/internal/server/server_test.go b/internal/server/server_test.go index afd0ca90..292643c1 100644 --- a/internal/server/server_test.go +++ b/internal/server/server_test.go @@ -806,6 +806,9 @@ func TestCategorizeLicenseCSS(t *testing.T) { {"MPL-2.0", "copyleft"}, {"", "unknown"}, {"Proprietary", "unknown"}, + {"mit", "permissive"}, + {"Apache 2.0", "permissive"}, + {"GPLv3", "copyleft"}, } for _, tc := range tests {