Cache Git LFS objects

README.md

@@ -925,7 +925,8 @@ git config --global url."http://localhost:8080/git/github/".insteadOf https://gi
 ```
 
 Each repository is cached as a bare clone and refreshed whenever its branches
-or tags change upstream. See [configuration](docs/configuration.md#git-repositories)
+or tags change upstream. Git LFS objects are cached in the artifact cache;
+git-lfs uses the proxy for them with no further setup. See [configuration](docs/configuration.md#git-repositories)
 for setting the rewrite through environment variables, size limits, private
 repositories and outage behavior.
 

docs/architecture.md

@@ -308,6 +308,7 @@ HTTP protocol handlers for each registry type.
 **GitHandler:**
 - Serves `/git/{name}/{repo}` from `internal/gitcache`, read-only over smart HTTP
 - Validates repository paths and the `repos` allowlist, and maps cache errors to statuses
+- Serves the Git LFS API under `{repo}.git/info/lfs/`: forwards download batches upstream and points each object at `objects/{oid}/{size}`, which is served from the artifact cache (ecosystem `git`, name `{name}/{repo}`, version the object ID); a miss uses the download URL kept from the batch or asks the upstream batch API for a fresh one, recorded without its signed query, copies the object from another repository when its content is already stored, and checks the object against its sha256 ID; a batch the upstream can't answer is answered from the cache
 
 ### `internal/gitcache`
 

docs/configuration.md

@@ -837,7 +837,42 @@ those yourself. The proxy locks `cache_dir` while it runs, through a
 `.gitcache-lock` file in it, so a second process given the same directory,
 such as another replica on a shared volume, fails to start; give each its own.
 
-Pushing, the dumb HTTP protocol and Git LFS are not supported. The settings
+Git LFS objects are cached in the artifact cache, so they use the configured
+storage backend and retention rather than `cache_dir`. git-lfs finds the LFS
+API under the rewritten remote, at `/git/{name}/{repo}.git/info/lfs`, with no
+further configuration. A batch request is passed to the upstream, which
+decides which objects the repository has, and each download is pointed at
+the proxy. An object the cache doesn't have is fetched from the download URL
+the batch returned, which the proxy keeps in memory until shortly before it
+expires, or else from one a fresh batch request returns. That URL is often
+on another host than the Git server, such as GitHub's or Hugging Face's
+object storage; the proxy fetches from whatever host the upstream names,
+with the usual address checks, and `upstream.auth` credentials only go to
+the hosts they are configured for. A download that isn't the object's size
+is cut off. If the object's content is already stored under another
+repository, such as a fork, it is copied from there instead of downloaded,
+once the upstream has confirmed this repository has it; the copy takes its
+own space.
+
+Large LFS objects may need longer timeouts. The proxy downloads an object it
+doesn't have before sending any of it, within `artifact_timeout`, and the
+whole response, download included, must fit within `write_timeout`; both
+default to 5 minutes. git-lfs gives up on a response after 30 seconds without
+data (`lfs.activitytimeout`) and retries. A retry waits for the download still
+running, so the object is served once it is cached, but a download that
+outlasts git-lfs's retries fails that clone or pull until it finishes.
+Download URLs are often signed; only the URL without its query is logged
+and recorded with the object. An object is only cached once its content
+matches its sha256 object ID. Objects are cached per repository, so one
+repository's objects are never served for another. When the upstream can't
+be reached or fails, a batch is answered with the objects cached for the
+repository, and the others are reported as unavailable; a batch the upstream
+refuses is refused. A cached object stays available after it is removed
+upstream, until retention removes it or you delete it through the
+[cache delete API](#cache-delete-api).
+
+Pushing, LFS uploads and locking, and the dumb HTTP protocol are not
+supported. The settings
 under `git` can also be set with `PROXY_GIT_CACHE_DIR`, `PROXY_GIT_MAX_SIZE`,
 `PROXY_GIT_MAX_REPO_SIZE`, `PROXY_GIT_REFRESH_WAIT` and
 `PROXY_GIT_STALE_MAX_AGE`.

internal/database/queries.go

@@ -265,6 +265,34 @@ func (db *DB) GetCachedArtifact(packagePURL, versionPURL, filename string) (*Cac
 	return row.artifact(versionPURL, filename), nil
 }
 
+// StoredContent is a stored object with a given content hash.
+type StoredContent struct {
+	StoragePath string         `db:"storage_path"`
+	Size        int64          `db:"size"`
+	ContentType sql.NullString `db:"content_type"`
+}
+
+// FindStoredContent returns a cached artifact whose content has the sha256
+// hash, under any name, or nil if none is stored. Pushed artifacts are left
+// out.
+func (db *DB) FindStoredContent(hash string) (*StoredContent, error) {
+	var found StoredContent
+	query := db.Rebind(`
+		SELECT storage_path, size, content_type FROM artifacts
+		WHERE content_hash = ? AND storage_path IS NOT NULL AND fetched_at IS NOT NULL
+		  AND size IS NOT NULL AND source <> 'local'
+		LIMIT 1
+	`)
+	err := db.Get(&found, query, hash)
+	if err == sql.ErrNoRows {
+		return nil, nil
+	}
+	if err != nil {
+		return nil, err
+	}
+	return &found, nil
+}
+
 type cachedArtifactRow struct {
 	Ecosystem   string         `db:"ecosystem"`
 	StoragePath string         `db:"storage_path"`

internal/database/schema.go

@@ -79,6 +79,7 @@ CREATE TABLE IF NOT EXISTS artifacts (
 );
 CREATE UNIQUE INDEX IF NOT EXISTS idx_artifacts_version_filename ON artifacts(version_purl, filename);
 CREATE INDEX IF NOT EXISTS idx_artifacts_storage_path ON artifacts(storage_path);
+CREATE INDEX IF NOT EXISTS idx_artifacts_content_hash ON artifacts(content_hash);
 CREATE INDEX IF NOT EXISTS idx_artifacts_last_accessed ON artifacts(last_accessed_at);
 
 CREATE TABLE IF NOT EXISTS pending_deletes (
@@ -220,6 +221,7 @@ CREATE TABLE IF NOT EXISTS artifacts (
 );
 CREATE UNIQUE INDEX IF NOT EXISTS idx_artifacts_version_filename ON artifacts(version_purl, filename);
 CREATE INDEX IF NOT EXISTS idx_artifacts_storage_path ON artifacts(storage_path);
+CREATE INDEX IF NOT EXISTS idx_artifacts_content_hash ON artifacts(content_hash);
 CREATE INDEX IF NOT EXISTS idx_artifacts_last_accessed ON artifacts(last_accessed_at);
 
 CREATE TABLE IF NOT EXISTS pending_deletes (
@@ -320,6 +322,7 @@ CREATE TABLE IF NOT EXISTS artifacts (
 );
 CREATE UNIQUE INDEX IF NOT EXISTS idx_artifacts_version_filename ON artifacts(version_purl, filename);
 CREATE INDEX IF NOT EXISTS idx_artifacts_storage_path ON artifacts(storage_path);
+CREATE INDEX IF NOT EXISTS idx_artifacts_content_hash ON artifacts(content_hash);
 CREATE INDEX IF NOT EXISTS idx_artifacts_last_accessed ON artifacts(last_accessed_at);
 `
 
@@ -343,6 +346,7 @@ CREATE TABLE IF NOT EXISTS artifacts (
 );
 CREATE UNIQUE INDEX IF NOT EXISTS idx_artifacts_version_filename ON artifacts(version_purl, filename);
 CREATE INDEX IF NOT EXISTS idx_artifacts_storage_path ON artifacts(storage_path);
+CREATE INDEX IF NOT EXISTS idx_artifacts_content_hash ON artifacts(content_hash);
 CREATE INDEX IF NOT EXISTS idx_artifacts_last_accessed ON artifacts(last_accessed_at);
 `
 
@@ -457,6 +461,7 @@ var migrations = []migration{
 	{"014_add_metadata_last_accessed", migrateAddMetadataLastAccessed},
 	{"015_add_job_runs", migrateAddJobRuns},
 	{"016_add_fetch_leases", migrateAddFetchLeases},
+	{"017_add_artifact_content_hash_index", migrateAddArtifactContentHashIndex},
 	{"018_add_upstream_holds", migrateAddUpstreamHolds},
 }
 
@@ -820,6 +825,16 @@ func migrateAddPendingDeletesQueuedAtIndex(db *DB) error {
 	return nil
 }
 
+// migrateAddArtifactContentHashIndex indexes artifacts by content, so an
+// artifact already stored under another name can be found by its hash.
+func migrateAddArtifactContentHashIndex(db *DB) error {
+	_, err := db.Exec(`CREATE INDEX IF NOT EXISTS idx_artifacts_content_hash ON artifacts(content_hash)`)
+	if err != nil {
+		return fmt.Errorf("creating artifacts content_hash index: %w", err)
+	}
+	return nil
+}
+
 // migrateAddMetadataDates creates the table of Last-Modified dates for
 // rewritten metadata, which replicas sharing the database agree on.
 func migrateAddMetadataDates(db *DB) error {

internal/handler/git.go

@@ -39,6 +39,11 @@ type GitHandler struct {
 	proxy     *Proxy
 	cache     *gitcache.Cache
 	upstreams map[string]gitUpstream
+	// baseURL is where clients reach the proxy, for LFS download URLs.
+	baseURL string
+	// lfsDownloads keeps upstream LFS download URLs between a batch and
+	// the GETs that follow it.
+	lfsDownloads *lfsDownloads
 	// writeStall is gitWriteStallTimeout, shorter in tests.
 	writeStall time.Duration
 }
@@ -56,10 +61,11 @@ type gitUpstream struct {
 // NewGitHandler creates a Git handler serving upstream.git from cache.
 func NewGitHandler(proxy *Proxy, cache *gitcache.Cache, upstream config.UpstreamConfig) *GitHandler {
 	h := &GitHandler{
-		proxy:      proxy,
-		cache:      cache,
-		upstreams:  make(map[string]gitUpstream, len(upstream.Git)),
-		writeStall: gitWriteStallTimeout,
+		proxy:        proxy,
+		cache:        cache,
+		upstreams:    make(map[string]gitUpstream, len(upstream.Git)),
+		writeStall:   gitWriteStallTimeout,
+		lfsDownloads: &lfsDownloads{},
 	}
 	for name, git := range upstream.Git {
 		git.URL = strings.TrimSuffix(git.URL, "/")
@@ -90,6 +96,10 @@ type gitRequest struct {
 }
 
 func (h *GitHandler) serve(w http.ResponseWriter, r *http.Request) {
+	if repoPath, endpoint, ok := strings.Cut(strings.TrimPrefix(r.URL.Path, "/"), gitLFSPath); ok {
+		h.serveLFS(w, r, repoPath, endpoint)
+		return
+	}
 	req, status, msg := h.parse(r)
 	if status != 0 {
 		http.Error(w, msg, status)
@@ -168,10 +178,38 @@ func (h *GitHandler) parse(r *http.Request) (gitRequest, int, string) {
 		return req, http.StatusNotFound, "not found"
 	}
 
+	repo, status, msg := h.repository(path)
+	if status != 0 {
+		return req, status, msg
+	}
+	req.key = repo.key
+	req.repoURL = repo.url
+	req.fetchWants = repo.upstream.fetchWants
+	return req, 0, ""
+}
+
+// gitRepository is a repository of an upstream.git host, as a request path
+// names it.
+type gitRepository struct {
+	// name is the upstream's name and repo the path under it, without .git.
+	name, repo string
+	upstream   gitUpstream
+	// key names the repository in the cache: {name}-{URL hash}/{repo}.git,
+	// with any .git the client gave removed from repo first, so both
+	// spellings share one copy.
+	key string
+	// url is the upstream repository, spelled as the client spelled it, and
+	// escapedRepo is repo escaped for a URL path.
+	url, escapedRepo string
+}
+
+// repository resolves {name}/{repo} to an upstream repository, or returns
+// the status and message to refuse it with.
+func (h *GitHandler) repository(path string) (gitRepository, int, string) {
 	name, repoPath, _ := strings.Cut(path, "/")
 	upstream, ok := h.upstreams[name]
 	if !ok {
-		return req, http.StatusNotFound, "unknown upstream"
+		return gitRepository{}, http.StatusNotFound, "unknown upstream"
 	}
 	segments := strings.Split(repoPath, "/")
 	last := len(segments) - 1
@@ -182,20 +220,25 @@ func (h *GitHandler) parse(r *http.Request) (gitRequest, int, string) {
 	escaped := make([]string, len(segments))
 	for i, segment := range segments {
 		if !validGitSegment(segment, i == last) {
-			return req, http.StatusNotFound, "invalid repository path"
+			return gitRepository{}, http.StatusNotFound, "invalid repository path"
 		}
 		escaped[i] = url.PathEscape(segment)
 	}
 	repo := strings.Join(segments, "/")
 	if !upstream.AllowsRepo(repo) {
 		// As for a missing repository, so a prober learns nothing about
 		// the allowlist.
-		return req, http.StatusNotFound, "repository not found"
+		return gitRepository{}, http.StatusNotFound, "repository not found"
 	}
-	req.key = upstream.dir + "/" + repo + ".git"
-	req.repoURL = upstream.URL + "/" + strings.Join(escaped, "/") + suffix
-	req.fetchWants = upstream.fetchWants
-	return req, 0, ""
+	escapedRepo := strings.Join(escaped, "/")
+	return gitRepository{
+		name:        name,
+		repo:        repo,
+		upstream:    upstream,
+		key:         upstream.dir + "/" + repo + ".git",
+		url:         upstream.URL + "/" + escapedRepo + suffix,
+		escapedRepo: escapedRepo,
+	}, 0, ""
 }
 
 // validGitSegment checks one segment of a repository path, which becomes both