Detect Git LFS pointer files

README.md

@@ -72,7 +72,7 @@ The format registry contains:
 
 - ZIP, TAR, native PHAR, ar, gzip, bzip2, xz, zstd, PDF, CFBF, PNG, JPEG, and GIF
 - ELF, Mach-O (thin and universal), PE/COFF, and WebAssembly
-- plain text, JSON, HTML, XML, and SVG
+- plain text, Git LFS pointers, JSON, HTML, XML, and SVG
 
 Detection uses bytes only. ZIP-based package types such as JAR, wheel, and
 NuGet remain `zip`, and compressed payloads are not opened. A `CA FE BA BE`
@@ -100,6 +100,12 @@ HTML, XML, and SVG signatures supply format metadata before the shared text
 rules run. The metadata remains present if malformed or control-bearing input
 is classified as unknown or binary.
 
+Git LFS pointers report `FormatLFSPointer` with kind `text`, MIME `text/plain`,
+and UTF-8 encoding. Detection checks the required fields, key order, and the
+1024-byte size limit, including unknown extension keys. Valid partial pointers
+report the same format with `ReasonNeedMore`; malformed pointers fall back to
+ordinary text classification.
+
 ## Performance
 
 The detector performs no allocations for the supplied fixtures. On an Apple

fuzz_test.go

@@ -12,6 +12,7 @@ func FuzzDetect(f *testing.F) {
 	seeds := [][]byte{
 		nil,
 		[]byte("hello\n"),
+		[]byte(lfsTestPointer),
 		[]byte("\xef\xbb\xbfhello"),
 		[]byte("\xff\xfeh\x00i\x00"),
 		[]byte("\xfe\xff\x00h\x00i"),
@@ -69,6 +70,9 @@ func FuzzDetect(f *testing.F) {
 				expectedPrefix.Format = FormatJSON
 				expectedPrefix.MIME = mimeJSON
 			}
+			if first.Format == FormatText && isLFSPointer(data, true) {
+				expectedPrefix.Format = FormatLFSPointer
+			}
 			if prefix.Reason == ReasonNeedMore {
 				expectedPrefix.Reason = ReasonNeedMore
 			}
@@ -117,6 +121,8 @@ func FuzzDetectPrefix(f *testing.F) {
 	seeds := [][]byte{
 		nil,
 		[]byte("hello"),
+		[]byte(lfsTestPointer),
+		[]byte(lfsTestVersion + "oid sha256:4d7a"),
 		[]byte("\xef\xbb"),
 		[]byte("\xff\xfeh"),
 		[]byte("PK\x03"),

lfs.go

@@ -0,0 +1,139 @@
+package magic
+
+import "bytes"
+
+const (
+	lfsPointerLimit = 1024
+	lfsHashLength   = 64
+	lfsSizeMax      = "9223372036854775807"
+)
+
+type lfsPointer struct {
+	previous []byte
+	oid      bool
+	size     bool
+}
+
+func isLFSPointer(data []byte, prefix bool) bool {
+	if len(data) == 0 || len(data) >= lfsPointerLimit || bytes.IndexByte(data, '\r') >= 0 {
+		return false
+	}
+	version, remaining, complete := bytes.Cut(data, []byte{'\n'})
+	if !validLFSVersion(version, prefix && !complete) {
+		return false
+	}
+	if !complete {
+		return prefix
+	}
+
+	pointer := lfsPointer{}
+	for len(remaining) > 0 {
+		line, rest, terminated := bytes.Cut(remaining, []byte{'\n'})
+		if !terminated && !prefix {
+			return false
+		}
+		if !pointer.line(line, !terminated) {
+			return false
+		}
+		remaining = rest
+	}
+	return prefix || pointer.oid && pointer.size
+}
+
+func validLFSVersion(line []byte, partial bool) bool {
+	for _, version := range [...]string{
+		"version https://git-lfs.github.com/spec/v1",
+		"version https://hawser.github.com/spec/v1",
+		"version http://git-media.io/v/2",
+	} {
+		if bytes.Equal(line, []byte(version)) || partial && bytes.HasPrefix([]byte(version), line) {
+			return true
+		}
+	}
+	return false
+}
+
+func (pointer *lfsPointer) line(line []byte, partial bool) bool {
+	key, value, separated := bytes.Cut(line, []byte{' '})
+	if !pointer.validKey(key, partial && !separated) {
+		return false
+	}
+	if !separated {
+		return partial
+	}
+	if len(value) > 0 && value[0] == ' ' {
+		return false
+	}
+	switch string(key) {
+	case "oid":
+		if !validLFSOID(value, partial) {
+			return false
+		}
+		pointer.oid = true
+	case "size":
+		if !validLFSSize(value, partial) {
+			return false
+		}
+		pointer.size = true
+	}
+	pointer.previous = key
+	return true
+}
+
+func (pointer *lfsPointer) validKey(key []byte, partial bool) bool {
+	if len(key) == 0 || bytes.Equal(key, []byte("version")) && !partial {
+		return false
+	}
+	for _, value := range key {
+		switch {
+		case value >= 'a' && value <= 'z', isDigit(value), value == '.', value == '-':
+		default:
+			return false
+		}
+	}
+	if bytes.Compare(key, pointer.previous) <= 0 && (!partial || !bytes.HasPrefix(pointer.previous, key)) {
+		return false
+	}
+	// Once a key sorts past a missing required field, later lines cannot supply it.
+	if !pointer.oid && bytes.Compare(key, []byte("oid")) > 0 {
+		return false
+	}
+	return pointer.size || bytes.Compare(key, []byte("size")) <= 0
+}
+
+func validLFSOID(value []byte, partial bool) bool {
+	const method = "sha256:"
+	if len(value) < len(method) {
+		return partial && bytes.HasPrefix([]byte(method), value)
+	}
+	if !bytes.HasPrefix(value, []byte(method)) {
+		return false
+	}
+	hash := value[len(method):]
+	if len(hash) > lfsHashLength || !partial && len(hash) != lfsHashLength {
+		return false
+	}
+	for _, digit := range hash {
+		if !isDigit(digit) && (digit < 'a' || digit > 'f') {
+			return false
+		}
+	}
+	return true
+}
+
+func validLFSSize(value []byte, partial bool) bool {
+	if len(value) == 0 {
+		return partial
+	}
+	if len(value) > 1 && value[0] == '0' {
+		return false
+	}
+	for _, digit := range value {
+		if !isDigit(digit) {
+			return false
+		}
+	}
+	// Leading zeros are rejected above, so equal lengths compare numerically.
+	return len(value) < len(lfsSizeMax) ||
+		len(value) == len(lfsSizeMax) && bytes.Compare(value, []byte(lfsSizeMax)) <= 0
+}

lfs_test.go

@@ -0,0 +1,192 @@
+package magic
+
+import (
+	"bytes"
+	"fmt"
+	"strings"
+	"testing"
+)
+
+const lfsTestVersion = "version https://git-lfs.github.com/spec/v1\n"
+const lfsTestOID = "oid sha256:4d7a214614ab2935c943f9e0ff69d22eadbb8f32b1258daaa5e2ca24d17e2393\n"
+const lfsTestPointer = lfsTestVersion + lfsTestOID + "size 12345\n"
+
+func TestLFSPointerDetection(t *testing.T) {
+	t.Parallel()
+
+	inputs := []string{
+		lfsTestPointer,
+		lfsTestVersion + lfsTestOID + "size 0\n",
+		strings.Replace(lfsTestPointer, "git-lfs", "hawser", 1),
+		strings.Replace(lfsTestPointer, "https://git-lfs.github.com/spec/v1", "http://git-media.io/v/2", 1),
+		lfsTestVersion + lfsTestOID + "size " + lfsSizeMax + "\n",
+		lfsTestVersion + "ext-0-custom sha256:abc\n" + lfsTestOID + "size 12345\n",
+		lfsTestVersion + "a.b-2 世界 with spaces\n" + lfsTestOID + "other value\nsize 1\nz value\n",
+		lfsTestPointer + "z \n",
+		lfsTestPointer + "z " + strings.Repeat("x", 1023-len(lfsTestPointer)-3) + "\n",
+	}
+	for index, input := range inputs {
+		t.Run(fmt.Sprint(index), func(t *testing.T) {
+			t.Parallel()
+			data := []byte(input)
+			want := Result{Kind: KindText, Format: FormatLFSPointer, MIME: mimeText, Encoding: EncodingUTF8}
+			assertResult(t, Detect(data), want)
+			want.Reason = ReasonNeedMore
+			assertResult(t, DetectPrefix(data), want)
+		})
+	}
+}
+
+func TestInvalidLFSPointers(t *testing.T) {
+	t.Parallel()
+
+	tests := map[string]string{
+		"embedded URL":         "hello\n" + lfsTestPointer,
+		"leading whitespace":   " " + lfsTestPointer,
+		"BOM":                  "\xef\xbb\xbf" + lfsTestPointer,
+		"CRLF":                 strings.ReplaceAll(lfsTestPointer, "\n", "\r\n"),
+		"uppercase version":    strings.Replace(lfsTestPointer, "https", "HTTPS", 1),
+		"wrong version":        strings.Replace(lfsTestPointer, "/v1", "/v2", 1),
+		"escaped version":      strings.Replace(lfsTestPointer, "spec", "%73pec", 1),
+		"uppercase hash":       strings.Replace(lfsTestPointer, "4d7a", "4D7A", 1),
+		"invalid hash":         strings.Replace(lfsTestPointer, "4d7a", "4g7a", 1),
+		"short hash":           strings.Replace(lfsTestPointer, "4d7a", "4d7", 1),
+		"long hash":            strings.Replace(lfsTestPointer, "4d7a", "4d7aa", 1),
+		"wrong hash method":    strings.Replace(lfsTestPointer, "sha256", "sha512", 1),
+		"negative size":        strings.Replace(lfsTestPointer, "12345", "-1", 1),
+		"signed size":          strings.Replace(lfsTestPointer, "12345", "+1", 1),
+		"fractional size":      strings.Replace(lfsTestPointer, "12345", "1.2", 1),
+		"noninteger size":      strings.Replace(lfsTestPointer, "12345", "one", 1),
+		"empty size":           strings.Replace(lfsTestPointer, "12345", "", 1),
+		"size leading zero":    strings.Replace(lfsTestPointer, "12345", "01", 1),
+		"size past int64":      strings.Replace(lfsTestPointer, "12345", "9223372036854775808", 1),
+		"size extra digit":     strings.Replace(lfsTestPointer, "12345", "92233720368547758070", 1),
+		"size trailing space":  strings.Replace(lfsTestPointer, "12345", "1 ", 1),
+		"out of order":         lfsTestVersion + "size 1\n" + lfsTestOID,
+		"missing oid":          lfsTestVersion + "size 1\n",
+		"skipped size":         lfsTestVersion + lfsTestOID + "z value\n",
+		"duplicate oid":        lfsTestVersion + lfsTestOID + lfsTestOID + "size 1\n",
+		"duplicate version":    lfsTestPointer + lfsTestVersion,
+		"duplicate extension":  lfsTestVersion + "a one\na two\n" + lfsTestOID + "size 1\n",
+		"unordered extensions": lfsTestVersion + "b one\na two\n" + lfsTestOID + "size 1\n",
+		"invalid key":          lfsTestVersion + "ext_custom value\n" + lfsTestOID + "size 1\n",
+		"uppercase key":        lfsTestVersion + "Ext value\n" + lfsTestOID + "size 1\n",
+		"nonascii key":         lfsTestPointer + "é value\n",
+		"empty key":            lfsTestVersion + " value\n" + lfsTestOID + "size 1\n",
+		"missing separator":    strings.Replace(lfsTestPointer, "size ", "size", 1),
+		"tab separator":        strings.Replace(lfsTestPointer, "size ", "size\t", 1),
+		"double separator":     strings.Replace(lfsTestPointer, "oid ", "oid  ", 1),
+		"extension separator":  lfsTestPointer + "z  value\n",
+		"return in extension":  lfsTestPointer + "z val\rue\n",
+		"extra newline":        lfsTestPointer + "\n",
+		"1024 bytes":           lfsTestPointer + "z " + strings.Repeat("x", 1024-len(lfsTestPointer)-3) + "\n",
+		"1025 bytes":           lfsTestPointer + "z " + strings.Repeat("x", 1025-len(lfsTestPointer)-3) + "\n",
+	}
+	for name, input := range tests {
+		t.Run(name, func(t *testing.T) {
+			t.Parallel()
+			want := Result{Kind: KindText, Format: FormatText, MIME: mimeText, Encoding: EncodingUTF8}
+			assertResult(t, Detect([]byte(input)), want)
+			want.Reason = ReasonNeedMore
+			assertResult(t, DetectPrefix([]byte(input)), want)
+		})
+	}
+}
+
+func TestLFSPointerPrefixes(t *testing.T) {
+	t.Parallel()
+
+	inputs := []string{
+		lfsTestPointer,
+		strings.Replace(lfsTestPointer, "git-lfs", "hawser", 1),
+		strings.Replace(lfsTestPointer, "https://git-lfs.github.com/spec/v1", "http://git-media.io/v/2", 1),
+		lfsTestVersion + "a.b-2 世界\n" + lfsTestOID + "oid-extra value\nsize 0\nsize-extra value\n",
+	}
+	for _, input := range inputs {
+		data := []byte(input)
+		for length := 1; length <= len(data); length++ {
+			got := DetectPrefix(data[:length])
+			want := Result{Kind: KindText, Format: FormatLFSPointer, MIME: mimeText, Encoding: EncodingUTF8, Reason: ReasonNeedMore}
+			if got != want {
+				t.Fatalf("DetectPrefix(%q) = %#v, want %#v", data[:length], got, want)
+			}
+		}
+	}
+
+	for _, input := range []string{lfsTestVersion, lfsTestVersion + lfsTestOID, strings.TrimSuffix(lfsTestPointer, "\n")} {
+		assertResult(t, Detect([]byte(input)), Result{Kind: KindText, Format: FormatText, MIME: mimeText, Encoding: EncodingUTF8})
+	}
+}
+
+func TestInvalidLFSPointerPrefixes(t *testing.T) {
+	t.Parallel()
+
+	inputs := []string{
+		"version https://git-lfs.github.com/spec/v2",
+		lfsTestVersion + "o!",
+		lfsTestVersion + "p",
+		lfsTestVersion + "oid sha257",
+		lfsTestVersion + "oid sha256:A",
+		lfsTestVersion + "oid sha256:" + strings.Repeat("a", 65),
+		lfsTestVersion + lfsTestOID + "oid ",
+		lfsTestVersion + lfsTestOID + "sz",
+		lfsTestVersion + lfsTestOID + "size -",
+		lfsTestVersion + lfsTestOID + "size 01",
+		lfsTestVersion + lfsTestOID + "size 1.",
+		lfsTestVersion + lfsTestOID + "size 9223372036854775808",
+		lfsTestPointer + "a",
+		lfsTestPointer + "z value\r",
+		lfsTestPointer + "z " + strings.Repeat("x", 1024-len(lfsTestPointer)-2),
+	}
+	for _, input := range inputs {
+		want := Result{Kind: KindText, Format: FormatText, MIME: mimeText, Encoding: EncodingUTF8, Reason: ReasonNeedMore}
+		if got := DetectPrefix([]byte(input)); got != want {
+			t.Fatalf("DetectPrefix(%q) = %#v, want %#v", input, got, want)
+		}
+	}
+}
+
+func TestLFSPointerTextValidation(t *testing.T) {
+	t.Parallel()
+
+	utf16 := []byte{0xff, 0xfe}
+	for _, value := range []byte(lfsTestPointer) {
+		utf16 = append(utf16, value, 0)
+	}
+	assertResult(t, Detect(utf16), Result{Kind: KindText, Format: FormatText, MIME: mimeText, Encoding: EncodingUTF16LE})
+	for _, suffix := range [][]byte{{0}, {0xff}, {'z', ' ', 1, '\n'}} {
+		data := append([]byte(lfsTestPointer), suffix...)
+		for _, result := range []Result{Detect(data), DetectPrefix(data)} {
+			if result.Format == FormatLFSPointer || result.Kind == KindText {
+				t.Fatalf("invalid text %x classified as %#v", data, result)
+			}
+		}
+	}
+	data := []byte(lfsTestPointer + "z \x01\n")
+	assertResult(t, DetectWithOptions(data, Options{TextControls: "\x01"}), Result{
+		Kind: KindText, Format: FormatLFSPointer, MIME: mimeText, Encoding: EncodingUTF8,
+	})
+}
+
+func TestLFSPointerAllocations(t *testing.T) {
+	inputs := [][]byte{
+		[]byte(lfsTestPointer),
+		[]byte(lfsTestVersion + "ext-0-custom value\n" + lfsTestOID + "size 1\n"),
+		[]byte(lfsTestVersion + lfsTestOID + "size " + lfsSizeMax + "\n"),
+		bytes.TrimSuffix([]byte(lfsTestPointer), []byte{'\n'}),
+	}
+	for _, input := range inputs {
+		if allocations := testing.AllocsPerRun(1000, func() {
+			Detect(input)
+			DetectPrefix(input)
+		}); allocations != 0 {
+			t.Fatalf("pointer detection allocated %.2f times", allocations)
+		}
+	}
+}
+
+func ExampleDetect_lfsPointer() {
+	result := Detect([]byte(lfsTestPointer))
+	fmt.Println(result.Kind, result.Format, result.MIME, result.Encoding)
+	// Output: text git-lfs-pointer text/plain utf-8
+}