From 33925297bc10730fa5dd50e4ef697f8e7f501384 Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Thu, 1 Oct 2026 14:51:00 +0100 Subject: [PATCH 1/2] Detect Git LFS pointer files --- README.md | 8 ++- fuzz_test.go | 6 ++ lfs.go | 135 +++++++++++++++++++++++++++++++++++++ lfs_test.go | 185 +++++++++++++++++++++++++++++++++++++++++++++++++++ magic.go | 48 +++++++------ 5 files changed, 359 insertions(+), 23 deletions(-) create mode 100644 lfs.go create mode 100644 lfs_test.go diff --git a/README.md b/README.md index 8441a7b..c60e689 100644 --- a/README.md +++ b/README.md @@ -72,7 +72,7 @@ The format registry contains: - ZIP, TAR, native PHAR, ar, gzip, bzip2, xz, zstd, PDF, CFBF, PNG, JPEG, and GIF - ELF, Mach-O (thin and universal), PE/COFF, and WebAssembly -- plain text, JSON, HTML, XML, and SVG +- plain text, Git LFS pointers, JSON, HTML, XML, and SVG Detection uses bytes only. ZIP-based package types such as JAR, wheel, and NuGet remain `zip`, and compressed payloads are not opened. A `CA FE BA BE` @@ -100,6 +100,12 @@ HTML, XML, and SVG signatures supply format metadata before the shared text rules run. The metadata remains present if malformed or control-bearing input is classified as unknown or binary. +Git LFS pointers report `FormatLFSPointer` with kind `text`, MIME `text/plain`, +and UTF-8 encoding. Detection checks the required fields, key order, and the +1024-byte size limit, including unknown extension keys. Valid partial pointers +report the same format with `ReasonNeedMore`; malformed pointers fall back to +ordinary text classification. + ## Performance The detector performs no allocations for the supplied fixtures. On an Apple diff --git a/fuzz_test.go b/fuzz_test.go index 2548ce0..ff942d5 100644 --- a/fuzz_test.go +++ b/fuzz_test.go @@ -12,6 +12,7 @@ func FuzzDetect(f *testing.F) { seeds := [][]byte{ nil, []byte("hello\n"), + []byte(lfsTestPointer), []byte("\xef\xbb\xbfhello"), []byte("\xff\xfeh\x00i\x00"), []byte("\xfe\xff\x00h\x00i"), @@ -69,6 +70,9 @@ func FuzzDetect(f *testing.F) { expectedPrefix.Format = FormatJSON expectedPrefix.MIME = mimeJSON } + if prefix.Format == FormatLFSPointer { + expectedPrefix.Format = FormatLFSPointer + } if prefix.Reason == ReasonNeedMore { expectedPrefix.Reason = ReasonNeedMore } @@ -117,6 +121,8 @@ func FuzzDetectPrefix(f *testing.F) { seeds := [][]byte{ nil, []byte("hello"), + []byte(lfsTestPointer), + []byte(lfsTestVersion + "oid sha256:4d7a"), []byte("\xef\xbb"), []byte("\xff\xfeh"), []byte("PK\x03"), diff --git a/lfs.go b/lfs.go new file mode 100644 index 0000000..7f039f0 --- /dev/null +++ b/lfs.go @@ -0,0 +1,135 @@ +package magic + +import "bytes" + +const ( + lfsPointerLimit = 1024 + lfsHashLength = 64 +) + +type lfsPointer struct { + previous []byte + oid bool + size bool +} + +func isLFSPointer(data []byte, prefix bool) bool { + if len(data) == 0 || len(data) >= lfsPointerLimit || bytes.IndexByte(data, '\r') >= 0 { + return false + } + version, remaining, complete := bytes.Cut(data, []byte{'\n'}) + if !validLFSVersion(version, prefix && !complete) { + return false + } + if !complete { + return prefix + } + + pointer := lfsPointer{} + for len(remaining) > 0 { + line, rest, terminated := bytes.Cut(remaining, []byte{'\n'}) + if !terminated && !prefix { + return false + } + if !pointer.line(line, !terminated) { + return false + } + remaining = rest + } + return prefix || pointer.oid && pointer.size +} + +func validLFSVersion(line []byte, partial bool) bool { + for _, version := range [...]string{ + "version https://git-lfs.github.com/spec/v1", + "version https://hawser.github.com/spec/v1", + } { + if bytes.Equal(line, []byte(version)) || partial && bytes.HasPrefix([]byte(version), line) { + return true + } + } + return false +} + +func (pointer *lfsPointer) line(line []byte, partial bool) bool { + key, value, separated := bytes.Cut(line, []byte{' '}) + if !pointer.validKey(key, partial && !separated) { + return false + } + if !separated { + return partial + } + if len(value) > 0 && value[0] == ' ' { + return false + } + switch string(key) { + case "oid": + if !validLFSOID(value, partial) { + return false + } + pointer.oid = true + case "size": + if !validLFSSize(value, partial) { + return false + } + pointer.size = true + } + pointer.previous = key + return true +} + +func (pointer *lfsPointer) validKey(key []byte, partial bool) bool { + if len(key) == 0 || bytes.Equal(key, []byte("version")) && !partial { + return false + } + for _, value := range key { + switch { + case value >= 'a' && value <= 'z', isDigit(value), value == '.', value == '-': + default: + return false + } + } + if bytes.Compare(key, pointer.previous) <= 0 && (!partial || !bytes.HasPrefix(pointer.previous, key)) { + return false + } + // Once a key sorts past a missing required field, later lines cannot supply it. + if !pointer.oid && bytes.Compare(key, []byte("oid")) > 0 { + return false + } + return pointer.size || bytes.Compare(key, []byte("size")) <= 0 +} + +func validLFSOID(value []byte, partial bool) bool { + const method = "sha256:" + if len(value) < len(method) { + return partial && bytes.HasPrefix([]byte(method), value) + } + if !bytes.HasPrefix(value, []byte(method)) { + return false + } + hash := value[len(method):] + if len(hash) > lfsHashLength || !partial && len(hash) != lfsHashLength { + return false + } + for _, digit := range hash { + if !isDigit(digit) && (digit < 'a' || digit > 'f') { + return false + } + } + return true +} + +func validLFSSize(value []byte, partial bool) bool { + if len(value) == 0 { + return partial + } + if len(value) > 1 && value[0] == '0' { + return false + } + for _, digit := range value { + if !isDigit(digit) { + return false + } + } + return true +} diff --git a/lfs_test.go b/lfs_test.go new file mode 100644 index 0000000..e5c68a2 --- /dev/null +++ b/lfs_test.go @@ -0,0 +1,185 @@ +package magic + +import ( + "bytes" + "fmt" + "strings" + "testing" +) + +const lfsTestVersion = "version https://git-lfs.github.com/spec/v1\n" +const lfsTestOID = "oid sha256:4d7a214614ab2935c943f9e0ff69d22eadbb8f32b1258daaa5e2ca24d17e2393\n" +const lfsTestPointer = lfsTestVersion + lfsTestOID + "size 12345\n" + +func TestLFSPointerDetection(t *testing.T) { + t.Parallel() + + inputs := []string{ + lfsTestPointer, + lfsTestVersion + lfsTestOID + "size 0\n", + strings.Replace(lfsTestPointer, "git-lfs", "hawser", 1), + lfsTestVersion + "ext-0-custom sha256:abc\n" + lfsTestOID + "size 12345\n", + lfsTestVersion + "a.b-2 世界 with spaces\n" + lfsTestOID + "other value\nsize 1\nz value\n", + lfsTestPointer + "z \n", + lfsTestPointer + "z " + strings.Repeat("x", 1023-len(lfsTestPointer)-3) + "\n", + } + for index, input := range inputs { + t.Run(fmt.Sprint(index), func(t *testing.T) { + t.Parallel() + data := []byte(input) + want := Result{Kind: KindText, Format: FormatLFSPointer, MIME: mimeText, Encoding: EncodingUTF8} + assertResult(t, Detect(data), want) + want.Reason = ReasonNeedMore + assertResult(t, DetectPrefix(data), want) + }) + } +} + +func TestInvalidLFSPointers(t *testing.T) { + t.Parallel() + + tests := map[string]string{ + "embedded URL": "hello\n" + lfsTestPointer, + "leading whitespace": " " + lfsTestPointer, + "BOM": "\xef\xbb\xbf" + lfsTestPointer, + "CRLF": strings.ReplaceAll(lfsTestPointer, "\n", "\r\n"), + "uppercase version": strings.Replace(lfsTestPointer, "https", "HTTPS", 1), + "wrong version": strings.Replace(lfsTestPointer, "/v1", "/v2", 1), + "escaped version": strings.Replace(lfsTestPointer, "spec", "%73pec", 1), + "uppercase hash": strings.Replace(lfsTestPointer, "4d7a", "4D7A", 1), + "invalid hash": strings.Replace(lfsTestPointer, "4d7a", "4g7a", 1), + "short hash": strings.Replace(lfsTestPointer, "4d7a", "4d7", 1), + "long hash": strings.Replace(lfsTestPointer, "4d7a", "4d7aa", 1), + "wrong hash method": strings.Replace(lfsTestPointer, "sha256", "sha512", 1), + "negative size": strings.Replace(lfsTestPointer, "12345", "-1", 1), + "signed size": strings.Replace(lfsTestPointer, "12345", "+1", 1), + "fractional size": strings.Replace(lfsTestPointer, "12345", "1.2", 1), + "noninteger size": strings.Replace(lfsTestPointer, "12345", "one", 1), + "empty size": strings.Replace(lfsTestPointer, "12345", "", 1), + "size leading zero": strings.Replace(lfsTestPointer, "12345", "01", 1), + "size trailing space": strings.Replace(lfsTestPointer, "12345", "1 ", 1), + "out of order": lfsTestVersion + "size 1\n" + lfsTestOID, + "missing oid": lfsTestVersion + "size 1\n", + "skipped size": lfsTestVersion + lfsTestOID + "z value\n", + "duplicate oid": lfsTestVersion + lfsTestOID + lfsTestOID + "size 1\n", + "duplicate version": lfsTestPointer + lfsTestVersion, + "duplicate extension": lfsTestVersion + "a one\na two\n" + lfsTestOID + "size 1\n", + "unordered extensions": lfsTestVersion + "b one\na two\n" + lfsTestOID + "size 1\n", + "invalid key": lfsTestVersion + "ext_custom value\n" + lfsTestOID + "size 1\n", + "uppercase key": lfsTestVersion + "Ext value\n" + lfsTestOID + "size 1\n", + "nonascii key": lfsTestPointer + "é value\n", + "empty key": lfsTestVersion + " value\n" + lfsTestOID + "size 1\n", + "missing separator": strings.Replace(lfsTestPointer, "size ", "size", 1), + "tab separator": strings.Replace(lfsTestPointer, "size ", "size\t", 1), + "double separator": strings.Replace(lfsTestPointer, "oid ", "oid ", 1), + "extension separator": lfsTestPointer + "z value\n", + "return in extension": lfsTestPointer + "z val\rue\n", + "extra newline": lfsTestPointer + "\n", + "1024 bytes": lfsTestPointer + "z " + strings.Repeat("x", 1024-len(lfsTestPointer)-3) + "\n", + "1025 bytes": lfsTestPointer + "z " + strings.Repeat("x", 1025-len(lfsTestPointer)-3) + "\n", + } + for name, input := range tests { + t.Run(name, func(t *testing.T) { + t.Parallel() + want := Result{Kind: KindText, Format: FormatText, MIME: mimeText, Encoding: EncodingUTF8} + assertResult(t, Detect([]byte(input)), want) + want.Reason = ReasonNeedMore + assertResult(t, DetectPrefix([]byte(input)), want) + }) + } +} + +func TestLFSPointerPrefixes(t *testing.T) { + t.Parallel() + + inputs := []string{ + lfsTestPointer, + strings.Replace(lfsTestPointer, "git-lfs", "hawser", 1), + lfsTestVersion + "a.b-2 世界\n" + lfsTestOID + "oid-extra value\nsize 0\nsize-extra value\n", + } + for _, input := range inputs { + data := []byte(input) + for length := 1; length <= len(data); length++ { + got := DetectPrefix(data[:length]) + want := Result{Kind: KindText, Format: FormatLFSPointer, MIME: mimeText, Encoding: EncodingUTF8, Reason: ReasonNeedMore} + if got != want { + t.Fatalf("DetectPrefix(%q) = %#v, want %#v", data[:length], got, want) + } + } + } + + for _, input := range []string{lfsTestVersion, lfsTestVersion + lfsTestOID, strings.TrimSuffix(lfsTestPointer, "\n")} { + assertResult(t, Detect([]byte(input)), Result{Kind: KindText, Format: FormatText, MIME: mimeText, Encoding: EncodingUTF8}) + } +} + +func TestInvalidLFSPointerPrefixes(t *testing.T) { + t.Parallel() + + inputs := []string{ + "version https://git-lfs.github.com/spec/v2", + lfsTestVersion + "o!", + lfsTestVersion + "p", + lfsTestVersion + "oid sha257", + lfsTestVersion + "oid sha256:A", + lfsTestVersion + "oid sha256:" + strings.Repeat("a", 65), + lfsTestVersion + lfsTestOID + "oid ", + lfsTestVersion + lfsTestOID + "sz", + lfsTestVersion + lfsTestOID + "size -", + lfsTestVersion + lfsTestOID + "size 01", + lfsTestVersion + lfsTestOID + "size 1.", + lfsTestPointer + "a", + lfsTestPointer + "z value\r", + lfsTestPointer + "z " + strings.Repeat("x", 1024-len(lfsTestPointer)-2), + } + for _, input := range inputs { + want := Result{Kind: KindText, Format: FormatText, MIME: mimeText, Encoding: EncodingUTF8, Reason: ReasonNeedMore} + if got := DetectPrefix([]byte(input)); got != want { + t.Fatalf("DetectPrefix(%q) = %#v, want %#v", input, got, want) + } + } +} + +func TestLFSPointerTextValidation(t *testing.T) { + t.Parallel() + + utf16 := []byte{0xff, 0xfe} + for _, value := range []byte(lfsTestPointer) { + utf16 = append(utf16, value, 0) + } + assertResult(t, Detect(utf16), Result{Kind: KindText, Format: FormatText, MIME: mimeText, Encoding: EncodingUTF16LE}) + for _, suffix := range [][]byte{{0}, {0xff}, {'z', ' ', 1, '\n'}} { + data := append([]byte(lfsTestPointer), suffix...) + for _, result := range []Result{Detect(data), DetectPrefix(data)} { + if result.Format == FormatLFSPointer || result.Kind == KindText { + t.Fatalf("invalid text %x classified as %#v", data, result) + } + } + } + data := []byte(lfsTestPointer + "z \x01\n") + assertResult(t, DetectWithOptions(data, Options{TextControls: "\x01"}), Result{ + Kind: KindText, Format: FormatLFSPointer, MIME: mimeText, Encoding: EncodingUTF8, + }) +} + +func TestLFSPointerAllocations(t *testing.T) { + inputs := [][]byte{ + []byte(lfsTestPointer), + []byte(lfsTestVersion + "ext-0-custom value\n" + lfsTestOID + "size 1\n"), + bytes.TrimSuffix([]byte(lfsTestPointer), []byte{'\n'}), + } + for _, input := range inputs { + if allocations := testing.AllocsPerRun(1000, func() { + Detect(input) + DetectPrefix(input) + }); allocations != 0 { + t.Fatalf("pointer detection allocated %.2f times", allocations) + } + } +} + +func ExampleDetect_lfsPointer() { + result := Detect([]byte(lfsTestPointer)) + fmt.Println(result.Kind, result.Format, result.MIME, result.Encoding) + // Output: text git-lfs-pointer text/plain utf-8 +} diff --git a/magic.go b/magic.go index 3462df3..92fdd8e 100644 --- a/magic.go +++ b/magic.go @@ -42,28 +42,29 @@ type Result struct { // Format values reported in Result.Format. const ( - FormatText = "text" - FormatHTML = "html" - FormatXML = "xml" - FormatSVG = "svg" - FormatJSON = "json" - FormatZIP = "zip" - FormatTAR = "tar" - FormatPHAR = "phar" - FormatGZIP = "gzip" - FormatBZIP2 = "bzip2" - FormatXZ = "xz" - FormatZstd = "zstd" - FormatPDF = "pdf" - FormatCFBF = "cfbf" - FormatPNG = "png" - FormatJPEG = "jpeg" - FormatGIF = "gif" - FormatELF = "elf" - FormatMachO = "mach-o" - FormatPE = "pe" - FormatWASM = "wasm" - FormatAR = "ar" + FormatLFSPointer = "git-lfs-pointer" + FormatText = "text" + FormatHTML = "html" + FormatXML = "xml" + FormatSVG = "svg" + FormatJSON = "json" + FormatZIP = "zip" + FormatTAR = "tar" + FormatPHAR = "phar" + FormatGZIP = "gzip" + FormatBZIP2 = "bzip2" + FormatXZ = "xz" + FormatZstd = "zstd" + FormatPDF = "pdf" + FormatCFBF = "cfbf" + FormatPNG = "png" + FormatJPEG = "jpeg" + FormatGIF = "gif" + FormatELF = "elf" + FormatMachO = "mach-o" + FormatPE = "pe" + FormatWASM = "wasm" + FormatAR = "ar" ) const ( @@ -150,6 +151,9 @@ func detect(data []byte, options Options) Result { } else if result.Kind == KindText { result.Format = FormatText result.MIME = mimeText + if result.Encoding == EncodingUTF8 && isLFSPointer(data, options.Prefix) { + result.Format = FormatLFSPointer + } } if options.Prefix && (binaryNeedsMore || prefixResultCanChange(result, len(data))) { From 12f7a4034a173411df57df9bc9ba0db178c59a72 Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Thu, 1 Oct 2026 15:13:00 +0100 Subject: [PATCH 2/2] Match git-lfs on version aliases and size bounds Accept the git-media.io alias git-lfs still reads, reject sizes beyond int64, and derive the fuzz prefix expectation from the input so the check can fail. --- fuzz_test.go | 2 +- lfs.go | 6 +++++- lfs_test.go | 7 +++++++ 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/fuzz_test.go b/fuzz_test.go index ff942d5..72be89a 100644 --- a/fuzz_test.go +++ b/fuzz_test.go @@ -70,7 +70,7 @@ func FuzzDetect(f *testing.F) { expectedPrefix.Format = FormatJSON expectedPrefix.MIME = mimeJSON } - if prefix.Format == FormatLFSPointer { + if first.Format == FormatText && isLFSPointer(data, true) { expectedPrefix.Format = FormatLFSPointer } if prefix.Reason == ReasonNeedMore { diff --git a/lfs.go b/lfs.go index 7f039f0..3a16cbb 100644 --- a/lfs.go +++ b/lfs.go @@ -5,6 +5,7 @@ import "bytes" const ( lfsPointerLimit = 1024 lfsHashLength = 64 + lfsSizeMax = "9223372036854775807" ) type lfsPointer struct { @@ -43,6 +44,7 @@ func validLFSVersion(line []byte, partial bool) bool { for _, version := range [...]string{ "version https://git-lfs.github.com/spec/v1", "version https://hawser.github.com/spec/v1", + "version http://git-media.io/v/2", } { if bytes.Equal(line, []byte(version)) || partial && bytes.HasPrefix([]byte(version), line) { return true @@ -131,5 +133,7 @@ func validLFSSize(value []byte, partial bool) bool { return false } } - return true + // Leading zeros are rejected above, so equal lengths compare numerically. + return len(value) < len(lfsSizeMax) || + len(value) == len(lfsSizeMax) && bytes.Compare(value, []byte(lfsSizeMax)) <= 0 } diff --git a/lfs_test.go b/lfs_test.go index e5c68a2..34e178a 100644 --- a/lfs_test.go +++ b/lfs_test.go @@ -18,6 +18,8 @@ func TestLFSPointerDetection(t *testing.T) { lfsTestPointer, lfsTestVersion + lfsTestOID + "size 0\n", strings.Replace(lfsTestPointer, "git-lfs", "hawser", 1), + strings.Replace(lfsTestPointer, "https://git-lfs.github.com/spec/v1", "http://git-media.io/v/2", 1), + lfsTestVersion + lfsTestOID + "size " + lfsSizeMax + "\n", lfsTestVersion + "ext-0-custom sha256:abc\n" + lfsTestOID + "size 12345\n", lfsTestVersion + "a.b-2 世界 with spaces\n" + lfsTestOID + "other value\nsize 1\nz value\n", lfsTestPointer + "z \n", @@ -57,6 +59,8 @@ func TestInvalidLFSPointers(t *testing.T) { "noninteger size": strings.Replace(lfsTestPointer, "12345", "one", 1), "empty size": strings.Replace(lfsTestPointer, "12345", "", 1), "size leading zero": strings.Replace(lfsTestPointer, "12345", "01", 1), + "size past int64": strings.Replace(lfsTestPointer, "12345", "9223372036854775808", 1), + "size extra digit": strings.Replace(lfsTestPointer, "12345", "92233720368547758070", 1), "size trailing space": strings.Replace(lfsTestPointer, "12345", "1 ", 1), "out of order": lfsTestVersion + "size 1\n" + lfsTestOID, "missing oid": lfsTestVersion + "size 1\n", @@ -95,6 +99,7 @@ func TestLFSPointerPrefixes(t *testing.T) { inputs := []string{ lfsTestPointer, strings.Replace(lfsTestPointer, "git-lfs", "hawser", 1), + strings.Replace(lfsTestPointer, "https://git-lfs.github.com/spec/v1", "http://git-media.io/v/2", 1), lfsTestVersion + "a.b-2 世界\n" + lfsTestOID + "oid-extra value\nsize 0\nsize-extra value\n", } for _, input := range inputs { @@ -128,6 +133,7 @@ func TestInvalidLFSPointerPrefixes(t *testing.T) { lfsTestVersion + lfsTestOID + "size -", lfsTestVersion + lfsTestOID + "size 01", lfsTestVersion + lfsTestOID + "size 1.", + lfsTestVersion + lfsTestOID + "size 9223372036854775808", lfsTestPointer + "a", lfsTestPointer + "z value\r", lfsTestPointer + "z " + strings.Repeat("x", 1024-len(lfsTestPointer)-2), @@ -166,6 +172,7 @@ func TestLFSPointerAllocations(t *testing.T) { inputs := [][]byte{ []byte(lfsTestPointer), []byte(lfsTestVersion + "ext-0-custom value\n" + lfsTestOID + "size 1\n"), + []byte(lfsTestVersion + lfsTestOID + "size " + lfsSizeMax + "\n"), bytes.TrimSuffix([]byte(lfsTestPointer), []byte{'\n'}), } for _, input := range inputs {