From 2e87024e27e75d00bae3623e9cf1344a4427e664 Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Thu, 1 Oct 2026 10:38:55 +0100 Subject: [PATCH] Support Unicode prefixes and configurable text controls --- README.md | 25 +++- fuzz_test.go | 13 +- magic.go | 30 ++-- testdata/fuzz/FuzzDetect/5af885026aa28acc | 2 + testdata/fuzz/FuzzDetect/80cb92b41e44e76d | 2 + text.go | 126 +++++++++++------ text_options_test.go | 164 ++++++++++++++++++++++ text_test.go | 22 +-- 8 files changed, 313 insertions(+), 71 deletions(-) create mode 100644 testdata/fuzz/FuzzDetect/5af885026aa28acc create mode 100644 testdata/fuzz/FuzzDetect/80cb92b41e44e76d create mode 100644 text_options_test.go diff --git a/README.md b/README.md index a5d45a6..8441a7b 100644 --- a/README.md +++ b/README.md @@ -45,11 +45,26 @@ is zero in the first release. Both functions are safe for concurrent use. They retain no input and use no mutable package state. +For text that uses formatting controls, supply an explicit allowlist: + +```go +result := magic.DetectWithOptions(prefix, magic.Options{ + Prefix: true, + TextControls: "\b\v\x1a", +}) +``` + +This permits backspace, vertical tab, and the DOS end marker in addition to the +default controls. The same policy applies to UTF-8 and decoded UTF-16 or UTF-32. +NUL remains binary even if listed, and binary format signatures take precedence. + ## Results `Kind` is `text`, `binary`, or `unknown`. `Format` and `MIME` describe the -physical content. `Encoding` is set for accepted UTF-8, UTF-16LE, or UTF-16BE -text and never appears as a MIME charset parameter. Compare `Format` and +physical content. `Encoding` identifies UTF-8, UTF-16LE, UTF-16BE, UTF-32LE, or +UTF-32BE and never appears as a MIME charset parameter. A BOM preserves encoding +metadata even if the content is malformed or contains disallowed controls; +check `Kind` before treating it as text. Compare `Format` and `Encoding` against the exported `Format*` and `Encoding*` constants rather than string literals. @@ -67,11 +82,15 @@ plausible, so Java class files fall through unclassified. PE requires the combine the result with filename or domain rules when it needs a semantic type. -Text accepts valid UTF-8 and BOM-marked UTF-16. Tab, line feed, form feed, +Text accepts valid UTF-8 and BOM-marked UTF-16 or UTF-32. Tab, line feed, form feed, carriage return, and escape are the permitted C0 controls. Other C0 controls classify the input as binary. Invalid UTF-8 without a NUL is unknown with `ReasonInvalidText`; callers that need Latin-1 can apply their own fallback. +A prefix ending inside a Unicode code point retains its encoding and reports +`ReasonNeedMore`. Complete input must contain complete code points. Invalid +interior sequences remain invalid in both modes. + JSON detection validates the complete input, including arrays and scalar top-level values. Surrounding JSON whitespace is accepted. A bounded prefix that contains valid or incomplete JSON syntax reports JSON with diff --git a/fuzz_test.go b/fuzz_test.go index a8425fd..2548ce0 100644 --- a/fuzz_test.go +++ b/fuzz_test.go @@ -15,6 +15,8 @@ func FuzzDetect(f *testing.F) { []byte("\xef\xbb\xbfhello"), []byte("\xff\xfeh\x00i\x00"), []byte("\xfe\xff\x00h\x00i"), + []byte("\xff\xfe\x00\x00A\x00\x00\x00"), + []byte("\x00\x00\xfe\xff\x00\x00\x00"), []byte("\xff\xfe\x01"), []byte("\xff\xfe\x01\x00"), []byte{'a', 0, 'b'}, @@ -70,7 +72,10 @@ func FuzzDetect(f *testing.F) { if prefix.Reason == ReasonNeedMore { expectedPrefix.Reason = ReasonNeedMore } - if prefix != expectedPrefix { + incompleteUnicode := first.Kind == KindUnknown && first.Reason == ReasonInvalidText && + prefix.Reason == ReasonNeedMore && + (prefix.Kind == KindText || prefix.Kind == KindBinary) + if prefix != expectedPrefix && !incompleteUnicode { t.Fatalf("DetectPrefix = %#v, incompatible with Detect = %#v", prefix, first) } } @@ -167,8 +172,10 @@ func assertResultInvariants(t testing.TB, result Result, prefix bool, inputLengt if result.Reason == ReasonNeedMore && !prefix { t.Fatalf("need-more for complete input: %#v", result) } - if result.Encoding != "" && result.Kind != KindText { - t.Fatalf("encoding set for non-text result: %#v", result) + switch result.Encoding { + case "", EncodingUTF8, EncodingUTF16LE, EncodingUTF16BE, EncodingUTF32LE, EncodingUTF32BE: + default: + t.Fatalf("invalid encoding: %#v", result) } if result.Kind == KindText && (result.Format == "" || result.MIME == "") { t.Fatalf("text result lacks metadata: %#v", result) diff --git a/magic.go b/magic.go index 8e289ac..3462df3 100644 --- a/magic.go +++ b/magic.go @@ -30,7 +30,7 @@ const ( // Result describes the physical format and text properties of content. // // MIME never includes a charset parameter. Encoding uses lowercase registered -// names. Empty fields mean that the corresponding property was not identified. +// names. A BOM identifies Encoding even when its content is invalid. type Result struct { Kind Kind MIME string @@ -96,11 +96,19 @@ const ( EncodingUTF8 = "utf-8" EncodingUTF16LE = "utf-16le" EncodingUTF16BE = "utf-16be" + EncodingUTF32LE = "utf-32le" + EncodingUTF32BE = "utf-32be" ) +type Options struct { + Prefix bool + // TextControls permits additional C0 bytes in text. NUL is always rejected. + TextControls string +} + // Detect classifies data as the complete content of a file. func Detect(data []byte) Result { - return detect(data, false) + return DetectWithOptions(data, Options{}) } // DetectPrefix classifies an intentionally bounded file prefix. @@ -108,13 +116,19 @@ func Detect(data []byte) Result { // ReasonNeedMore reports that later bytes could change the answer. NeedBytes // is reserved for a known minimum total length and is zero in this release. func DetectPrefix(prefix []byte) Result { - if len(prefix) == 0 { + return DetectWithOptions(prefix, Options{Prefix: true}) +} + +// DetectWithOptions applies the same control policy to UTF-8 and decoded Unicode. +// Binary format signatures take precedence over text options. +func DetectWithOptions(data []byte, options Options) Result { + if options.Prefix && len(data) == 0 { return Result{Kind: KindUnknown, Reason: ReasonNeedMore} } - return detect(prefix, true) + return detect(data, options) } -func detect(data []byte, prefix bool) Result { +func detect(data []byte, options Options) Result { format, mime, binaryNeedsMore := binaryFormatState(data) if format != "" { return Result{ @@ -125,11 +139,11 @@ func detect(data []byte, prefix bool) Result { } format, mime = textFormat(data) - if format == "" && isJSON(data, prefix) { + if format == "" && isJSON(data, options.Prefix) { format = FormatJSON mime = mimeJSON } - result := classifyText(data) + result := classifyText(data, options) if format != "" { result.Format = format result.MIME = mime @@ -138,7 +152,7 @@ func detect(data []byte, prefix bool) Result { result.MIME = mimeText } - if prefix && (binaryNeedsMore || prefixResultCanChange(result, len(data))) { + if options.Prefix && (binaryNeedsMore || prefixResultCanChange(result, len(data))) { result.Reason = ReasonNeedMore } diff --git a/testdata/fuzz/FuzzDetect/5af885026aa28acc b/testdata/fuzz/FuzzDetect/5af885026aa28acc new file mode 100644 index 0000000..ab090ea --- /dev/null +++ b/testdata/fuzz/FuzzDetect/5af885026aa28acc @@ -0,0 +1,2 @@ +go test fuzz v1 +[]byte("\x1a\xcf") diff --git a/testdata/fuzz/FuzzDetect/80cb92b41e44e76d b/testdata/fuzz/FuzzDetect/80cb92b41e44e76d new file mode 100644 index 0000000..fdda6d0 --- /dev/null +++ b/testdata/fuzz/FuzzDetect/80cb92b41e44e76d @@ -0,0 +1,2 @@ +go test fuzz v1 +[]byte("\xfe\xff\x00\x000") diff --git a/text.go b/text.go index 53dce8f..c3b7969 100644 --- a/text.go +++ b/text.go @@ -2,120 +2,154 @@ package magic import ( "bytes" + "encoding/binary" + "strings" + "unicode/utf16" "unicode/utf8" ) const ( utf8BOMLength = 3 utf16BOMLength = 2 + utf32BOMLength = 4 + utf16UnitSize = 2 + utf32UnitSize = 4 + lowSurrogate = 0xdc00 ) -func classifyText(data []byte) Result { +func classifyText(data []byte, options Options) Result { if len(data) == 0 { return Result{Kind: KindText} } - + if hasPrefix(data, "\xff\xfe\x00\x00") { + return classifyUTF32BOM(data[utf32BOMLength:], binary.LittleEndian, EncodingUTF32LE, options) + } + if hasPrefix(data, "\x00\x00\xfe\xff") { + return classifyUTF32BOM(data[utf32BOMLength:], binary.BigEndian, EncodingUTF32BE, options) + } if hasPrefix(data, "\xef\xbb\xbf") { - return classifyUTF8BOM(data[utf8BOMLength:]) + payload := data[utf8BOMLength:] + return encodedTextResult(EncodingUTF8, validUTF8(payload, options.Prefix), containsDisallowedControlUTF8(payload, options)) } if hasPrefix(data, "\xff\xfe") { - return classifyUTF16BOM(data[utf16BOMLength:], true) + return classifyUTF16BOM(data[utf16BOMLength:], binary.LittleEndian, EncodingUTF16LE, options) } if hasPrefix(data, "\xfe\xff") { - return classifyUTF16BOM(data[utf16BOMLength:], false) + return classifyUTF16BOM(data[utf16BOMLength:], binary.BigEndian, EncodingUTF16BE, options) } - - if !utf8.Valid(data) { + if !validUTF8(data, options.Prefix) { if containsNUL(data) { return Result{Kind: KindBinary} } return Result{Kind: KindUnknown, Reason: ReasonInvalidText} } - if containsDisallowedControlUTF8(data) { + if containsDisallowedControlUTF8(data, options) { return Result{Kind: KindBinary} } return Result{Kind: KindText, Encoding: EncodingUTF8} } -func classifyUTF8BOM(data []byte) Result { - if !utf8.Valid(data) { - return Result{Kind: KindUnknown, Reason: ReasonInvalidText} +func validUTF8(data []byte, prefix bool) bool { + if utf8.Valid(data) { + return true } - if containsDisallowedControlUTF8(data) { - return Result{Kind: KindBinary} + if !prefix { + return false } - return Result{Kind: KindText, Encoding: EncodingUTF8} + for len(data) > 0 { + if !utf8.FullRune(data) { + return true + } + r, size := utf8.DecodeRune(data) + if r == utf8.RuneError && size == 1 { + return false + } + data = data[size:] + } + return true } -func classifyUTF16BOM(data []byte, littleEndian bool) Result { - valid, disallowedControl := validUTF16(data, littleEndian) +func encodedTextResult(encoding string, valid, disallowedControl bool) Result { if !valid { - return Result{Kind: KindUnknown, Reason: ReasonInvalidText} + return Result{Kind: KindUnknown, Encoding: encoding, Reason: ReasonInvalidText} } if disallowedControl { - return Result{Kind: KindBinary} - } - - encoding := EncodingUTF16BE - if littleEndian { - encoding = EncodingUTF16LE + return Result{Kind: KindBinary, Encoding: encoding} } return Result{Kind: KindText, Encoding: encoding} } -func validUTF16(data []byte, littleEndian bool) (valid, disallowedControl bool) { - if len(data)%2 != 0 { - return false, false - } +func classifyUTF16BOM(data []byte, order binary.ByteOrder, encoding string, options Options) Result { + valid, controls := validUTF16(data, order, options) + return encodedTextResult(encoding, valid, controls) +} - for offset := 0; offset < len(data); offset += 2 { - unit := utf16Unit(data[offset], data[offset+1], littleEndian) +func validUTF16(data []byte, order binary.ByteOrder, options Options) (valid, disallowedControl bool) { + for len(data) > 0 { + if len(data) < utf16UnitSize { + return options.Prefix, disallowedControl + } + unit := rune(order.Uint16(data)) + data = data[utf16UnitSize:] switch { - case unit >= 0xd800 && unit <= 0xdbff: - if offset+3 >= len(data) { + case utf16.IsSurrogate(unit): + if unit >= lowSurrogate { return false, false } - next := utf16Unit(data[offset+2], data[offset+3], littleEndian) - if next < 0xdc00 || next > 0xdfff { + if len(data) < utf16UnitSize { + return options.Prefix, disallowedControl + } + next := rune(order.Uint16(data)) + if !utf16.IsSurrogate(next) || next < lowSurrogate { return false, false } - offset += 2 - case unit >= 0xdc00 && unit <= 0xdfff: - return false, false - case unit < 0x20 && !permittedControl(byte(unit)): + data = data[utf16UnitSize:] + case unit < 0x20 && !permittedControl(byte(unit), options): disallowedControl = true } } - return true, disallowedControl } -func utf16Unit(first, second byte, littleEndian bool) uint16 { - if littleEndian { - return uint16(first) | uint16(second)<<8 +func classifyUTF32BOM(data []byte, order binary.ByteOrder, encoding string, options Options) Result { + controls := false + for len(data) > 0 { + if len(data) < utf32UnitSize { + return encodedTextResult(encoding, options.Prefix, controls) + } + value := rune(order.Uint32(data)) + if !utf8.ValidRune(value) { + return encodedTextResult(encoding, false, false) + } + if value < 0x20 && !permittedControl(byte(value), options) { + controls = true + } + data = data[utf32UnitSize:] } - return uint16(first)<<8 | uint16(second) + return encodedTextResult(encoding, true, controls) } func containsNUL(data []byte) bool { return bytes.IndexByte(data, 0) >= 0 } -func containsDisallowedControlUTF8(data []byte) bool { +func containsDisallowedControlUTF8(data []byte, options Options) bool { // In valid UTF-8, every byte below 0x20 represents its ASCII code point. for _, value := range data { - if value < 0x20 && !permittedControl(value) { + if value < 0x20 && !permittedControl(value, options) { return true } } return false } -func permittedControl(value byte) bool { +func permittedControl(value byte, options Options) bool { switch value { + case 0: + return false case '\t', '\n', '\f', '\r', '\x1b': return true default: - return false + return strings.IndexByte(options.TextControls, value) >= 0 } } diff --git a/text_options_test.go b/text_options_test.go new file mode 100644 index 0000000..feb7a79 --- /dev/null +++ b/text_options_test.go @@ -0,0 +1,164 @@ +package magic_test + +import ( + "bytes" + "encoding/binary" + "fmt" + "strings" + "testing" + "unicode/utf16" + + "github.com/git-pkgs/magic" +) + +const sourceControls = "\x02\x03\b\v\x0f\x1a\x1f" + +type textCodec struct { + name string + width int + order binary.AppendByteOrder +} + +var textCodecs = []textCodec{ + {magic.EncodingUTF8, 1, nil}, + {magic.EncodingUTF16LE, 2, binary.LittleEndian}, + {magic.EncodingUTF16BE, 2, binary.BigEndian}, + {magic.EncodingUTF32LE, 4, binary.LittleEndian}, + {magic.EncodingUTF32BE, 4, binary.BigEndian}, +} + +func (codec textCodec) encode(source string) []byte { + if codec.width == 1 { + return []byte(source) + } + var encoded []byte + if codec.width == 2 { + for _, value := range utf16.Encode([]rune("\ufeff" + source)) { + encoded = codec.order.AppendUint16(encoded, value) + } + } else { + for _, value := range "\ufeff" + source { + encoded = codec.order.AppendUint32(encoded, uint32(value)) + } + } + return encoded +} + +func TestUnicodePrefixes(t *testing.T) { + for _, codec := range textCodecs { + t.Run(codec.name, func(t *testing.T) { + data := codec.encode("source \U0001f642 end") + for cut := codec.width; cut <= len(data); cut++ { + input := data[:cut] + got := magic.DetectPrefix(input) + if got.Kind != magic.KindText || got.Encoding != codec.name || got.Reason != magic.ReasonNeedMore { + t.Fatalf("cut %d: %+v", cut, got) + } + } + whole := magic.Detect(data) + if whole.Kind != magic.KindText || whole.Encoding != codec.name || whole.Reason != magic.ReasonNone { + t.Fatal(whole) + } + }) + } +} + +func TestMalformedUnicode(t *testing.T) { + for _, input := range [][]byte{ + []byte("\xff\xfe\x00\xdc"), + []byte("\xfe\xff\xd8\x00\x00a"), + []byte("\xff\xfe\x00\x00\x00\xd8\x00\x00"), + []byte("\x00\x00\xfe\xff\x00\x11\x00\x00"), + []byte("\xef\xbb\xbf\xff"), + } { + for _, prefix := range []bool{false, true} { + got := magic.DetectWithOptions(input, magic.Options{Prefix: prefix}) + if got.Kind != magic.KindUnknown || got.Encoding == "" { + t.Fatalf("%x prefix=%t: %+v", input, prefix, got) + } + } + } + for _, codec := range textCodecs[1:] { + data := codec.encode("\U0001f642") + for remove := 1; remove < len(data)-codec.width; remove++ { + got := magic.Detect(data[:len(data)-remove]) + if got.Kind != magic.KindUnknown || got.Reason != magic.ReasonInvalidText || got.Encoding != codec.name { + t.Fatalf("%s remove=%d: %+v", codec.name, remove, got) + } + } + } +} + +func TestUnicodePrefixWithNUL(t *testing.T) { + data := []byte("\xfe\xff\x00\x000") + prefix := magic.DetectPrefix(data) + complete := magic.Detect(data) + if prefix.Kind != magic.KindBinary || prefix.Encoding != magic.EncodingUTF16BE || prefix.Reason != magic.ReasonNeedMore { + t.Fatal(prefix) + } + if complete.Kind != magic.KindUnknown || complete.Encoding != magic.EncodingUTF16BE || complete.Reason != magic.ReasonInvalidText { + t.Fatal(complete) + } +} + +func TestUTF8PrefixWithControl(t *testing.T) { + data := []byte("\x1a\xcf") + if got := magic.DetectPrefix(data); got.Kind != magic.KindBinary || got.Reason != magic.ReasonNeedMore { + t.Fatal(got) + } + if got := magic.Detect(data); got.Kind != magic.KindUnknown || got.Reason != magic.ReasonInvalidText { + t.Fatal(got) + } + if got := magic.DetectWithOptions(data, magic.Options{Prefix: true, TextControls: "\x1a"}); got.Kind != magic.KindText || got.Encoding != magic.EncodingUTF8 { + t.Fatal(got) + } +} + +func TestTextControlOptions(t *testing.T) { + for _, codec := range textCodecs { + for control := byte(0); control < 32; control++ { + t.Run(fmt.Sprintf("%s/%02x", codec.name, control), func(t *testing.T) { + data := codec.encode("a" + string(control) + "b") + strict := strings.ContainsRune("\t\n\f\r\x1b", rune(control)) + for _, prefix := range []bool{false, true} { + for _, extra := range []string{"", sourceControls, "\x00" + sourceControls} { + want := magic.KindBinary + if strict || control != 0 && strings.IndexByte(extra, control) >= 0 { + want = magic.KindText + } + got := magic.DetectWithOptions(data, magic.Options{Prefix: prefix, TextControls: extra}) + if got.Kind != want { + t.Fatalf("prefix=%t controls=%q: %+v want %s", prefix, extra, got, want) + } + } + } + }) + } + } +} + +func TestTextControlPositions(t *testing.T) { + for _, control := range []byte(sourceControls) { + for _, content := range []string{string(control) + "text", "te" + string(control) + "xt", "text" + string(control), strings.Repeat(string(control), 1024)} { + got := magic.DetectWithOptions([]byte(content), magic.Options{TextControls: sourceControls}) + if got.Kind != magic.KindText { + t.Fatalf("%q: %+v", content, got) + } + } + } +} + +func TestTextOptionsPreserveSignatures(t *testing.T) { + for _, data := range [][]byte{ + []byte("%PDF-1.7"), []byte("\x89PNG\r\n\x1a\n"), []byte("PK\x03\x04"), []byte("\x7fELF"), + } { + before := bytes.Clone(data) + want := magic.Detect(data) + for _, prefix := range []bool{false, true} { + got := magic.DetectWithOptions(data, magic.Options{Prefix: prefix, TextControls: sourceControls}) + if got != want || got.Kind != magic.KindBinary || !bytes.Equal(data, before) { + t.Fatalf("%x: %+v want %+v", data, got, want) + } + } + } +} diff --git a/text_test.go b/text_test.go index 71ec36f..5fc4046 100644 --- a/text_test.go +++ b/text_test.go @@ -52,22 +52,22 @@ func TestTextDecisionTable(t *testing.T) { { name: "malformed UTF-8 BOM text", input: []byte("\xef\xbb\xbf\xff"), - expect: Result{Kind: KindUnknown, Reason: ReasonInvalidText}, + expect: Result{Kind: KindUnknown, Encoding: EncodingUTF8, Reason: ReasonInvalidText}, }, { name: "malformed UTF-16 has precedence over control", input: []byte("\xff\xfe\x01"), - expect: Result{Kind: KindUnknown, Reason: ReasonInvalidText}, + expect: Result{Kind: KindUnknown, Encoding: EncodingUTF16LE, Reason: ReasonInvalidText}, }, { name: "BOM text with disallowed control", input: []byte("\xff\xfe\x01\x00"), - expect: Result{Kind: KindBinary}, + expect: Result{Kind: KindBinary, Encoding: EncodingUTF16LE}, }, { name: "UTF-8 BOM text with disallowed control", input: []byte("\xef\xbb\xbf\x01"), - expect: Result{Kind: KindBinary}, + expect: Result{Kind: KindBinary, Encoding: EncodingUTF8}, }, { name: "NUL without UTF-16 BOM", @@ -135,27 +135,27 @@ func TestUTF16Validation(t *testing.T) { { name: "unpaired high surrogate", input: []byte("\xff\xfe\x3d\xd8"), - expect: Result{Kind: KindUnknown, Reason: ReasonInvalidText}, + expect: Result{Kind: KindUnknown, Encoding: EncodingUTF16LE, Reason: ReasonInvalidText}, }, { name: "high surrogate followed by ordinary unit", input: []byte("\xff\xfe\x3d\xd8A\x00"), - expect: Result{Kind: KindUnknown, Reason: ReasonInvalidText}, + expect: Result{Kind: KindUnknown, Encoding: EncodingUTF16LE, Reason: ReasonInvalidText}, }, { name: "unpaired low surrogate", input: []byte("\xfe\xff\xde\x00"), - expect: Result{Kind: KindUnknown, Reason: ReasonInvalidText}, + expect: Result{Kind: KindUnknown, Encoding: EncodingUTF16BE, Reason: ReasonInvalidText}, }, { - name: "UTF-32LE is binary", + name: "UTF-32LE text", input: []byte("\xff\xfe\x00\x00A\x00\x00\x00"), - expect: Result{Kind: KindBinary}, + expect: Result{Kind: KindText, Format: FormatText, MIME: mimeText, Encoding: EncodingUTF32LE}, }, { - name: "UTF-32BE is binary", + name: "UTF-32BE text", input: []byte("\x00\x00\xfe\xff\x00\x00\x00A"), - expect: Result{Kind: KindBinary}, + expect: Result{Kind: KindText, Format: FormatText, MIME: mimeText, Encoding: EncodingUTF32BE}, }, }