Support Unicode prefixes and configurable text controls

README.md

@@ -45,11 +45,26 @@ is zero in the first release.
 Both functions are safe for concurrent use. They retain no input and use no
 mutable package state.
 
+For text that uses formatting controls, supply an explicit allowlist:
+
+```go
+result := magic.DetectWithOptions(prefix, magic.Options{
+	Prefix:       true,
+	TextControls: "\b\v\x1a",
+})
+```
+
+This permits backspace, vertical tab, and the DOS end marker in addition to the
+default controls. The same policy applies to UTF-8 and decoded UTF-16 or UTF-32.
+NUL remains binary even if listed, and binary format signatures take precedence.
+
 ## Results
 
 `Kind` is `text`, `binary`, or `unknown`. `Format` and `MIME` describe the
-physical content. `Encoding` is set for accepted UTF-8, UTF-16LE, or UTF-16BE
-text and never appears as a MIME charset parameter. Compare `Format` and
+physical content. `Encoding` identifies UTF-8, UTF-16LE, UTF-16BE, UTF-32LE, or
+UTF-32BE and never appears as a MIME charset parameter. A BOM preserves encoding
+metadata even if the content is malformed or contains disallowed controls;
+check `Kind` before treating it as text. Compare `Format` and
 `Encoding` against the exported `Format*` and `Encoding*` constants rather
 than string literals.
 
@@ -67,11 +82,15 @@ plausible, so Java class files fall through unclassified. PE requires the
 combine the result with filename or domain rules when it needs a semantic
 type.
 
-Text accepts valid UTF-8 and BOM-marked UTF-16. Tab, line feed, form feed,
+Text accepts valid UTF-8 and BOM-marked UTF-16 or UTF-32. Tab, line feed, form feed,
 carriage return, and escape are the permitted C0 controls. Other C0 controls
 classify the input as binary. Invalid UTF-8 without a NUL is unknown with
 `ReasonInvalidText`; callers that need Latin-1 can apply their own fallback.
 
+A prefix ending inside a Unicode code point retains its encoding and reports
+`ReasonNeedMore`. Complete input must contain complete code points. Invalid
+interior sequences remain invalid in both modes.
+
 JSON detection validates the complete input, including arrays and scalar
 top-level values. Surrounding JSON whitespace is accepted. A bounded prefix
 that contains valid or incomplete JSON syntax reports JSON with

fuzz_test.go

@@ -15,6 +15,8 @@ func FuzzDetect(f *testing.F) {
 		[]byte("\xef\xbb\xbfhello"),
 		[]byte("\xff\xfeh\x00i\x00"),
 		[]byte("\xfe\xff\x00h\x00i"),
+		[]byte("\xff\xfe\x00\x00A\x00\x00\x00"),
+		[]byte("\x00\x00\xfe\xff\x00\x00\x00"),
 		[]byte("\xff\xfe\x01"),
 		[]byte("\xff\xfe\x01\x00"),
 		[]byte{'a', 0, 'b'},
@@ -70,7 +72,10 @@ func FuzzDetect(f *testing.F) {
 			if prefix.Reason == ReasonNeedMore {
 				expectedPrefix.Reason = ReasonNeedMore
 			}
-			if prefix != expectedPrefix {
+			incompleteUnicode := first.Kind == KindUnknown && first.Reason == ReasonInvalidText &&
+				prefix.Reason == ReasonNeedMore &&
+				(prefix.Kind == KindText || prefix.Kind == KindBinary)
+			if prefix != expectedPrefix && !incompleteUnicode {
 				t.Fatalf("DetectPrefix = %#v, incompatible with Detect = %#v", prefix, first)
 			}
 		}
@@ -167,8 +172,10 @@ func assertResultInvariants(t testing.TB, result Result, prefix bool, inputLengt
 	if result.Reason == ReasonNeedMore && !prefix {
 		t.Fatalf("need-more for complete input: %#v", result)
 	}
-	if result.Encoding != "" && result.Kind != KindText {
-		t.Fatalf("encoding set for non-text result: %#v", result)
+	switch result.Encoding {
+	case "", EncodingUTF8, EncodingUTF16LE, EncodingUTF16BE, EncodingUTF32LE, EncodingUTF32BE:
+	default:
+		t.Fatalf("invalid encoding: %#v", result)
 	}
 	if result.Kind == KindText && (result.Format == "" || result.MIME == "") {
 		t.Fatalf("text result lacks metadata: %#v", result)

magic.go

@@ -30,7 +30,7 @@ const (
 // Result describes the physical format and text properties of content.
 //
 // MIME never includes a charset parameter. Encoding uses lowercase registered
-// names. Empty fields mean that the corresponding property was not identified.
+// names. A BOM identifies Encoding even when its content is invalid.
 type Result struct {
 	Kind      Kind
 	MIME      string
@@ -96,25 +96,39 @@ const (
 	EncodingUTF8    = "utf-8"
 	EncodingUTF16LE = "utf-16le"
 	EncodingUTF16BE = "utf-16be"
+	EncodingUTF32LE = "utf-32le"
+	EncodingUTF32BE = "utf-32be"
 )
 
+type Options struct {
+	Prefix bool
+	// TextControls permits additional C0 bytes in text. NUL is always rejected.
+	TextControls string
+}
+
 // Detect classifies data as the complete content of a file.
 func Detect(data []byte) Result {
-	return detect(data, false)
+	return DetectWithOptions(data, Options{})
 }
 
 // DetectPrefix classifies an intentionally bounded file prefix.
 //
 // ReasonNeedMore reports that later bytes could change the answer. NeedBytes
 // is reserved for a known minimum total length and is zero in this release.
 func DetectPrefix(prefix []byte) Result {
-	if len(prefix) == 0 {
+	return DetectWithOptions(prefix, Options{Prefix: true})
+}
+
+// DetectWithOptions applies the same control policy to UTF-8 and decoded Unicode.
+// Binary format signatures take precedence over text options.
+func DetectWithOptions(data []byte, options Options) Result {
+	if options.Prefix && len(data) == 0 {
 		return Result{Kind: KindUnknown, Reason: ReasonNeedMore}
 	}
-	return detect(prefix, true)
+	return detect(data, options)
 }
 
-func detect(data []byte, prefix bool) Result {
+func detect(data []byte, options Options) Result {
 	format, mime, binaryNeedsMore := binaryFormatState(data)
 	if format != "" {
 		return Result{
@@ -125,11 +139,11 @@ func detect(data []byte, prefix bool) Result {
 	}
 
 	format, mime = textFormat(data)
-	if format == "" && isJSON(data, prefix) {
+	if format == "" && isJSON(data, options.Prefix) {
 		format = FormatJSON
 		mime = mimeJSON
 	}
-	result := classifyText(data)
+	result := classifyText(data, options)
 	if format != "" {
 		result.Format = format
 		result.MIME = mime
@@ -138,7 +152,7 @@ func detect(data []byte, prefix bool) Result {
 		result.MIME = mimeText
 	}
 
-	if prefix && (binaryNeedsMore || prefixResultCanChange(result, len(data))) {
+	if options.Prefix && (binaryNeedsMore || prefixResultCanChange(result, len(data))) {
 		result.Reason = ReasonNeedMore
 	}
 

testdata/fuzz/FuzzDetect/5af885026aa28acc

@@ -0,0 +1,2 @@
+go test fuzz v1
+[]byte("\x1a\xcf")

testdata/fuzz/FuzzDetect/80cb92b41e44e76d

@@ -0,0 +1,2 @@
+go test fuzz v1
+[]byte("\xfe\xff\x00\x000")

text.go

@@ -2,120 +2,154 @@ package magic
 
 import (
 	"bytes"
+	"encoding/binary"
+	"strings"
+	"unicode/utf16"
 	"unicode/utf8"
 )
 
 const (
 	utf8BOMLength  = 3
 	utf16BOMLength = 2
+	utf32BOMLength = 4
+	utf16UnitSize  = 2
+	utf32UnitSize  = 4
+	lowSurrogate   = 0xdc00
 )
 
-func classifyText(data []byte) Result {
+func classifyText(data []byte, options Options) Result {
 	if len(data) == 0 {
 		return Result{Kind: KindText}
 	}
-
+	if hasPrefix(data, "\xff\xfe\x00\x00") {
+		return classifyUTF32BOM(data[utf32BOMLength:], binary.LittleEndian, EncodingUTF32LE, options)
+	}
+	if hasPrefix(data, "\x00\x00\xfe\xff") {
+		return classifyUTF32BOM(data[utf32BOMLength:], binary.BigEndian, EncodingUTF32BE, options)
+	}
 	if hasPrefix(data, "\xef\xbb\xbf") {
-		return classifyUTF8BOM(data[utf8BOMLength:])
+		payload := data[utf8BOMLength:]
+		return encodedTextResult(EncodingUTF8, validUTF8(payload, options.Prefix), containsDisallowedControlUTF8(payload, options))
 	}
 	if hasPrefix(data, "\xff\xfe") {
-		return classifyUTF16BOM(data[utf16BOMLength:], true)
+		return classifyUTF16BOM(data[utf16BOMLength:], binary.LittleEndian, EncodingUTF16LE, options)
 	}
 	if hasPrefix(data, "\xfe\xff") {
-		return classifyUTF16BOM(data[utf16BOMLength:], false)
+		return classifyUTF16BOM(data[utf16BOMLength:], binary.BigEndian, EncodingUTF16BE, options)
 	}
-
-	if !utf8.Valid(data) {
+	if !validUTF8(data, options.Prefix) {
 		if containsNUL(data) {
 			return Result{Kind: KindBinary}
 		}
 		return Result{Kind: KindUnknown, Reason: ReasonInvalidText}
 	}
-	if containsDisallowedControlUTF8(data) {
+	if containsDisallowedControlUTF8(data, options) {
 		return Result{Kind: KindBinary}
 	}
 	return Result{Kind: KindText, Encoding: EncodingUTF8}
 }
 
-func classifyUTF8BOM(data []byte) Result {
-	if !utf8.Valid(data) {
-		return Result{Kind: KindUnknown, Reason: ReasonInvalidText}
+func validUTF8(data []byte, prefix bool) bool {
+	if utf8.Valid(data) {
+		return true
 	}
-	if containsDisallowedControlUTF8(data) {
-		return Result{Kind: KindBinary}
+	if !prefix {
+		return false
 	}
-	return Result{Kind: KindText, Encoding: EncodingUTF8}
+	for len(data) > 0 {
+		if !utf8.FullRune(data) {
+			return true
+		}
+		r, size := utf8.DecodeRune(data)
+		if r == utf8.RuneError && size == 1 {
+			return false
+		}
+		data = data[size:]
+	}
+	return true
 }
 
-func classifyUTF16BOM(data []byte, littleEndian bool) Result {
-	valid, disallowedControl := validUTF16(data, littleEndian)
+func encodedTextResult(encoding string, valid, disallowedControl bool) Result {
 	if !valid {
-		return Result{Kind: KindUnknown, Reason: ReasonInvalidText}
+		return Result{Kind: KindUnknown, Encoding: encoding, Reason: ReasonInvalidText}
 	}
 	if disallowedControl {
-		return Result{Kind: KindBinary}
-	}
-
-	encoding := EncodingUTF16BE
-	if littleEndian {
-		encoding = EncodingUTF16LE
+		return Result{Kind: KindBinary, Encoding: encoding}
 	}
 	return Result{Kind: KindText, Encoding: encoding}
 }
 
-func validUTF16(data []byte, littleEndian bool) (valid, disallowedControl bool) {
-	if len(data)%2 != 0 {
-		return false, false
-	}
+func classifyUTF16BOM(data []byte, order binary.ByteOrder, encoding string, options Options) Result {
+	valid, controls := validUTF16(data, order, options)
+	return encodedTextResult(encoding, valid, controls)
+}
 
-	for offset := 0; offset < len(data); offset += 2 {
-		unit := utf16Unit(data[offset], data[offset+1], littleEndian)
+func validUTF16(data []byte, order binary.ByteOrder, options Options) (valid, disallowedControl bool) {
+	for len(data) > 0 {
+		if len(data) < utf16UnitSize {
+			return options.Prefix, disallowedControl
+		}
+		unit := rune(order.Uint16(data))
+		data = data[utf16UnitSize:]
 		switch {
-		case unit >= 0xd800 && unit <= 0xdbff:
-			if offset+3 >= len(data) {
+		case utf16.IsSurrogate(unit):
+			if unit >= lowSurrogate {
 				return false, false
 			}
-			next := utf16Unit(data[offset+2], data[offset+3], littleEndian)
-			if next < 0xdc00 || next > 0xdfff {
+			if len(data) < utf16UnitSize {
+				return options.Prefix, disallowedControl
+			}
+			next := rune(order.Uint16(data))
+			if !utf16.IsSurrogate(next) || next < lowSurrogate {
 				return false, false
 			}
-			offset += 2
-		case unit >= 0xdc00 && unit <= 0xdfff:
-			return false, false
-		case unit < 0x20 && !permittedControl(byte(unit)):
+			data = data[utf16UnitSize:]
+		case unit < 0x20 && !permittedControl(byte(unit), options):
 			disallowedControl = true
 		}
 	}
-
 	return true, disallowedControl
 }
 
-func utf16Unit(first, second byte, littleEndian bool) uint16 {
-	if littleEndian {
-		return uint16(first) | uint16(second)<<8
+func classifyUTF32BOM(data []byte, order binary.ByteOrder, encoding string, options Options) Result {
+	controls := false
+	for len(data) > 0 {
+		if len(data) < utf32UnitSize {
+			return encodedTextResult(encoding, options.Prefix, controls)
+		}
+		value := rune(order.Uint32(data))
+		if !utf8.ValidRune(value) {
+			return encodedTextResult(encoding, false, false)
+		}
+		if value < 0x20 && !permittedControl(byte(value), options) {
+			controls = true
+		}
+		data = data[utf32UnitSize:]
 	}
-	return uint16(first)<<8 | uint16(second)
+	return encodedTextResult(encoding, true, controls)
 }
 
 func containsNUL(data []byte) bool {
 	return bytes.IndexByte(data, 0) >= 0
 }
 
-func containsDisallowedControlUTF8(data []byte) bool {
+func containsDisallowedControlUTF8(data []byte, options Options) bool {
 	// In valid UTF-8, every byte below 0x20 represents its ASCII code point.
 	for _, value := range data {
-		if value < 0x20 && !permittedControl(value) {
+		if value < 0x20 && !permittedControl(value, options) {
 			return true
 		}
 	}
 	return false
 }
 
-func permittedControl(value byte) bool {
+func permittedControl(value byte, options Options) bool {
 	switch value {
+	case 0:
+		return false
 	case '\t', '\n', '\f', '\r', '\x1b':
 		return true
 	default:
-		return false
+		return strings.IndexByte(options.TextControls, value) >= 0
 	}
 }