Add streaming text validation

README.md

@@ -58,6 +58,27 @@ This permits backspace, vertical tab, and the DOS end marker in addition to the
 default controls. The same policy applies to UTF-8 and decoded UTF-16 or UTF-32.
 NUL remains binary even if listed, and binary format signatures take precedence.
 
+To validate text while reading a file, write chunks to a `TextValidator`:
+
+```go
+var validator magic.TextValidator
+if _, err := io.Copy(&validator, reader); err != nil {
+    return err
+}
+result := validator.Result(magic.Options{})
+```
+
+The validator uses fixed memory, retains no input buffers, and handles Unicode code
+points and BOMs split across writes. `Result` returns a snapshot without ending
+the stream. Pass `Prefix: true` if reading stopped at a byte limit; incomplete
+trailing code points are then accepted. `TextControls` applies the same policy
+as `DetectWithOptions`, and can be chosen when requesting the result.
+
+`TextValidator` reports `Kind`, `Encoding`, and `Reason`. Format and MIME
+detection remain available through `Detect` and `DetectPrefix`, including for
+binary formats with text-like signatures. Each validator belongs to one stream;
+assign `magic.TextValidator{}` to reuse it for another file.
+
 ## Results
 
 `Kind` is `text`, `binary`, or `unknown`. `Format` and `MIME` describe the

text.go

@@ -1,12 +1,6 @@
 package magic
 
-import (
-	"bytes"
-	"encoding/binary"
-	"strings"
-	"unicode/utf16"
-	"unicode/utf8"
-)
+import "strings"
 
 const (
 	utf8BOMLength  = 3
@@ -15,58 +9,13 @@ const (
 	utf16UnitSize  = 2
 	utf32UnitSize  = 4
 	lowSurrogate   = 0xdc00
+	controlLimit   = 0x20
 )
 
 func classifyText(data []byte, options Options) Result {
-	if len(data) == 0 {
-		return Result{Kind: KindText}
-	}
-	if hasPrefix(data, "\xff\xfe\x00\x00") {
-		return classifyUTF32BOM(data[utf32BOMLength:], binary.LittleEndian, EncodingUTF32LE, options)
-	}
-	if hasPrefix(data, "\x00\x00\xfe\xff") {
-		return classifyUTF32BOM(data[utf32BOMLength:], binary.BigEndian, EncodingUTF32BE, options)
-	}
-	if hasPrefix(data, "\xef\xbb\xbf") {
-		payload := data[utf8BOMLength:]
-		return encodedTextResult(EncodingUTF8, validUTF8(payload, options.Prefix), containsDisallowedControlUTF8(payload, options))
-	}
-	if hasPrefix(data, "\xff\xfe") {
-		return classifyUTF16BOM(data[utf16BOMLength:], binary.LittleEndian, EncodingUTF16LE, options)
-	}
-	if hasPrefix(data, "\xfe\xff") {
-		return classifyUTF16BOM(data[utf16BOMLength:], binary.BigEndian, EncodingUTF16BE, options)
-	}
-	if !validUTF8(data, options.Prefix) {
-		if containsNUL(data) {
-			return Result{Kind: KindBinary}
-		}
-		return Result{Kind: KindUnknown, Reason: ReasonInvalidText}
-	}
-	if containsDisallowedControlUTF8(data, options) {
-		return Result{Kind: KindBinary}
-	}
-	return Result{Kind: KindText, Encoding: EncodingUTF8}
-}
-
-func validUTF8(data []byte, prefix bool) bool {
-	if utf8.Valid(data) {
-		return true
-	}
-	if !prefix {
-		return false
-	}
-	for len(data) > 0 {
-		if !utf8.FullRune(data) {
-			return true
-		}
-		r, size := utf8.DecodeRune(data)
-		if r == utf8.RuneError && size == 1 {
-			return false
-		}
-		data = data[size:]
-	}
-	return true
+	var validator TextValidator
+	_, _ = validator.Write(data)
+	return validator.result(options)
 }
 
 func encodedTextResult(encoding string, valid, disallowedControl bool) Result {
@@ -79,70 +28,6 @@ func encodedTextResult(encoding string, valid, disallowedControl bool) Result {
 	return Result{Kind: KindText, Encoding: encoding}
 }
 
-func classifyUTF16BOM(data []byte, order binary.ByteOrder, encoding string, options Options) Result {
-	valid, controls := validUTF16(data, order, options)
-	return encodedTextResult(encoding, valid, controls)
-}
-
-func validUTF16(data []byte, order binary.ByteOrder, options Options) (valid, disallowedControl bool) {
-	for len(data) > 0 {
-		if len(data) < utf16UnitSize {
-			return options.Prefix, disallowedControl
-		}
-		unit := rune(order.Uint16(data))
-		data = data[utf16UnitSize:]
-		switch {
-		case utf16.IsSurrogate(unit):
-			if unit >= lowSurrogate {
-				return false, false
-			}
-			if len(data) < utf16UnitSize {
-				return options.Prefix, disallowedControl
-			}
-			next := rune(order.Uint16(data))
-			if !utf16.IsSurrogate(next) || next < lowSurrogate {
-				return false, false
-			}
-			data = data[utf16UnitSize:]
-		case unit < 0x20 && !permittedControl(byte(unit), options):
-			disallowedControl = true
-		}
-	}
-	return true, disallowedControl
-}
-
-func classifyUTF32BOM(data []byte, order binary.ByteOrder, encoding string, options Options) Result {
-	controls := false
-	for len(data) > 0 {
-		if len(data) < utf32UnitSize {
-			return encodedTextResult(encoding, options.Prefix, controls)
-		}
-		value := rune(order.Uint32(data))
-		if !utf8.ValidRune(value) {
-			return encodedTextResult(encoding, false, false)
-		}
-		if value < 0x20 && !permittedControl(byte(value), options) {
-			controls = true
-		}
-		data = data[utf32UnitSize:]
-	}
-	return encodedTextResult(encoding, true, controls)
-}
-
-func containsNUL(data []byte) bool {
-	return bytes.IndexByte(data, 0) >= 0
-}
-
-func containsDisallowedControlUTF8(data []byte, options Options) bool {
-	// In valid UTF-8, every byte below 0x20 represents its ASCII code point.
-	for _, value := range data {
-		if value < 0x20 && !permittedControl(value, options) {
-			return true
-		}
-	}
-	return false
-}
-
 func permittedControl(value byte, options Options) bool {
 	switch value {
 	case 0:

text_validator.go

@@ -0,0 +1,223 @@
+package magic
+
+import (
+	"bytes"
+	"encoding/binary"
+	"unicode/utf16"
+	"unicode/utf8"
+)
+
+// TextValidator validates chunks in order with fixed memory and no input references.
+// The zero value is ready for use. Assign TextValidator{} to start another stream.
+// It checks text encoding and controls; format signatures require Detect or DetectPrefix.
+type TextValidator struct {
+	head, tail       [utf8.UTFMax]byte
+	headLen, tailLen int
+	encoding         string
+	controls         uint32
+	started, bom     bool
+	invalid          bool
+}
+
+// Write implements io.Writer. It always accepts all bytes; Result reports validity.
+func (v *TextValidator) Write(data []byte) (int, error) {
+	n := len(data)
+	if !v.started {
+		copied := copy(v.head[v.headLen:], data)
+		v.headLen += copied
+		data = data[copied:]
+		if v.headLen < len(v.head) {
+			return n, nil
+		}
+		v.start()
+	}
+	v.consume(data)
+	return n, nil
+}
+
+// Result snapshots the bytes written so far without ending the stream.
+// Options apply to this snapshot. Prefix permits an incomplete final code point.
+// Only Kind, Encoding and Reason are populated; prefixes report ReasonNeedMore.
+func (v TextValidator) Result(options Options) Result {
+	result := v.result(options)
+	if options.Prefix {
+		if v.headLen == 0 {
+			result.Kind = KindUnknown
+		}
+		result.Reason = ReasonNeedMore
+	}
+	return result
+}
+
+func (v *TextValidator) result(options Options) Result {
+	if v.headLen == 0 {
+		return Result{Kind: KindText}
+	}
+	if !v.started {
+		v.start()
+	}
+	valid := !v.invalid && (v.tailLen == 0 || options.Prefix)
+	controls := false
+	for control := byte(0); control < controlLimit; control++ {
+		if v.controls&(1<<control) != 0 && !permittedControl(control, options) {
+			controls = true
+			break
+		}
+	}
+	if v.bom {
+		return encodedTextResult(v.encoding, valid, controls)
+	}
+	if !valid {
+		if v.controls&1 != 0 {
+			return Result{Kind: KindBinary}
+		}
+		return Result{Kind: KindUnknown, Reason: ReasonInvalidText}
+	}
+	if controls {
+		return Result{Kind: KindBinary}
+	}
+	return Result{Kind: KindText, Encoding: EncodingUTF8}
+}
+
+func (v *TextValidator) start() {
+	data := v.head[:v.headLen]
+	v.encoding = EncodingUTF8
+	bom := 0
+	switch {
+	case hasPrefix(data, "\xff\xfe\x00\x00"):
+		v.encoding, bom = EncodingUTF32LE, utf32BOMLength
+	case hasPrefix(data, "\x00\x00\xfe\xff"):
+		v.encoding, bom = EncodingUTF32BE, utf32BOMLength
+	case hasPrefix(data, "\xef\xbb\xbf"):
+		bom = utf8BOMLength
+	case hasPrefix(data, "\xff\xfe"):
+		v.encoding, bom = EncodingUTF16LE, utf16BOMLength
+	case hasPrefix(data, "\xfe\xff"):
+		v.encoding, bom = EncodingUTF16BE, utf16BOMLength
+	}
+	v.started, v.bom = true, bom != 0
+	v.consume(data[bom:])
+}
+
+func (v *TextValidator) consume(data []byte) {
+	if v.invalid {
+		if !v.bom && v.encoding == EncodingUTF8 && bytes.IndexByte(data, 0) >= 0 {
+			v.controls |= 1
+		}
+		return
+	}
+	if v.encoding == EncodingUTF8 {
+		for _, value := range data {
+			v.control(rune(value))
+		}
+	}
+	for v.tailLen > 0 && len(data) > 0 {
+		v.tail[v.tailLen] = data[0]
+		v.tailLen++
+		data = data[1:]
+		if v.fullRune() {
+			tail, size := v.tail, v.tailLen
+			v.tailLen = 0
+			v.validate(tail[:size])
+			if v.invalid {
+				return
+			}
+		}
+	}
+	if v.tailLen == 0 {
+		v.validate(data)
+	}
+}
+
+func (v *TextValidator) fullRune() bool {
+	switch v.encoding {
+	case EncodingUTF16LE, EncodingUTF16BE:
+		if v.tailLen < utf16UnitSize {
+			return false
+		}
+		unit := v.unit(v.tail[:])
+		return !utf16.IsSurrogate(unit) || unit >= lowSurrogate || v.tailLen == len(v.tail)
+	case EncodingUTF32LE, EncodingUTF32BE:
+		return v.tailLen == utf32UnitSize
+	default:
+		return utf8.FullRune(v.tail[:v.tailLen])
+	}
+}
+
+func (v *TextValidator) validate(data []byte) {
+	switch v.encoding {
+	case EncodingUTF16LE, EncodingUTF16BE:
+		v.validateUTF16(data)
+	case EncodingUTF32LE, EncodingUTF32BE:
+		v.validateUTF32(data)
+	default:
+		v.validateUTF8(data)
+	}
+}
+
+func (v *TextValidator) validateUTF8(data []byte) {
+	end := len(data)
+	for i := len(data) - 1; i >= 0 && i >= len(data)-(utf8.UTFMax-1); i-- {
+		if utf8.RuneStart(data[i]) {
+			if !utf8.FullRune(data[i:]) {
+				end = i
+			}
+			break
+		}
+	}
+	v.invalid = !utf8.Valid(data[:end])
+	v.tailLen = copy(v.tail[:], data[end:])
+}
+
+func (v *TextValidator) validateUTF16(data []byte) {
+	for len(data) >= utf16UnitSize {
+		unit := v.unit(data)
+		if utf16.IsSurrogate(unit) {
+			if unit >= lowSurrogate {
+				v.invalid = true
+				return
+			}
+			if len(data) < 2*utf16UnitSize {
+				break
+			}
+			if utf16.DecodeRune(unit, v.unit(data[utf16UnitSize:])) == utf8.RuneError {
+				v.invalid = true
+				return
+			}
+			data = data[2*utf16UnitSize:]
+		} else {
+			v.control(unit)
+			data = data[utf16UnitSize:]
+		}
+	}
+	v.tailLen = copy(v.tail[:], data)
+}
+
+func (v *TextValidator) validateUTF32(data []byte) {
+	for len(data) >= utf32UnitSize {
+		value := binary.BigEndian.Uint32(data)
+		if v.encoding == EncodingUTF32LE {
+			value = binary.LittleEndian.Uint32(data)
+		}
+		if !utf8.ValidRune(rune(value)) {
+			v.invalid = true
+			return
+		}
+		v.control(rune(value))
+		data = data[utf32UnitSize:]
+	}
+	v.tailLen = copy(v.tail[:], data)
+}
+
+func (v *TextValidator) unit(data []byte) rune {
+	if v.encoding == EncodingUTF16LE {
+		return rune(binary.LittleEndian.Uint16(data))
+	}
+	return rune(binary.BigEndian.Uint16(data))
+}
+
+func (v *TextValidator) control(value rune) {
+	if value < controlLimit {
+		v.controls |= 1 << value
+	}
+}