From 3dfe84da8f102dc1c088e29b47e0889ab14da8ce Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Thu, 1 Oct 2026 15:20:16 +0100 Subject: [PATCH] Add streaming text validation --- README.md | 21 ++++ text.go | 125 +---------------------- text_validator.go | 223 +++++++++++++++++++++++++++++++++++++++++ text_validator_test.go | 196 ++++++++++++++++++++++++++++++++++++ 4 files changed, 445 insertions(+), 120 deletions(-) create mode 100644 text_validator.go create mode 100644 text_validator_test.go diff --git a/README.md b/README.md index c60e689..4185fb0 100644 --- a/README.md +++ b/README.md @@ -58,6 +58,27 @@ This permits backspace, vertical tab, and the DOS end marker in addition to the default controls. The same policy applies to UTF-8 and decoded UTF-16 or UTF-32. NUL remains binary even if listed, and binary format signatures take precedence. +To validate text while reading a file, write chunks to a `TextValidator`: + +```go +var validator magic.TextValidator +if _, err := io.Copy(&validator, reader); err != nil { + return err +} +result := validator.Result(magic.Options{}) +``` + +The validator uses fixed memory, retains no input buffers, and handles Unicode code +points and BOMs split across writes. `Result` returns a snapshot without ending +the stream. Pass `Prefix: true` if reading stopped at a byte limit; incomplete +trailing code points are then accepted. `TextControls` applies the same policy +as `DetectWithOptions`, and can be chosen when requesting the result. + +`TextValidator` reports `Kind`, `Encoding`, and `Reason`. Format and MIME +detection remain available through `Detect` and `DetectPrefix`, including for +binary formats with text-like signatures. Each validator belongs to one stream; +assign `magic.TextValidator{}` to reuse it for another file. + ## Results `Kind` is `text`, `binary`, or `unknown`. `Format` and `MIME` describe the diff --git a/text.go b/text.go index c3b7969..c6124cc 100644 --- a/text.go +++ b/text.go @@ -1,12 +1,6 @@ package magic -import ( - "bytes" - "encoding/binary" - "strings" - "unicode/utf16" - "unicode/utf8" -) +import "strings" const ( utf8BOMLength = 3 @@ -15,58 +9,13 @@ const ( utf16UnitSize = 2 utf32UnitSize = 4 lowSurrogate = 0xdc00 + controlLimit = 0x20 ) func classifyText(data []byte, options Options) Result { - if len(data) == 0 { - return Result{Kind: KindText} - } - if hasPrefix(data, "\xff\xfe\x00\x00") { - return classifyUTF32BOM(data[utf32BOMLength:], binary.LittleEndian, EncodingUTF32LE, options) - } - if hasPrefix(data, "\x00\x00\xfe\xff") { - return classifyUTF32BOM(data[utf32BOMLength:], binary.BigEndian, EncodingUTF32BE, options) - } - if hasPrefix(data, "\xef\xbb\xbf") { - payload := data[utf8BOMLength:] - return encodedTextResult(EncodingUTF8, validUTF8(payload, options.Prefix), containsDisallowedControlUTF8(payload, options)) - } - if hasPrefix(data, "\xff\xfe") { - return classifyUTF16BOM(data[utf16BOMLength:], binary.LittleEndian, EncodingUTF16LE, options) - } - if hasPrefix(data, "\xfe\xff") { - return classifyUTF16BOM(data[utf16BOMLength:], binary.BigEndian, EncodingUTF16BE, options) - } - if !validUTF8(data, options.Prefix) { - if containsNUL(data) { - return Result{Kind: KindBinary} - } - return Result{Kind: KindUnknown, Reason: ReasonInvalidText} - } - if containsDisallowedControlUTF8(data, options) { - return Result{Kind: KindBinary} - } - return Result{Kind: KindText, Encoding: EncodingUTF8} -} - -func validUTF8(data []byte, prefix bool) bool { - if utf8.Valid(data) { - return true - } - if !prefix { - return false - } - for len(data) > 0 { - if !utf8.FullRune(data) { - return true - } - r, size := utf8.DecodeRune(data) - if r == utf8.RuneError && size == 1 { - return false - } - data = data[size:] - } - return true + var validator TextValidator + _, _ = validator.Write(data) + return validator.result(options) } func encodedTextResult(encoding string, valid, disallowedControl bool) Result { @@ -79,70 +28,6 @@ func encodedTextResult(encoding string, valid, disallowedControl bool) Result { return Result{Kind: KindText, Encoding: encoding} } -func classifyUTF16BOM(data []byte, order binary.ByteOrder, encoding string, options Options) Result { - valid, controls := validUTF16(data, order, options) - return encodedTextResult(encoding, valid, controls) -} - -func validUTF16(data []byte, order binary.ByteOrder, options Options) (valid, disallowedControl bool) { - for len(data) > 0 { - if len(data) < utf16UnitSize { - return options.Prefix, disallowedControl - } - unit := rune(order.Uint16(data)) - data = data[utf16UnitSize:] - switch { - case utf16.IsSurrogate(unit): - if unit >= lowSurrogate { - return false, false - } - if len(data) < utf16UnitSize { - return options.Prefix, disallowedControl - } - next := rune(order.Uint16(data)) - if !utf16.IsSurrogate(next) || next < lowSurrogate { - return false, false - } - data = data[utf16UnitSize:] - case unit < 0x20 && !permittedControl(byte(unit), options): - disallowedControl = true - } - } - return true, disallowedControl -} - -func classifyUTF32BOM(data []byte, order binary.ByteOrder, encoding string, options Options) Result { - controls := false - for len(data) > 0 { - if len(data) < utf32UnitSize { - return encodedTextResult(encoding, options.Prefix, controls) - } - value := rune(order.Uint32(data)) - if !utf8.ValidRune(value) { - return encodedTextResult(encoding, false, false) - } - if value < 0x20 && !permittedControl(byte(value), options) { - controls = true - } - data = data[utf32UnitSize:] - } - return encodedTextResult(encoding, true, controls) -} - -func containsNUL(data []byte) bool { - return bytes.IndexByte(data, 0) >= 0 -} - -func containsDisallowedControlUTF8(data []byte, options Options) bool { - // In valid UTF-8, every byte below 0x20 represents its ASCII code point. - for _, value := range data { - if value < 0x20 && !permittedControl(value, options) { - return true - } - } - return false -} - func permittedControl(value byte, options Options) bool { switch value { case 0: diff --git a/text_validator.go b/text_validator.go new file mode 100644 index 0000000..7ad6c0c --- /dev/null +++ b/text_validator.go @@ -0,0 +1,223 @@ +package magic + +import ( + "bytes" + "encoding/binary" + "unicode/utf16" + "unicode/utf8" +) + +// TextValidator validates chunks in order with fixed memory and no input references. +// The zero value is ready for use. Assign TextValidator{} to start another stream. +// It checks text encoding and controls; format signatures require Detect or DetectPrefix. +type TextValidator struct { + head, tail [utf8.UTFMax]byte + headLen, tailLen int + encoding string + controls uint32 + started, bom bool + invalid bool +} + +// Write implements io.Writer. It always accepts all bytes; Result reports validity. +func (v *TextValidator) Write(data []byte) (int, error) { + n := len(data) + if !v.started { + copied := copy(v.head[v.headLen:], data) + v.headLen += copied + data = data[copied:] + if v.headLen < len(v.head) { + return n, nil + } + v.start() + } + v.consume(data) + return n, nil +} + +// Result snapshots the bytes written so far without ending the stream. +// Options apply to this snapshot. Prefix permits an incomplete final code point. +// Only Kind, Encoding and Reason are populated; prefixes report ReasonNeedMore. +func (v TextValidator) Result(options Options) Result { + result := v.result(options) + if options.Prefix { + if v.headLen == 0 { + result.Kind = KindUnknown + } + result.Reason = ReasonNeedMore + } + return result +} + +func (v *TextValidator) result(options Options) Result { + if v.headLen == 0 { + return Result{Kind: KindText} + } + if !v.started { + v.start() + } + valid := !v.invalid && (v.tailLen == 0 || options.Prefix) + controls := false + for control := byte(0); control < controlLimit; control++ { + if v.controls&(1<= 0 { + v.controls |= 1 + } + return + } + if v.encoding == EncodingUTF8 { + for _, value := range data { + v.control(rune(value)) + } + } + for v.tailLen > 0 && len(data) > 0 { + v.tail[v.tailLen] = data[0] + v.tailLen++ + data = data[1:] + if v.fullRune() { + tail, size := v.tail, v.tailLen + v.tailLen = 0 + v.validate(tail[:size]) + if v.invalid { + return + } + } + } + if v.tailLen == 0 { + v.validate(data) + } +} + +func (v *TextValidator) fullRune() bool { + switch v.encoding { + case EncodingUTF16LE, EncodingUTF16BE: + if v.tailLen < utf16UnitSize { + return false + } + unit := v.unit(v.tail[:]) + return !utf16.IsSurrogate(unit) || unit >= lowSurrogate || v.tailLen == len(v.tail) + case EncodingUTF32LE, EncodingUTF32BE: + return v.tailLen == utf32UnitSize + default: + return utf8.FullRune(v.tail[:v.tailLen]) + } +} + +func (v *TextValidator) validate(data []byte) { + switch v.encoding { + case EncodingUTF16LE, EncodingUTF16BE: + v.validateUTF16(data) + case EncodingUTF32LE, EncodingUTF32BE: + v.validateUTF32(data) + default: + v.validateUTF8(data) + } +} + +func (v *TextValidator) validateUTF8(data []byte) { + end := len(data) + for i := len(data) - 1; i >= 0 && i >= len(data)-(utf8.UTFMax-1); i-- { + if utf8.RuneStart(data[i]) { + if !utf8.FullRune(data[i:]) { + end = i + } + break + } + } + v.invalid = !utf8.Valid(data[:end]) + v.tailLen = copy(v.tail[:], data[end:]) +} + +func (v *TextValidator) validateUTF16(data []byte) { + for len(data) >= utf16UnitSize { + unit := v.unit(data) + if utf16.IsSurrogate(unit) { + if unit >= lowSurrogate { + v.invalid = true + return + } + if len(data) < 2*utf16UnitSize { + break + } + if utf16.DecodeRune(unit, v.unit(data[utf16UnitSize:])) == utf8.RuneError { + v.invalid = true + return + } + data = data[2*utf16UnitSize:] + } else { + v.control(unit) + data = data[utf16UnitSize:] + } + } + v.tailLen = copy(v.tail[:], data) +} + +func (v *TextValidator) validateUTF32(data []byte) { + for len(data) >= utf32UnitSize { + value := binary.BigEndian.Uint32(data) + if v.encoding == EncodingUTF32LE { + value = binary.LittleEndian.Uint32(data) + } + if !utf8.ValidRune(rune(value)) { + v.invalid = true + return + } + v.control(rune(value)) + data = data[utf32UnitSize:] + } + v.tailLen = copy(v.tail[:], data) +} + +func (v *TextValidator) unit(data []byte) rune { + if v.encoding == EncodingUTF16LE { + return rune(binary.LittleEndian.Uint16(data)) + } + return rune(binary.BigEndian.Uint16(data)) +} + +func (v *TextValidator) control(value rune) { + if value < controlLimit { + v.controls |= 1 << value + } +} diff --git a/text_validator_test.go b/text_validator_test.go new file mode 100644 index 0000000..09f8bac --- /dev/null +++ b/text_validator_test.go @@ -0,0 +1,196 @@ +package magic_test + +import ( + "bytes" + "fmt" + "io" + "strings" + "testing" + "testing/iotest" + + "github.com/git-pkgs/magic" +) + +func TestTextValidatorChunks(t *testing.T) { + inputs := [][]byte{ + nil, []byte("plain source\n"), []byte("\xef\xbb\xbfhéllo 世界\U0001f642"), + []byte("\xff"), []byte("\xff\x00"), []byte("\xffsource\x00"), []byte("\xef\xbb\xbf\xff\x00"), + []byte("\xff\xfe\x00\xdc"), []byte("\xfe\xff\xd8\x00\x00a"), + []byte("\xff\xfe\x00\x00\x00\xd8\x00\x00"), + []byte("\x00\x00\xfe\xff\x00\x11\x00\x00"), + } + for _, codec := range textCodecs { + inputs = append(inputs, codec.encode("source héllo 世界\U0001f642 tail\n")) + for control := byte(0); control < 32; control++ { + inputs = append(inputs, codec.encode("text"+string(control)+"\U0001f642")) + } + } + for _, data := range inputs { + for chunk := 1; chunk <= 7; chunk++ { + var validator magic.TextValidator + checkTextSnapshot(t, validator, nil) + for offset := 0; offset < len(data); { + end := min(offset+chunk, len(data)) + part := bytes.Clone(data[offset:end]) + n, err := validator.Write(part) + if n != len(part) || err != nil { + t.Fatalf("Write: %d, %v", n, err) + } + clear(part) + if n, err := validator.Write(nil); n != 0 || err != nil { + t.Fatalf("empty Write: %d, %v", n, err) + } + offset = end + checkTextSnapshot(t, validator, data[:offset]) + } + } + } +} + +func checkTextSnapshot(t *testing.T, validator magic.TextValidator, data []byte) { + t.Helper() + before := validator + for _, prefix := range []bool{false, true} { + for _, controls := range []string{"", "\x00" + sourceControls} { + options := magic.Options{Prefix: prefix, TextControls: controls} + want := magic.DetectWithOptions(data, options) + want.Format, want.MIME = "", "" + if prefix { + want.Reason = magic.ReasonNeedMore + } + if got := validator.Result(options); got != want { + t.Fatalf("%x options=%+v: got %+v, want %+v", data, options, got, want) + } + } + } + if validator != before { + t.Fatal("Result changed validator state") + } +} + +func TestTextValidatorReader(t *testing.T) { + for _, codec := range textCodecs { + data := codec.encode("package main\n// héllo 世界\U0001f642\nfunc main() {}\n") + var validator magic.TextValidator + n, err := io.Copy(&validator, iotest.OneByteReader(bytes.NewReader(data))) + if n != int64(len(data)) || err != nil { + t.Fatal(n, err) + } + got := validator.Result(magic.Options{}) + if got.Kind != magic.KindText || got.Encoding != codec.name || got.Reason != magic.ReasonNone { + t.Fatal(got) + } + } +} + +func TestTextValidatorLimitedReader(t *testing.T) { + reader := strings.NewReader("source \U0001f642 trailing content") + var validator magic.TextValidator + const limit = len("source ") + 1 + if n, err := io.Copy(&validator, io.LimitReader(reader, int64(limit))); n != int64(limit) || err != nil { + t.Fatal(n, err) + } + if reader.Len() != len("source \U0001f642 trailing content")-limit { + t.Fatal("read beyond the limit") + } + if got := validator.Result(magic.Options{Prefix: true}); got.Kind != magic.KindText || got.Reason != magic.ReasonNeedMore { + t.Fatal(got) + } + if got := validator.Result(magic.Options{}); got.Kind != magic.KindUnknown || got.Reason != magic.ReasonInvalidText { + t.Fatal(got) + } + if _, err := io.Copy(&validator, reader); err != nil { + t.Fatal(err) + } + if got := validator.Result(magic.Options{}); got.Kind != magic.KindText { + t.Fatal(got) + } +} + +func TestTextValidatorPolicyAndReset(t *testing.T) { + var validator magic.TextValidator + _, _ = validator.Write([]byte("source\x1a")) + if got := validator.Result(magic.Options{}); got.Kind != magic.KindBinary { + t.Fatal(got) + } + if got := validator.Result(magic.Options{TextControls: "\x1a"}); got.Kind != magic.KindText { + t.Fatal(got) + } + _, _ = validator.Write([]byte{0}) + if got := validator.Result(magic.Options{TextControls: "\x00\x1a"}); got.Kind != magic.KindBinary { + t.Fatal(got) + } + validator = magic.TextValidator{} + _, _ = validator.Write([]byte("GIF89a")) + if got := validator.Result(magic.Options{}); got.Kind != magic.KindText || got.Format != "" || got.MIME != "" { + t.Fatal(got) + } + if got := magic.Detect([]byte("GIF89a")); got.Kind != magic.KindBinary { + t.Fatal(got) + } +} + +func TestTextValidatorAllocations(t *testing.T) { + data := []byte("source 世界\U0001f642\n") + if got := testing.AllocsPerRun(100, func() { + var validator magic.TextValidator + for _, value := range data { + _, _ = validator.Write([]byte{value}) + } + _ = validator.Result(magic.Options{}) + }); got != 0 { + t.Fatalf("allocations: %v", got) + } +} + +func FuzzTextValidator(f *testing.F) { + for _, codec := range textCodecs { + f.Add(codec.encode("source 世界\U0001f642\x1a tail"), uint8(3)) + } + f.Add([]byte("\xff\xfe\x00"), uint8(1)) + f.Add([]byte("\xef\xbb\xbf\xff\x00"), uint8(2)) + f.Fuzz(func(t *testing.T, data []byte, size uint8) { + var whole, chunks magic.TextValidator + _, _ = whole.Write(data) + width := int(size) + 1 + for offset := 0; offset < len(data); offset += width { + _, _ = chunks.Write(data[offset:min(offset+width, len(data))]) + _ = chunks.Result(magic.Options{}) + } + for _, prefix := range []bool{false, true} { + options := magic.Options{Prefix: prefix, TextControls: sourceControls} + if got, want := chunks.Result(options), whole.Result(options); got != want { + t.Fatalf("chunk=%d: got %+v, want %+v", width, got, want) + } + } + }) +} + +func ExampleTextValidator() { + var validator magic.TextValidator + reader := strings.NewReader("package main\nfunc main() {}\n") + if _, err := io.Copy(&validator, reader); err != nil { + panic(err) + } + result := validator.Result(magic.Options{}) + fmt.Println(result.Kind, result.Encoding) + // Output: text utf-8 +} + +func BenchmarkTextValidator(b *testing.B) { + for _, codec := range textCodecs { + data := codec.encode(strings.Repeat("source 世界\U0001f642\n", 1<<14)) + b.Run(codec.name, func(b *testing.B) { + b.SetBytes(int64(len(data))) + b.ReportAllocs() + for b.Loop() { + var validator magic.TextValidator + const chunk = 32 << 10 + for offset := 0; offset < len(data); offset += chunk { + _, _ = validator.Write(data[offset:min(offset+chunk, len(data))]) + } + _ = validator.Result(magic.Options{}) + } + }) + } +}