From 67c1b784e9b282ca3874357d22f3bf6db5c9a677 Mon Sep 17 00:00:00 2001 From: Martin Hutchinson Date: Wed, 11 Nov 2026 09:41:36 +0000 Subject: [PATCH 1/2] Enforce strict C2SP tlog-proof framing and verbatim checkpoint parsing Unmarshal previously tokenized the entire proof line-by-line and reassembled the checkpoint afterward, silently appending a missing final newline to the checkpoint and accepting truncated proofs with no checkpoint at all. Marshal also emitted an invalid empty extra line when ExtraData was a non-nil empty slice. - Guard Marshal() with len(p.ExtraData) > 0 so empty non-nil ExtraData omits the extra line per C2SP tlog-proof@v1. - Replace bufio.Scanner in Unmarshal() with bytes.Cut so the raw Checkpoint bytes after the blank separator line are sliced and cloned verbatim without reassembly, while requiring newline-terminated header/hash lines, rejecting empty extra data and missing checkpoints, avoiding receiver mutation on error, and incidentally lifting Scanner's 64 KiB line limit. --- proof/tlog_proof.go | 59 +++++++----- proof/tlog_proof_test.go | 198 ++++++++++++++++++++++++++++++++++++++- 2 files changed, 228 insertions(+), 29 deletions(-) diff --git a/proof/tlog_proof.go b/proof/tlog_proof.go index 837c112..e3f505e 100644 --- a/proof/tlog_proof.go +++ b/proof/tlog_proof.go @@ -15,13 +15,11 @@ package proof import ( - "bufio" "bytes" "crypto/sha256" "encoding/base64" "fmt" "strconv" - "strings" ) const ( @@ -43,7 +41,7 @@ type TLogProof struct { func (p TLogProof) Marshal() []byte { var proof bytes.Buffer fmt.Fprintf(&proof, "%s\n", tlogProofHeaderV1) - if p.ExtraData != nil { + if len(p.ExtraData) > 0 { proof.WriteString("extra ") fmt.Fprintf(&proof, "%s\n", base64.StdEncoding.EncodeToString(p.ExtraData)) } @@ -60,43 +58,60 @@ func (p *TLogProof) Unmarshal(data []byte) error { if bytes.IndexByte(data, '\r') >= 0 { return fmt.Errorf("tlog proof contains carriage return") } - var err error - b := bufio.NewScanner(bytes.NewReader(data)) - if b.Scan(); b.Text() != tlogProofHeaderV1 { + line, rem, ok := bytes.Cut(data, []byte("\n")) + if !ok || string(line) != tlogProofHeaderV1 { return fmt.Errorf("tlog proof missing expected header") } + line, rem, ok = bytes.Cut(rem, []byte("\n")) + if !ok { + return fmt.Errorf("tlog proof missing required index") + } + // Handle optional extra line - var extra []byte - if b.Scan(); strings.HasPrefix(b.Text(), "extra ") { - e, _ := strings.CutPrefix(b.Text(), "extra ") + var ( + extra []byte + err error + ) + if after, hasExtra := bytes.CutPrefix(line, []byte("extra ")); hasExtra { + e := string(after) + if len(e) == 0 { + return fmt.Errorf("tlog proof empty extra data") + } extra, err = base64.StdEncoding.Strict().DecodeString(e) if err != nil { return fmt.Errorf("tlog proof extra data not base64 encoded: %w", err) } - b.Scan() + line, rem, ok = bytes.Cut(rem, []byte("\n")) + if !ok { + return fmt.Errorf("tlog proof missing required index") + } } - var idx uint64 - idxStr, ok := strings.CutPrefix(b.Text(), "index ") + afterIdx, ok := bytes.CutPrefix(line, []byte("index ")) if !ok { return fmt.Errorf("tlog proof missing required index") } + idxStr := string(afterIdx) if len(idxStr) > 1 && idxStr[0] == '0' { return fmt.Errorf("tlog proof index has leading zeros: %q", idxStr) } - idx, err = strconv.ParseUint(idxStr, 10, 63) + idx, err := strconv.ParseUint(idxStr, 10, 63) if err != nil { return fmt.Errorf("tlog proof index not a valid int64: %w", err) } var hashes [][sha256.Size]byte - for b.Scan() { - if b.Text() == "" { + for { + line, rem, ok = bytes.Cut(rem, []byte("\n")) + if !ok { + return fmt.Errorf("tlog proof missing blank line before checkpoint") + } + if len(line) == 0 { break } - hash, err := base64.StdEncoding.Strict().DecodeString(b.Text()) + hash, err := base64.StdEncoding.Strict().DecodeString(string(line)) if err != nil { return fmt.Errorf("tlog proof hash not base64 encoded: %w", err) } @@ -106,19 +121,13 @@ func (p *TLogProof) Unmarshal(data []byte) error { hashes = append(hashes, [sha256.Size]byte(hash)) } - var checkpoint bytes.Buffer - for b.Scan() { - checkpoint.Write(b.Bytes()) - checkpoint.WriteByte('\n') - } - - if err := b.Err(); err != nil { - return fmt.Errorf("scanning tlog proof: %w", err) + if len(rem) == 0 { + return fmt.Errorf("tlog proof missing checkpoint") } p.Index = idx p.Hashes = hashes - p.Checkpoint = checkpoint.Bytes() + p.Checkpoint = bytes.Clone(rem) p.ExtraData = extra return nil diff --git a/proof/tlog_proof_test.go b/proof/tlog_proof_test.go index 46102fe..0b3277a 100644 --- a/proof/tlog_proof_test.go +++ b/proof/tlog_proof_test.go @@ -65,6 +65,16 @@ func TestMarshal(t *testing.T) { }, want: "c2sp.org/tlog-proof@v1\nindex 0\n\ncheckpoint\n", }, + { + name: "proof with empty non-nil extra data omits extra line", + proof: TLogProof{ + Index: 0, + Hashes: [][sha256.Size]byte{h1}, + Checkpoint: []byte("checkpoint\n"), + ExtraData: []byte{}, + }, + want: fmt.Sprintf("c2sp.org/tlog-proof@v1\nindex 0\n%s\n\ncheckpoint\n", h1b64), + }, } for _, tt := range tests { @@ -82,11 +92,41 @@ func TestUnmarshalErrors(t *testing.T) { proof []byte wantErrSubstr string }{ + { + name: "empty input", + proof: []byte(""), + wantErrSubstr: "missing expected header", + }, + { + name: "header without newline", + proof: []byte("c2sp.org/tlog-proof@v1"), + wantErrSubstr: "missing expected header", + }, + { + name: "truncated after header", + proof: []byte("c2sp.org/tlog-proof@v1\n"), + wantErrSubstr: "missing required index", + }, { name: "missing header", proof: []byte("wrong-header\nindex 0\n\ncheckpoint\n"), wantErrSubstr: "missing expected header", }, + { + name: "empty extra data", + proof: []byte("c2sp.org/tlog-proof@v1\nextra \nindex 0\n\ncheckpoint\n"), + wantErrSubstr: "extra data", + }, + { + name: "extra line without newline", + proof: []byte("c2sp.org/tlog-proof@v1\nextra aGVsbG8="), + wantErrSubstr: "missing required index", + }, + { + name: "truncated after extra line", + proof: []byte("c2sp.org/tlog-proof@v1\nextra aGVsbG8=\n"), + wantErrSubstr: "missing required index", + }, { name: "invalid extra data encoding", proof: []byte("c2sp.org/tlog-proof@v1\nextra !!notbase64!!\nindex 0\n\ncheckpoint\n"), @@ -97,6 +137,11 @@ func TestUnmarshalErrors(t *testing.T) { proof: []byte("c2sp.org/tlog-proof@v1\n\n\ncheckpoint\n"), wantErrSubstr: "missing required index", }, + { + name: "index line without newline", + proof: []byte("c2sp.org/tlog-proof@v1\nindex 0"), + wantErrSubstr: "missing required index", + }, { name: "invalid index - not a number", proof: []byte("c2sp.org/tlog-proof@v1\nindex notanumber\n\ncheckpoint\n"), @@ -156,9 +201,39 @@ func TestUnmarshalErrors(t *testing.T) { wantErrSubstr: "hash not base64 encoded", }, { - name: "scanner error - buffer too large", - proof: []byte("c2sp.org/tlog-proof@v1\nindex 0\n" + strings.Repeat("a", 65*1024) + "\n"), - wantErrSubstr: "scanning tlog proof", + name: "oversized hash line (>64 KiB)", + proof: []byte("c2sp.org/tlog-proof@v1\nindex 0\n" + strings.Repeat("a", 65*1024) + "\n\ncheckpoint\n"), + wantErrSubstr: "hash length", + }, + { + name: "oversized invalid base64 hash line (>64 KiB)", + proof: []byte("c2sp.org/tlog-proof@v1\nindex 0\n" + strings.Repeat("!", 65*1024) + "\n\ncheckpoint\n"), + wantErrSubstr: "hash not base64 encoded", + }, + { + name: "missing blank line separator after index", + proof: []byte("c2sp.org/tlog-proof@v1\nindex 0\n"), + wantErrSubstr: "missing blank line", + }, + { + name: "missing blank line separator after hash", + proof: []byte("c2sp.org/tlog-proof@v1\nindex 0\n" + base64.StdEncoding.EncodeToString(make([]byte, 32)) + "\n"), + wantErrSubstr: "missing blank line", + }, + { + name: "hash line without newline", + proof: []byte("c2sp.org/tlog-proof@v1\nindex 0\n" + base64.StdEncoding.EncodeToString(make([]byte, 32))), + wantErrSubstr: "missing blank line", + }, + { + name: "empty checkpoint after blank line separator", + proof: []byte("c2sp.org/tlog-proof@v1\nindex 0\n\n"), + wantErrSubstr: "missing checkpoint", + }, + { + name: "empty checkpoint after hashes and blank line separator", + proof: []byte("c2sp.org/tlog-proof@v1\nindex 0\n" + base64.StdEncoding.EncodeToString(make([]byte, 32)) + "\n\n"), + wantErrSubstr: "missing checkpoint", }, { name: "CRLF on header line", @@ -195,7 +270,18 @@ func TestUnmarshalErrors(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - var p TLogProof + sentinel := TLogProof{ + Index: 42, + Hashes: [][sha256.Size]byte{sha256.Sum256([]byte("sentinel-hash"))}, + Checkpoint: []byte("sentinel-checkpoint\n"), + ExtraData: []byte("sentinel-extra"), + } + p := TLogProof{ + Index: sentinel.Index, + Hashes: append([][sha256.Size]byte(nil), sentinel.Hashes...), + Checkpoint: bytes.Clone(sentinel.Checkpoint), + ExtraData: bytes.Clone(sentinel.ExtraData), + } err := p.Unmarshal(tt.proof) if err == nil { @@ -205,6 +291,13 @@ func TestUnmarshalErrors(t *testing.T) { if !strings.Contains(err.Error(), tt.wantErrSubstr) { t.Errorf("error message doesn't contain %q, got: %v", tt.wantErrSubstr, err) } + + if p.Index != sentinel.Index || + len(p.Hashes) != len(sentinel.Hashes) || p.Hashes[0] != sentinel.Hashes[0] || + !bytes.Equal(p.Checkpoint, sentinel.Checkpoint) || + !bytes.Equal(p.ExtraData, sentinel.ExtraData) { + t.Errorf("Unmarshal mutated receiver on error: got %+v, want %+v", p, sentinel) + } }) } } @@ -231,6 +324,15 @@ func TestRoundTrip(t *testing.T) { ExtraData: []byte("some extra data"), }, }, + { + name: "large extra data (>64 KiB)", + proof: TLogProof{ + Index: 456, + Hashes: [][sha256.Size]byte{sha256.Sum256([]byte("c"))}, + Checkpoint: []byte("another checkpoint\n"), + ExtraData: bytes.Repeat([]byte("x"), 65*1024), + }, + }, { name: "empty hashes", proof: TLogProof{ @@ -247,6 +349,34 @@ func TestRoundTrip(t *testing.T) { Checkpoint: []byte("checkpoint\n"), }, }, + { + name: "multi-signature checkpoint with blank lines", + proof: TLogProof{ + Index: 73894, + Hashes: [][sha256.Size]byte{sha256.Sum256([]byte("leaf-sibling"))}, + Checkpoint: []byte("example.com/fancylog\n109482\nsFodV/vSp5O8n9a8QpW6PRY97tfOSW5bsc2Xl/EQi08=\n\n" + + "— example.com/fancylog hI2DJw1roloI=\n" + + "— witness1.example mJirIkljqY9v2B/5bg==\n"), + }, + }, + { + name: "checkpoint with trailing blank lines", + proof: TLogProof{ + Index: 1, + Hashes: nil, + Checkpoint: []byte("example.com/fancylog\n109482\nsFodV/vSp5O8n9a8QpW6PRY97tfOSW5bsc2Xl/EQi08=\n\n" + + "— example.com/fancylog hI2DJw1roloI=\n\n\n"), + }, + }, + { + name: "checkpoint without trailing newline", + proof: TLogProof{ + Index: 2, + Hashes: nil, + Checkpoint: []byte("example.com/fancylog\n109482\nsFodV/vSp5O8n9a8QpW6PRY97tfOSW5bsc2Xl/EQi08=\n\n" + + "— example.com/fancylog hI2DJw1roloI="), + }, + }, } for _, tt := range tests { @@ -278,3 +408,63 @@ func TestRoundTrip(t *testing.T) { }) } } + +func TestUnmarshalCheckpointDoesNotAliasInput(t *testing.T) { + raw := []byte("c2sp.org/tlog-proof@v1\nindex 0\n\nexample.com/log\n1\nhash==\n\n— example.com/log sig==\n") + p := TLogProof{ + ExtraData: []byte("stale-extra"), + } + if err := p.Unmarshal(raw); err != nil { + t.Fatalf("Unmarshal failed: %v", err) + } + if p.ExtraData != nil { + t.Errorf("ExtraData = %q, want nil when extra line is omitted", p.ExtraData) + } + + wantCP := bytes.Clone(p.Checkpoint) + // Mutate backing array of raw input. + for i := range raw { + raw[i] = 'X' + } + if !bytes.Equal(p.Checkpoint, wantCP) { + t.Errorf("Checkpoint aliased input buffer: got %q, want %q", p.Checkpoint, wantCP) + } +} + +func BenchmarkMarshal(b *testing.B) { + p := TLogProof{ + Index: 73894, + Hashes: [][sha256.Size]byte{ + sha256.Sum256([]byte("h1")), + sha256.Sum256([]byte("h2")), + sha256.Sum256([]byte("h3")), + sha256.Sum256([]byte("h4")), + }, + Checkpoint: []byte("example.com/fancylog\n109482\nsFodV/vSp5O8n9a8QpW6PRY97tfOSW5bsc2Xl/EQi08=\n\n— example.com/fancylog hI2DJw1roloI=\n"), + ExtraData: []byte("opaque application context"), + } + for b.Loop() { + _ = p.Marshal() + } +} + +func BenchmarkUnmarshal(b *testing.B) { + raw := TLogProof{ + Index: 73894, + Hashes: [][sha256.Size]byte{ + sha256.Sum256([]byte("h1")), + sha256.Sum256([]byte("h2")), + sha256.Sum256([]byte("h3")), + sha256.Sum256([]byte("h4")), + }, + Checkpoint: []byte("example.com/fancylog\n109482\nsFodV/vSp5O8n9a8QpW6PRY97tfOSW5bsc2Xl/EQi08=\n\n— example.com/fancylog hI2DJw1roloI=\n"), + ExtraData: []byte("opaque application context"), + }.Marshal() + + var p TLogProof + for b.Loop() { + if err := p.Unmarshal(raw); err != nil { + b.Fatal(err) + } + } +} From c9c5784f7f8b1c91ec8f7bc18b0e14088cd52cfe Mon Sep 17 00:00:00 2001 From: Martin Hutchinson Date: Tue, 17 Nov 2026 17:20:55 +0000 Subject: [PATCH 2/2] Simplify TLogProof.Unmarshal with strings.Cut and clearer variable names --- proof/tlog_proof.go | 31 +++++++++++++++---------------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/proof/tlog_proof.go b/proof/tlog_proof.go index e3f505e..a6d34af 100644 --- a/proof/tlog_proof.go +++ b/proof/tlog_proof.go @@ -20,6 +20,7 @@ import ( "encoding/base64" "fmt" "strconv" + "strings" ) const ( @@ -59,12 +60,12 @@ func (p *TLogProof) Unmarshal(data []byte) error { return fmt.Errorf("tlog proof contains carriage return") } - line, rem, ok := bytes.Cut(data, []byte("\n")) - if !ok || string(line) != tlogProofHeaderV1 { + line, rem, ok := strings.Cut(string(data), "\n") + if !ok || line != tlogProofHeaderV1 { return fmt.Errorf("tlog proof missing expected header") } - line, rem, ok = bytes.Cut(rem, []byte("\n")) + line, rem, ok = strings.Cut(rem, "\n") if !ok { return fmt.Errorf("tlog proof missing required index") } @@ -74,44 +75,42 @@ func (p *TLogProof) Unmarshal(data []byte) error { extra []byte err error ) - if after, hasExtra := bytes.CutPrefix(line, []byte("extra ")); hasExtra { - e := string(after) - if len(e) == 0 { + if extraData, hasExtra := strings.CutPrefix(line, "extra "); hasExtra { + if len(extraData) == 0 { return fmt.Errorf("tlog proof empty extra data") } - extra, err = base64.StdEncoding.Strict().DecodeString(e) + extra, err = base64.StdEncoding.Strict().DecodeString(extraData) if err != nil { return fmt.Errorf("tlog proof extra data not base64 encoded: %w", err) } - line, rem, ok = bytes.Cut(rem, []byte("\n")) + line, rem, ok = strings.Cut(rem, "\n") if !ok { return fmt.Errorf("tlog proof missing required index") } } - afterIdx, ok := bytes.CutPrefix(line, []byte("index ")) + idxData, ok := strings.CutPrefix(line, "index ") if !ok { return fmt.Errorf("tlog proof missing required index") } - idxStr := string(afterIdx) - if len(idxStr) > 1 && idxStr[0] == '0' { - return fmt.Errorf("tlog proof index has leading zeros: %q", idxStr) + if len(idxData) > 1 && idxData[0] == '0' { + return fmt.Errorf("tlog proof index has leading zeros: %q", idxData) } - idx, err := strconv.ParseUint(idxStr, 10, 63) + idx, err := strconv.ParseUint(idxData, 10, 63) if err != nil { return fmt.Errorf("tlog proof index not a valid int64: %w", err) } var hashes [][sha256.Size]byte for { - line, rem, ok = bytes.Cut(rem, []byte("\n")) + line, rem, ok = strings.Cut(rem, "\n") if !ok { return fmt.Errorf("tlog proof missing blank line before checkpoint") } if len(line) == 0 { break } - hash, err := base64.StdEncoding.Strict().DecodeString(string(line)) + hash, err := base64.StdEncoding.Strict().DecodeString(line) if err != nil { return fmt.Errorf("tlog proof hash not base64 encoded: %w", err) } @@ -127,7 +126,7 @@ func (p *TLogProof) Unmarshal(data []byte) error { p.Index = idx p.Hashes = hashes - p.Checkpoint = bytes.Clone(rem) + p.Checkpoint = []byte(rem) p.ExtraData = extra return nil