From faf0a56c511a748b0e481ebc7377d7c4ae46ef85 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 11 Nov 2026 23:30:26 +0000 Subject: [PATCH] Find metadata values the file writes with JSON escapes [patch] FindLocation searched the raw file text for the value a generator passes, which it got back from Deserialize and is therefore unescaped. A value the file writes with an escape, such as "C:\\temp", "say \"hi\"" or the "\u03A9" System.Text.Json writes for non-ASCII, was never found, and the scoped overload fell back to the wrong entry. Each string literal is now decoded and compared with the value, and the location covers the literal's escaped contents. Quotes inside // and /* */ comments no longer open a string. The raw search stays as the fallback for text that isn't a whole string, such as a number. Fixes ktsu-dev/SourceGeneratorToolkit#68 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_017vfJW2bqRiw92MR91EGjva --- .../MetadataFileTests.cs | 99 +++++++ SourceGeneratorToolkit/MetadataFile.cs | 245 +++++++++++++++++- 2 files changed, 330 insertions(+), 14 deletions(-) diff --git a/SourceGeneratorToolkit.Test/MetadataFileTests.cs b/SourceGeneratorToolkit.Test/MetadataFileTests.cs index 96b779c..c0ae1d8 100644 --- a/SourceGeneratorToolkit.Test/MetadataFileTests.cs +++ b/SourceGeneratorToolkit.Test/MetadataFileTests.cs @@ -222,6 +222,105 @@ public void ScopedFindLocationSkipsNestedObjectsAndBracesInStringsBeforeTheAncho Assert.AreEqual(2, location.GetLineSpan().StartLinePosition.Line); } + private const string EscapedValues = """ + { + "things": [ + { "name": "Alpha", "pattern": "C:\\temp" }, + { "name": "Caf\u00e9" }, + { "name": "say \"hi\"" }, + { "name": "\u03A9\u00B7m\u00b2", "pattern": "C:\\temp" } + ] + } + """; + + private static MetadataFile CreateEscaped() => + new("things.json", EscapedValues, SourceText.From(EscapedValues), "/repo/things.json"); + + private static string TextAt(Location location) => + EscapedValues.Substring(location.SourceSpan.Start, location.SourceSpan.Length); + + [TestMethod] + public void FindLocationFindsAValueWrittenWithAnEscapedBackslash() + { + MetadataFile file = CreateEscaped(); + + Location location = file.FindLocation(@"C:\temp"); + + Assert.AreEqual(2, location.GetLineSpan().StartLinePosition.Line); + Assert.AreEqual(@"C:\\temp", TextAt(location)); + } + + [TestMethod] + public void FindLocationFindsAValueWrittenWithAnEscapedQuote() + { + MetadataFile file = CreateEscaped(); + + Location location = file.FindLocation("say \"hi\""); + + Assert.AreEqual(4, location.GetLineSpan().StartLinePosition.Line); + Assert.AreEqual(@"say \""hi\""", TextAt(location)); + } + + [TestMethod] + public void FindLocationFindsAValueWrittenWithLowercaseUnicodeEscapes() + { + MetadataFile file = CreateEscaped(); + + Location location = file.FindLocation("Café"); + + Assert.AreEqual(3, location.GetLineSpan().StartLinePosition.Line); + Assert.AreEqual(@"Caf\u00e9", TextAt(location)); + } + + [TestMethod] + public void FindLocationFindsAValueWrittenWithMixedCaseUnicodeEscapes() + { + MetadataFile file = CreateEscaped(); + + Location location = file.FindLocation("Ω·m²"); + + Assert.AreEqual(5, location.GetLineSpan().StartLinePosition.Line); + Assert.AreEqual(@"\u03A9\u00B7m\u00b2", TextAt(location)); + } + + [TestMethod] + public void ScopedFindLocationFindsAnEscapedNeedleInTheAnchorsEntry() + { + MetadataFile file = CreateEscaped(); + + Location location = file.FindLocation("Ω·m²", @"C:\temp"); + + Assert.AreEqual(5, location.GetLineSpan().StartLinePosition.Line); + Assert.AreEqual(@"C:\\temp", TextAt(location)); + } + + [TestMethod] + public void ScopedFindLocationFindsAPlainNeedleAfterAnEscapedAnchor() + { + MetadataFile file = CreateEscaped(); + + Location location = file.FindLocation("Alpha", @"C:\temp"); + + Assert.AreEqual(2, location.GetLineSpan().StartLinePosition.Line); + Assert.AreEqual(@"C:\\temp", TextAt(location)); + } + + [TestMethod] + public void FindLocationIgnoresAQuoteInsideAComment() + { + const string text = """ + { + // the "old" name + "name": "old" + } + """; + MetadataFile file = new("things.json", text, SourceText.From(text), "/repo/things.json"); + + Location location = file.FindLocation("old"); + + Assert.AreEqual(2, location.GetLineSpan().StartLinePosition.Line); + } + [TestMethod] public void ScopedFindLocationReturnsNoneForAnEmptyNeedle() { diff --git a/SourceGeneratorToolkit/MetadataFile.cs b/SourceGeneratorToolkit/MetadataFile.cs index 614b697..05ad099 100644 --- a/SourceGeneratorToolkit/MetadataFile.cs +++ b/SourceGeneratorToolkit/MetadataFile.cs @@ -103,8 +103,8 @@ public Location FindLocation(string needle) return Location.None; } - int index = IndexOfToken(needle, 0); - return index < 0 ? Location.None : LocationAt(index, needle.Length); + int index = IndexOfToken(needle, 0, out int length); + return index < 0 ? Location.None : LocationAt(index, length); } /// @@ -138,28 +138,29 @@ public Location FindLocation(string anchor, string needle) return Location.None; } - int scope = string.IsNullOrEmpty(anchor) ? 0 : IndexOfToken(anchor, 0); + int scope = string.IsNullOrEmpty(anchor) ? 0 : IndexOfToken(anchor, 0, out _); if (scope < 0) { return FindLocation(needle); } int index = -1; + int length = 0; if (TryGetEnclosingObject(scope, out int open, out int close)) { - index = IndexOfToken(needle, scope, close); + index = IndexOfToken(needle, scope, close, out length); if (index < 0) { - index = IndexOfToken(needle, open, scope); + index = IndexOfToken(needle, open, scope, out length); } } if (index < 0) { - index = IndexOfToken(needle, scope); + index = IndexOfToken(needle, scope, out length); } - return index < 0 ? FindLocation(needle) : LocationAt(index, needle.Length); + return index < 0 ? FindLocation(needle) : LocationAt(index, length); } /// @@ -246,14 +247,15 @@ private bool TryGetEnclosingObject(int index, out int open, out int close) /// /// The text to find. /// Where the search starts. + /// The length of the match in the file. /// The index of the text itself, inside its quotes when it was found quoted, or -1. /// /// A plain substring search lands inside longer names: Meter is found in - /// "SquareMeter", which is a different entry. Looking for "Meter" first matches + /// "SquareMeter", which is a different entry. Matching whole JSON strings first finds /// only the whole string. The raw search remains the fallback so that text which is not a JSON /// string on its own, such as a number or an already quoted anchor, is still found. /// - private int IndexOfToken(string value, int startIndex) => IndexOfToken(value, startIndex, Text.Length); + private int IndexOfToken(string value, int startIndex, out int length) => IndexOfToken(value, startIndex, Text.Length, out length); /// /// Finds between and @@ -262,17 +264,232 @@ private bool TryGetEnclosingObject(int index, out int open, out int close) /// The text to find. /// Where the search starts. /// Where the search ends; a match must lie wholly before it. + /// + /// The length of the match in the file. For a JSON string it is the length of the literal's + /// escaped contents, which can differ from the length of . + /// /// The index of the text itself, inside its quotes when it was found quoted, or -1. - private int IndexOfToken(string value, int startIndex, int endIndex) + /// + /// A generator passes a value it got back from deserializing the file, which is unescaped, while + /// the file holds the escaped JSON form: C:\temp is written "C:\\temp", and + /// System.Text.Json writes Ω as "\u03A9". So each string literal is decoded and + /// compared with the value, rather than the value being searched for in the raw text (#68). + /// + private int IndexOfToken(string value, int startIndex, int endIndex, out int length) { - int count = endIndex - startIndex; - if (count <= 0) + length = 0; + if (endIndex - startIndex <= 0) { return -1; } - int quoted = Text.IndexOf("\"" + value + "\"", startIndex, count, StringComparison.Ordinal); - return quoted >= 0 ? quoted + 1 : Text.IndexOf(value, startIndex, count, StringComparison.Ordinal); + foreach (StringLiteral literal in GetStringLiterals()) + { + if (literal.OpeningQuote < startIndex) + { + continue; + } + + if (literal.ClosingQuote >= endIndex) + { + break; + } + + if (string.Equals(literal.Value, value, StringComparison.Ordinal)) + { + length = literal.ClosingQuote - literal.OpeningQuote - 1; + return literal.OpeningQuote + 1; + } + } + + int raw = Text.IndexOf(value, startIndex, endIndex - startIndex, StringComparison.Ordinal); + length = raw < 0 ? 0 : value.Length; + return raw; + } + + /// + /// A string literal in the file: where its quotes are and what it decodes to. + /// + /// The index of the opening quote. + /// The index of the closing quote. + /// + /// The decoded string, or when the literal holds an escape JSON does not allow. + /// + private readonly record struct StringLiteral(int OpeningQuote, int ClosingQuote, string? Value); + + private List? stringLiterals; + + /// + /// Gets the file's string literals, in order, scanning the file the first time they are needed. + /// + /// The closed string literals outside comments. + /// + /// The file is scanned from the start, as in , because only a + /// forward scan can tell which quotes open a string. // and /* */ comments are skipped, + /// since accepts them and a quote inside one opens no string. + /// An unclosed literal at the end of the file is left out. + /// + private List GetStringLiterals() + { + if (stringLiterals is not null) + { + return stringLiterals; + } + + List literals = []; + int i = 0; + while (i < Text.Length) + { + if (Text[i] == '"') + { + int close = ClosingQuoteOf(i); + if (close < 0) + { + break; + } + + literals.Add(new StringLiteral(i, close, Unescape(i + 1, close))); + i = close + 1; + } + else + { + i = PastComment(i); + } + } + + stringLiterals = literals; + return literals; + } + + /// + /// Moves past a // or /* */ comment starting at , or past one + /// character when none starts there. + /// + /// A position outside any string. + /// The index after the comment, or after the character. + private int PastComment(int index) + { + if (Text[index] != '/' || index + 1 >= Text.Length) + { + return index + 1; + } + + if (Text[index + 1] == '/') + { + int lineFeed = Text.IndexOf('\n', index); + return lineFeed < 0 ? Text.Length : lineFeed + 1; + } + + if (Text[index + 1] == '*') + { + int end = Text.IndexOf("*/", index + 2, StringComparison.Ordinal); + return end < 0 ? Text.Length : end + 2; + } + + return index + 1; + } + + /// + /// Finds the quote that closes the string opened at . + /// + /// The index of the opening quote. + /// The index of the closing quote, or -1 when the string is not closed. + private int ClosingQuoteOf(int openingQuote) + { + int i = openingQuote + 1; + while (i < Text.Length) + { + if (Text[i] == '"') + { + return i; + } + + // A backslash escapes the character after it, so an escaped quote does not close the string. + i += Text[i] == '\\' ? 2 : 1; + } + + return -1; + } + + /// + /// Decodes the contents of a JSON string literal. + /// + /// The index of the first character after the opening quote. + /// The index of the closing quote. + /// The decoded string, or when it holds an escape JSON does not allow. + private string? Unescape(int start, int end) + { + int backslash = Text.IndexOf('\\', start, end - start); + if (backslash < 0) + { + return Text[start..end]; + } + + System.Text.StringBuilder decoded = new(end - start); + decoded.Append(Text, start, backslash - start); + int i = backslash; + while (i < end) + { + if (Text[i] != '\\') + { + decoded.Append(Text[i]); + i++; + } + else if (!TryAppendEscape(i, end, decoded, out i)) + { + return null; + } + } + + return decoded.ToString(); + } + + /// + /// Decodes the escape sequence whose backslash is at . + /// + /// The index of the backslash. + /// The index of the closing quote. + /// The decoded text, which the escaped character is appended to. + /// The index after the escape sequence. + /// when the escape is one JSON does not allow. + private bool TryAppendEscape(int backslash, int end, System.Text.StringBuilder decoded, out int next) + { + next = backslash + 2; + if (next > end) + { + return false; + } + + char escape = Text[backslash + 1]; + if (escape == 'u') + { + next = backslash + 6; + if (next > end || !ushort.TryParse(Text.Substring(backslash + 2, 4), System.Globalization.NumberStyles.AllowHexSpecifier, System.Globalization.CultureInfo.InvariantCulture, out ushort code)) + { + return false; + } + + decoded.Append((char)code); + return true; + } + + char? unescaped = escape switch + { + '"' or '\\' or '/' => escape, + 'b' => '\b', + 'f' => '\f', + 'n' => '\n', + 'r' => '\r', + 't' => '\t', + _ => null, + }; + if (unescaped is not char simple) + { + return false; + } + + decoded.Append(simple); + return true; } ///