diff --git a/SourceGeneratorToolkit.Test/MetadataFileTests.cs b/SourceGeneratorToolkit.Test/MetadataFileTests.cs index 96b779c..c0ae1d8 100644 --- a/SourceGeneratorToolkit.Test/MetadataFileTests.cs +++ b/SourceGeneratorToolkit.Test/MetadataFileTests.cs @@ -222,6 +222,105 @@ public void ScopedFindLocationSkipsNestedObjectsAndBracesInStringsBeforeTheAncho Assert.AreEqual(2, location.GetLineSpan().StartLinePosition.Line); } + private const string EscapedValues = """ + { + "things": [ + { "name": "Alpha", "pattern": "C:\\temp" }, + { "name": "Caf\u00e9" }, + { "name": "say \"hi\"" }, + { "name": "\u03A9\u00B7m\u00b2", "pattern": "C:\\temp" } + ] + } + """; + + private static MetadataFile CreateEscaped() => + new("things.json", EscapedValues, SourceText.From(EscapedValues), "/repo/things.json"); + + private static string TextAt(Location location) => + EscapedValues.Substring(location.SourceSpan.Start, location.SourceSpan.Length); + + [TestMethod] + public void FindLocationFindsAValueWrittenWithAnEscapedBackslash() + { + MetadataFile file = CreateEscaped(); + + Location location = file.FindLocation(@"C:\temp"); + + Assert.AreEqual(2, location.GetLineSpan().StartLinePosition.Line); + Assert.AreEqual(@"C:\\temp", TextAt(location)); + } + + [TestMethod] + public void FindLocationFindsAValueWrittenWithAnEscapedQuote() + { + MetadataFile file = CreateEscaped(); + + Location location = file.FindLocation("say \"hi\""); + + Assert.AreEqual(4, location.GetLineSpan().StartLinePosition.Line); + Assert.AreEqual(@"say \""hi\""", TextAt(location)); + } + + [TestMethod] + public void FindLocationFindsAValueWrittenWithLowercaseUnicodeEscapes() + { + MetadataFile file = CreateEscaped(); + + Location location = file.FindLocation("Café"); + + Assert.AreEqual(3, location.GetLineSpan().StartLinePosition.Line); + Assert.AreEqual(@"Caf\u00e9", TextAt(location)); + } + + [TestMethod] + public void FindLocationFindsAValueWrittenWithMixedCaseUnicodeEscapes() + { + MetadataFile file = CreateEscaped(); + + Location location = file.FindLocation("Ω·m²"); + + Assert.AreEqual(5, location.GetLineSpan().StartLinePosition.Line); + Assert.AreEqual(@"\u03A9\u00B7m\u00b2", TextAt(location)); + } + + [TestMethod] + public void ScopedFindLocationFindsAnEscapedNeedleInTheAnchorsEntry() + { + MetadataFile file = CreateEscaped(); + + Location location = file.FindLocation("Ω·m²", @"C:\temp"); + + Assert.AreEqual(5, location.GetLineSpan().StartLinePosition.Line); + Assert.AreEqual(@"C:\\temp", TextAt(location)); + } + + [TestMethod] + public void ScopedFindLocationFindsAPlainNeedleAfterAnEscapedAnchor() + { + MetadataFile file = CreateEscaped(); + + Location location = file.FindLocation("Alpha", @"C:\temp"); + + Assert.AreEqual(2, location.GetLineSpan().StartLinePosition.Line); + Assert.AreEqual(@"C:\\temp", TextAt(location)); + } + + [TestMethod] + public void FindLocationIgnoresAQuoteInsideAComment() + { + const string text = """ + { + // the "old" name + "name": "old" + } + """; + MetadataFile file = new("things.json", text, SourceText.From(text), "/repo/things.json"); + + Location location = file.FindLocation("old"); + + Assert.AreEqual(2, location.GetLineSpan().StartLinePosition.Line); + } + [TestMethod] public void ScopedFindLocationReturnsNoneForAnEmptyNeedle() { diff --git a/SourceGeneratorToolkit/MetadataFile.cs b/SourceGeneratorToolkit/MetadataFile.cs index 614b697..05ad099 100644 --- a/SourceGeneratorToolkit/MetadataFile.cs +++ b/SourceGeneratorToolkit/MetadataFile.cs @@ -103,8 +103,8 @@ public Location FindLocation(string needle) return Location.None; } - int index = IndexOfToken(needle, 0); - return index < 0 ? Location.None : LocationAt(index, needle.Length); + int index = IndexOfToken(needle, 0, out int length); + return index < 0 ? Location.None : LocationAt(index, length); } /// @@ -138,28 +138,29 @@ public Location FindLocation(string anchor, string needle) return Location.None; } - int scope = string.IsNullOrEmpty(anchor) ? 0 : IndexOfToken(anchor, 0); + int scope = string.IsNullOrEmpty(anchor) ? 0 : IndexOfToken(anchor, 0, out _); if (scope < 0) { return FindLocation(needle); } int index = -1; + int length = 0; if (TryGetEnclosingObject(scope, out int open, out int close)) { - index = IndexOfToken(needle, scope, close); + index = IndexOfToken(needle, scope, close, out length); if (index < 0) { - index = IndexOfToken(needle, open, scope); + index = IndexOfToken(needle, open, scope, out length); } } if (index < 0) { - index = IndexOfToken(needle, scope); + index = IndexOfToken(needle, scope, out length); } - return index < 0 ? FindLocation(needle) : LocationAt(index, needle.Length); + return index < 0 ? FindLocation(needle) : LocationAt(index, length); } /// @@ -246,14 +247,15 @@ private bool TryGetEnclosingObject(int index, out int open, out int close) /// /// The text to find. /// Where the search starts. + /// The length of the match in the file. /// The index of the text itself, inside its quotes when it was found quoted, or -1. /// /// A plain substring search lands inside longer names: Meter is found in - /// "SquareMeter", which is a different entry. Looking for "Meter" first matches + /// "SquareMeter", which is a different entry. Matching whole JSON strings first finds /// only the whole string. The raw search remains the fallback so that text which is not a JSON /// string on its own, such as a number or an already quoted anchor, is still found. /// - private int IndexOfToken(string value, int startIndex) => IndexOfToken(value, startIndex, Text.Length); + private int IndexOfToken(string value, int startIndex, out int length) => IndexOfToken(value, startIndex, Text.Length, out length); /// /// Finds between and @@ -262,17 +264,232 @@ private bool TryGetEnclosingObject(int index, out int open, out int close) /// The text to find. /// Where the search starts. /// Where the search ends; a match must lie wholly before it. + /// + /// The length of the match in the file. For a JSON string it is the length of the literal's + /// escaped contents, which can differ from the length of . + /// /// The index of the text itself, inside its quotes when it was found quoted, or -1. - private int IndexOfToken(string value, int startIndex, int endIndex) + /// + /// A generator passes a value it got back from deserializing the file, which is unescaped, while + /// the file holds the escaped JSON form: C:\temp is written "C:\\temp", and + /// System.Text.Json writes Ω as "\u03A9". So each string literal is decoded and + /// compared with the value, rather than the value being searched for in the raw text (#68). + /// + private int IndexOfToken(string value, int startIndex, int endIndex, out int length) { - int count = endIndex - startIndex; - if (count <= 0) + length = 0; + if (endIndex - startIndex <= 0) { return -1; } - int quoted = Text.IndexOf("\"" + value + "\"", startIndex, count, StringComparison.Ordinal); - return quoted >= 0 ? quoted + 1 : Text.IndexOf(value, startIndex, count, StringComparison.Ordinal); + foreach (StringLiteral literal in GetStringLiterals()) + { + if (literal.OpeningQuote < startIndex) + { + continue; + } + + if (literal.ClosingQuote >= endIndex) + { + break; + } + + if (string.Equals(literal.Value, value, StringComparison.Ordinal)) + { + length = literal.ClosingQuote - literal.OpeningQuote - 1; + return literal.OpeningQuote + 1; + } + } + + int raw = Text.IndexOf(value, startIndex, endIndex - startIndex, StringComparison.Ordinal); + length = raw < 0 ? 0 : value.Length; + return raw; + } + + /// + /// A string literal in the file: where its quotes are and what it decodes to. + /// + /// The index of the opening quote. + /// The index of the closing quote. + /// + /// The decoded string, or when the literal holds an escape JSON does not allow. + /// + private readonly record struct StringLiteral(int OpeningQuote, int ClosingQuote, string? Value); + + private List? stringLiterals; + + /// + /// Gets the file's string literals, in order, scanning the file the first time they are needed. + /// + /// The closed string literals outside comments. + /// + /// The file is scanned from the start, as in , because only a + /// forward scan can tell which quotes open a string. // and /* */ comments are skipped, + /// since accepts them and a quote inside one opens no string. + /// An unclosed literal at the end of the file is left out. + /// + private List GetStringLiterals() + { + if (stringLiterals is not null) + { + return stringLiterals; + } + + List literals = []; + int i = 0; + while (i < Text.Length) + { + if (Text[i] == '"') + { + int close = ClosingQuoteOf(i); + if (close < 0) + { + break; + } + + literals.Add(new StringLiteral(i, close, Unescape(i + 1, close))); + i = close + 1; + } + else + { + i = PastComment(i); + } + } + + stringLiterals = literals; + return literals; + } + + /// + /// Moves past a // or /* */ comment starting at , or past one + /// character when none starts there. + /// + /// A position outside any string. + /// The index after the comment, or after the character. + private int PastComment(int index) + { + if (Text[index] != '/' || index + 1 >= Text.Length) + { + return index + 1; + } + + if (Text[index + 1] == '/') + { + int lineFeed = Text.IndexOf('\n', index); + return lineFeed < 0 ? Text.Length : lineFeed + 1; + } + + if (Text[index + 1] == '*') + { + int end = Text.IndexOf("*/", index + 2, StringComparison.Ordinal); + return end < 0 ? Text.Length : end + 2; + } + + return index + 1; + } + + /// + /// Finds the quote that closes the string opened at . + /// + /// The index of the opening quote. + /// The index of the closing quote, or -1 when the string is not closed. + private int ClosingQuoteOf(int openingQuote) + { + int i = openingQuote + 1; + while (i < Text.Length) + { + if (Text[i] == '"') + { + return i; + } + + // A backslash escapes the character after it, so an escaped quote does not close the string. + i += Text[i] == '\\' ? 2 : 1; + } + + return -1; + } + + /// + /// Decodes the contents of a JSON string literal. + /// + /// The index of the first character after the opening quote. + /// The index of the closing quote. + /// The decoded string, or when it holds an escape JSON does not allow. + private string? Unescape(int start, int end) + { + int backslash = Text.IndexOf('\\', start, end - start); + if (backslash < 0) + { + return Text[start..end]; + } + + System.Text.StringBuilder decoded = new(end - start); + decoded.Append(Text, start, backslash - start); + int i = backslash; + while (i < end) + { + if (Text[i] != '\\') + { + decoded.Append(Text[i]); + i++; + } + else if (!TryAppendEscape(i, end, decoded, out i)) + { + return null; + } + } + + return decoded.ToString(); + } + + /// + /// Decodes the escape sequence whose backslash is at . + /// + /// The index of the backslash. + /// The index of the closing quote. + /// The decoded text, which the escaped character is appended to. + /// The index after the escape sequence. + /// when the escape is one JSON does not allow. + private bool TryAppendEscape(int backslash, int end, System.Text.StringBuilder decoded, out int next) + { + next = backslash + 2; + if (next > end) + { + return false; + } + + char escape = Text[backslash + 1]; + if (escape == 'u') + { + next = backslash + 6; + if (next > end || !ushort.TryParse(Text.Substring(backslash + 2, 4), System.Globalization.NumberStyles.AllowHexSpecifier, System.Globalization.CultureInfo.InvariantCulture, out ushort code)) + { + return false; + } + + decoded.Append((char)code); + return true; + } + + char? unescaped = escape switch + { + '"' or '\\' or '/' => escape, + 'b' => '\b', + 'f' => '\f', + 'n' => '\n', + 'r' => '\r', + 't' => '\t', + _ => null, + }; + if (unescaped is not char simple) + { + return false; + } + + decoded.Append(simple); + return true; } ///