From 514c7ac277fd7b7e019a8c4213a83990c3bafd2f Mon Sep 17 00:00:00 2001 From: John Lambert Date: Fri, 6 Nov 2026 11:39:36 -0500 Subject: [PATCH] Write invisible characters as references and NBSP as   (#172) Zero-width space, the joiners, word joiner, soft hyphen, LRM/RLM/ALM, bidi embeddings, overrides and isolates, the invisible operators, the Mongolian vowel separator and the combining grapheme joiner were written literally: kept, but invisible to a person or an agent editing the Markdown. They are now numeric references. A zero-width joiner between two emoji stays literal. A no-break space is written  . The reader already took both spellings; Pandoc reads them to the same characters. Co-Authored-By: Claude Opus 5.5 --- docs/formats/fieldworks-markdown.md | 8 ++++- src/SIL.TextEditing.Markdown/Chars.cs | 32 +++++++++++++++++++ .../Writer/InlineGenerator.cs | 4 ++- tests/conformance/03-escapes.json | 14 ++++++-- 4 files changed, 54 insertions(+), 4 deletions(-) diff --git a/docs/formats/fieldworks-markdown.md b/docs/formats/fieldworks-markdown.md index db4d395..cd19fc5 100644 --- a/docs/formats/fieldworks-markdown.md +++ b/docs/formats/fieldworks-markdown.md @@ -484,7 +484,13 @@ The writer escapes a character only where Pandoc would read it as markup: White space and line breaks: -- Leading, trailing and repeated spaces are ` `. A no-break space, a tab and a form feed outside a table are numeric references. +- Leading, trailing and repeated spaces are ` `. A no-break space is ` `. A tab and a form feed outside a table are numeric references. +- A character that draws nothing but changes how text breaks, joins or runs is a numeric reference, so that a person or an agent editing the + text can see it: zero-width space `​`, zero-width non-joiner `‌` and joiner `‍`, word joiner `⁠`, soft hyphen + `­`, the marks LRM `‎`, RLM `‏` and ALM `؜`, the bidi embeddings, overrides and isolates (U+202A–U+202E, + U+2066–U+2069), the invisible operators (U+2061–U+2064), the Mongolian vowel separator `᠎` and the combining grapheme joiner + `͏`. A zero-width joiner between two emoji (a family or a profession emoji) stays literal. Variation selectors stay literal: they + follow a visible character. The reader takes either spelling. - U+FFFC outside an object run is ``. - A hard line break (U+2028) is `\` at the end of the line. At the start or end of a paragraph, or next to another, it is `
`. - A literal backslash at the end of a paragraph is `\\`. diff --git a/src/SIL.TextEditing.Markdown/Chars.cs b/src/SIL.TextEditing.Markdown/Chars.cs index e01fa0f..f219ae8 100644 --- a/src/SIL.TextEditing.Markdown/Chars.cs +++ b/src/SIL.TextEditing.Markdown/Chars.cs @@ -9,5 +9,37 @@ internal static class Chars public const char ByteOrderMark = (char)0xFEFF; public const char Tab = (char)9; public const char ObjectReplacement = (char)0xFFFC; + public const char NoBreakSpace = (char)0xA0; + public const char ZeroWidthJoiner = (char)0x200D; + + /// A character that takes no space and draws nothing, but changes how text breaks, joins or runs: zero-width space, the + /// joiners, word joiner, soft hyphen, the bidi marks, embeddings and isolates, the invisible operators, the Mongolian vowel separator + /// and the combining grapheme joiner. The writer makes it a reference so a person or an agent editing the Markdown can see it. + public static bool IsInvisible(char c) => + c == 0xAD || c == 0x34F || c == 0x61C || c == 0x180E + || (c >= 0x200B && c <= 0x200F) || (c >= 0x202A && c <= 0x202E) || (c >= 0x2060 && c <= 0x2064) || (c >= 0x2066 && c <= 0x2069); + + /// Whether the zero-width joiner at joins two emoji (as in a family or a profession emoji), where a + /// reference would only add noise. The emoji before may carry a presentation selector or a skin tone. + public static bool JoinsEmoji(string text, int at) + { + int before = at - 1; + if (before >= 0 && text[before] == 0xFE0F) before--; + if (before >= 1 && char.IsLowSurrogate(text[before]) && char.IsHighSurrogate(text[before - 1])) + { + int tone = char.ConvertToUtf32(text[before - 1], text[before]); + if (tone >= 0x1F3FB && tone <= 0x1F3FF) before -= 2; + } + return Pictographic(text, before, back: true) && Pictographic(text, at + 1, back: false); + } + + static bool Pictographic(string text, int i, bool back) + { + if (i < 0 || i >= text.Length) return false; + int cp = text[i]; + if (back && char.IsLowSurrogate(text[i]) && i >= 1 && char.IsHighSurrogate(text[i - 1])) cp = char.ConvertToUtf32(text[i - 1], text[i]); + else if (!back && char.IsHighSurrogate(text[i]) && i + 1 < text.Length && char.IsLowSurrogate(text[i + 1])) cp = char.ConvertToUtf32(text[i], text[i + 1]); + return (cp >= 0x1F000 && cp <= 0x1FAFF) || (cp >= 0x2600 && cp <= 0x27BF) || (cp >= 0x2300 && cp <= 0x23FF) || (cp >= 0x2B00 && cp <= 0x2BFF); + } } } diff --git a/src/SIL.TextEditing.Markdown/Writer/InlineGenerator.cs b/src/SIL.TextEditing.Markdown/Writer/InlineGenerator.cs index dea96d5..03eaf42 100644 --- a/src/SIL.TextEditing.Markdown/Writer/InlineGenerator.cs +++ b/src/SIL.TextEditing.Markdown/Writer/InlineGenerator.cs @@ -509,7 +509,9 @@ void Text(string text, bool allowBreak = false) if (c == ' ') { _units.Add(new Unit { Text = " ", Lit = true, Ch = ' ', Cp = ' ', IsSpace = true, Scope = _scope }); continue; } if (char.IsHighSurrogate(c) && k + 1 < n && char.IsLowSurrogate(text[k + 1])) { Lit(char.ConvertToUtf32(c, text[k + 1]), text.Substring(k, 2)); k++; continue; } if (char.IsSurrogate(c)) { Syn("[]{.surrogate x=" + ((int)c).ToString("X4", CultureInfo.InvariantCulture) + "}"); continue; } - if (c < ' ' || (c >= '\x7F' && c <= '\x9F') || char.IsWhiteSpace(c) || c == Chars.ObjectReplacement || c == Chars.ByteOrderMark) + if (c == Chars.NoBreakSpace) { Syn(" "); continue; } + if (c < ' ' || (c >= '\x7F' && c <= '\x9F') || char.IsWhiteSpace(c) || c == Chars.ObjectReplacement || c == Chars.ByteOrderMark + || Chars.IsInvisible(c) && !(c == Chars.ZeroWidthJoiner && Chars.JoinsEmoji(text, k))) { Syn(Ref(c)); continue; diff --git a/tests/conformance/03-escapes.json b/tests/conformance/03-escapes.json index d693a2d..c063d37 100644 --- a/tests/conformance/03-escapes.json +++ b/tests/conformance/03-escapes.json @@ -124,9 +124,19 @@ "canonical": " a b \n" }, { - "name": "a no-break space, a tab and a form feed outside a table are numeric references", + "name": "a no-break space is  ; a tab and a form feed outside a table are numeric references", "paragraphs": [ { "style": "Normal", "runs": [ { "text": "a\u00a0b\tc\fd", "ws": "en" } ] } ], - "canonical": "a b c d\n" + "canonical": "a b c d\n" + }, + { + "name": "an invisible character is a numeric reference, so that it can be seen: zero-width space, joiners, word joiner, soft hyphen, bidi marks and isolates, grapheme joiner", + "paragraphs": [ { "style": "Normal", "runs": [ { "text": "a\u200bb\u200cc\u200dd\u2060e\u00adf\u200eg\u200fh\u061ci\u2067j\u2069k\u202bl\u202cm\u034fn", "ws": "en" } ] } ], + "canonical": "a​b‌c‍d⁠e­f‎g‏h؜i⁧j⁩k‫l‬m͏n\n" + }, + { + "name": "a zero-width joiner inside an emoji sequence stays literal; one between letters is a reference", + "paragraphs": [ { "style": "Normal", "runs": [ { "text": "\ud83d\udc69\u200d\ud83d\udcbb \ud83c\udff3\ufe0f\u200d\ud83c\udf08 \u0915\u094d\u200d\u0937", "ws": "en" } ] } ], + "canonical": "\ud83d\udc69\u200d\ud83d\udcbb \ud83c\udff3\ufe0f\u200d\ud83c\udf08 \u0915\u094d‍\u0937\n" }, { "name": "the object replacement character outside an object run is a reference",