Write invisible characters as references and NBSP as   (#172)

docs/formats/fieldworks-markdown.md

@@ -484,7 +484,13 @@ The writer escapes a character only where Pandoc would read it as markup:
 
 White space and line breaks:
 
-- Leading, trailing and repeated spaces are ` `. A no-break space, a tab and a form feed outside a table are numeric references.
+- Leading, trailing and repeated spaces are ` `. A no-break space is ` `. A tab and a form feed outside a table are numeric references.
+- A character that draws nothing but changes how text breaks, joins or runs is a numeric reference, so that a person or an agent editing the
+  text can see it: zero-width space `​`, zero-width non-joiner `‌` and joiner `‍`, word joiner `⁠`, soft hyphen
+  `­`, the marks LRM `‎`, RLM `‏` and ALM `؜`, the bidi embeddings, overrides and isolates (U+202A–U+202E,
+  U+2066–U+2069), the invisible operators (U+2061–U+2064), the Mongolian vowel separator `᠎` and the combining grapheme joiner
+  `͏`. A zero-width joiner between two emoji (a family or a profession emoji) stays literal. Variation selectors stay literal: they
+  follow a visible character. The reader takes either spelling.
 - U+FFFC outside an object run is ``.
 - A hard line break (U+2028) is `\` at the end of the line. At the start or end of a paragraph, or next to another, it is `
`.
 - A literal backslash at the end of a paragraph is `\\`.

src/SIL.TextEditing.Markdown/Chars.cs

@@ -9,5 +9,37 @@ internal static class Chars
         public const char ByteOrderMark = (char)0xFEFF;
         public const char Tab = (char)9;
         public const char ObjectReplacement = (char)0xFFFC;
+        public const char NoBreakSpace = (char)0xA0;
+        public const char ZeroWidthJoiner = (char)0x200D;
+
+        /// <summary>A character that takes no space and draws nothing, but changes how text breaks, joins or runs: zero-width space, the
+        /// joiners, word joiner, soft hyphen, the bidi marks, embeddings and isolates, the invisible operators, the Mongolian vowel separator
+        /// and the combining grapheme joiner. The writer makes it a reference so a person or an agent editing the Markdown can see it.</summary>
+        public static bool IsInvisible(char c) =>
+            c == 0xAD || c == 0x34F || c == 0x61C || c == 0x180E
+            || (c >= 0x200B && c <= 0x200F) || (c >= 0x202A && c <= 0x202E) || (c >= 0x2060 && c <= 0x2064) || (c >= 0x2066 && c <= 0x2069);
+
+        /// <summary>Whether the zero-width joiner at <paramref name="at"/> joins two emoji (as in a family or a profession emoji), where a
+        /// reference would only add noise. The emoji before may carry a presentation selector or a skin tone.</summary>
+        public static bool JoinsEmoji(string text, int at)
+        {
+            int before = at - 1;
+            if (before >= 0 && text[before] == 0xFE0F) before--;
+            if (before >= 1 && char.IsLowSurrogate(text[before]) && char.IsHighSurrogate(text[before - 1]))
+            {
+                int tone = char.ConvertToUtf32(text[before - 1], text[before]);
+                if (tone >= 0x1F3FB && tone <= 0x1F3FF) before -= 2;
+            }
+            return Pictographic(text, before, back: true) && Pictographic(text, at + 1, back: false);
+        }
+
+        static bool Pictographic(string text, int i, bool back)
+        {
+            if (i < 0 || i >= text.Length) return false;
+            int cp = text[i];
+            if (back && char.IsLowSurrogate(text[i]) && i >= 1 && char.IsHighSurrogate(text[i - 1])) cp = char.ConvertToUtf32(text[i - 1], text[i]);
+            else if (!back && char.IsHighSurrogate(text[i]) && i + 1 < text.Length && char.IsLowSurrogate(text[i + 1])) cp = char.ConvertToUtf32(text[i], text[i + 1]);
+            return (cp >= 0x1F000 && cp <= 0x1FAFF) || (cp >= 0x2600 && cp <= 0x27BF) || (cp >= 0x2300 && cp <= 0x23FF) || (cp >= 0x2B00 && cp <= 0x2BFF);
+        }
     }
 }

src/SIL.TextEditing.Markdown/Writer/InlineGenerator.cs

@@ -509,7 +509,9 @@ void Text(string text, bool allowBreak = false)
                 if (c == ' ') { _units.Add(new Unit { Text = " ", Lit = true, Ch = ' ', Cp = ' ', IsSpace = true, Scope = _scope }); continue; }
                 if (char.IsHighSurrogate(c) && k + 1 < n && char.IsLowSurrogate(text[k + 1])) { Lit(char.ConvertToUtf32(c, text[k + 1]), text.Substring(k, 2)); k++; continue; }
                 if (char.IsSurrogate(c)) { Syn("[]{.surrogate x=" + ((int)c).ToString("X4", CultureInfo.InvariantCulture) + "}"); continue; }
-                if (c < ' ' || (c >= '\x7F' && c <= '\x9F') || char.IsWhiteSpace(c) || c == Chars.ObjectReplacement || c == Chars.ByteOrderMark)
+                if (c == Chars.NoBreakSpace) { Syn("&nbsp;"); continue; }
+                if (c < ' ' || (c >= '\x7F' && c <= '\x9F') || char.IsWhiteSpace(c) || c == Chars.ObjectReplacement || c == Chars.ByteOrderMark
+                    || Chars.IsInvisible(c) && !(c == Chars.ZeroWidthJoiner && Chars.JoinsEmoji(text, k)))
                 {
                     Syn(Ref(c));
                     continue;

tests/conformance/03-escapes.json

@@ -124,9 +124,19 @@
     "canonical": "  a  b \n"
   },
   {
-    "name": "a no-break space, a tab and a form feed outside a table are numeric references",
+    "name": "a no-break space is  ; a tab and a form feed outside a table are numeric references",
     "paragraphs": [ { "style": "Normal", "runs": [ { "text": "a\u00a0b\tc\fd", "ws": "en" } ] } ],
-    "canonical": "a b	cd\n"
+    "canonical": "a b	cd\n"
+  },
+  {
+    "name": "an invisible character is a numeric reference, so that it can be seen: zero-width space, joiners, word joiner, soft hyphen, bidi marks and isolates, grapheme joiner",
+    "paragraphs": [ { "style": "Normal", "runs": [ { "text": "a\u200bb\u200cc\u200dd\u2060e\u00adf\u200eg\u200fh\u061ci\u2067j\u2069k\u202bl\u202cm\u034fn", "ws": "en" } ] } ],
+    "canonical": "a​b‌c‍d⁠e­f‎g‏h؜i⁧j⁩k‫l‬m͏n\n"
+  },
+  {
+    "name": "a zero-width joiner inside an emoji sequence stays literal; one between letters is a reference",
+    "paragraphs": [ { "style": "Normal", "runs": [ { "text": "\ud83d\udc69\u200d\ud83d\udcbb \ud83c\udff3\ufe0f\u200d\ud83c\udf08 \u0915\u094d\u200d\u0937", "ws": "en" } ] } ],
+    "canonical": "\ud83d\udc69\u200d\ud83d\udcbb \ud83c\udff3\ufe0f\u200d\ud83c\udf08 \u0915\u094d‍\u0937\n"
   },
   {
     "name": "the object replacement character outside an object run is a reference",