fix: sixty, seventy normalizes to 60 70 (issue #42 DoD)

normalization/constants/protectors.py

@@ -10,6 +10,9 @@ class ProtectPlaceholder(str, Enum):
     UNIT_SLASH = "†"
     UNIT_DECIMAL = "‡"
     NUMBER_SEPARATOR = "¤"
+    # Comma between multi-digit numbers ("60, 70"): kept until after digit
+    # collapsing so the groups are not glued into "6070", then restored as space.
+    NUMBER_LIST_COMMA = "¶"
     SPELLING_SUFFIX = "xltrx"
 
 

normalization/pipeline/base.py

@@ -83,7 +83,10 @@ def validate(self) -> None:
             "protect_time_colon": {"restore_time_colon"},
             "protect_unit_decimal": {"restore_unit_decimal_as_word"},
             "protect_unit_slash": {"restore_unit_slash"},
-            "protect_number_separator_commas": {"remove_number_separator_markers"},
+            "protect_number_separator_commas": {
+                "remove_number_separator_markers",
+                "restore_number_list_commas",
+            },
             "protect_decimal_separator": {"restore_decimal_separator_with_word"},
             "protect_hyphenated_letter_spelling": {"remove_letter_spelling_markers"},
             "protect_space_separated_letter_spelling": {

normalization/presets/gladia-3.yaml

@@ -122,3 +122,6 @@ stages:
     # Final digit-collapse pass: late restores above (phone plus, unit slash)
     # can produce new adjacent-digit sequences that need collapsing.
     - remove_spaces_between_adjacent_digits # 3rd run: after late restores
+    # After every digit-collapse pass: turn ¶ list commas into spaces so
+    # "sixty, seventy" / "60, 70" stay "60 70" instead of "6070".
+    - restore_number_list_commas

normalization/steps/text/normalize_punctuation_between_number_words.py

@@ -4,13 +4,30 @@
 from normalization.steps.base import TextStep
 from normalization.steps.registry import register_step
 
+# British long-form numbers insert a comma after these scale words
+# ("one thousand, two hundred"). Not a list separator.
+_SCALE_WORDS_WITH_COMMA = ("thousand", "million", "billion", "trillion")
+
+
+def _word_alternation(words: list[str]) -> str:
+    sorted_words = sorted(words, key=lambda w: len(w), reverse=True)
+    return "|".join(re.escape(w) for w in sorted_words)
+
 
 @register_step
 class NormalizePunctuationBetweenNumberWordsStep(TextStep):
-    """Replace commas, dots, hyphens between number words with a single space.
+    """Normalize punctuation between spelled number words.
 
-    Handles: 'seven, zero' -> 'seven zero', 'two-one-three' -> 'two one three'.
-    Reads operators.config.number_words. No-op when None.
+    - Hyphens between any number_words: 'ninety-nine' -> 'ninety nine',
+      'two-one-three' -> 'two one three'.
+    - Commas after scale words: 'thousand, two' -> 'thousand two' (British
+      long form).
+    - Commas / dots / ellipsis / Arabic comma only between digit_words (0-9):
+      'seven, zero' -> 'seven zero'. Tens like 'sixty, seventy' keep their
+      comma so expand + list-comma protect yield '60 70', not '6070'.
+
+    Falls back to number_words for the digit comma/dot class when digit_words
+    is unset. No-op when number_words is empty.
     """
 
     name = "normalize_punctuation_between_number_words"
@@ -20,13 +37,33 @@ def __call__(self, text: str, operators: LanguageOperators) -> str:
         if not number_words:
             return text
 
-        sorted_words = sorted(number_words, key=lambda w: len(w), reverse=True)
-        alternation = "|".join(re.escape(w) for w in sorted_words)
-        number_group = f"(?:{alternation})"
-        pattern = re.compile(
-            # Include Arabic comma (U+060C) so digit-by-digit ASR like
-            # "صفر، خمسة" collapses the same way as "zero, five".
-            rf"({number_group})\s*[,.\-…\u060c]+\s*(?={number_group})",
+        number_group = f"(?:{_word_alternation(list(number_words))})"
+        # Compounds written with a hyphen (ninety-nine, two-one-three).
+        text = re.sub(
+            rf"({number_group})\s*-\s*(?={number_group})",
+            r"\1 ",
+            text,
+            flags=re.IGNORECASE,
+        )
+
+        scale_alt = "|".join(_SCALE_WORDS_WITH_COMMA)
+        text = re.sub(
+            rf"\b({scale_alt})\s*,\s*(?={number_group})",
+            r"\1 ",
+            text,
+            flags=re.IGNORECASE,
+        )
+
+        digit_words = operators.config.digit_words
+        comma_words = list(digit_words) if digit_words else list(number_words)
+        if not comma_words:
+            return text
+        digit_group = f"(?:{_word_alternation(comma_words)})"
+        # Include Arabic comma (U+060C) for digit-by-digit ASR.
+        text = re.sub(
+            rf"({digit_group})\s*[,.…\u060c]+\s*(?={digit_group})",
+            r"\1 ",
+            text,
             flags=re.IGNORECASE,
         )
-        return pattern.sub(r"\1 ", text)
+        return text

normalization/steps/text/placeholders.py

@@ -228,30 +228,31 @@ def _replacement(self, operators: LanguageOperators) -> str:
 # Number separator commas  (1, 2, 3 -> 1 ¤ 2 ¤ 3)
 # ---------------------------------------------------------------------------
 
+# Multi-digit list commas first ("60, 70") so they are not treated as joinable
+# digit-sequence separators.
+_RE_MULTI_DIGIT_LIST_COMMA = re.compile(r"(\d{2,}),\s+(?=\d{2,})")
 _RE_COMMA_SEP = re.compile(r"(\d+),\s+(?=\d)")
 _RE_DOT_SEP = re.compile(r"(\d+)\.\s+(?=\d)")
 
 
 @register_step
 class ProtectNumberSeparatorCommasStep(TextStep):
-    """Replace comma/dot+space between digits with ¤ marker. (1, 2, 3 -> 1 ¤ 2 ¤ 3)
+    """Protect commas/dots between digits before remove_symbols.
 
-    Two independent patterns (comma-separated and
-    dot-separated) both collapse to the same marker. ProtectStep handles a
-    single pattern mapping to a single placeholder.
+    - Multi-digit list commas ("60, 70") -> ¶ (restored as space after digit
+      collapsing, so the groups stay separate).
+    - Single-digit / mixed lists ("1, 2, 3") and spaced dots -> ¤ (stripped so
+      the groups join).
     """
 
     name = "protect_number_separator_commas"
 
     def __call__(self, text: str, operators: LanguageOperators) -> str:
-        text = _RE_COMMA_SEP.sub(
-            rf"\1 {ProtectPlaceholder.NUMBER_SEPARATOR.value} ",
-            text,
-        )
-        text = _RE_DOT_SEP.sub(
-            rf"\1 {ProtectPlaceholder.NUMBER_SEPARATOR.value} ",
-            text,
-        )
+        list_sep = ProtectPlaceholder.NUMBER_LIST_COMMA.value
+        join_sep = ProtectPlaceholder.NUMBER_SEPARATOR.value
+        text = _RE_MULTI_DIGIT_LIST_COMMA.sub(rf"\1 {list_sep} ", text)
+        text = _RE_COMMA_SEP.sub(rf"\1 {join_sep} ", text)
+        text = _RE_DOT_SEP.sub(rf"\1 {join_sep} ", text)
         return text
 
 
@@ -274,6 +275,20 @@ def __call__(self, text: str, operators: LanguageOperators) -> str:
         )
 
 
+@register_step
+class RestoreNumberListCommasStep(TextStep):
+    """Restore ¶ list-comma markers as a single space after digit collapsing."""
+
+    name = "restore_number_list_commas"
+
+    def __call__(self, text: str, operators: LanguageOperators) -> str:
+        return re.sub(
+            rf"\s*{re.escape(ProtectPlaceholder.NUMBER_LIST_COMMA.value)}\s*",
+            " ",
+            text,
+        )
+
+
 # ---------------------------------------------------------------------------
 # Decimal separator  (moved from protect_decimal_separator.py)
 # ---------------------------------------------------------------------------

normalization/steps/text/remove_spaces_between_adjacent_digits.py

@@ -20,7 +20,10 @@ class RemoveSpacesBetweenAdjacentDigitsStep(TextStep):
 
     def __call__(self, text: str, operators: LanguageOperators) -> str:
         decimal_word = operators.config.decimal_word
-        sep = ProtectPlaceholder.NUMBER_SEPARATOR.value
+        boundaries = (
+            ProtectPlaceholder.NUMBER_SEPARATOR.value,
+            ProtectPlaceholder.NUMBER_LIST_COMMA.value,
+        )
 
         if decimal_word is not None:
             text = text.replace(f" {decimal_word} ", "XDECWORDX")
@@ -38,12 +41,18 @@ def _protect_ordinal(match: re.Match) -> str:
 
             text = ordinal_re.sub(_protect_ordinal, text)
 
-        if sep not in text:
+        present = [b for b in boundaries if b in text]
+        if not present:
             text = _RE_SPACES_BETWEEN_DIGITS.sub(r"\1", text)
         else:
-            parts = text.split(sep)
-            parts = [_RE_SPACES_BETWEEN_DIGITS.sub(r"\1", p) for p in parts]
-            text = sep.join(parts)
+            # Split on every boundary marker so collapse cannot cross list commas
+            # ("60 ¶ 70") or joinable separators ("1 ¤ 2").
+            pattern = "|".join(re.escape(b) for b in present)
+            bits = re.split(f"({pattern})", text)
+            text = "".join(
+                bit if bit in present else _RE_SPACES_BETWEEN_DIGITS.sub(r"\1", bit)
+                for bit in bits
+            )
 
         for i, ordinal in enumerate(ordinals):
             text = text.replace(f"XORDINAL{i}X", ordinal.strip())

pyproject.toml

@@ -26,7 +26,6 @@ license = { file = "LICENSE" }
 requires-python = ">=3.10"
 dependencies = [
     "contractions>=0.1.73",
-    "num2words2>=1.0.21",
     "pyyaml>=6.0.3",
     "text2num>=3.0.0",
     "words2num2>=0.5.0",
@@ -45,7 +44,14 @@ Repository = "https://github.com/gladiaio/normalization"
 "Bug Tracker" = "https://github.com/gladiaio/normalization/issues"
 
 [dependency-groups]
-dev = ["pre-commit>=4.5.1", "ty>=0.0.20", "pytest>=9.0.3", "pygments>=2.20.0", "ruff>=0.15.2"]
+dev = [
+    "pre-commit>=4.5.1",
+    "ty>=0.0.20",
+    "pytest>=9.0.3",
+    "pygments>=2.20.0",
+    "ruff>=0.15.2",
+    "num2words2>=1.0.21",
+]
 docs = [
     "mkdocs>=1.6.1",
     "mkdocs-material>=9.7.6",

tests/e2e/files/gladia-3/en.csv

@@ -157,6 +157,6 @@ eleventh 2 thousaond and twelve,112012
 20 twenty,2020
 10 thousand,10000
 "That's fifty.",that is 50
-"where people hit sixty, seventy.",where people hit 6070
+"where people hit sixty, seventy.",where people hit 60 70
 nonfiction book,non fiction book
 non-fiction book,non fiction book

tests/unit/languages/english_number_normalizer_test.py

@@ -206,7 +206,7 @@ def test_for_not_rewritten_outside_number_point_context(
         ("That's fifty.", "that is 50"),
         ("I paid fifty dollars.", "i paid 50 dollars"),
         # Two written numbers side by side, of which this class alone expanded only the first.
-        ("where people hit sixty, seventy.", "where people hit 6070"),
+        ("where people hit sixty, seventy.", "where people hit 60 70"),
     ],
 )
 def test_words_to_num_across_punctuation_through_gladia_3_pipeline(