From c6ce6093a238f05b2ef815d9a9e39c8d05df1262 Mon Sep 17 00:00:00 2001 From: Jean-Louis Queguiner Date: Wed, 11 Nov 2026 14:34:04 -0500 Subject: [PATCH] fix: keep comma-separated multi-digit numbers apart (sixty, seventy -> 60 70) Issue #42 expected "60 70"; #43 left "6070" after digit collapse. Protect multi-digit list commas, stop stripping tens-list commas before expand, and move num2words2 to the dev group. Co-authored-by: Cursor --- normalization/constants/protectors.py | 3 + normalization/pipeline/base.py | 5 +- normalization/presets/gladia-3.yaml | 3 + ...malize_punctuation_between_number_words.py | 59 +++++++++++++++---- normalization/steps/text/placeholders.py | 39 ++++++++---- .../remove_spaces_between_adjacent_digits.py | 19 ++++-- pyproject.toml | 10 +++- tests/e2e/files/gladia-3/en.csv | 2 +- .../english_number_normalizer_test.py | 2 +- uv.lock | 4 +- 10 files changed, 111 insertions(+), 35 deletions(-) diff --git a/normalization/constants/protectors.py b/normalization/constants/protectors.py index f06f0e8..8e5ecab 100644 --- a/normalization/constants/protectors.py +++ b/normalization/constants/protectors.py @@ -10,6 +10,9 @@ class ProtectPlaceholder(str, Enum): UNIT_SLASH = "†" UNIT_DECIMAL = "‡" NUMBER_SEPARATOR = "¤" + # Comma between multi-digit numbers ("60, 70"): kept until after digit + # collapsing so the groups are not glued into "6070", then restored as space. + NUMBER_LIST_COMMA = "¶" SPELLING_SUFFIX = "xltrx" diff --git a/normalization/pipeline/base.py b/normalization/pipeline/base.py index 34ea8c6..82b69ff 100644 --- a/normalization/pipeline/base.py +++ b/normalization/pipeline/base.py @@ -83,7 +83,10 @@ def validate(self) -> None: "protect_time_colon": {"restore_time_colon"}, "protect_unit_decimal": {"restore_unit_decimal_as_word"}, "protect_unit_slash": {"restore_unit_slash"}, - "protect_number_separator_commas": {"remove_number_separator_markers"}, + "protect_number_separator_commas": { + "remove_number_separator_markers", + "restore_number_list_commas", + }, "protect_decimal_separator": {"restore_decimal_separator_with_word"}, "protect_hyphenated_letter_spelling": {"remove_letter_spelling_markers"}, "protect_space_separated_letter_spelling": { diff --git a/normalization/presets/gladia-3.yaml b/normalization/presets/gladia-3.yaml index b78e2bb..6d333e7 100644 --- a/normalization/presets/gladia-3.yaml +++ b/normalization/presets/gladia-3.yaml @@ -122,3 +122,6 @@ stages: # Final digit-collapse pass: late restores above (phone plus, unit slash) # can produce new adjacent-digit sequences that need collapsing. - remove_spaces_between_adjacent_digits # 3rd run: after late restores + # After every digit-collapse pass: turn ¶ list commas into spaces so + # "sixty, seventy" / "60, 70" stay "60 70" instead of "6070". + - restore_number_list_commas diff --git a/normalization/steps/text/normalize_punctuation_between_number_words.py b/normalization/steps/text/normalize_punctuation_between_number_words.py index aa2653e..81a36ea 100644 --- a/normalization/steps/text/normalize_punctuation_between_number_words.py +++ b/normalization/steps/text/normalize_punctuation_between_number_words.py @@ -4,13 +4,30 @@ from normalization.steps.base import TextStep from normalization.steps.registry import register_step +# British long-form numbers insert a comma after these scale words +# ("one thousand, two hundred"). Not a list separator. +_SCALE_WORDS_WITH_COMMA = ("thousand", "million", "billion", "trillion") + + +def _word_alternation(words: list[str]) -> str: + sorted_words = sorted(words, key=lambda w: len(w), reverse=True) + return "|".join(re.escape(w) for w in sorted_words) + @register_step class NormalizePunctuationBetweenNumberWordsStep(TextStep): - """Replace commas, dots, hyphens between number words with a single space. + """Normalize punctuation between spelled number words. - Handles: 'seven, zero' -> 'seven zero', 'two-one-three' -> 'two one three'. - Reads operators.config.number_words. No-op when None. + - Hyphens between any number_words: 'ninety-nine' -> 'ninety nine', + 'two-one-three' -> 'two one three'. + - Commas after scale words: 'thousand, two' -> 'thousand two' (British + long form). + - Commas / dots / ellipsis / Arabic comma only between digit_words (0-9): + 'seven, zero' -> 'seven zero'. Tens like 'sixty, seventy' keep their + comma so expand + list-comma protect yield '60 70', not '6070'. + + Falls back to number_words for the digit comma/dot class when digit_words + is unset. No-op when number_words is empty. """ name = "normalize_punctuation_between_number_words" @@ -20,13 +37,33 @@ def __call__(self, text: str, operators: LanguageOperators) -> str: if not number_words: return text - sorted_words = sorted(number_words, key=lambda w: len(w), reverse=True) - alternation = "|".join(re.escape(w) for w in sorted_words) - number_group = f"(?:{alternation})" - pattern = re.compile( - # Include Arabic comma (U+060C) so digit-by-digit ASR like - # "صفر، خمسة" collapses the same way as "zero, five". - rf"({number_group})\s*[,.\-…\u060c]+\s*(?={number_group})", + number_group = f"(?:{_word_alternation(list(number_words))})" + # Compounds written with a hyphen (ninety-nine, two-one-three). + text = re.sub( + rf"({number_group})\s*-\s*(?={number_group})", + r"\1 ", + text, + flags=re.IGNORECASE, + ) + + scale_alt = "|".join(_SCALE_WORDS_WITH_COMMA) + text = re.sub( + rf"\b({scale_alt})\s*,\s*(?={number_group})", + r"\1 ", + text, + flags=re.IGNORECASE, + ) + + digit_words = operators.config.digit_words + comma_words = list(digit_words) if digit_words else list(number_words) + if not comma_words: + return text + digit_group = f"(?:{_word_alternation(comma_words)})" + # Include Arabic comma (U+060C) for digit-by-digit ASR. + text = re.sub( + rf"({digit_group})\s*[,.…\u060c]+\s*(?={digit_group})", + r"\1 ", + text, flags=re.IGNORECASE, ) - return pattern.sub(r"\1 ", text) + return text diff --git a/normalization/steps/text/placeholders.py b/normalization/steps/text/placeholders.py index cff421e..4e8314c 100644 --- a/normalization/steps/text/placeholders.py +++ b/normalization/steps/text/placeholders.py @@ -228,30 +228,31 @@ def _replacement(self, operators: LanguageOperators) -> str: # Number separator commas (1, 2, 3 -> 1 ¤ 2 ¤ 3) # --------------------------------------------------------------------------- +# Multi-digit list commas first ("60, 70") so they are not treated as joinable +# digit-sequence separators. +_RE_MULTI_DIGIT_LIST_COMMA = re.compile(r"(\d{2,}),\s+(?=\d{2,})") _RE_COMMA_SEP = re.compile(r"(\d+),\s+(?=\d)") _RE_DOT_SEP = re.compile(r"(\d+)\.\s+(?=\d)") @register_step class ProtectNumberSeparatorCommasStep(TextStep): - """Replace comma/dot+space between digits with ¤ marker. (1, 2, 3 -> 1 ¤ 2 ¤ 3) + """Protect commas/dots between digits before remove_symbols. - Two independent patterns (comma-separated and - dot-separated) both collapse to the same marker. ProtectStep handles a - single pattern mapping to a single placeholder. + - Multi-digit list commas ("60, 70") -> ¶ (restored as space after digit + collapsing, so the groups stay separate). + - Single-digit / mixed lists ("1, 2, 3") and spaced dots -> ¤ (stripped so + the groups join). """ name = "protect_number_separator_commas" def __call__(self, text: str, operators: LanguageOperators) -> str: - text = _RE_COMMA_SEP.sub( - rf"\1 {ProtectPlaceholder.NUMBER_SEPARATOR.value} ", - text, - ) - text = _RE_DOT_SEP.sub( - rf"\1 {ProtectPlaceholder.NUMBER_SEPARATOR.value} ", - text, - ) + list_sep = ProtectPlaceholder.NUMBER_LIST_COMMA.value + join_sep = ProtectPlaceholder.NUMBER_SEPARATOR.value + text = _RE_MULTI_DIGIT_LIST_COMMA.sub(rf"\1 {list_sep} ", text) + text = _RE_COMMA_SEP.sub(rf"\1 {join_sep} ", text) + text = _RE_DOT_SEP.sub(rf"\1 {join_sep} ", text) return text @@ -274,6 +275,20 @@ def __call__(self, text: str, operators: LanguageOperators) -> str: ) +@register_step +class RestoreNumberListCommasStep(TextStep): + """Restore ¶ list-comma markers as a single space after digit collapsing.""" + + name = "restore_number_list_commas" + + def __call__(self, text: str, operators: LanguageOperators) -> str: + return re.sub( + rf"\s*{re.escape(ProtectPlaceholder.NUMBER_LIST_COMMA.value)}\s*", + " ", + text, + ) + + # --------------------------------------------------------------------------- # Decimal separator (moved from protect_decimal_separator.py) # --------------------------------------------------------------------------- diff --git a/normalization/steps/text/remove_spaces_between_adjacent_digits.py b/normalization/steps/text/remove_spaces_between_adjacent_digits.py index 7de4a4b..df6ec2d 100644 --- a/normalization/steps/text/remove_spaces_between_adjacent_digits.py +++ b/normalization/steps/text/remove_spaces_between_adjacent_digits.py @@ -20,7 +20,10 @@ class RemoveSpacesBetweenAdjacentDigitsStep(TextStep): def __call__(self, text: str, operators: LanguageOperators) -> str: decimal_word = operators.config.decimal_word - sep = ProtectPlaceholder.NUMBER_SEPARATOR.value + boundaries = ( + ProtectPlaceholder.NUMBER_SEPARATOR.value, + ProtectPlaceholder.NUMBER_LIST_COMMA.value, + ) if decimal_word is not None: text = text.replace(f" {decimal_word} ", "XDECWORDX") @@ -38,12 +41,18 @@ def _protect_ordinal(match: re.Match) -> str: text = ordinal_re.sub(_protect_ordinal, text) - if sep not in text: + present = [b for b in boundaries if b in text] + if not present: text = _RE_SPACES_BETWEEN_DIGITS.sub(r"\1", text) else: - parts = text.split(sep) - parts = [_RE_SPACES_BETWEEN_DIGITS.sub(r"\1", p) for p in parts] - text = sep.join(parts) + # Split on every boundary marker so collapse cannot cross list commas + # ("60 ¶ 70") or joinable separators ("1 ¤ 2"). + pattern = "|".join(re.escape(b) for b in present) + bits = re.split(f"({pattern})", text) + text = "".join( + bit if bit in present else _RE_SPACES_BETWEEN_DIGITS.sub(r"\1", bit) + for bit in bits + ) for i, ordinal in enumerate(ordinals): text = text.replace(f"XORDINAL{i}X", ordinal.strip()) diff --git a/pyproject.toml b/pyproject.toml index 0921eb2..c66db47 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,7 +26,6 @@ license = { file = "LICENSE" } requires-python = ">=3.10" dependencies = [ "contractions>=0.1.73", - "num2words2>=1.0.21", "pyyaml>=6.0.3", "text2num>=3.0.0", "words2num2>=0.5.0", @@ -45,7 +44,14 @@ Repository = "https://github.com/gladiaio/normalization" "Bug Tracker" = "https://github.com/gladiaio/normalization/issues" [dependency-groups] -dev = ["pre-commit>=4.5.1", "ty>=0.0.20", "pytest>=9.0.3", "pygments>=2.20.0", "ruff>=0.15.2"] +dev = [ + "pre-commit>=4.5.1", + "ty>=0.0.20", + "pytest>=9.0.3", + "pygments>=2.20.0", + "ruff>=0.15.2", + "num2words2>=1.0.21", +] docs = [ "mkdocs>=1.6.1", "mkdocs-material>=9.7.6", diff --git a/tests/e2e/files/gladia-3/en.csv b/tests/e2e/files/gladia-3/en.csv index 5667fb5..8d7b094 100644 --- a/tests/e2e/files/gladia-3/en.csv +++ b/tests/e2e/files/gladia-3/en.csv @@ -157,6 +157,6 @@ eleventh 2 thousaond and twelve,112012 20 twenty,2020 10 thousand,10000 "That's fifty.",that is 50 -"where people hit sixty, seventy.",where people hit 6070 +"where people hit sixty, seventy.",where people hit 60 70 nonfiction book,non fiction book non-fiction book,non fiction book diff --git a/tests/unit/languages/english_number_normalizer_test.py b/tests/unit/languages/english_number_normalizer_test.py index 1825093..9c2f1f4 100644 --- a/tests/unit/languages/english_number_normalizer_test.py +++ b/tests/unit/languages/english_number_normalizer_test.py @@ -206,7 +206,7 @@ def test_for_not_rewritten_outside_number_point_context( ("That's fifty.", "that is 50"), ("I paid fifty dollars.", "i paid 50 dollars"), # Two written numbers side by side, of which this class alone expanded only the first. - ("where people hit sixty, seventy.", "where people hit 6070"), + ("where people hit sixty, seventy.", "where people hit 60 70"), ], ) def test_words_to_num_across_punctuation_through_gladia_3_pipeline( diff --git a/uv.lock b/uv.lock index 744ffaa..3eb6f82 100644 --- a/uv.lock +++ b/uv.lock @@ -319,7 +319,6 @@ version = "0.1.0" source = { editable = "." } dependencies = [ { name = "contractions" }, - { name = "num2words2" }, { name = "pyyaml" }, { name = "text2num" }, { name = "words2num2" }, @@ -327,6 +326,7 @@ dependencies = [ [package.dev-dependencies] dev = [ + { name = "num2words2" }, { name = "pre-commit" }, { name = "pygments" }, { name = "pytest" }, @@ -342,7 +342,6 @@ docs = [ [package.metadata] requires-dist = [ { name = "contractions", specifier = ">=0.1.73" }, - { name = "num2words2", specifier = ">=1.0.21" }, { name = "pyyaml", specifier = ">=6.0.3" }, { name = "text2num", specifier = ">=3.0.0" }, { name = "words2num2", specifier = ">=0.5.0" }, @@ -350,6 +349,7 @@ requires-dist = [ [package.metadata.requires-dev] dev = [ + { name = "num2words2", specifier = ">=1.0.21" }, { name = "pre-commit", specifier = ">=4.5.1" }, { name = "pygments", specifier = ">=2.20.0" }, { name = "pytest", specifier = ">=9.0.3" },