fix: a written number is read across punctuation and next to another one

normalization/languages/english/number_normalizer.py

@@ -2,6 +2,8 @@
 from fractions import Fraction
 from typing import Iterator, Match
 
+from words2num2 import words2num_sentence
+
 
 class EnglishNumberNormalizer:
     """Convert any spelled-out numbers into arabic numbers, while handling:
@@ -161,6 +163,31 @@ def __init__(self):
             for key in mapping
         }
         self.literal_words = {"one", "ones"}
+        # The written numbers themselves, without the glue ("and", "point", "double") and
+        # without the literals this class turns a trailing digit into ("2.5.1" -> "2.5.one"):
+        # what the second pass has left to read, if anything (see __call__).
+        written = {
+            key
+            for mapping in [
+                self.zeros,
+                self.ones,
+                self.ones_suffixed,
+                self.tens,
+                self.tens_suffixed,
+                self.multipliers,
+                self.multipliers_suffixed,
+            ]
+            for key in mapping
+        } - self.literal_words
+        # longest first, so that "seventeen" is tried before "seven"
+        alternatives: list[str] = sorted(
+            (re.escape(word) for word in written),
+            key=lambda word: len(word),
+            reverse=True,
+        )
+        self._written_number = re.compile(
+            r"\b(?:" + "|".join(alternatives) + r")\b", re.IGNORECASE
+        )
 
     def process_words(self, words: list[str]) -> Iterator[str]:  # noqa: C901
         prefix: str | None = None
@@ -509,4 +536,13 @@ def __call__(self, s: str) -> str:
         s = self.preprocess(s)
         s = " ".join(word for word in self.process_words(s.split()) if word is not None)
         s = self.postprocess(s)
-        return s
+        # What this class leaves alone because of the punctuation around it, or because two
+        # written numbers sit side by side, words2num2's sentence mode still reads ("that is
+        # fifty." -> "that is 50.", "sixty, seventy" -> "60, 70"). It runs on this class's
+        # output, so the mixed digit-and-word forms it does not read ("20 thirteen" -> 2013,
+        # "25 hundred" -> 2500) keep the value this class gave them. It only runs when a
+        # written number is left: on digits and punctuation alone it reads a version or an IP
+        # address as a decimal ("2 dot 5 dot 1" -> "2 dot 50 point 1").
+        if self._written_number.search(s) is None:
+            return s
+        return words2num_sentence(s)

pyproject.toml

@@ -26,8 +26,10 @@ license = { file = "LICENSE" }
 requires-python = ">=3.10"
 dependencies = [
     "contractions>=0.1.73",
+    "num2words2>=1.0.21",
     "pyyaml>=6.0.3",
     "text2num>=3.0.0",
+    "words2num2>=0.5.0",
 ]
 
 [tool.setuptools.package-data]

tests/e2e/files/gladia-3/en.csv

@@ -156,5 +156,7 @@ eleventh 2 thousaond and twelve,112012
 20 one,21
 20 twenty,2020
 10 thousand,10000
+"That's fifty.",that is 50
+"where people hit sixty, seventy.",where people hit 6070
 nonfiction book,non fiction book
 non-fiction book,non fiction book

tests/unit/languages/english_number_normalizer_test.py

@@ -1,8 +1,19 @@
+from functools import lru_cache
+
 import pytest
+from num2words2 import num2words
 
 from normalization.languages.english.number_normalizer import EnglishNumberNormalizer
+from normalization.pipeline.base import NormalizationPipeline
 from normalization.pipeline.loader import load_pipeline
 
+
+@lru_cache(maxsize=1)
+def _pipeline() -> NormalizationPipeline:
+    """The published preset, built once for the whole module (it is read-only)."""
+    return load_pipeline("gladia-3", "en")
+
+
 _ONES_UNITS: dict[str, int] = {
     "one": 1,
     "two": 2,
@@ -186,3 +197,36 @@ def test_for_not_rewritten_outside_number_point_context(
 ) -> None:
     assert normalizer("for example") == "for example"
     assert normalizer("40 for example") == "40 for example"
+
+
+@pytest.mark.parametrize(
+    ("text", "expected"),
+    [
+        # A number word that ends the sentence, which this class alone left as words.
+        ("That's fifty.", "that is 50"),
+        ("I paid fifty dollars.", "i paid 50 dollars"),
+        # Two written numbers side by side, of which this class alone expanded only the first.
+        ("where people hit sixty, seventy.", "where people hit 6070"),
+    ],
+)
+def test_words_to_num_across_punctuation_through_gladia_3_pipeline(
+    text: str, expected: str
+) -> None:
+    assert _pipeline().normalize(text) == expected
+
+
+@pytest.mark.parametrize("text", ["version 2.5.1", "192.168.1.1", "1.1.1.1"])
+def test_digits_and_dots_are_not_read_as_numbers(text: str) -> None:
+    """The second pass reads written numbers only: a version or an address keeps its dots."""
+    assert _pipeline().normalize(text) == text.replace(".", " dot ")
+
+
+@pytest.mark.parametrize(
+    "number", [n for n in range(0, 121) if n != 1] + [999, 1234, 10000]
+)
+def test_every_number_num2words2_writes_expands_to_its_digits(number: int) -> None:
+    """words2num2 reads back what num2words2 writes, through the published preset. 1 is left
+    out: a lone "one" is a pronoun as often as a number, and this class keeps it."""
+    written = num2words(number, lang="en")
+    assert written is not None
+    assert _pipeline().normalize(written) == str(number)