diff --git a/normalization/languages/english/number_normalizer.py b/normalization/languages/english/number_normalizer.py index e435c6d..fe4c4dd 100644 --- a/normalization/languages/english/number_normalizer.py +++ b/normalization/languages/english/number_normalizer.py @@ -2,6 +2,8 @@ from fractions import Fraction from typing import Iterator, Match +from words2num2 import words2num_sentence + class EnglishNumberNormalizer: """Convert any spelled-out numbers into arabic numbers, while handling: @@ -161,6 +163,31 @@ def __init__(self): for key in mapping } self.literal_words = {"one", "ones"} + # The written numbers themselves, without the glue ("and", "point", "double") and + # without the literals this class turns a trailing digit into ("2.5.1" -> "2.5.one"): + # what the second pass has left to read, if anything (see __call__). + written = { + key + for mapping in [ + self.zeros, + self.ones, + self.ones_suffixed, + self.tens, + self.tens_suffixed, + self.multipliers, + self.multipliers_suffixed, + ] + for key in mapping + } - self.literal_words + # longest first, so that "seventeen" is tried before "seven" + alternatives: list[str] = sorted( + (re.escape(word) for word in written), + key=lambda word: len(word), + reverse=True, + ) + self._written_number = re.compile( + r"\b(?:" + "|".join(alternatives) + r")\b", re.IGNORECASE + ) def process_words(self, words: list[str]) -> Iterator[str]: # noqa: C901 prefix: str | None = None @@ -509,4 +536,13 @@ def __call__(self, s: str) -> str: s = self.preprocess(s) s = " ".join(word for word in self.process_words(s.split()) if word is not None) s = self.postprocess(s) - return s + # What this class leaves alone because of the punctuation around it, or because two + # written numbers sit side by side, words2num2's sentence mode still reads ("that is + # fifty." -> "that is 50.", "sixty, seventy" -> "60, 70"). It runs on this class's + # output, so the mixed digit-and-word forms it does not read ("20 thirteen" -> 2013, + # "25 hundred" -> 2500) keep the value this class gave them. It only runs when a + # written number is left: on digits and punctuation alone it reads a version or an IP + # address as a decimal ("2 dot 5 dot 1" -> "2 dot 50 point 1"). + if self._written_number.search(s) is None: + return s + return words2num_sentence(s) diff --git a/pyproject.toml b/pyproject.toml index 6476829..0921eb2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,8 +26,10 @@ license = { file = "LICENSE" } requires-python = ">=3.10" dependencies = [ "contractions>=0.1.73", + "num2words2>=1.0.21", "pyyaml>=6.0.3", "text2num>=3.0.0", + "words2num2>=0.5.0", ] [tool.setuptools.package-data] diff --git a/tests/e2e/files/gladia-3/en.csv b/tests/e2e/files/gladia-3/en.csv index 5ef15db..5667fb5 100644 --- a/tests/e2e/files/gladia-3/en.csv +++ b/tests/e2e/files/gladia-3/en.csv @@ -156,5 +156,7 @@ eleventh 2 thousaond and twelve,112012 20 one,21 20 twenty,2020 10 thousand,10000 +"That's fifty.",that is 50 +"where people hit sixty, seventy.",where people hit 6070 nonfiction book,non fiction book non-fiction book,non fiction book diff --git a/tests/unit/languages/english_number_normalizer_test.py b/tests/unit/languages/english_number_normalizer_test.py index e282b62..1825093 100644 --- a/tests/unit/languages/english_number_normalizer_test.py +++ b/tests/unit/languages/english_number_normalizer_test.py @@ -1,8 +1,19 @@ +from functools import lru_cache + import pytest +from num2words2 import num2words from normalization.languages.english.number_normalizer import EnglishNumberNormalizer +from normalization.pipeline.base import NormalizationPipeline from normalization.pipeline.loader import load_pipeline + +@lru_cache(maxsize=1) +def _pipeline() -> NormalizationPipeline: + """The published preset, built once for the whole module (it is read-only).""" + return load_pipeline("gladia-3", "en") + + _ONES_UNITS: dict[str, int] = { "one": 1, "two": 2, @@ -186,3 +197,36 @@ def test_for_not_rewritten_outside_number_point_context( ) -> None: assert normalizer("for example") == "for example" assert normalizer("40 for example") == "40 for example" + + +@pytest.mark.parametrize( + ("text", "expected"), + [ + # A number word that ends the sentence, which this class alone left as words. + ("That's fifty.", "that is 50"), + ("I paid fifty dollars.", "i paid 50 dollars"), + # Two written numbers side by side, of which this class alone expanded only the first. + ("where people hit sixty, seventy.", "where people hit 6070"), + ], +) +def test_words_to_num_across_punctuation_through_gladia_3_pipeline( + text: str, expected: str +) -> None: + assert _pipeline().normalize(text) == expected + + +@pytest.mark.parametrize("text", ["version 2.5.1", "192.168.1.1", "1.1.1.1"]) +def test_digits_and_dots_are_not_read_as_numbers(text: str) -> None: + """The second pass reads written numbers only: a version or an address keeps its dots.""" + assert _pipeline().normalize(text) == text.replace(".", " dot ") + + +@pytest.mark.parametrize( + "number", [n for n in range(0, 121) if n != 1] + [999, 1234, 10000] +) +def test_every_number_num2words2_writes_expands_to_its_digits(number: int) -> None: + """words2num2 reads back what num2words2 writes, through the published preset. 1 is left + out: a lone "one" is a pronoun as often as a number, and this class keeps it.""" + written = num2words(number, lang="en") + assert written is not None + assert _pipeline().normalize(written) == str(number) diff --git a/uv.lock b/uv.lock index 1fe0ad7..744ffaa 100644 --- a/uv.lock +++ b/uv.lock @@ -319,8 +319,10 @@ version = "0.1.0" source = { editable = "." } dependencies = [ { name = "contractions" }, + { name = "num2words2" }, { name = "pyyaml" }, { name = "text2num" }, + { name = "words2num2" }, ] [package.dev-dependencies] @@ -340,8 +342,10 @@ docs = [ [package.metadata] requires-dist = [ { name = "contractions", specifier = ">=0.1.73" }, + { name = "num2words2", specifier = ">=1.0.21" }, { name = "pyyaml", specifier = ">=6.0.3" }, { name = "text2num", specifier = ">=3.0.0" }, + { name = "words2num2", specifier = ">=0.5.0" }, ] [package.metadata.requires-dev] @@ -683,6 +687,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/88/b2/d0896bdcdc8d28a7fc5717c305f1a861c26e18c05047949fb371034d98bd/nodeenv-1.10.0-py2.py3-none-any.whl", hash = "sha256:5bb13e3eed2923615535339b3c620e76779af4cb4c6a90deccc9e36b274d3827", size = 23438, upload-time = "2025-12-20T14:08:52.782Z" }, ] +[[package]] +name = "num2words2" +version = "1.0.21" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/46/7f/96f60ad3091401205041c64679be2bdce639d01bbf48110c7567f4349b4d/num2words2-1.0.21.tar.gz", hash = "sha256:a1c57b15c257db6059a57f2293c3aa8d29b5e8d955c3bbcfe6deacff06b4dcdf", size = 2811817, upload-time = "2026-11-09T13:13:46.1Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/23/1a/b3e32d20dbb0b853bd3b4b8ab84a5839c24a4a9f44550bc7c29191c54750/num2words2-1.0.21-cp38-abi3-macosx_10_12_x86_64.whl", hash = "sha256:2fb9b10631369514f5a2d955c81a9d17193a589e7d524d721f38a37216ee0fc1", size = 3597789, upload-time = "2026-11-09T13:13:36.565Z" }, + { url = "https://files.pythonhosted.org/packages/7b/25/e4f82a116e08cc691b7ecd01ac0516f84288c68fbc2311740cef98a6e886/num2words2-1.0.21-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:1ebe0195b382171aa6f671216fc48dbd6951287b3f057789d3ae577dcc65af77", size = 3297647, upload-time = "2026-11-09T13:13:38.477Z" }, + { url = "https://files.pythonhosted.org/packages/79/aa/e33fd0ec229b5159d06d54e6f1974b2f453be326309e24424f281b3edbd3/num2words2-1.0.21-cp38-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:e24fc22041f7c02a355f044445d6ca49bbad2e8e395e529d7469f1ae0095c1bf", size = 3456334, upload-time = "2026-11-09T13:13:40.478Z" }, + { url = "https://files.pythonhosted.org/packages/a9/89/bb6235237a0370a431e152c827f6c08ce557b23144efb44bb5cb4eb80913/num2words2-1.0.21-cp38-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:a8db28e6782d4f2776c2e9418cd63f11dc529ac4763692b4de6dc2b610aba494", size = 3732982, upload-time = "2026-11-09T13:13:42.387Z" }, + { url = "https://files.pythonhosted.org/packages/a4/cc/2b2361e52e3c60687c4e4bc2d81706fb4805b17079f305306e35ff438acd/num2words2-1.0.21-cp38-abi3-win_amd64.whl", hash = "sha256:99470993bd562f57b17a694e251380ee55a3c6731835fc1bc51161bdf161c7d6", size = 4116950, upload-time = "2026-11-09T13:13:44.438Z" }, +] + [[package]] name = "packaging" version = "26.3" @@ -1325,3 +1342,16 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/92/9c/5a39ec8880c08e50f75d90f4e0e27b95dc91a04deb5e6e04871dd4b7e006/watchdog-7.1.0-py3-none-win_arm64.whl", hash = "sha256:5ed4b94365c53ffd51d88e47ca48d6afae02fb1bd849175a9a10927a2fbfb932", size = 103260, upload-time = "2026-11-05T09:33:18.987Z" }, { url = "https://files.pythonhosted.org/packages/c0/aa/7a7210433e000f2491ddec7756d312009b0a47847e5c3457ccb03852cfdf/watchdog-7.1.0-py3-none-win_ia64.whl", hash = "sha256:0cfc5a6b3d4e76fc3e18fd371ee79d57945004da4105b589f71bb16966fd3e67", size = 103256, upload-time = "2026-11-05T09:33:20.159Z" }, ] + +[[package]] +name = "words2num2" +version = "0.5.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/74/39/d76fe36a22755a520aaec24016d52124d9da26dcd280266b8c67bb35c45e/words2num2-0.5.0.tar.gz", hash = "sha256:a549d3d7392950b09b90ec0a4da2f4c885fe82686f447b24071029eeb6578129", size = 443017, upload-time = "2026-11-09T22:13:33.734Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/85/7e/1124dd5525b82dfe94f438875d79f8cf3aa9557bf1d53ca39e8935894695/words2num2-0.5.0-cp38-abi3-macosx_10_12_x86_64.whl", hash = "sha256:31da57a63dec36030dbd2e727d7734dab5deb167bb1f533cfee140b843da5cb5", size = 3980421, upload-time = "2026-11-09T22:13:25.215Z" }, + { url = "https://files.pythonhosted.org/packages/4e/e1/8b30953b501c2f895b00659bab3c88e48eacb6b9a82e4518c79e743cc464/words2num2-0.5.0-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:e5f585dc1411222b835d98a666843073a7b32a02e5a22c7562f5c11e4aae8061", size = 3635139, upload-time = "2026-11-09T22:13:26.986Z" }, + { url = "https://files.pythonhosted.org/packages/82/11/b97ffc407fdcd69eb7efea28bfee40ff55edb148cbaab965b58d5331b565/words2num2-0.5.0-cp38-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:632a725339c1bfa74c9dc2d94cf05d79b7ee9d5031b5c56c6e4d4252078c420a", size = 3962602, upload-time = "2026-11-09T22:13:28.481Z" }, + { url = "https://files.pythonhosted.org/packages/ad/3d/b7b8d253a1536d8464d02939661781f003d5a21e38c8c36d1c8587ce1207/words2num2-0.5.0-cp38-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:6b506455e07c0aefd1d1ad7271a1b872494165fb21b3f50bb26432eb9d3ea1e7", size = 4183549, upload-time = "2026-11-09T22:13:30.382Z" }, + { url = "https://files.pythonhosted.org/packages/23/94/ccf8df95d4eb5672e425baa8510d38647a631094d171ea7a6336ff127022/words2num2-0.5.0-cp38-abi3-win_amd64.whl", hash = "sha256:f8ce2ea7fe2104e12bcc9b7d709d5779c8dc2167b4963b30e4fa933afc85e2df", size = 4085072, upload-time = "2026-11-09T22:13:31.959Z" }, +]