From a34baf6d63051e526b8e1e4eb2ec947d92039eee Mon Sep 17 00:00:00 2001 From: karamouche Date: Wed, 21 Oct 2026 11:04:51 -0400 Subject: [PATCH 01/11] feat: add apply_character_replacements step driven by language config Adds an optional character_replacements field to LanguageConfig and a step that applies it first in gladia-3 text_pre. The step is a no-op for every language that leaves the field as None, so existing outputs are unchanged. --- docs/reference/steps.md | 10 ++++++++ .../languages/base/language_config.py | 11 +++++++++ normalization/presets/gladia-3.yaml | 5 ++++ normalization/steps/text/__init__.py | 2 ++ .../text/apply_character_replacements.py | 22 +++++++++++++++++ .../text/apply_character_replacements_test.py | 24 +++++++++++++++++++ 6 files changed, 74 insertions(+) create mode 100644 normalization/steps/text/apply_character_replacements.py create mode 100644 tests/unit/steps/text/apply_character_replacements_test.py diff --git a/docs/reference/steps.md b/docs/reference/steps.md index 1b6d764..04e797d 100644 --- a/docs/reference/steps.md +++ b/docs/reference/steps.md @@ -4,6 +4,16 @@ Auto-generated — do not edit by hand. Run `scripts/generate_step_docs.py` to u ## Text Steps +### `apply_character_replacements` + +**Base class:** `TextStep` + +Replace or delete single characters using ``character_replacements`` from the language config. + +Runs first in the pipeline so that script-level variants (letter forms, native +digits, vowel marks) are unified before protection, numeric, and symbol steps. +Skipped when the language does not define ``character_replacements``. + ### `apply_sentence_level_replacements` **Base class:** `TextStep` diff --git a/normalization/languages/base/language_config.py b/normalization/languages/base/language_config.py index 3387cfc..e0d9c38 100644 --- a/normalization/languages/base/language_config.py +++ b/normalization/languages/base/language_config.py @@ -28,6 +28,12 @@ CurrencySymbols: TypeAlias = dict[str, str] """Maps currency symbols to their corresponding words.""" +CharacterReplacements: TypeAlias = dict[str, str] +"""Maps single characters to their canonical replacement (possibly empty to delete). +Applied character by character, before any other text step. +Example (Arabic): {'أ': 'ا', 'ـ': '', '٣': '3'} +""" + @dataclass class LanguageConfig: @@ -91,3 +97,8 @@ class LanguageConfig: pm_word: str | None = None """Canonical PM time designator (e.g. "pm" for English). Used by am/pm time formatting steps. None = am/pm steps are skipped.""" + character_replacements: CharacterReplacements | None = None + """Maps single characters to their canonical form; an empty value deletes the character. + Used by apply_character_replacements to unify script-level variants (letter forms, + native digits, vowel marks) before any other step sees the text. + None = step is skipped.""" diff --git a/normalization/presets/gladia-3.yaml b/normalization/presets/gladia-3.yaml index ca3f6ff..b78e2bb 100644 --- a/normalization/presets/gladia-3.yaml +++ b/normalization/presets/gladia-3.yaml @@ -2,6 +2,11 @@ name: gladia-3 stages: text_pre: + # --- Phase 0: Script-level character unification --- + # No-op unless the language defines character_replacements. Runs first so + # native digits / separators are visible to every protect and numeric step. + - apply_character_replacements + # --- Phase 1: Protect symbols from being erased by later cleanup --- # Run first so that email @/., phone +, time :, unit decimal/slash, and # number separators all survive remove_symbols later in this stage. diff --git a/normalization/steps/text/__init__.py b/normalization/steps/text/__init__.py index 23b0f46..90ea8e2 100644 --- a/normalization/steps/text/__init__.py +++ b/normalization/steps/text/__init__.py @@ -1,4 +1,5 @@ from . import ( + apply_character_replacements, apply_sentence_level_replacements, casefold_text, convert_comparison_operators_to_words, @@ -39,6 +40,7 @@ ) __all__ = [ + "apply_character_replacements", "apply_sentence_level_replacements", "casefold_text", "convert_comparison_operators_to_words", diff --git a/normalization/steps/text/apply_character_replacements.py b/normalization/steps/text/apply_character_replacements.py new file mode 100644 index 0000000..f7c3d79 --- /dev/null +++ b/normalization/steps/text/apply_character_replacements.py @@ -0,0 +1,22 @@ +from normalization.languages.base import LanguageOperators +from normalization.steps.base import TextStep +from normalization.steps.registry import register_step + + +@register_step +class ApplyCharacterReplacementsStep(TextStep): + """Replace or delete single characters using ``character_replacements`` from the language config. + + Runs first in the pipeline so that script-level variants (letter forms, native + digits, vowel marks) are unified before protection, numeric, and symbol steps. + Skipped when the language does not define ``character_replacements``. + """ + + name = "apply_character_replacements" + + def __call__(self, text: str, operators: LanguageOperators) -> str: + replacements = operators.config.character_replacements + # TODO: populate character_replacements for languages with script-level variants + if replacements is None: + return text + return text.translate(str.maketrans(replacements)) diff --git a/tests/unit/steps/text/apply_character_replacements_test.py b/tests/unit/steps/text/apply_character_replacements_test.py new file mode 100644 index 0000000..5a88344 --- /dev/null +++ b/tests/unit/steps/text/apply_character_replacements_test.py @@ -0,0 +1,24 @@ +from normalization.languages.base import LanguageConfig, LanguageOperators +from normalization.steps.text.apply_character_replacements import ( + ApplyCharacterReplacementsStep, +) + +from .conftest import assert_text_step_registered + + +def test_step_is_registered(): + assert_text_step_registered(ApplyCharacterReplacementsStep) + + +def test_skipped_when_config_is_none(operators: LanguageOperators): + assert ApplyCharacterReplacementsStep()("abc ـ ٣", operators) == "abc ـ ٣" + + +def test_replaces_and_deletes_characters(): + operators = LanguageOperators( + LanguageConfig( + code="test", + character_replacements={"a": "b", "-": "", "٣": "3"}, + ) + ) + assert ApplyCharacterReplacementsStep()("a-a ٣", operators) == "bb 3" From ec9df94054cfecf6450e945cb02022c5bbcc5180 Mon Sep 17 00:00:00 2001 From: karamouche Date: Wed, 21 Oct 2026 11:05:31 -0400 Subject: [PATCH 02/11] feat(ar): add Arabic language with symbols, currencies and fillers Registers the 'ar' language with symbol and currency words, Arabic filler words and an empty word replacement table. --- normalization/languages/__init__.py | 2 + normalization/languages/arabic/__init__.py | 7 +++ normalization/languages/arabic/operators.py | 46 +++++++++++++++++++ .../languages/arabic/replacements.py | 7 +++ tests/unit/languages/arabic_operators_test.py | 41 +++++++++++++++++ 5 files changed, 103 insertions(+) create mode 100644 normalization/languages/arabic/__init__.py create mode 100644 normalization/languages/arabic/operators.py create mode 100644 normalization/languages/arabic/replacements.py create mode 100644 tests/unit/languages/arabic_operators_test.py diff --git a/normalization/languages/__init__.py b/normalization/languages/__init__.py index 3d0c998..0d446fd 100644 --- a/normalization/languages/__init__.py +++ b/normalization/languages/__init__.py @@ -1,4 +1,5 @@ from . import ( + arabic, danish, dutch, english, @@ -18,6 +19,7 @@ register_language(LanguageOperators) __all__ = [ + "arabic", "danish", "dutch", "english", diff --git a/normalization/languages/arabic/__init__.py b/normalization/languages/arabic/__init__.py new file mode 100644 index 0000000..17e90f1 --- /dev/null +++ b/normalization/languages/arabic/__init__.py @@ -0,0 +1,7 @@ +from .operators import ArabicOperators +from .replacements import ARABIC_REPLACEMENTS + +__all__ = [ + "ArabicOperators", + "ARABIC_REPLACEMENTS", +] diff --git a/normalization/languages/arabic/operators.py b/normalization/languages/arabic/operators.py new file mode 100644 index 0000000..bc750a6 --- /dev/null +++ b/normalization/languages/arabic/operators.py @@ -0,0 +1,46 @@ +from normalization.languages.arabic.replacements import ARABIC_REPLACEMENTS +from normalization.languages.base import LanguageConfig, LanguageOperators +from normalization.languages.registry import register_language + +# Words inserted by steps (symbols, currencies, fillers) are written in the +# normalized surface form so they match the rest of the normalized text. +ARABIC_CONFIG = LanguageConfig( + code="ar", + decimal_separator=".", + decimal_word="فاصله", + thousand_separator=",", + symbols_to_words={ + "@": "ات", + ".": "نقطه", + "+": "زائد", + "=": "يساوي", + ">": "اكبر من", + "<": "اصغر من", + "°": "درجه", + "°C": "درجه مئويه", + "°F": "درجه فهرنهايت", + "%": "بالمئه", + }, + currency_symbol_to_word={ + "$": "دولار", + "€": "يورو", + "£": "جنيه", + "¥": "ين", + }, + filler_words=[ + "اه", + "اهه", + "امم", + "اممم", + "ممم", + ], +) + + +@register_language +class ArabicOperators(LanguageOperators): + def __init__(self) -> None: + super().__init__(ARABIC_CONFIG) + + def get_word_replacements(self) -> dict[str, str]: + return ARABIC_REPLACEMENTS diff --git a/normalization/languages/arabic/replacements.py b/normalization/languages/arabic/replacements.py new file mode 100644 index 0000000..57b5e04 --- /dev/null +++ b/normalization/languages/arabic/replacements.py @@ -0,0 +1,7 @@ +"""Word-level substitutions for Arabic. + +Keys and values must use the normalized surface form produced by the gladia-3 +pipeline (see ``character_replacements`` in ``operators.py``). +""" + +ARABIC_REPLACEMENTS: dict[str, str] = {} diff --git a/tests/unit/languages/arabic_operators_test.py b/tests/unit/languages/arabic_operators_test.py new file mode 100644 index 0000000..02e8650 --- /dev/null +++ b/tests/unit/languages/arabic_operators_test.py @@ -0,0 +1,41 @@ +import pytest + +from normalization.languages.arabic.operators import ArabicOperators +from normalization.languages.registry import get_language_registry +from normalization.pipeline.loader import load_pipeline + + +@pytest.fixture +def operators() -> ArabicOperators: + return ArabicOperators() + + +@pytest.fixture(scope="module") +def pipeline(): + return load_pipeline("gladia-3", "ar") + + +def test_arabic_is_registered() -> None: + assert "ar" in get_language_registry() + + +def test_arabic_registry_produces_arabic_operators() -> None: + instance = get_language_registry()["ar"]() + assert isinstance(instance, ArabicOperators) + + +def test_config_code(operators: ArabicOperators) -> None: + assert operators.config.code == "ar" + + +@pytest.mark.parametrize( + ("text", "expected"), + [ + ("يكلف $50", "يكلف 50 دولار"), + ("20%", "20 بالميه"), + ("12.5", "12 فاصله 5"), + ("امم نعم", "نعم"), + ], +) +def test_pipeline_symbols_and_fillers(pipeline, text: str, expected: str) -> None: + assert pipeline.normalize(text) == expected From ae7d074f9dcd7be2d05030447a97541471527bed Mon Sep 17 00:00:00 2001 From: karamouche Date: Wed, 21 Oct 2026 11:05:44 -0400 Subject: [PATCH 03/11] feat(ar): strip harakat and tatweel before word splitting MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Vowel marks were turned into spaces by remove_symbols, splitting vocalized words into single letters (مَرْحَبًا -> م ر ح ب ا). They are now deleted up front, together with the typographic tatweel. --- normalization/languages/arabic/operators.py | 9 +++++++++ tests/unit/languages/arabic_operators_test.py | 15 +++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/normalization/languages/arabic/operators.py b/normalization/languages/arabic/operators.py index bc750a6..8210687 100644 --- a/normalization/languages/arabic/operators.py +++ b/normalization/languages/arabic/operators.py @@ -2,6 +2,14 @@ from normalization.languages.base import LanguageConfig, LanguageOperators from normalization.languages.registry import register_language +# Short vowels, tanween, shadda, sukun and related marks (U+064B–U+065F) plus +# the superscript alef. Transcripts rarely agree on vocalization, and +# remove_symbols would otherwise turn each mark into a word-splitting space. +_HARAKAT: dict[str, str] = {chr(c): "" for c in range(0x064B, 0x0660)} | {"\u0670": ""} + +# Tatweel / kashida: purely typographic letter elongation (عـــربي → عربي). +_TATWEEL: dict[str, str] = {"\u0640": ""} + # Words inserted by steps (symbols, currencies, fillers) are written in the # normalized surface form so they match the rest of the normalized text. ARABIC_CONFIG = LanguageConfig( @@ -27,6 +35,7 @@ "£": "جنيه", "¥": "ين", }, + character_replacements={**_HARAKAT, **_TATWEEL}, filler_words=[ "اه", "اهه", diff --git a/tests/unit/languages/arabic_operators_test.py b/tests/unit/languages/arabic_operators_test.py index 02e8650..6e264a9 100644 --- a/tests/unit/languages/arabic_operators_test.py +++ b/tests/unit/languages/arabic_operators_test.py @@ -39,3 +39,18 @@ def test_config_code(operators: ArabicOperators) -> None: ) def test_pipeline_symbols_and_fillers(pipeline, text: str, expected: str) -> None: assert pipeline.normalize(text) == expected + + +@pytest.mark.parametrize( + ("text", "expected"), + [ + ("مَرْحَبًا بِكُمْ", "مرحبا بكم"), + ("مُحَمَّدٌ", "محمد"), + ("هٰذا", "هذا"), + ("عـــربي", "عربي"), + ], +) +def test_pipeline_strips_harakat_and_tatweel( + pipeline, text: str, expected: str +) -> None: + assert pipeline.normalize(text) == expected From abfafef9ba50f09cd5b64ae31f0dda535b5baa41 Mon Sep 17 00:00:00 2001 From: karamouche Date: Wed, 21 Oct 2026 11:05:57 -0400 Subject: [PATCH 04/11] feat(ar): unify alef, alef maqsura and ta marbuta variants Folds hamzated/madda/wasla alef to bare alef, alef maqsura to yeh and ta marbuta to heh, plus Persian yeh/keheh lookalikes, so spelling variants of the same word no longer count as WER errors. --- normalization/languages/arabic/operators.py | 15 ++++++++++++++- tests/unit/languages/arabic_operators_test.py | 15 +++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/normalization/languages/arabic/operators.py b/normalization/languages/arabic/operators.py index 8210687..aed5845 100644 --- a/normalization/languages/arabic/operators.py +++ b/normalization/languages/arabic/operators.py @@ -10,6 +10,19 @@ # Tatweel / kashida: purely typographic letter elongation (عـــربي → عربي). _TATWEEL: dict[str, str] = {"\u0640": ""} +# Orthographic variants that transcribers and STT engines use interchangeably. +# Folding them is standard practice for Arabic WER scoring. +_LETTER_VARIANTS: dict[str, str] = { + "أ": "ا", # alef with hamza above + "إ": "ا", # alef with hamza below + "آ": "ا", # alef with madda + "ٱ": "ا", # alef wasla + "ى": "ي", # alef maqsura + "ة": "ه", # ta marbuta + "ی": "ي", # Persian yeh + "ک": "ك", # keheh +} + # Words inserted by steps (symbols, currencies, fillers) are written in the # normalized surface form so they match the rest of the normalized text. ARABIC_CONFIG = LanguageConfig( @@ -35,7 +48,7 @@ "£": "جنيه", "¥": "ين", }, - character_replacements={**_HARAKAT, **_TATWEEL}, + character_replacements={**_HARAKAT, **_TATWEEL, **_LETTER_VARIANTS}, filler_words=[ "اه", "اهه", diff --git a/tests/unit/languages/arabic_operators_test.py b/tests/unit/languages/arabic_operators_test.py index 6e264a9..3214eef 100644 --- a/tests/unit/languages/arabic_operators_test.py +++ b/tests/unit/languages/arabic_operators_test.py @@ -54,3 +54,18 @@ def test_pipeline_strips_harakat_and_tatweel( pipeline, text: str, expected: str ) -> None: assert pipeline.normalize(text) == expected + + +@pytest.mark.parametrize( + ("text", "expected"), + [ + ("أنا ذهبت إلى المدرسة", "انا ذهبت الي المدرسه"), + ("آمن", "امن"), + ("ٱلله", "الله"), + ("مستشفى", "مستشفي"), + ("مستشفي", "مستشفي"), + ("کتاب عربی", "كتاب عربي"), + ], +) +def test_pipeline_unifies_letter_variants(pipeline, text: str, expected: str) -> None: + assert pipeline.normalize(text) == expected From 1dcdbdf51dc3437e5942ba4245503ca9b803c462 Mon Sep 17 00:00:00 2001 From: karamouche Date: Wed, 21 Oct 2026 11:06:08 -0400 Subject: [PATCH 05/11] feat(ar): convert Arabic-Indic digits and separators to ASCII MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Maps Arabic-Indic and Eastern Arabic-Indic digits to 0-9 and the Arabic decimal separator, thousands separator and percent sign to their ASCII counterparts. Previously ١٢٫٥ collapsed to ١٢٥. --- normalization/languages/arabic/operators.py | 20 ++++++++++++++++++- tests/unit/languages/arabic_operators_test.py | 16 +++++++++++++++ 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/normalization/languages/arabic/operators.py b/normalization/languages/arabic/operators.py index aed5845..f148d3e 100644 --- a/normalization/languages/arabic/operators.py +++ b/normalization/languages/arabic/operators.py @@ -23,6 +23,19 @@ "ک": "ك", # keheh } +# Arabic-Indic (U+0660–U+0669) and Eastern Arabic-Indic (U+06F0–U+06F9) digits +# become ASCII so every numeric step sees them; the Arabic decimal/thousands +# separators and percent sign map to the characters those steps expect. +_DIGITS_AND_SEPARATORS: dict[str, str] = ( + {chr(0x0660 + d): str(d) for d in range(10)} + | {chr(0x06F0 + d): str(d) for d in range(10)} + | { + "٫": ".", # Arabic decimal separator + "٬": ",", # Arabic thousands separator + "٪": "%", # Arabic percent sign + } +) + # Words inserted by steps (symbols, currencies, fillers) are written in the # normalized surface form so they match the rest of the normalized text. ARABIC_CONFIG = LanguageConfig( @@ -48,7 +61,12 @@ "£": "جنيه", "¥": "ين", }, - character_replacements={**_HARAKAT, **_TATWEEL, **_LETTER_VARIANTS}, + character_replacements={ + **_HARAKAT, + **_TATWEEL, + **_LETTER_VARIANTS, + **_DIGITS_AND_SEPARATORS, + }, filler_words=[ "اه", "اهه", diff --git a/tests/unit/languages/arabic_operators_test.py b/tests/unit/languages/arabic_operators_test.py index 3214eef..4dd8207 100644 --- a/tests/unit/languages/arabic_operators_test.py +++ b/tests/unit/languages/arabic_operators_test.py @@ -69,3 +69,19 @@ def test_pipeline_strips_harakat_and_tatweel( ) def test_pipeline_unifies_letter_variants(pipeline, text: str, expected: str) -> None: assert pipeline.normalize(text) == expected + + +@pytest.mark.parametrize( + ("text", "expected"), + [ + ("عندي ٣ كتب", "عندي 3 كتب"), + ("۱۲۳", "123"), + ("١٢٫٥ كيلو", "12 فاصله 5 كيلو"), + ("١٬٠٠٠ دولار", "1000 دولار"), + ("٢٠٪", "20 بالميه"), + ], +) +def test_pipeline_converts_native_digits_and_separators( + pipeline, text: str, expected: str +) -> None: + assert pipeline.normalize(text) == expected From e5133a7cf69620a6477726b3c6deca232086d706 Mon Sep 17 00:00:00 2001 From: karamouche Date: Wed, 21 Oct 2026 11:06:22 -0400 Subject: [PATCH 06/11] test(ar): add gladia-3 end-to-end cases for Arabic --- tests/e2e/files/gladia-3/ar.csv | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 tests/e2e/files/gladia-3/ar.csv diff --git a/tests/e2e/files/gladia-3/ar.csv b/tests/e2e/files/gladia-3/ar.csv new file mode 100644 index 0000000..edabbef --- /dev/null +++ b/tests/e2e/files/gladia-3/ar.csv @@ -0,0 +1,20 @@ +input,expected +مَرْحَبًا بِكُمْ,مرحبا بكم +عـــربي,عربي +أنا ذهبت إلى المدرسة,انا ذهبت الي المدرسه +مستشفى,مستشفي +كيف حالك؟ بخير، شكرا؛,كيف حالك بخير شكرا +« نص »,نص +عندي ٣ كتب,عندي 3 كتب +۱۲۳,123 +١٢٫٥ كيلو,12 فاصله 5 كيلو +١٬٠٠٠ دولار,1000 دولار +نسبة ٢٠٪,نسبه 20 بالميه +يكلف $50,يكلف 50 دولار +آه، أنا هنا,انا هنا +امم نعم,نعم +admin@example.com,admin ات example نقطه com +2 > 1,2 اكبر من 1 +درجة الحرارة 30°C,درجه الحراره 30 درجه ميويه +مسؤول,مسوول +کتاب عربی,كتاب عربي From aba7e1c756fe684f3101f4f7a6070fe5bd001900 Mon Sep 17 00:00:00 2001 From: karamouche Date: Wed, 21 Oct 2026 11:06:29 -0400 Subject: [PATCH 07/11] docs: list Arabic in supported languages --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 7ffa32c..aa47045 100644 --- a/README.md +++ b/README.md @@ -143,6 +143,7 @@ Pipelines are defined declaratively in **YAML presets**. Each preset lists the s | Code | Language | | ---- | -------- | +| `ar` | Arabic | | `da` | Danish | | `de` | German | | `en` | English | From eb3ffbecf83842f27c5ae525f447114228da32f9 Mon Sep 17 00:00:00 2001 From: karamouche Date: Wed, 21 Oct 2026 19:42:14 -0400 Subject: [PATCH 08/11] fix(ar): keep real words out of the filler list MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Once alef is folded, the plural "أمم" (nations) normalizes to "امم" and was deleted as a hesitation. "اه" is also the dialectal "yes" in Egyptian and Levantine Arabic. Only the elongated forms (اهه, اممم, ممم) are fillers now. --- normalization/languages/arabic/operators.py | 4 ++-- tests/e2e/files/gladia-3/ar.csv | 6 ++++-- tests/unit/languages/arabic_operators_test.py | 15 ++++++++++++++- 3 files changed, 20 insertions(+), 5 deletions(-) diff --git a/normalization/languages/arabic/operators.py b/normalization/languages/arabic/operators.py index f148d3e..28c110f 100644 --- a/normalization/languages/arabic/operators.py +++ b/normalization/languages/arabic/operators.py @@ -67,10 +67,10 @@ **_LETTER_VARIANTS, **_DIGITS_AND_SEPARATORS, }, + # "اه" (dialectal "yes") and "امم" ("nations" once alef is folded) are real + # words, so only the elongated hesitation forms are treated as fillers. filler_words=[ - "اه", "اهه", - "امم", "اممم", "ممم", ], diff --git a/tests/e2e/files/gladia-3/ar.csv b/tests/e2e/files/gladia-3/ar.csv index edabbef..0e67989 100644 --- a/tests/e2e/files/gladia-3/ar.csv +++ b/tests/e2e/files/gladia-3/ar.csv @@ -11,8 +11,10 @@ input,expected ١٬٠٠٠ دولار,1000 دولار نسبة ٢٠٪,نسبه 20 بالميه يكلف $50,يكلف 50 دولار -آه، أنا هنا,انا هنا -امم نعم,نعم +آهه، أنا هنا,انا هنا +اممم نعم,نعم +أمم كثيرة,امم كثيره +اه والله,اه والله admin@example.com,admin ات example نقطه com 2 > 1,2 اكبر من 1 درجة الحرارة 30°C,درجه الحراره 30 درجه ميويه diff --git a/tests/unit/languages/arabic_operators_test.py b/tests/unit/languages/arabic_operators_test.py index 4dd8207..8700ea2 100644 --- a/tests/unit/languages/arabic_operators_test.py +++ b/tests/unit/languages/arabic_operators_test.py @@ -34,13 +34,26 @@ def test_config_code(operators: ArabicOperators) -> None: ("يكلف $50", "يكلف 50 دولار"), ("20%", "20 بالميه"), ("12.5", "12 فاصله 5"), - ("امم نعم", "نعم"), + ("اممم نعم", "نعم"), ], ) def test_pipeline_symbols_and_fillers(pipeline, text: str, expected: str) -> None: assert pipeline.normalize(text) == expected +@pytest.mark.parametrize( + ("text", "expected"), + [ + ("أمم كثيرة", "امم كثيره"), + ("اه والله", "اه والله"), + ], +) +def test_pipeline_keeps_real_words_that_look_like_fillers( + pipeline, text: str, expected: str +) -> None: + assert pipeline.normalize(text) == expected + + @pytest.mark.parametrize( ("text", "expected"), [ From 3da083f4d325799575534a5c857d720bdf47d02d Mon Sep 17 00:00:00 2001 From: karamouche Date: Wed, 21 Oct 2026 19:42:42 -0400 Subject: [PATCH 09/11] fix(ar): fold hamza seats explicitly and write config words in normalized form MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit remove_diacritics already folded ؤ to و and ئ to ي through NFKD, but only as a side effect, so symbol words such as "زائد" and "بالمئه" did not match what the pipeline outputs and replacement keys with hamza seats could never match. The folding is now part of character_replacements, the config words are written in their final form, and a test guards that invariant. --- normalization/languages/arabic/operators.py | 11 +++++++---- normalization/languages/arabic/replacements.py | 6 ++++-- tests/e2e/files/gladia-3/ar.csv | 1 + tests/unit/languages/arabic_operators_test.py | 17 +++++++++++++++++ 4 files changed, 29 insertions(+), 6 deletions(-) diff --git a/normalization/languages/arabic/operators.py b/normalization/languages/arabic/operators.py index 28c110f..a73dc91 100644 --- a/normalization/languages/arabic/operators.py +++ b/normalization/languages/arabic/operators.py @@ -17,6 +17,8 @@ "إ": "ا", # alef with hamza below "آ": "ا", # alef with madda "ٱ": "ا", # alef wasla + "ؤ": "و", # waw with hamza above + "ئ": "ي", # yeh with hamza above "ى": "ي", # alef maqsura "ة": "ه", # ta marbuta "ی": "ي", # Persian yeh @@ -37,7 +39,8 @@ ) # Words inserted by steps (symbols, currencies, fillers) are written in the -# normalized surface form so they match the rest of the normalized text. +# normalized surface form (no hamza seats, alef maqsura or ta marbuta) so they +# match the rest of the normalized text. ARABIC_CONFIG = LanguageConfig( code="ar", decimal_separator=".", @@ -46,14 +49,14 @@ symbols_to_words={ "@": "ات", ".": "نقطه", - "+": "زائد", + "+": "زايد", "=": "يساوي", ">": "اكبر من", "<": "اصغر من", "°": "درجه", - "°C": "درجه مئويه", + "°C": "درجه ميويه", "°F": "درجه فهرنهايت", - "%": "بالمئه", + "%": "بالميه", }, currency_symbol_to_word={ "$": "دولار", diff --git a/normalization/languages/arabic/replacements.py b/normalization/languages/arabic/replacements.py index 57b5e04..5fbc0d5 100644 --- a/normalization/languages/arabic/replacements.py +++ b/normalization/languages/arabic/replacements.py @@ -1,7 +1,9 @@ """Word-level substitutions for Arabic. -Keys and values must use the normalized surface form produced by the gladia-3 -pipeline (see ``character_replacements`` in ``operators.py``). +Word replacements run after every text_pre step, so keys and values must use +the normalized surface form: no harakat or tatweel, bare alef for أ/إ/آ/ٱ, +و for ؤ, ي for ئ and ى, ه for ة (see ``character_replacements`` in +``operators.py``). A key spelled with any of the folded letters never matches. """ ARABIC_REPLACEMENTS: dict[str, str] = {} diff --git a/tests/e2e/files/gladia-3/ar.csv b/tests/e2e/files/gladia-3/ar.csv index 0e67989..4a55010 100644 --- a/tests/e2e/files/gladia-3/ar.csv +++ b/tests/e2e/files/gladia-3/ar.csv @@ -19,4 +19,5 @@ admin@example.com,admin ات example نقطه com 2 > 1,2 اكبر من 1 درجة الحرارة 30°C,درجه الحراره 30 درجه ميويه مسؤول,مسوول +رئيس الوزراء,رييس الوزراء کتاب عربی,كتاب عربي diff --git a/tests/unit/languages/arabic_operators_test.py b/tests/unit/languages/arabic_operators_test.py index 8700ea2..f413940 100644 --- a/tests/unit/languages/arabic_operators_test.py +++ b/tests/unit/languages/arabic_operators_test.py @@ -28,6 +28,21 @@ def test_config_code(operators: ArabicOperators) -> None: assert operators.config.code == "ar" +def test_config_words_are_in_normalized_form(operators: ArabicOperators) -> None: + config = operators.config + assert config.character_replacements is not None + assert config.decimal_word is not None + folded = {k for k, v in config.character_replacements.items() if k != v} + words = [ + *config.symbols_to_words.values(), + *config.currency_symbol_to_word.values(), + *config.filler_words, + config.decimal_word, + ] + for word in words: + assert not folded & set(word), word + + @pytest.mark.parametrize( ("text", "expected"), [ @@ -78,6 +93,8 @@ def test_pipeline_strips_harakat_and_tatweel( ("مستشفى", "مستشفي"), ("مستشفي", "مستشفي"), ("کتاب عربی", "كتاب عربي"), + ("سؤال", "سوال"), + ("رئيس", "رييس"), ], ) def test_pipeline_unifies_letter_variants(pipeline, text: str, expected: str) -> None: From 6659a60c5f1d9b1a8591639947a988b90f520bc3 Mon Sep 17 00:00:00 2001 From: karamouche Date: Wed, 21 Oct 2026 19:53:39 -0400 Subject: [PATCH 10/11] feat(ar): convert spelled-out Arabic numbers to digits MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds ArabicNumberNormalizer behind expand_written_numbers, so references and hypotheses that write the same number as words or digits normalize alike (خمسة وعشرون and ٢٥ both give 25). Covers MSA cardinals up to billions with gender and case variants, two-word teens, units-before-tens compounds chained with و, glued and separate hundreds, ألف/آلاف/مليون/ملايين groups, digits followed by a scale word, and common Egyptian and Levantine spellings. و is only consumed where it builds a compound, so ثلاثة وأربعة stays "3 و4" like ٣ و٤. Standalone واحد/أحد stay words, as English keeps "one". digit_words enables digit-by-digit readings. Ordinals, fractions (نص, ربع) and dual nouns are out of scope. --- .../languages/arabic/number_normalizer.py | 390 ++++++++++++++++++ normalization/languages/arabic/operators.py | 28 ++ tests/e2e/files/gladia-3/ar.csv | 10 + .../arabic_number_normalizer_test.py | 190 +++++++++ tests/unit/languages/arabic_operators_test.py | 23 ++ 5 files changed, 641 insertions(+) create mode 100644 normalization/languages/arabic/number_normalizer.py create mode 100644 tests/unit/languages/arabic_number_normalizer_test.py diff --git a/normalization/languages/arabic/number_normalizer.py b/normalization/languages/arabic/number_normalizer.py new file mode 100644 index 0000000..69df77e --- /dev/null +++ b/normalization/languages/arabic/number_normalizer.py @@ -0,0 +1,390 @@ +"""Arabic number normalizer (STT-oriented). + +``text2num.alpha2digit`` does not support Arabic, so this module parses spelled-out +cardinals for common transcript patterns: Modern Standard Arabic with its gender and +case variants (``ثلاث``/``ثلاثة``, ``عشرون``/``عشرين``), plus frequent Egyptian and +Levantine spellings (``تلاتة``, ``اتناشر``, ``تلتمية``). + +Arabic puts units before tens and chains every further component with the +conjunction ``و`` glued to the next word: ``مئة وخمسة وعشرون`` = 125, +``ألفين وخمسة وعشرين`` = 2025. ``و`` only joins a unit to a tens word, a hundreds +word to its remainder, or a larger scale group to a smaller one; anywhere else it is +the plain "and" and is left alone (``ثلاثة وأربعة`` → ``3 و4``). + +Lookups go through the language's ``character_replacements`` so tables can be +written in their usual spelling and still match folded text (``مئة`` and ``مية`` +both fold to ``ميه``). Standalone ``واحد``/``أحد`` stay words, like English "one", +because they double as pronouns. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass + +_ZERO = "صفر" + +# Words for 1: converted only inside a larger number. +_ONES: dict[str, int] = {"واحد": 1, "واحدة": 1, "أحد": 1, "إحدى": 1} + +_UNITS: dict[str, int] = { + "اثنان": 2, + "اثنين": 2, + "اثنتان": 2, + "اثنتين": 2, + "اتنين": 2, + "ثنين": 2, + "ثنتين": 2, + "ثلاثة": 3, + "ثلاث": 3, + "تلاتة": 3, + "تلات": 3, + "أربعة": 4, + "أربع": 4, + "خمسة": 5, + "خمس": 5, + "ستة": 6, + "ست": 6, + "سبعة": 7, + "سبع": 7, + "ثمانية": 8, + "ثماني": 8, + "ثمان": 8, + "تمانية": 8, + "تمنية": 8, + "تسعة": 9, + "تسع": 9, +} + +# First word of a two-word teen (أحد عشر, اثنا عشر, ثلاثة عشر, ...). +_TEEN_FIRST: dict[str, int] = { + "أحد": 1, + "إحدى": 1, + "اثنا": 2, + "اثني": 2, + "اثنتا": 2, + "اثنتي": 2, + **_UNITS, +} + +_TEN: dict[str, int] = {"عشرة": 10, "عشر": 10} + +# Glued dialect teens (Egyptian / Levantine). +_DIALECT_TEENS: dict[str, int] = { + "حداشر": 11, + "احداشر": 11, + "اتناشر": 12, + "اطناشر": 12, + "تلتاشر": 13, + "تلاتاشر": 13, + "اربعتاشر": 14, + "اربعطاشر": 14, + "خمستاشر": 15, + "خمسطاشر": 15, + "ستاشر": 16, + "سطاشر": 16, + "سبعتاشر": 17, + "سبعطاشر": 17, + "تمنتاشر": 18, + "تمانتاشر": 18, + "تمنطاشر": 18, + "تسعتاشر": 19, + "تسعطاشر": 19, +} + +_TENS: dict[str, int] = { + "عشرون": 20, + "عشرين": 20, + "ثلاثون": 30, + "ثلاثين": 30, + "تلاتين": 30, + "أربعون": 40, + "أربعين": 40, + "خمسون": 50, + "خمسين": 50, + "ستون": 60, + "ستين": 60, + "سبعون": 70, + "سبعين": 70, + "ثمانون": 80, + "ثمانين": 80, + "تمانين": 80, + "تسعون": 90, + "تسعين": 90, +} + +_HUNDRED_WORDS = ("مئة", "مائة", "مية") + +# Unit stems glued to a hundred word (ثلاثمئة, تلتمية, ...). +_HUNDRED_PREFIXES: dict[str, int] = { + "ثلاث": 3, + "تلت": 3, + "تلات": 3, + "أربع": 4, + "ربع": 4, + "خمس": 5, + "ست": 6, + "سبع": 7, + "ثمان": 8, + "ثماني": 8, + "تمن": 8, + "تمان": 8, + "تسع": 9, +} + +_HUNDREDS: dict[str, int] = { + **dict.fromkeys(_HUNDRED_WORDS, 100), + "مئتان": 200, + "مئتين": 200, + "مائتان": 200, + "مائتين": 200, + **{ + prefix + word: value * 100 + for prefix, value in _HUNDRED_PREFIXES.items() + for word in _HUNDRED_WORDS + }, +} + +# Scale words that follow a count (ثلاثة آلاف, خمسة ملايين). +_MULTIPLIERS: dict[str, int] = { + "ألف": 1_000, + "ألفا": 1_000, + "آلاف": 1_000, + "تلاف": 1_000, + "مليون": 1_000_000, + "مليونا": 1_000_000, + "ملايين": 1_000_000, + "مليار": 1_000_000_000, + "مليارا": 1_000_000_000, + "مليارات": 1_000_000_000, + "بليون": 1_000_000_000, +} + +# Scale words that carry their own count (ألف = 1000, ألفين = 2000), with their scale. +_STANDALONE_SCALES: dict[str, tuple[int, int]] = { + "ألف": (1_000, 1_000), + "ألفا": (1_000, 1_000), + "ألفان": (2_000, 1_000), + "ألفين": (2_000, 1_000), + "مليون": (1_000_000, 1_000_000), + "مليونان": (2_000_000, 1_000_000), + "مليونين": (2_000_000, 1_000_000), + "مليار": (1_000_000_000, 1_000_000_000), + "ملياران": (2_000_000_000, 1_000_000_000), + "مليارين": (2_000_000_000, 1_000_000_000), + "بليون": (1_000_000_000, 1_000_000_000), +} + +_CONJUNCTION = "و" + +_RE_PUNCT_EDGES = re.compile(r"^(\W*)(.*?)(\W*)$", re.DOTALL) + + +@dataclass +class _Token: + raw: str + """Original text, emitted unchanged when the token is not part of a number.""" + lead: str = "" + trail: str = "" + word: str | None = None + """Folded number word (or ASCII digits) with any conjunction stripped.""" + conj: str = "" + """``"و"`` when glued to the word, ``"و "`` when written as its own token.""" + + +class ArabicNumberNormalizer: + """Convert Arabic spelled-out numbers to digits.""" + + def __init__(self, character_replacements: dict[str, str] | None = None) -> None: + self._table = str.maketrans(character_replacements or {}) + fold = self._fold + self._ones = {fold(k): v for k, v in _ONES.items()} + self._units = {fold(k): v for k, v in _UNITS.items()} + self._teen_first = {fold(k): v for k, v in _TEEN_FIRST.items()} + self._ten = {fold(k): v for k, v in _TEN.items()} + self._dialect_teens = {fold(k): v for k, v in _DIALECT_TEENS.items()} + self._tens = {fold(k): v for k, v in _TENS.items()} + self._hundred_words = {fold(w) for w in _HUNDRED_WORDS} + self._hundreds = {fold(k): v for k, v in _HUNDREDS.items()} + self._multipliers = {fold(k): v for k, v in _MULTIPLIERS.items()} + self._standalone_scales = {fold(k): v for k, v in _STANDALONE_SCALES.items()} + self._zero = fold(_ZERO) + self._wahid = fold("واحد") + self._lexicon = { + self._zero, + *self._ones, + *self._teen_first, + *self._ten, + *self._dialect_teens, + *self._tens, + *self._hundreds, + *self._multipliers, + *self._standalone_scales, + } + + def _fold(self, word: str) -> str: + return word.translate(self._table) + + def __call__(self, text: str) -> str: + if not text.strip(): + return text + toks = self._tokenize(text.split()) + out: list[str] = [] + i = 0 + while i < len(toks): + parsed = self._parse_number(toks, i) + single_one = ( + parsed is not None and parsed[0] == i + 1 and toks[i].word in self._ones + ) + if parsed is None or single_one: + out.append(toks[i].raw) + i += 1 + continue + end, value = parsed + first, last = toks[i], toks[end - 1] + out.append(f"{first.lead}{first.conj}{value}{last.trail}") + i = end + return " ".join(out) + + def _is_number_word(self, word: str) -> bool: + return word in self._lexicon or (word.isascii() and word.isdigit()) + + def _tokenize(self, words: list[str]) -> list[_Token]: + toks: list[_Token] = [] + i = 0 + while i < len(words): + raw = words[i] + lead, word, trail = self._split(raw) + next_word = self._split(words[i + 1])[1] if i + 1 < len(words) else "" + # "وأحد" folds to "واحد"; after a number and before عشر it is و + أحد + # (مئة وأحد عشر = 111), since "واحد عشر" is not a valid teen. + is_and_ahad = ( + word == self._wahid + and next_word in self._ten + and bool(toks) + and toks[-1].word is not None + ) + if self._is_number_word(word) and not is_and_ahad: + toks.append(_Token(raw, lead, trail, word)) + elif ( + word.startswith(_CONJUNCTION) + and not lead + and self._is_number_word(word[1:]) + ): + toks.append(_Token(raw, lead, trail, word[1:], _CONJUNCTION)) + elif word == _CONJUNCTION and not lead and not trail and next_word: + next_lead, _, next_trail = self._split(words[i + 1]) + if next_lead or next_word not in self._lexicon: + toks.append(_Token(raw)) + else: + merged = f"{raw} {words[i + 1]}" + conj = f"{_CONJUNCTION} " + toks.append(_Token(merged, "", next_trail, next_word, conj)) + i += 1 + else: + toks.append(_Token(raw)) + i += 1 + return toks + + def _split(self, raw: str) -> tuple[str, str, str]: + """Fold a raw token and split it into (leading punct, word, trailing punct).""" + match = _RE_PUNCT_EDGES.match(self._fold(raw)) + assert match is not None + return match.group(1), match.group(2), match.group(3) + + @staticmethod + def _joinable(toks: list[_Token], i: int) -> bool: + """Whether toks[i] can continue the number ending at toks[i - 1].""" + return i < len(toks) and not toks[i - 1].trail and not toks[i].lead + + def _parse_number(self, toks: list[_Token], i: int) -> tuple[int, int] | None: + term = self._parse_term(toks, i) + if term is None: + return None + end, total, scale = term + # Larger groups are followed by smaller ones: ألف وتسعمائة وتسعين. + while scale > 1 and self._joinable(toks, end) and toks[end].conj: + nxt = self._parse_term(toks, end) + if nxt is None or nxt[2] >= scale: + break + end, value, scale = nxt + total += value + return end, total + + def _parse_term(self, toks: list[_Token], i: int) -> tuple[int, int, int] | None: + """Parse one scale group; returns (end, value, scale).""" + word = toks[i].word + if word is None: + return None + if word in self._standalone_scales: + value, scale = self._standalone_scales[word] + return i + 1, value, scale + if word.isascii() and word.isdigit(): + j = i + 1 + if self._is_multiplier(toks, j): + mult = self._multipliers[toks[j].word or ""] + return j + 1, int(word) * mult, mult + return None + parsed = self._parse_below_1000(toks, i) + if parsed is None: + return None + j, value = parsed + if self._is_multiplier(toks, j): + mult = self._multipliers[toks[j].word or ""] + return j + 1, value * mult, mult + return j, value, 1 + + def _is_multiplier(self, toks: list[_Token], j: int) -> bool: + return ( + self._joinable(toks, j) + and not toks[j].conj + and toks[j].word in self._multipliers + ) + + def _parse_below_1000(self, toks: list[_Token], i: int) -> tuple[int, int] | None: + word = toks[i].word + if word in self._hundreds: + hundreds, j = self._hundreds[word], i + 1 + elif ( + word in self._units + and self._joinable(toks, i + 1) + and not toks[i + 1].conj + and toks[i + 1].word in self._hundred_words + ): + hundreds, j = self._units[word] * 100, i + 2 + else: + return self._parse_below_100(toks, i) + if self._joinable(toks, j) and toks[j].conj: + rest = self._parse_below_100(toks, j) + if rest is not None: + return rest[0], hundreds + rest[1] + return j, hundreds + + def _parse_below_100(self, toks: list[_Token], i: int) -> tuple[int, int] | None: + word = toks[i].word + if word is None: + return None + if word in self._dialect_teens: + return i + 1, self._dialect_teens[word] + if ( + word in self._teen_first + and self._joinable(toks, i + 1) + and not toks[i + 1].conj + and toks[i + 1].word in self._ten + ): + return i + 2, 10 + self._teen_first[word] + if word in self._tens: + return i + 1, self._tens[word] + unit = self._units.get(word, self._ones.get(word)) + if unit is not None: + nxt = i + 1 + if self._joinable(toks, nxt) and toks[nxt].conj: + tens = self._tens.get(toks[nxt].word or "") + if tens is not None: + return nxt + 1, unit + tens + return nxt, unit + if word in self._ten: + return i + 1, 10 + if word == self._zero: + return i + 1, 0 + return None diff --git a/normalization/languages/arabic/operators.py b/normalization/languages/arabic/operators.py index a73dc91..5f4e6be 100644 --- a/normalization/languages/arabic/operators.py +++ b/normalization/languages/arabic/operators.py @@ -1,3 +1,4 @@ +from normalization.languages.arabic.number_normalizer import ArabicNumberNormalizer from normalization.languages.arabic.replacements import ARABIC_REPLACEMENTS from normalization.languages.base import LanguageConfig, LanguageOperators from normalization.languages.registry import register_language @@ -38,6 +39,25 @@ } ) +# Digit words for digit-by-digit readings (phone numbers, codes). Only the full +# forms: short forms like ست / سبع are also common nouns. +_ARABIC_DIGIT_WORDS: dict[str, str] = { + "صفر": "0", + "واحد": "1", + "اثنين": "2", + "اثنان": "2", + "اتنين": "2", + "ثلاثه": "3", + "تلاته": "3", + "اربعه": "4", + "خمسه": "5", + "سته": "6", + "سبعه": "7", + "ثمانيه": "8", + "تمانيه": "8", + "تسعه": "9", +} + # Words inserted by steps (symbols, currencies, fillers) are written in the # normalized surface form (no hamza seats, alef maqsura or ta marbuta) so they # match the rest of the normalized text. @@ -77,6 +97,7 @@ "اممم", "ممم", ], + digit_words=_ARABIC_DIGIT_WORDS, ) @@ -84,6 +105,13 @@ class ArabicOperators(LanguageOperators): def __init__(self) -> None: super().__init__(ARABIC_CONFIG) + self._number_normalizer = ArabicNumberNormalizer( + ARABIC_CONFIG.character_replacements + ) + + def expand_written_numbers(self, text: str) -> str: + """Convert Arabic spelled-out numbers to digits (e.g. خمسة وعشرون → 25).""" + return self._number_normalizer(text) def get_word_replacements(self) -> dict[str, str]: return ARABIC_REPLACEMENTS diff --git a/tests/e2e/files/gladia-3/ar.csv b/tests/e2e/files/gladia-3/ar.csv index 4a55010..f25a8b4 100644 --- a/tests/e2e/files/gladia-3/ar.csv +++ b/tests/e2e/files/gladia-3/ar.csv @@ -21,3 +21,13 @@ admin@example.com,admin ات example نقطه com مسؤول,مسوول رئيس الوزراء,رييس الوزراء کتاب عربی,كتاب عربي +خمسة وعشرون,25 +مئة وخمسة وعشرون,125 +ألفين وخمسة وعشرين,2025 +ثلاثة آلاف دولار,3000 دولار +عشرون بالمئة,20 بالميه +ثلاثة فاصلة خمسة,3 فاصله 5 +ثلاثة وأربعة,3 و4 +تلتمية وخمسين,350 +جاء واحد من الطلاب,جاء واحد من الطلاب +اتصل على صفر خمسة صفر واحد اثنين ثلاثة,اتصل علي 050123 diff --git a/tests/unit/languages/arabic_number_normalizer_test.py b/tests/unit/languages/arabic_number_normalizer_test.py new file mode 100644 index 0000000..e9a72bb --- /dev/null +++ b/tests/unit/languages/arabic_number_normalizer_test.py @@ -0,0 +1,190 @@ +import pytest + +from normalization.languages.arabic.number_normalizer import ArabicNumberNormalizer +from normalization.languages.arabic.operators import ARABIC_CONFIG + + +@pytest.fixture(scope="module") +def normalizer() -> ArabicNumberNormalizer: + return ArabicNumberNormalizer(ARABIC_CONFIG.character_replacements) + + +@pytest.mark.parametrize( + ("text", "expected"), + [ + ("صفر", "0"), + ("اثنان", "2"), + ("اثنين", "2"), + ("ثلاثة", "3"), + ("ثلاث", "3"), + ("أربعة", "4"), + ("خمسة", "5"), + ("ستة", "6"), + ("سبعة", "7"), + ("ثمانية", "8"), + ("ثماني", "8"), + ("تسعة", "9"), + ("عشرة", "10"), + ("عشر", "10"), + ], +) +def test_units(normalizer: ArabicNumberNormalizer, text: str, expected: str) -> None: + assert normalizer(text) == expected + + +@pytest.mark.parametrize( + "text", + ["واحد", "واحدة", "أحد", "إحدى", "ولد واحد", "يوم الأحد"], +) +def test_standalone_one_is_kept_as_a_word( + normalizer: ArabicNumberNormalizer, text: str +) -> None: + # Like English "one", these double as pronouns ("someone", "one of"). + assert normalizer(text) == text + + +@pytest.mark.parametrize( + ("text", "expected"), + [ + ("أحد عشر", "11"), + ("إحدى عشرة", "11"), + ("اثنا عشر", "12"), + ("اثني عشر", "12"), + ("اثنتا عشرة", "12"), + ("ثلاثة عشر", "13"), + ("خمس عشرة", "15"), + ("تسعة عشر", "19"), + ], +) +def test_teens(normalizer: ArabicNumberNormalizer, text: str, expected: str) -> None: + assert normalizer(text) == expected + + +@pytest.mark.parametrize( + ("text", "expected"), + [ + ("عشرون", "20"), + ("عشرين", "20"), + ("ثلاثون", "30"), + ("أربعين", "40"), + ("تسعون", "90"), + ("واحد وعشرون", "21"), + ("خمسة وعشرون", "25"), + ("خمسة و عشرين", "25"), + ("ثلاثة وتسعين", "93"), + ], +) +def test_tens_and_compounds( + normalizer: ArabicNumberNormalizer, text: str, expected: str +) -> None: + assert normalizer(text) == expected + + +@pytest.mark.parametrize( + ("text", "expected"), + [ + ("مئة", "100"), + ("مائة", "100"), + ("مية", "100"), + ("مئتان", "200"), + ("مئتين", "200"), + ("ثلاثمئة", "300"), + ("ثلاثمائة", "300"), + ("ثلاث مئة", "300"), + ("خمسمية", "500"), + ("تسعمائة", "900"), + ("مئة وخمسة", "105"), + ("مئة وواحد", "101"), + ("مئة وخمسة وعشرون", "125"), + ("ثلاثمئة وأحد عشر", "311"), + ], +) +def test_hundreds(normalizer: ArabicNumberNormalizer, text: str, expected: str) -> None: + assert normalizer(text) == expected + + +@pytest.mark.parametrize( + ("text", "expected"), + [ + ("ألف", "1000"), + ("ألفان", "2000"), + ("ألفين", "2000"), + ("ثلاثة آلاف", "3000"), + ("عشرة آلاف", "10000"), + ("أحد عشر ألفاً", "11000"), + ("خمسة وعشرين ألف", "25000"), + ("مئة ألف", "100000"), + ("ألف وتسعمائة وتسعين", "1990"), + ("ألفين وخمسة وعشرين", "2025"), + ("ثلاثة آلاف وخمسمائة", "3500"), + ("مليون", "1000000"), + ("مليونين", "2000000"), + ("ثلاثة ملايين", "3000000"), + ("ثلاثة ملايين وخمسمائة ألف", "3500000"), + ("مليار", "1000000000"), + ("خمسة مليارات", "5000000000"), + ], +) +def test_thousands_and_above( + normalizer: ArabicNumberNormalizer, text: str, expected: str +) -> None: + assert normalizer(text) == expected + + +@pytest.mark.parametrize( + ("text", "expected"), + [ + ("3 آلاف", "3000"), + ("٥ ملايين", "5000000"), + ("25 ألف", "25000"), + ], +) +def test_digits_followed_by_a_scale_word( + normalizer: ArabicNumberNormalizer, text: str, expected: str +) -> None: + assert normalizer(text) == expected + + +@pytest.mark.parametrize( + ("text", "expected"), + [ + ("اتنين", "2"), + ("تلاتة", "3"), + ("تمانية", "8"), + ("حداشر", "11"), + ("اتناشر", "12"), + ("خمستاشر", "15"), + ("تلاتين", "30"), + ("تلتمية", "300"), + ("ميتين", "200"), + ("تلات تلاف", "3000"), + ], +) +def test_dialect_spellings( + normalizer: ArabicNumberNormalizer, text: str, expected: str +) -> None: + assert normalizer(text) == expected + + +@pytest.mark.parametrize( + ("text", "expected"), + [ + # "و" only joins a unit to a tens word or a smaller group to a larger + # one; anything else is the plain conjunction "and". + ("ثلاثة وأربعة", "3 و4"), + ("عشرين وخمسة", "20 و5"), + ("و خمسة", "و 5"), + ("عندي خمسة وعشرون كتاب", "عندي 25 كتاب"), + ("سنة ألفين وعشرين", "سنة 2020"), + ("خمسة، ستة", "5، 6"), + ("الساعة ثلاثة", "الساعة 3"), + ("ثلاثة فاصلة خمسة", "3 فاصلة 5"), + ("عشرين بالمئة", "20 بالمئة"), + ("من غير أرقام", "من غير أرقام"), + ("", ""), + ], +) +def test_numbers_in_context( + normalizer: ArabicNumberNormalizer, text: str, expected: str +) -> None: + assert normalizer(text) == expected diff --git a/tests/unit/languages/arabic_operators_test.py b/tests/unit/languages/arabic_operators_test.py index f413940..6bcd4e8 100644 --- a/tests/unit/languages/arabic_operators_test.py +++ b/tests/unit/languages/arabic_operators_test.py @@ -37,6 +37,7 @@ def test_config_words_are_in_normalized_form(operators: ArabicOperators) -> None *config.symbols_to_words.values(), *config.currency_symbol_to_word.values(), *config.filler_words, + *(config.digit_words or {}), config.decimal_word, ] for word in words: @@ -115,3 +116,25 @@ def test_pipeline_converts_native_digits_and_separators( pipeline, text: str, expected: str ) -> None: assert pipeline.normalize(text) == expected + + +@pytest.mark.parametrize( + ("words", "digits"), + [ + ("عندي خمسة وعشرون كتاباً", "عندي ٢٥ كتاباً"), + ("سنة ألفين وخمسة وعشرين", "سنة ٢٠٢٥"), + ("ثلاثة فاصلة خمسة", "٣٫٥"), + ("عشرون بالمئة", "٢٠٪"), + ("ثلاثة آلاف دولار", "٣٬٠٠٠ دولار"), + ("ثلاثة وأربعة", "٣ و٤"), + ("مئة وخمسة وعشرون", "١٢٥"), + ], +) +def test_pipeline_spelled_out_numbers_match_digits( + pipeline, words: str, digits: str +) -> None: + assert pipeline.normalize(words) == pipeline.normalize(digits) + + +def test_pipeline_collapses_digit_word_sequences(pipeline) -> None: + assert pipeline.normalize("واحد اثنين ثلاثة أربعة") == "1234" From 1edc13780fc3d6cf80dcc39522fa0dbb45d125ce Mon Sep 17 00:00:00 2001 From: Jean-Louis Queguiner Date: Wed, 11 Nov 2026 10:27:04 -0500 Subject: [PATCH 11/11] fix(ar): enable digit-word punctuation collapse including Arabic comma MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wire number_words from digit vocabulary and treat U+060C like ASCII commas so ASR sequences such as صفر، خمسة fold before digit conversion. Co-authored-by: Cursor --- normalization/languages/arabic/operators.py | 4 ++++ .../steps/text/normalize_punctuation_between_number_words.py | 4 +++- tests/e2e/files/gladia-3/ar.csv | 1 + 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/normalization/languages/arabic/operators.py b/normalization/languages/arabic/operators.py index 5f4e6be..d710683 100644 --- a/normalization/languages/arabic/operators.py +++ b/normalization/languages/arabic/operators.py @@ -98,6 +98,10 @@ "ممم", ], digit_words=_ARABIC_DIGIT_WORDS, + # Same vocabulary as digit_words: used by + # normalize_punctuation_between_number_words so "صفر، خمسة" becomes + # "صفر خمسة" before convert_digit_word_sequences_to_digits. + number_words=list(_ARABIC_DIGIT_WORDS), ) diff --git a/normalization/steps/text/normalize_punctuation_between_number_words.py b/normalization/steps/text/normalize_punctuation_between_number_words.py index f088ef0..aa2653e 100644 --- a/normalization/steps/text/normalize_punctuation_between_number_words.py +++ b/normalization/steps/text/normalize_punctuation_between_number_words.py @@ -24,7 +24,9 @@ def __call__(self, text: str, operators: LanguageOperators) -> str: alternation = "|".join(re.escape(w) for w in sorted_words) number_group = f"(?:{alternation})" pattern = re.compile( - rf"({number_group})\s*[,.\-…]+\s*(?={number_group})", + # Include Arabic comma (U+060C) so digit-by-digit ASR like + # "صفر، خمسة" collapses the same way as "zero, five". + rf"({number_group})\s*[,.\-…\u060c]+\s*(?={number_group})", flags=re.IGNORECASE, ) return pattern.sub(r"\1 ", text) diff --git a/tests/e2e/files/gladia-3/ar.csv b/tests/e2e/files/gladia-3/ar.csv index f25a8b4..0efb480 100644 --- a/tests/e2e/files/gladia-3/ar.csv +++ b/tests/e2e/files/gladia-3/ar.csv @@ -31,3 +31,4 @@ admin@example.com,admin ات example نقطه com تلتمية وخمسين,350 جاء واحد من الطلاب,جاء واحد من الطلاب اتصل على صفر خمسة صفر واحد اثنين ثلاثة,اتصل علي 050123 +صفر، خمسة، واحد,051