diff --git a/README.md b/README.md index 7ffa32c..aa47045 100644 --- a/README.md +++ b/README.md @@ -143,6 +143,7 @@ Pipelines are defined declaratively in **YAML presets**. Each preset lists the s | Code | Language | | ---- | -------- | +| `ar` | Arabic | | `da` | Danish | | `de` | German | | `en` | English | diff --git a/docs/reference/steps.md b/docs/reference/steps.md index 1b6d764..04e797d 100644 --- a/docs/reference/steps.md +++ b/docs/reference/steps.md @@ -4,6 +4,16 @@ Auto-generated — do not edit by hand. Run `scripts/generate_step_docs.py` to u ## Text Steps +### `apply_character_replacements` + +**Base class:** `TextStep` + +Replace or delete single characters using ``character_replacements`` from the language config. + +Runs first in the pipeline so that script-level variants (letter forms, native +digits, vowel marks) are unified before protection, numeric, and symbol steps. +Skipped when the language does not define ``character_replacements``. + ### `apply_sentence_level_replacements` **Base class:** `TextStep` diff --git a/normalization/languages/__init__.py b/normalization/languages/__init__.py index 3d0c998..0d446fd 100644 --- a/normalization/languages/__init__.py +++ b/normalization/languages/__init__.py @@ -1,4 +1,5 @@ from . import ( + arabic, danish, dutch, english, @@ -18,6 +19,7 @@ register_language(LanguageOperators) __all__ = [ + "arabic", "danish", "dutch", "english", diff --git a/normalization/languages/arabic/__init__.py b/normalization/languages/arabic/__init__.py new file mode 100644 index 0000000..17e90f1 --- /dev/null +++ b/normalization/languages/arabic/__init__.py @@ -0,0 +1,7 @@ +from .operators import ArabicOperators +from .replacements import ARABIC_REPLACEMENTS + +__all__ = [ + "ArabicOperators", + "ARABIC_REPLACEMENTS", +] diff --git a/normalization/languages/arabic/number_normalizer.py b/normalization/languages/arabic/number_normalizer.py new file mode 100644 index 0000000..69df77e --- /dev/null +++ b/normalization/languages/arabic/number_normalizer.py @@ -0,0 +1,390 @@ +"""Arabic number normalizer (STT-oriented). + +``text2num.alpha2digit`` does not support Arabic, so this module parses spelled-out +cardinals for common transcript patterns: Modern Standard Arabic with its gender and +case variants (``ثلاث``/``ثلاثة``, ``عشرون``/``عشرين``), plus frequent Egyptian and +Levantine spellings (``تلاتة``, ``اتناشر``, ``تلتمية``). + +Arabic puts units before tens and chains every further component with the +conjunction ``و`` glued to the next word: ``مئة وخمسة وعشرون`` = 125, +``ألفين وخمسة وعشرين`` = 2025. ``و`` only joins a unit to a tens word, a hundreds +word to its remainder, or a larger scale group to a smaller one; anywhere else it is +the plain "and" and is left alone (``ثلاثة وأربعة`` → ``3 و4``). + +Lookups go through the language's ``character_replacements`` so tables can be +written in their usual spelling and still match folded text (``مئة`` and ``مية`` +both fold to ``ميه``). Standalone ``واحد``/``أحد`` stay words, like English "one", +because they double as pronouns. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass + +_ZERO = "صفر" + +# Words for 1: converted only inside a larger number. +_ONES: dict[str, int] = {"واحد": 1, "واحدة": 1, "أحد": 1, "إحدى": 1} + +_UNITS: dict[str, int] = { + "اثنان": 2, + "اثنين": 2, + "اثنتان": 2, + "اثنتين": 2, + "اتنين": 2, + "ثنين": 2, + "ثنتين": 2, + "ثلاثة": 3, + "ثلاث": 3, + "تلاتة": 3, + "تلات": 3, + "أربعة": 4, + "أربع": 4, + "خمسة": 5, + "خمس": 5, + "ستة": 6, + "ست": 6, + "سبعة": 7, + "سبع": 7, + "ثمانية": 8, + "ثماني": 8, + "ثمان": 8, + "تمانية": 8, + "تمنية": 8, + "تسعة": 9, + "تسع": 9, +} + +# First word of a two-word teen (أحد عشر, اثنا عشر, ثلاثة عشر, ...). +_TEEN_FIRST: dict[str, int] = { + "أحد": 1, + "إحدى": 1, + "اثنا": 2, + "اثني": 2, + "اثنتا": 2, + "اثنتي": 2, + **_UNITS, +} + +_TEN: dict[str, int] = {"عشرة": 10, "عشر": 10} + +# Glued dialect teens (Egyptian / Levantine). +_DIALECT_TEENS: dict[str, int] = { + "حداشر": 11, + "احداشر": 11, + "اتناشر": 12, + "اطناشر": 12, + "تلتاشر": 13, + "تلاتاشر": 13, + "اربعتاشر": 14, + "اربعطاشر": 14, + "خمستاشر": 15, + "خمسطاشر": 15, + "ستاشر": 16, + "سطاشر": 16, + "سبعتاشر": 17, + "سبعطاشر": 17, + "تمنتاشر": 18, + "تمانتاشر": 18, + "تمنطاشر": 18, + "تسعتاشر": 19, + "تسعطاشر": 19, +} + +_TENS: dict[str, int] = { + "عشرون": 20, + "عشرين": 20, + "ثلاثون": 30, + "ثلاثين": 30, + "تلاتين": 30, + "أربعون": 40, + "أربعين": 40, + "خمسون": 50, + "خمسين": 50, + "ستون": 60, + "ستين": 60, + "سبعون": 70, + "سبعين": 70, + "ثمانون": 80, + "ثمانين": 80, + "تمانين": 80, + "تسعون": 90, + "تسعين": 90, +} + +_HUNDRED_WORDS = ("مئة", "مائة", "مية") + +# Unit stems glued to a hundred word (ثلاثمئة, تلتمية, ...). +_HUNDRED_PREFIXES: dict[str, int] = { + "ثلاث": 3, + "تلت": 3, + "تلات": 3, + "أربع": 4, + "ربع": 4, + "خمس": 5, + "ست": 6, + "سبع": 7, + "ثمان": 8, + "ثماني": 8, + "تمن": 8, + "تمان": 8, + "تسع": 9, +} + +_HUNDREDS: dict[str, int] = { + **dict.fromkeys(_HUNDRED_WORDS, 100), + "مئتان": 200, + "مئتين": 200, + "مائتان": 200, + "مائتين": 200, + **{ + prefix + word: value * 100 + for prefix, value in _HUNDRED_PREFIXES.items() + for word in _HUNDRED_WORDS + }, +} + +# Scale words that follow a count (ثلاثة آلاف, خمسة ملايين). +_MULTIPLIERS: dict[str, int] = { + "ألف": 1_000, + "ألفا": 1_000, + "آلاف": 1_000, + "تلاف": 1_000, + "مليون": 1_000_000, + "مليونا": 1_000_000, + "ملايين": 1_000_000, + "مليار": 1_000_000_000, + "مليارا": 1_000_000_000, + "مليارات": 1_000_000_000, + "بليون": 1_000_000_000, +} + +# Scale words that carry their own count (ألف = 1000, ألفين = 2000), with their scale. +_STANDALONE_SCALES: dict[str, tuple[int, int]] = { + "ألف": (1_000, 1_000), + "ألفا": (1_000, 1_000), + "ألفان": (2_000, 1_000), + "ألفين": (2_000, 1_000), + "مليون": (1_000_000, 1_000_000), + "مليونان": (2_000_000, 1_000_000), + "مليونين": (2_000_000, 1_000_000), + "مليار": (1_000_000_000, 1_000_000_000), + "ملياران": (2_000_000_000, 1_000_000_000), + "مليارين": (2_000_000_000, 1_000_000_000), + "بليون": (1_000_000_000, 1_000_000_000), +} + +_CONJUNCTION = "و" + +_RE_PUNCT_EDGES = re.compile(r"^(\W*)(.*?)(\W*)$", re.DOTALL) + + +@dataclass +class _Token: + raw: str + """Original text, emitted unchanged when the token is not part of a number.""" + lead: str = "" + trail: str = "" + word: str | None = None + """Folded number word (or ASCII digits) with any conjunction stripped.""" + conj: str = "" + """``"و"`` when glued to the word, ``"و "`` when written as its own token.""" + + +class ArabicNumberNormalizer: + """Convert Arabic spelled-out numbers to digits.""" + + def __init__(self, character_replacements: dict[str, str] | None = None) -> None: + self._table = str.maketrans(character_replacements or {}) + fold = self._fold + self._ones = {fold(k): v for k, v in _ONES.items()} + self._units = {fold(k): v for k, v in _UNITS.items()} + self._teen_first = {fold(k): v for k, v in _TEEN_FIRST.items()} + self._ten = {fold(k): v for k, v in _TEN.items()} + self._dialect_teens = {fold(k): v for k, v in _DIALECT_TEENS.items()} + self._tens = {fold(k): v for k, v in _TENS.items()} + self._hundred_words = {fold(w) for w in _HUNDRED_WORDS} + self._hundreds = {fold(k): v for k, v in _HUNDREDS.items()} + self._multipliers = {fold(k): v for k, v in _MULTIPLIERS.items()} + self._standalone_scales = {fold(k): v for k, v in _STANDALONE_SCALES.items()} + self._zero = fold(_ZERO) + self._wahid = fold("واحد") + self._lexicon = { + self._zero, + *self._ones, + *self._teen_first, + *self._ten, + *self._dialect_teens, + *self._tens, + *self._hundreds, + *self._multipliers, + *self._standalone_scales, + } + + def _fold(self, word: str) -> str: + return word.translate(self._table) + + def __call__(self, text: str) -> str: + if not text.strip(): + return text + toks = self._tokenize(text.split()) + out: list[str] = [] + i = 0 + while i < len(toks): + parsed = self._parse_number(toks, i) + single_one = ( + parsed is not None and parsed[0] == i + 1 and toks[i].word in self._ones + ) + if parsed is None or single_one: + out.append(toks[i].raw) + i += 1 + continue + end, value = parsed + first, last = toks[i], toks[end - 1] + out.append(f"{first.lead}{first.conj}{value}{last.trail}") + i = end + return " ".join(out) + + def _is_number_word(self, word: str) -> bool: + return word in self._lexicon or (word.isascii() and word.isdigit()) + + def _tokenize(self, words: list[str]) -> list[_Token]: + toks: list[_Token] = [] + i = 0 + while i < len(words): + raw = words[i] + lead, word, trail = self._split(raw) + next_word = self._split(words[i + 1])[1] if i + 1 < len(words) else "" + # "وأحد" folds to "واحد"; after a number and before عشر it is و + أحد + # (مئة وأحد عشر = 111), since "واحد عشر" is not a valid teen. + is_and_ahad = ( + word == self._wahid + and next_word in self._ten + and bool(toks) + and toks[-1].word is not None + ) + if self._is_number_word(word) and not is_and_ahad: + toks.append(_Token(raw, lead, trail, word)) + elif ( + word.startswith(_CONJUNCTION) + and not lead + and self._is_number_word(word[1:]) + ): + toks.append(_Token(raw, lead, trail, word[1:], _CONJUNCTION)) + elif word == _CONJUNCTION and not lead and not trail and next_word: + next_lead, _, next_trail = self._split(words[i + 1]) + if next_lead or next_word not in self._lexicon: + toks.append(_Token(raw)) + else: + merged = f"{raw} {words[i + 1]}" + conj = f"{_CONJUNCTION} " + toks.append(_Token(merged, "", next_trail, next_word, conj)) + i += 1 + else: + toks.append(_Token(raw)) + i += 1 + return toks + + def _split(self, raw: str) -> tuple[str, str, str]: + """Fold a raw token and split it into (leading punct, word, trailing punct).""" + match = _RE_PUNCT_EDGES.match(self._fold(raw)) + assert match is not None + return match.group(1), match.group(2), match.group(3) + + @staticmethod + def _joinable(toks: list[_Token], i: int) -> bool: + """Whether toks[i] can continue the number ending at toks[i - 1].""" + return i < len(toks) and not toks[i - 1].trail and not toks[i].lead + + def _parse_number(self, toks: list[_Token], i: int) -> tuple[int, int] | None: + term = self._parse_term(toks, i) + if term is None: + return None + end, total, scale = term + # Larger groups are followed by smaller ones: ألف وتسعمائة وتسعين. + while scale > 1 and self._joinable(toks, end) and toks[end].conj: + nxt = self._parse_term(toks, end) + if nxt is None or nxt[2] >= scale: + break + end, value, scale = nxt + total += value + return end, total + + def _parse_term(self, toks: list[_Token], i: int) -> tuple[int, int, int] | None: + """Parse one scale group; returns (end, value, scale).""" + word = toks[i].word + if word is None: + return None + if word in self._standalone_scales: + value, scale = self._standalone_scales[word] + return i + 1, value, scale + if word.isascii() and word.isdigit(): + j = i + 1 + if self._is_multiplier(toks, j): + mult = self._multipliers[toks[j].word or ""] + return j + 1, int(word) * mult, mult + return None + parsed = self._parse_below_1000(toks, i) + if parsed is None: + return None + j, value = parsed + if self._is_multiplier(toks, j): + mult = self._multipliers[toks[j].word or ""] + return j + 1, value * mult, mult + return j, value, 1 + + def _is_multiplier(self, toks: list[_Token], j: int) -> bool: + return ( + self._joinable(toks, j) + and not toks[j].conj + and toks[j].word in self._multipliers + ) + + def _parse_below_1000(self, toks: list[_Token], i: int) -> tuple[int, int] | None: + word = toks[i].word + if word in self._hundreds: + hundreds, j = self._hundreds[word], i + 1 + elif ( + word in self._units + and self._joinable(toks, i + 1) + and not toks[i + 1].conj + and toks[i + 1].word in self._hundred_words + ): + hundreds, j = self._units[word] * 100, i + 2 + else: + return self._parse_below_100(toks, i) + if self._joinable(toks, j) and toks[j].conj: + rest = self._parse_below_100(toks, j) + if rest is not None: + return rest[0], hundreds + rest[1] + return j, hundreds + + def _parse_below_100(self, toks: list[_Token], i: int) -> tuple[int, int] | None: + word = toks[i].word + if word is None: + return None + if word in self._dialect_teens: + return i + 1, self._dialect_teens[word] + if ( + word in self._teen_first + and self._joinable(toks, i + 1) + and not toks[i + 1].conj + and toks[i + 1].word in self._ten + ): + return i + 2, 10 + self._teen_first[word] + if word in self._tens: + return i + 1, self._tens[word] + unit = self._units.get(word, self._ones.get(word)) + if unit is not None: + nxt = i + 1 + if self._joinable(toks, nxt) and toks[nxt].conj: + tens = self._tens.get(toks[nxt].word or "") + if tens is not None: + return nxt + 1, unit + tens + return nxt, unit + if word in self._ten: + return i + 1, 10 + if word == self._zero: + return i + 1, 0 + return None diff --git a/normalization/languages/arabic/operators.py b/normalization/languages/arabic/operators.py new file mode 100644 index 0000000..d710683 --- /dev/null +++ b/normalization/languages/arabic/operators.py @@ -0,0 +1,121 @@ +from normalization.languages.arabic.number_normalizer import ArabicNumberNormalizer +from normalization.languages.arabic.replacements import ARABIC_REPLACEMENTS +from normalization.languages.base import LanguageConfig, LanguageOperators +from normalization.languages.registry import register_language + +# Short vowels, tanween, shadda, sukun and related marks (U+064B–U+065F) plus +# the superscript alef. Transcripts rarely agree on vocalization, and +# remove_symbols would otherwise turn each mark into a word-splitting space. +_HARAKAT: dict[str, str] = {chr(c): "" for c in range(0x064B, 0x0660)} | {"\u0670": ""} + +# Tatweel / kashida: purely typographic letter elongation (عـــربي → عربي). +_TATWEEL: dict[str, str] = {"\u0640": ""} + +# Orthographic variants that transcribers and STT engines use interchangeably. +# Folding them is standard practice for Arabic WER scoring. +_LETTER_VARIANTS: dict[str, str] = { + "أ": "ا", # alef with hamza above + "إ": "ا", # alef with hamza below + "آ": "ا", # alef with madda + "ٱ": "ا", # alef wasla + "ؤ": "و", # waw with hamza above + "ئ": "ي", # yeh with hamza above + "ى": "ي", # alef maqsura + "ة": "ه", # ta marbuta + "ی": "ي", # Persian yeh + "ک": "ك", # keheh +} + +# Arabic-Indic (U+0660–U+0669) and Eastern Arabic-Indic (U+06F0–U+06F9) digits +# become ASCII so every numeric step sees them; the Arabic decimal/thousands +# separators and percent sign map to the characters those steps expect. +_DIGITS_AND_SEPARATORS: dict[str, str] = ( + {chr(0x0660 + d): str(d) for d in range(10)} + | {chr(0x06F0 + d): str(d) for d in range(10)} + | { + "٫": ".", # Arabic decimal separator + "٬": ",", # Arabic thousands separator + "٪": "%", # Arabic percent sign + } +) + +# Digit words for digit-by-digit readings (phone numbers, codes). Only the full +# forms: short forms like ست / سبع are also common nouns. +_ARABIC_DIGIT_WORDS: dict[str, str] = { + "صفر": "0", + "واحد": "1", + "اثنين": "2", + "اثنان": "2", + "اتنين": "2", + "ثلاثه": "3", + "تلاته": "3", + "اربعه": "4", + "خمسه": "5", + "سته": "6", + "سبعه": "7", + "ثمانيه": "8", + "تمانيه": "8", + "تسعه": "9", +} + +# Words inserted by steps (symbols, currencies, fillers) are written in the +# normalized surface form (no hamza seats, alef maqsura or ta marbuta) so they +# match the rest of the normalized text. +ARABIC_CONFIG = LanguageConfig( + code="ar", + decimal_separator=".", + decimal_word="فاصله", + thousand_separator=",", + symbols_to_words={ + "@": "ات", + ".": "نقطه", + "+": "زايد", + "=": "يساوي", + ">": "اكبر من", + "<": "اصغر من", + "°": "درجه", + "°C": "درجه ميويه", + "°F": "درجه فهرنهايت", + "%": "بالميه", + }, + currency_symbol_to_word={ + "$": "دولار", + "€": "يورو", + "£": "جنيه", + "¥": "ين", + }, + character_replacements={ + **_HARAKAT, + **_TATWEEL, + **_LETTER_VARIANTS, + **_DIGITS_AND_SEPARATORS, + }, + # "اه" (dialectal "yes") and "امم" ("nations" once alef is folded) are real + # words, so only the elongated hesitation forms are treated as fillers. + filler_words=[ + "اهه", + "اممم", + "ممم", + ], + digit_words=_ARABIC_DIGIT_WORDS, + # Same vocabulary as digit_words: used by + # normalize_punctuation_between_number_words so "صفر، خمسة" becomes + # "صفر خمسة" before convert_digit_word_sequences_to_digits. + number_words=list(_ARABIC_DIGIT_WORDS), +) + + +@register_language +class ArabicOperators(LanguageOperators): + def __init__(self) -> None: + super().__init__(ARABIC_CONFIG) + self._number_normalizer = ArabicNumberNormalizer( + ARABIC_CONFIG.character_replacements + ) + + def expand_written_numbers(self, text: str) -> str: + """Convert Arabic spelled-out numbers to digits (e.g. خمسة وعشرون → 25).""" + return self._number_normalizer(text) + + def get_word_replacements(self) -> dict[str, str]: + return ARABIC_REPLACEMENTS diff --git a/normalization/languages/arabic/replacements.py b/normalization/languages/arabic/replacements.py new file mode 100644 index 0000000..5fbc0d5 --- /dev/null +++ b/normalization/languages/arabic/replacements.py @@ -0,0 +1,9 @@ +"""Word-level substitutions for Arabic. + +Word replacements run after every text_pre step, so keys and values must use +the normalized surface form: no harakat or tatweel, bare alef for أ/إ/آ/ٱ, +و for ؤ, ي for ئ and ى, ه for ة (see ``character_replacements`` in +``operators.py``). A key spelled with any of the folded letters never matches. +""" + +ARABIC_REPLACEMENTS: dict[str, str] = {} diff --git a/normalization/languages/base/language_config.py b/normalization/languages/base/language_config.py index 3387cfc..e0d9c38 100644 --- a/normalization/languages/base/language_config.py +++ b/normalization/languages/base/language_config.py @@ -28,6 +28,12 @@ CurrencySymbols: TypeAlias = dict[str, str] """Maps currency symbols to their corresponding words.""" +CharacterReplacements: TypeAlias = dict[str, str] +"""Maps single characters to their canonical replacement (possibly empty to delete). +Applied character by character, before any other text step. +Example (Arabic): {'أ': 'ا', 'ـ': '', '٣': '3'} +""" + @dataclass class LanguageConfig: @@ -91,3 +97,8 @@ class LanguageConfig: pm_word: str | None = None """Canonical PM time designator (e.g. "pm" for English). Used by am/pm time formatting steps. None = am/pm steps are skipped.""" + character_replacements: CharacterReplacements | None = None + """Maps single characters to their canonical form; an empty value deletes the character. + Used by apply_character_replacements to unify script-level variants (letter forms, + native digits, vowel marks) before any other step sees the text. + None = step is skipped.""" diff --git a/normalization/presets/gladia-3.yaml b/normalization/presets/gladia-3.yaml index ca3f6ff..b78e2bb 100644 --- a/normalization/presets/gladia-3.yaml +++ b/normalization/presets/gladia-3.yaml @@ -2,6 +2,11 @@ name: gladia-3 stages: text_pre: + # --- Phase 0: Script-level character unification --- + # No-op unless the language defines character_replacements. Runs first so + # native digits / separators are visible to every protect and numeric step. + - apply_character_replacements + # --- Phase 1: Protect symbols from being erased by later cleanup --- # Run first so that email @/., phone +, time :, unit decimal/slash, and # number separators all survive remove_symbols later in this stage. diff --git a/normalization/steps/text/__init__.py b/normalization/steps/text/__init__.py index 23b0f46..90ea8e2 100644 --- a/normalization/steps/text/__init__.py +++ b/normalization/steps/text/__init__.py @@ -1,4 +1,5 @@ from . import ( + apply_character_replacements, apply_sentence_level_replacements, casefold_text, convert_comparison_operators_to_words, @@ -39,6 +40,7 @@ ) __all__ = [ + "apply_character_replacements", "apply_sentence_level_replacements", "casefold_text", "convert_comparison_operators_to_words", diff --git a/normalization/steps/text/apply_character_replacements.py b/normalization/steps/text/apply_character_replacements.py new file mode 100644 index 0000000..f7c3d79 --- /dev/null +++ b/normalization/steps/text/apply_character_replacements.py @@ -0,0 +1,22 @@ +from normalization.languages.base import LanguageOperators +from normalization.steps.base import TextStep +from normalization.steps.registry import register_step + + +@register_step +class ApplyCharacterReplacementsStep(TextStep): + """Replace or delete single characters using ``character_replacements`` from the language config. + + Runs first in the pipeline so that script-level variants (letter forms, native + digits, vowel marks) are unified before protection, numeric, and symbol steps. + Skipped when the language does not define ``character_replacements``. + """ + + name = "apply_character_replacements" + + def __call__(self, text: str, operators: LanguageOperators) -> str: + replacements = operators.config.character_replacements + # TODO: populate character_replacements for languages with script-level variants + if replacements is None: + return text + return text.translate(str.maketrans(replacements)) diff --git a/normalization/steps/text/normalize_punctuation_between_number_words.py b/normalization/steps/text/normalize_punctuation_between_number_words.py index f088ef0..aa2653e 100644 --- a/normalization/steps/text/normalize_punctuation_between_number_words.py +++ b/normalization/steps/text/normalize_punctuation_between_number_words.py @@ -24,7 +24,9 @@ def __call__(self, text: str, operators: LanguageOperators) -> str: alternation = "|".join(re.escape(w) for w in sorted_words) number_group = f"(?:{alternation})" pattern = re.compile( - rf"({number_group})\s*[,.\-…]+\s*(?={number_group})", + # Include Arabic comma (U+060C) so digit-by-digit ASR like + # "صفر، خمسة" collapses the same way as "zero, five". + rf"({number_group})\s*[,.\-…\u060c]+\s*(?={number_group})", flags=re.IGNORECASE, ) return pattern.sub(r"\1 ", text) diff --git a/tests/e2e/files/gladia-3/ar.csv b/tests/e2e/files/gladia-3/ar.csv new file mode 100644 index 0000000..0efb480 --- /dev/null +++ b/tests/e2e/files/gladia-3/ar.csv @@ -0,0 +1,34 @@ +input,expected +مَرْحَبًا بِكُمْ,مرحبا بكم +عـــربي,عربي +أنا ذهبت إلى المدرسة,انا ذهبت الي المدرسه +مستشفى,مستشفي +كيف حالك؟ بخير، شكرا؛,كيف حالك بخير شكرا +« نص »,نص +عندي ٣ كتب,عندي 3 كتب +۱۲۳,123 +١٢٫٥ كيلو,12 فاصله 5 كيلو +١٬٠٠٠ دولار,1000 دولار +نسبة ٢٠٪,نسبه 20 بالميه +يكلف $50,يكلف 50 دولار +آهه، أنا هنا,انا هنا +اممم نعم,نعم +أمم كثيرة,امم كثيره +اه والله,اه والله +admin@example.com,admin ات example نقطه com +2 > 1,2 اكبر من 1 +درجة الحرارة 30°C,درجه الحراره 30 درجه ميويه +مسؤول,مسوول +رئيس الوزراء,رييس الوزراء +کتاب عربی,كتاب عربي +خمسة وعشرون,25 +مئة وخمسة وعشرون,125 +ألفين وخمسة وعشرين,2025 +ثلاثة آلاف دولار,3000 دولار +عشرون بالمئة,20 بالميه +ثلاثة فاصلة خمسة,3 فاصله 5 +ثلاثة وأربعة,3 و4 +تلتمية وخمسين,350 +جاء واحد من الطلاب,جاء واحد من الطلاب +اتصل على صفر خمسة صفر واحد اثنين ثلاثة,اتصل علي 050123 +صفر، خمسة، واحد,051 diff --git a/tests/unit/languages/arabic_number_normalizer_test.py b/tests/unit/languages/arabic_number_normalizer_test.py new file mode 100644 index 0000000..e9a72bb --- /dev/null +++ b/tests/unit/languages/arabic_number_normalizer_test.py @@ -0,0 +1,190 @@ +import pytest + +from normalization.languages.arabic.number_normalizer import ArabicNumberNormalizer +from normalization.languages.arabic.operators import ARABIC_CONFIG + + +@pytest.fixture(scope="module") +def normalizer() -> ArabicNumberNormalizer: + return ArabicNumberNormalizer(ARABIC_CONFIG.character_replacements) + + +@pytest.mark.parametrize( + ("text", "expected"), + [ + ("صفر", "0"), + ("اثنان", "2"), + ("اثنين", "2"), + ("ثلاثة", "3"), + ("ثلاث", "3"), + ("أربعة", "4"), + ("خمسة", "5"), + ("ستة", "6"), + ("سبعة", "7"), + ("ثمانية", "8"), + ("ثماني", "8"), + ("تسعة", "9"), + ("عشرة", "10"), + ("عشر", "10"), + ], +) +def test_units(normalizer: ArabicNumberNormalizer, text: str, expected: str) -> None: + assert normalizer(text) == expected + + +@pytest.mark.parametrize( + "text", + ["واحد", "واحدة", "أحد", "إحدى", "ولد واحد", "يوم الأحد"], +) +def test_standalone_one_is_kept_as_a_word( + normalizer: ArabicNumberNormalizer, text: str +) -> None: + # Like English "one", these double as pronouns ("someone", "one of"). + assert normalizer(text) == text + + +@pytest.mark.parametrize( + ("text", "expected"), + [ + ("أحد عشر", "11"), + ("إحدى عشرة", "11"), + ("اثنا عشر", "12"), + ("اثني عشر", "12"), + ("اثنتا عشرة", "12"), + ("ثلاثة عشر", "13"), + ("خمس عشرة", "15"), + ("تسعة عشر", "19"), + ], +) +def test_teens(normalizer: ArabicNumberNormalizer, text: str, expected: str) -> None: + assert normalizer(text) == expected + + +@pytest.mark.parametrize( + ("text", "expected"), + [ + ("عشرون", "20"), + ("عشرين", "20"), + ("ثلاثون", "30"), + ("أربعين", "40"), + ("تسعون", "90"), + ("واحد وعشرون", "21"), + ("خمسة وعشرون", "25"), + ("خمسة و عشرين", "25"), + ("ثلاثة وتسعين", "93"), + ], +) +def test_tens_and_compounds( + normalizer: ArabicNumberNormalizer, text: str, expected: str +) -> None: + assert normalizer(text) == expected + + +@pytest.mark.parametrize( + ("text", "expected"), + [ + ("مئة", "100"), + ("مائة", "100"), + ("مية", "100"), + ("مئتان", "200"), + ("مئتين", "200"), + ("ثلاثمئة", "300"), + ("ثلاثمائة", "300"), + ("ثلاث مئة", "300"), + ("خمسمية", "500"), + ("تسعمائة", "900"), + ("مئة وخمسة", "105"), + ("مئة وواحد", "101"), + ("مئة وخمسة وعشرون", "125"), + ("ثلاثمئة وأحد عشر", "311"), + ], +) +def test_hundreds(normalizer: ArabicNumberNormalizer, text: str, expected: str) -> None: + assert normalizer(text) == expected + + +@pytest.mark.parametrize( + ("text", "expected"), + [ + ("ألف", "1000"), + ("ألفان", "2000"), + ("ألفين", "2000"), + ("ثلاثة آلاف", "3000"), + ("عشرة آلاف", "10000"), + ("أحد عشر ألفاً", "11000"), + ("خمسة وعشرين ألف", "25000"), + ("مئة ألف", "100000"), + ("ألف وتسعمائة وتسعين", "1990"), + ("ألفين وخمسة وعشرين", "2025"), + ("ثلاثة آلاف وخمسمائة", "3500"), + ("مليون", "1000000"), + ("مليونين", "2000000"), + ("ثلاثة ملايين", "3000000"), + ("ثلاثة ملايين وخمسمائة ألف", "3500000"), + ("مليار", "1000000000"), + ("خمسة مليارات", "5000000000"), + ], +) +def test_thousands_and_above( + normalizer: ArabicNumberNormalizer, text: str, expected: str +) -> None: + assert normalizer(text) == expected + + +@pytest.mark.parametrize( + ("text", "expected"), + [ + ("3 آلاف", "3000"), + ("٥ ملايين", "5000000"), + ("25 ألف", "25000"), + ], +) +def test_digits_followed_by_a_scale_word( + normalizer: ArabicNumberNormalizer, text: str, expected: str +) -> None: + assert normalizer(text) == expected + + +@pytest.mark.parametrize( + ("text", "expected"), + [ + ("اتنين", "2"), + ("تلاتة", "3"), + ("تمانية", "8"), + ("حداشر", "11"), + ("اتناشر", "12"), + ("خمستاشر", "15"), + ("تلاتين", "30"), + ("تلتمية", "300"), + ("ميتين", "200"), + ("تلات تلاف", "3000"), + ], +) +def test_dialect_spellings( + normalizer: ArabicNumberNormalizer, text: str, expected: str +) -> None: + assert normalizer(text) == expected + + +@pytest.mark.parametrize( + ("text", "expected"), + [ + # "و" only joins a unit to a tens word or a smaller group to a larger + # one; anything else is the plain conjunction "and". + ("ثلاثة وأربعة", "3 و4"), + ("عشرين وخمسة", "20 و5"), + ("و خمسة", "و 5"), + ("عندي خمسة وعشرون كتاب", "عندي 25 كتاب"), + ("سنة ألفين وعشرين", "سنة 2020"), + ("خمسة، ستة", "5، 6"), + ("الساعة ثلاثة", "الساعة 3"), + ("ثلاثة فاصلة خمسة", "3 فاصلة 5"), + ("عشرين بالمئة", "20 بالمئة"), + ("من غير أرقام", "من غير أرقام"), + ("", ""), + ], +) +def test_numbers_in_context( + normalizer: ArabicNumberNormalizer, text: str, expected: str +) -> None: + assert normalizer(text) == expected diff --git a/tests/unit/languages/arabic_operators_test.py b/tests/unit/languages/arabic_operators_test.py new file mode 100644 index 0000000..6bcd4e8 --- /dev/null +++ b/tests/unit/languages/arabic_operators_test.py @@ -0,0 +1,140 @@ +import pytest + +from normalization.languages.arabic.operators import ArabicOperators +from normalization.languages.registry import get_language_registry +from normalization.pipeline.loader import load_pipeline + + +@pytest.fixture +def operators() -> ArabicOperators: + return ArabicOperators() + + +@pytest.fixture(scope="module") +def pipeline(): + return load_pipeline("gladia-3", "ar") + + +def test_arabic_is_registered() -> None: + assert "ar" in get_language_registry() + + +def test_arabic_registry_produces_arabic_operators() -> None: + instance = get_language_registry()["ar"]() + assert isinstance(instance, ArabicOperators) + + +def test_config_code(operators: ArabicOperators) -> None: + assert operators.config.code == "ar" + + +def test_config_words_are_in_normalized_form(operators: ArabicOperators) -> None: + config = operators.config + assert config.character_replacements is not None + assert config.decimal_word is not None + folded = {k for k, v in config.character_replacements.items() if k != v} + words = [ + *config.symbols_to_words.values(), + *config.currency_symbol_to_word.values(), + *config.filler_words, + *(config.digit_words or {}), + config.decimal_word, + ] + for word in words: + assert not folded & set(word), word + + +@pytest.mark.parametrize( + ("text", "expected"), + [ + ("يكلف $50", "يكلف 50 دولار"), + ("20%", "20 بالميه"), + ("12.5", "12 فاصله 5"), + ("اممم نعم", "نعم"), + ], +) +def test_pipeline_symbols_and_fillers(pipeline, text: str, expected: str) -> None: + assert pipeline.normalize(text) == expected + + +@pytest.mark.parametrize( + ("text", "expected"), + [ + ("أمم كثيرة", "امم كثيره"), + ("اه والله", "اه والله"), + ], +) +def test_pipeline_keeps_real_words_that_look_like_fillers( + pipeline, text: str, expected: str +) -> None: + assert pipeline.normalize(text) == expected + + +@pytest.mark.parametrize( + ("text", "expected"), + [ + ("مَرْحَبًا بِكُمْ", "مرحبا بكم"), + ("مُحَمَّدٌ", "محمد"), + ("هٰذا", "هذا"), + ("عـــربي", "عربي"), + ], +) +def test_pipeline_strips_harakat_and_tatweel( + pipeline, text: str, expected: str +) -> None: + assert pipeline.normalize(text) == expected + + +@pytest.mark.parametrize( + ("text", "expected"), + [ + ("أنا ذهبت إلى المدرسة", "انا ذهبت الي المدرسه"), + ("آمن", "امن"), + ("ٱلله", "الله"), + ("مستشفى", "مستشفي"), + ("مستشفي", "مستشفي"), + ("کتاب عربی", "كتاب عربي"), + ("سؤال", "سوال"), + ("رئيس", "رييس"), + ], +) +def test_pipeline_unifies_letter_variants(pipeline, text: str, expected: str) -> None: + assert pipeline.normalize(text) == expected + + +@pytest.mark.parametrize( + ("text", "expected"), + [ + ("عندي ٣ كتب", "عندي 3 كتب"), + ("۱۲۳", "123"), + ("١٢٫٥ كيلو", "12 فاصله 5 كيلو"), + ("١٬٠٠٠ دولار", "1000 دولار"), + ("٢٠٪", "20 بالميه"), + ], +) +def test_pipeline_converts_native_digits_and_separators( + pipeline, text: str, expected: str +) -> None: + assert pipeline.normalize(text) == expected + + +@pytest.mark.parametrize( + ("words", "digits"), + [ + ("عندي خمسة وعشرون كتاباً", "عندي ٢٥ كتاباً"), + ("سنة ألفين وخمسة وعشرين", "سنة ٢٠٢٥"), + ("ثلاثة فاصلة خمسة", "٣٫٥"), + ("عشرون بالمئة", "٢٠٪"), + ("ثلاثة آلاف دولار", "٣٬٠٠٠ دولار"), + ("ثلاثة وأربعة", "٣ و٤"), + ("مئة وخمسة وعشرون", "١٢٥"), + ], +) +def test_pipeline_spelled_out_numbers_match_digits( + pipeline, words: str, digits: str +) -> None: + assert pipeline.normalize(words) == pipeline.normalize(digits) + + +def test_pipeline_collapses_digit_word_sequences(pipeline) -> None: + assert pipeline.normalize("واحد اثنين ثلاثة أربعة") == "1234" diff --git a/tests/unit/steps/text/apply_character_replacements_test.py b/tests/unit/steps/text/apply_character_replacements_test.py new file mode 100644 index 0000000..5a88344 --- /dev/null +++ b/tests/unit/steps/text/apply_character_replacements_test.py @@ -0,0 +1,24 @@ +from normalization.languages.base import LanguageConfig, LanguageOperators +from normalization.steps.text.apply_character_replacements import ( + ApplyCharacterReplacementsStep, +) + +from .conftest import assert_text_step_registered + + +def test_step_is_registered(): + assert_text_step_registered(ApplyCharacterReplacementsStep) + + +def test_skipped_when_config_is_none(operators: LanguageOperators): + assert ApplyCharacterReplacementsStep()("abc ـ ٣", operators) == "abc ـ ٣" + + +def test_replaces_and_deletes_characters(): + operators = LanguageOperators( + LanguageConfig( + code="test", + character_replacements={"a": "b", "-": "", "٣": "3"}, + ) + ) + assert ApplyCharacterReplacementsStep()("a-a ٣", operators) == "bb 3"