From 213ca3bec348cc0c1b4a026281d88f126a76110c Mon Sep 17 00:00:00 2001
From: Oliver Kriska <oliver@ideax.sk>
Date: Wed, 28 Oct 2026 18:06:06 +0100
Subject: [PATCH] Prune fuzzy candidates using Jaro similarity bounds

Prepare grapheme counts lazily and skip candidates that cannot reach the
threshold or improve on the best match. Keep platform Jaro scoring for
survivors, the original traversal order and first-winner tie behavior.

Retain the full scan for empty or malformed input so pruning cannot hide
scorer exceptions. Cover inclusive thresholds, repeated Unicode graphemes,
ties, zero scores and malformed input through the merge API.
---
 lib/gettext/merger.ex        | 110 +++++++++++++++++++++++++++++++----
 test/gettext/merger_test.exs | 100 +++++++++++++++++++++++++++++++
 2 files changed, 200 insertions(+), 10 deletions(-)

diff --git a/lib/gettext/merger.ex b/lib/gettext/merger.ex
index bfb4d1d..d210695 100644
--- a/lib/gettext/merger.ex
+++ b/lib/gettext/merger.ex
@@ -78,8 +78,9 @@ defmodule Gettext.Merger do
 
     old = Map.new(old, &{Message.key(&1), &1})
 
-    {messages, {stats, unused}} =
-      Enum.map_reduce(new, {stats, _unused = old}, fn message, {stats_acc, unused} ->
+    {messages, {stats, unused, _candidates}} =
+      Enum.map_reduce(new, {stats, _unused = old, nil}, fn message,
+                                                           {stats_acc, unused, candidates} ->
         key = Message.key(message)
         message = adjust_number_of_plural_forms(message, plural_forms)
 
@@ -88,10 +89,12 @@ defmodule Gettext.Merger do
             stats = update_in(stats_acc.exact_matches, &(&1 + 1))
 
             {merge_two_messages(exact_match, message, custom_flags_to_keep),
-             {stats, Map.delete(unused, key)}}
+             {stats, Map.delete(unused, key), candidates}}
 
           :error when fuzzy? ->
-            case maybe_merge_fuzzy(message, old, key, fuzzy_threshold) do
+            candidates = candidates || prepare_fuzzy_candidates(old, key)
+
+            case maybe_merge_fuzzy(message, candidates || old, key, fuzzy_threshold) do
               {:matched, match, fuzzy_merged} ->
                 stats_acc = update_in(stats_acc.fuzzy_matches, &(&1 + 1))
                 unused = Map.delete(unused, Message.key(match))
@@ -105,16 +108,16 @@ defmodule Gettext.Merger do
                     fuzzy_merged
                   end
 
-                {fuzzy_merged, {stats_acc, unused}}
+                {fuzzy_merged, {stats_acc, unused, candidates}}
 
               :nomatch ->
                 stats_acc = update_in(stats_acc.new, &(&1 + 1))
-                {message, {stats_acc, unused}}
+                {message, {stats_acc, unused, candidates}}
             end
 
           :error ->
             stats_acc = update_in(stats_acc.new, &(&1 + 1))
-            {message, {stats_acc, unused}}
+            {message, {stats_acc, unused, candidates}}
         end
       end)
 
@@ -143,15 +146,48 @@ defmodule Gettext.Merger do
     message
   end
 
-  defp maybe_merge_fuzzy(message, old, key, fuzzy_threshold) do
-    if matched = find_fuzzy_match(old, key, fuzzy_threshold) do
+  defp maybe_merge_fuzzy(message, candidates, key, fuzzy_threshold) do
+    if matched = find_fuzzy_match(candidates, key, fuzzy_threshold) do
       {:matched, matched, Fuzzy.merge(message, matched)}
     else
       :nomatch
     end
   end
 
-  defp find_fuzzy_match(messages, key, threshold) do
+  defp prepare_fuzzy_candidates(messages, {_context, msgid}) do
+    cond do
+      # Jaro handles empty strings without traversing the other string. Avoid
+      # indexing for these lookups, and leave malformed bytes to the scorer.
+      msgid == "" or not String.valid?(msgid) ->
+        nil
+
+      Enum.all?(messages, fn {{_context, old_msgid}, _message} -> String.valid?(old_msgid) end) ->
+        Enum.map(messages, fn {{_context, msgid}, message} ->
+          graphemes = String.graphemes(msgid)
+          {message, msgid, length(graphemes), Enum.frequencies(graphemes)}
+        end)
+
+      true ->
+        # Pruning or stopping at a perfect match could hide a scorer exception
+        # from another candidate. Preserve the full scan for malformed input.
+        messages
+    end
+  end
+
+  defp find_fuzzy_match(messages, key, threshold) when is_map(messages) do
+    find_unpruned_match(messages, key, threshold)
+  end
+
+  defp find_fuzzy_match(candidates, {_context, msgid} = key, threshold) do
+    if msgid == "" or not String.valid?(msgid) do
+      messages = Enum.map(candidates, fn {message, msgid, _, _} -> {{nil, msgid}, message} end)
+      find_unpruned_match(messages, key, threshold)
+    else
+      find_best_candidate(candidates, msgid, threshold)
+    end
+  end
+
+  defp find_unpruned_match(messages, key, threshold) do
     matcher = Fuzzy.matcher(threshold)
 
     candidates =
@@ -168,6 +204,60 @@ defmodule Gettext.Merger do
     end
   end
 
+  defp find_best_candidate(candidates, msgid, threshold) do
+    graphemes = String.graphemes(msgid)
+    size = length(graphemes)
+    counts = graphemes |> Enum.frequencies() |> Enum.sort_by(fn {_, count} -> -count end)
+
+    {message, _distance} =
+      Enum.reduce_while(candidates, {nil, threshold}, fn
+        {message, old_msgid, old_size, old_counts}, {best, best_distance} = acc ->
+          if jaro_upper_bound(min(size, old_size), size, old_size) >= best_distance and
+               enough_common_graphemes?(counts, old_counts, size, old_size, size, best_distance) do
+            distance = String.jaro_distance(old_msgid, msgid)
+
+            if distance >= best_distance and (is_nil(best) or distance > best_distance) do
+              if distance == 1.0,
+                do: {:halt, {message, distance}},
+                else: {:cont, {message, distance}}
+            else
+              {:cont, acc}
+            end
+          else
+            {:cont, acc}
+          end
+      end)
+
+    message
+  end
+
+  # Matching graphemes cannot outnumber either string or their shared grapheme
+  # counts. Assuming no transpositions gives an upper bound on Jaro similarity.
+  # Keep the scorer's arithmetic order so inclusive thresholds stay inclusive.
+  defp jaro_upper_bound(_matches, 0, 0), do: 1.0
+  defp jaro_upper_bound(_matches, 0, _size), do: 0.0
+  defp jaro_upper_bound(_matches, _size, 0), do: 0.0
+
+  defp jaro_upper_bound(matches, size, old_size),
+    do: (matches / size + matches / old_size + 1.0) / 3
+
+  defp enough_common_graphemes?([], _old, _size, _old_size, _remaining, _target), do: true
+
+  defp enough_common_graphemes?(
+         [{grapheme, count} | rest],
+         old,
+         size,
+         old_size,
+         remaining,
+         target
+       ) do
+    missing = max(count - Map.get(old, grapheme, 0), 0)
+    remaining = remaining - missing
+
+    (missing == 0 or jaro_upper_bound(min(remaining, old_size), size, old_size) >= target) and
+      enough_common_graphemes?(rest, old, size, old_size, remaining, target)
+  end
+
   # msgid, msgid_plural: they're the same
   # msgctxt: it's the same, even if it's not present (nil)
   # msgstr: new.msgstr should be empty since it comes from a POT file
diff --git a/test/gettext/merger_test.exs b/test/gettext/merger_test.exs
index 7db679d..3cab30b 100644
--- a/test/gettext/merger_test.exs
+++ b/test/gettext/merger_test.exs
@@ -444,6 +444,106 @@ defmodule Gettext.MergerTest do
       assert new_message.msgctxt == "context"
     end
 
+    test "fuzzy matching includes the threshold and counts repeated Unicode graphemes" do
+      # Four matching graphemes out of four and eight, with no transpositions:
+      # the Jaro similarity is (1 + 1/2 + 1) / 3 = 5/6.
+      cases =
+        for suffix <- ["WXYZ", String.duplicate("e\u0301", 4), String.duplicate("👩‍👩‍👧‍👦", 4)],
+            do: {"abcd", "abcd" <> suffix, 5 / 6}
+
+      # Three repeated matching graphemes: (3/4 + 3/6 + 1) / 3 = 3/4.
+      for {old, new, score} <- [{"aaab", "aaaccc", 0.75}, {"aaaaaa", "aaab", 0.75} | cases],
+          {threshold, expected} <- [{score, ["translation"]}, {score + 1.0e-15, [""]}] do
+        old_po = %Messages{messages: [%Message.Singular{msgid: [old], msgstr: ["translation"]}]}
+        new_pot = %Messages{messages: [%Message.Singular{msgid: [new]}]}
+
+        assert {%Messages{messages: [message]}, _stats} =
+                 Merger.merge(
+                   old_po,
+                   new_pot,
+                   "en",
+                   [fuzzy: true, fuzzy_threshold: threshold],
+                   []
+                 )
+
+        assert message.msgstr == expected
+      end
+    end
+
+    test "fuzzy matching chooses the best score and keeps the first match on ties" do
+      old_po = %Messages{
+        messages: [
+          %Message.Singular{msgctxt: ["c"], msgid: ["abcd"], msgstr: ["second best"]},
+          %Message.Singular{msgctxt: ["b"], msgid: ["abcd"], msgstr: ["first best"]},
+          %Message.Singular{msgctxt: ["a"], msgid: ["abce"], msgstr: ["worse"]}
+        ]
+      }
+
+      new_pot = %Messages{messages: [%Message.Singular{msgid: ["abcd!"]}]}
+
+      assert {%Messages{messages: [message]}, %{fuzzy_matches: 1}} =
+               Merger.merge(old_po, new_pot, "en", [fuzzy: true, fuzzy_threshold: 0.7], [])
+
+      assert message.msgstr == ["first best"]
+    end
+
+    test "a zero fuzzy threshold also matches empty and dissimilar messages" do
+      for {old, new} <- [{"", "abcd"}, {"abcd", ""}, {"abcd", "wxyz"}] do
+        old_po = %Messages{messages: [%Message.Singular{msgid: [old], msgstr: ["translation"]}]}
+        new_pot = %Messages{messages: [%Message.Singular{msgid: [new]}]}
+
+        assert {%Messages{messages: [message]}, %{fuzzy_matches: 1}} =
+                 Merger.merge(old_po, new_pot, "en", [fuzzy: true, fuzzy_threshold: 0.0], [])
+
+        assert message.msgstr == ["translation"]
+      end
+    end
+
+    test "fuzzy matching preserves the scorer's handling of malformed UTF-8" do
+      for {old, new, translation} <- [
+            {[<<255>>], "z", [""]},
+            {["z"], <<255>>, [""]},
+            {["abc", <<255>>], "abc", ["translation"]},
+            {[<<255>>], <<255>>, ["translation"]},
+            {[<<255>>], "", [""]},
+            {[""], <<255>>, [""]}
+          ],
+          prefix <- [[], ["a!"]] do
+        old_po = %Messages{
+          messages:
+            Enum.with_index(old, fn text, i ->
+              %Message.Singular{msgctxt: [to_string(i)], msgid: [text], msgstr: ["translation"]}
+            end)
+        }
+
+        new_pot = %Messages{
+          messages: Enum.map(prefix ++ [new], &%Message.Singular{msgctxt: ["new"], msgid: [&1]})
+        }
+
+        # Older scorers accept malformed bytes; newer ones may raise, except
+        # for their equality/empty fast paths. Preserve the platform's behavior.
+        expected =
+          try do
+            for text <- prefix ++ [new], previous <- old, do: String.jaro_distance(previous, text)
+
+            :ok
+          rescue
+            error -> {:error, error.__struct__}
+          end
+
+        merge = fn -> Merger.merge(old_po, new_pot, "en", @opts, @gettext_config) end
+
+        case expected do
+          :ok ->
+            {po, _stats} = merge.()
+            assert List.last(po.messages).msgstr == translation
+
+          {:error, exception} ->
+            assert_raise exception, merge
+        end
+      end
+    end
+
     test "if there's a Plural-Forms header, it's used to determine number of plural forms" do
       old_po = %Messages{
         headers: [~s(Plural-Forms: nplurals=3;plural=n>1;)],