diff --git a/lib/gettext/merger.ex b/lib/gettext/merger.ex index bfb4d1d..d210695 100644 --- a/lib/gettext/merger.ex +++ b/lib/gettext/merger.ex @@ -78,8 +78,9 @@ defmodule Gettext.Merger do old = Map.new(old, &{Message.key(&1), &1}) - {messages, {stats, unused}} = - Enum.map_reduce(new, {stats, _unused = old}, fn message, {stats_acc, unused} -> + {messages, {stats, unused, _candidates}} = + Enum.map_reduce(new, {stats, _unused = old, nil}, fn message, + {stats_acc, unused, candidates} -> key = Message.key(message) message = adjust_number_of_plural_forms(message, plural_forms) @@ -88,10 +89,12 @@ defmodule Gettext.Merger do stats = update_in(stats_acc.exact_matches, &(&1 + 1)) {merge_two_messages(exact_match, message, custom_flags_to_keep), - {stats, Map.delete(unused, key)}} + {stats, Map.delete(unused, key), candidates}} :error when fuzzy? -> - case maybe_merge_fuzzy(message, old, key, fuzzy_threshold) do + candidates = candidates || prepare_fuzzy_candidates(old, key) + + case maybe_merge_fuzzy(message, candidates || old, key, fuzzy_threshold) do {:matched, match, fuzzy_merged} -> stats_acc = update_in(stats_acc.fuzzy_matches, &(&1 + 1)) unused = Map.delete(unused, Message.key(match)) @@ -105,16 +108,16 @@ defmodule Gettext.Merger do fuzzy_merged end - {fuzzy_merged, {stats_acc, unused}} + {fuzzy_merged, {stats_acc, unused, candidates}} :nomatch -> stats_acc = update_in(stats_acc.new, &(&1 + 1)) - {message, {stats_acc, unused}} + {message, {stats_acc, unused, candidates}} end :error -> stats_acc = update_in(stats_acc.new, &(&1 + 1)) - {message, {stats_acc, unused}} + {message, {stats_acc, unused, candidates}} end end) @@ -143,15 +146,48 @@ defmodule Gettext.Merger do message end - defp maybe_merge_fuzzy(message, old, key, fuzzy_threshold) do - if matched = find_fuzzy_match(old, key, fuzzy_threshold) do + defp maybe_merge_fuzzy(message, candidates, key, fuzzy_threshold) do + if matched = find_fuzzy_match(candidates, key, fuzzy_threshold) do {:matched, matched, Fuzzy.merge(message, matched)} else :nomatch end end - defp find_fuzzy_match(messages, key, threshold) do + defp prepare_fuzzy_candidates(messages, {_context, msgid}) do + cond do + # Jaro handles empty strings without traversing the other string. Avoid + # indexing for these lookups, and leave malformed bytes to the scorer. + msgid == "" or not String.valid?(msgid) -> + nil + + Enum.all?(messages, fn {{_context, old_msgid}, _message} -> String.valid?(old_msgid) end) -> + Enum.map(messages, fn {{_context, msgid}, message} -> + graphemes = String.graphemes(msgid) + {message, msgid, length(graphemes), Enum.frequencies(graphemes)} + end) + + true -> + # Pruning or stopping at a perfect match could hide a scorer exception + # from another candidate. Preserve the full scan for malformed input. + messages + end + end + + defp find_fuzzy_match(messages, key, threshold) when is_map(messages) do + find_unpruned_match(messages, key, threshold) + end + + defp find_fuzzy_match(candidates, {_context, msgid} = key, threshold) do + if msgid == "" or not String.valid?(msgid) do + messages = Enum.map(candidates, fn {message, msgid, _, _} -> {{nil, msgid}, message} end) + find_unpruned_match(messages, key, threshold) + else + find_best_candidate(candidates, msgid, threshold) + end + end + + defp find_unpruned_match(messages, key, threshold) do matcher = Fuzzy.matcher(threshold) candidates = @@ -168,6 +204,60 @@ defmodule Gettext.Merger do end end + defp find_best_candidate(candidates, msgid, threshold) do + graphemes = String.graphemes(msgid) + size = length(graphemes) + counts = graphemes |> Enum.frequencies() |> Enum.sort_by(fn {_, count} -> -count end) + + {message, _distance} = + Enum.reduce_while(candidates, {nil, threshold}, fn + {message, old_msgid, old_size, old_counts}, {best, best_distance} = acc -> + if jaro_upper_bound(min(size, old_size), size, old_size) >= best_distance and + enough_common_graphemes?(counts, old_counts, size, old_size, size, best_distance) do + distance = String.jaro_distance(old_msgid, msgid) + + if distance >= best_distance and (is_nil(best) or distance > best_distance) do + if distance == 1.0, + do: {:halt, {message, distance}}, + else: {:cont, {message, distance}} + else + {:cont, acc} + end + else + {:cont, acc} + end + end) + + message + end + + # Matching graphemes cannot outnumber either string or their shared grapheme + # counts. Assuming no transpositions gives an upper bound on Jaro similarity. + # Keep the scorer's arithmetic order so inclusive thresholds stay inclusive. + defp jaro_upper_bound(_matches, 0, 0), do: 1.0 + defp jaro_upper_bound(_matches, 0, _size), do: 0.0 + defp jaro_upper_bound(_matches, _size, 0), do: 0.0 + + defp jaro_upper_bound(matches, size, old_size), + do: (matches / size + matches / old_size + 1.0) / 3 + + defp enough_common_graphemes?([], _old, _size, _old_size, _remaining, _target), do: true + + defp enough_common_graphemes?( + [{grapheme, count} | rest], + old, + size, + old_size, + remaining, + target + ) do + missing = max(count - Map.get(old, grapheme, 0), 0) + remaining = remaining - missing + + (missing == 0 or jaro_upper_bound(min(remaining, old_size), size, old_size) >= target) and + enough_common_graphemes?(rest, old, size, old_size, remaining, target) + end + # msgid, msgid_plural: they're the same # msgctxt: it's the same, even if it's not present (nil) # msgstr: new.msgstr should be empty since it comes from a POT file diff --git a/test/gettext/merger_test.exs b/test/gettext/merger_test.exs index 7db679d..3cab30b 100644 --- a/test/gettext/merger_test.exs +++ b/test/gettext/merger_test.exs @@ -444,6 +444,106 @@ defmodule Gettext.MergerTest do assert new_message.msgctxt == "context" end + test "fuzzy matching includes the threshold and counts repeated Unicode graphemes" do + # Four matching graphemes out of four and eight, with no transpositions: + # the Jaro similarity is (1 + 1/2 + 1) / 3 = 5/6. + cases = + for suffix <- ["WXYZ", String.duplicate("e\u0301", 4), String.duplicate("๐Ÿ‘ฉโ€๐Ÿ‘ฉโ€๐Ÿ‘งโ€๐Ÿ‘ฆ", 4)], + do: {"abcd", "abcd" <> suffix, 5 / 6} + + # Three repeated matching graphemes: (3/4 + 3/6 + 1) / 3 = 3/4. + for {old, new, score} <- [{"aaab", "aaaccc", 0.75}, {"aaaaaa", "aaab", 0.75} | cases], + {threshold, expected} <- [{score, ["translation"]}, {score + 1.0e-15, [""]}] do + old_po = %Messages{messages: [%Message.Singular{msgid: [old], msgstr: ["translation"]}]} + new_pot = %Messages{messages: [%Message.Singular{msgid: [new]}]} + + assert {%Messages{messages: [message]}, _stats} = + Merger.merge( + old_po, + new_pot, + "en", + [fuzzy: true, fuzzy_threshold: threshold], + [] + ) + + assert message.msgstr == expected + end + end + + test "fuzzy matching chooses the best score and keeps the first match on ties" do + old_po = %Messages{ + messages: [ + %Message.Singular{msgctxt: ["c"], msgid: ["abcd"], msgstr: ["second best"]}, + %Message.Singular{msgctxt: ["b"], msgid: ["abcd"], msgstr: ["first best"]}, + %Message.Singular{msgctxt: ["a"], msgid: ["abce"], msgstr: ["worse"]} + ] + } + + new_pot = %Messages{messages: [%Message.Singular{msgid: ["abcd!"]}]} + + assert {%Messages{messages: [message]}, %{fuzzy_matches: 1}} = + Merger.merge(old_po, new_pot, "en", [fuzzy: true, fuzzy_threshold: 0.7], []) + + assert message.msgstr == ["first best"] + end + + test "a zero fuzzy threshold also matches empty and dissimilar messages" do + for {old, new} <- [{"", "abcd"}, {"abcd", ""}, {"abcd", "wxyz"}] do + old_po = %Messages{messages: [%Message.Singular{msgid: [old], msgstr: ["translation"]}]} + new_pot = %Messages{messages: [%Message.Singular{msgid: [new]}]} + + assert {%Messages{messages: [message]}, %{fuzzy_matches: 1}} = + Merger.merge(old_po, new_pot, "en", [fuzzy: true, fuzzy_threshold: 0.0], []) + + assert message.msgstr == ["translation"] + end + end + + test "fuzzy matching preserves the scorer's handling of malformed UTF-8" do + for {old, new, translation} <- [ + {[<<255>>], "z", [""]}, + {["z"], <<255>>, [""]}, + {["abc", <<255>>], "abc", ["translation"]}, + {[<<255>>], <<255>>, ["translation"]}, + {[<<255>>], "", [""]}, + {[""], <<255>>, [""]} + ], + prefix <- [[], ["a!"]] do + old_po = %Messages{ + messages: + Enum.with_index(old, fn text, i -> + %Message.Singular{msgctxt: [to_string(i)], msgid: [text], msgstr: ["translation"]} + end) + } + + new_pot = %Messages{ + messages: Enum.map(prefix ++ [new], &%Message.Singular{msgctxt: ["new"], msgid: [&1]}) + } + + # Older scorers accept malformed bytes; newer ones may raise, except + # for their equality/empty fast paths. Preserve the platform's behavior. + expected = + try do + for text <- prefix ++ [new], previous <- old, do: String.jaro_distance(previous, text) + + :ok + rescue + error -> {:error, error.__struct__} + end + + merge = fn -> Merger.merge(old_po, new_pot, "en", @opts, @gettext_config) end + + case expected do + :ok -> + {po, _stats} = merge.() + assert List.last(po.messages).msgstr == translation + + {:error, exception} -> + assert_raise exception, merge + end + end + end + test "if there's a Plural-Forms header, it's used to determine number of plural forms" do old_po = %Messages{ headers: [~s(Plural-Forms: nplurals=3;plural=n>1;)],