Keep the final-template prune on in grammars with partial rules wherever no partial word can be involved

docs/review/hermitcrab.md

@@ -11,7 +11,7 @@ Governs `src/SIL.Machine.Morphology.HermitCrab/**/*.cs`.
   whether equivalent analyses can be merged safely. The merge key
   (`AnalysisMergeKey`) compares exactly these fields: shape, stratum identity, syntactic
   feature structure, realizational feature structure, non-head count, per-rule
-  unapplication counts, and `FinalTemplateState`. Check a new field against that list.
+  unapplication counts, `FinalTemplateState`, and `NeedsPartial`. Check a new field against that list.
 - Inspect allocations and retained object lifetimes only in changed inner loops. If the
   change claims a performance improvement, require a reproducible benchmark or measured
   artifact in addition to semantic regression tests.

src/SIL.Machine.Morphology.HermitCrab/AnalysisAffixTemplateRule.cs

@@ -41,27 +41,33 @@ public IEnumerable<Word> Apply(Word input)
             if (_morpher.TraceManager.IsTracing)
                 _morpher.TraceManager.BeginUnapplyTemplate(_template, input);
 
-            // Do not allow a final template to unapply if we don't need to worry about partial rules
-            // and a non-template was last unapplied.
-            if (
-                (!_morpher.HasPartialRules || _morpher.AlwaysEnforceFinalTemplates)
-                && input.FinalTemplateState == FinalTemplateState.NonTemplate
+            // In synthesis a non-template rule follows a final template only if a partial rule applied earlier.
+            bool afterNonTemplate =
+                _morpher.TracksFinalTemplateState
                 && _template.IsFinal
-            )
+                && input.FinalTemplateState == FinalTemplateState.NonTemplate;
+            if (afterNonTemplate && _morpher.IsFinalTemplatePrunable(_template, input))
             {
                 if (_morpher.TraceManager.IsTracing)
                 {
                     _morpher.TraceManager.EndUnapplyTemplate(
                         _template,
                         input,
                         false,
-                        FailureReason.NonPartialRuleProhibitedAfterFinalTemplate
+                        FailureReason.NonPartialRuleProhibitedAfterFinalTemplate,
+                        null
                     );
                 }
                 return Enumerable.Empty<Word>();
             }
 
             Word inWord = input.Clone();
+            // The prune check above passed, so the word is partial at this template or the analysis fails.
+            if (afterNonTemplate)
+            {
+                inWord.NeedsPartial = true;
+                inWord.OwedPartialTemplate = _template;
+            }
             inWord.Freeze();
 
             var output = new HashSet<Word>(FreezableEqualityComparer<Word>.Default);
@@ -70,6 +76,14 @@ public IEnumerable<Word> Apply(Word input)
             else
                 ParallelApplySlots(inWord, output);
 
+            // The output with every slot skipped did not unapply the template, so it owes nothing.
+            if (afterNonTemplate && output.Remove(inWord))
+            {
+                Word unchanged = input.Clone();
+                unchanged.Freeze();
+                output.Add(unchanged);
+            }
+
             return output;
         }
 
@@ -83,13 +97,13 @@ private void ApplySlots(Word inWord, int index, HashSet<Word> output)
                 if (!_template.Slots[i].Optional)
                 {
                     if (_morpher.TraceManager.IsTracing)
-                        _morpher.TraceManager.EndUnapplyTemplate(_template, inWord, false, FailureReason.None);
+                        _morpher.TraceManager.EndUnapplyTemplate(_template, inWord, false, FailureReason.None, null);
                     return;
                 }
             }
 
             if (_morpher.TraceManager.IsTracing)
-                _morpher.TraceManager.EndUnapplyTemplate(_template, inWord, true, FailureReason.None);
+                _morpher.TraceManager.EndUnapplyTemplate(_template, inWord, true, FailureReason.None, null);
             output.Add(inWord);
         }
 
@@ -125,7 +139,8 @@ private void ParallelApplySlots(Word inWord, HashSet<Word> output)
                                         _template,
                                         work.Item1,
                                         false,
-                                        FailureReason.None
+                                        FailureReason.None,
+                                        null
                                     );
                                 add = false;
                                 break;
@@ -139,7 +154,8 @@ private void ParallelApplySlots(Word inWord, HashSet<Word> output)
                                     _template,
                                     work.Item1,
                                     true,
-                                    FailureReason.None
+                                    FailureReason.None,
+                                    null
                                 );
                             outStack.Push(work.Item1);
                         }

src/SIL.Machine.Morphology.HermitCrab/AnalysisStratumRule.cs

@@ -184,6 +184,11 @@ internal IEnumerable<Word> Apply(Word input, ref int alternativeCount)
 
         private IEnumerable<Word> ApplyMorphologicalRules(Word input)
         {
+            if (_morpher.OwesImpossiblePartial(input))
+            {
+                _morpher.TraceOwedPartialDropped(input);
+                yield break;
+            }
             foreach (Word mruleOutWord in _mrulesRule.Apply(input).Distinct(FreezableEqualityComparer<Word>.Default))
             {
                 switch (_stratum.MorphologicalRuleOrder)
@@ -203,6 +208,11 @@ private IEnumerable<Word> ApplyMorphologicalRules(Word input)
 
         private IEnumerable<Word> ApplyTemplates(Word input)
         {
+            if (_morpher.OwesImpossiblePartial(input))
+            {
+                _morpher.TraceOwedPartialDropped(input);
+                yield break;
+            }
             foreach (Word tempOutWord in _templatesRule.Apply(input).Distinct(FreezableEqualityComparer<Word>.Default))
             {
                 switch (_stratum.MorphologicalRuleOrder)
@@ -235,6 +245,7 @@ private IEnumerable<Word> ApplyTemplates(Word input)
             private readonly int _nonHeadCount;
             private readonly IReadOnlyDictionary<IMorphologicalRule, int> _ruleCounts;
             private readonly FinalTemplateState _finalTemplateState;
+            private readonly bool _needsPartial;
             private readonly int _hashCode;
 
             // Freezes word.SyntacticFeatureStruct, which Word.Freeze leaves unfrozen.
@@ -253,6 +264,7 @@ public AnalysisMergeKey(Word word)
                 _nonHeadCount = word.NonHeadCount;
                 _ruleCounts = word.UnappliedRuleCounts;
                 _finalTemplateState = word.FinalTemplateState;
+                _needsPartial = word.NeedsPartial;
 
                 _shape.Freeze();
                 _syntacticFS.Freeze();
@@ -265,6 +277,7 @@ public AnalysisMergeKey(Word word)
                 hash = hash * 31 + _realizationalFS.GetFrozenHashCode();
                 hash = hash * 31 + _nonHeadCount;
                 hash = hash * 31 + _finalTemplateState.GetHashCode();
+                hash = hash * 31 + (_needsPartial ? 1 : 0);
                 if (_ruleCounts != null)
                 {
                     int multisetHash = 0;
@@ -292,7 +305,7 @@ public bool Equals(AnalysisMergeKey other)
                     || !_realizationalFS.ValueEquals(other._realizationalFS)
                 )
                     return false;
-                if (_finalTemplateState != other._finalTemplateState)
+                if (_finalTemplateState != other._finalTemplateState || _needsPartial != other._needsPartial)
                     return false;
                 return RuleCountsEqual(_ruleCounts, other._ruleCounts);
             }

src/SIL.Machine.Morphology.HermitCrab/FinalTemplatePartialReachability.cs

@@ -0,0 +1,142 @@
+using System.Collections.Generic;
+using System.Linq;
+using SIL.Machine.FeatureModel;
+using SIL.Machine.Morphology.HermitCrab.MorphologicalRules;
+
+namespace SIL.Machine.Morphology.HermitCrab
+{
+    /// <summary>
+    /// Finds the affix templates that a partial word can reach in synthesis. A word is partial at a template only
+    /// if a partial affix-process rule that is neither a template-slot nor a clitic rule applied before it (a partial root takes no template), in the template's
+    /// stratum or an earlier one. Its part of speech is then whatever that rule leaves, changed by any rules
+    /// applied since. The search over-approximates: it tracks only the part-of-speech feature, treats an
+    /// unspecified or variable part of speech as every part of speech, ignores application counts, rule order,
+    /// template gating and every other feature, and lets every rule of a stratum apply any number of times. A
+    /// template it does not return can therefore never see a partial word.
+    /// </summary>
+    internal static class FinalTemplatePartialReachability
+    {
+        public static HashSet<AffixTemplate> Compute(Language lang)
+        {
+            var result = new HashSet<AffixTemplate>();
+            SymbolicFeature posFeature = lang.SyntacticFeatureSystem.PartOfSpeechFeature;
+            HashSet<FeatureSymbol> all =
+                posFeature == null
+                    ? new HashSet<FeatureSymbol>()
+                    : new HashSet<FeatureSymbol>(posFeature.PossibleSymbols);
+
+            // Part-of-speech values a partial word may carry on leaving the strata processed so far.
+            var reach = new HashSet<FeatureSymbol>();
+            bool anyPartial = false;
+            foreach (Stratum stratum in lang.Strata) // synthesis order
+            {
+                List<IMorphologicalRule> rules = stratum
+                    .MorphologicalRules.Concat(stratum.AffixTemplates.SelectMany(t => t.Slots).SelectMany(s => s.Rules))
+                    .Distinct()
+                    .ToList();
+
+                foreach (AffixProcessRule rule in rules.OfType<AffixProcessRule>().Where(IsSynthesisPartial))
+                {
+                    anyPartial = true;
+                    reach.UnionWith(
+                        Apply(all, all, rule.RequiredSyntacticFeatureStruct, rule.OutSyntacticFeatureStruct)
+                    );
+                }
+                if (!anyPartial)
+                    continue;
+                if (all.Count == 0)
+                {
+                    // No parts of speech to tell templates apart: every later template is reachable.
+                    result.UnionWith(stratum.AffixTemplates);
+                    continue;
+                }
+
+                // Close the set under every rule of this stratum.
+                bool changed = true;
+                while (changed)
+                {
+                    changed = false;
+                    foreach (IMorphologicalRule rule in rules)
+                    {
+                        if (reach.Count == 0)
+                            break;
+                        IEnumerable<FeatureSymbol> next = ApplyRule(all, reach, rule);
+                        foreach (FeatureSymbol symbol in next.ToList())
+                            changed |= reach.Add(symbol);
+                    }
+                }
+
+                foreach (AffixTemplate template in stratum.AffixTemplates)
+                {
+                    if (reach.Count == 0)
+                        continue;
+                    HashSet<FeatureSymbol> required = Constraint(all, template.RequiredSyntacticFeatureStruct);
+                    if (required == null || required.Overlaps(reach))
+                        result.Add(template);
+                }
+            }
+            return result;
+        }
+
+        // Synthesis marks a word partial only for a non-template, non-clitic partial rule
+        // (SynthesisAffixProcessRule), and a slot or clitic rule is exempt from the final-template check.
+        internal static bool IsSynthesisPartial(AffixProcessRule rule)
+        {
+            return rule.IsPartial && !rule.IsTemplateRule && !rule.GetIsCliticRule();
+        }
+
+        private static IEnumerable<FeatureSymbol> ApplyRule(
+            HashSet<FeatureSymbol> all,
+            HashSet<FeatureSymbol> input,
+            IMorphologicalRule rule
+        )
+        {
+            switch (rule)
+            {
+                case AffixProcessRule apr:
+                    return Apply(all, input, apr.RequiredSyntacticFeatureStruct, apr.OutSyntacticFeatureStruct);
+                case CompoundingRule cr:
+                    return Apply(all, input, cr.HeadRequiredSyntacticFeatureStruct, cr.OutSyntacticFeatureStruct);
+                case RealizationalAffixProcessRule rr:
+                    return Apply(all, input, rr.RequiredSyntacticFeatureStruct, rr.RealizationalFeatureStruct);
+                default:
+                    // Unknown rule kind: assume it can produce any part of speech.
+                    return all;
+            }
+        }
+
+        /// <summary>Unify with <paramref name="required"/>, then priority-union <paramref name="output"/>.</summary>
+        private static IEnumerable<FeatureSymbol> Apply(
+            HashSet<FeatureSymbol> all,
+            HashSet<FeatureSymbol> input,
+            FeatureStruct required,
+            FeatureStruct output
+        )
+        {
+            HashSet<FeatureSymbol> req = Constraint(all, required);
+            bool applicable = req == null ? input.Count > 0 : req.Overlaps(input);
+            if (!applicable)
+                return Enumerable.Empty<FeatureSymbol>();
+            HashSet<FeatureSymbol> outPos = Constraint(all, output);
+            if (outPos != null)
+                return outPos;
+            if (req == null)
+                return input;
+            return input.Where(req.Contains);
+        }
+
+        /// <summary>The part-of-speech values a feature structure allows, or null if it leaves them open.</summary>
+        private static HashSet<FeatureSymbol> Constraint(HashSet<FeatureSymbol> all, FeatureStruct fs)
+        {
+            if (fs == null)
+                return null;
+            if (!fs.TryGetValue(SyntacticFeatureSystem.PartOfSpeechID, out SymbolicFeatureValue value))
+                return null;
+            if (value.IsVariable)
+                return null;
+            var symbols = new HashSet<FeatureSymbol>(value.Values);
+            // A value that names no known symbol is treated as unconstrained, to stay conservative.
+            return symbols.Count == 0 ? null : symbols;
+        }
+    }
+}

src/SIL.Machine.Morphology.HermitCrab/ITraceManager.cs

@@ -28,6 +28,7 @@ public enum FailureReason
         MaxApplicationCount,
         AllomorphNotApplicable,
         LexicalLookupFeatures,
+        OwedPartialCannotUnapply,
     }
 
     public interface ITraceManager
@@ -45,7 +46,13 @@ public interface ITraceManager
         void PhonologicalRuleNotUnapplied(IPhonologicalRule rule, int subruleIndex, Word input);
 
         void BeginUnapplyTemplate(AffixTemplate template, Word input);
-        void EndUnapplyTemplate(AffixTemplate template, Word output, bool unapplied, FailureReason reason);
+        void EndUnapplyTemplate(
+            AffixTemplate template,
+            Word output,
+            bool unapplied,
+            FailureReason reason,
+            object failureObj
+        );
 
         void MorphologicalRuleUnapplied(IMorphologicalRule rule, int subruleIndex, Word input, Word output);
         void MorphologicalRuleNotUnapplied(IMorphologicalRule rule, int subruleIndex, Word input);