Prune affix template partials whose required slots cannot find their material, in order and at the word edges

src/SIL.Machine.Morphology.HermitCrab/AnalysisAffixTemplateRule.cs

@@ -14,6 +14,7 @@ internal class AnalysisAffixTemplateRule : IRule<Word, ShapeNode>
         private readonly Morpher _morpher;
         private readonly AffixTemplate _template;
         private readonly List<IRule<Word, ShapeNode>> _rules;
+        private readonly RequiredSlotMaterial _lookahead;
 
         public AnalysisAffixTemplateRule(Morpher morpher, AffixTemplate template)
         {
@@ -26,6 +27,7 @@ public AnalysisAffixTemplateRule(Morpher morpher, AffixTemplate template)
                     FreezableEqualityComparer<Word>.Default
                 ))
             );
+            _lookahead = new RequiredSlotMaterial(template);
         }
 
         public IEnumerable<Word> Apply(Word input)
@@ -55,51 +57,129 @@ public IEnumerable<Word> Apply(Word input)
                         _template,
                         input,
                         false,
-                        FailureReason.NonPartialRuleProhibitedAfterFinalTemplate
+                        FailureReason.NonPartialRuleProhibitedAfterFinalTemplate,
+                        null
                     );
                 }
                 return Enumerable.Empty<Word>();
             }
 
+            int[] fired = null;
+            if (RequiredSlotCannotUnapply(input, _rules.Count - 1, ref fired))
+                return Enumerable.Empty<Word>();
+
             Word inWord = input.Clone();
             inWord.Freeze();
 
             var output = new HashSet<Word>(FreezableEqualityComparer<Word>.Default);
             if (_morpher.MaxDegreeOfParallelism == 1)
-                ApplySlots(inWord, _rules.Count - 1, output);
+                ApplySlots(inWord, _rules.Count - 1, output, fired);
             else
-                ParallelApplySlots(inWord, output);
+                ParallelApplySlots(inWord, output, fired);
 
             return output;
         }
 
-        private void ApplySlots(Word inWord, int index, HashSet<Word> output)
+        // Testing runs from the next slot to unapply and stops after an impure slot, which makes the test
+        // for any slot behind it unsound. Shadow mode prunes nothing and carries the fired slots in fired.
+        private bool RequiredSlotCannotUnapply(Word word, int lastSlot, ref int[] fired)
+        {
+            RequiredSlotLookaheadMode mode = _morpher.RequiredSlotLookahead;
+            if (mode == RequiredSlotLookaheadMode.Off)
+                return false;
+
+            long unifications = 0;
+            bool cannot = false;
+            SlotWalk walk = SlotWalk.Start();
+            for (int j = lastSlot; j >= 0; j--)
+            {
+                if (
+                    !_template.Slots[j].Optional
+                    && !IsFired(fired, j)
+                    && _lookahead.CannotUnapply(j, word.Shape, ref walk, ref unifications)
+                )
+                {
+                    if (mode == RequiredSlotLookaheadMode.On)
+                    {
+                        cannot = true;
+                        if (_morpher.TraceManager.IsTracing)
+                        {
+                            _morpher.TraceManager.EndUnapplyTemplate(
+                                _template,
+                                word,
+                                false,
+                                FailureReason.UnreachableRequiredSlot,
+                                _template.Slots[j]
+                            );
+                        }
+                        break;
+                    }
+                    fired = fired == null ? new[] { j } : fired.Append(j).ToArray();
+                    _morpher.RequiredSlotShadow?.Invoke("fired", _template, j);
+                }
+                _lookahead.Advance(j, ref walk);
+                if (!_lookahead.IsPure(j))
+                    break;
+            }
+            if (unifications != 0)
+                _morpher.RequiredSlotCheckObserver?.Invoke(unifications);
+            return cannot;
+        }
+
+        private static bool IsFired(int[] fired, int slot)
+        {
+            return fired != null && Array.IndexOf(fired, slot) >= 0;
+        }
+
+        private void ReportShadow(string kind, int[] fired, int slot)
+        {
+            if (fired == null || _morpher.RequiredSlotShadow == null)
+                return;
+            if (slot < 0)
+            {
+                foreach (int j in fired)
+                    _morpher.RequiredSlotShadow(kind, _template, j);
+            }
+            else if (IsFired(fired, slot))
+            {
+                _morpher.RequiredSlotShadow(kind, _template, slot);
+            }
+        }
+
+        private void ApplySlots(Word inWord, int index, HashSet<Word> output, int[] fired)
         {
             for (int i = index; i >= 0; i--)
             {
                 foreach (Word outWord in _rules[i].Apply(inWord))
-                    ApplySlots(outWord, i - 1, output);
+                {
+                    ReportShadow("violated", fired, i);
+                    int[] outFired = fired;
+                    if (RequiredSlotCannotUnapply(outWord, i - 1, ref outFired))
+                        continue;
+                    ApplySlots(outWord, i - 1, output, outFired);
+                }
 
                 if (!_template.Slots[i].Optional)
                 {
                     if (_morpher.TraceManager.IsTracing)
-                        _morpher.TraceManager.EndUnapplyTemplate(_template, inWord, false, FailureReason.None);
+                        _morpher.TraceManager.EndUnapplyTemplate(_template, inWord, false, FailureReason.None, null);
                     return;
                 }
             }
 
             if (_morpher.TraceManager.IsTracing)
-                _morpher.TraceManager.EndUnapplyTemplate(_template, inWord, true, FailureReason.None);
+                _morpher.TraceManager.EndUnapplyTemplate(_template, inWord, true, FailureReason.None, null);
+            ReportShadow("completed-under-fired", fired, -1);
             output.Add(inWord);
         }
 
-        private void ParallelApplySlots(Word inWord, HashSet<Word> output)
+        private void ParallelApplySlots(Word inWord, HashSet<Word> output, int[] fired)
         {
             ParallelOptions parallelOptions = _morpher.CreateParallelOptions();
             var outStack = new ConcurrentStack<Word>();
-            var from = new ConcurrentStack<Tuple<Word, int>>();
-            from.Push(Tuple.Create(inWord, _rules.Count - 1));
-            var to = new ConcurrentStack<Tuple<Word, int>>();
+            var from = new ConcurrentStack<Tuple<Word, int, int[]>>();
+            from.Push(Tuple.Create(inWord, _rules.Count - 1, fired));
+            var to = new ConcurrentStack<Tuple<Word, int, int[]>>();
             while (!from.IsEmpty)
             {
                 to.Clear();
@@ -111,9 +191,18 @@ private void ParallelApplySlots(Word inWord, HashSet<Word> output)
                         bool add = true;
                         for (int i = work.Item2; i >= 0; i--)
                         {
-                            Tuple<Word, int>[] workItems = _rules[i]
+                            int next = i - 1;
+                            Tuple<Word, int, int[]>[] workItems = _rules[i]
                                 .Apply(work.Item1)
-                                .Select(res => Tuple.Create(res, i - 1))
+                                .Select(res =>
+                                {
+                                    ReportShadow("violated", work.Item3, i);
+                                    int[] resFired = work.Item3;
+                                    if (!RequiredSlotCannotUnapply(res, next, ref resFired))
+                                        return Tuple.Create(res, next, resFired);
+                                    return null;
+                                })
+                                .Where(item => item != null)
                                 .ToArray();
                             if (workItems.Length > 0)
                                 to.PushRange(workItems);
@@ -125,7 +214,8 @@ private void ParallelApplySlots(Word inWord, HashSet<Word> output)
                                         _template,
                                         work.Item1,
                                         false,
-                                        FailureReason.None
+                                        FailureReason.None,
+                                        null
                                     );
                                 add = false;
                                 break;
@@ -139,13 +229,15 @@ private void ParallelApplySlots(Word inWord, HashSet<Word> output)
                                     _template,
                                     work.Item1,
                                     true,
-                                    FailureReason.None
+                                    FailureReason.None,
+                                    null
                                 );
+                            ReportShadow("completed-under-fired", work.Item3, -1);
                             outStack.Push(work.Item1);
                         }
                     }
                 );
-                ConcurrentStack<Tuple<Word, int>> temp = from;
+                ConcurrentStack<Tuple<Word, int, int[]>> temp = from;
                 from = to;
                 to = temp;
             }

src/SIL.Machine.Morphology.HermitCrab/ITraceManager.cs

@@ -27,6 +27,7 @@ public enum FailureReason
         NonPartialRuleRequiredAfterNonFinalTemplate,
         MaxApplicationCount,
         AllomorphNotApplicable,
+        UnreachableRequiredSlot,
     }
 
     public interface ITraceManager
@@ -44,7 +45,13 @@ public interface ITraceManager
         void PhonologicalRuleNotUnapplied(IPhonologicalRule rule, int subruleIndex, Word input);
 
         void BeginUnapplyTemplate(AffixTemplate template, Word input);
-        void EndUnapplyTemplate(AffixTemplate template, Word output, bool unapplied, FailureReason reason);
+        void EndUnapplyTemplate(
+            AffixTemplate template,
+            Word output,
+            bool unapplied,
+            FailureReason reason,
+            object failureObj
+        );
 
         void MorphologicalRuleUnapplied(IMorphologicalRule rule, int subruleIndex, Word input, Word output);
         void MorphologicalRuleNotUnapplied(IMorphologicalRule rule, int subruleIndex, Word input);

src/SIL.Machine.Morphology.HermitCrab/Morpher.cs

@@ -88,6 +88,38 @@ public ITraceManager TraceManager
 
         public int DeletionReapplications { get; set; }
 
+        /// <summary>
+        /// Whether affix template walks drop partials in which a required slot can no longer unapply because
+        /// the material it inserts is absent. See <see cref="RequiredSlotMaterial"/>.
+        /// </summary>
+        internal RequiredSlotLookaheadMode RequiredSlotLookahead { get; set; } = RequiredSlotLookaheadMode.On;
+
+        /// <summary>
+        /// Enable required-slot pruning during affix template analysis. On by default, also while
+        /// <see cref="ITraceManager.IsTracing"/> is set: each pruned partial is traced as
+        /// <see cref="FailureReason.UnreachableRequiredSlot"/> with the slot. Set it before parsing; changing it
+        /// during a parse alters how much is pruned, not the analyses returned.
+        /// </summary>
+        public bool PruneRequiredSlots
+        {
+            get { return RequiredSlotLookahead != RequiredSlotLookaheadMode.Off; }
+            set { RequiredSlotLookahead = value ? RequiredSlotLookaheadMode.On : RequiredSlotLookaheadMode.Off; }
+        }
+
+        /// <summary>
+        /// Called in <see cref="RequiredSlotLookaheadMode.Shadow"/> with a kind and the template and slot it is about:
+        /// "fired" (the test is true for a required slot), "violated" (a rule of a fired slot then unapplied under
+        /// that partial) and "completed-under-fired" (the template walk completed with a fired slot on its path).
+        /// Any "violated" or "completed-under-fired" is a soundness failure.
+        /// </summary>
+        internal Action<string, AffixTemplate, int> RequiredSlotShadow { get; set; }
+
+        /// <summary>
+        /// Called after each required-slot check that unified, with the number of unifications it performed.
+        /// Null unless a diagnostic sets it; it may be called from several threads.
+        /// </summary>
+        internal Action<long> RequiredSlotCheckObserver { get; set; }
+
         public int MaxStemCount { get; set; }
 
         /// <summary>