perf(hermitcrab): keep the required-slot look-ahead testing behind slots that write back deleted stem segments

src/SIL.Machine.Morphology.HermitCrab/AnalysisAffixTemplateRule.cs

@@ -80,8 +80,8 @@ public IEnumerable<Word> Apply(Word input)
             return output;
         }
 
-        // Testing runs from the next slot to unapply and stops after an impure slot, which makes the test
-        // for any slot behind it unsound. Shadow mode prunes nothing and carries the fired slots in fired.
+        // Testing runs from the next slot to unapply and stops after a slot whose written-back segments cannot be
+        // bounded. Shadow mode prunes nothing and carries the fired slots in fired.
         private bool RequiredSlotCannotUnapply(Word word, int lastSlot, ref int[] fired)
         {
             RequiredSlotLookaheadMode mode = _morpher.RequiredSlotLookahead;
@@ -118,7 +118,7 @@ private bool RequiredSlotCannotUnapply(Word word, int lastSlot, ref int[] fired)
                     _morpher.RequiredSlotShadow?.Invoke("fired", _template, j);
                 }
                 _lookahead.Advance(j, ref walk);
-                if (!_lookahead.IsPure(j))
+                if (!_lookahead.CanWalkPast(j))
                     break;
             }
             if (unifications != 0)

src/SIL.Machine.Morphology.HermitCrab/RequiredSlotMaterial.cs

@@ -47,6 +47,11 @@
         /// <summary>False once a slot walked so far can reorder the nodes it copies.</summary>
         public bool Ordered;
 
+        /// <summary>
+        /// The slots walked so far that can write segments the shape does not hold, as a bit mask of slot indexes.
+        /// </summary>
+        public int Suppliers;
+
         public static SlotWalk Start()
         {
             return new SlotWalk
@@ -94,23 +99,45 @@
     /// (<see cref="SlotWalk.ReachLeft"/>), and the nodes of a required ordered slot lie after the prefix and before
     /// the suffix it inserts (<see cref="SlotWalk.Lo"/>, <see cref="SlotWalk.Hi"/>). Both only hold while every
     /// slot walked so far is ordered, and the reach only while none can remove from the middle.
+    ///
+    /// Supply. The Presence argument above fails for an impure slot, whose unapplication can write back a segment
+    /// the shape lacks (<see cref="AnalysisMorphologicalTransform.GenerateShape"/> untruncates the segments of an
+    /// uncopied part). Such a slot is walked past, and is called a supplier, only when it copies and inserts and
+    /// nothing more, because a modified node could change any feature of any node. Its budget is the most
+    /// segments any of its allomorphs writes back, and its supply is the constraints of those segments; a constraint with variables
+    /// is recorded as a null supply, which covers any required constraint. A required constraint that unifies with
+    /// no node of the shape is missing, and is covered only if some supplier already walked has a supply that
+    /// unifies with it (<c>AllomorphSummary.Cover</c>). The test passes when the missing constraints number no more
+    /// than the summed budget of the suppliers covering them, so it can only be weaker than the pure case. Slots
+    /// walked past as suppliers are recorded in <see cref="SlotWalk.Suppliers"/>. Once a supplier has been walked,
+    /// the shape may later gain nodes at any position, so the ordered test, the edge window and the reach no
+    /// longer hold and only the unordered presence test with the budget runs (<see cref="Advance"/>). A change to
+    /// <c>Embeds</c> must not assume that every slot walked is pure.
     /// </remarks>
     internal class RequiredSlotMaterial
     {
+        // Slots at or above this index cannot be recorded in SlotWalk.Suppliers.
+        private const int MaxSupplierSlots = 31;
+
         private readonly SlotSummary[] _slots;
 
         public RequiredSlotMaterial(AffixTemplate template)
         {
             _slots = template.Slots.Select(Summarize).ToArray();
+            for (int i = 0; i < _slots.Length; i++)
+            {
+                foreach (AllomorphSummary allomorph in _slots[i].Allomorphs)
+                    allomorph.Cover = CoverMasks(allomorph, i);
+            }
         }
 
         /// <summary>
-        /// Whether every rule of the slot is pure, so that unapplying the slot leaves the segments needed by
-        /// slots before it no easier to find.
+        /// Whether the walk may go on past the slot: it is pure, or the segments it can write are bounded, so that
+        /// the slots before it are tested with those segments allowed to be missing from the shape.
         /// </summary>
-        public bool IsPure(int slot)
+        public bool CanWalkPast(int slot)
         {
-            return _slots[slot].IsPure;
+            return _slots[slot].IsPure || (_slots[slot].Suppliable && slot < MaxSupplierSlots);
         }
 
         /// <summary>
@@ -120,6 +147,8 @@
         public void Advance(int slot, ref SlotWalk walk)
         {
             SlotSummary summary = _slots[slot];
+            if (!summary.IsPure && summary.Suppliable && slot < MaxSupplierSlots)
+                walk.Suppliers |= 1 << slot;
             if (!walk.Ordered || !summary.IsOrdered)
             {
                 walk.Ordered = false;
@@ -135,7 +164,8 @@
 
         /// <summary>
         /// True if no allomorph of the slot can unapply to a word with this shape, provided that every slot
-        /// walked between this word and the slot is pure. When the slot can unapply and is ordered, narrows the
+        /// walked between this word and the slot is pure or has its written-back segments bounded and counted
+        /// in the walk. When the slot can unapply and is ordered, narrows the
         /// window of the walk for the slots before it.
         /// </summary>
         /// <param name="slot">The slot to test.</param>
@@ -160,7 +190,7 @@
                 int suffixStart = int.MaxValue;
                 bool found = walk.Ordered
                     ? Embeds(allomorph, shape, ref walk, ref unifications, out prefixEnd, out suffixStart)
-                    : AllPresent(allomorph, shape, ref unifications);
+                    : AllPresent(allomorph, shape, walk.Suppliers, ref unifications);
                 if (!found)
                     continue;
 
@@ -257,12 +287,15 @@
             return taken == allomorph.Post;
         }
 
-        // Once a slot that can reorder nodes has been walked the order of the shape says nothing about the order of
-        // the shape the slot unapplies to, so each constraint is only looked for on its own.
-        private static bool AllPresent(AllomorphSummary allomorph, Shape shape, ref long unifications)
+        // The order of the shape says nothing once a reordering slot is walked, so each constraint is looked for
+        // alone. One with no node may be met by a segment a walked supplier writes, once each.
+        private bool AllPresent(AllomorphSummary allomorph, Shape shape, int suppliers, ref long unifications)
         {
-            foreach (FeatureStruct required in allomorph.Sequence)
+            int missing = 0;
+            int covering = 0;
+            for (int i = 0; i < allomorph.Sequence.Length; i++)
             {
+                FeatureStruct required = allomorph.Sequence[i];
                 if (required == null)
                     continue;
 
@@ -275,10 +308,46 @@
                         break;
                     }
                 }
-                if (!found)
+                if (found)
+                    continue;
+
+                int coverable = allomorph.Cover[i] & suppliers;
+                if (coverable == 0)
                     return false;
+                missing++;
+                covering |= coverable;
             }
-            return true;
+            if (missing == 0)
+                return true;
+
+            int budget = 0;
+            for (int slot = 0; covering != 0; slot++, covering >>= 1)
+            {
+                if ((covering & 1) != 0)
+                    budget += _slots[slot].Budget;
+            }
+            return missing <= budget;
+        }
+
+        // For each constraint, the slots that can write a segment it unifies with.
+        private int[] CoverMasks(AllomorphSummary allomorph, int own)
+        {
+            var masks = new int[allomorph.Sequence.Length];
+            for (int i = 0; i < masks.Length; i++)
+            {
+                FeatureStruct required = allomorph.Sequence[i];
+                if (required == null)
+                    continue;
+
+                for (int slot = 0; slot < _slots.Length && slot < MaxSupplierSlots; slot++)
+                {
+                    if (slot == own || !_slots[slot].Suppliable || _slots[slot].IsPure)
+                        continue;
+                    if (_slots[slot].Supply.Any(fs => fs == null || fs.IsUnifiable(required, false, null)))
+                        masks[i] |= 1 << slot;
+                }
+            }
+            return masks;
         }
 
         private static bool IsUnifiable(ShapeNode node, FeatureStruct required, ref long unifications)
@@ -291,6 +360,7 @@
         {
             var summary = new SlotSummary();
             var allomorphs = new List<AllomorphSummary>();
+            var supply = new List<FeatureStruct>();
             foreach (MorphemicMorphologicalRule rule in slot.Rules)
             {
                 IList<AffixProcessAllomorph> ruleAllomorphs;
@@ -321,6 +391,11 @@
                     bool pure = IsPure(allomorph);
                     if (!pure)
                         summary.IsPure = false;
+                    int restored = Restored(allomorph, supply);
+                    if (restored < 0)
+                        summary.Suppliable = false;
+                    else
+                        summary.Budget = Math.Max(summary.Budget, restored);
                     if (!pure || !IsOrdered(allomorph))
                         summary.IsOrdered = false;
 
@@ -334,13 +409,15 @@
                 }
             }
             summary.Allomorphs = allomorphs.ToArray();
+            summary.Supply = supply.ToArray();
             return summary;
         }
 
         private static void MakeOpaque(SlotSummary summary)
         {
             summary.MayAlwaysUnapply = true;
             summary.IsPure = false;
+            summary.Suppliable = false;
             summary.IsOrdered = false;
             summary.ReachLeft = SlotWalk.Unbounded;
             summary.ReachRight = SlotWalk.Unbounded;
@@ -453,6 +530,53 @@
             return true;
         }
 
+        // The most segments the allomorph writes back, their constraints added to supply (null with variables).
+        // -1 if it does more than copy and insert, as a modified node can change any feature of any node.
+        private static int Restored(AffixProcessAllomorph allomorph, List<FeatureStruct> supply)
+        {
+            var copied = new HashSet<string>();
+            foreach (MorphologicalOutputAction action in allomorph.Rhs)
+            {
+                if (action is CopyFromInput)
+                    copied.Add(action.PartName);
+                else if (!IsInsertion(action))
+                    return -1;
+            }
+
+            int total = 0;
+            foreach (Pattern<Word, ShapeNode> part in allomorph.Lhs)
+            {
+                if (RestoredSegments(part) == 0)
+                    continue;
+                if (copied.Contains(part.Name) && MinSegments(part) > 0)
+                    continue;
+                total += RestoredSegments(part);
+                AddRestoredConstraints(part, supply);
+            }
+            return total;
+        }
+
+        private static void AddRestoredConstraints(PatternNode<Word, ShapeNode> node, List<FeatureStruct> supply)
+        {
+            foreach (PatternNode<Word, ShapeNode> child in node.Children)
+            {
+                if (child is Constraint<Word, ShapeNode> constraint && constraint.Type() == HCFeatureSystem.Segment)
+                {
+                    FeatureStruct copy = null;
+                    if (!constraint.FeatureStruct.HasVariables)
+                    {
+                        copy = constraint.FeatureStruct.Clone();
+                        copy.Freeze();
+                    }
+                    supply.Add(copy);
+                }
+                else if (!(child is Quantifier<Word, ShapeNode> quantifier) || quantifier.MaxOccur > 0)
+                {
+                    AddRestoredConstraints(child, supply);
+                }
+            }
+        }
+
         // GenerateShape writes parts in left-hand order, so copying them in another order, or one twice, can
         // reorder or repeat nodes.
         private static bool IsOrdered(AffixProcessAllomorph allomorph)
@@ -519,6 +643,7 @@
         private class AllomorphSummary
         {
             public FeatureStruct[] Sequence;
+            public int[] Cover;
             public int Pre;
             public int Post;
             public int ReachLeft;
@@ -530,6 +655,9 @@
             public bool IsPure = true;
             public bool IsOrdered = true;
             public bool MayAlwaysUnapply;
+            public bool Suppliable = true;
+            public int Budget;
+            public FeatureStruct[] Supply;
             public int ReachLeft;
             public int ReachRight;
             public AllomorphSummary[] Allomorphs;

tests/SIL.Machine.Morphology.HermitCrab.Tests/AnalysisAffixTemplateFusionTests.cs

@@ -175,9 +175,9 @@ public void Apply_SlotWithABoundedRunThatMayBeEmpty_RestoresASegmentTheInnerSlot
     }
 
     [Test]
-    public void Apply_SlotWithABoundedRunThatMayBeEmpty_StaysImpureWhenTheMaterialIsAbsent()
+    public void Apply_SlotWithABoundedRunThatMayBeEmpty_StaysImpureWhenTheRunCouldBeTheMissingSegment()
     {
-        AffixTemplate template = TemplateOf(Slot(false, Suffix("ti")), Slot(true, BoundedRunSuffix("z")));
+        AffixTemplate template = TemplateOf(Slot(false, Suffix("i")), Slot(true, BoundedRunSuffix("z")));
 
         (WalkResult off, WalkResult on) = WalkBoth(template, "sagz");
 

tests/SIL.Machine.Morphology.HermitCrab.Tests/AnalysisAffixTemplateRuleTests.cs

@@ -280,10 +280,12 @@ public void Apply_RequiredSlotWithNullAffix_IsNeverPruned()
         });
     }
 
-    // The input lacks the inner slot's "ti"; a pure outer rule permits pruning before it is tried.
+    // An optional slot inserting "z"; the word lacks the inner slot's segments, so a pure rule prunes at entry.
+    // An impure rule does not if it can restore them: those cases use the one segment it may restore.
     private (WalkResult Off, WalkResult On) WalkWithOptionalSlot(
         Pattern<Word, ShapeNode>[] lhs,
-        MorphologicalOutputAction[] rhs
+        MorphologicalOutputAction[] rhs,
+        string inner = "ti"
     )
     {
         var rule = new AffixProcessRule
@@ -301,7 +303,7 @@ MorphologicalOutputAction[] rhs
             allomorph.Rhs.Add(action);
         allomorph.Rhs.Add(new InsertSegments(Table3, "z"));
         rule.Allomorphs.Add(allomorph);
-        AffixTemplate template = TemplateOf(Slot(false, SuffixRule("ti", "ti")), Slot(true, rule));
+        AffixTemplate template = TemplateOf(Slot(false, SuffixRule(inner, inner)), Slot(true, rule));
         return WalkBoth(template, "sagz");
     }
 
@@ -310,7 +312,8 @@ public void Apply_CopiedPartCappedAtOneThatMayMatchNothing_MakesTheSlotImpure()
     {
         (WalkResult off, WalkResult on) = WalkWithOptionalSlot(
             new[] { Pattern<Word, ShapeNode>.New("1").Annotation(Any).Optional.Value },
-            new MorphologicalOutputAction[] { new CopyFromInput("1") }
+            new MorphologicalOutputAction[] { new CopyFromInput("1") },
+            "i"
         );
 
         Assert.That(on.RuleAttempts, Is.GreaterThan(0), "the optional rule is still tried at entry");
@@ -369,7 +372,8 @@ public void Apply_CopiedAlternationWithAnOptionalBranch_MakesTheSlotImpure()
                     .Optional.Or.Annotation(Character(Table3, "u"))
                     .Value,
             },
-            new MorphologicalOutputAction[] { new CopyFromInput("1"), new CopyFromInput("2") }
+            new MorphologicalOutputAction[] { new CopyFromInput("1"), new CopyFromInput("2") },
+            "i"
         );
 
         Assert.That(on.RuleAttempts, Is.GreaterThan(0), "the optional rule is still tried at entry");
@@ -510,7 +514,7 @@ public void Compile_UntypedConstraints_DisablesTheSummaryConservatively(bool unt
         var summary = new RequiredSlotMaterial(TemplateOf(Slot(false, rule)));
         var walk = SlotWalk.Start();
         long unifications = 0;
-        Assert.That(summary.IsPure(0), Is.False);
+        Assert.That(summary.CanWalkPast(0), Is.False);
         Assert.That(summary.CannotUnapply(0, Input("sag").Shape, ref walk, ref unifications), Is.False);
     }