From d48b95b55021cee9b1bc0f3985b15088405c6c55 Mon Sep 17 00:00:00 2001 From: gspark Date: Wed, 4 Nov 2026 11:50:38 +0900 Subject: [PATCH] fix: refresh output-spec hook signatures after torch.export (#292) The forward hooks record each leaf module's output shape as a string while export is tracing. A later guard can still merge symbols: with torch 2.10 + transformers 4.46 LLaMA, position_ids gets its own batch symbol that is unified with input_ids' only after layer 0's input_layernorm ran, so its spec read (s72, s70, 64) while the node carried (s44, s70, 64). The no-op alias fold then did not match and the norm was indexed as a 2-tuple (B=1 IndexError, B=2 invalid reshape in KV-cache generation). Re-render the signatures from the recorded FakeTensors after export, and add a CPU test that the exported KV graph computes the model's logits. Co-Authored-By: Claude Opus 5.5 --- opt_prime/opt_prime/IR.py | 8 ++++++++ opt_prime/tests/test_kv_cache_batch.py | 13 +++++++++++++ 2 files changed, 21 insertions(+) diff --git a/opt_prime/opt_prime/IR.py b/opt_prime/opt_prime/IR.py index da7594c..24c00e6 100644 --- a/opt_prime/opt_prime/IR.py +++ b/opt_prime/opt_prime/IR.py @@ -2027,6 +2027,14 @@ def _retrieve_IR_export(self, model: nn.Module, use_kv_cache: bool = False, for_ if export_kwargs: _strip_runtime_asserts(exported_program) + # Re-render the hook signatures now that export is done. The shape strings were taken while tracing, and a + # later guard can still merge symbols: with torch 2.10 + transformers 4.46 LLaMA, position_ids gets its own + # batch symbol that rotary_emb later unifies with input_ids', so layer 0's input_layernorm was recorded as + # (s72, s70, 64) while its node carries (s44, s70, 64). The stale string kept the no-op alias fold from + # matching and the norm was indexed as a 2-tuple (#292). + _out_specs = {fqn: [_tensor_output_sig(s[3]) if s is not None else None for s in spec] + for fqn, spec in _out_specs.items()} + # Reconstruct module-level graph from nn_module_stack metadata # (bypasses unflatten() which is broken for LLaMA — PyTorch #147348) self.gm = build_module_graph_from_export(exported_program, model, input_names, diff --git a/opt_prime/tests/test_kv_cache_batch.py b/opt_prime/tests/test_kv_cache_batch.py index 8397168..b3a6036 100644 --- a/opt_prime/tests/test_kv_cache_batch.py +++ b/opt_prime/tests/test_kv_cache_batch.py @@ -42,6 +42,19 @@ def test_hftracer_kv_cache_trace(tiny): torch.testing.assert_close(out["logits"], ref) +def test_export_kv_cache_trace(tiny): + # torch 2.10 + transformers 4.46: a stale hook signature left layer 0's input_layernorm indexed as a 2-tuple + # (#292); the exported graph must compute the model's logits. + ir = IR(tiny, SimpleNamespace(model2type={"hf": 50, "sy": 51, "vt": 52})) + ir.retrieve_IR(tiny, use_kv_cache=True, dynamo_capture=True) + ids = torch.randint(0, 128, (2, 6), generator=torch.Generator().manual_seed(0)) + pos = torch.arange(6).expand(2, -1) + with torch.no_grad(): + out = ir.gm(input_ids=ids, position_ids=pos)[0] # (logits, *present key/values) + ref = tiny(ids).logits + torch.testing.assert_close(out, ref) + + @needs_gpu @pytest.mark.parametrize("dynamo_capture", [True, pytest.param(False, marks=needs_hf_fx)]) @pytest.mark.parametrize("batch", [1, 2])