diff --git a/opt_prime/opt_prime/IR.py b/opt_prime/opt_prime/IR.py index da7594c..24c00e6 100644 --- a/opt_prime/opt_prime/IR.py +++ b/opt_prime/opt_prime/IR.py @@ -2027,6 +2027,14 @@ def _retrieve_IR_export(self, model: nn.Module, use_kv_cache: bool = False, for_ if export_kwargs: _strip_runtime_asserts(exported_program) + # Re-render the hook signatures now that export is done. The shape strings were taken while tracing, and a + # later guard can still merge symbols: with torch 2.10 + transformers 4.46 LLaMA, position_ids gets its own + # batch symbol that rotary_emb later unifies with input_ids', so layer 0's input_layernorm was recorded as + # (s72, s70, 64) while its node carries (s44, s70, 64). The stale string kept the no-op alias fold from + # matching and the norm was indexed as a 2-tuple (#292). + _out_specs = {fqn: [_tensor_output_sig(s[3]) if s is not None else None for s in spec] + for fqn, spec in _out_specs.items()} + # Reconstruct module-level graph from nn_module_stack metadata # (bypasses unflatten() which is broken for LLaMA — PyTorch #147348) self.gm = build_module_graph_from_export(exported_program, model, input_names, diff --git a/opt_prime/tests/test_kv_cache_batch.py b/opt_prime/tests/test_kv_cache_batch.py index 8397168..b3a6036 100644 --- a/opt_prime/tests/test_kv_cache_batch.py +++ b/opt_prime/tests/test_kv_cache_batch.py @@ -42,6 +42,19 @@ def test_hftracer_kv_cache_trace(tiny): torch.testing.assert_close(out["logits"], ref) +def test_export_kv_cache_trace(tiny): + # torch 2.10 + transformers 4.46: a stale hook signature left layer 0's input_layernorm indexed as a 2-tuple + # (#292); the exported graph must compute the model's logits. + ir = IR(tiny, SimpleNamespace(model2type={"hf": 50, "sy": 51, "vt": 52})) + ir.retrieve_IR(tiny, use_kv_cache=True, dynamo_capture=True) + ids = torch.randint(0, 128, (2, 6), generator=torch.Generator().manual_seed(0)) + pos = torch.arange(6).expand(2, -1) + with torch.no_grad(): + out = ir.gm(input_ids=ids, position_ids=pos)[0] # (logits, *present key/values) + ref = tiny(ids).logits + torch.testing.assert_close(out, ref) + + @needs_gpu @pytest.mark.parametrize("dynamo_capture", [True, pytest.param(False, marks=needs_hf_fx)]) @pytest.mark.parametrize("batch", [1, 2])