Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions opt_prime/opt_prime/IR.py
Original file line number Diff line number Diff line change
Expand Up @@ -2027,6 +2027,14 @@ def _retrieve_IR_export(self, model: nn.Module, use_kv_cache: bool = False, for_
if export_kwargs:
_strip_runtime_asserts(exported_program)

# Re-render the hook signatures now that export is done. The shape strings were taken while tracing, and a
# later guard can still merge symbols: with torch 2.10 + transformers 4.46 LLaMA, position_ids gets its own
# batch symbol that rotary_emb later unifies with input_ids', so layer 0's input_layernorm was recorded as
# (s72, s70, 64) while its node carries (s44, s70, 64). The stale string kept the no-op alias fold from
# matching and the norm was indexed as a 2-tuple (#292).
_out_specs = {fqn: [_tensor_output_sig(s[3]) if s is not None else None for s in spec]
for fqn, spec in _out_specs.items()}

# Reconstruct module-level graph from nn_module_stack metadata
# (bypasses unflatten() which is broken for LLaMA — PyTorch #147348)
self.gm = build_module_graph_from_export(exported_program, model, input_names,
Expand Down
13 changes: 13 additions & 0 deletions opt_prime/tests/test_kv_cache_batch.py
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,19 @@ def test_hftracer_kv_cache_trace(tiny):
torch.testing.assert_close(out["logits"], ref)


def test_export_kv_cache_trace(tiny):
# torch 2.10 + transformers 4.46: a stale hook signature left layer 0's input_layernorm indexed as a 2-tuple
# (#292); the exported graph must compute the model's logits.
ir = IR(tiny, SimpleNamespace(model2type={"hf": 50, "sy": 51, "vt": 52}))
ir.retrieve_IR(tiny, use_kv_cache=True, dynamo_capture=True)
ids = torch.randint(0, 128, (2, 6), generator=torch.Generator().manual_seed(0))
pos = torch.arange(6).expand(2, -1)
with torch.no_grad():
out = ir.gm(input_ids=ids, position_ids=pos)[0] # (logits, *present key/values)
ref = tiny(ids).logits
torch.testing.assert_close(out, ref)


@needs_gpu
@pytest.mark.parametrize("dynamo_capture", [True, pytest.param(False, marks=needs_hf_fx)])
@pytest.mark.parametrize("batch", [1, 2])
Expand Down
Loading