Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions opt_prime/opt_prime/IR.py
Original file line number Diff line number Diff line change
Expand Up @@ -1786,6 +1786,11 @@ def _retrieve_IR_trace(self, model: nn.Module, use_kv_cache: bool = False):
for p in sig.parameters.values()
if p.name not in input_names
}
# With use_cache left at its default, the model returns its DynamicCache, which HFTracer cannot emit as a
# graph output (ProxyableDynamicCache, #191). KV-cache inference does not use that output anyway
# (CachedSDPA modules are injected into the graph afterwards), so trace without it.
if use_kv_cache and 'use_cache' in concrete_args:
concrete_args['use_cache'] = False

tracer = hf_fx.HFTracer()

Expand Down
29 changes: 25 additions & 4 deletions opt_prime/tests/test_kv_cache_batch.py
Original file line number Diff line number Diff line change
@@ -1,17 +1,21 @@
#
# Copyright (c) 2026-present, ETRI, All rights reserved.
#
# opt_prime.inference: greedy generation with the KV cache (torch.export path) matches HF at batch size 1 and 2.
# opt_prime.inference: greedy generation with the KV cache matches HF at batch size 1 and 2, on the torch.export path
# and on the HFTracer path (transformers 4.x only, #191).
import copy
import os
import sys
from types import SimpleNamespace

import pytest
import torch

sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from opt_prime.IR import _HF_FX_AVAILABLE, IR # noqa: E402

pytestmark = pytest.mark.skipif(not torch.cuda.is_available(), reason="Optimus_Inference needs a GPU")
needs_gpu = pytest.mark.skipif(not torch.cuda.is_available(), reason="Optimus_Inference needs a GPU")
needs_hf_fx = pytest.mark.skipif(not _HF_FX_AVAILABLE, reason="HFTracer needs transformers < 5")


@pytest.fixture(scope="module")
Expand All @@ -24,14 +28,31 @@ def tiny():
return AutoModelForCausalLM.from_config(cfg).eval()


@needs_hf_fx
def test_hftracer_kv_cache_trace(tiny):
# The model's DynamicCache output used to stop HFTracer (ProxyableDynamicCache); the graph must trace and
# compute the model's logits.
ir = IR(tiny, SimpleNamespace(model2type={"hf": 50, "sy": 51, "vt": 52}))
ir.retrieve_IR(tiny, use_kv_cache=True, dynamo_capture=False)
ids = torch.randint(0, 128, (2, 6), generator=torch.Generator().manual_seed(0))
pos = torch.arange(6).expand(2, -1)
with torch.no_grad():
out = ir.gm(input_ids=ids, position_ids=pos)
ref = tiny(ids).logits
torch.testing.assert_close(out["logits"], ref)


@needs_gpu
@pytest.mark.parametrize("dynamo_capture", [True, pytest.param(False, marks=needs_hf_fx)])
@pytest.mark.parametrize("batch", [1, 2])
def test_kv_cache_generation_matches_hf(tiny, batch):
def test_kv_cache_generation_matches_hf(tiny, batch, dynamo_capture):
from opt_prime.inference import Optimus_Inference
ids = torch.randint(0, 128, (batch, 6), generator=torch.Generator().manual_seed(batch))
with torch.no_grad():
ref = copy.deepcopy(tiny).cuda().generate(ids.cuda(), max_new_tokens=5, do_sample=False,
pad_token_id=0).cpu()
eng = Optimus_Inference(copy.deepcopy(tiny), use_kv_cache=True, dynamo_capture=True, dtype=torch.float32)
eng = Optimus_Inference(copy.deepcopy(tiny), use_kv_cache=True, dynamo_capture=dynamo_capture,
dtype=torch.float32)
eng.eval()
out = eng.generate(ids, max_new_tokens=5, do_sample=False, verbose=False).cpu()
assert torch.equal(out, ref)
Loading