Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 24 additions & 6 deletions opt_prime/opt_prime/accel/pd/paged.py
Original file line number Diff line number Diff line change
Expand Up @@ -445,12 +445,30 @@ def span_index(self, slot, start, end):
return self.physical([slot] * (end - start), torch.arange(start, end, device=self.device))

def write_span(self, slot, start, keys, values):
"""keys/values [layers, kv_heads, n, head_dim] for positions start .. start+n of `slot` (a list of those, one
per KV group, for hybrid models), each group in one indexed copy (the PD admission of KV that another engine
prefilled)"""
keys = keys if isinstance(keys, (list, tuple)) else [keys]
values = values if isinstance(values, (list, tuple)) else [values]
idx = self.span_index(slot, start, start + keys[0].shape[2])
"""Import [layers, kv_heads, n, head_dim] KV at `start`: either model-layer order for uniform geometry,
or one tensor per receiver group. Hybrid dense inputs are split by model-layer indices; grouped inputs
also support differing head geometries. Validate all shapes before copying any group."""
if isinstance(keys, torch.Tensor) and isinstance(values, torch.Tensor) and len(self.groups) == 1:
keys, values = [keys], [values]
if isinstance(keys, torch.Tensor) and isinstance(values, torch.Tensor):
layers = max(self.layer_group) + 1
if keys.ndim != 4 or values.shape != keys.shape or keys.shape[0] != layers:
raise ValueError("dense KV span must contain every model layer with matching K/V shapes")
# Subprocess producers export model-layer order; pools group attention geometries.
keys = [keys if g.layers == tuple(range(layers)) else keys[list(g.layers)] for g in self.groups]
values = [values if g.layers == tuple(range(layers)) else values[list(g.layers)] for g in self.groups]
if not isinstance(keys, (list, tuple)) or not isinstance(values, (list, tuple)):
raise ValueError("KV spans must both be dense tensors or grouped sequences")
if len(keys) != len(self.groups) or len(values) != len(self.groups):
raise ValueError("KV span group count must match the receiver")
n = None
for g, k, v in zip(self.groups, keys, values):
if not isinstance(k, torch.Tensor) or not isinstance(v, torch.Tensor) or k.ndim != 4:
raise ValueError("each KV group must contain four-dimensional tensors")
n = k.shape[2] if n is None else n
if k.shape != (len(g.layers), g.kv_heads, n, g.head_dim) or v.shape != k.shape:
raise ValueError("KV span shapes must match receiver groups and share a token count")
idx = self.span_index(slot, start, start + n)
for pk, pv, k, v in zip(self.gk, self.gv, keys, values):
pk[:, :, idx] = k.to(self.device, pk.dtype, non_blocking=True)
pv[:, :, idx] = v.to(self.device, pv.dtype, non_blocking=True)
Expand Down
65 changes: 65 additions & 0 deletions opt_prime/tests/test_pd_dense_hybrid.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,65 @@
"""Dense subprocess KV must preserve model-layer order in grouped GPU pools (#258)."""
import pytest
import torch
from opt_prime.accel.pd.paged import PagedKV
from opt_prime.accel.pd.torch_models import KVGroup


def pool():
groups = (KVGroup('sliding', 4, 1, 2, (0, 2)), KVGroup('full', None, 1, 2, (1,)))
cache = PagedKV(3, 1, 1, 2, torch.float32, 'cpu', pages=4, page_size=4,
max_context=16, groups=groups)
assert cache.alloc(0, 12) == 0
return cache


def test_dense_span_maps_model_layers_at_nonzero_start():
cache = pool()
keys = torch.arange(30).reshape(3, 1, 5, 2).float()
cache.write_span(0, 3, keys, keys + 100)
idx = cache.span_index(0, 3, 8)
for layer in range(3):
k, v = cache.layer_kv(layer)
torch.testing.assert_close(k[:, idx], keys[layer])
torch.testing.assert_close(v[:, idx], keys[layer] + 100)
assert all(not x[:, :, :3].any() for x in cache.gk)


@pytest.mark.parametrize('bad', ['missing_group', 'bad_later_shape'])
def test_invalid_grouped_span_rejected_before_any_write(bad):
cache = pool()
keys = [torch.ones(2, 1, 5, 2), torch.ones(1, 1, 5, 2)]
values = [x + 1 for x in keys]
if bad == 'missing_group':
keys, values = keys[:1], values[:1]
else:
values[1] = torch.ones(1, 1, 4, 2)
with pytest.raises(ValueError):
cache.write_span(0, 0, keys, values)
assert all(not x.any() for x in cache.gk + cache.gv)


def test_dense_gemma_pd_matches_independent_hf_beyond_window():
from dataclasses import replace
from test_pd_hybrid import _gemma3, _greedy, _prompts
from opt_prime.accel.pd import PDRequest
from opt_prime.accel.pd.pd_batching import TorchPrefillEngine, PrefillWorker, PDBatcher
import copy

class DenseEngine(TorchPrefillEngine):
def prefill(self, request, send_from=0):
out = super().prefill(request, send_from)
def dense(groups):
by_layer = {layer: span[i] for g, span in zip(self.kv.groups, groups)
for i, layer in enumerate(g.layers)}
return torch.stack([by_layer[i] for i in range(len(by_layer))])
return replace(out, keys=dense(out.keys), values=dense(out.values))

model = _gemma3()
engine = DenseEngine(copy.deepcopy(model), max_context=128, chunk=16, page_size=8)
batcher = PDBatcher(model, PrefillWorker(engine), slots=2, max_context=128, page_size=8,
prefix_cache=False)
prompts = _prompts(3, 25, 40, seed=258)
futures = [batcher.submit(PDRequest(str(i), p, 20)) for i, p in enumerate(prompts)]
batcher.run_until_idle()
assert [f.result() for f in futures] == [_greedy(model, p, 20) for p in prompts]
Loading