Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion opt_prime/opt_prime/accel/pd/paged.py
Original file line number Diff line number Diff line change
Expand Up @@ -415,7 +415,7 @@ def drain_events(self):
return out

def physical(self, slots, positions):
s = torch.as_tensor(slots, device=self.device)
s = torch.as_tensor(slots, dtype=torch.long, device=self.device)
p = torch.as_tensor(positions, device=self.device)
return self.table[s, p // self.page_size].long() * self.page_size + p % self.page_size

Expand Down
21 changes: 21 additions & 0 deletions opt_prime/tests/test_pd_paged_empty_span.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
"""An empty KV span (nothing left to import, e.g. a fully cached prompt) must index as a no-op (#47)."""
import torch
from opt_prime.accel.pd.paged import PagedKV


def pool():
cache = PagedKV(2, 1, 1, 2, torch.float32, 'cpu', pages=4, page_size=4, max_context=16)
assert cache.alloc(0, 8) == 0
return cache


def test_empty_span_index_is_long_and_empty():
idx = pool().span_index(0, 5, 5)
assert idx.dtype == torch.long and idx.numel() == 0


def test_empty_span_write_is_noop():
cache = pool()
empty = torch.ones(2, 1, 0, 2)
cache.write_span(0, 5, empty, empty)
assert all(not x.any() for x in cache.gk + cache.gv)
Loading