From a3f4d5cec084f50afc55c019a7f79bdb2b2b4643 Mon Sep 17 00:00:00 2001 From: gspark Date: Wed, 4 Nov 2026 15:54:51 +0900 Subject: [PATCH] feat: KV transport on hetero-kv in accel/pd (#298) accel/pd/transport.py is Optimus' glue to the standalone hetero-kv library: TorchCacheTransport (TorchCopyEngine over KVCacheManager/LayeredKVCache storage), SessionTransport (any hetero-kv engine) and p2p_transport (hetero-npu-p2p RNGD backend as a hetero-kv engine, one selection per DMA span). LayerProgress is hetero_kv.LayerProgress. torch_cache.py, layer_progress.py and hetero_p2p.py are removed; cache storage moves to torch_engine.py, KVCacheManager gains storage(). hetero-kv is the new `pd` extra, pinned to d4625f2; CPU CI installs Rust and it. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/cpu-tests.yml | 3 +- opt_prime/docs/CODE_REVIEW_GUIDE_KO.md | 2 +- opt_prime/docs/CONTRIBUTING_PD.md | 11 +- opt_prime/docs/HETEROGENEOUS_PD.md | 38 ++- opt_prime/docs/NATIVE_SHARED_DIAGNOSTIC.md | 2 +- opt_prime/docs/NPU_BACKEND_EXTENSIONS.md | 4 +- opt_prime/examples/pd/benchmark_torch.py | 2 +- opt_prime/examples/pd/validate_checkpoint.py | 2 +- opt_prime/examples/pd/validate_koni.py | 6 +- opt_prime/examples/pd/validate_moe_pd.py | 2 +- opt_prime/opt_prime/accel/pd/hetero_p2p.py | 32 --- .../opt_prime/accel/pd/layer_progress.py | 63 ----- opt_prime/opt_prime/accel/pd/npu/factory.py | 4 +- opt_prime/opt_prime/accel/pd/pd_batching.py | 2 +- .../opt_prime/accel/pd/prefill_server.py | 2 +- opt_prime/opt_prime/accel/pd/torch_cache.py | 80 ------ opt_prime/opt_prime/accel/pd/torch_engine.py | 50 +++- opt_prime/opt_prime/accel/pd/transport.py | 241 ++++++++++++++++++ opt_prime/opt_prime/kv_cache.py | 4 + opt_prime/pyproject.toml | 2 + opt_prime/tests/test_pd.py | 116 +++++++-- opt_prime/tests/test_pd_layer_progress.py | 13 +- opt_prime/tests/test_pd_transport.py | 68 +++++ 23 files changed, 505 insertions(+), 244 deletions(-) delete mode 100644 opt_prime/opt_prime/accel/pd/hetero_p2p.py delete mode 100644 opt_prime/opt_prime/accel/pd/layer_progress.py delete mode 100644 opt_prime/opt_prime/accel/pd/torch_cache.py create mode 100644 opt_prime/opt_prime/accel/pd/transport.py create mode 100644 opt_prime/tests/test_pd_transport.py diff --git a/.github/workflows/cpu-tests.yml b/.github/workflows/cpu-tests.yml index 12e95ea4..947e7183 100644 --- a/.github/workflows/cpu-tests.yml +++ b/.github/workflows/cpu-tests.yml @@ -20,11 +20,12 @@ jobs: with: python-version: "3.10" cache: pip + - uses: dtolnay/rust-toolchain@stable # builds hetero-kv (pd extra); pip caches the wheel by commit - name: Install run: | python -m pip install --upgrade pip python -m pip install torch==2.10.0 --index-url https://download.pytorch.org/whl/cpu - python -m pip install -e "opt_prime[test]" "transformers==5.1.0" triton + python -m pip install -e "opt_prime[test,pd]" "transformers==5.1.0" triton - name: opt_prime tests working-directory: opt_prime run: python -m pytest -q tests diff --git a/opt_prime/docs/CODE_REVIEW_GUIDE_KO.md b/opt_prime/docs/CODE_REVIEW_GUIDE_KO.md index c8bf42bd..4c856582 100644 --- a/opt_prime/docs/CODE_REVIEW_GUIDE_KO.md +++ b/opt_prime/docs/CODE_REVIEW_GUIDE_KO.md @@ -13,7 +13,7 @@ | 3 | `examples/pd/validate_native_no_primer.py` | 실제 ATOM 검증. 첫 요청 전 prefill 금지, 이후 참조 방식과 전체 vocabulary logits 비교 | | 4 | `examples/pd/native_rngd_prefix_worker.py` | 별도 Furiosa SDK 프로세스에서 요청마다 실제 RNGD prefill. 현재는 NPZ 파일을 이용한 호스트 전달 검증 도구 | | 5 | `opt_prime/pd/contracts.py`, `runtime.py` | 모델 revision·물리 KV 규격, 수명, prefill→reserve→transfer→decode→finish 계약 | -| 6 | `opt_prime/pd/npu/engines.py`, `hetero_p2p.py` | 기존 커스텀 NPU 엔진과 P2P 전송을 공통 런타임에 연결하는 어댑터 | +| 6 | `opt_prime/pd/npu/engines.py`, `transport.py` | 기존 커스텀 NPU 엔진과 P2P 전송을 공통 런타임에 연결하는 어댑터 | | 7 | `opt_prime/pd/llmd.py` | llm-d stage 서비스, 요청/취소 상태, 소유 스레드 실행과 HTTP 경계 | ## 제조사 제공 기능과 우리 코드 diff --git a/opt_prime/docs/CONTRIBUTING_PD.md b/opt_prime/docs/CONTRIBUTING_PD.md index 5f9d7c84..1bd4bbd9 100644 --- a/opt_prime/docs/CONTRIBUTING_PD.md +++ b/opt_prime/docs/CONTRIBUTING_PD.md @@ -14,16 +14,15 @@ dependencies, see [NPU backend extensions](NPU_BACKEND_EXTENSIONS.md). | `contracts.py` | Request, continuation, physical KV identity, engine/transport protocols | Import SDKs or infer device addresses | | `runtime.py` | Single-flight execution and poisoned-lane owner retention | Model-specific layout conversion or service routing | | `torch_models.py` | Explicit family support and cache geometry constraints | Load weights or silently accept hybrid/sliding caches | -| `torch_cache.py` | Optimus KVCacheManager storage and explicit tensor-copy transport | Choose a fallback after DMA failure | -| `torch_engine.py` | Model forward calls, continuation handling, greedy decoding | HTTP request handling or vendor driver calls | +| `torch_engine.py` | Model forward calls, continuation handling, greedy decoding; per-layer cache storage | HTTP request handling or vendor driver calls | +| `transport.py` | Optimus glue to hetero-kv: tensor-copy and P2P transports on hetero-kv sessions | Choose a fallback after DMA failure; bypass completion validation | | `koni.py` | Existing RNGD prefix and owned ATOM worker adaptation | Reimplement allocation/export ABI | -| `hetero_p2p.py` | Bridge to checked CopySession | Bypass completion validation | | `llmd.py` (feature branch) | HTTP/control-plane ownership queue | Run DMA inside HTTP worker threads | Earlier `opt_prime.accel.pd.runtime.KVSpec` and `opt_prime.accel.pd.torch_engine.TorchCacheTransport` imports remain compatibility -exports. New code should import contracts from `opt_prime.accel.pd`, and the Torch -transport from `opt_prime.accel.pd.torch_cache`. +exports. New code should import contracts from `opt_prime.accel.pd`, and transports +from `opt_prime.accel.pd.transport`. ## Adding a Torch model @@ -124,5 +123,5 @@ A native checkpoint success does **not** register it with `KoniRngdEngine` or `KoniAtomWorkerEngine`. Qwen requires Q/K/V projection biases, tied embedding handling, checkpoint-specific layer/head dimensions and cache layouts in both compiled graphs. Each new physical layout needs export/import/packing and fence -validation before it can use `HeteroP2PTransport`. Keep vendor-native baseline +validation before it can use `p2p_transport`. Keep vendor-native baseline support and end-to-end heterogeneous PD support as separate evidence columns. diff --git a/opt_prime/docs/HETEROGENEOUS_PD.md b/opt_prime/docs/HETEROGENEOUS_PD.md index a2ce1133..96995c7f 100644 --- a/opt_prime/docs/HETEROGENEOUS_PD.md +++ b/opt_prime/docs/HETEROGENEOUS_PD.md @@ -15,9 +15,10 @@ Caller / future llm-d service adapter \ / Transport | - optional HeteroP2PTransport + accel/pd/transport.py (Optimus glue) | - hetero_p2p.CopySession + checked backend + hetero-kv Session + engine (TorchCopyEngine, + P2PBackendEngine over a checked hetero_p2p backend) | supported driver / device pair ``` @@ -25,8 +26,12 @@ Caller / future llm-d service adapter `from opt_prime import PDRuntime` and `from opt_prime.accel.pd import ...` do not initialize PyTorch, CUDA, or a vendor SDK. Existing `Optimus_Inference` imports and behavior remain unchanged. Run from the repository with `PYTHONPATH=opt_prime`. -The optional P2P bridge imports `hetero_p2p` only when constructed. Install that -separate project in the same environment; no vendor implementation is vendored. +KV transports (`opt_prime.accel.pd.transport`) run on the separate library +[hetero-kv](https://github.com/ai-computing/hetero-kv) (`pd` extra): it owns the +reserve -> transfer -> fence -> admit -> finish lifecycle, poisoning and explicit +routes; Optimus keeps its engine contracts and maps its caches onto hetero-kv +buffers. The RNGD -> ATOM path imports `hetero_p2p` only when a transfer runs. +Install that separate project in the same environment; no vendor implementation is vendored. ## Interface and lifetime contract @@ -64,12 +69,12 @@ object garbage collection is not a recovery procedure. ```python from opt_prime import PDRuntime -from opt_prime.accel.pd.hetero_p2p import HeteroP2PTransport +from opt_prime.accel.pd.transport import p2p_transport # Real endpoint adapters must implement the contracts above. -transport = HeteroP2PTransport( +transport = p2p_transport( checked_backend, build_plan_from_live_leases, mark_atom_cache_visible, - source="rngd:3", destination="atom:2", spec=prefill_engine.spec, + source="/dev/rngd/npu3ch0", destination="atom:2", spec=prefill_engine.spec, ) runtime = PDRuntime(prefill_engine, decode_engine, transport) tokens = runtime.generate(request) @@ -79,8 +84,13 @@ The snippet illustrates dependency injection; use the bundled KONI endpoint adapters and explicit validation runner to bind existing hardware resources. `build_plan_from_live_leases` must produce the existing validated `TransferPlan`: RNGD-owned source allocation, ATOM-owned exported DMA-BUF destinations and bounded -copy spans. The bridge checks the physical format and delegates completion receipt, -owner retention and decode sequencing to the real `CopySession`. Hardware backend +copy spans. `p2p_transport` checks the physical format, maps the plan onto +hetero-kv buffers (one selection per DMA span) and runs it through a hetero-kv +`Session`: `P2PBackendEngine` executes each destination's spans as a validated +`TransferPlan` with the original command grouping and checks the completion receipt +as `CopySession` did; the session fences before `notify`, retains owners and +poisons on uncertainty (fenced = `CopySession` "ready", admitted = "decoding"). Other +device pairs use `SessionTransport` with any hetero-kv engine. Hardware backend construction must retain its driver/SDK/helper ABI checks. The transfer adapter never discovers raw addresses or attempts reverse import. @@ -119,13 +129,17 @@ process; do not kill it as a routine timeout cleanup. ```bash python -m pip install -e ./opt_prime # SDK-free contracts +# KV transports (hetero-kv; builds from source, needs a Rust toolchain): +python -m pip install -e './opt_prime[pd]' # Optional isolated Torch validation environment (not the NPU SDK environment): -python -m pip install -e './opt_prime[torch-pd]' +python -m pip install -e './opt_prime[torch-pd,pd]' ``` The Torch validation extra pins the versions actually tested. Keep vendor SDK Python environments separate; the core package has no mandatory torch dependency. KONI uses the existing RNGD Python environment and a separate ATOM owning process. +An SDK environment that streams layers (`pd.prefill_server`) or runs a transport +also needs hetero-kv installed. The hardware runner accepts `--config` and `--hetero-repo` explicitly rather than embedding local artifact paths in the engine package. @@ -151,14 +165,14 @@ explicit, independently validated adapter rather than a transport side effect. ```bash PYTHONPATH=opt_prime python3 -m unittest discover -s opt_prime/tests -v -# Also exercise the real optional CopySession with a fake CPU backend: +# Also exercise the hetero_p2p plan path with a fake CPU backend: PYTHONPATH=opt_prime:/path/to/hetero-npu-p2p/src \ python3 -m unittest discover -s opt_prime/tests -v ``` Tests include 24 actual GPT-2/Llama parity cases and cover request reuse, output-token accounting, incompatible revisions, unsupported routes, transfer/decode failures, owner retention, lazy imports and -P2P completion-before-notify ordering. The small randomly initialized CPU models test execution parity, not task quality. +P2P completion-before-notify ordering on the hetero-kv session. The small randomly initialized CPU models test execution parity, not task quality. Contract fixtures and CPU tests do not measure NPU performance, MFU or power. Measured results and exact scope: [validation record](validation/RESULTS.md). diff --git a/opt_prime/docs/NATIVE_SHARED_DIAGNOSTIC.md b/opt_prime/docs/NATIVE_SHARED_DIAGNOSTIC.md index 3bea563d..adee00ad 100644 --- a/opt_prime/docs/NATIVE_SHARED_DIAGNOSTIC.md +++ b/opt_prime/docs/NATIVE_SHARED_DIAGNOSTIC.md @@ -25,7 +25,7 @@ Initial0.5B r1/r2 lacked dedicated warmup and are preserved, not headline result `hetero-npu-p2p/src/hetero_p2p/atom_experimental.py` exports verified live eager allocations at pinned thunk ABI. `qwen_atom_worker.py` supplies external KV tensor -addresses; `npu/engines.py` builds the plan and `pd/hetero_p2p.py` uses CopySession. +addresses; `npu/engines.py` builds the plan and `pd/transport.py` runs it on a hetero-kv session. RNGD writes ATOM-owned exported buffers directly, then the owner marks KV updated. Installed `rebel/kv_cache.py` native `set_kv_cache_for_layer` accepts **host** data. diff --git a/opt_prime/docs/NPU_BACKEND_EXTENSIONS.md b/opt_prime/docs/NPU_BACKEND_EXTENSIONS.md index bfc8dfda..a67ed25a 100644 --- a/opt_prime/docs/NPU_BACKEND_EXTENSIONS.md +++ b/opt_prime/docs/NPU_BACKEND_EXTENSIONS.md @@ -10,13 +10,13 @@ serving/control plane. Neither branch is the upstream main branch. | Layer | Files | Responsibility | New device/model work | |---|---|---|---| | Common lifecycle | `contracts.py`, `runtime.py` | Requests, continuation boundary, physical KV identity, fenced stages, retained owners on error | Usually none | -| Torch backend | `torch_models.py`, `torch_cache.py`, `torch_engine.py` | Dense HuggingFace model execution and explicit tensor copy | Add family policy and numerical tests; actual GPU validation still required | +| Torch backend | `torch_models.py`, `torch_engine.py`, `transport.py` | Dense HuggingFace model execution and explicit tensor copy | Add family policy and numerical tests; actual GPU validation still required | | Dense NPU model contract | `npu/model.py` | Immutable checkpoint geometry, full-attention policy, QKV bias/tied-weight checks, per-head DMA alignment | Add explicit semantics; reject unsupported architectures | | RNGD graph | `npu/rngd.py` | Model-sized biased attention/MLP prefill; device-resident BF16 KV | Implement supported compiler operations and precision policy | | ATOM graph | `npu/atom.py` | Model-sized resident paged decode | Compile and validate exact input/cache contract | | ATOM runtime | `npu/atom_runtime.py`, `npu/atom_device.py` | Pinned SDK runtime, physical-device binding, cache visibility | Replace version-specific implementation for a new SDK/device | | Endpoint adapters | `npu/engines.py` | Adapt these graphs/worker to the common prefill/decode protocols | Preserve lease ownership and explicit completion fences | -| Transport | `hetero_p2p.py`, external `hetero-npu-p2p` library | Validate physical route/layout, export/import/copy completion | Implement a separately negotiated route; no implicit fallback | +| Transport | `transport.py` on hetero-kv, external `hetero-npu-p2p` backend | Validate physical route/layout, export/import/copy completion | Implement a separately negotiated route; no implicit fallback | | Serving | `llmd.py` on feature branch | Request validation, source preparation, serial owner, HTTP | Independent of model dimensions and vendor graph implementation | | Experiments | `examples/pd/*qwen*.py` | Build artifacts, compare references, finite test runs and diagnostics | Extend measured cases; do not put test traffic into the serving engine | diff --git a/opt_prime/examples/pd/benchmark_torch.py b/opt_prime/examples/pd/benchmark_torch.py index 95f83b3c..6519e70a 100644 --- a/opt_prime/examples/pd/benchmark_torch.py +++ b/opt_prime/examples/pd/benchmark_torch.py @@ -12,7 +12,7 @@ MistralConfig, MistralForCausalLM) from opt_prime.accel.pd import PDRequest, PDRuntime from opt_prime.accel.pd.torch_engine import TorchCausalLMEngine -from opt_prime.accel.pd.torch_cache import TorchCacheTransport +from opt_prime.accel.pd.transport import TorchCacheTransport class Timed: diff --git a/opt_prime/examples/pd/validate_checkpoint.py b/opt_prime/examples/pd/validate_checkpoint.py index b561db41..22c370ac 100644 --- a/opt_prime/examples/pd/validate_checkpoint.py +++ b/opt_prime/examples/pd/validate_checkpoint.py @@ -8,7 +8,7 @@ from transformers import AutoModelForCausalLM, AutoTokenizer from opt_prime.accel.pd import PDRequest, PDRuntime from opt_prime.accel.pd.torch_engine import TorchCausalLMEngine -from opt_prime.accel.pd.torch_cache import TorchCacheTransport +from opt_prime.accel.pd.transport import TorchCacheTransport def main(): diff --git a/opt_prime/examples/pd/validate_koni.py b/opt_prime/examples/pd/validate_koni.py index 4217805d..8b51716c 100644 --- a/opt_prime/examples/pd/validate_koni.py +++ b/opt_prime/examples/pd/validate_koni.py @@ -32,7 +32,7 @@ def main(): from hetero_p2p import RngdPdmaBackend from opt_prime.accel.pd import PDRuntime, PDRequest from opt_prime.accel.pd.koni import KoniRngdEngine, KoniAtomWorkerEngine - from opt_prime.accel.pd.hetero_p2p import HeteroP2PTransport + from opt_prime.accel.pd.transport import p2p_transport device = cfg['atom_devices'][0] if cfg['atom_devices'] != [device] or device not in (2, 3): raise ValueError('one authorized ATOM2/3 required') @@ -137,8 +137,8 @@ def _next_token(self, x, P): print('REFERENCES_READY', flush=True) source = KoniRngdEngine(engine, packer, binding, revision=model_dir.name, max_prefix=capacity) decoder = KoniAtomWorkerEngine(owner, spec=source.spec, device=device, batch_size=batch) - transport = HeteroP2PTransport(backend, decoder.build_plan, lambda lease: None, - source=binding.device_path, destination=f'atom:{device}', spec=source.spec) + transport = p2p_transport(backend, decoder.build_plan, lambda lease: None, + source=binding.device_path, destination=f'atom:{device}', spec=source.spec) runtime = PDRuntime(source, decoder, transport) rows = [] for repeat in range(cfg.get('repeats', 2)): diff --git a/opt_prime/examples/pd/validate_moe_pd.py b/opt_prime/examples/pd/validate_moe_pd.py index 47af1da2..7ca90b18 100644 --- a/opt_prime/examples/pd/validate_moe_pd.py +++ b/opt_prime/examples/pd/validate_moe_pd.py @@ -17,7 +17,7 @@ from transformers import AutoModelForCausalLM, AutoTokenizer from opt_prime.accel.pd import PDRequest, PDRuntime -from opt_prime.accel.pd.torch_cache import TorchCacheTransport +from opt_prime.accel.pd.transport import TorchCacheTransport from opt_prime.accel.pd.torch_engine import TorchCausalLMEngine PROMPTS = ("The capital of France is", diff --git a/opt_prime/opt_prime/accel/pd/hetero_p2p.py b/opt_prime/opt_prime/accel/pd/hetero_p2p.py deleted file mode 100644 index 07601c36..00000000 --- a/opt_prime/opt_prime/accel/pd/hetero_p2p.py +++ /dev/null @@ -1,32 +0,0 @@ -"""Optional bridge to the separately installed hetero-npu-p2p library.""" - - -class HeteroP2PTransport: - """Explicit same-host route; plan_builder must use live owned allocations. - - Backend construction retains existing SDK/driver/helper ABI checks. - notify(destination) makes the completed cache visible to the decoder. - No address discovery, reverse import, or implicit host fallback is added. - """ - def __init__(self, backend, plan_builder, notify, *, source, destination, spec): - from hetero_p2p.core import CopySession - self.session = CopySession(backend) - self.plan_builder, self.notify = plan_builder, notify - self.source, self.destination, self.spec = source, destination, spec - - def supports(self, source, destination, spec): - return (source, destination, spec) == (self.source, self.destination, self.spec) - - def transfer(self, request_id, source, destination): - if source.spec != destination.spec or not self.supports(source.endpoint, destination.endpoint, source.spec): - raise ValueError('unsupported P2P route or physical format') - plan = self.plan_builder(source, destination) - if (plan.source.format != self.spec.representation or - any(d.format != self.spec.representation for d in plan.destinations)): - raise ValueError('DMA plan format differs from negotiated representation') - receipt = self.session.admit(request_id, plan, lambda: self.notify(destination)) - self.session.start_decode(request_id) - return receipt - - def finish(self, request_id): - self.session.finish_decode(request_id, passed=True) diff --git a/opt_prime/opt_prime/accel/pd/layer_progress.py b/opt_prime/opt_prime/accel/pd/layer_progress.py deleted file mode 100644 index a13fd219..00000000 --- a/opt_prime/opt_prime/accel/pd/layer_progress.py +++ /dev/null @@ -1,63 +0,0 @@ -"""Validated, bounded-by-coverage notifications for a reserved KV slab. - -This tracks source writes only. A consumer must finish its own transfers before -publishing the cache or releasing the reservation; source completion is not a fence. -""" -from threading import Lock - - -class LayerProgress: - def __init__(self, layers, start, end): - if any(type(x) is not int for x in (layers,start,end)) or layers<=0 or not 0<=start<=end: - raise ValueError('invalid layer-progress geometry') - self.layers,self.start,self.end=layers,start,end - self._coverage=[[] for _ in range(layers)] - self._pending=[] - self._closed=False - self._lock=Lock() - - def publish(self, layer_start, layer_end, token_start, token_end): - span=(layer_start,layer_end,token_start,token_end) - with self._lock: - if self._closed: - raise ValueError('layer progress is closed') - if (any(type(x) is not int for x in span) - or not 0<=layer_start self.max_seq_len: - raise ValueError('cache capacity exceeded') - self.k[layer][:, :, position:position + n] = keys - self.v[layer][:, :, position:position + n] = values - - def get(self, layer): - return self.k[layer][:, :, :self._len], self.v[layer][:, :, :self._len] - - def set_seq_len(self, n): - self._len = n - - def get_seq_len(self): - return self._len - - -def fence(device): - if torch.device(device).type == 'cuda': - torch.cuda.synchronize(device) - - -class TorchCacheTransport: - """Explicit tensor-copy transport; never selected as a P2P fallback.""" - def __init__(self, *, source, destination, spec): - self.source, self.destination, self.spec = source, destination, spec - - def supports(self, source, destination, spec): - return (source, destination, spec) == (self.source, self.destination, self.spec) - - @torch.inference_mode() - def transfer(self, request_id, source, destination): - if source.spec != destination.spec or not self.supports(source.endpoint, destination.endpoint, source.spec): - raise ValueError('unconfigured tensor-copy route') - src, dst = source.payload.manager, destination.payload.manager - position = src.get_seq_len() - if not 0 < position <= dst.max_seq_len: - raise ValueError('invalid source sequence length') - copied = 0 - for i in range(src.layer_start, src.layer_end): - keys, values = src.get(i) - dst.update(i, keys.to(dst.device), values.to(dst.device), position=0) - copied += keys.numel() * keys.element_size() + values.numel() * values.element_size() - dst.set_seq_len(position) - fence(src.device) - fence(dst.device) - return dict(transport='torch-copy', bytes=copied, cache_position=position, completed=True) - - def finish(self, request_id): - pass - - def recover(self): - """tensor copies leave nothing outstanding once the devices are fenced (see TorchCausalLMEngine.recover)""" - return True diff --git a/opt_prime/opt_prime/accel/pd/torch_engine.py b/opt_prime/opt_prime/accel/pd/torch_engine.py index d2938f49..56cfdd3c 100644 --- a/opt_prime/opt_prime/accel/pd/torch_engine.py +++ b/opt_prime/opt_prime/accel/pd/torch_engine.py @@ -3,14 +3,60 @@ Models are supplied by the caller. No downloads or vendor SDK imports occur here. """ import inspect +from dataclasses import dataclass import torch from transformers.cache_utils import DynamicCache from opt_prime.kv_cache import KVCacheManager from opt_prime.accel.pd.contracts import KVLease, KVSpec, PrefillResult from opt_prime.accel.pd.sampling import select_token from opt_prime.accel.pd.torch_models import describe_model -# Compatibility exports: existing users may import transport from this module. -from opt_prime.accel.pd.torch_cache import LayeredKVCache, TorchCache, TorchCacheTransport, fence +# Compatibility export: existing users import the transport from this module. +from opt_prime.accel.pd.transport import TorchCacheTransport # noqa: F401 + + +@dataclass +class TorchCache: + manager: KVCacheManager + + +class LayeredKVCache: + """Single-request K/V storage with its own shape per layer (models whose layers keep differently shaped K/V, + e.g. Gemma 4). Same interface as the parts of KVCacheManager the PD engine and transport use.""" + + def __init__(self, layer_dims, max_seq_len, dtype, device): + self.layer_start, self.layer_end = 0, len(layer_dims) + self.max_seq_len, self.dtype, self.device = max_seq_len, dtype, torch.device(device) + self.k = [torch.zeros(1, h, max_seq_len, d, dtype=dtype, device=self.device) for h, d in layer_dims] + self.v = [torch.zeros(1, h, max_seq_len, d, dtype=dtype, device=self.device) for h, d in layer_dims] + self._len = 0 + + def allocate(self, batch_size=1): + if batch_size != 1: + raise ValueError('one request per layered cache') + + def update(self, layer, keys, values, position=0): + n = keys.shape[2] + if position + n > self.max_seq_len: + raise ValueError('cache capacity exceeded') + self.k[layer][:, :, position:position + n] = keys + self.v[layer][:, :, position:position + n] = values + + def get(self, layer): + return self.k[layer][:, :, :self._len], self.v[layer][:, :, :self._len] + + def storage(self, layer): + return self.k[layer], self.v[layer] + + def set_seq_len(self, n): + self._len = n + + def get_seq_len(self): + return self._len + + +def fence(device): + if torch.device(device).type == 'cuda': + torch.cuda.synchronize(device) class TorchCausalLMEngine: diff --git a/opt_prime/opt_prime/accel/pd/transport.py b/opt_prime/opt_prime/accel/pd/transport.py new file mode 100644 index 00000000..b310b1ab --- /dev/null +++ b/opt_prime/opt_prime/accel/pd/transport.py @@ -0,0 +1,241 @@ +"""KV transports on hetero-kv sessions (https://github.com/ai-computing/hetero-kv). + +hetero-kv moves the bytes and owns the lifecycle reserve -> transfer -> fence -> admit -> finish: decode is admitted +only after every completion is fenced, an uncertain completion poisons the session until an engine quiesce, and +routes are explicit (no fallback). This module is Optimus' glue: it maps Optimus caches and DMA plans onto hetero-kv +buffers. Importing it does not import torch or a vendor SDK. +""" +from dataclasses import dataclass +import os +import re + +import hetero_kv as hk + + +def _settle(session): + """Bring a session back to idle after the lane failed; True on success.""" + rid = session.request_id + try: + if session.state == "admitted": + try: + session.finish(rid, passed=False) # the decoder failed while using the cache + except hk.PoisonedError: + pass # expected: a failed consumer poisons + elif session.state in ("reserved", "fenced"): + session.cancel(rid) + elif session.state == "transferring": + session.cancel(rid) + session.fence(rid) # never abandon in-flight copies + try: + session.admit(rid) + except hk.CancelledError: + pass + if session.state == "poisoned": + session.recover() + except hk.HeteroKVError: + return False + return session.state == "idle" + + +@dataclass(frozen=True) +class TransferPlan: + """A plan builder's result: the source buffer, one destination buffer per KV group, and the selections copied + into each group.""" + source: object + destinations: tuple + selections: tuple + + +class SessionTransport: + """Explicit route over any hetero-kv engine between two device locations (e.g. rngd:3 -> atom:2). + + plan_builder(source_lease, destination_lease) -> TransferPlan must use live, owned allocations; + notify(destination_lease) makes the fenced cache visible to the decoder. + """ + def __init__(self, engine, plan_builder, notify, *, source, destination, spec, src_location, dst_location, + representation): + self.representation = hk.canonical_representation(representation) + self.session = hk.Session(hk.Route(engine, src_location, dst_location, self.representation)) + self.plan_builder, self.notify = plan_builder, notify + self.source, self.destination, self.spec = source, destination, spec + + def supports(self, source, destination, spec): + return (source, destination, spec) == (self.source, self.destination, self.spec) + + def transfer(self, request_id, source, destination): + if source.spec != destination.spec or not self.supports(source.endpoint, destination.endpoint, source.spec): + raise ValueError("unsupported P2P route or physical format") + plan = self.plan_builder(source, destination) + if any(b.spec.representation != self.representation for b in (plan.source, *plan.destinations)): + raise ValueError("DMA plan format differs from negotiated representation") + if len(plan.selections) != len(plan.destinations): + raise ValueError("one selection list per destination group required") + s = self.session + s.reserve(request_id, list(plan.destinations)) + try: + for group, selections in enumerate(plan.selections): + if selections: + s.transfer(request_id, plan.source, list(selections), group=group) + s.fence(request_id) + return s.admit(request_id, lambda: self.notify(destination)) + except (hk.InvalidError, hk.UnsupportedError, hk.StateError): + _settle(s) # rejected before the engine saw anything + raise + + def finish(self, request_id): + self.session.finish(request_id, passed=True) + + def recover(self): + return _settle(self.session) + + +_ROW = 4096 # hetero-npu-p2p DMA spans are 4 KiB aligned + + +class P2PBackendEngine: + """hetero-kv engine over a hetero-npu-p2p DMA backend (`backend.transfer(plan) -> receipt`, e.g. RngdPdmaBackend + for RNGD -> ATOM DMA-BUF). Each job runs as one validated hetero_p2p TransferPlan with the original plan's + command grouping; the receipt is checked as CopySession did. No quiesce: a poisoned lane stays poisoned.""" + name = "hetero-p2p" + + def __init__(self, backend, src_location, dst_location, representation): + self.backend = backend + self.capabilities = [(src_location, dst_location, hk.canonical_representation(representation))] + + def submit(self, job): + from hetero_p2p.core import CopySpan, TransferPlan as DmaPlan + original = job.src[0].obj + plan = DmaPlan(original.source, tuple(r.obj for r in job.dst), + tuple(CopySpan(di, so, do, n) for _, so, di, do, n in job.spans), + original.descriptors_per_command).validate() + receipt = self.backend.transfer(plan) + if (receipt.get("cqe_result") != 0 or receipt.get("completed") != plan.commands + or receipt.get("requested") != plan.commands or receipt.get("bytes_requested") != plan.bytes): + raise RuntimeError("incomplete or inconsistent DMA completion receipt") + return plan.bytes + + def poll(self, ticket): + return True + + def wait(self, ticket): + return ticket + + def quiesce(self): + return False + + +def p2p_plan(dma, src_location, dst_location, representation): + """hetero-kv buffers for a validated hetero_p2p TransferPlan: flat 4 KiB rows over the source allocation and + each destination DMA-BUF, one selection per DMA span, so every job's spans are exactly the plan's.""" + element = hk.Layout(representation, 1, 1, 1, 1, kv="latent").region_bytes + + def flat(location, addr, nbytes, obj): + layout = hk.Layout(representation, 1, nbytes // _ROW, 1, _ROW // element, kv="latent") + return hk.KVBuffer(hk.KVSpec("hetero-p2p", "flat-4k-v1", layout), + [hk.Region.external(location, addr, layout.region_bytes, obj)]) + + source = flat(src_location, dma.source.address, dma.source.size, dma) + destinations = tuple(flat(dst_location, 0, os.fstat(d.fd).st_size, d) for d in dma.destinations) + selections = tuple(tuple(hk.Selection.contiguous(s.source_offset // _ROW, s.destination_offset // _ROW, + s.length // _ROW) + for s in dma.spans if s.destination == i) + for i in range(len(destinations))) + return TransferPlan(source, destinations, selections) + + +def p2p_transport(backend, plan_builder, notify, *, source, destination, spec): + """SessionTransport over a hetero-npu-p2p backend from an RNGD device path (/dev/rngd/npuNchM) to atom:N. + plan_builder returns a hetero_p2p TransferPlan in the negotiated physical format (spec.representation, e.g. + dlfloat16-token-major-v1; its element type is the hetero-kv representation).""" + m = re.fullmatch(r"/dev/rngd/npu(\d+)ch\d+", str(source)) + if m is None or not re.fullmatch(r"atom:\d+", str(destination)): + raise ValueError("explicit RNGD device path and ATOM device required") + locations = (f"rngd:{m.group(1)}", destination) + representation = hk.canonical_representation(spec.representation.split("-")[0]) + + def build(src, dst): + dma = plan_builder(src, dst) + if (dma.source.format != spec.representation + or any(d.format != spec.representation for d in dma.destinations)): + raise ValueError("DMA plan format differs from negotiated representation") + return p2p_plan(dma.validate(), *locations, representation) + + return SessionTransport(P2PBackendEngine(backend, *locations, representation), build, notify, + source=source, destination=destination, spec=spec, src_location=locations[0], + dst_location=locations[1], representation=representation) + + +class TorchCacheTransport: + """Explicit tensor-copy transport, never selected as a P2P fallback. Copies the source prefix [0, seq_len) of + every layer and batch entry with hetero-kv's TorchCopyEngine, fences, then publishes seq_len on the destination + as the admission step.""" + def __init__(self, *, source, destination, spec): + self.source, self.destination, self.spec = source, destination, spec + self._sessions = {} + self._active = {} + + def supports(self, source, destination, spec): + return (source, destination, spec) == (self.source, self.destination, self.spec) + + def _session(self, src_location, dst_location, representation): + from hetero_kv.torch_engine import TorchCopyEngine + key = (src_location, dst_location, representation) + if key not in self._sessions: + self._sessions[key] = hk.Session(hk.Route(TorchCopyEngine(), *key)) + return self._sessions[key] + + def transfer(self, request_id, source, destination): + from hetero_kv.torch_engine import tensor_location + if source.spec != destination.spec or not self.supports(source.endpoint, destination.endpoint, source.spec): + raise ValueError("unconfigured tensor-copy route") + src, dst = source.payload.manager, destination.payload.manager + position = src.get_seq_len() + if not 0 < position <= dst.max_seq_len: + raise ValueError("invalid source sequence length") + src_groups, src_t = _groups(src, source.spec) + dst_groups, dst_t = _groups(dst, destination.spec) + if len(src_groups) != len(dst_groups): + raise ValueError("source and destination layer groups differ") + session = self._session(tensor_location(src_t.device), tensor_location(dst_t.device), + hk.canonical_representation(str(src_t.dtype))) + session.reserve(request_id, dst_groups) + try: + # One transfer per (group, batch entry): disjoint byte ranges of the same tensors, in flight together. + for group, buf in enumerate(src_groups): + for b in range(src_t.shape[0]): + session.transfer(request_id, buf, hk.Selection(src_blocks=(b,), dst_blocks=(b,), tokens=position), + group=group) + receipt = session.fence(request_id) + session.admit(request_id, lambda: dst.set_seq_len(position)) + except (hk.InvalidError, hk.UnsupportedError, hk.StateError): + _settle(session) + raise + self._active[request_id] = session + return dict(transport="torch-copy", bytes=receipt["bytes"], cache_position=position, completed=True) + + def finish(self, request_id): + session = self._active.pop(request_id, None) + if session is not None: + session.finish(request_id, passed=True) + + def recover(self): + """Quiesce (Torch device sync) and settle every session; True when all are idle.""" + ok = all([_settle(s) for s in self._sessions.values()]) + if ok: + self._active.clear() + return ok + + +def _groups(manager, spec): + """KV buffers over a cache's full-capacity [batch, heads, capacity, dim] tensors: one bhsd group, or one per layer + when layer shapes differ (e.g. Gemma 4). Blocks are batch entries; block_size is the capacity.""" + from hetero_kv.torch_engine import tensor_region + pairs = [manager.storage(i) for i in range(manager.layer_start, manager.layer_end)] + chunks = [pairs] if len({tuple(k.shape) for k, _ in pairs}) == 1 else [[p] for p in pairs] + groups = [] + for chunk in chunks: + batch, heads, capacity, dim = chunk[0][0].shape + layout = hk.Layout.preset("bhsd", str(chunk[0][0].dtype), len(chunk), capacity, heads, dim, blocks=batch) + groups.append(hk.KVBuffer(hk.KVSpec(spec.model_id, spec.revision, layout), + [tensor_region(t) for pair in chunk for t in pair])) + return groups, pairs[0][0] diff --git a/opt_prime/opt_prime/kv_cache.py b/opt_prime/opt_prime/kv_cache.py index 7c4c5cb0..7fa5d6b8 100644 --- a/opt_prime/opt_prime/kv_cache.py +++ b/opt_prime/opt_prime/kv_cache.py @@ -193,6 +193,10 @@ def get(self, layer_idx: int) -> Optional[Tuple[torch.Tensor, torch.Tensor]]: value_cache[:, :, :self._seq_len, :], ) + def storage(self, layer_idx: int) -> Tuple[torch.Tensor, torch.Tensor]: + """Full-capacity (key, value) tensors of a layer, [batch, num_heads, max_seq_len, head_dim] (KV transport).""" + return self._cache[layer_idx] + def get_seq_len(self) -> int: """Get current sequence length in cache.""" return self._seq_len diff --git a/opt_prime/pyproject.toml b/opt_prime/pyproject.toml index b725e1f1..359990ef 100644 --- a/opt_prime/pyproject.toml +++ b/opt_prime/pyproject.toml @@ -12,6 +12,8 @@ dependencies = ["torch", "transformers", "numpy", "psutil"] [project.optional-dependencies] torch-pd = ["torch==2.11.0", "transformers==5.8.1"] +# KV transport for accel.pd (built from source: needs a Rust toolchain) +pd = ["hetero-kv @ git+https://github.com/ai-computing/hetero-kv@d4625f28ba24d7c8bbe4ffab95c13f852fe35922#subdirectory=crates/hetero-kv-py"] test = ["pytest", "requests", "pyzmq", "safetensors", "scipy"] [tool.setuptools.packages.find] diff --git a/opt_prime/tests/test_pd.py b/opt_prime/tests/test_pd.py index 7670ba2d..de20ff53 100644 --- a/opt_prime/tests/test_pd.py +++ b/opt_prime/tests/test_pd.py @@ -125,39 +125,109 @@ def test_reentrant_request_is_rejected(self): def test_no_eager_sdk_or_torch_import(self): subprocess.run([sys.executable, '-c', - 'import sys; import opt_prime.accel.pd; import opt_prime.accel.pd.hetero_p2p; ' + 'import sys; import opt_prime.accel.pd; import opt_prime.accel.pd.transport; ' 'assert "torch" not in sys.modules; assert "hetero_p2p" not in sys.modules'], check=True) + def bridge(self, events, fail=False): + # hetero-kv session over a CPU-only fake engine; no DMA. + import ctypes + import hetero_kv as hk + from opt_prime.accel.pd.transport import SessionTransport, TransferPlan + class Engine: + name, capabilities = 'fake-dma', [('rngd', 'atom', 'fp16')] + def submit(self, job): + return job + def poll(self, job): + return False + def wait(self, job): + if fail: + raise RuntimeError('DMA completion unknown') + for si, so, di, do, n in job.spans: + ctypes.memmove(job.dst[di].addr + do, job.src[si].addr + so, n) + events.append('fence') + return job.nbytes + def quiesce(self): + return True + def buffer(location, data): + spec = hk.KVSpec(SPEC.model_id, SPEC.revision, hk.Layout('fp16', 2, 8, 1, 4)) + return hk.KVBuffer(spec, [hk.Region.external(location, ctypes.addressof(data), len(data), data)]) + self.memory = [ctypes.create_string_buffer(bytes(range(256)), 256), ctypes.create_string_buffer(256)] + src, dst = buffer('rngd:3', self.memory[0]), buffer('atom:2', self.memory[1]) + plan = TransferPlan(src, (dst,), ((hk.Selection.prefix(8),),)) + return SessionTransport(Engine(), lambda s, d: plan, lambda d: events.append('notify'), + source='rngd:3', destination='atom:2', spec=SPEC, src_location='rngd:3', dst_location='atom:2', + representation='fp16') + def test_p2p_bridge_fence_and_notify(self): - # Real CopySession contract with a CPU-only fake plan/backend; no DMA. - try: - from hetero_p2p.core import CopySession - except ImportError: - self.skipTest('optional hetero_p2p dependency absent') - from opt_prime.accel.pd.hetero_p2p import HeteroP2PTransport events = [] - class Allocation: - format = SPEC.representation - class Plan: - source = Allocation() - destinations = [Allocation()] - commands, bytes = 2, 8192 - def validate(self): - return self - class Backend: - def transfer(self, plan): - events.append('fence') - return dict(cqe_result=0, completed=2, requested=2, bytes_requested=8192) - bridge = HeteroP2PTransport(Backend(), lambda s, d: Plan(), lambda d: events.append('notify'), - source='rngd:3', destination='atom:2', spec=SPEC) + bridge = self.bridge(events) source = KVLease(SPEC, self, None, 'rngd:3') destination = KVLease(SPEC, self, None, 'atom:2') - bridge.transfer('x', source, destination) + with self.assertRaisesRegex(ValueError, 'unsupported P2P route'): + bridge.transfer('x', KVLease(SPEC, self, None, 'rngd:0'), destination) + receipt = bridge.transfer('x', source, destination) self.assertEqual(events, ['fence', 'notify']) - self.assertEqual(bridge.session.state, 'decoding') + self.assertEqual(receipt['bytes'], 2 * 2 * 8 * 4 * 2) + self.assertEqual(self.memory[1].raw, self.memory[0].raw) + self.assertEqual(bridge.session.state, 'admitted') bridge.finish('x') self.assertEqual(bridge.session.state, 'idle') + def test_p2p_bridge_uncertain_completion_poisons_until_recovered(self): + events = [] + bridge = self.bridge(events, fail=True) + with self.assertRaises(RuntimeError): + bridge.transfer('x', KVLease(SPEC, self, None, 'rngd:3'), KVLease(SPEC, self, None, 'atom:2')) + self.assertEqual((bridge.session.state, events), ('poisoned', [])) + self.assertTrue(bridge.recover()) + self.assertEqual(bridge.session.state, 'idle') + + def test_p2p_backend_runs_the_validated_dma_plan(self): + # hetero-npu-p2p TransferPlan through p2p_transport: same spans, grouping and receipt checks as CopySession. + try: + from hetero_p2p.core import CopySpan, DestinationAllocation, SourceAllocation, TransferPlan + except ImportError: + self.skipTest('optional hetero_p2p dependency absent') + import tempfile + from opt_prime.accel.pd.transport import p2p_transport + fmt = 'dlfloat16-token-major-v1' + spec = KVSpec('test-model', 'immutable-revision', fmt, 'linear-v1', (4,)) + plans, events = [], [] + class Backend: + cqe = 0 + def transfer(self, plan): + plans.append(plan) + return dict(cqe_result=self.cqe, completed=plan.commands, requested=plan.commands, + bytes_requested=plan.bytes) + with tempfile.TemporaryFile() as a, tempfile.TemporaryFile() as b: + a.truncate(8 * 4096) + b.truncate(4 * 4096) + owner = object() + targets = (DestinationAllocation(a.fileno(), owner, fmt), DestinationAllocation(b.fileno(), owner, fmt)) + spans = (CopySpan(0, 0, 4096, 8192), CopySpan(0, 8192, 16384, 4096), + CopySpan(1, 12288, 0, 4096), CopySpan(1, 16384, 8192, 4096)) + dma = TransferPlan(SourceAllocation(4096, 5 * 4096, owner, fmt), targets, spans, 2) + transport = p2p_transport(Backend(), lambda s, d: dma, lambda d: events.append('notify'), + source='/dev/rngd/npu3ch0', destination='atom:2', spec=spec) + receipt = transport.transfer('x', KVLease(spec, self, None, '/dev/rngd/npu3ch0'), + KVLease(spec, self, None, 'atom:2')) + self.assertEqual((receipt['bytes'], events), (dma.bytes, ['notify'])) + self.assertEqual([(p.spans, p.descriptors_per_command) for p in plans], + [((CopySpan(0, 0, 4096, 8192), CopySpan(0, 8192, 16384, 4096)), 2), + ((CopySpan(0, 12288, 0, 4096), CopySpan(0, 16384, 8192, 4096)), 2)]) + self.assertEqual([p.destinations for p in plans], [targets[:1], targets[1:]]) + transport.finish('x') + self.assertEqual(transport.session.state, 'idle') + # A failed completion poisons before notify; the DMA lane has no recovery, as with CopySession. + failing = Backend() + failing.cqe = -5 + transport = p2p_transport(failing, lambda s, d: dma, lambda d: events.append('notify'), + source='/dev/rngd/npu3ch0', destination='atom:2', spec=spec) + with self.assertRaisesRegex(RuntimeError, 'inconsistent DMA completion receipt'): + transport.transfer('y', KVLease(spec, self, None, '/dev/rngd/npu3ch0'), + KVLease(spec, self, None, 'atom:2')) + self.assertEqual((transport.session.state, events), ('poisoned', ['notify'])) + self.assertFalse(transport.recover()) if __name__ == '__main__': unittest.main() diff --git a/opt_prime/tests/test_pd_layer_progress.py b/opt_prime/tests/test_pd_layer_progress.py index 99f5277b..b1663ef2 100644 --- a/opt_prime/tests/test_pd_layer_progress.py +++ b/opt_prime/tests/test_pd_layer_progress.py @@ -1,15 +1,6 @@ -"""Partial KV must cover every layer/position without overlaps before publication.""" -import importlib.util -from pathlib import Path +"""Partial KV must cover every layer/position without overlaps before publication (hetero-kv LayerProgress).""" import pytest - -path=Path(__file__).parents[1]/'opt_prime/accel/pd/layer_progress.py' - -def tracker(*args): - assert path.exists(), 'layer-progress validator is not implemented' - spec=importlib.util.spec_from_file_location('layer_progress',path) - m=importlib.util.module_from_spec(spec);spec.loader.exec_module(m) - return m.LayerProgress(*args) +from hetero_kv import LayerProgress as tracker def test_partial_prefix_and_out_of_order_layers_require_complete_coverage(): diff --git a/opt_prime/tests/test_pd_transport.py b/opt_prime/tests/test_pd_transport.py new file mode 100644 index 00000000..2a2dec22 --- /dev/null +++ b/opt_prime/tests/test_pd_transport.py @@ -0,0 +1,68 @@ +"""TorchCacheTransport on hetero-kv keeps the tensor-copy contract: prefix [0, seq_len) of every layer and batch +entry, receipt bytes, published seq_len, and a reusable lane after rejections and recovery.""" +import pytest +import torch + +from opt_prime.accel.pd import KVLease, KVSpec +from opt_prime.accel.pd.torch_engine import LayeredKVCache, TorchCache, TorchCacheTransport +from opt_prime.kv_cache import KVCacheManager + +SPEC = KVSpec("tiny", "seed17", "torch.float32", "torch-bhsd-full-attention-v1:test", (1,)) + + +def kv_manager(layers, batch, heads, dim, dtype): + def make(capacity): + m = KVCacheManager(layers, batch, capacity, heads, dim, dtype=dtype, device="cpu") + m.allocate(batch) + return m + return make + + +def fill(manager, n): + gen = torch.Generator().manual_seed(n) + for i in range(manager.layer_start, manager.layer_end): + for t in manager.storage(i): + t[:, :, :n] = torch.randn(t[:, :, :n].shape, generator=gen).to(t.dtype) + manager.set_seq_len(n) + + +def leases(src, dst, endpoint="destination"): + return KVLease(SPEC, src, TorchCache(src), "source"), KVLease(SPEC, dst, TorchCache(dst), endpoint) + + +@pytest.mark.parametrize("make", [ + kv_manager(3, 1, 2, 8, torch.float32), + kv_manager(2, 2, 2, 4, torch.bfloat16), # batch > 1 + lambda capacity: LayeredKVCache([(2, 8), (1, 16), (2, 8)], capacity, torch.float32, "cpu"), +]) +def test_copies_prefix_and_publishes_seq_len(make): + src, dst = make(16), make(16) + fill(src, 7) + transport = TorchCacheTransport(source="source", destination="destination", spec=SPEC) + expected = sum(t.numel() * t.element_size() for i in range(src.layer_end) for t in src.get(i)) + receipt = transport.transfer("r", *leases(src, dst)) + assert receipt == dict(transport="torch-copy", bytes=expected, cache_position=7, completed=True) + assert dst.get_seq_len() == 7 + for i in range(src.layer_end): + for a, b in zip(src.get(i), dst.get(i)): + torch.testing.assert_close(b, a, rtol=0, atol=0) + transport.finish("r") + fill(src, 3) + assert transport.transfer("r2", *leases(src, dst))["cache_position"] == 3 + transport.finish("r2") + assert transport.recover() is True + + +def test_rejections_and_recovery_leave_lane_usable(): + make = kv_manager(1, 1, 1, 4, torch.float32) + src, dst = make(8), make(8) + transport = TorchCacheTransport(source="source", destination="destination", spec=SPEC) + with pytest.raises(ValueError, match="invalid source sequence length"): + transport.transfer("r", *leases(src, dst)) + with pytest.raises(ValueError, match="unconfigured tensor-copy route"): + transport.transfer("r", *leases(src, dst, "elsewhere")) + fill(src, 8) + transport.transfer("r", *leases(src, dst)) + assert transport.recover() is True # decoder failed: PDRuntime.recover() settles the lane + transport.transfer("r3", *leases(src, dst)) + transport.finish("r3")