From 60eb5400e75b176817ab353b53878e18b1c96b14 Mon Sep 17 00:00:00 2001 From: huidesheng <1832140001@qq.com> Date: Wed, 18 Nov 2026 03:47:12 +0800 Subject: [PATCH 1/3] feat(scheduler): support mixed prefill and decode batching Add opt-in decode-first mixed batching with chunked prefill support and scheduling regression tests. --- examples/test_infer.py | 6 + python/infinilm/base_config.py | 6 + python/infinilm/config/engine_config.py | 3 + python/infinilm/llm/llm.py | 14 ++ python/infinilm/llm/scheduler.py | 184 ++++++++++-------- .../processors/basic_llm_processor.py | 4 +- python/infinilm/server/inference_server.py | 4 + test/runtime/test_chunked_prefill_sched.py | 99 +++++++++- 8 files changed, 237 insertions(+), 83 deletions(-) diff --git a/examples/test_infer.py b/examples/test_infer.py index 70714244b..d6785a95e 100644 --- a/examples/test_infer.py +++ b/examples/test_infer.py @@ -39,6 +39,8 @@ def test( weight_load_mode="async", enable_prefix_caching=True, pre_transpose=False, + chunked_prefill_size=0, + enable_mixed_batching=False, ): if video_path is not None: from infinilm.processors.videonsa_processor import decode_video_frames @@ -71,6 +73,8 @@ def test( top_k=top_k, top_p=top_p, enable_graph=enable_graph, + chunked_prefill_size=chunked_prefill_size, + enable_mixed_batching=enable_mixed_batching, attn_backend=attn_backend, use_mla=use_mla, skip_load=skip_load, @@ -167,6 +171,8 @@ def test( ep=ep, enable_paged_attn=enable_paged_attn, enable_graph=enable_graph, + chunked_prefill_size=cfg.chunked_prefill_size, + enable_mixed_batching=cfg.enable_mixed_batching, num_blocks=cfg.num_blocks, block_size=cfg.block_size, top_k=cfg.top_k, diff --git a/python/infinilm/base_config.py b/python/infinilm/base_config.py index 06c6d5e4a..e6672398d 100644 --- a/python/infinilm/base_config.py +++ b/python/infinilm/base_config.py @@ -79,6 +79,7 @@ def __init__(self): self.pre_transpose = self.args.pre_transpose self.num_blocks = self.args.num_blocks self.chunked_prefill_size = self.args.chunked_prefill_size + self.enable_mixed_batching = self.args.enable_mixed_batching self.block_size = self.args.block_size self.max_cache_len = self.args.max_cache_len self.kv_cache_dtype = self.args.kv_cache_dtype @@ -250,6 +251,11 @@ def _add_common_args(self): self.parser.add_argument( "--num-blocks", type=int, default=512, help="number of KV cache blocks" ) + self.parser.add_argument( + "--enable-mixed-batching", + action="store_true", + help="batch running decode tokens with whole or chunked prefills in paged text inference", + ) self.parser.add_argument( "--chunked-prefill-size", type=int, diff --git a/python/infinilm/config/engine_config.py b/python/infinilm/config/engine_config.py index e49ecef5c..20234d5f5 100644 --- a/python/infinilm/config/engine_config.py +++ b/python/infinilm/config/engine_config.py @@ -70,6 +70,7 @@ class EngineConfig: enable_prefix_caching: bool = True static_cache_capacity: int = 1 chunked_prefill_size: int = 0 + enable_mixed_batching: bool = False def __post_init__(self) -> None: if ( @@ -97,6 +98,8 @@ def __post_init__(self) -> None: and self.cache_type != "paged" ): raise ValueError("kv_transfer_config requires cache_type='paged'") + if self.enable_mixed_batching and self.cache_type != "paged": + raise ValueError("mixed batching requires paged attention") if type(self.chunked_prefill_size) is not int or ( self.chunked_prefill_size < 0 ): diff --git a/python/infinilm/llm/llm.py b/python/infinilm/llm/llm.py index 0d5ffdd04..6f3e9ea64 100644 --- a/python/infinilm/llm/llm.py +++ b/python/infinilm/llm/llm.py @@ -173,6 +173,15 @@ def __init__(self, config: EngineConfig): ) assert 1024 <= max_num_batched_tokens <= max_position_embeddings + if config.enable_mixed_batching and connector is None and ( + type(self.processor).build_model_inputs + is not BasicLLMProcessor.build_model_inputs + or self.model_runner.speculative_runner is not None + or config.use_mla + ): + raise ValueError( + "mixed batching requires the standard paged text processor and attention path" + ) self.scheduler = Scheduler( max_batch_size=config.max_batch_size, num_blocks=config.num_blocks, @@ -188,6 +197,7 @@ def __init__(self, config: EngineConfig): else None ), chunked_prefill_size=config.chunked_prefill_size, + enable_mixed_batching=config.enable_mixed_batching, ) if model_engine.model_type == "videonsa": self.model_runner.bind_nsa_scheduler(self.scheduler) @@ -556,6 +566,7 @@ def __init__( enable_prefix_caching: bool = True, static_cache_capacity: int = 1, chunked_prefill_size: int = 0, + enable_mixed_batching: bool = False, ): """Initialize LLM. @@ -596,6 +607,7 @@ def __init__( max_batch_size=max_batch_size, static_cache_capacity=static_cache_capacity, chunked_prefill_size=chunked_prefill_size, + enable_mixed_batching=enable_mixed_batching, max_tokens=max_tokens, num_blocks=num_blocks, block_size=block_size, @@ -786,6 +798,7 @@ def __init__( enable_prefix_caching: bool = True, static_cache_capacity: int = 1, chunked_prefill_size: int = 0, + enable_mixed_batching: bool = False, ): """Initialize AsyncLLMEngine. @@ -830,6 +843,7 @@ def __init__( max_batch_size=max_batch_size, static_cache_capacity=static_cache_capacity, chunked_prefill_size=chunked_prefill_size, + enable_mixed_batching=enable_mixed_batching, max_tokens=max_tokens, num_blocks=num_blocks, block_size=block_size, diff --git a/python/infinilm/llm/scheduler.py b/python/infinilm/llm/scheduler.py index efdc81381..b1767adbf 100644 --- a/python/infinilm/llm/scheduler.py +++ b/python/infinilm/llm/scheduler.py @@ -160,10 +160,15 @@ def __init__( is_prefill: bool = False, speculative_cache_ops: Optional[SpeculativeCacheOps] = None, nsa_owner=None, + num_decode_requests: int | None = None, ): self.scheduled_requests = scheduled_requests self.num_requests = len(scheduled_requests) - self.is_prefill = is_prefill + self.is_prefill = is_prefill # True if the batch contains prefill work. + # Decode rows lead a mixed batch. Pure batches use the same boundary. + if num_decode_requests is None: + num_decode_requests = 0 if is_prefill else self.num_requests + self.num_decode_requests = num_decode_requests self.speculative_cache_ops = speculative_cache_ops self.kv_connector_metadata = None self.nsa_owner = nsa_owner @@ -190,6 +195,7 @@ def __init__( enable_prefix_caching: bool = True, nsa_owner_pool_epoch: int | None = None, chunked_prefill_size: int = 0, + enable_mixed_batching: bool = False, ): if nsa_owner_pool_epoch is not None and ( connector is not None or has_mamba_cache @@ -257,11 +263,8 @@ def __init__( f"chunked_prefill_size must be >= block_size " f"({block_size}), got {chunked_prefill_size}" ) - # Keep the chunk block-aligned even when the token budget - # is not: a resumed chunk reads KV back at the block-table - # boundary (allocate_slots requires num_computed_tokens to - # cover every fully cached block), so a misaligned chunk - # would make num_computed_tokens precede that boundary. + # Intermediate chunks must end at a page boundary so the next + # step can extend the cached block table. chunked_prefill_size = min( chunked_prefill_size // block_size * block_size, max_num_batched_tokens // block_size * block_size, @@ -273,7 +276,17 @@ def __init__( "chunked prefill" ) self.chunked_prefill_size = chunked_prefill_size - # When chunked prefill is enabled, alternate between + # On this branch a KV connector owns the P/D handoff and disables + # chunking above. A shared launch configuration can still request mixing. + self.enable_mixed_batching = enable_mixed_batching and connector is None + if self.enable_mixed_batching and (has_mamba_cache or nsa_owner_pool_epoch is not None): + raise ValueError( + "mixed batching requires standard local paged attention" + ) + # A whole prompt deferred by decode tokens gets the first row next time. + # Like other dequeued requests it is already owned by _active_requests. + self._whole_prefill_retry: InferenceRequest | None = None + # The separate-batch policy alternates between # prefill and decode batches so a long chunked prompt # cannot starve decode work already in the running queue. self._last_batch_was_prefill = False @@ -432,9 +445,20 @@ def schedule(self) -> Optional[SchedulerOutput]: deferred_requests = [] scheduled_requests = [] seen_request_objects = set() - is_prefill = False - current_num_batched_tokens = 0 - current_prefill_extra_blocks = 0 + retry_whole = self._whole_prefill_retry is not None + if self.enable_mixed_batching and not retry_whole: + self._schedule_running( + scheduled_requests, + seen_request_objects, + min(self.max_batch_size, self.max_num_batched_tokens), + ) + num_decode_requests = len(scheduled_requests) + current_num_batched_tokens = num_decode_requests + # These decode rows have left the running queue. Admission must still + # reserve their remaining KV pages when admitting a new prefill. + current_prefill_extra_blocks = sum( + self._get_prefill_extra_blocks(req) for req in scheduled_requests + ) # With chunked prefill enabled, a long prompt is split # over several prefill batches. Always returning the @@ -443,12 +467,15 @@ def schedule(self) -> Optional[SchedulerOutput]: # batch, run decode first if any decode work exists. skip_waiting = ( self.chunked_prefill_size is not None + and not self.enable_mixed_batching and self._last_batch_was_prefill and self.running_queue.sync_q.qsize() > 0 ) # Process Waiting queue (prefill phase) waiting_to_inspect = 0 if skip_waiting else self.waiting_queue.sync_q.qsize() + if retry_whole: + waiting_to_inspect += 1 while ( waiting_to_inspect > 0 and len(scheduled_requests) < self.max_batch_size @@ -456,7 +483,11 @@ def schedule(self) -> Optional[SchedulerOutput]: ): waiting_to_inspect -= 1 try: - req = self.waiting_queue.sync_q.get_nowait() + if self._whole_prefill_retry is not None: + req = self._whole_prefill_retry + self._whole_prefill_retry = None + else: + req = self.waiting_queue.sync_q.get_nowait() self._dequeued_request = req except queue.Empty: break @@ -514,14 +545,12 @@ def schedule(self) -> Optional[SchedulerOutput]: # stay referenced by the request, so a resumed chunk # attends to the KV of earlier chunks. num_scheduled_tokens = None - if ( - self.chunked_prefill_size is not None - and not load_kv_async - and not req.has_multimodal_inputs - and num_new_tokens > self.chunked_prefill_size - ): - num_scheduled_tokens = self.chunked_prefill_size - + if not load_kv_async and not req.has_multimodal_inputs: + size = self._prefill_step_size( + num_new_tokens, current_num_batched_tokens + ) + if size < num_new_tokens: + num_scheduled_tokens = size # Early token budget check: skip can_accept_request and allocate_slots # for requests that would exceed the per-schedule token budget. if not load_kv_async: @@ -537,7 +566,10 @@ def schedule(self) -> Optional[SchedulerOutput]: ): if num_local_computed_tokens > 0: self.cache_manager.free_blocks(cached_block_table) - deferred_requests.append(req) + if num_decode_requests and self.chunked_prefill_size is None: + self._whole_prefill_retry = req + else: + deferred_requests.append(req) break if not self.can_accept_request( @@ -619,10 +651,8 @@ def schedule(self) -> Optional[SchedulerOutput]: load_kv_async = False computed_start = req.num_computed_tokens remaining = req.get_prompt_length() - computed_start - num_scheduled = ( - min(remaining, self.chunked_prefill_size) - if self.chunked_prefill_size is not None - else remaining + num_scheduled = self._prefill_step_size( + remaining, current_num_batched_tokens ) if self._exceeds_token_budget( current_num_batched_tokens, @@ -694,63 +724,15 @@ def schedule(self) -> Optional[SchedulerOutput]: for req in deferred_requests: self.waiting_queue.sync_q.put(req) - # Return prefill batch if any waiting requests were scheduled - if scheduled_requests: - is_prefill = True - self._last_batch_was_prefill = True - if self._nsa_owner_clock is not None: - self.cache_manager.validate_nsa_request_writes( - scheduled_requests, decode=not is_prefill - ) - scheduler_output = SchedulerOutput( - scheduled_requests=scheduled_requests, - is_prefill=is_prefill, - speculative_cache_ops=self.speculative_cache_ops, - nsa_owner=( - capture_owner_batch( - self.cache_manager, scheduled_requests, is_prefill - ) - if self._nsa_owner_clock is not None - else None - ), + is_prefill = len(scheduled_requests) > num_decode_requests + if not scheduled_requests and (not self.enable_mixed_batching or retry_whole): + self._schedule_running( + scheduled_requests, seen_request_objects, self.max_batch_size ) - if self.connector is not None: - meta = self.connector.build_connector_meta() - scheduler_output.kv_connector_metadata = meta - return self._publish_nsa_handoff(scheduler_output) - - # Process Running queue (decode phase) - running_to_inspect = self.running_queue.sync_q.qsize() - while running_to_inspect > 0 and len(scheduled_requests) < self.max_batch_size: - running_to_inspect -= 1 - try: - req = self.running_queue.sync_q.get_nowait() - self._dequeued_request = req - except queue.Empty: - break - if id(req) in seen_request_objects: - self._completion_failed = True - raise RuntimeError("paged request appeared twice in one schedule") - seen_request_objects.add(id(req)) - self._claim_request(req) - self._dequeued_request = None - # Skip requests that were already finished (e.g., timed out/canceled while running) - if req.is_finished(): - self.complete_requests([req]) - continue - - # Decode phase: allocate slot for newly generated token - req.block_table, new_slot = self.cache_manager.append_slot( - req.block_table, req.get_total_length() - ) - req.slot_mapping = [new_slot] - req.num_blocks = len(req.block_table) - req.num_local_cached_tokens = req.get_total_length() - 1 - scheduled_requests.append(req) # Promote completed remote KV transfers (lower priority than running queue). # Cleanup (is_finished, failed re-queue) runs unconditionally; batch append only if slots remain. - if self.connector is not None and self.remote_kv_requests: + if not is_prefill and self.connector is not None and self.remote_kv_requests: for req_id in list(self.remote_kv_requests.keys()): req = self.remote_kv_requests[req_id] if req.is_finished(): @@ -774,10 +756,8 @@ def schedule(self) -> Optional[SchedulerOutput]: else: break # Defer promotion to next schedule() if batch is full - # Return decode batch if any running requests were scheduled if scheduled_requests: - is_prefill = False - self._last_batch_was_prefill = False + self._last_batch_was_prefill = is_prefill if self._nsa_owner_clock is not None: self.cache_manager.validate_nsa_request_writes( scheduled_requests, decode=not is_prefill @@ -785,6 +765,7 @@ def schedule(self) -> Optional[SchedulerOutput]: scheduler_output = SchedulerOutput( scheduled_requests=scheduled_requests, is_prefill=is_prefill, + num_decode_requests=num_decode_requests if is_prefill else None, speculative_cache_ops=self.speculative_cache_ops, nsa_owner=( capture_owner_batch( @@ -812,6 +793,48 @@ def schedule(self) -> Optional[SchedulerOutput]: self._last_batch_was_prefill = False return None + def _prefill_step_size(self, remaining: int, used: int) -> int: + if self.chunked_prefill_size is None: + return remaining + size = min(remaining, self.chunked_prefill_size) + budget = self.max_num_batched_tokens - used + if self.enable_mixed_batching and size > budget: + # Only full pages can be resumed. A final slice can be shorter. + aligned = budget // self.block_size * self.block_size + if aligned > 0: + return aligned + return size + + def _schedule_running(self, scheduled_requests, seen_request_objects, limit): + # Process Running queue (decode phase) + running_to_inspect = self.running_queue.sync_q.qsize() + while running_to_inspect > 0 and len(scheduled_requests) < limit: + running_to_inspect -= 1 + try: + req = self.running_queue.sync_q.get_nowait() + self._dequeued_request = req + except queue.Empty: + break + if id(req) in seen_request_objects: + self._completion_failed = True + raise RuntimeError("paged request appeared twice in one schedule") + seen_request_objects.add(id(req)) + self._claim_request(req) + self._dequeued_request = None + # Skip requests that were already finished (e.g., timed out/canceled while running) + if req.is_finished(): + self.complete_requests([req]) + continue + + # Decode phase: allocate slot for newly generated token + req.block_table, new_slot = self.cache_manager.append_slot( + req.block_table, req.get_total_length() + ) + req.slot_mapping = [new_slot] + req.num_blocks = len(req.block_table) + req.num_local_cached_tokens = req.get_total_length() - 1 + scheduled_requests.append(req) + def update_waiting_for_remote_kv(self, request: InferenceRequest): self.remote_kv_requests.pop(request.request_id, None) self.pending_kv_decode_blocks -= ( @@ -956,6 +979,7 @@ def shutdown_drain(self, finalize_request) -> None: self._claim_request(req) self._dequeued_request = None + self._whole_prefill_retry = None active = list(self._active_requests.values()) for req in active: finalize_request(req) diff --git a/python/infinilm/processors/basic_llm_processor.py b/python/infinilm/processors/basic_llm_processor.py index e3a6e7712..1c9488cf6 100644 --- a/python/infinilm/processors/basic_llm_processor.py +++ b/python/infinilm/processors/basic_llm_processor.py @@ -307,9 +307,9 @@ def _build_model_input_from_batch_scheduler_output( ) current_offset = 0 - for req in scheduler_output.scheduled_requests: + for row, req in enumerate(scheduler_output.scheduled_requests): num_cached = req.num_local_cached_tokens - if scheduler_output.is_prefill: + if row >= scheduler_output.num_decode_requests: # Prefill phase req_tokens = req.get_input_tokens() # A chunked prefill schedules only part of the diff --git a/python/infinilm/server/inference_server.py b/python/infinilm/server/inference_server.py index 0669f02dd..dff66f1e7 100644 --- a/python/infinilm/server/inference_server.py +++ b/python/infinilm/server/inference_server.py @@ -148,6 +148,7 @@ def __init__( pre_transpose: bool = False, tool_call_parser: str = "none", chunked_prefill_size: int = 0, + enable_mixed_batching: bool = False, ): """Initialize inference server. @@ -213,6 +214,7 @@ def __init__( raise ValueError("Unsupported tool call parser") self.tool_call_parser = tool_call_parser self.chunked_prefill_size = chunked_prefill_size + self.enable_mixed_batching = enable_mixed_batching self.engine: AsyncLLMEngine = None @@ -257,6 +259,7 @@ async def lifespan(app: FastAPI): enable_prefix_caching=self.enable_prefix_caching, pre_transpose=self.pre_transpose, chunked_prefill_size=self.chunked_prefill_size, + enable_mixed_batching=self.enable_mixed_batching, ) self.engine.start() logger.info(f"Engine initialized with model at {self.model_path}") @@ -782,6 +785,7 @@ def main(): pre_transpose=cfg.pre_transpose, tool_call_parser=cfg.tool_call_parser, chunked_prefill_size=cfg.chunked_prefill_size, + enable_mixed_batching=cfg.enable_mixed_batching, ) server.start() diff --git a/test/runtime/test_chunked_prefill_sched.py b/test/runtime/test_chunked_prefill_sched.py index f1a2b5c32..88704f11a 100644 --- a/test/runtime/test_chunked_prefill_sched.py +++ b/test/runtime/test_chunked_prefill_sched.py @@ -91,6 +91,100 @@ def blocks_held(s): class ChunkedPrefillSchedTest(unittest.TestCase): + def test_mixed_decode_and_resumed_prefill_share_budget(self): + from infinilm.processors.basic_llm_processor import BasicLLMProcessor + + processor = BasicLLMProcessor.__new__(BasicLLMProcessor) + s = make_scheduler( + chunked_prefill_size=64, enable_mixed_batching=True, max_num_batched_tokens=64 + ) + short = make_request("short", 8) + long = make_request("long", 111) + s.add_request(short) + s.schedule() + short.append_generated_token_id(314) + s.complete_requests([short]) + s.add_request(long) + for expected in (48, 63): + batch = s.schedule() + self.assertEqual(batch.scheduled_requests, [short, long]) + self.assertEqual(batch.num_decode_requests, 1) + self.assertEqual(len(short.slot_mapping), 1) + self.assertEqual(len(long.slot_mapping), expected) + inputs = processor.build_model_inputs(batch) + start = long.num_local_cached_tokens + end = start + expected + self.assertEqual(inputs["input_offsets"].to_numpy().tolist(), [0, 1, 1 + expected]) + self.assertEqual( + inputs["input_ids"].to_numpy().tolist(), + [[314, *long.prompt_token_ids[start:end]]], + ) + self.assertEqual( + inputs["position_ids"].to_numpy().tolist(), + [short.get_total_length() - 1, *range(start, end)], + ) + self.assertEqual( + inputs["slot_mapping"].to_numpy().tolist(), + short.slot_mapping + long.slot_mapping, + ) + short.append_generated_token_id(314) + long.num_computed_tokens = long.num_local_cached_tokens + expected + if not long.chunked_prefill_pending: + long.append_generated_token_id(42) + s.complete_requests(batch.scheduled_requests) + batch = s.schedule() + self.assertFalse(batch.is_prefill) + self.assertEqual(batch.num_decode_requests, 2) + for req in batch.scheduled_requests: + req.status = RequestStatus.FINISHED + s.complete_requests(batch.scheduled_requests) + self.assertEqual(blocks_held(s), 0) + + def test_mixed_whole_prompt_fits_or_retries_without_splitting(self): + s = make_scheduler(enable_mixed_batching=True, max_num_batched_tokens=64) + running, fits, overflow = ( + make_request("running", 8), make_request("fits", 40), make_request("overflow", 80) + ) + s.add_request(running) + s.schedule() + running.append_generated_token_id(42) + s.complete_requests([running]) + s.add_request(fits) + s.add_request(overflow) + first = s.schedule() + self.assertEqual(first.scheduled_requests, [running, fits]) + self.assertEqual(first.num_decode_requests, 1) + self.assertEqual(len(fits.slot_mapping), 40) + self.assertFalse(fits.chunked_prefill_pending) + self.assertFalse(overflow.block_table) + for req in first.scheduled_requests: + req.append_generated_token_id(42) + s.complete_requests(first.scheduled_requests) + # Even with running decodes and a newly arrived prompt, the large + # prompt gets a complete forward using the existing single-row exception. + s.add_request(make_request("new", 16)) + second = s.schedule() + self.assertEqual(second.scheduled_requests, [overflow]) + self.assertEqual(second.num_decode_requests, 0) + self.assertEqual(len(overflow.slot_mapping), 80) + self.assertFalse(overflow.chunked_prefill_pending) + self.assertIsNone(overflow.num_scheduled_tokens) + + def test_mixed_admission_reserves_running_decode_pages(self): + s = make_scheduler( + chunked_prefill_size=16, enable_mixed_batching=True, num_blocks=2 + ) + short = make_request("short", 15, max_tokens=16) + s.add_request(short) + s.schedule() + short.append_generated_token_id(42) + s.complete_requests([short]) + waiting = make_request("waiting", 2) + s.add_request(waiting) + batch = s.schedule() + self.assertEqual(batch.scheduled_requests, [short]) + self.assertEqual(waiting.block_table, []) + def test_disabled_by_default_full_prefill(self): s = make_scheduler() req = make_request("r1", 300) @@ -563,8 +657,11 @@ class FakeConnector: pass with self.assertLogs("infinilm.llm.scheduler", level="WARNING"): - s = make_scheduler(connector=FakeConnector(), chunked_prefill_size=64) + s = make_scheduler( + connector=FakeConnector(), chunked_prefill_size=64, enable_mixed_batching=True + ) self.assertIsNone(s.chunked_prefill_size) + self.assertFalse(s.enable_mixed_batching) def test_decode_batch_unlimited_by_token_budget(self): # Decode batches bypass max_num_batched_tokens (only From f4473fb3f32f88d45eda8285ffb0e8802ba59261 Mon Sep 17 00:00:00 2001 From: huidesheng <1832140001@qq.com> Date: Fri, 20 Nov 2026 05:06:01 +0800 Subject: [PATCH 2/3] fix(scheduler): honor mixed batching with KV connectors --- python/infinilm/llm/llm.py | 2 +- python/infinilm/llm/scheduler.py | 64 +++++++++--------- test/runtime/test_chunked_prefill_sched.py | 77 +++++++++++++++++++++- 3 files changed, 107 insertions(+), 36 deletions(-) diff --git a/python/infinilm/llm/llm.py b/python/infinilm/llm/llm.py index 6f3e9ea64..610a2ad0e 100644 --- a/python/infinilm/llm/llm.py +++ b/python/infinilm/llm/llm.py @@ -173,7 +173,7 @@ def __init__(self, config: EngineConfig): ) assert 1024 <= max_num_batched_tokens <= max_position_embeddings - if config.enable_mixed_batching and connector is None and ( + if config.enable_mixed_batching and ( type(self.processor).build_model_inputs is not BasicLLMProcessor.build_model_inputs or self.model_runner.speculative_runner is not None diff --git a/python/infinilm/llm/scheduler.py b/python/infinilm/llm/scheduler.py index b1767adbf..72ab65c99 100644 --- a/python/infinilm/llm/scheduler.py +++ b/python/infinilm/llm/scheduler.py @@ -276,12 +276,10 @@ def __init__( "chunked prefill" ) self.chunked_prefill_size = chunked_prefill_size - # On this branch a KV connector owns the P/D handoff and disables - # chunking above. A shared launch configuration can still request mixing. - self.enable_mixed_batching = enable_mixed_batching and connector is None + self.enable_mixed_batching = enable_mixed_batching if self.enable_mixed_batching and (has_mamba_cache or nsa_owner_pool_epoch is not None): raise ValueError( - "mixed batching requires standard local paged attention" + "mixed batching requires standard paged attention" ) # A whole prompt deferred by decode tokens gets the first row next time. # Like other dequeued requests it is already owned by _active_requests. @@ -695,7 +693,12 @@ def schedule(self) -> Optional[SchedulerOutput]: num_tokens_this_step, len(scheduled_requests), ): - deferred_requests.append(req) + # A failed transfer may retain a trusted local prefix. + # Its remaining whole prefill needs the same retry as a new one. + if num_decode_requests and self.chunked_prefill_size is None: + self._whole_prefill_retry = req + else: + deferred_requests.append(req) break self.commit_computed_tokens(req, req.num_computed_tokens) @@ -726,35 +729,10 @@ def schedule(self) -> Optional[SchedulerOutput]: is_prefill = len(scheduled_requests) > num_decode_requests if not scheduled_requests and (not self.enable_mixed_batching or retry_whole): - self._schedule_running( - scheduled_requests, seen_request_objects, self.max_batch_size - ) - - # Promote completed remote KV transfers (lower priority than running queue). - # Cleanup (is_finished, failed re-queue) runs unconditionally; batch append only if slots remain. - if not is_prefill and self.connector is not None and self.remote_kv_requests: - for req_id in list(self.remote_kv_requests.keys()): - req = self.remote_kv_requests[req_id] - if req.is_finished(): - self.complete_requests([req]) - continue - if req_id in self.failed_receiving_kv_req_ids: - logger.warning( - f"Request {req_id[:8]}... failed receiving KV, re-queuing for prefill." - ) - self.update_waiting_for_remote_kv(req) - req.status = RequestStatus.WAITING - self.waiting_queue.sync_q.put(req) - elif req_id in self.finished_receiving_kv_req_ids: - if len(scheduled_requests) < self.max_batch_size: - logger.info( - f"Request {req_id[:8]}... finished receiving KV, scheduling for decode." - ) - self.update_waiting_for_remote_kv(req) - req.status = RequestStatus.RUNNING - scheduled_requests.append(req) - else: - break # Defer promotion to next schedule() if batch is full + limit = self.max_batch_size + if self.enable_mixed_batching: + limit = min(limit, self.max_num_batched_tokens) + self._schedule_running(scheduled_requests, seen_request_objects, limit) if scheduled_requests: self._last_batch_was_prefill = is_prefill @@ -835,6 +813,24 @@ def _schedule_running(self, scheduled_requests, seen_request_objects, limit): req.num_local_cached_tokens = req.get_total_length() - 1 scheduled_requests.append(req) + # A received prompt already owns the slot for its first decode step. + # Promote it after running decodes, before the prefill budget is computed. + if not self.remote_kv_requests: + return + for req_id, req in list(self.remote_kv_requests.items()): + if req.is_finished(): + self.complete_requests([req]) + elif req_id in self.failed_receiving_kv_req_ids: + logger.warning("Request %s failed receiving KV, re-queuing for prefill.", req_id) + self.update_waiting_for_remote_kv(req) + req.status = RequestStatus.WAITING + self.waiting_queue.sync_q.put(req) + elif req_id in self.finished_receiving_kv_req_ids and len(scheduled_requests) < limit: + logger.info("Request %s finished receiving KV, scheduling for decode.", req_id) + self.update_waiting_for_remote_kv(req) + req.status = RequestStatus.RUNNING + scheduled_requests.append(req) + def update_waiting_for_remote_kv(self, request: InferenceRequest): self.remote_kv_requests.pop(request.request_id, None) self.pending_kv_decode_blocks -= ( diff --git a/test/runtime/test_chunked_prefill_sched.py b/test/runtime/test_chunked_prefill_sched.py index 88704f11a..66c5970e9 100644 --- a/test/runtime/test_chunked_prefill_sched.py +++ b/test/runtime/test_chunked_prefill_sched.py @@ -661,7 +661,82 @@ class FakeConnector: connector=FakeConnector(), chunked_prefill_size=64, enable_mixed_batching=True ) self.assertIsNone(s.chunked_prefill_size) - self.assertFalse(s.enable_mixed_batching) + self.assertTrue(s.enable_mixed_batching) + + def test_mooncake_decode_and_failed_receive_share_batch(self): + from infinilm.config.kv_transfer import KVTransferConfig + from infinilm.kv_connector.mooncake.mooncake_connector_scheduler import ( + MooncakeConnectorScheduler, + ) + from infinilm.llm.model_runner.model_runner import KVConnectorOutput, ModelRunnerOutput + from infinilm.processors.basic_llm_processor import BasicLLMProcessor + + for budget in (5, 6): + with self.subTest(budget=budget): + connector = MooncakeConnectorScheduler( + KVTransferConfig(kv_connector="MooncakeConnector", kv_role="kv_consumer"), + engine_id="D", + ) + s = make_scheduler( + connector=connector, enable_mixed_batching=True, + max_num_batched_tokens=budget, + ) + ready, failed = make_request("ready", 20), make_request("failed", 20) + for req in (ready, failed): + req.kv_transfer_params = { + "do_remote_prefill": True, "remote_engine_id": "P", + "remote_bootstrap_addr": "localhost", "transfer_id": req.request_id, + } + s.add_request(req) + running = make_request("running", 3) + s.add_request(running) + first = s.schedule() + self.assertEqual(first.scheduled_requests, [running]) + self.assertEqual(set(first.kv_connector_metadata.reqs_to_recv["P"]), {"ready", "failed"}) + running.append_generated_token_id(42) + s.complete_requests([running]) + + # Only the second page failed. The consumer can reuse the + # first 16 tokens and recompute the final four locally. + s.update_from_output(ModelRunnerOutput(kv_connector_output=KVConnectorOutput( + finished_recving={"ready"}, failed_recving={"failed"}, + invalid_block_ids={failed.block_table[1]}, + ))) + remote_slot = list(ready.slot_mapping) + batch = s.schedule() + self.assertEqual(batch.num_decode_requests, 2) + self.assertEqual(ready.slot_mapping, remote_slot) + self.assertEqual(len(remote_slot), 1) + self.assertEqual(s.remote_kv_requests, {}) + self.assertEqual(s.pending_kv_decode_blocks, 0) + if budget == 5: + self.assertEqual(batch.scheduled_requests, [running, ready]) + for req in batch.scheduled_requests: + req.append_generated_token_id(43) + s.complete_requests(batch.scheduled_requests) + batch = s.schedule() + self.assertEqual(batch.scheduled_requests, [failed]) + self.assertEqual(batch.num_decode_requests, 0) + else: + self.assertEqual(batch.scheduled_requests, [running, ready, failed]) + processor = BasicLLMProcessor.__new__(BasicLLMProcessor) + inputs = processor.build_model_inputs(batch) + self.assertEqual(inputs["input_offsets"].to_numpy().tolist(), [0, 1, 2, 6]) + self.assertEqual(inputs["input_ids"].to_numpy().tolist(), [ + [42, ready.prompt_token_ids[-1], *failed.prompt_token_ids[16:]] + ]) + self.assertFalse(batch.kv_connector_metadata.reqs_to_recv) + self.assertEqual(failed.num_local_cached_tokens, 16) + for req in batch.scheduled_requests: + req.status = RequestStatus.FINISHED + s.complete_requests(batch.scheduled_requests) + if budget == 5: + final = s.schedule() + self.assertEqual(final.scheduled_requests, [running, ready]) + for req in final.scheduled_requests: + req.status = RequestStatus.FINISHED + s.complete_requests(final.scheduled_requests) + self.assertEqual(blocks_held(s), 0) def test_decode_batch_unlimited_by_token_budget(self): # Decode batches bypass max_num_batched_tokens (only From 81118629d7a8c80abf5840389c803bb05a2cde77 Mon Sep 17 00:00:00 2001 From: huidesheng <1832140001@qq.com> Date: Sat, 21 Nov 2026 05:15:13 +0800 Subject: [PATCH 3/3] fix(scheduler): disable mixed batching with KV connectors and warn when requested --- python/infinilm/llm/llm.py | 2 +- python/infinilm/llm/scheduler.py | 66 ++++++++------- test/runtime/test_chunked_prefill_sched.py | 96 +++------------------- 3 files changed, 48 insertions(+), 116 deletions(-) diff --git a/python/infinilm/llm/llm.py b/python/infinilm/llm/llm.py index 610a2ad0e..6f3e9ea64 100644 --- a/python/infinilm/llm/llm.py +++ b/python/infinilm/llm/llm.py @@ -173,7 +173,7 @@ def __init__(self, config: EngineConfig): ) assert 1024 <= max_num_batched_tokens <= max_position_embeddings - if config.enable_mixed_batching and ( + if config.enable_mixed_batching and connector is None and ( type(self.processor).build_model_inputs is not BasicLLMProcessor.build_model_inputs or self.model_runner.speculative_runner is not None diff --git a/python/infinilm/llm/scheduler.py b/python/infinilm/llm/scheduler.py index 72ab65c99..adcb3fee7 100644 --- a/python/infinilm/llm/scheduler.py +++ b/python/infinilm/llm/scheduler.py @@ -276,10 +276,16 @@ def __init__( "chunked prefill" ) self.chunked_prefill_size = chunked_prefill_size + if enable_mixed_batching and connector is not None: + logger.warning( + "--enable-mixed-batching disabled because a KV connector is configured; " + "using separate prefill and decode batches" + ) + enable_mixed_batching = False self.enable_mixed_batching = enable_mixed_batching if self.enable_mixed_batching and (has_mamba_cache or nsa_owner_pool_epoch is not None): raise ValueError( - "mixed batching requires standard paged attention" + "mixed batching requires standard local paged attention" ) # A whole prompt deferred by decode tokens gets the first row next time. # Like other dequeued requests it is already owned by _active_requests. @@ -693,12 +699,7 @@ def schedule(self) -> Optional[SchedulerOutput]: num_tokens_this_step, len(scheduled_requests), ): - # A failed transfer may retain a trusted local prefix. - # Its remaining whole prefill needs the same retry as a new one. - if num_decode_requests and self.chunked_prefill_size is None: - self._whole_prefill_retry = req - else: - deferred_requests.append(req) + deferred_requests.append(req) break self.commit_computed_tokens(req, req.num_computed_tokens) @@ -729,10 +730,35 @@ def schedule(self) -> Optional[SchedulerOutput]: is_prefill = len(scheduled_requests) > num_decode_requests if not scheduled_requests and (not self.enable_mixed_batching or retry_whole): - limit = self.max_batch_size - if self.enable_mixed_batching: - limit = min(limit, self.max_num_batched_tokens) - self._schedule_running(scheduled_requests, seen_request_objects, limit) + self._schedule_running( + scheduled_requests, seen_request_objects, self.max_batch_size + ) + + # Promote completed remote KV transfers (lower priority than running queue). + # Cleanup (is_finished, failed re-queue) runs unconditionally; batch append only if slots remain. + if not is_prefill and self.connector is not None and self.remote_kv_requests: + for req_id in list(self.remote_kv_requests.keys()): + req = self.remote_kv_requests[req_id] + if req.is_finished(): + self.complete_requests([req]) + continue + if req_id in self.failed_receiving_kv_req_ids: + logger.warning( + f"Request {req_id[:8]}... failed receiving KV, re-queuing for prefill." + ) + self.update_waiting_for_remote_kv(req) + req.status = RequestStatus.WAITING + self.waiting_queue.sync_q.put(req) + elif req_id in self.finished_receiving_kv_req_ids: + if len(scheduled_requests) < self.max_batch_size: + logger.info( + f"Request {req_id[:8]}... finished receiving KV, scheduling for decode." + ) + self.update_waiting_for_remote_kv(req) + req.status = RequestStatus.RUNNING + scheduled_requests.append(req) + else: + break # Defer promotion to next schedule() if batch is full if scheduled_requests: self._last_batch_was_prefill = is_prefill @@ -813,24 +839,6 @@ def _schedule_running(self, scheduled_requests, seen_request_objects, limit): req.num_local_cached_tokens = req.get_total_length() - 1 scheduled_requests.append(req) - # A received prompt already owns the slot for its first decode step. - # Promote it after running decodes, before the prefill budget is computed. - if not self.remote_kv_requests: - return - for req_id, req in list(self.remote_kv_requests.items()): - if req.is_finished(): - self.complete_requests([req]) - elif req_id in self.failed_receiving_kv_req_ids: - logger.warning("Request %s failed receiving KV, re-queuing for prefill.", req_id) - self.update_waiting_for_remote_kv(req) - req.status = RequestStatus.WAITING - self.waiting_queue.sync_q.put(req) - elif req_id in self.finished_receiving_kv_req_ids and len(scheduled_requests) < limit: - logger.info("Request %s finished receiving KV, scheduling for decode.", req_id) - self.update_waiting_for_remote_kv(req) - req.status = RequestStatus.RUNNING - scheduled_requests.append(req) - def update_waiting_for_remote_kv(self, request: InferenceRequest): self.remote_kv_requests.pop(request.request_id, None) self.pending_kv_decode_blocks -= ( diff --git a/test/runtime/test_chunked_prefill_sched.py b/test/runtime/test_chunked_prefill_sched.py index 66c5970e9..4500e2824 100644 --- a/test/runtime/test_chunked_prefill_sched.py +++ b/test/runtime/test_chunked_prefill_sched.py @@ -650,93 +650,17 @@ def test_mamba_cache_chunked_resume(self): self.assertTrue(saw_resumed) def test_connector_disabled_chunked_flag(self): - # chunked_prefill_size is force-disabled with a - # connector (constructor warning), so the - # skip_waiting branch can never fire there. - class FakeConnector: - pass - - with self.assertLogs("infinilm.llm.scheduler", level="WARNING"): - s = make_scheduler( - connector=FakeConnector(), chunked_prefill_size=64, enable_mixed_batching=True - ) + with self.assertLogs("infinilm.llm.scheduler", level="WARNING") as logs: + s = make_scheduler(connector=object(), chunked_prefill_size=64) self.assertIsNone(s.chunked_prefill_size) - self.assertTrue(s.enable_mixed_batching) - - def test_mooncake_decode_and_failed_receive_share_batch(self): - from infinilm.config.kv_transfer import KVTransferConfig - from infinilm.kv_connector.mooncake.mooncake_connector_scheduler import ( - MooncakeConnectorScheduler, - ) - from infinilm.llm.model_runner.model_runner import KVConnectorOutput, ModelRunnerOutput - from infinilm.processors.basic_llm_processor import BasicLLMProcessor - - for budget in (5, 6): - with self.subTest(budget=budget): - connector = MooncakeConnectorScheduler( - KVTransferConfig(kv_connector="MooncakeConnector", kv_role="kv_consumer"), - engine_id="D", - ) - s = make_scheduler( - connector=connector, enable_mixed_batching=True, - max_num_batched_tokens=budget, - ) - ready, failed = make_request("ready", 20), make_request("failed", 20) - for req in (ready, failed): - req.kv_transfer_params = { - "do_remote_prefill": True, "remote_engine_id": "P", - "remote_bootstrap_addr": "localhost", "transfer_id": req.request_id, - } - s.add_request(req) - running = make_request("running", 3) - s.add_request(running) - first = s.schedule() - self.assertEqual(first.scheduled_requests, [running]) - self.assertEqual(set(first.kv_connector_metadata.reqs_to_recv["P"]), {"ready", "failed"}) - running.append_generated_token_id(42) - s.complete_requests([running]) - - # Only the second page failed. The consumer can reuse the - # first 16 tokens and recompute the final four locally. - s.update_from_output(ModelRunnerOutput(kv_connector_output=KVConnectorOutput( - finished_recving={"ready"}, failed_recving={"failed"}, - invalid_block_ids={failed.block_table[1]}, - ))) - remote_slot = list(ready.slot_mapping) - batch = s.schedule() - self.assertEqual(batch.num_decode_requests, 2) - self.assertEqual(ready.slot_mapping, remote_slot) - self.assertEqual(len(remote_slot), 1) - self.assertEqual(s.remote_kv_requests, {}) - self.assertEqual(s.pending_kv_decode_blocks, 0) - if budget == 5: - self.assertEqual(batch.scheduled_requests, [running, ready]) - for req in batch.scheduled_requests: - req.append_generated_token_id(43) - s.complete_requests(batch.scheduled_requests) - batch = s.schedule() - self.assertEqual(batch.scheduled_requests, [failed]) - self.assertEqual(batch.num_decode_requests, 0) - else: - self.assertEqual(batch.scheduled_requests, [running, ready, failed]) - processor = BasicLLMProcessor.__new__(BasicLLMProcessor) - inputs = processor.build_model_inputs(batch) - self.assertEqual(inputs["input_offsets"].to_numpy().tolist(), [0, 1, 2, 6]) - self.assertEqual(inputs["input_ids"].to_numpy().tolist(), [ - [42, ready.prompt_token_ids[-1], *failed.prompt_token_ids[16:]] - ]) - self.assertFalse(batch.kv_connector_metadata.reqs_to_recv) - self.assertEqual(failed.num_local_cached_tokens, 16) - for req in batch.scheduled_requests: - req.status = RequestStatus.FINISHED - s.complete_requests(batch.scheduled_requests) - if budget == 5: - final = s.schedule() - self.assertEqual(final.scheduled_requests, [running, ready]) - for req in final.scheduled_requests: - req.status = RequestStatus.FINISHED - s.complete_requests(final.scheduled_requests) - self.assertEqual(blocks_held(s), 0) + self.assertIn("chunked_prefill_size disabled", logs.records[0].getMessage()) + + def test_connector_warns_when_mixed_batching_requested(self): + with self.assertLogs("infinilm.llm.scheduler", level="WARNING") as logs: + s = make_scheduler(connector=object(), enable_mixed_batching=True) + self.assertFalse(s.enable_mixed_batching) + self.assertIn("--enable-mixed-batching disabled", logs.records[0].getMessage()) + self.assertIn("KV connector", logs.records[0].getMessage()) def test_decode_batch_unlimited_by_token_budget(self): # Decode batches bypass max_num_batched_tokens (only