diff --git a/examples/test_infer.py b/examples/test_infer.py index 70714244b..d6785a95e 100644 --- a/examples/test_infer.py +++ b/examples/test_infer.py @@ -39,6 +39,8 @@ def test( weight_load_mode="async", enable_prefix_caching=True, pre_transpose=False, + chunked_prefill_size=0, + enable_mixed_batching=False, ): if video_path is not None: from infinilm.processors.videonsa_processor import decode_video_frames @@ -71,6 +73,8 @@ def test( top_k=top_k, top_p=top_p, enable_graph=enable_graph, + chunked_prefill_size=chunked_prefill_size, + enable_mixed_batching=enable_mixed_batching, attn_backend=attn_backend, use_mla=use_mla, skip_load=skip_load, @@ -167,6 +171,8 @@ def test( ep=ep, enable_paged_attn=enable_paged_attn, enable_graph=enable_graph, + chunked_prefill_size=cfg.chunked_prefill_size, + enable_mixed_batching=cfg.enable_mixed_batching, num_blocks=cfg.num_blocks, block_size=cfg.block_size, top_k=cfg.top_k, diff --git a/python/infinilm/base_config.py b/python/infinilm/base_config.py index 06c6d5e4a..e6672398d 100644 --- a/python/infinilm/base_config.py +++ b/python/infinilm/base_config.py @@ -79,6 +79,7 @@ def __init__(self): self.pre_transpose = self.args.pre_transpose self.num_blocks = self.args.num_blocks self.chunked_prefill_size = self.args.chunked_prefill_size + self.enable_mixed_batching = self.args.enable_mixed_batching self.block_size = self.args.block_size self.max_cache_len = self.args.max_cache_len self.kv_cache_dtype = self.args.kv_cache_dtype @@ -250,6 +251,11 @@ def _add_common_args(self): self.parser.add_argument( "--num-blocks", type=int, default=512, help="number of KV cache blocks" ) + self.parser.add_argument( + "--enable-mixed-batching", + action="store_true", + help="batch running decode tokens with whole or chunked prefills in paged text inference", + ) self.parser.add_argument( "--chunked-prefill-size", type=int, diff --git a/python/infinilm/config/engine_config.py b/python/infinilm/config/engine_config.py index e49ecef5c..20234d5f5 100644 --- a/python/infinilm/config/engine_config.py +++ b/python/infinilm/config/engine_config.py @@ -70,6 +70,7 @@ class EngineConfig: enable_prefix_caching: bool = True static_cache_capacity: int = 1 chunked_prefill_size: int = 0 + enable_mixed_batching: bool = False def __post_init__(self) -> None: if ( @@ -97,6 +98,8 @@ def __post_init__(self) -> None: and self.cache_type != "paged" ): raise ValueError("kv_transfer_config requires cache_type='paged'") + if self.enable_mixed_batching and self.cache_type != "paged": + raise ValueError("mixed batching requires paged attention") if type(self.chunked_prefill_size) is not int or ( self.chunked_prefill_size < 0 ): diff --git a/python/infinilm/llm/llm.py b/python/infinilm/llm/llm.py index 0d5ffdd04..6f3e9ea64 100644 --- a/python/infinilm/llm/llm.py +++ b/python/infinilm/llm/llm.py @@ -173,6 +173,15 @@ def __init__(self, config: EngineConfig): ) assert 1024 <= max_num_batched_tokens <= max_position_embeddings + if config.enable_mixed_batching and connector is None and ( + type(self.processor).build_model_inputs + is not BasicLLMProcessor.build_model_inputs + or self.model_runner.speculative_runner is not None + or config.use_mla + ): + raise ValueError( + "mixed batching requires the standard paged text processor and attention path" + ) self.scheduler = Scheduler( max_batch_size=config.max_batch_size, num_blocks=config.num_blocks, @@ -188,6 +197,7 @@ def __init__(self, config: EngineConfig): else None ), chunked_prefill_size=config.chunked_prefill_size, + enable_mixed_batching=config.enable_mixed_batching, ) if model_engine.model_type == "videonsa": self.model_runner.bind_nsa_scheduler(self.scheduler) @@ -556,6 +566,7 @@ def __init__( enable_prefix_caching: bool = True, static_cache_capacity: int = 1, chunked_prefill_size: int = 0, + enable_mixed_batching: bool = False, ): """Initialize LLM. @@ -596,6 +607,7 @@ def __init__( max_batch_size=max_batch_size, static_cache_capacity=static_cache_capacity, chunked_prefill_size=chunked_prefill_size, + enable_mixed_batching=enable_mixed_batching, max_tokens=max_tokens, num_blocks=num_blocks, block_size=block_size, @@ -786,6 +798,7 @@ def __init__( enable_prefix_caching: bool = True, static_cache_capacity: int = 1, chunked_prefill_size: int = 0, + enable_mixed_batching: bool = False, ): """Initialize AsyncLLMEngine. @@ -830,6 +843,7 @@ def __init__( max_batch_size=max_batch_size, static_cache_capacity=static_cache_capacity, chunked_prefill_size=chunked_prefill_size, + enable_mixed_batching=enable_mixed_batching, max_tokens=max_tokens, num_blocks=num_blocks, block_size=block_size, diff --git a/python/infinilm/llm/scheduler.py b/python/infinilm/llm/scheduler.py index efdc81381..adcb3fee7 100644 --- a/python/infinilm/llm/scheduler.py +++ b/python/infinilm/llm/scheduler.py @@ -160,10 +160,15 @@ def __init__( is_prefill: bool = False, speculative_cache_ops: Optional[SpeculativeCacheOps] = None, nsa_owner=None, + num_decode_requests: int | None = None, ): self.scheduled_requests = scheduled_requests self.num_requests = len(scheduled_requests) - self.is_prefill = is_prefill + self.is_prefill = is_prefill # True if the batch contains prefill work. + # Decode rows lead a mixed batch. Pure batches use the same boundary. + if num_decode_requests is None: + num_decode_requests = 0 if is_prefill else self.num_requests + self.num_decode_requests = num_decode_requests self.speculative_cache_ops = speculative_cache_ops self.kv_connector_metadata = None self.nsa_owner = nsa_owner @@ -190,6 +195,7 @@ def __init__( enable_prefix_caching: bool = True, nsa_owner_pool_epoch: int | None = None, chunked_prefill_size: int = 0, + enable_mixed_batching: bool = False, ): if nsa_owner_pool_epoch is not None and ( connector is not None or has_mamba_cache @@ -257,11 +263,8 @@ def __init__( f"chunked_prefill_size must be >= block_size " f"({block_size}), got {chunked_prefill_size}" ) - # Keep the chunk block-aligned even when the token budget - # is not: a resumed chunk reads KV back at the block-table - # boundary (allocate_slots requires num_computed_tokens to - # cover every fully cached block), so a misaligned chunk - # would make num_computed_tokens precede that boundary. + # Intermediate chunks must end at a page boundary so the next + # step can extend the cached block table. chunked_prefill_size = min( chunked_prefill_size // block_size * block_size, max_num_batched_tokens // block_size * block_size, @@ -273,7 +276,21 @@ def __init__( "chunked prefill" ) self.chunked_prefill_size = chunked_prefill_size - # When chunked prefill is enabled, alternate between + if enable_mixed_batching and connector is not None: + logger.warning( + "--enable-mixed-batching disabled because a KV connector is configured; " + "using separate prefill and decode batches" + ) + enable_mixed_batching = False + self.enable_mixed_batching = enable_mixed_batching + if self.enable_mixed_batching and (has_mamba_cache or nsa_owner_pool_epoch is not None): + raise ValueError( + "mixed batching requires standard local paged attention" + ) + # A whole prompt deferred by decode tokens gets the first row next time. + # Like other dequeued requests it is already owned by _active_requests. + self._whole_prefill_retry: InferenceRequest | None = None + # The separate-batch policy alternates between # prefill and decode batches so a long chunked prompt # cannot starve decode work already in the running queue. self._last_batch_was_prefill = False @@ -432,9 +449,20 @@ def schedule(self) -> Optional[SchedulerOutput]: deferred_requests = [] scheduled_requests = [] seen_request_objects = set() - is_prefill = False - current_num_batched_tokens = 0 - current_prefill_extra_blocks = 0 + retry_whole = self._whole_prefill_retry is not None + if self.enable_mixed_batching and not retry_whole: + self._schedule_running( + scheduled_requests, + seen_request_objects, + min(self.max_batch_size, self.max_num_batched_tokens), + ) + num_decode_requests = len(scheduled_requests) + current_num_batched_tokens = num_decode_requests + # These decode rows have left the running queue. Admission must still + # reserve their remaining KV pages when admitting a new prefill. + current_prefill_extra_blocks = sum( + self._get_prefill_extra_blocks(req) for req in scheduled_requests + ) # With chunked prefill enabled, a long prompt is split # over several prefill batches. Always returning the @@ -443,12 +471,15 @@ def schedule(self) -> Optional[SchedulerOutput]: # batch, run decode first if any decode work exists. skip_waiting = ( self.chunked_prefill_size is not None + and not self.enable_mixed_batching and self._last_batch_was_prefill and self.running_queue.sync_q.qsize() > 0 ) # Process Waiting queue (prefill phase) waiting_to_inspect = 0 if skip_waiting else self.waiting_queue.sync_q.qsize() + if retry_whole: + waiting_to_inspect += 1 while ( waiting_to_inspect > 0 and len(scheduled_requests) < self.max_batch_size @@ -456,7 +487,11 @@ def schedule(self) -> Optional[SchedulerOutput]: ): waiting_to_inspect -= 1 try: - req = self.waiting_queue.sync_q.get_nowait() + if self._whole_prefill_retry is not None: + req = self._whole_prefill_retry + self._whole_prefill_retry = None + else: + req = self.waiting_queue.sync_q.get_nowait() self._dequeued_request = req except queue.Empty: break @@ -514,14 +549,12 @@ def schedule(self) -> Optional[SchedulerOutput]: # stay referenced by the request, so a resumed chunk # attends to the KV of earlier chunks. num_scheduled_tokens = None - if ( - self.chunked_prefill_size is not None - and not load_kv_async - and not req.has_multimodal_inputs - and num_new_tokens > self.chunked_prefill_size - ): - num_scheduled_tokens = self.chunked_prefill_size - + if not load_kv_async and not req.has_multimodal_inputs: + size = self._prefill_step_size( + num_new_tokens, current_num_batched_tokens + ) + if size < num_new_tokens: + num_scheduled_tokens = size # Early token budget check: skip can_accept_request and allocate_slots # for requests that would exceed the per-schedule token budget. if not load_kv_async: @@ -537,7 +570,10 @@ def schedule(self) -> Optional[SchedulerOutput]: ): if num_local_computed_tokens > 0: self.cache_manager.free_blocks(cached_block_table) - deferred_requests.append(req) + if num_decode_requests and self.chunked_prefill_size is None: + self._whole_prefill_retry = req + else: + deferred_requests.append(req) break if not self.can_accept_request( @@ -619,10 +655,8 @@ def schedule(self) -> Optional[SchedulerOutput]: load_kv_async = False computed_start = req.num_computed_tokens remaining = req.get_prompt_length() - computed_start - num_scheduled = ( - min(remaining, self.chunked_prefill_size) - if self.chunked_prefill_size is not None - else remaining + num_scheduled = self._prefill_step_size( + remaining, current_num_batched_tokens ) if self._exceeds_token_budget( current_num_batched_tokens, @@ -694,63 +728,15 @@ def schedule(self) -> Optional[SchedulerOutput]: for req in deferred_requests: self.waiting_queue.sync_q.put(req) - # Return prefill batch if any waiting requests were scheduled - if scheduled_requests: - is_prefill = True - self._last_batch_was_prefill = True - if self._nsa_owner_clock is not None: - self.cache_manager.validate_nsa_request_writes( - scheduled_requests, decode=not is_prefill - ) - scheduler_output = SchedulerOutput( - scheduled_requests=scheduled_requests, - is_prefill=is_prefill, - speculative_cache_ops=self.speculative_cache_ops, - nsa_owner=( - capture_owner_batch( - self.cache_manager, scheduled_requests, is_prefill - ) - if self._nsa_owner_clock is not None - else None - ), + is_prefill = len(scheduled_requests) > num_decode_requests + if not scheduled_requests and (not self.enable_mixed_batching or retry_whole): + self._schedule_running( + scheduled_requests, seen_request_objects, self.max_batch_size ) - if self.connector is not None: - meta = self.connector.build_connector_meta() - scheduler_output.kv_connector_metadata = meta - return self._publish_nsa_handoff(scheduler_output) - - # Process Running queue (decode phase) - running_to_inspect = self.running_queue.sync_q.qsize() - while running_to_inspect > 0 and len(scheduled_requests) < self.max_batch_size: - running_to_inspect -= 1 - try: - req = self.running_queue.sync_q.get_nowait() - self._dequeued_request = req - except queue.Empty: - break - if id(req) in seen_request_objects: - self._completion_failed = True - raise RuntimeError("paged request appeared twice in one schedule") - seen_request_objects.add(id(req)) - self._claim_request(req) - self._dequeued_request = None - # Skip requests that were already finished (e.g., timed out/canceled while running) - if req.is_finished(): - self.complete_requests([req]) - continue - - # Decode phase: allocate slot for newly generated token - req.block_table, new_slot = self.cache_manager.append_slot( - req.block_table, req.get_total_length() - ) - req.slot_mapping = [new_slot] - req.num_blocks = len(req.block_table) - req.num_local_cached_tokens = req.get_total_length() - 1 - scheduled_requests.append(req) # Promote completed remote KV transfers (lower priority than running queue). # Cleanup (is_finished, failed re-queue) runs unconditionally; batch append only if slots remain. - if self.connector is not None and self.remote_kv_requests: + if not is_prefill and self.connector is not None and self.remote_kv_requests: for req_id in list(self.remote_kv_requests.keys()): req = self.remote_kv_requests[req_id] if req.is_finished(): @@ -774,10 +760,8 @@ def schedule(self) -> Optional[SchedulerOutput]: else: break # Defer promotion to next schedule() if batch is full - # Return decode batch if any running requests were scheduled if scheduled_requests: - is_prefill = False - self._last_batch_was_prefill = False + self._last_batch_was_prefill = is_prefill if self._nsa_owner_clock is not None: self.cache_manager.validate_nsa_request_writes( scheduled_requests, decode=not is_prefill @@ -785,6 +769,7 @@ def schedule(self) -> Optional[SchedulerOutput]: scheduler_output = SchedulerOutput( scheduled_requests=scheduled_requests, is_prefill=is_prefill, + num_decode_requests=num_decode_requests if is_prefill else None, speculative_cache_ops=self.speculative_cache_ops, nsa_owner=( capture_owner_batch( @@ -812,6 +797,48 @@ def schedule(self) -> Optional[SchedulerOutput]: self._last_batch_was_prefill = False return None + def _prefill_step_size(self, remaining: int, used: int) -> int: + if self.chunked_prefill_size is None: + return remaining + size = min(remaining, self.chunked_prefill_size) + budget = self.max_num_batched_tokens - used + if self.enable_mixed_batching and size > budget: + # Only full pages can be resumed. A final slice can be shorter. + aligned = budget // self.block_size * self.block_size + if aligned > 0: + return aligned + return size + + def _schedule_running(self, scheduled_requests, seen_request_objects, limit): + # Process Running queue (decode phase) + running_to_inspect = self.running_queue.sync_q.qsize() + while running_to_inspect > 0 and len(scheduled_requests) < limit: + running_to_inspect -= 1 + try: + req = self.running_queue.sync_q.get_nowait() + self._dequeued_request = req + except queue.Empty: + break + if id(req) in seen_request_objects: + self._completion_failed = True + raise RuntimeError("paged request appeared twice in one schedule") + seen_request_objects.add(id(req)) + self._claim_request(req) + self._dequeued_request = None + # Skip requests that were already finished (e.g., timed out/canceled while running) + if req.is_finished(): + self.complete_requests([req]) + continue + + # Decode phase: allocate slot for newly generated token + req.block_table, new_slot = self.cache_manager.append_slot( + req.block_table, req.get_total_length() + ) + req.slot_mapping = [new_slot] + req.num_blocks = len(req.block_table) + req.num_local_cached_tokens = req.get_total_length() - 1 + scheduled_requests.append(req) + def update_waiting_for_remote_kv(self, request: InferenceRequest): self.remote_kv_requests.pop(request.request_id, None) self.pending_kv_decode_blocks -= ( @@ -956,6 +983,7 @@ def shutdown_drain(self, finalize_request) -> None: self._claim_request(req) self._dequeued_request = None + self._whole_prefill_retry = None active = list(self._active_requests.values()) for req in active: finalize_request(req) diff --git a/python/infinilm/processors/basic_llm_processor.py b/python/infinilm/processors/basic_llm_processor.py index e3a6e7712..1c9488cf6 100644 --- a/python/infinilm/processors/basic_llm_processor.py +++ b/python/infinilm/processors/basic_llm_processor.py @@ -307,9 +307,9 @@ def _build_model_input_from_batch_scheduler_output( ) current_offset = 0 - for req in scheduler_output.scheduled_requests: + for row, req in enumerate(scheduler_output.scheduled_requests): num_cached = req.num_local_cached_tokens - if scheduler_output.is_prefill: + if row >= scheduler_output.num_decode_requests: # Prefill phase req_tokens = req.get_input_tokens() # A chunked prefill schedules only part of the diff --git a/python/infinilm/server/inference_server.py b/python/infinilm/server/inference_server.py index 0669f02dd..dff66f1e7 100644 --- a/python/infinilm/server/inference_server.py +++ b/python/infinilm/server/inference_server.py @@ -148,6 +148,7 @@ def __init__( pre_transpose: bool = False, tool_call_parser: str = "none", chunked_prefill_size: int = 0, + enable_mixed_batching: bool = False, ): """Initialize inference server. @@ -213,6 +214,7 @@ def __init__( raise ValueError("Unsupported tool call parser") self.tool_call_parser = tool_call_parser self.chunked_prefill_size = chunked_prefill_size + self.enable_mixed_batching = enable_mixed_batching self.engine: AsyncLLMEngine = None @@ -257,6 +259,7 @@ async def lifespan(app: FastAPI): enable_prefix_caching=self.enable_prefix_caching, pre_transpose=self.pre_transpose, chunked_prefill_size=self.chunked_prefill_size, + enable_mixed_batching=self.enable_mixed_batching, ) self.engine.start() logger.info(f"Engine initialized with model at {self.model_path}") @@ -782,6 +785,7 @@ def main(): pre_transpose=cfg.pre_transpose, tool_call_parser=cfg.tool_call_parser, chunked_prefill_size=cfg.chunked_prefill_size, + enable_mixed_batching=cfg.enable_mixed_batching, ) server.start() diff --git a/test/runtime/test_chunked_prefill_sched.py b/test/runtime/test_chunked_prefill_sched.py index f1a2b5c32..4500e2824 100644 --- a/test/runtime/test_chunked_prefill_sched.py +++ b/test/runtime/test_chunked_prefill_sched.py @@ -91,6 +91,100 @@ def blocks_held(s): class ChunkedPrefillSchedTest(unittest.TestCase): + def test_mixed_decode_and_resumed_prefill_share_budget(self): + from infinilm.processors.basic_llm_processor import BasicLLMProcessor + + processor = BasicLLMProcessor.__new__(BasicLLMProcessor) + s = make_scheduler( + chunked_prefill_size=64, enable_mixed_batching=True, max_num_batched_tokens=64 + ) + short = make_request("short", 8) + long = make_request("long", 111) + s.add_request(short) + s.schedule() + short.append_generated_token_id(314) + s.complete_requests([short]) + s.add_request(long) + for expected in (48, 63): + batch = s.schedule() + self.assertEqual(batch.scheduled_requests, [short, long]) + self.assertEqual(batch.num_decode_requests, 1) + self.assertEqual(len(short.slot_mapping), 1) + self.assertEqual(len(long.slot_mapping), expected) + inputs = processor.build_model_inputs(batch) + start = long.num_local_cached_tokens + end = start + expected + self.assertEqual(inputs["input_offsets"].to_numpy().tolist(), [0, 1, 1 + expected]) + self.assertEqual( + inputs["input_ids"].to_numpy().tolist(), + [[314, *long.prompt_token_ids[start:end]]], + ) + self.assertEqual( + inputs["position_ids"].to_numpy().tolist(), + [short.get_total_length() - 1, *range(start, end)], + ) + self.assertEqual( + inputs["slot_mapping"].to_numpy().tolist(), + short.slot_mapping + long.slot_mapping, + ) + short.append_generated_token_id(314) + long.num_computed_tokens = long.num_local_cached_tokens + expected + if not long.chunked_prefill_pending: + long.append_generated_token_id(42) + s.complete_requests(batch.scheduled_requests) + batch = s.schedule() + self.assertFalse(batch.is_prefill) + self.assertEqual(batch.num_decode_requests, 2) + for req in batch.scheduled_requests: + req.status = RequestStatus.FINISHED + s.complete_requests(batch.scheduled_requests) + self.assertEqual(blocks_held(s), 0) + + def test_mixed_whole_prompt_fits_or_retries_without_splitting(self): + s = make_scheduler(enable_mixed_batching=True, max_num_batched_tokens=64) + running, fits, overflow = ( + make_request("running", 8), make_request("fits", 40), make_request("overflow", 80) + ) + s.add_request(running) + s.schedule() + running.append_generated_token_id(42) + s.complete_requests([running]) + s.add_request(fits) + s.add_request(overflow) + first = s.schedule() + self.assertEqual(first.scheduled_requests, [running, fits]) + self.assertEqual(first.num_decode_requests, 1) + self.assertEqual(len(fits.slot_mapping), 40) + self.assertFalse(fits.chunked_prefill_pending) + self.assertFalse(overflow.block_table) + for req in first.scheduled_requests: + req.append_generated_token_id(42) + s.complete_requests(first.scheduled_requests) + # Even with running decodes and a newly arrived prompt, the large + # prompt gets a complete forward using the existing single-row exception. + s.add_request(make_request("new", 16)) + second = s.schedule() + self.assertEqual(second.scheduled_requests, [overflow]) + self.assertEqual(second.num_decode_requests, 0) + self.assertEqual(len(overflow.slot_mapping), 80) + self.assertFalse(overflow.chunked_prefill_pending) + self.assertIsNone(overflow.num_scheduled_tokens) + + def test_mixed_admission_reserves_running_decode_pages(self): + s = make_scheduler( + chunked_prefill_size=16, enable_mixed_batching=True, num_blocks=2 + ) + short = make_request("short", 15, max_tokens=16) + s.add_request(short) + s.schedule() + short.append_generated_token_id(42) + s.complete_requests([short]) + waiting = make_request("waiting", 2) + s.add_request(waiting) + batch = s.schedule() + self.assertEqual(batch.scheduled_requests, [short]) + self.assertEqual(waiting.block_table, []) + def test_disabled_by_default_full_prefill(self): s = make_scheduler() req = make_request("r1", 300) @@ -556,15 +650,17 @@ def test_mamba_cache_chunked_resume(self): self.assertTrue(saw_resumed) def test_connector_disabled_chunked_flag(self): - # chunked_prefill_size is force-disabled with a - # connector (constructor warning), so the - # skip_waiting branch can never fire there. - class FakeConnector: - pass - - with self.assertLogs("infinilm.llm.scheduler", level="WARNING"): - s = make_scheduler(connector=FakeConnector(), chunked_prefill_size=64) + with self.assertLogs("infinilm.llm.scheduler", level="WARNING") as logs: + s = make_scheduler(connector=object(), chunked_prefill_size=64) self.assertIsNone(s.chunked_prefill_size) + self.assertIn("chunked_prefill_size disabled", logs.records[0].getMessage()) + + def test_connector_warns_when_mixed_batching_requested(self): + with self.assertLogs("infinilm.llm.scheduler", level="WARNING") as logs: + s = make_scheduler(connector=object(), enable_mixed_batching=True) + self.assertFalse(s.enable_mixed_batching) + self.assertIn("--enable-mixed-batching disabled", logs.records[0].getMessage()) + self.assertIn("KV connector", logs.records[0].getMessage()) def test_decode_batch_unlimited_by_token_budget(self): # Decode batches bypass max_num_batched_tokens (only