Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions examples/test_infer.py
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,8 @@ def test(
weight_load_mode="async",
enable_prefix_caching=True,
pre_transpose=False,
chunked_prefill_size=0,
enable_mixed_batching=False,
):
if video_path is not None:
from infinilm.processors.videonsa_processor import decode_video_frames
Expand Down Expand Up @@ -71,6 +73,8 @@ def test(
top_k=top_k,
top_p=top_p,
enable_graph=enable_graph,
chunked_prefill_size=chunked_prefill_size,
enable_mixed_batching=enable_mixed_batching,
attn_backend=attn_backend,
use_mla=use_mla,
skip_load=skip_load,
Expand Down Expand Up @@ -167,6 +171,8 @@ def test(
ep=ep,
enable_paged_attn=enable_paged_attn,
enable_graph=enable_graph,
chunked_prefill_size=cfg.chunked_prefill_size,
enable_mixed_batching=cfg.enable_mixed_batching,
num_blocks=cfg.num_blocks,
block_size=cfg.block_size,
top_k=cfg.top_k,
Expand Down
6 changes: 6 additions & 0 deletions python/infinilm/base_config.py
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,7 @@ def __init__(self):
self.pre_transpose = self.args.pre_transpose
self.num_blocks = self.args.num_blocks
self.chunked_prefill_size = self.args.chunked_prefill_size
self.enable_mixed_batching = self.args.enable_mixed_batching
self.block_size = self.args.block_size
self.max_cache_len = self.args.max_cache_len
self.kv_cache_dtype = self.args.kv_cache_dtype
Expand Down Expand Up @@ -250,6 +251,11 @@ def _add_common_args(self):
self.parser.add_argument(
"--num-blocks", type=int, default=512, help="number of KV cache blocks"
)
self.parser.add_argument(
"--enable-mixed-batching",
action="store_true",
help="batch running decode tokens with whole or chunked prefills in paged text inference",
)
self.parser.add_argument(
"--chunked-prefill-size",
type=int,
Expand Down
3 changes: 3 additions & 0 deletions python/infinilm/config/engine_config.py
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,7 @@ class EngineConfig:
enable_prefix_caching: bool = True
static_cache_capacity: int = 1
chunked_prefill_size: int = 0
enable_mixed_batching: bool = False

def __post_init__(self) -> None:
if (
Expand Down Expand Up @@ -97,6 +98,8 @@ def __post_init__(self) -> None:
and self.cache_type != "paged"
):
raise ValueError("kv_transfer_config requires cache_type='paged'")
if self.enable_mixed_batching and self.cache_type != "paged":
raise ValueError("mixed batching requires paged attention")
if type(self.chunked_prefill_size) is not int or (
self.chunked_prefill_size < 0
):
Expand Down
14 changes: 14 additions & 0 deletions python/infinilm/llm/llm.py
Original file line number Diff line number Diff line change
Expand Up @@ -173,6 +173,15 @@ def __init__(self, config: EngineConfig):
)
assert 1024 <= max_num_batched_tokens <= max_position_embeddings

if config.enable_mixed_batching and connector is None and (
type(self.processor).build_model_inputs
is not BasicLLMProcessor.build_model_inputs
or self.model_runner.speculative_runner is not None
or config.use_mla
):
raise ValueError(
"mixed batching requires the standard paged text processor and attention path"
)
self.scheduler = Scheduler(
max_batch_size=config.max_batch_size,
num_blocks=config.num_blocks,
Expand All @@ -188,6 +197,7 @@ def __init__(self, config: EngineConfig):
else None
),
chunked_prefill_size=config.chunked_prefill_size,
enable_mixed_batching=config.enable_mixed_batching,
)
if model_engine.model_type == "videonsa":
self.model_runner.bind_nsa_scheduler(self.scheduler)
Expand Down Expand Up @@ -556,6 +566,7 @@ def __init__(
enable_prefix_caching: bool = True,
static_cache_capacity: int = 1,
chunked_prefill_size: int = 0,
enable_mixed_batching: bool = False,
):
"""Initialize LLM.

Expand Down Expand Up @@ -596,6 +607,7 @@ def __init__(
max_batch_size=max_batch_size,
static_cache_capacity=static_cache_capacity,
chunked_prefill_size=chunked_prefill_size,
enable_mixed_batching=enable_mixed_batching,
max_tokens=max_tokens,
num_blocks=num_blocks,
block_size=block_size,
Expand Down Expand Up @@ -786,6 +798,7 @@ def __init__(
enable_prefix_caching: bool = True,
static_cache_capacity: int = 1,
chunked_prefill_size: int = 0,
enable_mixed_batching: bool = False,
):
"""Initialize AsyncLLMEngine.

Expand Down Expand Up @@ -830,6 +843,7 @@ def __init__(
max_batch_size=max_batch_size,
static_cache_capacity=static_cache_capacity,
chunked_prefill_size=chunked_prefill_size,
enable_mixed_batching=enable_mixed_batching,
max_tokens=max_tokens,
num_blocks=num_blocks,
block_size=block_size,
Expand Down
Loading