From 2a43a7f9eb1974957daa07f8817799a360df7cc8 Mon Sep 17 00:00:00 2001 From: chao-xu-spec Date: Thu, 19 Nov 2026 08:44:21 +0000 Subject: [PATCH] indexer: half-run work split in the four-GPU decode scorer The four-GPU decode scorer gave a workgroup one whole 16-block run per rank, or one 4-block unit. With 53 to 200 runs per rank that left most CUs idle (whole runs) or paid for 16-byte peer stores (units). A new half-run mode (8 blocks per item, two pages per wave, 32-byte peer stores) doubles the busy workgroups. The mode is chosen on the device from the sequence lengths. Scores are bitwise identical to main. - Three modes per call, chosen on the device from the rank's run count: units up to 52 runs, halves up to 200, whole runs above. Block placement, scores and slots are unchanged, so the output does not depend on the mode. - Half-run items stage through LDS with 8-byte writes and a rank stores 32 B per (head, row). Signed-off-by: chao-xu-spec --- .../split4/decode_score.py | 94 ++++++++++++++----- .../test_split4_rank_owned.py | 19 ++++ .../test_split4_rank_owned_ranks4.py | 50 ++++++++++ 3 files changed, 140 insertions(+), 23 deletions(-) diff --git a/amd_kernels/indexer/indexer_nvfp4_d128_p128_top16/split4/decode_score.py b/amd_kernels/indexer/indexer_nvfp4_d128_p128_top16/split4/decode_score.py index d8a5a0c..cb4f763 100644 --- a/amd_kernels/indexer/indexer_nvfp4_d128_p128_top16/split4/decode_score.py +++ b/amd_kernels/indexer/indexer_nvfp4_d128_p128_top16/split4/decode_score.py @@ -14,7 +14,8 @@ Work: the rank's runs (its 16 blocks of a chunk) of every request, prefix-summed in LDS; workgroup g takes runs g, g + grid, ..., wave w quarter w (4 blocks) of each, the next page loading tile by tile under the current one. A call of at most UNIT_MAX / 4 runs (from the device-side lengths) splits each run into four 4-block units instead, wave w -its block w. A wave keeps its request's Q columns in registers and 64 pages' table entries in one VGPR (readlane per +its block w, and up to HALF_MAX / 2 runs into two 8-block halves, wave w its blocks 2 w and 2 w + 1 (32 B peer stores). +A wave keeps its request's Q columns in registers and 64 pages' table entries in one VGPR (readlane per page). The quarters meet in LDS and one wave stores the run: 64 B per (head, row), sc0 sc1 (16 B pieces cost more over xGMI). @@ -99,7 +100,10 @@ def grid_for(num_reqs: int, ql: int) -> int: SC1 = 0x10 # load cache policy: coherent across the XCDs' L2s # units of 4 blocks (one page per wave) up to this many: more workgroups shorten a wave's page chain; above it their # 16 B peer stores lose (M 8 at 32K 17.2 -> 13.3 us; 512 units at 512K +6.5%) -UNIT_MAX = 256 +UNIT_MAX = 208 +# half runs (8 blocks: two pages per wave, 32 B peer stores) above UNIT_MAX up to this many: twice the busy +# workgroups of whole runs, which win again from about 200 runs per rank +HALF_MAX = 400 # the fused top-k (one waiting workgroup per row, the standalone four-wave selector's body) up to this many rows: the # waiting workgroups slow the call as their count grows (128K M 40-128 -0.7..-1.0 us vs two launches) TK_MAX_ROWS = 128 @@ -300,6 +304,8 @@ def kernel( # contiguous split spreads them and stacks some CUs: 16 requests 15.0 -> 18.5 us) total = _uni(incl_l[NR - 1]) unit = total * NW <= UNIT_MAX + # half runs (2 pages per wave) up to HALF_MAX of them: between the unit split and whole runs + half = total * 2 <= HALF_MAX lut8 = fx.logical_divide( fx.make_view( @@ -324,6 +330,14 @@ def kernel( fx.make_layout(1, 1), ) atom4 = fx.make_copy_atom(fx.UniversalCopy32b(), fx.Int32) + stage8 = fx.logical_divide( + fx.make_view( + fx.recast_iter(fx.Int32, smem.stage.ptr), + fx.make_layout(2 * NG * 16 * NW * QD, 1), + ), + fx.make_layout(2, 1), + ) + atom8 = fx.make_copy_atom(fx.UniversalCopy64b(), fx.Int32) def lut_load(sbyte): r = fx.make_rmem_tensor(fx.make_layout(2, 1), fx.Int32) @@ -510,11 +524,15 @@ def row_value(mx, seq, q_off, b): ) rs = rs + i32_nwg - def scan(UNIT): - """The pages of this workgroup's items: units (``UNIT``, a quarter of - a run, wave w its block w) or runs (wave w its quarter w).""" - qdv = 1 if UNIT else QD # pages per item and wave - items = total * NW if UNIT else total + def scan(MODE): + """The pages of this workgroup's items: units (``MODE`` 2: a quarter of + a run, wave w its block w), halves (``MODE`` 1: half a run, wave w its + blocks 2 w, 2 w + 1) or runs (``MODE`` 0: wave w its quarter w).""" + UNIT = MODE == 2 + HALF = MODE == 1 + qdv = 1 if UNIT else (2 if HALF else QD) # pages per item and wave + qsh = qdv.bit_length() - 1 + items = total * (QD // qdv) nitems = (pid < items).select((items - pid + i32_nwg - 1) // i32_nwg, 0) nslots = nitems * qdv # pages of this wave w0 = fx.Int32(0) @@ -523,8 +541,8 @@ def scan(UNIT): # grid's: a run's block 4 w + s % 4, or unit u's block 4 u + w (run # k = item / 4), of request rj (binary search in the prefix) s = w0 + lane - it = pid + (s if UNIT else s >> 2) * i32_nwg - k = (it >> 2) if UNIT else it + it = pid + (s >> qsh) * i32_nwg + k = it >> (2 - qsh) rj = fx.Int32(0) for bs in range_constexpr(NR_LOG): step = NR >> (bs + 1) @@ -534,7 +552,12 @@ def scan(UNIT): excl = (rj > 0).select(fx.Int32(incl_l[_max(rj - 1, 0)]), fx.Int32(0)) seqj = fx.Int32(seq_l[rj]) nblkj = (seqj + (BLK - 1)) // BLK - in_run = ((it & 3) * NW + wave) if UNIT else (wave * QD + (s & 3)) + if const_expr(UNIT): + in_run = (it & 3) * NW + wave + elif const_expr(HALF): + in_run = (it & 1) * (2 * NW) + wave * 2 + (s & 1) + else: + in_run = wave * QD + (s & 3) bj = (k - excl) * CHUNK + in_run * WORLD + i32_me frow = seqj - (QL - 1) rbf = ((frow < 0).select(fx.Int32(0), frow) + (BLK - 1)) // BLK @@ -580,12 +603,12 @@ def scan(UNIT): # a unit is one slot, so its lane polls all four sources want = lane < n word = mine + fx.Int64( - (SC_IQ + (0 if UNIT else lane % WORLD) * SC_MAX_REQS + rj) * 4 + (SC_IQ + (lane % WORLD if MODE == 0 else 0) * SC_MAX_REQS + rj) * 4 ) def poll(): lo = _flag_read(word) - if const_expr(UNIT): + if const_expr(MODE != 0): for p in range_constexpr(1, WORLD): lo = _min( lo, _flag_read(word + fx.Int64(p * SC_MAX_REQS * 4)) @@ -657,7 +680,7 @@ def poll(): c5, s5 = res[2][5] c6, s6 = res[2][6] c7, s7 = res[2][7] - e = fx.Int32(0) if UNIT else (w0 + j) & 3 + e = fx.Int32(0) if UNIT else (w0 + j) & (qdv - 1) ov0 = row_value(res[0], seq, st_qoff, b) v00 = (e == 0).select(ov0, v00) v01 = (e == 1).select(ov0, v01) @@ -669,13 +692,13 @@ def poll(): v11 = (e == 1).select(ov1, v11) v12 = (e == 2).select(ov1, v12) v13 = (e == 3).select(ov1, v13) - done = (j >= 0) if UNIT else (e == QD - 1) + done = (j >= 0) if UNIT else (e == qdv - 1) if done: # the item's pages meet in LDS ([slot][group][col][wave]: a # unit's 4 B or a run quarter's 16 B per wave); wave m % NW # stores 16 B per lane: a unit's 4 blocks (lanes kl 0) or # quarter kl of the run - m = (w0 + j) if UNIT else (w0 + j) >> 2 + m = (w0 + j) >> qsh slot = m % 2 vsets = [[v00, v01, v02, v03], [v10, v11, v12, v13]] if kl == 0: @@ -685,6 +708,14 @@ def poll(): val = fx.make_rmem_tensor(fx.make_layout(1, 1), fx.Int32) val.store(Vec.from_elements([_bits(vsets[g][0])], fx.Int32)) fx.copy(atom4, val, fx.slice(stage4, (None, base))) + elif const_expr(HALF): + val = fx.make_rmem_tensor(fx.make_layout(2, 1), fx.Int32) + val.store( + Vec.from_elements( + [_bits(vsets[g][0]), _bits(vsets[g][1])], fx.Int32 + ) + ) + fx.copy(atom8, val, fx.slice(stage8, (None, base))) else: val = fx.make_rmem_tensor(fx.make_layout(4, 1), fx.Int32) val.store( @@ -693,20 +724,31 @@ def poll(): ) ) fx.copy(atom16, val, fx.slice(stage16, (None, base))) - if const_expr(UNIT): + if const_expr(MODE != 0): fx.rocdl.s_waitcnt(lgkmcnt=0) fx.gpu.barrier() if wave == m % NW: # the 16 B's 4-block group within this rank's 16 of # the chunk - qsel = (b % CHUNK) // (NW * WORLD) if UNIT else kl + if const_expr(UNIT): + qsel = (b % CHUNK) // (NW * WORLD) + elif const_expr(HALF): + qsel = ((b % CHUNK) // (2 * NW * WORLD)) * 2 + (kl & 1) + else: + qsel = kl for g in range_constexpr(NG): base = (slot * NG + g) * 16 + col got = fx.make_rmem_tensor(fx.make_layout(4, 1), fx.Int32) fx.copy( atom16, fx.slice( - stage16, (None, base if UNIT else base * NW + kl) + stage16, + ( + None, + base + if UNIT + else (base * 2 + (kl & 1) if HALF else base * NW + kl), + ), ), got, ) @@ -717,9 +759,12 @@ def poll(): + qsel * 4 ) for h in range_constexpr(HEADS): - ok_h = ( - (st_head == h) & (kl == 0) if UNIT else st_head == h - ) + if const_expr(UNIT): + ok_h = (st_head == h) & (kl == 0) + elif const_expr(HALF): + ok_h = (st_head == h) & (kl < 2) + else: + ok_h = st_head == h if const_expr(4 * g + 4 > QL): ok_h = ok_h & row_ok[g] buffer_ops.buffer_store( @@ -734,9 +779,12 @@ def poll(): # one code path per split (selects in the page loop cost run calls ~1%) if unit: - scan(True) + scan(2) else: - scan(False) + if half: + scan(1) + else: + scan(0) # Every global write of the workgroup is an sc0 sc1 store, visible system-wide once drained (a system release # per workgroup writes back L2 every time: 512 workgroups +45 us). The last workgroup (dispatched last, so diff --git a/tests/indexer/indexer_nvfp4_d128_p128_top16/test_split4_rank_owned.py b/tests/indexer/indexer_nvfp4_d128_p128_top16/test_split4_rank_owned.py index 2c7096b..070ffa1 100644 --- a/tests/indexer/indexer_nvfp4_d128_p128_top16/test_split4_rank_owned.py +++ b/tests/indexer/indexer_nvfp4_d128_p128_top16/test_split4_rank_owned.py @@ -265,6 +265,25 @@ def test_decode_indices_rank_owned_order_past_one_window(ql, strips): assert torch.equal(got, topk_reference(blk, nblk_rows, int(seq.max()))[0]) +# Requests of k whole 64-block chunks give every rank k runs each; a request one block longer gives rank 0 one more. +# The scorer splits a run into units up to 52 runs per rank, into halves up to 200, else keeps it whole. +RUN_BOUNDARIES = [(13, 4), (10, 20), (6, 40)] + + +@pytest.mark.parametrize("ql", [4, 8]) +@pytest.mark.parametrize("n,chunks", RUN_BOUNDARIES) +@pytest.mark.parametrize("longer", [False, True], ids=["even", "rank0_one_run_more"]) +@torch.inference_mode() +def test_decode_rank_owned_run_count_boundaries(ql, n, chunks, longer): + """Per-rank run totals at both sides of each split-mode boundary (52 / 53, 200 / 201, 240), with the ranks on + different sides: the scores are bitwise the single-head scorer's and the top-k its reference.""" + require_gfx950() + lens = [chunks * 64 * BLK] * n + if longer: + lens[0] += BLK + _check_decode(lens, ql, n * chunks, 0, 1) + + @torch.inference_mode() def test_decode_rank_owned_long_rows(): """The fused decode and its top-k on requests of 150K-300K tokens (one or two top-k windows).""" diff --git a/tests/indexer/indexer_nvfp4_d128_p128_top16/test_split4_rank_owned_ranks4.py b/tests/indexer/indexer_nvfp4_d128_p128_top16/test_split4_rank_owned_ranks4.py index d4bc171..633e388 100644 --- a/tests/indexer/indexer_nvfp4_d128_p128_top16/test_split4_rank_owned_ranks4.py +++ b/tests/indexer/indexer_nvfp4_d128_p128_top16/test_split4_rank_owned_ranks4.py @@ -96,3 +96,53 @@ def test_split4_rank_owned_four_gpus(tmp_path): with open(p) as f: msgs.append(f.read()) raise AssertionError("\n".join(msgs) or "worker failed") from None + + +def _run_boundary_worker(rank, port): + import torch.distributed as dist + from amd_kernels import warm_up + from amd_kernels.indexer.indexer_nvfp4_d128_p128_top16.split4.host import IndexSplit4RankOwned + from .test_split4_rank_owned import _nvfp4_pages, _single_head_scores, _topk_decode_indices + + torch.set_num_threads(1) + warm_up.set_auto(True) + warm_up.set_workers(1) + torch.cuda.set_device(rank) + dist.init_process_group('gloo', init_method=f'tcp://127.0.0.1:{port}', rank=rank, world_size=4) + ql = 8 + # (requests, 64-block chunks per request, the first request one block longer): per-rank runs 52, 53 on rank 0 + # only, 200, 201 on rank 0 only + for n, chunks, longer in ((13, 4, 0), (13, 4, 1), (10, 20, 0), (10, 20, 1)): + lens = [chunks * 64 * 128 + longer * 128] + [chunks * 64 * 128] * (n - 1) + cp = IndexSplit4RankOwned(rank, dist.group.WORLD, torch.device('cuda', rank), n * ql, 8192, ql) + cg = torch.Generator(device='cuda').manual_seed(n * chunks + longer) + cache = _nvfp4_pages(2048, cg) + table = torch.randint(0, 2048, (n, 8192), generator=cg, dtype=torch.int32, device='cuda') + q = (torch.randn(n * ql, 4, 128, generator=cg, device='cuda') * .5).to(torch.float8_e4m3fn) + seq = torch.tensor(lens, dtype=torch.int32, device='cuda') + out = torch.empty(1, n * ql, 16, dtype=torch.int32, device='cuda') + graph = None + for step in range(3): + if step == 0: + cp.score_topk(q[:, rank].contiguous(), cache, table, seq, 2, 3, 8192, out, 1 << 20, sharded=False) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + cp.score_topk(q[:, rank].contiguous(), cache, table, seq, 2, 3, 8192, out, 1 << 20, sharded=False) + out.fill_(-7) + graph.replay() + torch.cuda.synchronize() + expected = _single_head_scores(q, cache, table, seq, ql, 2, 3)[rank:rank + 1] + want = _topk_decode_indices(expected, seq, ql, False) + assert torch.equal(out[0], want), (rank, n, chunks, longer, step) + assert warm_up.late_compiles() == 0 + dist.destroy_process_group() + + +def test_split4_run_count_boundaries_four_gpus(): + """Per-rank run totals at both sides of the unit / half-run / whole-run boundaries, ranks on different sides, + captured in a graph and replayed: the ids equal the single-head scorer's top-16.""" + require_gfx950() + if torch.cuda.device_count() < 4: + pytest.skip('needs four GPUs') + import torch.multiprocessing as mp + mp.spawn(_run_boundary_worker, args=(_free_port(),), nprocs=4, join=True)